<commit_before>import sys
import logging
from datetime import datetime

from modularodm import Q

from framework.transactions.context import TokuTransaction
from website.app import init_app
from website import mails, settings

from scripts.utils import add_file_logger

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

def main(dry_run=True):
    #find all emails to be sent, pops the top one for each user(to obey the once
    #a week requirement), checks to see if one has been sent this week, and if
    #not send the email, otherwise leave it in the queue

    user_queue = {}
    for email in find_queued_mails_ready_to_be_sent():
        user_queue.setdefault(email.user._id, []).append(email)

    emails_to_be_sent = pop_and_verify_mails_for_each_user(user_queue)

    for mail in emails_to_be_sent:
        if not dry_run:
            with TokuTransaction():
                try:
                    sent_ = mail.send_mail()
                    message = 'Email of type {0} sent to {1}'.format(mail.email_type, mail.to_addr) if sent_ else \
                        'Email of type {0} failed to be sent to {1}'.format(mail.email_type, mail.to_addr)
                    logger.warn(message)
                except:
                    logger.warn('Email of type {0} to be sent to {1} caused an ERROR'.format(mail.email_type, mail.to_addr))
                    pass

def find_queued_mails_ready_to_be_sent():
    return mails.QueuedMail.find(
        Q('send_at', 'lt', datetime.utcnow()) &
        Q('sent_at', 'eq', None)
    )

def pop_and_verify_mails_for_each_user(user_queue):
    for user_emails in user_queue.values():
        mail = user_emails[0]
        mails_past_week = mails.QueuedMail.find(
            Q('user', 'eq', mail.user) &
            Q('sent_at', 'gt', datetime.utcnow() - settings.WAIT_BETWEEN_MAILS)
        )
        if not mails_past_week.count():
            yield mail

if __name__ == '__main__':
    dry_run = 'dry' in sys.argv
    init_app(routes=False)
    if not dry_run:
        add_file_logger(logger, __file__)
    main(dry_run=dry_run)
<commit_msg>Add date and error type to log<commit_after>import sys
import logging
from datetime import datetime

from modularodm import Q

from framework.transactions.context import TokuTransaction
from website.app import init_app
from website import mails, settings

from scripts.utils import add_file_logger

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

def main(dry_run=True):
    #find all emails to be sent, pops the top one for each user(to obey the once
    #a week requirement), checks to see if one has been sent this week, and if
    #not send the email, otherwise leave it in the queue

    user_queue = {}
    for email in find_queued_mails_ready_to_be_sent():
        user_queue.setdefault(email.user._id, []).append(email)

    emails_to_be_sent = pop_and_verify_mails_for_each_user(user_queue)

    logger.warn('Emails being sent at {0}:{1}:{2}:{3}'.format(datetime.utcnow().year, datetime.utcnow().month, datetime.utcnow().day, datetime.utcnow().minute))

    for mail in emails_to_be_sent:
        if not dry_run:
            with TokuTransaction():
                try:
                    sent_ = mail.send_mail()
                    message = 'Email of type {0} sent to {1}'.format(mail.email_type, mail.to_addr) if sent_ else \
                        'Email of type {0} failed to be sent to {1}'.format(mail.email_type, mail.to_addr)
                    logger.warn(message)
                except Exception as error:
                    logger.error('Email of type {0} to be sent to {1} caused an ERROR'.format(mail.email_type, mail.to_addr))
                    logger.exception(error)
                    pass

def find_queued_mails_ready_to_be_sent():
    return mails.QueuedMail.find(
        Q('send_at', 'lt', datetime.utcnow()) &
        Q('sent_at', 'eq', None)
    )

def pop_and_verify_mails_for_each_user(user_queue):
    for user_emails in user_queue.values():
        mail = user_emails[0]
        mails_past_week = mails.QueuedMail.find(
            Q('user', 'eq', mail.user) &
            Q('sent_at', 'gt', datetime.utcnow() - settings.WAIT_BETWEEN_MAILS)
        )
        if not mails_past_week.count():
            yield mail

if __name__ == '__main__':
    dry_run = 'dry' in sys.argv
    init_app(routes=False)
    if not dry_run:
        add_file_logger(logger, __file__)
    main(dry_run=dry_run)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Licensed under the MIT license
# http://opensource.org/licenses/mit-license.php

# Copyright 2006,2007 Frank Scholz <coherence@beebits.net>
#
# a little helper to get the proper ElementTree package

import re
import exceptions

try:
    import cElementTree as ET
    import elementtree
    #print "we are on CET"
except ImportError:
    try:
        from elementtree import ElementTree as ET
        import elementtree
        #print "simply using ET"
    except ImportError:
        """ this seems to be necessary with the python2.5 on the Maemo platform """
        try:
            from xml.etree import cElementTree as ET
            from xml import etree as elementtree
        except ImportError:
            try:
                from xml.etree import ElementTree as ET
                from xml import etree as elementtree
            except ImportError:
                #print "no ElementTree module found, critical error"
                raise ImportError, "no ElementTree module found, critical error"

utf8_escape = re.compile(eval(r'u"[&<>\"]+"'))
escape = re.compile(eval(r'u"[&<>\"\u0080-\uffff]+"'))

def encode_entity(text, pattern=escape):
    # map reserved and non-ascii characters to numerical entities
    def escape_entities(m, map=elementtree.ElementTree._escape_map):
        out = []
        append = out.append
        for char in m.group():
            t = map.get(char)
            if t is None:
                t = "&#%d;" % ord(char)
            append(t)
        return ''.join(out)
    try:
        return elementtree.ElementTree._encode(pattern.sub(escape_entities, text), 'ascii')
    except TypeError:
        elementtree.ElementTree._raise_serialization_error(text)

def new_encode_entity(text, pattern=utf8_escape):
    # map reserved and non-ascii characters to numerical entities
    def escape_entities(m, map=elementtree.ElementTree._escape_map):
        out = []
        append = out.append
        for char in m.group():
            t = map.get(char)
            if t is None:
                t = "&#%d;" % ord(char)
            append(t)
        if type(text) == unicode:
            return ''.join(out)
        else:
            return u''.encode('utf-8').join(out)
    try:
        if type(text) == unicode:
            return elementtree.ElementTree._encode(escape.sub(escape_entities, text), 'ascii')
        else:
            return elementtree.ElementTree._encode(utf8_escape.sub(escape_entities, text.decode('utf-8')), 'utf-8')
    except TypeError:

        elementtree.ElementTree._raise_serialization_error(text)

elementtree.ElementTree._encode_entity = new_encode_entity

# it seems there are some ElementTree libs out there
# which have the alias XMLParser and some that haven't.
#
# So we just use the XMLTreeBuilder method for now
# if XMLParser isn't available.

if not hasattr(ET, 'XMLParser'):
    def XMLParser(encoding='utf-8'):
        return ET.XMLTreeBuilder()

    ET.XMLParser = XMLParser

def namespace_map_update(namespaces):
    #try:
    #    from xml.etree import ElementTree
    #except ImportError:
    #    from elementtree import ElementTree

    elementtree.ElementTree._namespace_map.update(namespaces)

class ElementInterface(elementtree.ElementTree._ElementInterface):
    """ helper class """

def indent(elem, level=0):
    """ generate pretty looking XML, based upon:
        http://effbot.org/zone/element-lib.htm#prettyprint
    """
    i = "\n" + level*"  "
    if len(elem):
        if not elem.text or not elem.text.strip():
            elem.text = i + "  "
        for elem in elem:
            indent(elem, level+1)
            if not elem.tail or not elem.tail.strip():
                elem.tail = i
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
    else:
        if level and (not elem.tail or not elem.tail.strip()):
            elem.tail = i

def parse_xml(data, encoding="utf-8",dump_invalid_data=False):
    try:
        p = ET.XMLParser(encoding=encoding)
    except exceptions.TypeError:
        p = ET.XMLParser()

    # my version of twisted.web returns page_infos as a dictionary in
    # the second item of the data list
    if isinstance(data, (list, tuple)):
        data, _ = data

    try:
        data = data.encode(encoding)
    except UnicodeDecodeError:
        pass

    # Guess from who we're getting this?
    data = data.replace('\x00','')
    try:
        p.feed(data)
    except Exception, error:
        if dump_invalid_data:
            print error, repr(data)
        p.close()
        raise Exception, error
    else:
        return ET.ElementTree(p.close())
<commit_msg>consolidated messages about source modules for ElementTree API<commit_after># -*- coding: utf-8 -*-

# Licensed under the MIT license
# http://opensource.org/licenses/mit-license.php

# Copyright 2006,2007 Frank Scholz <coherence@beebits.net>
#
# a little helper to get the proper ElementTree package

import re
import exceptions

try:
    import cElementTree as ET
    import elementtree
    #print "ElementTree: using cElementTree as ET"
except ImportError:
    try:
        from elementtree import ElementTree as ET
        import elementtree
        #print "ElementTree: using elementtree.ElementTree as ET"
    except ImportError:
        """ this seems to be necessary with the python2.5 on the Maemo platform """
        try:
            from xml.etree import cElementTree as ET
            from xml import etree as elementtree
            #print "ElementTree: using xml.etree.cElementTree as ET"
        except ImportError:
            try:
                from xml.etree import ElementTree as ET
                from xml import etree as elementtree
                #print "ElementTree: using xml.etree.ElementTree as ET"
            except ImportError:
                #print "ElementTree: no ElementTree module found, critical error"
                raise ImportError, "ElementTree: no ElementTree module found, critical error"

utf8_escape = re.compile(eval(r'u"[&<>\"]+"'))
escape = re.compile(eval(r'u"[&<>\"\u0080-\uffff]+"'))

def encode_entity(text, pattern=escape):
    # map reserved and non-ascii characters to numerical entities
    def escape_entities(m, map=elementtree.ElementTree._escape_map):
        out = []
        append = out.append
        for char in m.group():
            t = map.get(char)
            if t is None:
                t = "&#%d;" % ord(char)
            append(t)
        return ''.join(out)
    try:
        return elementtree.ElementTree._encode(pattern.sub(escape_entities, text), 'ascii')
    except TypeError:
        elementtree.ElementTree._raise_serialization_error(text)

def new_encode_entity(text, pattern=utf8_escape):
    # map reserved and non-ascii characters to numerical entities
    def escape_entities(m, map=elementtree.ElementTree._escape_map):
        out = []
        append = out.append
        for char in m.group():
            t = map.get(char)
            if t is None:
                t = "&#%d;" % ord(char)
            append(t)
        if type(text) == unicode:
            return ''.join(out)
        else:
            return u''.encode('utf-8').join(out)
    try:
        if type(text) == unicode:
            return elementtree.ElementTree._encode(escape.sub(escape_entities, text), 'ascii')
        else:
            return elementtree.ElementTree._encode(utf8_escape.sub(escape_entities, text.decode('utf-8')), 'utf-8')
    except TypeError:

        elementtree.ElementTree._raise_serialization_error(text)

elementtree.ElementTree._encode_entity = new_encode_entity

# it seems there are some ElementTree libs out there
# which have the alias XMLParser and some that haven't.
#
# So we just use the XMLTreeBuilder method for now
# if XMLParser isn't available.

if not hasattr(ET, 'XMLParser'):
    def XMLParser(encoding='utf-8'):
        return ET.XMLTreeBuilder()

    ET.XMLParser = XMLParser

def namespace_map_update(namespaces):
    #try:
    #    from xml.etree import ElementTree
    #except ImportError:
    #    from elementtree import ElementTree

    elementtree.ElementTree._namespace_map.update(namespaces)

class ElementInterface(elementtree.ElementTree._ElementInterface):
    """ helper class """

def indent(elem, level=0):
    """ generate pretty looking XML, based upon:
        http://effbot.org/zone/element-lib.htm#prettyprint
    """
    i = "\n" + level*"  "
    if len(elem):
        if not elem.text or not elem.text.strip():
            elem.text = i + "  "
        for elem in elem:
            indent(elem, level+1)
            if not elem.tail or not elem.tail.strip():
                elem.tail = i
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
    else:
        if level and (not elem.tail or not elem.tail.strip()):
            elem.tail = i

def parse_xml(data, encoding="utf-8",dump_invalid_data=False):
    try:
        p = ET.XMLParser(encoding=encoding)
    except exceptions.TypeError:
        p = ET.XMLParser()

    # my version of twisted.web returns page_infos as a dictionary in
    # the second item of the data list
    if isinstance(data, (list, tuple)):
        data, _ = data

    try:
        data = data.encode(encoding)
    except UnicodeDecodeError:
        pass

    # Guess from who we're getting this?
    data = data.replace('\x00','')
    try:
        p.feed(data)
    except Exception, error:
        if dump_invalid_data:
            print error, repr(data)
        p.close()
        raise Exception, error
    else:
        return ET.ElementTree(p.close())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
=====================
Fitting Distributions
=====================

Histograms, PDF fits, Kernel Density.
"""

# Author: Moritz Lotze <mlotze@km3net.de>
# License: BSD-3

import matplotlib.pyplot as plt
import numpy as np
import statsmodels.api as sm

from astropy.stats import knuth_bin_width
from scipy.stats import norm
from sklearn.mixture import GaussianMixture
from sklearn.grid_search import GridSearchCV
from sklearn.neighbors import KernelDensity

import km3pipe.style
km3pipe.style.use('default')

##############################################################################
# First generate some pseudodata: A bimodal gaussian, + noise.

N = 100
bmg = np.concatenate((np.random.normal(15, 1, 0.3 * N),
                      np.random.normal(20, 1, 0.7 * N)))
noise_bmg = 0.5
data = np.random.normal(bmg, noise_bmg)[:, np.newaxis]

# make X axis for plots
x = np.linspace(5, 35, N+1)

##############################################################################
# Histograms (nonparametric)
# --------------------------
#
# The simplest nonparametric density estimation tool is the Histogram.
# Choosing the binning manually can be tedious, however:
#
# 15 bins, spaced from ``data.min()`` to ``data.max()``.

plt.hist(data, bins=15, alpha=.5, normed=True)

##############################################################################
# Auto Binning (recommended)
# ~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Use builtin (numpy's) heuristics to figure out best binning.

plt.hist(data, bins='auto', alpha=.5, normed=True)

##############################################################################
# Knuth’s rule:
# ^^^^^^^^^^^^^
#
# A fixed-width, Bayesian approach to determining the optimal bin
# width of a histogram.

width, knuth_binlims = knuth_bin_width(data=data[:, 0], return_bins=True)

plt.hist(data, bins=knuth_binlims, alpha=.5, normed=True)

##############################################################################
# Bayesian Blocks
# ^^^^^^^^^^^^^^^
#
# TODO: Compute optimal segmentation of data with Scargle’s Bayesian Blocks.
# Produces bins of uneven width.

##############################################################################
# Fit Distribution via Maximum Likelihood
# ---------------------------------------
#
# If we have a hypothesis what the distribution looks like (e.g. gaussian), and want to fit its parameters.
#
# The nice thing is, you can define your own PDFs in scipy and fit it.
# Or take one from the dozens of pre-defined ones.
#
# However, there is no *bimodal* gaussian implemented in scipy yet :/
# In this case, either define it yourself, or use a GMM (below)

mu, sig = norm.fit(data)

plt.fill(x, norm(mu, sig).pdf(x), alpha=.5, label='Fitted')
plt.legend()
print('Unimodal Gaussian Fit:  Mean {:.4}, stdev {:.4}'.format(mu, sig))
plt.hist(data, bins='auto', alpha=.3, normed=True)

##############################################################################
# As expected, the result is rather silly, since we are only fitting *one*
# of the two gaussians.

##############################################################################
# Fit Gaussian Mixture Model (GMM)
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Assuming the data is the sum of one or more gaussians.
# Easily handles multidimensional case as well.

gmm = GaussianMixture(n_components=2, covariance_type='spherical')
gmm.fit(data)

mu1 = gmm.means_[0, 0]
mu2 = gmm.means_[1, 0]
sig1, sig2 = gmm.covariances_
wgt1, wgt2 = gmm.weights_
print('''Fit:
      1: Mean {:.4}, stdev {:.4}, weight {:.4}
      2: Mean {:.4}, stdev {:.4}, weight {:.4}
'''.format(mu1, sig1, wgt1, mu2, sig2, wgt2))

plt.hist(data, bins='auto', alpha=.3, normed=True)
plt.vlines((mu1, mu2), ymin=0, ymax=0.35, label='Fitted Means')
plt.legend()
plt.title('Gaussian Mixture Model')

##############################################################################
# Kernel Density: (non-parametric)
# --------------------------------
#
# If we have no strong assumptions about the underlying pdf.
#
# "Smooth out" each event with a kernel (e.g. gaussian) of
# a certain bandwidth, then add together all these mini-functions.
#
# The "bandwidth" (width of the kernel function) depends on the data, and
# can be estimated using cross-validation + maximum likelihood

##############################################################################
# in Statsmodels

dens = sm.nonparametric.KDEUnivariate(data)
dens.fit()

kde_sm = dens.evaluate(x)
plt.fill(x, kde_sm, alpha=.5, label='KDE')
plt.hist(data, bins='auto', alpha=.3, normed=True)

##############################################################################
# in scikit-learn

params = {'bandwidth': np.logspace(-2, 2, 50)}
grid = GridSearchCV(KernelDensity(), params)
grid.fit(data)

print("best bandwidth: {0}".format(grid.best_estimator_.bandwidth))

# use the best estimator to compute the kernel density estimate
kde_best = grid.best_estimator_
kde_sk = np.exp(
    kde_best.score_samples(x[:, np.newaxis])
)
plt.fill(x, kde_sk, alpha=.5, label='KDE')
plt.hist(data, bins='auto', alpha=.3, normed=True)

##############################################################################
# References
# ----------
#
# - B.W. Silverman, “Density Estimation for Statistics and Data Analysis”
# - Hastie, Tibshirani and Friedman, “The Elements of Statistical Learning: Data Mining, Inference, and Prediction”, Springer (2009)
# - Liu, R., Yang, L. “Kernel estimation of multivariate cumulative distribution function.” Journal of Nonparametric Statistics (2008)
# - Scargle, J., et al. "Studies in astronomical time series analysis. vi. Bayesian Block Representations." The Astrophysical Journal 764.2 (2013)
<commit_msg>plot GMM dists<commit_after># -*- coding: utf-8 -*-
"""
=====================
Fitting Distributions
=====================

Histograms, PDF fits, Kernel Density.
"""

# Author: Moritz Lotze <mlotze@km3net.de>
# License: BSD-3

import matplotlib.pyplot as plt
import numpy as np
import statsmodels.api as sm

from astropy.stats import knuth_bin_width
from scipy.stats import norm
from sklearn.mixture import GaussianMixture
from sklearn.grid_search import GridSearchCV
from sklearn.neighbors import KernelDensity

import km3pipe.style
km3pipe.style.use('default')

##############################################################################
# First generate some pseudodata: A bimodal gaussian, + noise.

N = 100
bmg = np.concatenate((np.random.normal(15, 1, 0.3 * N),
                      np.random.normal(20, 1, 0.7 * N)))
noise_bmg = 0.5
data = np.random.normal(bmg, noise_bmg)[:, np.newaxis]

# make X axis for plots
x = np.linspace(5, 35, N+1)

##############################################################################
# Histograms (nonparametric)
# --------------------------
#
# The simplest nonparametric density estimation tool is the Histogram.
# Choosing the binning manually can be tedious, however:
#
# 15 bins, spaced from ``data.min()`` to ``data.max()``.

plt.hist(data, bins=15, alpha=.5, normed=True)

##############################################################################
# Auto Binning (recommended)
# ~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Use builtin (numpy's) heuristics to figure out best binning.

plt.hist(data, bins='auto', alpha=.5, normed=True)

##############################################################################
# Knuth’s rule:
# ^^^^^^^^^^^^^
#
# A fixed-width, Bayesian approach to determining the optimal bin
# width of a histogram.

width, knuth_binlims = knuth_bin_width(data=data[:, 0], return_bins=True)

plt.hist(data, bins=knuth_binlims, alpha=.5, normed=True)

##############################################################################
# Bayesian Blocks
# ^^^^^^^^^^^^^^^
#
# TODO: Compute optimal segmentation of data with Scargle’s Bayesian Blocks.
# Produces bins of uneven width.

##############################################################################
# Fit Distribution via Maximum Likelihood
# ---------------------------------------
#
# If we have a hypothesis what the distribution looks like (e.g. gaussian),
# and want to fit its parameters.
#
# The nice thing is, you can define your own PDFs in scipy and fit it.
# Or take one from the dozens of pre-defined ones.
#
# However, there is no *bimodal* gaussian implemented in scipy yet :/
# In this case, either define it yourself, or use a GMM (below)

mu, sig = norm.fit(data)

plt.fill(x, norm(mu, sig).pdf(x), alpha=.5, label='Fitted')
plt.legend()
print('Unimodal Gaussian Fit:  Mean {:.4}, stdev {:.4}'.format(mu, sig))
plt.hist(data, bins='auto', alpha=.3, normed=True)

##############################################################################
# As expected, the result is rather silly, since we are only fitting *one*
# of the two gaussians.

##############################################################################
# Fit Gaussian Mixture Model (GMM)
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Assuming the data is the sum of one or more gaussians.
# Easily handles multidimensional case as well.

gmm = GaussianMixture(n_components=2, covariance_type='spherical')
gmm.fit(data)

mu1 = gmm.means_[0, 0]
mu2 = gmm.means_[1, 0]
var1, var2 = gmm.covariances_
wgt1, wgt2 = gmm.weights_
print('''Fit:
      1: Mean {:.4}, var {:.4}, weight {:.4}
      2: Mean {:.4}, var {:.4}, weight {:.4}
'''.format(mu1, var1, wgt1, mu2, var2, wgt2))

plt.hist(data, bins='auto', alpha=.3, normed=True)
plt.vlines((mu1, mu2), ymin=0, ymax=0.35, label='Fitted Means')
plt.plot(x, norm.pdf(x, mu1, np.sqrt(var1)))
plt.plot(x, norm.pdf(x, mu2, np.sqrt(var2)))
plt.legend()
plt.title('Gaussian Mixture Model')

##############################################################################
# Kernel Density: (non-parametric)
# --------------------------------
#
# If we have no strong assumptions about the underlying pdf.
#
# "Smooth out" each event with a kernel (e.g. gaussian) of
# a certain bandwidth, then add together all these mini-functions.
#
# The "bandwidth" (width of the kernel function) depends on the data, and
# can be estimated using cross-validation + maximum likelihood

##############################################################################
# in Statsmodels

dens = sm.nonparametric.KDEUnivariate(data)
dens.fit()

kde_sm = dens.evaluate(x)
plt.fill(x, kde_sm, alpha=.5, label='KDE')
plt.hist(data, bins='auto', alpha=.3, normed=True)

##############################################################################
# in scikit-learn

params = {'bandwidth': np.logspace(-2, 2, 50)}
grid = GridSearchCV(KernelDensity(), params)
grid.fit(data)

print("best bandwidth: {0}".format(grid.best_estimator_.bandwidth))

# use the best estimator to compute the kernel density estimate
kde_best = grid.best_estimator_
kde_sk = np.exp(
    kde_best.score_samples(x[:, np.newaxis])
)
plt.fill(x, kde_sk, alpha=.5, label='KDE')
plt.hist(data, bins='auto', alpha=.3, normed=True)

##############################################################################
# References
# ----------
#
# - B.W. Silverman, “Density Estimation for Statistics and Data Analysis”
# - Hastie, Tibshirani and Friedman, “The Elements of Statistical Learning: Data Mining, Inference, and Prediction”, Springer (2009)
# - Liu, R., Yang, L. “Kernel estimation of multivariate cumulative distribution function.” Journal of Nonparametric Statistics (2008)
# - Scargle, J., et al. "Studies in astronomical time series analysis. vi. Bayesian Block Representations." The Astrophysical Journal 764.2 (2013)
<|endoftext|>


<commit_before># -*- encoding: utf8 -*-
#
# Copyright (c) 2007 The PyAMF Project. All rights reserved.
#
# Nick Joyce
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""
Simple Python echo test U{WGSI<http://wsgi.org>} server for Flash Remoting.

You can use this example with the echo_test.swf client on the
U{EchoTest<http://pyamf.org/wiki/EchoTest>} wiki page.

@author: U{Nick Joyce<mailto:nick@boxdesign.co.uk>}
"""

import pyamf

class RemoteClass(object):
    """
    This Python class is mapped to the clientside Actionscript class.
    """
    pass

class ExternalizableClass(object):
    pass

def read_ec(obj, input):
    assert input.readBoolean() == True
    assert input.readBoolean() == False
    assert input.readByte() == 0
    assert input.readByte() == -1
    assert input.readByte() == 1
    assert input.readByte() == 127
    assert input.readByte() == -127
    assert input.readDouble() == 1.0
    assert input.readFloat() == 2.0
    assert input.readInt() == 0
    assert input.readInt() == -1
    assert input.readInt() == 1
    input.readMultiByte(7, 'iso-8859-1')
    input.readMultiByte(14, 'utf8')
    assert input.readObject() == [1, 'one', 1.0]
    assert input.readShort() == 0
    assert input.readShort() == -1
    assert input.readShort() == 1
    assert input.readUnsignedInt() == 0
    assert input.readUnsignedInt() == 1
    assert input.readUTF() == "Hello world!"
    assert input.readUTFBytes(12) == "Hello world!"

def write_ec(obj, output):
    output.writeBoolean(True)
    output.writeBoolean(False)
    output.writeByte(0)
    output.writeByte(-1)
    output.writeByte(1)
    output.writeByte(127)
    output.writeByte(-127)
    output.writeDouble(1.0)
    output.writeFloat(2.0)
    output.writeInt(0)
    output.writeInt(-1)
    output.writeInt(1)
    output.writeMultiByte(u"\xe4\xf6\xfc\xc4\xd6\xdc\xdf", 'iso-8859-1')
    output.writeMultiByte(u"\xe4\xf6\xfc\xc4\xd6\xdc\xdf", 'utf8')
    output.writeObject([1, 'one', 1])
    output.writeShort(0)
    output.writeShort(-1)
    output.writeShort(1)
    output.writeUnsignedInt(0)
    output.writeUnsignedInt(1)
    output.writeUTF("Hello world!")
    output.writeUTFBytes("Hello world!")

#: Map ActionScript class to Python class
pyamf.register_class(RemoteClass, 'org.red5.server.webapp.echo.RemoteClass')
pyamf.register_class(ExternalizableClass, 'org.red5.server.webapp.echo.ExternalizableClass',
    write_func=write_ec, read_func=read_ec)

def echo(data):
    """
    Return data back to the client.

    @type data:
    @param data: Decoded AS->Python data
    """
    return data

if __name__ == '__main__':
    import sys
    from __init__ import parse_args, run_server

    options = parse_args(sys.argv[1:])

    run_server('Echo Test', options[0], {'echo': echo})
<commit_msg>Added docs to echo_test<commit_after># -*- encoding: utf8 -*-
#
# Copyright (c) 2007 The PyAMF Project. All rights reserved.
#
# Nick Joyce
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""
Simple Python echo test U{WGSI<http://wsgi.org>} server for Flash Remoting.

You can use this example with the echo_test.swf client on the
U{EchoTest<http://pyamf.org/wiki/EchoTest>} wiki page.

@author: U{Nick Joyce<mailto:nick@boxdesign.co.uk>}
"""

from pyamf import register_class

ECHO_NS = 'org.red5.server.webapp.echo'

class RemoteClass(object):
    """
    This Python class is mapped to the clientside Actionscript class.
    """
    pass

class ExternalizableClass(object):
    pass

def echo(data):
    """
    Return data back to the client.

    @type data: mixed
    @param data: Decoded AS->Python data
    """
    # Simply return the data back to the client
    return data

def read_ec(obj, input):
    """
    This function is invoked when the obj needs to be unserialized.

    @type obj: L{ExternalizableClass}
    @param obj: The object in question.
    @param input: The input stream to read from
    @type input L{pyamf.compat.DataInput}
    """
    assert input.readBoolean() == True
    assert input.readBoolean() == False
    assert input.readByte() == 0
    assert input.readByte() == -1
    assert input.readByte() == 1
    assert input.readByte() == 127
    assert input.readByte() == -127
    assert input.readDouble() == 1.0
    assert input.readFloat() == 2.0
    assert input.readInt() == 0
    assert input.readInt() == -1
    assert input.readInt() == 1
    input.readMultiByte(7, 'iso-8859-1')
    input.readMultiByte(14, 'utf8')
    assert input.readObject() == [1, 'one', 1.0]
    assert input.readShort() == 0
    assert input.readShort() == -1
    assert input.readShort() == 1
    assert input.readUnsignedInt() == 0
    assert input.readUnsignedInt() == 1
    assert input.readUTF() == "Hello world!"
    assert input.readUTFBytes(12) == "Hello world!"

def write_ec(obj, output):
    """
    This function is invoked when the obj needs to be serialized.

    @type obj: L{ExternalizableClass}
    @param obj: The object in question.
    @param input: The output stream to write to
    @type input L{pyamf.compat.DataOutput}
    """
    output.writeBoolean(True)
    output.writeBoolean(False)
    output.writeByte(0)
    output.writeByte(-1)
    output.writeByte(1)
    output.writeByte(127)
    output.writeByte(-127)
    output.writeDouble(1.0)
    output.writeFloat(2.0)
    output.writeInt(0)
    output.writeInt(-1)
    output.writeInt(1)
    output.writeMultiByte(u"\xe4\xf6\xfc\xc4\xd6\xdc\xdf", 'iso-8859-1')
    output.writeMultiByte(u"\xe4\xf6\xfc\xc4\xd6\xdc\xdf", 'utf8')
    output.writeObject([1, 'one', 1])
    output.writeShort(0)
    output.writeShort(-1)
    output.writeShort(1)
    output.writeUnsignedInt(0)
    output.writeUnsignedInt(1)
    output.writeUTF("Hello world!")
    output.writeUTFBytes("Hello world!")

#: Map ActionScript class to Python class
register_class(RemoteClass, '%s.%s' % (ECHO_NS, 'RemoteClass'))
register_class(ExternalizableClass, '%s.%s' % (ECHO_NS, 'ExternalizableClass'),
    write_func=write_ec, read_func=read_ec)

if __name__ == '__main__':
    import sys
    from __init__ import parse_args, run_server

    options = parse_args(sys.argv[1:])

    run_server('Echo Test', options[0], {'echo': echo})
<|endoftext|>


<commit_before># coding=utf-8
from . import main
from flask import render_template
from .tools import get_rlimits
import os
import psutil
import datetime
import netifaces

@main.route('/')
def index():
    info = os.uname()
    boot_time = datetime.datetime.fromtimestamp(psutil.boot_time())
    boot_time_format = boot_time.strftime("%Y-%m-%d %H:%M:%S")
    now_time = datetime.datetime.now()
    now_time_format = now_time.strftime("%Y-%m-%d %H:%M:%S")
    up_time = "{} 天 {} 小时 {} 分钟 {} 秒".format(
        (now_time - boot_time).days,
        str(now_time - boot_time).split('.')[0].split(':')[0],
        str(now_time - boot_time).split('.')[0].split(':')[1],
        str(now_time - boot_time).split('.')[0].split(':')[2]
    )

    return render_template('index.html',
                           sys_name=info[1],
                           kernel_name=info[0],
                           kernel_no=info[2],
                           kernel_version=info[3],
                           sys_framework=info[4],
                           now_time=now_time_format,
                           boot_time=boot_time_format,
                           up_time=up_time
                           )

@main.route('/users')
def all_user():
    users = psutil.users()

    return render_template('users.html', users=users)

@main.route('/cpu')
def cpu_info():
    logical_core_num = psutil.cpu_count()
    physical_core_num = psutil.cpu_count(logical=False)
    load_avg = os.getloadavg()
    cpu_time_percent = psutil.cpu_times_percent()
    else_percent = 0.0
    for i in range(5, 10):
        else_percent += cpu_time_percent[i]
    cpu_freq = psutil.cpu_freq()

    return render_template('cpu.html',
                           physical_core_num=physical_core_num,
                           logical_core_num=logical_core_num,
                           load_avg=load_avg,
                           cpu_time_percent=cpu_time_percent,
                           else_percent=else_percent,
                           cpu_freq=cpu_freq
                           )

@main.route('/memory')
def memory():
    memory_info = psutil.virtual_memory()
    swap_memory = psutil.swap_memory()

    return render_template('memory.html', memory_info=memory_info, swap_memory=swap_memory)

@main.route('/disks')
def disks():
    physical_disks_info = []
    physical_disk_partitions = psutil.disk_partitions()
    for physical_disk_partition in physical_disk_partitions:
        physical_disk_usage = psutil.disk_usage(physical_disk_partition.mountpoint)
        physical_disk = {
            'device': physical_disk_partition.device,
            'mount_point': physical_disk_partition.mountpoint,
            'type': physical_disk_partition.fstype,
            'options': physical_disk_partition.opts,
            'space_total': physical_disk_usage.total,
            'space_used': physical_disk_usage.used,
            'used_percent': physical_disk_usage.percent,
            'space_free': physical_disk_usage.free
        }
        physical_disks_info.append(physical_disk)

    disks_info = []
    disk_partitions_all = psutil.disk_partitions(all=True)
    for disk_partition in disk_partitions_all:
        disk_usage = psutil.disk_usage(disk_partition.mountpoint)
        disk = {
            'device': disk_partition.device,
            'mount_point': disk_partition.mountpoint,
            'type': disk_partition.fstype,
            'options': disk_partition.opts,
            'space_total': disk_usage.total,
            'space_used': disk_usage.used,
            'used_percent': disk_usage.percent,
            'space_free': disk_usage.free
        }
        disks_info.append(disk)

    disks_io_info = psutil.disk_io_counters(perdisk=True)

    return render_template('disks.html',
                           physical_disks_info=physical_disks_info,
                           disks_info=disks_info,
                           disks_io_info=disks_io_info
                           )

@main.route('/network')
def network():
    net_io_counts = psutil.net_io_counters(pernic=True)
    interfaces_io = []
    for interface in net_io_counts:
        interface_dict = {}
        addrs = netifaces.ifaddresses(interface)
        ipv4_addr_info = addrs.get(2, None)
        ipv4_addr = None
        if ipv4_addr_info:
            ipv4_addr = ipv4_addr_info[0].get('addr', None)
        ipv6_addr_info = addrs.get(10, None)
        ipv6_addr = None
        if ipv6_addr_info:
            ipv6_addr = ipv6_addr_info[0].get('addr', None)
        interface_dict[interface] = [
            addrs[17][0]['addr'],  # MAC address
            ipv4_addr,
            ipv6_addr,
            net_io_counts[interface].bytes_sent,
            net_io_counts[interface].bytes_recv,
            net_io_counts[interface].packets_sent,
            net_io_counts[interface].packets_recv,
            net_io_counts[interface].errin,
            net_io_counts[interface].errout,
            net_io_counts[interface].dropin,
            net_io_counts[interface].dropout
        ]
        interfaces_io.append(interface_dict)

        net_connections = psutil.net_connections(kind='all')

    return render_template('network.html', interfaces_io=interfaces_io, net_connections=net_connections)

@main.route('/processes')
def all_process():
    processes = []
    for p in psutil.process_iter():
        process_info = {
            'name': p.name(),
            'pid': p.pid,
            'username': p.username(),
            'cpu_percent': p.cpu_percent(),
            'memory_percent': p.memory_percent(),
            'memory_rss': p.memory_info().rss,
            'memory_vms': p.memory_info().vms,
            'status': p.status(),
            'created_time': p.create_time(),
            'cmdline': ' '.join(p.cmdline())
        }
        processes.append(process_info)
    return render_template('processes.html', processes=processes)

@main.route('/process/<int:pid>')
def process(pid):
    the_process = psutil.Process(pid)
    parent_process = the_process.parent()
    process_info = the_process.as_dict()
    process_info['parent'] = parent_process.name() if parent_process else None
    process_info['cpu_affinity'] = [str(n) for n in process_info['cpu_affinity']]
    process_info['limits'] = get_rlimits(the_process)

    return render_template('process.html', process_info=process_info)
<commit_msg>后台代码网络信息部分改善<commit_after># coding=utf-8
from . import main
from flask import render_template
from .tools import get_rlimits
import os
import psutil
import datetime
import netifaces

@main.route('/')
def index():
    info = os.uname()
    boot_time = datetime.datetime.fromtimestamp(psutil.boot_time())
    boot_time_format = boot_time.strftime("%Y-%m-%d %H:%M:%S")
    now_time = datetime.datetime.now()
    now_time_format = now_time.strftime("%Y-%m-%d %H:%M:%S")
    up_time = "{} 天 {} 小时 {} 分钟 {} 秒".format(
        (now_time - boot_time).days,
        str(now_time - boot_time).split('.')[0].split(':')[0],
        str(now_time - boot_time).split('.')[0].split(':')[1],
        str(now_time - boot_time).split('.')[0].split(':')[2]
    )

    return render_template('index.html',
                           sys_name=info[1],
                           kernel_name=info[0],
                           kernel_no=info[2],
                           kernel_version=info[3],
                           sys_framework=info[4],
                           now_time=now_time_format,
                           boot_time=boot_time_format,
                           up_time=up_time
                           )

@main.route('/users')
def all_user():
    users = psutil.users()

    return render_template('users.html', users=users)

@main.route('/cpu')
def cpu_info():
    logical_core_num = psutil.cpu_count()
    physical_core_num = psutil.cpu_count(logical=False)
    load_avg = os.getloadavg()
    cpu_time_percent = psutil.cpu_times_percent()
    else_percent = 0.0
    for i in range(5, 10):
        else_percent += cpu_time_percent[i]
    cpu_freq = psutil.cpu_freq()

    return render_template('cpu.html',
                           physical_core_num=physical_core_num,
                           logical_core_num=logical_core_num,
                           load_avg=load_avg,
                           cpu_time_percent=cpu_time_percent,
                           else_percent=else_percent,
                           cpu_freq=cpu_freq
                           )

@main.route('/memory')
def memory():
    memory_info = psutil.virtual_memory()
    swap_memory = psutil.swap_memory()

    return render_template('memory.html', memory_info=memory_info, swap_memory=swap_memory)

@main.route('/disks')
def disks():
    physical_disks_info = []
    physical_disk_partitions = psutil.disk_partitions()
    for physical_disk_partition in physical_disk_partitions:
        physical_disk_usage = psutil.disk_usage(physical_disk_partition.mountpoint)
        physical_disk = {
            'device': physical_disk_partition.device,
            'mount_point': physical_disk_partition.mountpoint,
            'type': physical_disk_partition.fstype,
            'options': physical_disk_partition.opts,
            'space_total': physical_disk_usage.total,
            'space_used': physical_disk_usage.used,
            'used_percent': physical_disk_usage.percent,
            'space_free': physical_disk_usage.free
        }
        physical_disks_info.append(physical_disk)

    disks_info = []
    disk_partitions_all = psutil.disk_partitions(all=True)
    for disk_partition in disk_partitions_all:
        disk_usage = psutil.disk_usage(disk_partition.mountpoint)
        disk = {
            'device': disk_partition.device,
            'mount_point': disk_partition.mountpoint,
            'type': disk_partition.fstype,
            'options': disk_partition.opts,
            'space_total': disk_usage.total,
            'space_used': disk_usage.used,
            'used_percent': disk_usage.percent,
            'space_free': disk_usage.free
        }
        disks_info.append(disk)

    disks_io_info = psutil.disk_io_counters(perdisk=True)

    return render_template('disks.html',
                           physical_disks_info=physical_disks_info,
                           disks_info=disks_info,
                           disks_io_info=disks_io_info
                           )

@main.route('/network')
def network():
    net_io_counts = psutil.net_io_counters(pernic=True)
    interfaces_io = []
    for interface in net_io_counts:
        interface_dict = {}
        addrs = netifaces.ifaddresses(interface)
        ipv4_addr_info = addrs.get(2, None)
        ipv4_addr = None
        if ipv4_addr_info:
            ipv4_addr = ipv4_addr_info[0].get('addr', None)
        ipv6_addr_info = addrs.get(10, None)
        ipv6_addr = None
        if ipv6_addr_info:
            ipv6_addr = ipv6_addr_info[0].get('addr', None)
        interface_dict[interface] = [
            addrs[17][0]['addr'],  # MAC address
            ipv4_addr,
            ipv6_addr,
            net_io_counts[interface].bytes_sent,
            net_io_counts[interface].bytes_recv,
            net_io_counts[interface].packets_sent,
            net_io_counts[interface].packets_recv,
            net_io_counts[interface].errin,
            net_io_counts[interface].errout,
            net_io_counts[interface].dropin,
            net_io_counts[interface].dropout
        ]
        interfaces_io.append(interface_dict)

    net_connections = psutil.net_connections(kind='all')

    return render_template('network.html', interfaces_io=interfaces_io, net_connections=net_connections)

@main.route('/processes')
def all_process():
    processes = []
    for p in psutil.process_iter():
        process_info = {
            'name': p.name(),
            'pid': p.pid,
            'username': p.username(),
            'cpu_percent': p.cpu_percent(),
            'memory_percent': p.memory_percent(),
            'memory_rss': p.memory_info().rss,
            'memory_vms': p.memory_info().vms,
            'status': p.status(),
            'created_time': p.create_time(),
            'cmdline': ' '.join(p.cmdline())
        }
        processes.append(process_info)
    return render_template('processes.html', processes=processes)

@main.route('/process/<int:pid>')
def process(pid):
    the_process = psutil.Process(pid)
    parent_process = the_process.parent()
    process_info = the_process.as_dict()
    process_info['parent'] = parent_process.name() if parent_process else None
    process_info['cpu_affinity'] = [str(n) for n in process_info['cpu_affinity']]
    process_info['limits'] = get_rlimits(the_process)

    return render_template('process.html', process_info=process_info)
<|endoftext|>


<commit_before>import re
import nltk
import textblob

from textblob.blob import BaseBlob
from textblob.en.taggers import PatternTagger

# BaseBlob uses NLTKTagger as a pos_tagger, but it works wrong
BaseBlob.pos_tagger = PatternTagger()

from typing import Optional

from .base import RegexDetector
from ..filth import SkypeFilth

class SkypeDetector(RegexDetector):
    """Skype usernames tend to be used inline in dirty dirty text quite
    often but also appear as ``skype: {{SKYPE}}`` quite a bit. This method
    looks at words within ``word_radius`` words of "skype" for things that
    appear to be misspelled or have punctuation in them as a means to
    identify skype usernames.

    Default ``word_radius`` is 10, corresponding with the rough scale of
    half of a sentence before or after the word "skype" is used. Increasing
    the ``word_radius`` will increase the false positive rate and
    decreasing the ``word_radius`` will increase the false negative rate.
    """
    filth_cls = SkypeFilth
    name = 'skype'

    word_radius = 10

    # these two regular expressions are used to validate a skype usernames.
    # _TOKEN is the core regular expression that is used to chunk text into
    # tokens to make sure all valid skype usernames are considered the same
    # token. Importantly, the word "skype" must pass the _SKYPE regex.
    # SKYPE_TOKEN is used to tokenize text and SKYPE_USERNAME is the same thing
    # but with the 6-32 character limit imposed on the username. adapted from
    # http://bit.ly/1FQs1hD
    _SKYPE = r'[a-zA-Z][a-zA-Z0-9_\-\,\.]'
    SKYPE_TOKEN = re.compile(_SKYPE+'+')
    SKYPE_USERNAME = re.compile(_SKYPE+'{5,31}')

    def iter_filth(self, text, document_name: Optional[str] = None):

        # find 'skype' in the text using a customized tokenizer. this makes
        # sure that all valid skype usernames are kept as tokens and not split
        # into different words
        tokenizer = nltk.tokenize.regexp.RegexpTokenizer(
            self.SKYPE_TOKEN
        )
        blob = textblob.TextBlob(text, tokenizer=tokenizer)
        skype_indices, tokens = [], []
        for i, token in enumerate(blob.tokens):
            tokens.append(token)
            if 'skype' in token.lower():
                skype_indices.append(i)

        # go through the words before and after skype words to identify
        # potential skype usernames.
        skype_usernames = []
        for i in skype_indices:
            jmin = max(i-self.word_radius, 0)
            jmax = min(i+self.word_radius+1, len(tokens))
            for j in list(range(jmin, i)) + list(range(i+1, jmax)):
                token = tokens[j]
                if self.SKYPE_USERNAME.match(token):

                    # this token is a valid skype username. Most skype
                    # usernames appear to be misspelled words. Word.spellcheck
                    # does not handle the situation of an all caps word very
                    # well, so we cast these to all lower case before checking
                    # whether the word is misspelled
                    if token.isupper():
                        token = token.lower()
                    word = textblob.Word(token)
                    suggestions = word.spellcheck()
                    corrected_word, score = suggestions[0]
                    if score < 0.5:
                        skype_usernames.append(token)

        # replace all skype usernames
        if skype_usernames:
            self.regex = re.compile('|'.join(skype_usernames))
            return super(SkypeDetector, self).iter_filth(text, document_name=document_name)

        return []
<commit_msg>register detector and pep8<commit_after>import re
import nltk
import textblob

from textblob.blob import BaseBlob
from textblob.en.taggers import PatternTagger

from typing import Optional

from . import register_detector
from .base import RegexDetector
from ..filth import SkypeFilth

# BaseBlob uses NLTKTagger as a pos_tagger, but it works wrong
BaseBlob.pos_tagger = PatternTagger()

class SkypeDetector(RegexDetector):
    """Skype usernames tend to be used inline in dirty dirty text quite
    often but also appear as ``skype: {{SKYPE}}`` quite a bit. This method
    looks at words within ``word_radius`` words of "skype" for things that
    appear to be misspelled or have punctuation in them as a means to
    identify skype usernames.

    Default ``word_radius`` is 10, corresponding with the rough scale of
    half of a sentence before or after the word "skype" is used. Increasing
    the ``word_radius`` will increase the false positive rate and
    decreasing the ``word_radius`` will increase the false negative rate.
    """
    filth_cls = SkypeFilth
    name = 'skype'

    word_radius = 10

    # these two regular expressions are used to validate a skype usernames.
    # _TOKEN is the core regular expression that is used to chunk text into
    # tokens to make sure all valid skype usernames are considered the same
    # token. Importantly, the word "skype" must pass the _SKYPE regex.
    # SKYPE_TOKEN is used to tokenize text and SKYPE_USERNAME is the same thing
    # but with the 6-32 character limit imposed on the username. adapted from
    # http://bit.ly/1FQs1hD
    _SKYPE = r'[a-zA-Z][a-zA-Z0-9_\-\,\.]'
    SKYPE_TOKEN = re.compile(_SKYPE+'+')
    SKYPE_USERNAME = re.compile(_SKYPE+'{5,31}')

    def iter_filth(self, text, document_name: Optional[str] = None):

        # find 'skype' in the text using a customized tokenizer. this makes
        # sure that all valid skype usernames are kept as tokens and not split
        # into different words
        tokenizer = nltk.tokenize.regexp.RegexpTokenizer(
            self.SKYPE_TOKEN
        )
        blob = textblob.TextBlob(text, tokenizer=tokenizer)
        skype_indices, tokens = [], []
        for i, token in enumerate(blob.tokens):
            tokens.append(token)
            if 'skype' in token.lower():
                skype_indices.append(i)

        # go through the words before and after skype words to identify
        # potential skype usernames.
        skype_usernames = []
        for i in skype_indices:
            jmin = max(i-self.word_radius, 0)
            jmax = min(i+self.word_radius+1, len(tokens))
            for j in list(range(jmin, i)) + list(range(i+1, jmax)):
                token = tokens[j]
                if self.SKYPE_USERNAME.match(token):

                    # this token is a valid skype username. Most skype
                    # usernames appear to be misspelled words. Word.spellcheck
                    # does not handle the situation of an all caps word very
                    # well, so we cast these to all lower case before checking
                    # whether the word is misspelled
                    if token.isupper():
                        token = token.lower()
                    word = textblob.Word(token)
                    suggestions = word.spellcheck()
                    corrected_word, score = suggestions[0]
                    if score < 0.5:
                        skype_usernames.append(token)

        # replace all skype usernames
        if skype_usernames:
            self.regex = re.compile('|'.join(skype_usernames))
            return super(SkypeDetector, self).iter_filth(text, document_name=document_name)

        return []

register_detector(SkypeDetector, autoload=False)
<|endoftext|>


<commit_before>import bpy
import os
import pdb

COLOURS = {"PHOSPHATE": (255, 255, 0),
           "SUGAR": (255, 0, 0),
           "GUANINE": (0, 255, 0),
           "ADENINE": (0, 0, 255),
           "CYTOSINE": (0, 255, 255),
           "THYMINE": (255, 0, 255)}

MATERIALS = {}
for (k, v) in COLOURS.items():
    # Make Material
    mat = bpy.data.materials.new(k)
    mat.diffuse_color = v
    mat.diffuse_shader = 'LAMBERT'
    mat.diffuse_intensity = 1.0
    mat.specular_color = v
    mat.specular_shader = 'COOKTORR'
    mat.specular_intensity = 0.5
    mat.alpha = (1)
    mat.ambient = 0.05
    MATERIALS[k] = mat

def do_world():
    """Make the world look nice
    """
    bpy.data.worlds["World"].use_sky_blend = 1
    bpy.data.worlds["World"].horizon_color = (0.27, 1, 1)
    bpy.data.worlds["World"].zenith_color = (1, 1, 1)
    bpy.data.worlds["World"].ambient_color = (1, 1, 1)
    bpy.data.worlds["World"].light_settings.use_environment_light = 1
    return None

def make_movie(infile, camera_positions, camera_directions):
    """
    """
    return None

def make_render(infile, camera_position, camera_rotation, outfile, cut=100):
    """Render a frame

    make_render(infile, camera_position, camera_rotation, outfile, cut=100)

    args:
        infile: blender file to Render
        camera_position: 3-element list for position
        camera_rotation: 3-element list for rotation (degrees)
        outfile: output image render filename
    """
    assert os.path.exists(infile), "Could not find infile"
    assert len(camera_position) == 3, "camera position invalid"
    assert len(camera_position) == 3, "camera direction invalid"
    print("Opening {}".format(infile))
    bpy.ops.wm.open_mainfile(filepath=infile)
    camera_rotation = [3.14159/180.*r for r in camera_rotation]
    cam = bpy.data.cameras.new("RenderCamera")
    cam_ob = bpy.data.objects.new("RenderCamera", cam)
    bpy.context.scene.objects.link(cam_ob)
    cam_ob.rotation_euler = camera_rotation
    cam_ob.location = camera_position

    bpy.context.scene.objects.active =\
        bpy.context.scene.objects["RenderCamera"]
    bpy.ops.render.render()
    bpy.ops.image.save_as(outfile)
    return None

def assemble_geometry(infile, outfile, units, filepath, placement_dict,
                      ellipse=None):
    """Assemble placement volumes into a blender file.

    assemble_geometry(infile, outfile, units, filepath, placement_dict,
                      **kwargs)

    args:
        infile: input filename (note [1])
        outfile: output filename (must have suffix .blend)
        units: size of each placement volume cube (note [2])
        filepath: directory where input and output file should be
        placement_dict: dictionary, note[3].

    kwargs:
        ellipse: a list of three semi-major axis values that can serve as
                 an xyz mask to restrict placements.
                 Placements are only made inside the ellipse defined by
                 the xyz values given.

    note:
        [1]
        structure of input file (whitespace separated columns)
            IDX KIND POS_X POS_Y POS_Z EUL_PSI EUL_THETA EUL_PHI

        [2]
        The file instructing blender where to place volumes may be parametrised
        so it can work with a variety of placement groups. This parameter
        is a multiplicative factor that multiplies the POS_X, POS_Y and POS_Z
        columns.
        For example, the input file may provide placements only in integer
        units in a grid, and then the multiplicative value here can be used
        to specify the size of each placement, ex: 50 units (units are relative
        to whatever units are in the placement files)

        [3]
        placement_dict contains key/value pairs for:
            (kindname, (blendfile, groupname))
            kindname is the name the group has in the input placement file
            blendfile should have an absolute path
            groupname is the name of the desired group in the the blendfile
    """
    units = float(units)
    assert outfile[-6:] == ".blend", "outfile param needs .blend suffix"
    if ellipse is not None:
        assert len(ellipse) == 3, "ellipse kwarg contains 3 elements"

    bpy.ops.wm.read_homefile()
    for t in ["MESH", "SURFACE", "CURVE", "META", "FONT", "CAMERA", "LAMP"]:
        bpy.ops.object.select_by_type(type=t)
        bpy.ops.object.delete()

    # set up file
    do_world()

    # read_in
    infile  = open(os.path.join(filepath, infile), "r")
    names = []
    positions = []
    rotations = []
    for line in infile:
        if line[0] != "#":
            ll = line.split()
            pos = [float(ii)*units for ii in ll[2:5]]
            rot = [float(ii) for ii in ll[5:8]]
            valid = True
            if ellipse is not None:
                r = (pos[0]/ellipse[0])**2 + (pos[1]/ellipse[1])**2 +\
                    (pos[2]/ellipse[2])**2
                if r > 1:
                    valid = False
            if valid is True:
                names.append(ll[1])
                positions.append(pos)
                rotations.append(rot)

    for (kind, (path, groupname)) in placement_dict.items():
        with bpy.data.libraries.load(path, link=True) as (data_from, data_to):
            data_to.groups = data_from.groups

    for (ii, (kind, pos, rot)) in enumerate(zip(names, positions, rotations)):
        print("Placement", ii, kind, pos, rot)
        bpy.ops.object.group_instance_add(group=placement_dict[kind][1],
                                          location=pos,
                                          rotation=rot)

    outfile = os.path.join(filepath, outfile)
    bpy.ops.wm.save_as_mainfile(filepath=outfile)
    return None

def placement_volume(infile, outfile, filepath):
    """Load a placement volume into blender and save it

    args:
        infile: string, input file name (see below for structure)
        outfile: output file (with .blend extension)
        filepath: directory where both infile and outfile should/will be.

    note:
        [1]
        structure of input file:
        col00:    NAME
        col01:    SHAPE
        col02:    CHAIN_ID
        col03:    STRAND_ID
        col04:    BP_INDEX
        col05-07: SIZE_X SIZE_Y SIZE_Z
        col08-10: POS_X POS_Y POS_Z
        col11-13: ROT_X ROT_Y ROT_Z

        [2]
        A default groupname is created for each placement, corresponding
        to the value of outfile, without the .blend suffix
    """
    assert outfile[-6:] == ".blend", "outfile param needs .blend suffix"

    bpy.ops.wm.read_homefile()
    for t in ["MESH", "SURFACE", "CURVE", "META", "FONT", "CAMERA", "LAMP"]:
        bpy.ops.object.select_by_type(type=t)
        bpy.ops.object.delete()

    do_world()

    infile  = open(os.path.join(filepath, infile), "r")
    names = []
    sizes = []
    positions = []
    rotations = []
    for line in infile:
        if line[0] != "#":
            ll = line.split()
            names.append(ll[0])
            sz = [float(ii) for ii in ll[5:8]]
            pos = [float(ii) for ii in ll[8:11]]
            rot = [float(ii) for ii in ll[11:14]]

            sizes.append(sz)
            positions.append(pos)
            rotations.append(rot)

    groupname = outfile.replace(".blend", "")
    bpy.ops.group.create(name=groupname)
    for (ii, (name, sz, pos, rot)) in\
            enumerate(zip(names, sizes, positions, rotations)):
        if (ii//100)*100 == ii:
            print("{0}/{1}".format(ii, len(names)))
        bpy.ops.mesh.primitive_uv_sphere_add(segments=16, ring_count=8,
                                             size=1,
                                             view_align=False,
                                             enter_editmode=False,
                                             location=pos,
                                             rotation=rot)
        bpy.ops.object.group_link(group=groupname)

        ob = bpy.context.object
        ob.scale = sz
        try:
            color = COLOURS[name.upper()]
        except KeyError:
            print("Material not defined for {}".format(name.upper()))
            color = (100, 100, 100)

        mat = bpy.data.materials.new(k)
        mat.diffuse_color = color
        mat.diffuse_shader = 'LAMBERT'
        mat.diffuse_intensity = 1.0
        mat.specular_color = (1, 1, 1)
        mat.specular_shader = 'COOKTORR'
        mat.specular_intensity = 0.5
        mat.alpha = (1)
        mat.ambient = 0.05
        ob.data.materials.append(mat)

    outfile = os.path.join(filepath, outfile)
    bpy.ops.wm.save_as_mainfile(filepath=outfile)
    return None
<commit_msg>writing the vis renderer functions<commit_after>import bpy
import os
import math
from mathutils import Vector

COLOURS = {"PHOSPHATE": (255, 255, 0),
           "SUGAR": (255, 0, 0),
           "GUANINE": (0, 255, 0),
           "ADENINE": (0, 0, 255),
           "CYTOSINE": (0, 255, 255),
           "THYMINE": (255, 0, 255)}

MATERIALS = {}
for (k, v) in COLOURS.items():
    # Make Material
    mat = bpy.data.materials.new(k)
    mat.diffuse_color = v
    mat.diffuse_shader = 'LAMBERT'
    mat.diffuse_intensity = 1.0
    mat.specular_color = v
    mat.specular_shader = 'COOKTORR'
    mat.specular_intensity = 0.5
    mat.alpha = (1)
    mat.ambient = 0.05
    MATERIALS[k] = mat

def do_world():
    """Make the world look nice
    """
    bpy.data.worlds["World"].use_sky_blend = 1
    bpy.data.worlds["World"].horizon_color = (0.27, 1, 1)
    bpy.data.worlds["World"].zenith_color = (1, 1, 1)
    bpy.data.worlds["World"].ambient_color = (1, 1, 1)
    bpy.data.worlds["World"].light_settings.use_environment_light = 1
    bpy.data.worlds["World"].light_settings.environment_energy = 0.1
    return None

def make_movie(infile, centre, distance, outfile, clip=100):
    """Make a movie

    make_movie(infile, centre, distance, outfile, clip=100)

    For a given infile, the camera will spin around the geometry
        (around the z-axis), then fly to through along the x-axis to the
        origin, before spinning once more.
    """
    def point_camera(obj_camera, point):
        loc_camera = obj_camera.matrix_world.to_translation()
        direction = Vector(point) - loc_camera
        # point the cameras '-Z' and use its 'Y' as up
        rot_quat = direction.to_track_quat('-Z', 'Y')
        # assume we're using euler rotation
        obj_camera.rotation_euler = rot_quat.to_euler()

    assert os.path.exists(infile), "Could not find infile"
    assert len(centre) == 3, "centre position invalid"
    assert distance > 0, "distance is a positive integer"

    print("Opening {}".format(infile))
    bpy.ops.wm.open_mainfile(filepath=infile)
    cam = bpy.data.cameras.new("RenderCamera")

    cam_ob = bpy.data.objects.new("RenderCamera", cam)
    sceneKey = bpy.data.scenes.keys()[0]
    bpy.data.scenes[sceneKey].objects.link(cam_ob)
    scene = bpy.data.scenes[sceneKey]
    scene.frame_start = 1
    scene.frame_end = 101
    positions = []
    for ii in range(24):
        angle = (ii + 1) * 3.14159 / 12.  # a little less than pi
        positions.append([distance*math.cos(angle),
                          distance*math.sin(angle),
                          0])
    cam_ob.location = (distance, 0, 0)
    point_camera(cam_ob, (0, 0, 0))
    the_frame = 0
    cam_ob.keyframe_insert(data_path="location", frame=the_frame)
    cam_ob.keyframe_insert(data_path="rotation_euler", frame=the_frame)
    for position in positions:
        the_frame = the_frame + 2
        bpy.data.scenes[sceneKey].frame_set(the_frame)
        cam_ob.location = position
        point_camera(cam_ob, (0, 0, 0))
        cam_ob.keyframe_insert(data_path="location", frame=the_frame)
        cam_ob.keyframe_insert(data_path="rotation_euler", frame=the_frame)

    # zoom to center
    the_frame = the_frame + 10
    bpy.data.scenes[sceneKey].frame_set(the_frame)
    cam_ob.location = (0, 0, 0)
    cam_ob.keyframe_insert(data_path="location", frame=the_frame)
    for ii in range(6):
        angle = (ii + 1) * 3.14159 / 12.  # a little less than pi
        the_frame = the_frame + 2
        bpy.data.scenes[sceneKey].frame_set(the_frame)
        cam_ob.rotation_euler[2] = angle
        cam_ob.keyframe_insert(data_path="rotation_euler", frame=the_frame)

    for ii in range(6):
        angle = (ii + 1) * 3.14159 / 12.  # a little less than pi
        the_frame = the_frame + 2
        bpy.data.scenes[sceneKey].frame_set(the_frame)
        cam_ob.rotation_euler[1] = angle
        cam_ob.keyframe_insert(data_path="rotation_euler", frame=the_frame)

    # zoom back out
    the_frame = the_frame + 10
    bpy.data.scenes[sceneKey].frame_set(the_frame)
    cam_ob.location = [0, 0, -1*distance]
    cam_ob.keyframe_insert(data_path="location", frame=the_frame)

    bpy.data.cameras["RenderCamera"].clip_end = clip
    bpy.data.scenes[sceneKey].camera = cam_ob
    bpy.data.scenes[sceneKey].render.image_settings.file_format = 'H264'
    bpy.data.scenes[sceneKey].render.fps = 24
    bpy.data.scenes[sceneKey].render.frame_map_old = 1
    bpy.data.scenes[sceneKey].render.frame_map_new = 24
    bpy.data.scenes[sceneKey].render.filepath = outfile
    bpy.ops.render.render()
    return None

def make_render(infile, camera_position, camera_rotation, outfile, clip=100):
    """Render a frame

    make_render(infile, camera_position, camera_rotation, outfile, clip=100)

    args:
        infile: blender file to Render
        camera_position: 3-element list for position
        camera_rotation: 3-element list for rotation (degrees)
        outfile: output image render filename
    """
    assert os.path.exists(infile), "Could not find infile"
    assert len(camera_position) == 3, "camera position invalid"
    assert len(camera_position) == 3, "camera direction invalid"
    print("Opening {}".format(infile))
    bpy.ops.wm.open_mainfile(filepath=infile)
    camera_rotation = [3.14159/180.*r for r in camera_rotation]
    cam = bpy.data.cameras.new("RenderCamera")
    cam_ob = bpy.data.objects.new("RenderCamera", cam)
    bpy.data.scenes[sceneKey].objects.link(cam_ob)
    cam_ob.rotation_euler = camera_rotation
    cam_ob.location = camera_position
    bpy.data.cameras["RenderCamera"].clip_end = clip
    sceneKey = bpy.data.scenes.keys()[0]
    bpy.data.scenes[sceneKey].camera = cam_ob
    bpy.context.scene[sceneKey].render.image_settings.file_format = 'PNG'
    bpy.data.scenes[sceneKey].render.filepath = outfile
    bpy.ops.render.render(write_still=True)
    return None

def assemble_geometry(infile, outfile, units, filepath, placement_dict,
                      ellipse=None):
    """Assemble placement volumes into a blender file.

    assemble_geometry(infile, outfile, units, filepath, placement_dict,
                      **kwargs)

    args:
        infile: input filename (note [1])
        outfile: output filename (must have suffix .blend)
        units: size of each placement volume cube (note [2])
        filepath: directory where input and output file should be
        placement_dict: dictionary, note[3].

    kwargs:
        ellipse: a list of three semi-major axis values that can serve as
                 an xyz mask to restrict placements.
                 Placements are only made inside the ellipse defined by
                 the xyz values given.

    note:
        [1]
        structure of input file (whitespace separated columns)
            IDX KIND POS_X POS_Y POS_Z EUL_PSI EUL_THETA EUL_PHI

        [2]
        The file instructing blender where to place volumes may be parametrised
        so it can work with a variety of placement groups. This parameter
        is a multiplicative factor that multiplies the POS_X, POS_Y and POS_Z
        columns.
        For example, the input file may provide placements only in integer
        units in a grid, and then the multiplicative value here can be used
        to specify the size of each placement, ex: 50 units (units are relative
        to whatever units are in the placement files)

        [3]
        placement_dict contains key/value pairs for:
            (kindname, (blendfile, groupname))
            kindname is the name the group has in the input placement file
            blendfile should have an absolute path
            groupname is the name of the desired group in the the blendfile
    """
    units = float(units)
    assert outfile[-6:] == ".blend", "outfile param needs .blend suffix"
    if ellipse is not None:
        assert len(ellipse) == 3, "ellipse kwarg contains 3 elements"

    bpy.ops.wm.read_homefile()
    for t in ["MESH", "SURFACE", "CURVE", "META", "FONT", "CAMERA", "LAMP"]:
        bpy.ops.object.select_by_type(type=t)
        bpy.ops.object.delete()

    # set up file
    do_world()

    # read_in
    infile  = open(os.path.join(filepath, infile), "r")
    names = []
    positions = []
    rotations = []
    for line in infile:
        if line[0] != "#":
            ll = line.split()
            pos = [float(ii)*units for ii in ll[2:5]]
            rot = [float(ii) for ii in ll[5:8]]
            valid = True
            if ellipse is not None:
                r = (pos[0]/ellipse[0])**2 + (pos[1]/ellipse[1])**2 +\
                    (pos[2]/ellipse[2])**2
                if r > 1:
                    valid = False
            if valid is True:
                names.append(ll[1])
                positions.append(pos)
                rotations.append(rot)

    for (kind, (path, groupname)) in placement_dict.items():
        with bpy.data.libraries.load(path, link=True) as (data_from, data_to):
            data_to.groups = data_from.groups

    for (ii, (kind, pos, rot)) in enumerate(zip(names, positions, rotations)):
        print("Placement", ii, kind, pos, rot)
        bpy.ops.object.group_instance_add(group=placement_dict[kind][1],
                                          location=pos,
                                          rotation=rot)

    outfile = os.path.join(filepath, outfile)
    bpy.ops.wm.save_as_mainfile(filepath=outfile)
    return None

def placement_volume(infile, outfile, filepath):
    """Load a placement volume into blender and save it

    args:
        infile: string, input file name (see below for structure)
        outfile: output file (with .blend extension)
        filepath: directory where both infile and outfile should/will be.

    note:
        [1]
        structure of input file:
        col00:    NAME
        col01:    SHAPE
        col02:    CHAIN_ID
        col03:    STRAND_ID
        col04:    BP_INDEX
        col05-07: SIZE_X SIZE_Y SIZE_Z
        col08-10: POS_X POS_Y POS_Z
        col11-13: ROT_X ROT_Y ROT_Z

        [2]
        A default groupname is created for each placement, corresponding
        to the value of outfile, without the .blend suffix
    """
    assert outfile[-6:] == ".blend", "outfile param needs .blend suffix"

    bpy.ops.wm.read_homefile()
    for t in ["MESH", "SURFACE", "CURVE", "META", "FONT", "CAMERA", "LAMP"]:
        bpy.ops.object.select_by_type(type=t)
        bpy.ops.object.delete()

    do_world()

    infile  = open(os.path.join(filepath, infile), "r")
    names = []
    sizes = []
    positions = []
    rotations = []
    for line in infile:
        if line[0] != "#":
            ll = line.split()
            names.append(ll[0])
            sz = [float(ii) for ii in ll[5:8]]
            pos = [float(ii) for ii in ll[8:11]]
            rot = [float(ii) for ii in ll[11:14]]

            sizes.append(sz)
            positions.append(pos)
            rotations.append(rot)

    groupname = outfile.replace(".blend", "")
    bpy.ops.group.create(name=groupname)
    for (ii, (name, sz, pos, rot)) in\
            enumerate(zip(names, sizes, positions, rotations)):
        if (ii//100)*100 == ii:
            print("{0}/{1}".format(ii, len(names)))
        bpy.ops.mesh.primitive_uv_sphere_add(segments=16, ring_count=8,
                                             size=1,
                                             view_align=False,
                                             enter_editmode=False,
                                             location=pos,
                                             rotation=rot)
        bpy.ops.object.group_link(group=groupname)

        ob = bpy.context.object
        ob.scale = sz
        try:
            color = COLOURS[name.upper()]
        except KeyError:
            print("Material not defined for {}".format(name.upper()))
            color = (100, 100, 100)

        mat = bpy.data.materials.new(k)
        mat.diffuse_color = color
        mat.diffuse_shader = 'LAMBERT'
        mat.diffuse_intensity = 1.0
        mat.specular_color = (1, 1, 1)
        mat.specular_shader = 'COOKTORR'
        mat.specular_intensity = 0.5
        mat.alpha = (1)
        mat.ambient = 0.05
        ob.data.materials.append(mat)

    outfile = os.path.join(filepath, outfile)
    bpy.ops.wm.save_as_mainfile(filepath=outfile)
    return None
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import tornado.websocket
import tornado.web
import os
import webbrowser
from tornado.ioloop import IOLoop
import logging
import argparse
import threading
import time

from pkg_resources import resource_filename

from volta.analysis.wizard import VoltaWorker, EventPoller, PhoneWorker

logger = logging.getLogger(__name__)

class grabberThread(threading.Thread):
    """
    Drain a generator to a destination that answers to put(), in a thread
    """
    def __init__(self, source):
        super(grabberThread, self).__init__()
        self.source = source
        self._finished = threading.Event()
        self._interrupted = threading.Event()

    def run(self):
        self.source.startTest()
        self._finished.set()

    def wait(self, timeout=None):
        self._finished.wait(timeout=timeout)

    def close(self):
        self._interrupted.set()

class WizardWebSocket(tornado.websocket.WebSocketHandler):
    def open(self):
        self.write_message("Volta wizard started")
        self.volta = VoltaWorker()
        self.phone = PhoneWorker()
        self.write_message(u'Подключите коробку')
        # 1 - подключение коробки
        self.volta.device = EventPoller(self.volta.isUsbConnected)
        self.write_message(u'Коробка найдена! Подключите телефон.')
        # 2 - подключение телефона
        phone = EventPoller(self.phone.isPhoneConnected)
        self.write_message(u'Телефон найден: %s' % phone)

    def on_message(self, message):
        # 3 - установка apk
        # TODO
        # 4 - параметризация теста
        if message:
            self.duration = int(message)
        else:
            self.write_message(u'Вы не ввели длительность теста.')
        self.volta.setTestDuration(self.duration)
        self.write_message(u'Длительность теста будет %s секунд.' % self.duration)
        # pre5 - сброс logcat
        self.write_message(u'Чистим логи на телефоне (logcat)')
        EventPoller(self.phone.clearLogcat)
        self.write_message(u'Теперь отключите телефон')
        # 5 - отключение телефона
        EventPoller(self.phone.isPhoneDisconnected)
        self.write_message(u'Начинается тест, не забудьте помигать фонариком!')
        # 6 - запуск теста, мигание фонариком
        grabber = grabberThread(self.volta)
        grabber.run()

        counter = 0
        while counter < self.duration:
            progress = 100 * counter / self.duration
            if progress < 100:
                self.write_message('Прогресс: %s %%' % progress)
            step=10
            counter = counter + step
            time.sleep(step)

        grabber.wait(self.duration * 2)
        EventPoller(self.volta.isTestFinished)
        self.write_message(u'Готово. Подключите телефон')
        # 7 - подключение телефона
        EventPoller(self.phone.isPhoneConnected)
        self.write_message(u'Найден телефон, забираем логи с телефона')
        EventPoller(self.phone.dumpLogcatEvents)
        EventPoller(self.phone.getInfoAboutDevice)
        # 8 - заливка логов
        self.write_message(u'Собираем логи и заливаем логи в Лунапарк. '
                           u'Это может занять какое-то время.')
        jobid = self.volta.upload('./output.bin', './events.log')
        self.write_message(u'Lunapark URL' % jobid)
        # work finished, closing the connection
        self.close()

    def on_close(self):
        self.write_message('Volta wizard finished')

class IndexPage(tornado.web.RequestHandler):
    def get(self):
        """ index page w/ buttons """
        self.render(
            resource_filename(__name__, 'index.html'),
            title="Volta UI"
        )

def make_app():
    static_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'static')
    return tornado.web.Application([
        (r"/", IndexPage),
        (r"/wizard", WizardWebSocket),
        (r"/static/(.*)", tornado.web.StaticFileHandler, {"path": static_path}),
    ])

def main():
    logging.basicConfig(level=logging.DEBUG)

    parser = argparse.ArgumentParser(description='Configures ui tornado server.')
    parser.add_argument('--port', dest='port', default=8888, help='port for webserver (default: 8888)')
    args = parser.parse_args()

    app = make_app()
    app.listen(args.port)
    url = "http://localhost:{port}".format(port=args.port)
    webbrowser.open(url,new=2) #new=2 means open in new tab if possible
    IOLoop.instance().start()

if __name__ == "__main__":
    main()
<commit_msg>bugfixes<commit_after># -*- coding: utf-8 -*-

import tornado.websocket
import tornado.web
import os
import webbrowser
from tornado.ioloop import IOLoop
import logging
import argparse
import threading
import time

from pkg_resources import resource_filename

from volta.analysis.wizard import VoltaWorker, EventPoller, PhoneWorker

logger = logging.getLogger(__name__)

class grabberThread(threading.Thread):
    """
    Drain a generator to a destination that answers to put(), in a thread
    """
    def __init__(self, source):
        super(grabberThread, self).__init__()
        self.source = source
        self._finished = threading.Event()
        self._interrupted = threading.Event()

    def run(self):
        self.source.startTest()
        self._finished.set()

    def wait(self, timeout=None):
        self._finished.wait(timeout=timeout)

    def close(self):
        self._interrupted.set()

class WizardWebSocket(tornado.websocket.WebSocketHandler):
    def open(self):
        self.write_message("Volta wizard started")
        self.volta = VoltaWorker()
        self.phone = PhoneWorker()
        self.write_message(u'Подключите коробку')
        # 1 - подключение коробки
        self.volta.device = EventPoller(self.volta.isUsbConnected)
        self.write_message(u'Коробка найдена! Подключите телефон.')
        # 2 - подключение телефона
        phone = EventPoller(self.phone.isPhoneConnected)
        self.write_message(u'Телефон найден: %s' % phone)

    def on_message(self, message):
        # 3 - установка apk
        # TODO
        # 4 - параметризация теста
        if message:
            self.duration = int(message)
        else:
            self.write_message(u'Вы не ввели длительность теста.')
        self.volta.setTestDuration(self.duration)
        self.write_message(u'Длительность теста будет %s секунд.' % self.duration)
        # pre5 - сброс logcat
        self.write_message(u'Чистим логи на телефоне (logcat)')
        EventPoller(self.phone.clearLogcat)
        self.write_message(u'Теперь отключите телефон')
        # 5 - отключение телефона
        EventPoller(self.phone.isPhoneDisconnected)
        self.write_message(u'Начинается тест, не забудьте помигать фонариком!')
        # 6 - запуск теста, мигание фонариком
        grabber = grabberThread(self.volta)
        grabber.run()

        counter = 0
        while counter < self.duration:
            progress = 100 * counter / self.duration
            if progress < 100:
                self.write_message('Прогресс: %s %%' % progress)
            step=10
            counter = counter + step
            time.sleep(step)

        grabber.wait(self.duration * 2)
        EventPoller(self.volta.isTestFinished)
        self.write_message(u'Готово. Подключите телефон')
        # 7 - подключение телефона
        EventPoller(self.phone.isPhoneConnected)
        self.write_message(u'Найден телефон, забираем логи с телефона')
        EventPoller(self.phone.dumpLogcatEvents)
        EventPoller(self.phone.getInfoAboutDevice)
        # 8 - заливка логов
        self.write_message(u'Собираем логи и заливаем логи в Лунапарк. '
                           u'Это может занять какое-то время.')
        jobid = self.volta.upload('./output.bin', './events.log')
        self.write_message(u'Lunapark URL' % jobid)
        # work finished, closing the connection
        self.close()

    def on_close(self):
        self.write_message('Volta wizard finished')

class IndexPage(tornado.web.RequestHandler):
    def get(self):
        """ index page w/ buttons """
        self.render(
            resource_filename(__name__, 'index.html'),
            title="Volta UI"
        )

def make_app():
    static_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'static')
    return tornado.web.Application([
        (r"/", IndexPage),
        (r"/wizard", WizardWebSocket),
        (r"/static/(.*)", tornado.web.StaticFileHandler, {"path": static_path}),
    ])

def main():
    logging.basicConfig(level=logging.DEBUG)

    parser = argparse.ArgumentParser(description='Configures ui tornado server.')
    parser.add_argument('--port', dest='port', default=9998, help='port for webserver (default: 9998)')
    args = parser.parse_args()

    app = make_app()
    app.listen(args.port)
    url = "http://localhost:{port}".format(port=args.port)
    webbrowser.open(url,new=2) #new=2 means open in new tab if possible
    IOLoop.instance().start()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (C) 2008  José Pablo Fernández Silva
#
# This file is part of Bonvrtaro.
#
# Bonvortaro is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# Bonvortaro is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with Bonvortaro.  If not, see <http://www.gnu.org/licenses/>.

from django.db import models

#WORD_TYPES = (
#    ("s", "suffix"),
#    ("p", "prefix"),
#    ("n", "noun"),
#    ("nn", "plural noun"),
#    ("a", "adjective"),
#    ("aa", "plural adjective"),
#    ("v", "verb"),
#    ("d", "adverb")
#)

class Root(models.Model):
    root = models.CharField(max_length=70)
    kind = models.CharField(max_length=70, blank=True) #, choices=WORD_TYPES)

    # Not used, unknown or Retavortaro specific.
    begining = models.CharField(max_length=70, blank=True)
    ending = models.CharField(max_length=70, blank=True)
    ofc = models.CharField(max_length=70, blank=True)
    #fnt = models.CharField(max_length=20)
    mrk = models.CharField(max_length=100, blank=True)

    def __unicode__(self):
        return u"%s %s %s" % (self.begining, self.root, self.ending)

class Word(models.Model):
    word = models.CharField(max_length=70)
    kind = models.CharField(max_length=70, blank=True) #, choices=WORD_TYPES)
    first = models.BooleanField()

    # Not used, unknown or Retavortaro specific.
    root = models.ForeignKey(Root)
    begining = models.CharField(max_length=70, blank=True)
    ending = models.CharField(max_length=70, blank=True)
    ofc = models.CharField(max_length=70, blank=True)
    #fnt = models.CharField(max_length=20)
    mrk = models.CharField(max_length=100, blank=True)

    def __unicode__(self):
        return u"%s %s %s" % (self.begining, self.root.root, self.ending)
<commit_msg>For words, the unicode representation is just the word.<commit_after># -*- coding: utf-8 -*-
# Copyright (C) 2008  José Pablo Fernández Silva
#
# This file is part of Bonvrtaro.
#
# Bonvortaro is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# Bonvortaro is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with Bonvortaro.  If not, see <http://www.gnu.org/licenses/>.

from django.db import models

#WORD_TYPES = (
#    ("s", "suffix"),
#    ("p", "prefix"),
#    ("n", "noun"),
#    ("nn", "plural noun"),
#    ("a", "adjective"),
#    ("aa", "plural adjective"),
#    ("v", "verb"),
#    ("d", "adverb")
#)

class Root(models.Model):
    root = models.CharField(max_length=70)
    kind = models.CharField(max_length=70, blank=True) #, choices=WORD_TYPES)

    # Not used, unknown or Retavortaro specific.
    begining = models.CharField(max_length=70, blank=True)
    ending = models.CharField(max_length=70, blank=True)
    ofc = models.CharField(max_length=70, blank=True)
    #fnt = models.CharField(max_length=20)
    mrk = models.CharField(max_length=100, blank=True)

    def __unicode__(self):
        return u"%s %s %s" % (self.begining, self.root, self.ending)

class Word(models.Model):
    word = models.CharField(max_length=70)
    kind = models.CharField(max_length=70, blank=True) #, choices=WORD_TYPES)
    first = models.BooleanField()

    # Not used, unknown or Retavortaro specific.
    root = models.ForeignKey(Root)
    begining = models.CharField(max_length=70, blank=True)
    ending = models.CharField(max_length=70, blank=True)
    ofc = models.CharField(max_length=70, blank=True)
    #fnt = models.CharField(max_length=20)
    mrk = models.CharField(max_length=100, blank=True)

    def __unicode__(self):
        return self.word
<|endoftext|>


<commit_before>#!/usr/bin/env python3
__doc__="""A collection of utility functions."""

import logging
import numpy

import vitables

def divide(A, B, rep=0.0):
    """Compute :math:`A / B` silencing warnings.

    Given two :class:`numpy.ndarray`, compute the element wise division
    suppressing divide by zero warnings and invalid entries.  All
    resulting NaNs, Infs, and places where ``B`` is essentially 0 are
    replaced by ``rep``.

    Parameters
    ----------

    A : :class:`numpy.ndarray`
        The numerator.
    B : :class:`numpy.ndarray`
        The denominator.
    rep : scalar, optional
        The value to replace bad values.

    Returns
    -------

    C : :class:`numpy.ndarray`
        ``A / B`` with bad values set to ``rep``.

    """
    with numpy.errstate(invalid="ignore", divide="ignore"):
        C = A / B

    C[numpy.isclose(B, 0)] = rep
    C[numpy.isnan(C)] = rep
    C[numpy.isinf(C)] = rep
    return C

def setup_logger(name):
    """Add the GUI's logging window as a stream handler.

    By default, the stream logger is removed during the invocation of
    ViTables.  The logging window in the GUI is a stream handler for the
    ViTables logger *only*.  This method will add the logging window in
    the GUI as a stream handler for the named logger.  The method checks
    to see if ViTables is an active application.  If it is not, nothing
    is done.

    """
    logger = logging.getLogger(name)
    app = vitables.utils.getApp()
    if app is not None:
        stream = logging.StreamHandler(app.gui.logger)
        stream.setFormatter(
            logging.Formatter(vitables.vtgui._GUI_LOG_FORMAT)
        )
        logger.addHandler(stream)

    return

<commit_msg>Add parameter list to function<commit_after>#!/usr/bin/env python3
__doc__="""A collection of utility functions."""

import logging
import numpy

import vitables

def divide(A, B, rep=0.0):
    """Compute :math:`A / B` silencing warnings.

    Given two :class:`numpy.ndarray`, compute the element wise division
    suppressing divide by zero warnings and invalid entries.  All
    resulting NaNs, Infs, and places where ``B`` is essentially 0 are
    replaced by ``rep``.

    Parameters
    ----------

    A : :class:`numpy.ndarray`
        The numerator.
    B : :class:`numpy.ndarray`
        The denominator.
    rep : scalar, optional
        The value to replace bad values.

    Returns
    -------

    C : :class:`numpy.ndarray`
        ``A / B`` with bad values set to ``rep``.

    """
    with numpy.errstate(invalid="ignore", divide="ignore"):
        C = A / B

    C[numpy.isclose(B, 0)] = rep
    C[numpy.isnan(C)] = rep
    C[numpy.isinf(C)] = rep
    return C

def setup_logger(name):
    """Add the GUI's logging window as a stream handler.

    By default, the stream logger is removed during the invocation of
    ViTables.  The logging window in the GUI is a stream handler for the
    ViTables logger *only*.  This method will add the logging window in
    the GUI as a stream handler for the named logger.  The method checks
    to see if ViTables is an active application.  If it is not, nothing
    is done.

    Parameters
    ----------

    name : string
        The name of the module to add to the ViTables logging window.

    """
    logger = logging.getLogger(name)
    app = vitables.utils.getApp()
    if app is not None:
        stream = logging.StreamHandler(app.gui.logger)
        stream.setFormatter(
            logging.Formatter(vitables.vtgui._GUI_LOG_FORMAT)
        )
        logger.addHandler(stream)

    return

<|endoftext|>


<commit_before>import subprocess
import os.path
from celery import shared_task
from time import sleep
from tempfile import mkdtemp
from shutil import copyfile, rmtree
import logging

from django.conf import settings
from .models import Job, Result
from .parser import parse_sea_output

@shared_task()
def run_simple_calculation(job_id):
    job = Job.objects.get(pk=job_id)
    with open(job.structure.name, 'r') as f:
        lines = f.readlines()
        num_lines = len(lines)
    sleep(30.) # seconds
    job.status = Job.STATUS.done
    job.save()
    return num_lines

@shared_task()
def run_sea_calculation(job_id):
    job = Job.objects.get(pk=job_id)
    temp_dir = mkdtemp()
    temp_path = os.path.join(temp_dir, 'struct')
    temp_struct_path = temp_path + '.gro'
    temp_topo_path = temp_path + '.top'
    copyfile(job.structure.path, temp_struct_path)
    copyfile(job.topology.path, temp_topo_path)
    solvate_cmd = os.path.join(settings.SEA_HOME, "bin", "solvate")
    input_path = temp_path
    arg_list = [solvate_cmd, "-s", input_path, "-i", str(job.iterations)]
    logging.info( "%s" % ' '.join(arg_list))
    try:
        output_str = subprocess.check_output(arg_list, stderr=subprocess.STDOUT)
        output = parse_sea_output(output_str)
        create_result_obj(job, output)
        status = Job.STATUS.done
    except subprocess.CalledProcessError, e:
        output_str = "There was an error:\n%s" % e.output
        output_str += "return code: %s\n" % e.returncode
        status = Job.STATUS.error

    sleep(10.) # seconds
    job.status = status
    job.save()
    rmtree(temp_dir)
    return "%s\n%s" % (status, output_str)

def create_result_obj(job, output):
    r = Result.objects.create(job=job, **output)
    return
<commit_msg>Remove files when calculation is complete.<commit_after>import subprocess
import os.path
from celery import shared_task
from time import sleep
from tempfile import mkdtemp
from shutil import copyfile, rmtree
import logging
from django.conf import settings

from .models import Job, Result
from .parser import parse_sea_output

@shared_task()
def run_sea_calculation(job_id):
    job = Job.objects.get(pk=job_id)
    temp_dir = mkdtemp()
    temp_path = os.path.join(temp_dir, 'struct')
    temp_struct_path = temp_path + '.gro'
    temp_topo_path = temp_path + '.top'
    copyfile(job.structure.path, temp_struct_path)
    copyfile(job.topology.path, temp_topo_path)
    solvate_cmd = os.path.join(settings.SEA_HOME, "bin", "solvate")
    input_path = temp_path
    arg_list = [solvate_cmd, "-s", input_path, "-i", str(job.iterations)]
    logging.info( "%s" % ' '.join(arg_list))
    try:
        output_str = subprocess.check_output(arg_list, stderr=subprocess.STDOUT)
        output = parse_sea_output(output_str)
        create_result_obj(job, output)
        status = Job.STATUS.done
    except subprocess.CalledProcessError, e:
        output_str = "There was an error:\n%s" % e.output
        output_str += "return code: %s\n" % e.returncode
        status = Job.STATUS.error

    sleep(10.) # seconds
    job.status = status
    job.save()
    rmtree(temp_dir)
    job.structure.delete()
    job.topology.delete()
    return "%s\n%s" % (status, output_str)

def create_result_obj(job, output):
    r = Result.objects.create(job=job, **output)
    return
<|endoftext|>