<commit_before># coding: utf-8

# Import python module
import os
import sys
import datetime

# Import third module
from mako.exceptions import RichTraceback
import tornado.web

# Import yweb module
from yweb.conf import settings

# Import my module
from apps.auth.models import User, Session, AuthCode
from yweb.utils.translation import trans_real
from yweb.utils.translation import ugettext_lazy
_ = ugettext_lazy

class RequestHandler(tornado.web.RequestHandler):

    def __init__(self, application, request, **kwargs):
        super(RequestHandler, self).__init__(
            application, request, **kwargs )
        self.template_path = None
        self.title = None
        self.data = {}

        # TODO: hack django 的 lazy translation 机制
        trans_real._default = trans_real.translation(self.locale_code)

    def render_string(self, template_path, **kwargs):
        '''渲染模板，返回字符串
        '''
        if not template_path:
            # 模板未指定，出错
            return _('No template!')

        args = dict(
            handler=self,
            request=self.request,
            current_user=self.current_user,
#            locale=self.locale,
#            _=self.locale.translate,
            _=ugettext_lazy,
            static_url=self.static_url,
            xsrf_form_html=self.xsrf_form_html,
            reverse_url=self.reverse_url,
            module=self.load_module,
            s=lambda x: u"\"{0}\"".format(x),
        )

        args.update(self.data)
        args.update(kwargs)

        # 如果标题未设置，设置一个默认值
        if self.title:
            args['title'] = self.title
        else:
            args['title'] = settings.DEFAULT_HTML_TITLE

        try:
            html = self.application.template_lookup.get_template(
                template_path).render(**args)
        except:
            html = self.application.template_lookup.get_template(
                'mako_failed.html').render(
                    traceback = RichTraceback())

        return html

    def render(self, template_path=None, **kwargs):
        '''渲染模板，直接返回给客户端
        '''

        if not template_path:
            template_path = self.template_path

        html = self.render_string(template_path, **kwargs)

        self.finish(html)

    def get_current_user(self):

        skey = self.get_secure_cookie('session_key')
        session = self.db.query(Session).filter_by(skey=skey).first()
        if not session:
            return None

        now = datetime.datetime.now()

        # Does session expired ?
        if session.expire_date < now:
            self.db.remove( session )
            self.db.commit()
            return None

        uid = session.get_uid()
        if not uid:
            return None

        user = self.db.query(User).get(uid)

        if user:
            if user.is_locked:
                return None
            else:
                user.last_active = now
                #self.db.commit()
                return user
        else:
            return None

    def get_user_locale(self):

        user_locale = self.get_cookie("user_locale")

        if user_locale:
            return tornado.locale.get(user_locale)
        else:
            # Use the Accept-Language header
            return None

    @property
    def locale_code(self):
        '''获取 language code

        为了使用 Django 的翻译机制
        '''

        user_locale = self.get_cookie("user_locale")
        if user_locale:
            return user_locale
        else:
            return 'en_US'

    @property
    def db(self):
        return self.application.db_session()

    def on_finish(self):
        self.application.db_session.remove()

    def get(self):
        '''提供默认的 HTTP GET 请求

        Python不太爱隐藏细节，但是程序员很懒。
        '''
        if self.template_path:
            self.render()
        else:
            # 可能 Handler 没有提供 get 方法
            raise tornado.web.HTTPError(405)

    def static_url(self, path, app=None):
        '''静态文件地址

        Tornado 的 static_url 扩展。但不计算 version，可由 nginx 提供。

        支持指定 app，获取其 static 文件 url
        '''

        if app:
            url = None
            data = self.settings.get('app_static_urls')
            if data:
                url = data.get(app)

            # 默认 url
            if not url:
                url = '/{0}/static/'.format(app)

            url += path
        else:
            url = self.settings.get('static_url_prefix', '/static/') + path

        return url

    def load_module(self, module_name, *args, **kwargs):
        '''加载 ui module
        '''

        ui_modules = self.settings['y_ui_modules']

        if module_name not in ui_modules.keys():
            return '
' + \
            'Can not find module "{0}"
'.format(module_name)

        # TODO：保存该 module 为激活的，在 render 时可以像 tornado
        # 里一样，加载 html, css, js 内容

        Module = ui_modules.get(module_name)

        return Module(self).render(*args, **kwargs)

    def check_authcode(self):
        '''检查 authcode 安全
        '''

        authcode_key = self.get_argument('authcode_key', None)
        authcode_code = self.get_argument('authcode_code', None)
        if not (authcode_key and authcode_code):
            return False

        ac = self.db.query(AuthCode).get(authcode_key)
        if ac and ac.code.lower() == authcode_code.lower():
            self.db.delete(ac)
            self.db.commit()
            return True

        return False

class DefaultIndexHandler(RequestHandler):
    '''默认的视图

    如果用户还没有指定自己的首页视图，就会调用这个视图
    '''

    def get(self):
#        self.write("
I'm runing!
")
        self.render('default_index.html')

class NotFoundHandler(RequestHandler):
    '''404错误视图

    如果用户访问的 URL 不存在，就会调用这个视图
    '''

    def prepare(self):
        self.set_status(404)
        self.render("/404.html")

class ApiRequestHandler(RequestHandler):

    def write_fail(self, message='', data=None):

        d = { 'status': 'fail', 'message': message, 'data': data }

        self.write( d )

    def write_success(self, message='', data=None):

        d = { 'status': 'success', 'message': message, 'data': data }

        self.write( d )

# 参考 tornado/web.py:UIModule
class UIModule(object):
    """A re-usable, modular UI unit on a page.

    UI modules often execute additional queries, and they can include
    additional CSS and JavaScript that will be included in the output
    page, which is automatically inserted on page render.
    """
    def __init__(self, handler):
        self.handler = handler
        self.request = handler.request
        self.ui = handler.ui
        self.locale = handler.locale

    @property
    def current_user(self):
        return self.handler.current_user

    def render(self, *args, **kwargs):
        """Overridden in subclasses to return this module's output."""
        raise NotImplementedError()

    def embedded_javascript(self):
        """Returns a JavaScript string that will be embedded in the page."""
        return None

    def javascript_files(self):
        """Returns a list of JavaScript files required by this module."""
        return None

    def embedded_css(self):
        """Returns a CSS string that will be embedded in the page."""
        return None

    def css_files(self):
        """Returns a list of CSS files required by this module."""
        return None

    def html_head(self):
        """Returns a CSS string that will be put in the  element"""
        return None

    def html_body(self):
        """Returns an HTML string that will be put in the  element"""
        return None

    def render_string(self, path, **kwargs):
        """Renders a template and returns it as a string."""
        return self.handler.render_string(path, **kwargs)

<commit_msg>用户的默认 locale 设置为 zh_CN<commit_after># coding: utf-8

# Import python module
import os
import sys
import datetime

# Import third module
from mako.exceptions import RichTraceback
import tornado.web

# Import yweb module
from yweb.conf import settings

# Import my module
from apps.auth.models import User, Session, AuthCode
from yweb.utils.translation import trans_real
from yweb.utils.translation import ugettext_lazy
_ = ugettext_lazy

class RequestHandler(tornado.web.RequestHandler):

    def __init__(self, application, request, **kwargs):
        super(RequestHandler, self).__init__(
            application, request, **kwargs )
        self.template_path = None
        self.title = None
        self.data = {}

        # TODO: hack django 的 lazy translation 机制
        trans_real._default = trans_real.translation(self.locale_code)

    def render_string(self, template_path, **kwargs):
        '''渲染模板，返回字符串
        '''
        if not template_path:
            # 模板未指定，出错
            return _('No template!')

        args = dict(
            handler=self,
            request=self.request,
            current_user=self.current_user,
#            locale=self.locale,
#            _=self.locale.translate,
            _=ugettext_lazy,
            static_url=self.static_url,
            xsrf_form_html=self.xsrf_form_html,
            reverse_url=self.reverse_url,
            module=self.load_module,
            s=lambda x: u"\"{0}\"".format(x),
        )

        args.update(self.data)
        args.update(kwargs)

        # 如果标题未设置，设置一个默认值
        if self.title:
            args['title'] = self.title
        else:
            args['title'] = settings.DEFAULT_HTML_TITLE

        try:
            html = self.application.template_lookup.get_template(
                template_path).render(**args)
        except:
            html = self.application.template_lookup.get_template(
                'mako_failed.html').render(
                    traceback = RichTraceback())

        return html

    def render(self, template_path=None, **kwargs):
        '''渲染模板，直接返回给客户端
        '''

        if not template_path:
            template_path = self.template_path

        html = self.render_string(template_path, **kwargs)

        self.finish(html)

    def get_current_user(self):

        skey = self.get_secure_cookie('session_key')
        session = self.db.query(Session).filter_by(skey=skey).first()
        if not session:
            return None

        now = datetime.datetime.now()

        # Does session expired ?
        if session.expire_date < now:
            self.db.remove( session )
            self.db.commit()
            return None

        uid = session.get_uid()
        if not uid:
            return None

        user = self.db.query(User).get(uid)

        if user:
            if user.is_locked:
                return None
            else:
                user.last_active = now
                #self.db.commit()
                return user
        else:
            return None

    def get_user_locale(self):

        user_locale = self.get_cookie("user_locale")

        if user_locale:
            return tornado.locale.get(user_locale)
        else:
            # Use the Accept-Language header
            return None

    @property
    def locale_code(self):
        '''获取 language code

        为了使用 Django 的翻译机制
        '''

        user_locale = self.get_cookie("user_locale")
        if user_locale:
            return user_locale
        else:
            return 'zh_CN'

    @property
    def db(self):
        return self.application.db_session()

    def on_finish(self):
        self.application.db_session.remove()

    def get(self):
        '''提供默认的 HTTP GET 请求

        Python不太爱隐藏细节，但是程序员很懒。
        '''
        if self.template_path:
            self.render()
        else:
            # 可能 Handler 没有提供 get 方法
            raise tornado.web.HTTPError(405)

    def static_url(self, path, app=None):
        '''静态文件地址

        Tornado 的 static_url 扩展。但不计算 version，可由 nginx 提供。

        支持指定 app，获取其 static 文件 url
        '''

        if app:
            url = None
            data = self.settings.get('app_static_urls')
            if data:
                url = data.get(app)

            # 默认 url
            if not url:
                url = '/{0}/static/'.format(app)

            url += path
        else:
            url = self.settings.get('static_url_prefix', '/static/') + path

        return url

    def load_module(self, module_name, *args, **kwargs):
        '''加载 ui module
        '''

        ui_modules = self.settings['y_ui_modules']

        if module_name not in ui_modules.keys():
            return '
' + \
            'Can not find module "{0}"
'.format(module_name)

        # TODO：保存该 module 为激活的，在 render 时可以像 tornado
        # 里一样，加载 html, css, js 内容

        Module = ui_modules.get(module_name)

        return Module(self).render(*args, **kwargs)

    def check_authcode(self):
        '''检查 authcode 安全
        '''

        authcode_key = self.get_argument('authcode_key', None)
        authcode_code = self.get_argument('authcode_code', None)
        if not (authcode_key and authcode_code):
            return False

        ac = self.db.query(AuthCode).get(authcode_key)
        if ac and ac.code.lower() == authcode_code.lower():
            self.db.delete(ac)
            self.db.commit()
            return True

        return False

class DefaultIndexHandler(RequestHandler):
    '''默认的视图

    如果用户还没有指定自己的首页视图，就会调用这个视图
    '''

    def get(self):
#        self.write("
I'm runing!
")
        self.render('default_index.html')

class NotFoundHandler(RequestHandler):
    '''404错误视图

    如果用户访问的 URL 不存在，就会调用这个视图
    '''

    def prepare(self):
        self.set_status(404)
        self.render("/404.html")

class ApiRequestHandler(RequestHandler):

    def write_fail(self, message='', data=None):

        d = { 'status': 'fail', 'message': message, 'data': data }

        self.write( d )

    def write_success(self, message='', data=None):

        d = { 'status': 'success', 'message': message, 'data': data }

        self.write( d )

# 参考 tornado/web.py:UIModule
class UIModule(object):
    """A re-usable, modular UI unit on a page.

    UI modules often execute additional queries, and they can include
    additional CSS and JavaScript that will be included in the output
    page, which is automatically inserted on page render.
    """
    def __init__(self, handler):
        self.handler = handler
        self.request = handler.request
        self.ui = handler.ui
        self.locale = handler.locale

    @property
    def current_user(self):
        return self.handler.current_user

    def render(self, *args, **kwargs):
        """Overridden in subclasses to return this module's output."""
        raise NotImplementedError()

    def embedded_javascript(self):
        """Returns a JavaScript string that will be embedded in the page."""
        return None

    def javascript_files(self):
        """Returns a list of JavaScript files required by this module."""
        return None

    def embedded_css(self):
        """Returns a CSS string that will be embedded in the page."""
        return None

    def css_files(self):
        """Returns a list of CSS files required by this module."""
        return None

    def html_head(self):
        """Returns a CSS string that will be put in the  element"""
        return None

    def html_body(self):
        """Returns an HTML string that will be put in the  element"""
        return None

    def render_string(self, path, **kwargs):
        """Renders a template and returns it as a string."""
        return self.handler.render_string(path, **kwargs)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# noinspection PyUnresolvedReferences
from .settings import *

ETC_DIR = os.path.join(BASE_DIR, 'etc')

# Django
DEBUG = False

ALLOWED_HOSTS = ['localhost', '127.0.0.1', '::1', '23.105.201.47', '.moonfruit.top']

with open(os.path.join(ETC_DIR, 'secret_key.txt')) as f:
    SECRET_KEY = f.read().strip()

STATIC_ROOT = os.path.join(VAR_DIR, 'static')
MEDIA_ROOT = os.path.join(VAR_DIR, 'media')

os.makedirs(STATIC_ROOT, exist_ok=True)
os.makedirs(MEDIA_ROOT, exist_ok=True)

# Security
SECURE_BROWSER_XSS_FILTER = True
SECURE_CONTENT_TYPE_NOSNIFF = True
SECURE_HSTS_SECONDS = 31536000
SECURE_HSTS_INCLUDE_SUBDOMAINS = True
SECURE_SSL_REDIRECT = True

SESSION_COOKIE_SECURE = True

CSRF_COOKIE_SECURE = True
CSRF_COOKIE_HTTPONLY = True

X_FRAME_OPTIONS = 'DENY'

# Log
LOGGING.update({
    'root': {
        'handlers': ['file'],
        'level': 'DEBUG',
    },
})

# Cache
CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache',
        'LOCATION': '/run/memcached/memcached.sock',
    }
}

# Email
EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
# noinspection SpellCheckingInspection
EMAIL_HOST_PASSWORD = ''

# YYFeed
YYFEED_CACHE = {
    '()': 'yyutil.cache.MemCache',
    'servers': ['/run/memcached/memcached.sock'],
}
<commit_msg>for production<commit_after># -*- coding: utf-8 -*-
# noinspection PyUnresolvedReferences
from .settings import *

ETC_DIR = os.path.join(BASE_DIR, 'etc')

# Django
DEBUG = False

ALLOWED_HOSTS = ['localhost', '127.0.0.1', '::1', '23.105.201.47', '.moonfruit.top']

with open(os.path.join(ETC_DIR, 'secret_key.txt')) as f:
    SECRET_KEY = f.read().strip()

STATIC_ROOT = os.path.join(VAR_DIR, 'static')
MEDIA_ROOT = os.path.join(VAR_DIR, 'media')

os.makedirs(STATIC_ROOT, exist_ok=True)
os.makedirs(MEDIA_ROOT, exist_ok=True)

# Security
SECURE_BROWSER_XSS_FILTER = True
SECURE_CONTENT_TYPE_NOSNIFF = True
SECURE_HSTS_PRELOAD = True
SECURE_HSTS_SECONDS = 31536000
SECURE_HSTS_INCLUDE_SUBDOMAINS = True
SECURE_SSL_REDIRECT = True

SESSION_COOKIE_SECURE = True

CSRF_COOKIE_SECURE = True
CSRF_COOKIE_HTTPONLY = True

X_FRAME_OPTIONS = 'DENY'

# Log
LOGGING.update({
    'root': {
        'handlers': ['file'],
        'level': 'DEBUG',
    },
})

# Cache
CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache',
        'LOCATION': '/run/memcached/memcached.sock',
    }
}

# Email
EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
# noinspection SpellCheckingInspection
EMAIL_HOST_PASSWORD = ''

# YYFeed
YYFEED_CACHE = {
    '()': 'yyutil.cache.MemCache',
    'servers': ['/run/memcached/memcached.sock'],
}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''Basic zerto object'''

class ZertoObject(object):

    def __str__(self):
        return str(getattr(self, 'values', None))

    def __repr__(self):
        return '<{} {}>'.format(
            self.__class__.__name__, getattr(self, 'values', None))

# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
<commit_msg>python  2.6 compatible<commit_after># -*- coding: utf-8 -*-
'''Basic zerto object'''

class ZertoObject(object):

    def __str__(self):
        return str(getattr(self, 'values', None))

    def __repr__(self):
        return '<{0} {1}>'.format(
            self.__class__.__name__, getattr(self, 'values', None))

# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
   gauge.__about__
   ~~~~~~~~~~~~~~~
"""
__version__ = '0.4.1'
__license__ = 'BSD'
__author__ = 'What! Studio'
__maintainer__ = 'Heungsub Lee'
__maintainer_email__ = 'sub@nexon.co.kr'
<commit_msg>Bump version: 0.4.2<commit_after># -*- coding: utf-8 -*-
"""
   gauge.__about__
   ~~~~~~~~~~~~~~~
"""
__version__ = '0.4.2'
__license__ = 'BSD'
__author__ = 'What! Studio'
__maintainer__ = 'Heungsub Lee'
__maintainer_email__ = 'sub@nexon.co.kr'
<|endoftext|>


<commit_before># -*- coding: utf8 -*-

from PyQt4 import QtGui, QtCore

class speciesListDialog(QtGui.QDialog):

	_tableview = None

	def __init__(self, parent, app):
		QtGui.QDialog.__init__(self, parent)
		self._app = app
		self._parent = parent
		self.initUI()
		self.setWindowTitle('List species')
		self.show()

	def initUI(self):
		layout = QtGui.QVBoxLayout(self)

		tablemodel = SpeciesTableModel(self._app.map.species, self)
		self._tableview = QtGui.QTableView()
		self._tableview.setItemDelegate(EditableRowDelegate(self._tableview))
		self._tableview.setModel(tablemodel)

		closeButton = QtGui.QPushButton("Close")
		closeButton.clicked.connect(self.close)

		form = self.creationForm()

		layout.addWidget(self._tableview)
		layout.addLayout(form)
		layout.addWidget(closeButton)
		self.setLayout(layout)

	def creationForm(self):
		form = QtGui.QGridLayout()

		nameLabel = QtGui.QLabel("Species name")
		self._nameField = QtGui.QLineEdit()
		self._nameField.textChanged.connect(self.updateCreateButton)
		descriptionLabel = QtGui.QLabel("Species Description")
		self._descriptionField = QtGui.QTextEdit()
		self._descriptionField.textChanged.connect(self.updateCreateButton)

		self._saveButton = QtGui.QPushButton("Create")
		self._saveButton.setEnabled(False)
		self._saveButton.clicked.connect(self.createSpecies)

		form.addWidget(nameLabel, 0, 0)
		form.addWidget(self._nameField, 0, 1)
		form.addWidget(descriptionLabel, 1, 0)
		form.addWidget(self._descriptionField, 1, 1)
		form.addWidget(self._saveButton, 2, 1)

		return form

	def updateCreateButton(self):
		self._saveButton.setEnabled(str(self._nameField.text()).strip() != "")

	def createSpecies(self):
		name = str(self._nameField.text()).strip()
		description = str(self._descriptionField.toPlainText()).strip()

		if name is "":
			return False

		self._app.addSpecies(name, description)

		tablemodel = SpeciesTableModel(self._app.map.species, self)
		self._tableview.setModel(tablemodel)

class SpeciesTableModel(QtCore.QAbstractTableModel):
	def __init__(self, datain, parent = None, *args):
		QtCore.QAbstractTableModel.__init__(self, parent, *args)
		self.arraydata = datain

	def rowCount(self, parent):
		return len(self.arraydata)

	def columnCount(self, parent):
		if len(self.arraydata) == 0:
			return 0

		return len(self.arraydata[0])

	def setData(self, index, value, role):
		if role == QtCore.Qt.DisplayRole:
			self.arraydata[index.row()][index.column()] = str(value)
			return True
		return False

	def data(self, index, role):
		if not index.isValid():
			return None
		elif role == QtCore.Qt.EditRole:
			return str(self.arraydata[index.row()][index.column()])
		elif role != QtCore.Qt.DisplayRole:
			return None
		return (self.arraydata[index.row()][index.column()])

	def flags(self, index):
		return QtCore.Qt.ItemIsEditable | QtCore.Qt.ItemIsEnabled | QtCore.Qt.ItemIsSelectable

class EditableRowDelegate(QtGui.QItemDelegate):
	_table = None

	def __init__(self, table, *args):
		QtGui.QItemDelegate.__init__(self, *args)
		self._table = table

	def createEditor(self, parent, option, index):
		if index.column() == 1:
			self._editor = QtGui.QTextEdit(parent)
			self._editor.textChanged.connect(self.updateValue)
		else:
			self._editor = QtGui.QLineEdit(parent)
			self._editor.editingFinished.connect(self.updateValue)
		self._editedIndex = index
		return self._editor

	def setEditorData(self, editor, index):
		self._editedIndex = index
		editor.setText(index.model().data(index, QtCore.Qt.EditRole));

	def updateValue(self):
		try:
			value = self._editor.text()
			if value == "":
				return False
		except AttributeError:
			value = self._editor.toPlainText()

		self._editedIndex.model().setData(self._editedIndex, value, QtCore.Qt.DisplayRole)
		self._table.setModel(self._editedIndex.model())
<commit_msg>comments added for the widget listing the world's species<commit_after># -*- coding: utf8 -*-

from PyQt4 import QtGui, QtCore

class speciesListDialog(QtGui.QDialog):
	"""
	Class to list the current world's species.
	"""
	_tableview = None

	def __init__(self, parent, app):
		"""
		Initialisation of the window, creates the GUI and displays the window.
		"""
		QtGui.QDialog.__init__(self, parent)
		self._app = app
		self._parent = parent
		self.initUI()
		self.setWindowTitle('List species')
		self.show()

	def initUI(self):
		"""
		Creates the GUI.
		The GUI is composed of a table listing the existing species, a form to
		add a species and a button to close the window.
		"""
		layout = QtGui.QVBoxLayout(self)

		tablemodel = SpeciesTableModel(self._app.map.species, self)
		self._tableview = QtGui.QTableView()
		self._tableview.setItemDelegate(EditableRowDelegate(self._tableview))
		self._tableview.setModel(tablemodel)

		closeButton = QtGui.QPushButton("Close")
		closeButton.clicked.connect(self.close)

		form = self.creationForm()

		layout.addWidget(self._tableview)
		layout.addLayout(form)
		layout.addWidget(closeButton)
		self.setLayout(layout)

	def creationForm(self):
		"""
		Method which creates the form to add a species.
		Returns a layout containing the form elements.
		"""
		form = QtGui.QGridLayout()

		nameLabel = QtGui.QLabel("Species name")
		self._nameField = QtGui.QLineEdit()
		self._nameField.textChanged.connect(self.updateCreateButton)
		descriptionLabel = QtGui.QLabel("Species Description")
		self._descriptionField = QtGui.QTextEdit()
		self._descriptionField.textChanged.connect(self.updateCreateButton)

		self._saveButton = QtGui.QPushButton("Create")
		self._saveButton.setEnabled(False)
		self._saveButton.clicked.connect(self.createSpecies)

		form.addWidget(nameLabel, 0, 0)
		form.addWidget(self._nameField, 0, 1)
		form.addWidget(descriptionLabel, 1, 0)
		form.addWidget(self._descriptionField, 1, 1)
		form.addWidget(self._saveButton, 2, 1)

		return form

	def updateCreateButton(self):
		"""
		Method called when the form's fields are edited. The "create" button is
		enabled if the name field is not empty.
		"""
		self._saveButton.setEnabled(str(self._nameField.text()).strip() != "")

	def createSpecies(self):
		"""
		Method called when the "create" button is pressed. The filled data are
		checked and if they are correct, the species is created.
		"""
		name = str(self._nameField.text()).strip()
		description = str(self._descriptionField.toPlainText()).strip()

		if name is "":
			return False

		self._app.addSpecies(name, description)

		tablemodel = SpeciesTableModel(self._app.map.species, self)
		self._tableview.setModel(tablemodel)

class SpeciesTableModel(QtCore.QAbstractTableModel):
	"""
	Model class for the species list
	"""
	def __init__(self, datain, parent = None, *args):
		QtCore.QAbstractTableModel.__init__(self, parent, *args)
		self.arraydata = datain

	def rowCount(self, parent):
		"""
		Method to get the number of rows of the table.
		"""
		return len(self.arraydata)

	def columnCount(self, parent):
		"""
		Method to get the number of columns of the table.
		"""
		if len(self.arraydata) == 0:
			return 0

		return len(self.arraydata[0])

	def setData(self, index, value, role):
		"""
		Method to update a cell of the table, depending on a given role.
		"""
		if role == QtCore.Qt.DisplayRole:
			self.arraydata[index.row()][index.column()] = str(value)
			return True
		return False

	def data(self, index, role):
		"""
		Method to get the value of a cell of the table, depending on a given
		role.
		"""
		if not index.isValid():
			return None
		elif role == QtCore.Qt.EditRole:
			return str(self.arraydata[index.row()][index.column()])
		elif role != QtCore.Qt.DisplayRole:
			return None
		return (self.arraydata[index.row()][index.column()])

	def flags(self, index):
		"""
		Method to set the table's cells flags
		"""
		return QtCore.Qt.ItemIsEditable | QtCore.Qt.ItemIsEnabled | QtCore.Qt.ItemIsSelectable

class EditableRowDelegate(QtGui.QItemDelegate):
	"""
	Delegate class for the table items.
	Will create a special text or line edit field when a field is edited.
	"""
	_table = None

	def __init__(self, table, *args):
		QtGui.QItemDelegate.__init__(self, *args)
		self._table = table

	def createEditor(self, parent, option, index):
		"""
		Method called when a cell is edited, to create the according field.
		If the cell is a description cell (index 1), a textedit field is
		created, else a lineedit field is created.
		"""
		if index.column() == 1:
			self._editor = QtGui.QTextEdit(parent)
			self._editor.textChanged.connect(self.updateValue)
		else:
			self._editor = QtGui.QLineEdit(parent)
			self._editor.editingFinished.connect(self.updateValue)
		self._editedIndex = index
		return self._editor

	def setEditorData(self, editor, index):
		"""
		Method to fill the field's value.
		"""
		self._editedIndex = index
		editor.setText(index.model().data(index, QtCore.Qt.EditRole));

	def updateValue(self):
		"""
		Method called when a field lose the focus, to update the according
		species's value.
		"""
		try:
			value = self._editor.text()
			if value == "":
				return False
		except AttributeError:
			value = self._editor.toPlainText()

		self._editedIndex.model().setData(self._editedIndex, value, QtCore.Qt.DisplayRole)
		self._table.setModel(self._editedIndex.model())
<|endoftext|>


<commit_before>from datetime import datetime
from celery.result import AsyncResult
from celery.signals import task_prerun, task_postrun
from django.db import models

class Permissions(models.Model):
    pass
    class Meta:
        permissions =(
                ("can_publish", "Can publish a finding aid"),
                ("can_preview", "Can preview a finding aid"),
        )

# store details about pdf reload celery task results for display on admin page
class TaskResult(models.Model):
    label = models.CharField(max_length=100)
    eadid = models.CharField(max_length=50)
    created = models.DateTimeField(default=datetime.now)
    task_id = models.CharField(max_length=100)
    task_start = models.DateTimeField(blank=True, null=True)
    task_end = models.DateTimeField(blank=True, null=True)

    def task(self):
        return AsyncResult(self.task_id)

# listeners to celery signals to store start and end time for tasks
# NOTE: these functions do not filter on the sender/task function

def taskresult_start(sender, task_id, **kwargs):
    try:
        tr = TaskResult.objects.get(task_id=task_id)
        tr.task_start = datetime.now()
        tr.save()
    except Exception:
        pass
task_prerun.connect(taskresult_start)

def taskresult_end(sender, task_id, **kwargs):
    try:
        tr = TaskResult.objects.get(task_id=task_id)
        tr.task_end = datetime.now()
        tr.save()
    except Exception:
        pass
task_postrun.connect(taskresult_end)

<commit_msg>Change of 'permissions class to Findingaids<commit_after>from datetime import datetime
from celery.result import AsyncResult
from celery.signals import task_prerun, task_postrun
from django.db import models

class Findingaids(models.Model):
    """
    This model is used as a place holder for custom permissions needed with the
    admin portion of the site.
    """
    pass
    class Meta:
        permissions =(
                ("can_publish", "Can publish a finding aid"),
                ("can_preview", "Can preview a finding aid"),
                ("can_delete", "Can delete a finding aid"),
        )

# store details about pdf reload celery task results for display on admin page
class TaskResult(models.Model):
    label = models.CharField(max_length=100)
    eadid = models.CharField(max_length=50)
    created = models.DateTimeField(default=datetime.now)
    task_id = models.CharField(max_length=100)
    task_start = models.DateTimeField(blank=True, null=True)
    task_end = models.DateTimeField(blank=True, null=True)

    def task(self):
        return AsyncResult(self.task_id)

# listeners to celery signals to store start and end time for tasks
# NOTE: these functions do not filter on the sender/task function

def taskresult_start(sender, task_id, **kwargs):
    try:
        tr = TaskResult.objects.get(task_id=task_id)
        tr.task_start = datetime.now()
        tr.save()
    except Exception:
        pass
task_prerun.connect(taskresult_start)

def taskresult_end(sender, task_id, **kwargs):
    try:
        tr = TaskResult.objects.get(task_id=task_id)
        tr.task_end = datetime.now()
        tr.save()
    except Exception:
        pass
task_postrun.connect(taskresult_end)

<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import re
import os
inFilename = sys.argv[1]
if os.path.isfile(inFilename):
    namelength = inFilename.rfind(".")
    name = inFilename[0:namelength]
    exten = inFilename[namelength:]
    ReuseDH = name+"-reuse"+exten
    StackDH = name+"-stack"+exten

print "inFilename:", inFilename
print "outFilename:", ReuseDH,StackDH

fpRead = open(inFilename, "r")
RDHWrite = open(ReuseDH, "w+")
SDHWrite = open(StackDH,"w+")

#core0ReuseDis pattern
core0ReuseDisPattern = re.compile(r'.*(system.l2.core0ReuseDis::)([0-9]+)\s+([0-9]+)')
core0ReuseSamplePattern = re.compile(r'.*(system.l2.core0ReuseDis::sample).* ([0-9]+)')
#core1ReuseDis pattern
core1ReuseDisPattern = re.compile(r'.*(system.l2.core1ReussDis::)([0-9]+)\s+([0-9]+)')
core1ReuseSamplePattern = re.compile(r'.*(system.l2.core1ReuseDis::sample).* ([0-9]+)')
#l2Stack historgram Pattern
l2StackHisPattern = re.compile(r'.*(system.l2.l2StackHis).*([0-9]+)\s+([0-9]+)')
#l2 cachehits Pattern
l2cachehitsPattern = re.compile(r'.*(system.l2.cachehits).*([0-9|\.]+)')
#core memread pattern
core0memreadPattern = re.compile(r'.*(system.switch_cpus0.commit.op_class_0::MemRead).* ([0-9]+)\s+.*')
core1memreadPattern = re.compile(r'.*(system.switch_cpus1.commit.op_class_0::MemRead).* ([0-9]+)\s+.*')
#core memwrite Pattern
core0memwritePattern = re.compile(r'.*(system.switch_cpus0.commit.op_class_0::MemWrite).* ([0-9]+)\s+.*')
core1memwritePattern = re.compile(r'.*(system.switch_cpus1.commit.op_class_0::MemWrite).* ([0-9]+)\s+.*')
#core commit total Pattern
core0commitPattern = re.compile(r'.*(system.switch_cpus0.commit.op_class_0::total).* ([0-9|\.]+)')
core1commitPattern = re.compile(r'.*(system.switch_cpus1.commit.op_class_0::total).* ([0-9|\.]+)')
#core cpi pattern
core0cpiPattern = re.compile(r'.*(system.switch_cpus0.cpi).* (([0-9|\.]+)|(nan))')
core1cpiPattern = re.compile(r'.*(system.switch_cpus1.cpi).* (([0-9|\.]+)|(nan))')
#thread pattern
threadbeginPattern = re.compile(r'.*Begin Simulation Statistics.*')
threadendPattern =re.compile(r'.*End Simulation Statistics.*')
lines = fpRead.readline()
debug = 1

def check(a,b):
    if b-a==1:
        return 0
    else:
        c = b-a
        return c

while lines:
    threadbeginmatch = threadbeginPattern.search(lines)
#    print "----------------------- reading lines------------------"
    if threadbeginmatch:
        core0flag = True
        core1flag = True
        l2flag = True
        core0=[]
        core1=[]
        l2 = []
        core0pos=1
        core1pos=1
        l2pos = 1
        l2R = []
        l2ES=[]
        pos0=0
        pos1=0
#        print "------------------------ thread matched ------------------"
        threadlines = fpRead.readline()
        threadendmatch = threadendPattern.match(threadlines)
    	while threadlines:
            core0Dismatch = core0ReuseDisPattern.search(threadlines)
            core1Dismatch = core1ReuseDisPattern.search(threadlines)
            core0samplematch = core0ReuseSamplePattern.search(threadlines)
            core1samplematch = core1ReuseSamplePattern.search(threadlines)
            l2Hismatch = l2StackHisPattern.search(threadlines)
            l2cachehitsmatch = l2cachehitsPattern.search(threadlines)
            core0readmatch =  core0memreadPattern.search(threadlines)
            core0writematch = core0memwritePattern.search(threadlines)
            core0commitmatch = core0commitPattern.search(threadlines)
            core0cpimatch = core0cpiPattern.search(threadlines)
            core1readmatch =  core1memreadPattern.search(threadlines)
            core1writematch = core1memwritePattern.search(threadlines)
            core1commitmatch = core1commitPattern.search(threadlines)
            core1cpimatch = core1cpiPattern.search(threadlines)
            threadendmatch = threadendPattern.match(threadlines)
            if core0samplematch:
                core0sample = core0samplematch.group(2)
            if core0Dismatch:
                pos0 = int(core0Dismatch.group(2))
                if core0flag:
                    core0flag = False
                    core0pos = pos0
                    dis0 = pos0
                    while(dis0-1) > 0:
                        core0.append(0)
                        dis0 = dis0-1
                val0 = int(core0Dismatch.group(3))
                dis0 = check(core0pos,pos0)
                if dis0!=0:
                    while (dis0-1) > 0:
                        core0.append(0)
                        dis0 = dis0-1
                core0.append(val0)
                core0pos = pos0
            if core1samplematch:
                core1sample = core1samplematch.group(2)
            if core1Dismatch:
                pos1 = int(core1Dismatch.group(2))
                if core1flag:
                    core1flag = False
                    core1pos = pos1
                    dis1 = pos1
                    while(dis1-1) >0 :
                        core1.append(0)
                        dis1 = dis1-1
                val1 = int(core1Dismatch.group(3))
                dis1 = check(core1pos,pos1)
                if dis1!=0:
                    while (dis1-1) > 0:
                        core1.append(0)
                        dis1 = dis1-1
                core1.append(val1)
                core1pos = pos1
            if l2Hismatch:
                pos2 = int(l2Hismatch.group(2))
                if l2flag:
                    l2flag = False
                    l2pos = pos2
                    dis2 = pos2
                    while(dis2-1) > 0:
                        l2.append(0)
                        dis2 = dis2-1
                val2 = int(l2Hismatch.group(3))
                dis2 = check(l2pos,pos2)
                if dis2!=0:
                    while (dis2-1) > 0:
                        l2.append(0)
                        dis2 = dis2-1
                l2.append(val2)
                l2pos = pos2
            if l2cachehitsmatch:
                cachehits = l2cachehitsmatch.group(2)
            if core0readmatch:
                read0 = int(core0readmatch.group(2))
            if core0writematch:
                write0 = int(core0writematch.group(2))
            if core0commitmatch:
                commit0 = float(core0commitmatch.group(2))
            if core0cpimatch:
                cpi0 = core0cpimatch.group(2)
                if(cpi0!='nan'):
                    cpi0=float(cpi0)
                else:
                    cpi0=0
            if core1readmatch:
                read1 = int(core1readmatch.group(2))
            if core1writematch:
                write1 = int(core1writematch.group(2))
            if core1commitmatch:
                commit1 = float(core1commitmatch.group(2))
            if core1cpimatch:
                cpi1 = core1cpimatch.group(2)
                if(cpi1!='nan'):
                    cpi1=float(cpi1)
                else:
                    cpi1=0
            if threadendmatch:
                #full the histogram
                dis0 = check(pos0,300)
                dis1 = check(pos1,300)
                dis2 = check(pos2,30)
                if dis0!=0:
                    if pos0 == 299:
                        core0.append(0)
                    else:
                        while dis0 > 0 :
                            core0.append(0)
                            dis0 = dis0-1
                if dis1!=0:
                    if pos1 == 299:
                        core1.append(0)
                    else:
                        while dis1 >0:
                            core1.append(0)
                            dis1 = dis1-1
                if dis2!=0:
                    if pos2 == 29:
                        l2.append(0)
                    else:
                        while dis2 > 0:
                            l2.append(0)
                            dis2 = dis2-1
                print len(core0)
                assert len(core0)==300, "core0 len error"
                assert len(core1)==300, "core1 len error"
                assert len(l2)==30,"l2 len error"
                if ((cpi1!=0) and (cpi0!=0)):
                    fac0=(read0+write0)/commit0
                    fac1=(read1+write1)/commit1
                    cpic0=cpi0/cpi1
                    cpic1=cpi1/cpi0
                    temp0 = (fac1/fac0)*cpic0
                    temp1 = (fac0/fac1)*cpic1
                    core0coe = float('%.2f'%temp0)
                    core1coe = float('%.2f'%temp1)
                    for i in range(300):
                        core0[i] = core0[i]*(1+core0coe)
                        core1[i] = core1[i]*(1+core1coe)
                        l2R.append(int(core0[i]+core1[i]))
                else:
                    for j in range(300):
                        l2R.append(int(core0[j]+core1[j]))
                assert len(l2R)==300, "l2R error"
                for k in range(300):
                    RDHWrite.write("%d\n" %l2R[k])
                for m in range(30):
                    SDHWrite.write("%d " %l2[m])
                RDHWrite.write('thread end\n')
                SDHWrite.write("\n")
                break
            threadlines = fpRead.readline()
    lines = fpRead.readline()
fpRead.close()
RDHWrite.close()
SDHWrite.close()

<commit_msg>modified the new_script and add the remark of it<commit_after>#!/usr/bin/env python

### this is the python script created by Ferris Cheung 2017-01-01
### It is used to get the shared Cache Reuse Distance Distribution which is added up by diff core's RDD-----L2R,and it's output as name-reuse.txt
### and it gets the shared Cache Stack Distance Distribution which is collected in the l2 cache port ----L2E,and it's output as name-stack.txt
import sys
import re
import os
###
### input file and output file define
###
inFilename = sys.argv[1]
if os.path.isfile(inFilename):
    namelength = inFilename.rfind(".")
    name = inFilename[0:namelength]
    exten = inFilename[namelength:]
    ReuseDH = name+"-reuse"+exten
    StackDH = name+"-stack"+exten

print "inFilename:", inFilename
print "outFilename:", ReuseDH,StackDH

#the input file
fpRead = open(inFilename, "r")
#the output file Reuse Distance Distribution
RDHWrite = open(ReuseDH, "w+")
#the output file Stack Distance Distribution
SDHWrite = open(StackDH,"w+")

#core0ReuseDis pattern
core0ReuseDisPattern = re.compile(r'.*(system.l2.core0ReuseDis::)([0-9]+)\s+([0-9]+)')
core0ReuseSamplePattern = re.compile(r'.*(system.l2.core0ReuseDis::sample).* ([0-9]+)')
#core1ReuseDis pattern
core1ReuseDisPattern = re.compile(r'.*(system.l2.core1ReussDis::)([0-9]+)\s+([0-9]+)')
core1ReuseSamplePattern = re.compile(r'.*(system.l2.core1ReuseDis::sample).* ([0-9]+)')
#l2Stack historgram Pattern
l2StackHisPattern = re.compile(r'.*(system.l2.l2StackHis).*([0-9]+)\s+([0-9]+)')
#l2 cachehits Pattern
l2cachehitsPattern = re.compile(r'.*(system.l2.cachehits).*([0-9|\.]+)')
#core memread pattern
core0memreadPattern = re.compile(r'.*(system.switch_cpus0.commit.op_class_0::MemRead).* ([0-9]+)\s+.*')
core1memreadPattern = re.compile(r'.*(system.switch_cpus1.commit.op_class_0::MemRead).* ([0-9]+)\s+.*')
#core memwrite Pattern
core0memwritePattern = re.compile(r'.*(system.switch_cpus0.commit.op_class_0::MemWrite).* ([0-9]+)\s+.*')
core1memwritePattern = re.compile(r'.*(system.switch_cpus1.commit.op_class_0::MemWrite).* ([0-9]+)\s+.*')
#core commit total Pattern
core0commitPattern = re.compile(r'.*(system.switch_cpus0.commit.op_class_0::total).* ([0-9|\.]+)')
core1commitPattern = re.compile(r'.*(system.switch_cpus1.commit.op_class_0::total).* ([0-9|\.]+)')
#core cpi pattern
core0cpiPattern = re.compile(r'.*(system.switch_cpus0.cpi).* (([0-9|\.]+)|(nan))')
core1cpiPattern = re.compile(r'.*(system.switch_cpus1.cpi).* (([0-9|\.]+)|(nan))')
#thread pattern
threadbeginPattern = re.compile(r'.*Begin Simulation Statistics.*')
threadendPattern =re.compile(r'.*End Simulation Statistics.*')
lines = fpRead.readline()
#debug is the flag to denote which thread is working on
debug = 1

###
### func check is mean to check the distribution is continuous or not
###
def check(a,b):
    if b-a==1:
        return 0
    else:
        c = b-a
        return c

while lines:
    threadbeginmatch = threadbeginPattern.search(lines)
#    print "----------------------- reading lines------------------"
    if threadbeginmatch:
        #this three flag is to denote the distribution is first time to work
        #when it is true which means we are collect this thread's first distribution
        #once we collect the thread's first distribution ,we set the flag to false
        core0flag = True
        core1flag = True
        l2flag = True
        #this three list is the container of distrubtion
        core0=[]
        core1=[]
        l2 = []
        #this thres pos is the pointer of the last distribution we collected
        #use it to check the distribution is continuous or not
        core0pos=1
        core1pos=1
        l2pos = 1
        l2R = []
        l2ES=[]
        #this three pos is the pointer of the Distribution we are collecting
        pos0=0
        pos1=0
        pos2=0
#        print "------------------------ thread matched ------------------"
        threadlines = fpRead.readline()
        threadendmatch = threadendPattern.match(threadlines)
    	while threadlines:
            core0Dismatch = core0ReuseDisPattern.search(threadlines)
            core1Dismatch = core1ReuseDisPattern.search(threadlines)
            core0samplematch = core0ReuseSamplePattern.search(threadlines)
            core1samplematch = core1ReuseSamplePattern.search(threadlines)
            l2Hismatch = l2StackHisPattern.search(threadlines)
            l2cachehitsmatch = l2cachehitsPattern.search(threadlines)
            core0readmatch =  core0memreadPattern.search(threadlines)
            core0writematch = core0memwritePattern.search(threadlines)
            core0commitmatch = core0commitPattern.search(threadlines)
            core0cpimatch = core0cpiPattern.search(threadlines)
            core1readmatch =  core1memreadPattern.search(threadlines)
            core1writematch = core1memwritePattern.search(threadlines)
            core1commitmatch = core1commitPattern.search(threadlines)
            core1cpimatch = core1cpiPattern.search(threadlines)
            threadendmatch = threadendPattern.match(threadlines)
            if core0samplematch:
                core0sample = core0samplematch.group(2)
            if core0Dismatch:
                pos0 = int(core0Dismatch.group(2))
                #this part add the 0 to the distribution
                #when our distribtuion begin with the number bigger than 1
                if core0flag:
                    core0flag = False
                    core0pos = pos0
                    dis0 = pos0
                    while(dis0-1) > 0:
                        core0.append(0)
                        dis0 = dis0-1
                val0 = int(core0Dismatch.group(3))
                #this part the add the 0 to the distribution
                #when our distribution is not continous
                dis0 = check(core0pos,pos0)
                if dis0!=0:
                    while (dis0-1) > 0:
                        core0.append(0)
                        dis0 = dis0-1
                core0.append(val0)
                core0pos = pos0
            if core1samplematch:
                core1sample = core1samplematch.group(2)
            if core1Dismatch:
                pos1 = int(core1Dismatch.group(2))
                if core1flag:
                    core1flag = False
                    core1pos = pos1
                    dis1 = pos1
                    while(dis1-1) >0 :
                        core1.append(0)
                        dis1 = dis1-1
                val1 = int(core1Dismatch.group(3))
                dis1 = check(core1pos,pos1)
                if dis1!=0:
                    while (dis1-1) > 0:
                        core1.append(0)
                        dis1 = dis1-1
                core1.append(val1)
                core1pos = pos1
            if l2Hismatch:
                pos2 = int(l2Hismatch.group(2))
                if l2flag:
                    l2flag = False
                    l2pos = pos2
                    dis2 = pos2
                    while(dis2-1) > 0:
                        l2.append(0)
                        dis2 = dis2-1
                val2 = int(l2Hismatch.group(3))
                dis2 = check(l2pos,pos2)
                if dis2!=0:
                    while (dis2-1) > 0:
                        l2.append(0)
                        dis2 = dis2-1
                l2.append(val2)
                l2pos = pos2
            if l2cachehitsmatch:
                cachehits = l2cachehitsmatch.group(2)
            if core0readmatch:
                read0 = int(core0readmatch.group(2))
            if core0writematch:
                write0 = int(core0writematch.group(2))
            if core0commitmatch:
                commit0 = float(core0commitmatch.group(2))
            if core0cpimatch:
                cpi0 = core0cpimatch.group(2)
                if(cpi0!='nan'):
                    cpi0=float(cpi0)
                else:
                    cpi0=0
            if core1readmatch:
                read1 = int(core1readmatch.group(2))
            if core1writematch:
                write1 = int(core1writematch.group(2))
            if core1commitmatch:
                commit1 = float(core1commitmatch.group(2))
            if core1cpimatch:
                cpi1 = core1cpimatch.group(2)
                if(cpi1!='nan'):
                    cpi1=float(cpi1)
                else:
                    cpi1=0
            if threadendmatch:
                #full the histogram
                dis0 = check(pos0,300)
                dis1 = check(pos1,300)
                dis2 = check(pos2,30)
                if (dis0==0):
                    if pos0==299:
                        core0.append(0)
                else:
                    while dis0 > 0 :
                        core0.append(0)
                        dis0 = dis0-1
                if (dis1==0):
                    if pos1 == 299:
                        core1.append(0)
                else:
                    while dis1 > 0:
                        core1.append(0)
                        dis1 = dis1-1
                if (dis2==0):
                    if pos2 == 29:
                        l2.append(0)
                else:
                    while dis2 > 0:
                        l2.append(0)
                        dis2 = dis2-1
                print len(core0)
                assert len(core0)==300, "core0 len error"
                assert len(core1)==300, "core1 len error"
                assert len(l2)==30,"l2 len error"
                ##
                ##this part is to calc the added up reuse distance distribution
                ##when the cpi0 and cpi1 are both exist we do the calc
                ##when it's not ,add the direct
                ##
                if ((cpi1!=0) and (cpi0!=0)):
                    fac0=(read0+write0)/commit0
                    fac1=(read1+write1)/commit1
                    cpic0=cpi0/cpi1
                    cpic1=cpi1/cpi0
                    temp0 = (fac1/fac0)*cpic0
                    temp1 = (fac0/fac1)*cpic1
                    core0coe = float('%.2f'%temp0)
                    core1coe = float('%.2f'%temp1)
                    for i in range(300):
                        core0[i] = core0[i]*(1+core0coe)
                        core1[i] = core1[i]*(1+core1coe)
                        l2R.append(int(core0[i]+core1[i]))
                else:
                    for j in range(300):
                        l2R.append(int(core0[j]+core1[j]))
                assert len(l2R)==300, "l2R error"
                for k in range(300):
                    RDHWrite.write("%d\n" %l2R[k])
                for m in range(30):
                    SDHWrite.write("%d " %l2[m])
                RDHWrite.write('thread end\n')
                SDHWrite.write("\n")
                break
            threadlines = fpRead.readline()
    lines = fpRead.readline()
fpRead.close()
RDHWrite.close()
SDHWrite.close()

<|endoftext|>


<commit_before>#!/usr/bin/env python
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import itertools
import logging
import re
import time
from datetime import datetime

from django.utils.translation import ugettext as _

from jobbrowser.apis.base_api import Api

LOG = logging.getLogger(__name__)

try:
  from beeswax.models import Session
  from impala.server import get_api as get_impalad_api, _get_impala_server_url
except Exception, e:
  LOG.exception('Some application are not enabled: %s' % e)

class QueryApi(Api):

  def __init__(self, user):
    self.user = user
    session = Session.objects.get_session(self.user, application='impala')
    self.server_url = _get_impala_server_url(session)

  def apps(self, filters):
    kwargs = {}

    api = get_impalad_api(user=self.user, url=self.server_url)

    jobs = api.get_queries(**kwargs)

    filter_list = self._get_filter_list(filters)
    jobs_iter = itertools.chain(jobs['in_flight_queries'], jobs['completed_queries'])
    jobs_iter_filtered = self._n_filter(filter_list, jobs_iter)

    return {
      'apps': [{
        'id': job['query_id'],
        'name': job['stmt'][:100] + ('...' if len(job['stmt']) > 100 else ''),
        'status': job['state'],
        'apiStatus': self._api_status(job['state']),
        'type': job['stmt_type'],
        'user': job['effective_user'],
        'queue': job['resource_pool'],
        'progress': job['progress'],
        'canWrite': job in jobs['in_flight_queries'],
        'duration': self._time_in_ms_groups(re.search(r"\s*(([\d.]*)([a-z]*))(([\d.]*)([a-z]*))?(([\d.]*)([a-z]*))?", job['duration'], re.MULTILINE).groups()),
        'submitted': job['start_time'],
        # Extra specific
        'rows_fetched': job['rows_fetched'],
        'waiting': job['waiting'],
        'waiting_time': job['waiting_time']
      } for job in jobs_iter_filtered],
      'total': jobs['num_in_flight_queries'] + jobs['num_executing_queries'] + jobs['num_waiting_queries']
    }

  def _time_in_ms_groups(self, groups):
    time = 0
    for x in range(0, len(groups), 3):
      if groups[x+1]:
        time += self._time_in_ms(groups[x+1], groups[x+2])
    return time

  def _time_in_ms(self, time, period):
    if period == 'ns':
      return float(time) / 1000
    elif period == 'ms':
      return float(time)
    elif period == 's':
      return float(time) * 1000
    elif period == 'm':
      return float(time) * 60000 #1000*60
    elif period == 'h':
      return float(time) * 3600000 #1000*60*60
    elif period == 'd':
      return float(time) * 86400000  # 1000*60*60*24
    else:
      return float(time)

  def app(self, appid):
    api = get_impalad_api(user=self.user, url=self.server_url)

    query = api.get_query_profile(query_id=appid)
    if query.get('error'):
      return {
        'status': -1,
        'message': query.get('error')
      }

    user = re.search(r"^\s*User:\s?([^\n\r]*)$", query['profile'], re.MULTILINE).group(1)
    status = re.search(r"^\s*Query State:\s?([^\n\r]*)$$", query['profile'], re.MULTILINE).group(1)
    stmt = re.search(r"^\s*Sql Statement:\s?([^\n\r]*)$$", query['profile'], re.MULTILINE).group(1)
    partitions = re.findall(r"partitions=\s*(\d)+\s*\/\s*(\d)+", query['profile'])
    end_time = re.search(r"^\s*End Time:\s?([^\n\r]*)$", query['profile'], re.MULTILINE).group(1)
    submitted = re.search(r"^\s*Start Time:\s?([^\n\r]*)$", query['profile'], re.MULTILINE).group(1)

    progress = 0
    if end_time:
      progress = 100
    elif partitions:
      for partition in partitions:
        progress += float(partition[0]) / float(partition[1])
      progress /= len(partitions)
      progress *= 100

    if end_time:
      end_time_ms = int(time.mktime(datetime.strptime(end_time[:26], '%Y-%m-%d %H:%M:%S.%f').timetuple()))*1000
      start_time_ms = int(time.mktime(datetime.strptime(submitted[:26], '%Y-%m-%d %H:%M:%S.%f').timetuple()))*1000
      duration_ms = end_time_ms - start_time_ms
    else:
      duration_ms = 0

    common = {
        'id': appid,
        'name': stmt,
        'status': status,
        'apiStatus': self._api_status(status),
        'user': user,
        'progress': progress,
        'duration': duration_ms,
        'submitted': submitted,
        'type': 'queries'
    }

    common['properties'] = {
      'memory': '',
      'profile': '',
      'plan': ''
    }

    return common

  def action(self, appid, action):
    message = {'message': '', 'status': 0}

    if action.get('action') == 'kill':
      api = get_impalad_api(user=self.user, url=self.server_url)

      for _id in appid:
        result = api.kill(_id)
        if result.get('error'):
          message['message'] = result.get('error')
          message['status'] = -1
        elif result.get('contents') and message.get('status') != -1:
          message['message'] = result.get('contents')

    return message;

  def logs(self, appid, app_type, log_name=None):
    return {'logs': ''}

  def profile(self, appid, app_type, app_property, app_filters):
    if app_property == 'memory':
      return self._memory(appid, app_type, app_property, app_filters)
    elif app_property == 'profile':
      return self._query_profile(appid)
    else:
      return self._query(appid)

  def _memory(self, appid, app_type, app_property, app_filters):
    api = get_impalad_api(user=self.user, url=self.server_url)
    return api.get_query_memory(query_id=appid);

  def _query(self, appid):
    api = get_impalad_api(user=self.user, url=self.server_url)
    return api.get_query(query_id=appid)

  def _query_profile(self, appid):
    api = get_impalad_api(user=self.user, url=self.server_url)
    return api.get_query_profile(query_id=appid)

  def _api_status_filter(self, status):
    if status in ['RUNNING', 'CREATED']:
      return 'RUNNING'
    elif status in ['FINISHED']:
      return 'COMPLETED'
    else:
      return 'FAILED'

  def _api_status(self, status):
    if status in ['RUNNING', 'CREATED']:
      return 'RUNNING'
    elif status in ['FINISHED']:
      return 'SUCCEEDED'
    else:
      return 'FAILED'

  def _get_filter_list(self, filters):
    filter_list = []
    if filters.get("text"):
      filter_names = {
        'user':'effective_user',
        'id':'query_id',
        'name':'state',
        'type':'stmt_type',
        'status':'status'
      }

      def makeLambda(name, value):
        return lambda app: app[name] == value

      for key, name in filter_names.items():
          text_filter = re.search(r"\s*("+key+")\s*:([^ ]+)", filters.get("text"))
          if text_filter and text_filter.group(1) == key:
            filter_list.append(makeLambda(name, text_filter.group(2).strip()))
    if filters.get("time"):
      time_filter = filters.get("time")
      period_ms = self._time_in_ms(float(time_filter.get("time_value")), time_filter.get("time_unit")[0:1])
      current_ms = time.time() * 1000.0
      filter_list.append(lambda app: current_ms - (time.mktime(datetime.strptime(app['start_time'][:26], '%Y-%m-%d %H:%M:%S.%f').timetuple()) * 1000) < period_ms)
    if filters.get("states"):
      filter_list.append(lambda app: self._api_status_filter(app['state']).lower() in filters.get("states"))

    return filter_list

  def _n_filter(self, filters, tuples):
    for f in filters:
      tuples = filter(f, tuples)
    return tuples<commit_msg>HUE-7420 [jb] total number of impala queries incorrect<commit_after>#!/usr/bin/env python
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import itertools
import logging
import re
import time
from datetime import datetime

from django.utils.translation import ugettext as _

from jobbrowser.apis.base_api import Api

LOG = logging.getLogger(__name__)

try:
  from beeswax.models import Session
  from impala.server import get_api as get_impalad_api, _get_impala_server_url
except Exception, e:
  LOG.exception('Some application are not enabled: %s' % e)

class QueryApi(Api):

  def __init__(self, user):
    self.user = user
    session = Session.objects.get_session(self.user, application='impala')
    self.server_url = _get_impala_server_url(session)

  def apps(self, filters):
    kwargs = {}

    api = get_impalad_api(user=self.user, url=self.server_url)

    jobs = api.get_queries(**kwargs)

    filter_list = self._get_filter_list(filters)
    jobs_iter = itertools.chain(jobs['in_flight_queries'], jobs['completed_queries'])
    jobs_iter_filtered = self._n_filter(filter_list, jobs_iter)

    return {
      'apps': [{
        'id': job['query_id'],
        'name': job['stmt'][:100] + ('...' if len(job['stmt']) > 100 else ''),
        'status': job['state'],
        'apiStatus': self._api_status(job['state']),
        'type': job['stmt_type'],
        'user': job['effective_user'],
        'queue': job['resource_pool'],
        'progress': job['progress'],
        'canWrite': job in jobs['in_flight_queries'],
        'duration': self._time_in_ms_groups(re.search(r"\s*(([\d.]*)([a-z]*))(([\d.]*)([a-z]*))?(([\d.]*)([a-z]*))?", job['duration'], re.MULTILINE).groups()),
        'submitted': job['start_time'],
        # Extra specific
        'rows_fetched': job['rows_fetched'],
        'waiting': job['waiting'],
        'waiting_time': job['waiting_time']
      } for job in jobs_iter_filtered],
      'total': len(jobs['in_flight_queries']) + len(jobs['completed_queries'])
    }

  def _time_in_ms_groups(self, groups):
    time = 0
    for x in range(0, len(groups), 3):
      if groups[x+1]:
        time += self._time_in_ms(groups[x+1], groups[x+2])
    return time

  def _time_in_ms(self, time, period):
    if period == 'ns':
      return float(time) / 1000
    elif period == 'ms':
      return float(time)
    elif period == 's':
      return float(time) * 1000
    elif period == 'm':
      return float(time) * 60000 #1000*60
    elif period == 'h':
      return float(time) * 3600000 #1000*60*60
    elif period == 'd':
      return float(time) * 86400000  # 1000*60*60*24
    else:
      return float(time)

  def app(self, appid):
    api = get_impalad_api(user=self.user, url=self.server_url)

    query = api.get_query_profile(query_id=appid)
    if query.get('error'):
      return {
        'status': -1,
        'message': query.get('error')
      }

    user = re.search(r"^\s*User:\s?([^\n\r]*)$", query['profile'], re.MULTILINE).group(1)
    status = re.search(r"^\s*Query State:\s?([^\n\r]*)$$", query['profile'], re.MULTILINE).group(1)
    stmt = re.search(r"^\s*Sql Statement:\s?([^\n\r]*)$$", query['profile'], re.MULTILINE).group(1)
    partitions = re.findall(r"partitions=\s*(\d)+\s*\/\s*(\d)+", query['profile'])
    end_time = re.search(r"^\s*End Time:\s?([^\n\r]*)$", query['profile'], re.MULTILINE).group(1)
    submitted = re.search(r"^\s*Start Time:\s?([^\n\r]*)$", query['profile'], re.MULTILINE).group(1)

    progress = 0
    if end_time:
      progress = 100
    elif partitions:
      for partition in partitions:
        progress += float(partition[0]) / float(partition[1])
      progress /= len(partitions)
      progress *= 100

    if end_time:
      end_time_ms = int(time.mktime(datetime.strptime(end_time[:26], '%Y-%m-%d %H:%M:%S.%f').timetuple()))*1000
      start_time_ms = int(time.mktime(datetime.strptime(submitted[:26], '%Y-%m-%d %H:%M:%S.%f').timetuple()))*1000
      duration_ms = end_time_ms - start_time_ms
    else:
      duration_ms = 0

    common = {
        'id': appid,
        'name': stmt,
        'status': status,
        'apiStatus': self._api_status(status),
        'user': user,
        'progress': progress,
        'duration': duration_ms,
        'submitted': submitted,
        'type': 'queries'
    }

    common['properties'] = {
      'memory': '',
      'profile': '',
      'plan': ''
    }

    return common

  def action(self, appid, action):
    message = {'message': '', 'status': 0}

    if action.get('action') == 'kill':
      api = get_impalad_api(user=self.user, url=self.server_url)

      for _id in appid:
        result = api.kill(_id)
        if result.get('error'):
          message['message'] = result.get('error')
          message['status'] = -1
        elif result.get('contents') and message.get('status') != -1:
          message['message'] = result.get('contents')

    return message;

  def logs(self, appid, app_type, log_name=None):
    return {'logs': ''}

  def profile(self, appid, app_type, app_property, app_filters):
    if app_property == 'memory':
      return self._memory(appid, app_type, app_property, app_filters)
    elif app_property == 'profile':
      return self._query_profile(appid)
    else:
      return self._query(appid)

  def _memory(self, appid, app_type, app_property, app_filters):
    api = get_impalad_api(user=self.user, url=self.server_url)
    return api.get_query_memory(query_id=appid);

  def _query(self, appid):
    api = get_impalad_api(user=self.user, url=self.server_url)
    return api.get_query(query_id=appid)

  def _query_profile(self, appid):
    api = get_impalad_api(user=self.user, url=self.server_url)
    return api.get_query_profile(query_id=appid)

  def _api_status_filter(self, status):
    if status in ['RUNNING', 'CREATED']:
      return 'RUNNING'
    elif status in ['FINISHED']:
      return 'COMPLETED'
    else:
      return 'FAILED'

  def _api_status(self, status):
    if status in ['RUNNING', 'CREATED']:
      return 'RUNNING'
    elif status in ['FINISHED']:
      return 'SUCCEEDED'
    else:
      return 'FAILED'

  def _get_filter_list(self, filters):
    filter_list = []
    if filters.get("text"):
      filter_names = {
        'user':'effective_user',
        'id':'query_id',
        'name':'state',
        'type':'stmt_type',
        'status':'status'
      }

      def makeLambda(name, value):
        return lambda app: app[name] == value

      for key, name in filter_names.items():
          text_filter = re.search(r"\s*("+key+")\s*:([^ ]+)", filters.get("text"))
          if text_filter and text_filter.group(1) == key:
            filter_list.append(makeLambda(name, text_filter.group(2).strip()))
    if filters.get("time"):
      time_filter = filters.get("time")
      period_ms = self._time_in_ms(float(time_filter.get("time_value")), time_filter.get("time_unit")[0:1])
      current_ms = time.time() * 1000.0
      filter_list.append(lambda app: current_ms - (time.mktime(datetime.strptime(app['start_time'][:26], '%Y-%m-%d %H:%M:%S.%f').timetuple()) * 1000) < period_ms)
    if filters.get("states"):
      filter_list.append(lambda app: self._api_status_filter(app['state']).lower() in filters.get("states"))

    return filter_list

  def _n_filter(self, filters, tuples):
    for f in filters:
      tuples = filter(f, tuples)
    return tuples<|endoftext|>


<commit_before>"""
Cron job command for running each of the prediction pipelines as needed.
"""
import sys
import time

from django.core.management.base import BaseCommand

from apps.predict.models import PredictStrain, get_timeout
from apps.pipeline.models import ProgramRun

class Command(BaseCommand):
    help = """Schedule each of the pipeline tasks with the shell or LSF"""

    def handle(self, **options):
        """Called from the command line"""
        skipped, errors = 0, 0
        for strain in PredictStrain.objects.filter(piperun__isnull=True,\
                dataset__created__gt=get_timeout()):
            dl = strain.files_status
            sys.stderr.write("Strain: {}, Files: {}".format(strain, dl))
            if dl is 0:
                skipped += 1
            elif dl is 1:
                skipped += 1
            elif dl is 2:
                errors += 1
            elif strain.run():
                sys.stderr.write("SUBMITTED [OK]\n")
            else:
                sys.stderr.write("SUBMITTED [ERROR]\n")
            time.sleep(1)

        if skipped or errors:
            sys.stderr.write("DOWNLOADS WARN: %d SKIPPED, %d ERRORS\n" % (skipped, errors))

        for strain in PredictStrain.objects.filter(
                piperun__programs__is_submitted=False,
                piperun__programs__is_complete=True,
                piperun__programs__is_error=True,
                ).distinct():
            sys.stderr.write("Re-Strain: %s " % str(strain))
            if strain.piperun.rerun():
                sys.stderr.write("SUBMITTED [OK]\n")
            else:
                sys.stderr.write("SUBMITTED [ERROR]\n")
            time.sleep(1)

        for progrun in ProgramRun.objects.filter(is_submitted=True, is_complete=False, is_error=False):
            sys.stderr.write("Run Status: %s " % str(progrun))
            if progrun.update_status():
                sys.stderr.write(" [COMPLETE]\n")
                progrun.piperun.predictstrain_set.first().update_status()
            elif progrun.is_error:
                sys.stderr.write(" [ERROR]\n")
            else:
                if 'fatal' in str(progrun).lower():
                    sys.stderr.write(" [FAILED]\n")
                    progrun.is_error = True
                    progrun.save()
                else:
                    sys.stderr.write(" [WAITING]\n")

<commit_msg>Time out each job if it's too old<commit_after>"""
Cron job command for running each of the prediction pipelines as needed.
"""
import sys
import time

from django.core.management.base import BaseCommand

from apps.predict.models import PredictStrain, get_timeout
from apps.pipeline.models import ProgramRun

class Command(BaseCommand):
    help = """Schedule each of the pipeline tasks with the shell or LSF"""

    def handle(self, **options):
        """Called from the command line"""
        skipped, errors = 0, 0
        for strain in PredictStrain.objects.filter(piperun__isnull=True,\
                dataset__created__gt=get_timeout()):
            dl = strain.files_status
            sys.stderr.write("Strain: {}, Files: {}".format(strain, dl))
            if dl is 0:
                skipped += 1
            elif dl is 1:
                skipped += 1
            elif dl is 2:
                errors += 1
            elif strain.run():
                sys.stderr.write("SUBMITTED [OK]\n")
            else:
                sys.stderr.write("SUBMITTED [ERROR]\n")
            time.sleep(1)

        if skipped or errors:
            sys.stderr.write("DOWNLOADS WARN: %d SKIPPED, %d ERRORS\n" % (skipped, errors))

        for strain in PredictStrain.objects.filter(
                piperun__programs__is_submitted=False,
                piperun__programs__is_complete=True,
                piperun__programs__is_error=True,
                ).distinct():
            sys.stderr.write("Re-Strain: %s " % str(strain))
            if strain.piperun.rerun():
                sys.stderr.write("SUBMITTED [OK]\n")
            else:
                sys.stderr.write("SUBMITTED [ERROR]\n")
            time.sleep(1)

        for progrun in ProgramRun.objects.filter(is_submitted=True, is_complete=False, is_error=False):
            sys.stderr.write("Run Status: %s " % str(progrun))
            if progrun.update_status():
                sys.stderr.write(" [COMPLETE]\n")
                progrun.piperun.predictstrain_set.first().update_status()
            elif progrun.is_error:
                sys.stderr.write(" [ERROR]\n")
            else:
                if 'fatal' in str(progrun).lower():
                    sys.stderr.write(" [FAILED]\n")
                    progrun.is_error = True
                    progrun.save()
                elif progrun.created < get_timeout():
                    sys.stderr.write(" [TIMEOUT]\n")
                    progrun.error_text = str(progrun.error_text) + '[JOB TIMEOUT]'
                    progrun.is_error = True
                    progrun.save()
                else:
                    sys.stderr.write(" [WAITING]\n")
<|endoftext|>


<commit_before>from .algorithm import UserAverage, UserAverage_Life
from apps.recommenders.UserAverage.benchmark.models import BenchUserAverage
from django.utils import timezone

import logging
logger = logging.getLogger(__name__)

def runUserAverage(songSetLimit):
    logger.info("[Start Run User Average - Benchmark]")
    startedAt = timezone.now()
    UserAverage(songSetLimit=songSetLimit)
    finishedAt = timezone.now()
    BenchUserAverage.objects.create(
        life=UserAverage_Life.objects.last(),
        started_at=startedAt,
        finished_at=finishedAt
    )
    logger.info(
        "Benchmark: Start at - "
        + str(startedAt)
        + " || Finished at -"
        + str(finishedAt)
    )
    logger.info("[Start Run User Average] - Benchmark")
<commit_msg>subindo linha<commit_after>from django.utils import timezone
from .algorithm import UserAverage, UserAverage_Life
from apps.recommenders.UserAverage.benchmark.models import BenchUserAverage

import logging
logger = logging.getLogger(__name__)

def runUserAverage(songSetLimit):
    logger.info("[Start Run User Average - Benchmark]")
    startedAt = timezone.now()
    UserAverage(songSetLimit=songSetLimit)
    finishedAt = timezone.now()
    BenchUserAverage.objects.create(
        life=UserAverage_Life.objects.last(),
        started_at=startedAt,
        finished_at=finishedAt
    )
    logger.info(
        "Benchmark: Start at - "
        + str(startedAt)
        + " || Finished at -"
        + str(finishedAt)
    )
    logger.info("[Start Run User Average] - Benchmark")
<|endoftext|>


<commit_before>"""Test the search config
"""
import pytest

from jobfunnel.config import SearchConfig
from jobfunnel.resources import Locale, Remoteness, enums

@pytest.mark.parametrize("keywords, exp_query_str", [
    (['b33f', 'd3ad'], 'b33f d3ad'),
    (['trumpet'], 'trumpet'),
])
def test_search_config_query_string(mocker, keywords, exp_query_str):
    """Test that search config can build keyword query string correctly.
    """
    cfg = SearchConfig(
        keywords=keywords,
        province_or_state=mocker.Mock(),
        locale=Locale.CANADA_FRENCH,
        providers=mocker.Mock(),
    )

    # FUT
    query_str = cfg.query_string

    # Assertions
    assert query_str == exp_query_str

@pytest.mark.parametrize("locale, domain, exp_domain", [
    (Locale.CANADA_ENGLISH, None, 'ca'),
    (Locale.CANADA_FRENCH, None, 'ca'),
    (Locale.USA_ENGLISH, None, 'com'),
    (Locale.UK_ENGLISH, None, 'co.uk'),
    (Locale.FRANCE_FRENCH, None, 'fr'),
    (Locale.USA_ENGLISH, 'xyz', 'xyz'),
    (None, None, None),
])
def test_search_config_init(mocker, locale, domain, exp_domain):
    """Make sure the init functions as we expect wrt to domain selection
    """
    # FUT
    if not locale:
        # test our error
        with pytest.raises(ValueError, match=r"Unknown domain for locale.*"):
            cfg = SearchConfig(
                keywords=mocker.Mock(),
                province_or_state=mocker.Mock(),
                locale=-1,  # AKA an unknown Enum entry to Locale
                providers=mocker.Mock(),
            )
    else:
        cfg = SearchConfig(
            keywords=mocker.Mock(),
            province_or_state=mocker.Mock(),
            locale=locale,
            domain=domain,
            providers=mocker.Mock(),
        )

        # Assertions
        assert cfg.domain == exp_domain

@pytest.mark.parametrize('keywords, province_or_state, locale, providers, in_city',
                         [(['Ice Cream', 'Spiderman'], None, Locale.USA_ENGLISH, [enums.Provider.INDEED], 'Austin')])
def test_search_config_validate_invalid_province(keywords, province_or_state, locale, providers, in_city):
    cfg = SearchConfig(keywords, province_or_state, locale, providers, city=in_city)

    with pytest.raises(AssertionError, match="Province/State not set"):
        cfg.validate()

@pytest.mark.parametrize('keywords, province_or_state, locale, providers, in_city',
                         [(['Python', 'Space'], 'Texas', Locale.USA_ENGLISH, [enums.Provider.INDEED], None)])
def test_search_config_validate_invalid_city(keywords, province_or_state, locale, providers, in_city):
    cfg = SearchConfig(keywords, province_or_state, locale, providers, city=in_city)

    with pytest.raises(AssertionError, match="City not set"):
        cfg.validate()

@pytest.mark.parametrize('keywords, province_or_state,  locale, providers, in_city,  in_domain',
                         [(['Python', 'Space'], 'Texas', None, [enums.Provider.INDEED], 'Austin', 'com')])
def test_search_config_validate_invalid_locale(keywords, province_or_state, locale, providers, in_city, in_domain):
    cfg = SearchConfig(keywords, province_or_state, locale, providers, city=in_city, domain=in_domain)

    with pytest.raises(AssertionError, match="Locale not set"):
        cfg.validate()

@pytest.mark.parametrize('keywords, province_or_state, locale, providers, in_city',
                         [(['Ice Cream', 'Spiderman'], 'Texas', Locale.USA_ENGLISH, [],
                           'Austin')])
def test_search_config_validate_invalid_providers(keywords, province_or_state, locale, providers, in_city):
    cfg = SearchConfig(keywords, province_or_state, locale, providers, city=in_city)

    with pytest.raises(AssertionError, match="Providers not set"):
        cfg.validate()

@pytest.mark.parametrize('keywords, province_or_state, locale, providers, in_city',
                         [([], 'Texas', Locale.USA_ENGLISH, [enums.Provider.INDEED], 'Austin')])
def test_search_config_validate_invalid_keywords(keywords, province_or_state, locale, providers, in_city):
    cfg = SearchConfig(keywords, province_or_state, locale, providers, city=in_city)

    with pytest.raises(AssertionError, match='Keywords not set'):
        cfg.validate()

@pytest.mark.parametrize('keywords, province_or_state, locale, providers, in_city, in_max_listing_days',
                         [(['Ice Cream', 'Spiderman'], Locale.USA_ENGLISH, Locale.USA_ENGLISH, [enums.Provider.INDEED],
                           'Austin', -1)])
def test_search_config_validate_invalid_max_posting_days(keywords, province_or_state, locale, providers, in_city,
                                                 in_max_listing_days):
    cfg = SearchConfig(keywords, province_or_state, locale, providers, city=in_city, max_listing_days=in_max_listing_days)

    with pytest.raises(AssertionError, match="Cannot set max posting days < 1"):
        cfg.validate()

@pytest.mark.parametrize('keywords, province_or_state, locale, providers, in_city',
                         [(['Ice Cream', 'Spiderman'], 'Texas', Locale.USA_ENGLISH, [enums.Provider.INDEED], 'Austin')])
def test_search_config_validate_domain(keywords, province_or_state, locale, providers, in_city):
    cfg = SearchConfig(keywords, province_or_state, locale, providers, city=in_city)

    # We have to force an invalid domain because the constructor ensures that it is valid.
    cfg.domain = None

    with pytest.raises(AssertionError, match="Domain not set"):
        cfg.validate()

@pytest.mark.parametrize('keywords, province_or_state, locale, providers, in_city, in_remoteness',
                         [(['Ice Cream', 'Spiderman'], 'Texas', Locale.USA_ENGLISH, [enums.Provider.INDEED], 'Austin',
                           Remoteness.UNKNOWN)])
def test_search_config_validate_remoteness(keywords, province_or_state, locale, providers, in_city, in_remoteness):
    cfg = SearchConfig(keywords, province_or_state, locale, providers, city=in_city, remoteness= in_remoteness)

    with pytest.raises(AssertionError, match="Remoteness is UNKNOWN!"):
        cfg.validate()
<commit_msg>Add a configuration test for the german locale<commit_after>"""Test the search config
"""
import pytest

from jobfunnel.config import SearchConfig
from jobfunnel.resources import Locale, Remoteness, enums

@pytest.mark.parametrize("keywords, exp_query_str", [
    (['b33f', 'd3ad'], 'b33f d3ad'),
    (['trumpet'], 'trumpet'),
])
def test_search_config_query_string(mocker, keywords, exp_query_str):
    """Test that search config can build keyword query string correctly.
    """
    cfg = SearchConfig(
        keywords=keywords,
        province_or_state=mocker.Mock(),
        locale=Locale.CANADA_FRENCH,
        providers=mocker.Mock(),
    )

    # FUT
    query_str = cfg.query_string

    # Assertions
    assert query_str == exp_query_str

@pytest.mark.parametrize("locale, domain, exp_domain", [
    (Locale.CANADA_ENGLISH, None, 'ca'),
    (Locale.CANADA_FRENCH, None, 'ca'),
    (Locale.USA_ENGLISH, None, 'com'),
    (Locale.UK_ENGLISH, None, 'co.uk'),
    (Locale.FRANCE_FRENCH, None, 'fr'),
    (Locale.GERMANY_GERMAN, None, 'de'),
    (Locale.USA_ENGLISH, 'xyz', 'xyz'),
    (None, None, None),
])
def test_search_config_init(mocker, locale, domain, exp_domain):
    """Make sure the init functions as we expect wrt to domain selection
    """
    # FUT
    if not locale:
        # test our error
        with pytest.raises(ValueError, match=r"Unknown domain for locale.*"):
            cfg = SearchConfig(
                keywords=mocker.Mock(),
                province_or_state=mocker.Mock(),
                locale=-1,  # AKA an unknown Enum entry to Locale
                providers=mocker.Mock(),
            )
    else:
        cfg = SearchConfig(
            keywords=mocker.Mock(),
            province_or_state=mocker.Mock(),
            locale=locale,
            domain=domain,
            providers=mocker.Mock(),
        )

        # Assertions
        assert cfg.domain == exp_domain

@pytest.mark.parametrize('keywords, province_or_state, locale, providers, in_city',
                         [(['Ice Cream', 'Spiderman'], None, Locale.USA_ENGLISH, [enums.Provider.INDEED], 'Austin')])
def test_search_config_validate_invalid_province(keywords, province_or_state, locale, providers, in_city):
    cfg = SearchConfig(keywords, province_or_state, locale, providers, city=in_city)

    with pytest.raises(AssertionError, match="Province/State not set"):
        cfg.validate()

@pytest.mark.parametrize('keywords, province_or_state, locale, providers, in_city',
                         [(['Python', 'Space'], 'Texas', Locale.USA_ENGLISH, [enums.Provider.INDEED], None)])
def test_search_config_validate_invalid_city(keywords, province_or_state, locale, providers, in_city):
    cfg = SearchConfig(keywords, province_or_state, locale, providers, city=in_city)

    with pytest.raises(AssertionError, match="City not set"):
        cfg.validate()

@pytest.mark.parametrize('keywords, province_or_state,  locale, providers, in_city,  in_domain',
                         [(['Python', 'Space'], 'Texas', None, [enums.Provider.INDEED], 'Austin', 'com')])
def test_search_config_validate_invalid_locale(keywords, province_or_state, locale, providers, in_city, in_domain):
    cfg = SearchConfig(keywords, province_or_state, locale, providers, city=in_city, domain=in_domain)

    with pytest.raises(AssertionError, match="Locale not set"):
        cfg.validate()

@pytest.mark.parametrize('keywords, province_or_state, locale, providers, in_city',
                         [(['Ice Cream', 'Spiderman'], 'Texas', Locale.USA_ENGLISH, [],
                           'Austin')])
def test_search_config_validate_invalid_providers(keywords, province_or_state, locale, providers, in_city):
    cfg = SearchConfig(keywords, province_or_state, locale, providers, city=in_city)

    with pytest.raises(AssertionError, match="Providers not set"):
        cfg.validate()

@pytest.mark.parametrize('keywords, province_or_state, locale, providers, in_city',
                         [([], 'Texas', Locale.USA_ENGLISH, [enums.Provider.INDEED], 'Austin')])
def test_search_config_validate_invalid_keywords(keywords, province_or_state, locale, providers, in_city):
    cfg = SearchConfig(keywords, province_or_state, locale, providers, city=in_city)

    with pytest.raises(AssertionError, match='Keywords not set'):
        cfg.validate()

@pytest.mark.parametrize('keywords, province_or_state, locale, providers, in_city, in_max_listing_days',
                         [(['Ice Cream', 'Spiderman'], Locale.USA_ENGLISH, Locale.USA_ENGLISH, [enums.Provider.INDEED],
                           'Austin', -1)])
def test_search_config_validate_invalid_max_posting_days(keywords, province_or_state, locale, providers, in_city,
                                                 in_max_listing_days):
    cfg = SearchConfig(keywords, province_or_state, locale, providers, city=in_city, max_listing_days=in_max_listing_days)

    with pytest.raises(AssertionError, match="Cannot set max posting days < 1"):
        cfg.validate()

@pytest.mark.parametrize('keywords, province_or_state, locale, providers, in_city',
                         [(['Ice Cream', 'Spiderman'], 'Texas', Locale.USA_ENGLISH, [enums.Provider.INDEED], 'Austin')])
def test_search_config_validate_domain(keywords, province_or_state, locale, providers, in_city):
    cfg = SearchConfig(keywords, province_or_state, locale, providers, city=in_city)

    # We have to force an invalid domain because the constructor ensures that it is valid.
    cfg.domain = None

    with pytest.raises(AssertionError, match="Domain not set"):
        cfg.validate()

@pytest.mark.parametrize('keywords, province_or_state, locale, providers, in_city, in_remoteness',
                         [(['Ice Cream', 'Spiderman'], 'Texas', Locale.USA_ENGLISH, [enums.Provider.INDEED], 'Austin',
                           Remoteness.UNKNOWN)])
def test_search_config_validate_remoteness(keywords, province_or_state, locale, providers, in_city, in_remoteness):
    cfg = SearchConfig(keywords, province_or_state, locale, providers, city=in_city, remoteness= in_remoteness)

    with pytest.raises(AssertionError, match="Remoteness is UNKNOWN!"):
        cfg.validate()
<|endoftext|>