<commit_before>#!/usr/bin/python
# A rule-based style and grammar checker
# Copyright (C) 2002,2003 Daniel Naber <daniel.naber@t-online.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
# USA

import getopt
import os
import re
import socket
import string
import sys
import time

import Tagger
import Rules
import SentenceSplitter

class TextChecker:
	"""A rule-based style and grammar checker."""

	def __init__(self, grammar, falsefriends, words, builtin, \
		textlanguage, mothertongue, max_sentence_length):
		# which rules are activated (lists)?
		#fixme: not used???
		self.grammar = grammar
		self.falsefriends = falsefriends
		self.words = words
		self.builtin = builtin
		self.textlanguage = textlanguage
		self.mothertongue = mothertongue
		self.max_sentence_length = max_sentence_length
		self.tagger = Tagger.Tagger()
		self.tagger.bindData()
		self.rules = Rules.Rules(self.max_sentence_length, self.grammar)
		return

	def checkFile(self, filename):
		"""Check a text file and return the results as an XML formatted list
		of possible errors."""
		f = open(filename)
		text = f.read()
		f.close()
		(rule_matches, result, tagged_words) = self.check(text)
		return (rule_matches, result, tagged_words)

	def check(self, text):
		"""Check a text string and return the results as an XML formatted list
		of possible errors."""
		splitter = SentenceSplitter.SentenceSplitter()
		sentences = splitter.split(text)
		#print sentences
		#tx = time.time()
		rule_matches = []
		all_tagged_words = []
		char_counter = 0
		for sentence in sentences:
			tagged_words = self.tagger.tagText(sentence)
			tagged_words.insert(0, ('', None, 'SENT_START'))
			tagged_words.append(('', None, 'SENT_END'))
			#print tagged_words
			all_tagged_words.extend(tagged_words)
			#print "time1: %.2fsec" % (time.time()-tx)
			#tx = time.time()
			for rule in self.rules.rules:
				matches = rule.match(tagged_words, char_counter)
				rule_matches.extend(matches)
				#print "time2: %.2fsec" % (time.time()-tx)
				#tx = time.time()
			for triple in sentence:
				char_counter = char_counter + len(triple[0])
		rule_match_list = []
		for rule_match in rule_matches:
			rule_match_list.append(rule_match.toXML())
		xml_part = "<errors>\n%s\n</errors>\n" % string.join(rule_match_list, "\n")
		#print "time3: %.2fsec" % (time.time()-tx)
		# TODO: optionally return tagged text
		#print "2=>%.2f" % (time.time()-tx)
		return (rule_matches, xml_part, all_tagged_words)

def cleanEntities(s):
	"""Replace only the most common BNC entities with their
	ASCII respresentation."""
	s = re.compile("&?").sub("&", s)
	s = re.compile("£?").sub("", s)
	s = s.replace("&ast", "*")
	s = s.replace("&aacute", "a")
	s = s.replace("&agr", "a")
	s = s.replace("&bquo", "\"")
	s = s.replace("&equo", "\"")
	s = s.replace("&quot", "'")
	s = s.replace("&deg", "")
	s = s.replace("&dollar", "$")
	s = s.replace("&eacute", "e")
	s = s.replace("&egrave", "e")
	s = s.replace("&percnt", "&")
	s = s.replace("&ndash", "-")
	s = s.replace("&mdash", "--")
	s = s.replace("&hellip", "...")
	s = s.replace("&lsqb", "[")
	s = s.replace("&rsqb", "]")
	s = s.replace("&uuml", "")
	s = s.replace("&auml", "")
	s = s.replace("&uml", "")
	s = s.replace("&Uuml", "")
	s = s.replace("&Auml", "")
	s = s.replace("&Ouml", "")
	return s

def checkBNCFiles(directory, checker):
	"""Recursively load all files from a directory, extract
	all paragraphs and feed them to the style and grammar checker
	one by one."""
	para_regex = re.compile("
(.*?)
", re.DOTALL)
	xml_regex = re.compile("<.*?>", re.DOTALL)
	whitespace_regex = re.compile("\s+", re.DOTALL)
	files = []
	filemode = 0
	if os.path.isfile(directory):		# call with a filename is okay
		files = [directory]
		filemode = 1
	else:
		files = os.listdir(directory)
	for file in files:
		filename = None
		if filemode:
			filename = file
		else:
			filename = os.path.join(directory, file)
		if os.path.isdir(filename):
			#print filename
			checkBNCFiles(filename, checker)
		elif os.path.isfile(filename):
			print "FILE=%s" % file
			f = open(filename)
			s = f.read()
			f.close()
			matches = para_regex.findall(s)
			for match in matches:
				s = xml_regex.sub("", match)
				s = whitespace_regex.sub(" ", s)
				s = cleanEntities(s)
				s = s.strip()
				#print s
				(rule_matches, result, tagged_words) = checker.check(s)
				if len(rule_matches) == 0:
					pass
					#print >> sys.stderr, "No errors found."
				else:
					print result
	return

def usage():
	print "Usage: TextChecker.py [OPTION] <filename>"
	print "  -h, --help               Show this help"
	print "  -c, --check              Check directory with BNC files in SGML format"
	print "  -g, --grammar=...        Use only these grammar rules"
	print "  -f, --falsefriends=...   Use only these false friend rules"
	print "  -m, --mothertongue=...   Your native language"
	print "  -l, --sentencelength=... Maximum sentence length"
	return

def main():
	options = None
	rest = None
	try:
		(options, rest) = getopt.getopt(sys.argv[1:], 'hcg:f:w:l:', \
			['help', 'check', 'grammar=', 'falsefriends=', 'words=', \
			'mothertongue=', 'textlanguage=', 'sentencelength='])
	except getopt.GetoptError,e :
		print >> sys.stderr, "Error: ", e
		usage()
		sys.exit(2)
	grammar = None
	falsefriends = None
	words = None
	# todo: use?
	builtin = []
	textlanguage = mothertongue = None
	max_sentence_length = None

	for o, a in options:
		if o in ("-g", "--grammar"):
			grammar = a.split(",")
		elif o in ("-f", "--falsefriends"):
			falsefriends = a.split(",")
		elif o in ("-w", "--words"):
			words = a.split(",")
		elif o in ("-m", "--mothertongue"):
			mothertongue = a
		elif o in ("-t", "--textlanguage"):
			textlanguage = a
		elif o in ("-l", "--sentencelength"):
			max_sentence_length = a

	for o, a in options:
		if o in ("-h", "--help"):
			usage()
			sys.exit(0)
		elif o in ("-c", "--check"):
			checker = TextChecker(grammar, falsefriends, words, builtin, \
				textlanguage, mothertongue, max_sentence_length)
			for filename in rest:
				checkBNCFiles(filename, checker)
			sys.exit(0)

	if len(rest) == 1:
		checker = TextChecker(grammar, falsefriends, words, builtin, \
			textlanguage, mothertongue, max_sentence_length)
		(rule_matches, result, tagged_words) = checker.checkFile(rest[0])
		if not result:
			print >> sys.stderr, "No errors found."
		else:
			print result
	else:
		usage()
	return

if __name__ == "__main__":
    main()
<commit_msg>move stuff; count paragraphs<commit_after>#!/usr/bin/python
# A rule-based style and grammar checker
# Copyright (C) 2002,2003 Daniel Naber <daniel.naber@t-online.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
# USA

import getopt
import os
import pre as re
import socket
import string
import sys
import time

import Tagger
import Rules
import SentenceSplitter

class TextChecker:
	"""A rule-based style and grammar checker."""

	def __init__(self, grammar, falsefriends, words, builtin, \
		textlanguage, mothertongue, max_sentence_length):
		# which rules are activated (lists)?
		#fixme: not used???
		self.grammar = grammar
		self.falsefriends = falsefriends
		self.words = words
		self.builtin = builtin
		self.textlanguage = textlanguage
		self.mothertongue = mothertongue
		self.max_sentence_length = max_sentence_length
		self.tagger = Tagger.Tagger()
		self.tagger.bindData()
		self.rules = Rules.Rules(self.max_sentence_length, self.grammar)
		self.bnc_paras = 0
		return

	def checkFile(self, filename):
		"""Check a text file and return the results as an XML formatted list
		of possible errors."""
		f = open(filename)
		text = f.read()
		f.close()
		(rule_matches, result, tagged_words) = self.check(text)
		return (rule_matches, result, tagged_words)

	def check(self, text):
		"""Check a text string and return the results as an XML formatted list
		of possible errors."""
		splitter = SentenceSplitter.SentenceSplitter()
		sentences = splitter.split(text)
		#print sentences
		#tx = time.time()
		rule_matches = []
		all_tagged_words = []
		char_counter = 0
		for sentence in sentences:
			tagged_words = self.tagger.tagText(sentence)
			tagged_words.insert(0, ('', None, 'SENT_START'))
			tagged_words.append(('', None, 'SENT_END'))
			#print tagged_words
			all_tagged_words.extend(tagged_words)
			#print "time1: %.2fsec" % (time.time()-tx)
			#tx = time.time()
			for rule in self.rules.rules:
				matches = rule.match(tagged_words, char_counter)
				rule_matches.extend(matches)
				#print "time2: %.2fsec" % (time.time()-tx)
				#tx = time.time()
			for triple in sentence:
				char_counter = char_counter + len(triple[0])
		rule_match_list = []
		for rule_match in rule_matches:
			rule_match_list.append(rule_match.toXML())
		xml_part = "<errors>\n%s\n</errors>\n" % string.join(rule_match_list, "\n")
		#print "time3: %.2fsec" % (time.time()-tx)
		# TODO: optionally return tagged text
		#print "2=>%.2f" % (time.time()-tx)
		return (rule_matches, xml_part, all_tagged_words)

	def cleanEntities(self, s):
		"""Replace only the most common BNC entities with their
		ASCII respresentation."""
		s = re.compile("&?").sub("&", s)
		s = re.compile("£?").sub("", s)
		s = s.replace("&ast", "*")
		s = s.replace("&aacute", "a")
		s = s.replace("&agr", "a")
		s = s.replace("&bquo", "\"")
		s = s.replace("&equo", "\"")
		s = s.replace("&quot", "'")
		s = s.replace("&deg", "")
		s = s.replace("&dollar", "$")
		s = s.replace("&eacute", "e")
		s = s.replace("&egrave", "e")
		s = s.replace("&percnt", "&")
		s = s.replace("&ndash", "-")
		s = s.replace("&mdash", "--")
		s = s.replace("&hellip", "...")
		s = s.replace("&lsqb", "[")
		s = s.replace("&rsqb", "]")
		s = s.replace("&uuml", "")
		s = s.replace("&auml", "")
		s = s.replace("&uml", "")
		s = s.replace("&Uuml", "")
		s = s.replace("&Auml", "")
		s = s.replace("&Ouml", "")
		return s

	def checkBNCFiles(self, directory, checker):
		"""Recursively load all files from a directory, extract
		all paragraphs and feed them to the style and grammar checker
		one by one."""
		para_regex = re.compile("
(.*?)
", re.DOTALL)
		xml_regex = re.compile("<.*?>", re.DOTALL)
		whitespace_regex = re.compile("\s+", re.DOTALL)
		files = []
		filemode = 0
		if os.path.isfile(directory):		# call with a filename is okay
			files = [directory]
			filemode = 1
		else:
			files = os.listdir(directory)
		for file in files:
			filename = None
			if filemode:
				filename = file
			else:
				filename = os.path.join(directory, file)
			if os.path.isdir(filename):
				#print filename
				self.checkBNCFiles(filename, checker)
			elif os.path.isfile(filename):
				print >> sys.stderr, "FILE=%s" % file
				f = open(filename)
				s = f.read()
				f.close()
				matches = para_regex.findall(s)
				for match in matches:
					#print len(match)
					self.bnc_paras = self.bnc_paras + 1
					s = xml_regex.sub("", match)
					s = whitespace_regex.sub(" ", s)
					s = self.cleanEntities(s)
					s = s.strip()
					#print s
					(rule_matches, result, tagged_words) = checker.check(s)
					if len(rule_matches) == 0:
						pass
						#print >> sys.stderr, "No errors found."
					else:
						print result
		return

def usage():
	print "Usage: TextChecker.py [OPTION] <filename>"
	print "  -h, --help               Show this help"
	print "  -c, --check              Check directory with BNC files in SGML format"
	print "  -g, --grammar=...        Use only these grammar rules"
	print "  -f, --falsefriends=...   Use only these false friend rules"
	print "  -m, --mothertongue=...   Your native language"
	print "  -l, --sentencelength=... Maximum sentence length"
	return

def main():
	options = None
	rest = None
	try:
		(options, rest) = getopt.getopt(sys.argv[1:], 'hcg:f:w:l:', \
			['help', 'check', 'grammar=', 'falsefriends=', 'words=', \
			'mothertongue=', 'textlanguage=', 'sentencelength='])
	except getopt.GetoptError,e :
		print >> sys.stderr, "Error: ", e
		usage()
		sys.exit(2)
	grammar = None
	falsefriends = None
	words = None
	# todo: use?
	builtin = []
	textlanguage = mothertongue = None
	max_sentence_length = None

	for o, a in options:
		if o in ("-g", "--grammar"):
			grammar = a.split(",")
		elif o in ("-f", "--falsefriends"):
			falsefriends = a.split(",")
		elif o in ("-w", "--words"):
			words = a.split(",")
		elif o in ("-m", "--mothertongue"):
			mothertongue = a
		elif o in ("-t", "--textlanguage"):
			textlanguage = a
		elif o in ("-l", "--sentencelength"):
			max_sentence_length = a

	for o, a in options:
		if o in ("-h", "--help"):
			usage()
			sys.exit(0)
		elif o in ("-c", "--check"):
			checker = TextChecker(grammar, falsefriends, words, builtin, \
				textlanguage, mothertongue, max_sentence_length)
			for filename in rest:
				checker.checkBNCFiles(filename, checker)
			print >> sys.stderr, "Checked %d paragraphs." % checker.bnc_paras
			sys.exit(0)

	if len(rest) == 1:
		checker = TextChecker(grammar, falsefriends, words, builtin, \
			textlanguage, mothertongue, max_sentence_length)
		(rule_matches, result, tagged_words) = checker.checkFile(rest[0])
		if not result:
			print >> sys.stderr, "No errors found."
		else:
			print result
	else:
		usage()
	return

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Mario Lassnig, mario.lassnig@cern.ch, 2016-2017
# - Daniel Drizhuk, d.drizhuk@gmail.com, 2017
# - Tobias Wegner, tobias.wegner@cern.ch, 2017
# - Paul Nilsson, paul.nilsson@cern.ch, 2017

import Queue
import json
import os
import subprocess
import threading
import time

from pilot.control.job import send_state
from pilot.util.container import execute

import logging
logger = logging.getLogger(__name__)

def control(queues, traces, args):
    """
    (add description)

    :param queues:
    :param traces:
    :param args:
    :return:
    """

    threads = [threading.Thread(target=validate_pre,
                                kwargs={'queues': queues,
                                        'traces': traces,
                                        'args': args}),
               threading.Thread(target=execute_payloads,
                                kwargs={'queues': queues,
                                        'traces': traces,
                                        'args': args}),
               threading.Thread(target=validate_post,
                                kwargs={'queues': queues,
                                        'traces': traces,
                                        'args': args})]

    [t.start() for t in threads]

def validate_pre(queues, traces, args):
    """
    (add description)

    :param queues:
    :param traces:
    :param args:
    :return:
    """
    while not args.graceful_stop.is_set():
        try:
            job = queues.payloads.get(block=True, timeout=1)
        except Queue.Empty:
            continue

        if _validate_payload(job):
            queues.validated_payloads.put(job)
        else:
            queues.failed_payloads.put(job)

def _validate_payload(job):
    """
    (add description)

    :param job:
    :return:
    """
    # valid = random.uniform(0, 100)
    # if valid > 99:
    #     logger.warning('payload did not validate correctly -- skipping')
    #     job['errno'] = random.randint(0, 100)
    #     job['errmsg'] = 'payload failed random validation'
    #     return False
    return True

def setup_payload(job, out, err):
    """
    (add description)

    :param job:
    :param out:
    :param err:
    :return:
    """
    # log = logger.getChild(str(job['PandaID']))

    # try:
    # create symbolic link for sqlite200 and geomDB in job dir
    #    for db_name in ['sqlite200', 'geomDB']:
    #         src = '/cvmfs/atlas.cern.ch/repo/sw/database/DBRelease/current/%s' % db_name
    #         link_name = 'job-%s/%s' % (job['PandaID'], db_name)
    #         os.symlink(src, link_name)
    # except Exception as e:
    #     log.error('could not create symbolic links to database files: %s' % e)
    #     return False

    return True

def run_payload(job, out, err):
    """
    (add description)

    :param job:
    :param out:
    :param err:
    :return:
    """
    log = logger.getChild(str(job['PandaID']))

    # get the payload command from the user specific code
    pilot_user = os.environ.get('PILOT_USER', 'generic').lower()
    user = __import__('pilot.user.%s.common' % pilot_user, globals(), locals(), [pilot_user], -1)
    cmd = user.get_payload_command(job)
    log.info("payload execution command: %s" % cmd)

    # athena_version = job['homepackage'].split('/')[1]
    # asetup = 'source $ATLAS_LOCAL_ROOT_BASE/user/atlasLocalSetup.sh --quiet; '\
    #          'source $AtlasSetup/scripts/asetup.sh %s,here; ' % athena_version
    # cmd = job['transformation'] + ' ' + job['jobPars']

    # log.debug('executable=%s' % asetup + cmd)

    try:
        proc = subprocess.Popen(cmd,
                                bufsize=-1,
                                stdout=out,
                                stderr=err,
                                cwd=job['working_dir'],
                                shell=True)
    except Exception as e:
        log.error('could not execute: %s' % str(e))
        return None

    log.info('started -- pid=%s executable=%s' % (proc.pid, cmd))

    return proc

def wait_graceful(args, proc, job):
    """
    (add description)

    :param args:
    :param proc:
    :param job:
    :return:
    """
    log = logger.getChild(str(job['PandaID']))

    breaker = False
    exit_code = None
    while True:
        for i in xrange(100):
            if args.graceful_stop.is_set():
                breaker = True
                log.debug('breaking -- sending SIGTERM pid=%s' % proc.pid)
                proc.terminate()
                break
            time.sleep(0.1)
        if breaker:
            log.debug('breaking -- sleep 3s before sending SIGKILL pid=%s' % proc.pid)
            time.sleep(3)
            proc.kill()
            break

        exit_code = proc.poll()
        log.info('running: pid=%s exit_code=%s' % (proc.pid, exit_code))
        if exit_code is not None:
            break
        else:
            # send_state(job, args, 'running')
            continue

    return exit_code

def execute_payloads(queues, traces, args):
    """
    Execute queued payloads.

    :param queues:
    :param traces:
    :param args:
    :return:
    """
    while not args.graceful_stop.is_set():
        try:
            job = queues.validated_payloads.get(block=True, timeout=1)
            log = logger.getChild(str(job['PandaID']))

            q_snapshot = list(queues.finished_data_in.queue)
            peek = [s_job for s_job in q_snapshot if job['PandaID'] == s_job['PandaID']]
            if len(peek) == 0:
                queues.validated_payloads.put(job)
                for i in xrange(10):
                    if args.graceful_stop.is_set():
                        break
                    time.sleep(0.1)
                continue

            log.debug('opening payload stdout/err logs')
            out = open(os.path.join(job['working_dir'], 'payload.stdout'), 'wb')
            err = open(os.path.join(job['working_dir'], 'payload.stderr'), 'wb')

            log.debug('setting up payload environment')
            send_state(job, args, 'starting')

            exit_code = 1
            if setup_payload(job, out, err):
                log.debug('running payload')
                send_state(job, args, 'running')
                proc = run_payload(job, out, err)
                if proc is not None:
                    exit_code = wait_graceful(args, proc, job)
                    log.info('finished pid=%s exit_code=%s' % (proc.pid, exit_code))

            log.debug('closing payload stdout/err logs')
            out.close()
            err.close()

            if exit_code == 0:
                queues.finished_payloads.put(job)
            else:
                queues.failed_payloads.put(job)

        except Queue.Empty:
            continue

def validate_post(queues, traces, args):
    """
    (add description)

    :param queues:
    :param traces:
    :param args:
    :return:
    """

    while not args.graceful_stop.is_set():
        try:
            job = queues.finished_payloads.get(block=True, timeout=1)
        except Queue.Empty:
            continue
        log = logger.getChild(str(job['PandaID']))

        log.debug('adding job report for stageout')
        with open(os.path.join(job['working_dir'], 'jobReport.json')) as data_file:
            job['job_report'] = json.load(data_file)

        queues.data_out.put(job)
<commit_msg>Now using execute() in run_payload()<commit_after>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Mario Lassnig, mario.lassnig@cern.ch, 2016-2017
# - Daniel Drizhuk, d.drizhuk@gmail.com, 2017
# - Tobias Wegner, tobias.wegner@cern.ch, 2017
# - Paul Nilsson, paul.nilsson@cern.ch, 2017

import Queue
import json
import os
import subprocess
import threading
import time

from pilot.control.job import send_state
from pilot.util.container import execute

import logging
logger = logging.getLogger(__name__)

def control(queues, traces, args):
    """
    (add description)

    :param queues:
    :param traces:
    :param args:
    :return:
    """

    threads = [threading.Thread(target=validate_pre,
                                kwargs={'queues': queues,
                                        'traces': traces,
                                        'args': args}),
               threading.Thread(target=execute_payloads,
                                kwargs={'queues': queues,
                                        'traces': traces,
                                        'args': args}),
               threading.Thread(target=validate_post,
                                kwargs={'queues': queues,
                                        'traces': traces,
                                        'args': args})]

    [t.start() for t in threads]

def validate_pre(queues, traces, args):
    """
    (add description)

    :param queues:
    :param traces:
    :param args:
    :return:
    """
    while not args.graceful_stop.is_set():
        try:
            job = queues.payloads.get(block=True, timeout=1)
        except Queue.Empty:
            continue

        if _validate_payload(job):
            queues.validated_payloads.put(job)
        else:
            queues.failed_payloads.put(job)

def _validate_payload(job):
    """
    (add description)

    :param job:
    :return:
    """
    # valid = random.uniform(0, 100)
    # if valid > 99:
    #     logger.warning('payload did not validate correctly -- skipping')
    #     job['errno'] = random.randint(0, 100)
    #     job['errmsg'] = 'payload failed random validation'
    #     return False
    return True

def setup_payload(job, out, err):
    """
    (add description)

    :param job:
    :param out:
    :param err:
    :return:
    """
    # log = logger.getChild(str(job['PandaID']))

    # try:
    # create symbolic link for sqlite200 and geomDB in job dir
    #    for db_name in ['sqlite200', 'geomDB']:
    #         src = '/cvmfs/atlas.cern.ch/repo/sw/database/DBRelease/current/%s' % db_name
    #         link_name = 'job-%s/%s' % (job['PandaID'], db_name)
    #         os.symlink(src, link_name)
    # except Exception as e:
    #     log.error('could not create symbolic links to database files: %s' % e)
    #     return False

    return True

def run_payload(job, out, err):
    """
    (add description)

    :param job:
    :param out:
    :param err:
    :return:
    """
    log = logger.getChild(str(job['PandaID']))

    # get the payload command from the user specific code
    pilot_user = os.environ.get('PILOT_USER', 'generic').lower()
    user = __import__('pilot.user.%s.common' % pilot_user, globals(), locals(), [pilot_user], -1)
    cmd = user.get_payload_command(job)
    log.info("payload execution command: %s" % cmd)

    # athena_version = job['homepackage'].split('/')[1]
    # asetup = 'source $ATLAS_LOCAL_ROOT_BASE/user/atlasLocalSetup.sh --quiet; '\
    #          'source $AtlasSetup/scripts/asetup.sh %s,here; ' % athena_version
    # cmd = job['transformation'] + ' ' + job['jobPars']

    # log.debug('executable=%s' % asetup + cmd)

    # replace platform and workdir with new function get_payload_options() or someting from experiment specific code
    try:
        exit_code, stdout, stderr = execute(cmd, platform=job['cmtConfig'], workdir=job['working_dir'])

    # try:
    #     proc = subprocess.Popen(cmd,
    #                             bufsize=-1,
    #                             stdout=out,
    #                             stderr=err,
    #                             cwd=job['working_dir'],
    #                             shell=True)
    except Exception as e:
        log.error('could not execute: %s' % str(e))
        return None

    log.info('started -- pid=%s executable=%s' % (proc.pid, cmd))

    return proc

def wait_graceful(args, proc, job):
    """
    (add description)

    :param args:
    :param proc:
    :param job:
    :return:
    """
    log = logger.getChild(str(job['PandaID']))

    breaker = False
    exit_code = None
    while True:
        for i in xrange(100):
            if args.graceful_stop.is_set():
                breaker = True
                log.debug('breaking -- sending SIGTERM pid=%s' % proc.pid)
                proc.terminate()
                break
            time.sleep(0.1)
        if breaker:
            log.debug('breaking -- sleep 3s before sending SIGKILL pid=%s' % proc.pid)
            time.sleep(3)
            proc.kill()
            break

        exit_code = proc.poll()
        log.info('running: pid=%s exit_code=%s' % (proc.pid, exit_code))
        if exit_code is not None:
            break
        else:
            # send_state(job, args, 'running')
            continue

    return exit_code

def execute_payloads(queues, traces, args):
    """
    Execute queued payloads.

    :param queues:
    :param traces:
    :param args:
    :return:
    """
    while not args.graceful_stop.is_set():
        try:
            job = queues.validated_payloads.get(block=True, timeout=1)
            log = logger.getChild(str(job['PandaID']))

            q_snapshot = list(queues.finished_data_in.queue)
            peek = [s_job for s_job in q_snapshot if job['PandaID'] == s_job['PandaID']]
            if len(peek) == 0:
                queues.validated_payloads.put(job)
                for i in xrange(10):
                    if args.graceful_stop.is_set():
                        break
                    time.sleep(0.1)
                continue

            log.debug('opening payload stdout/err logs')
            out = open(os.path.join(job['working_dir'], 'payload.stdout'), 'wb')
            err = open(os.path.join(job['working_dir'], 'payload.stderr'), 'wb')

            log.debug('setting up payload environment')
            send_state(job, args, 'starting')

            exit_code = 1
            if setup_payload(job, out, err):
                log.debug('running payload')
                send_state(job, args, 'running')
                proc = run_payload(job, out, err)
                if proc is not None:
                    exit_code = wait_graceful(args, proc, job)
                    log.info('finished pid=%s exit_code=%s' % (proc.pid, exit_code))

            log.debug('closing payload stdout/err logs')
            out.close()
            err.close()

            if exit_code == 0:
                queues.finished_payloads.put(job)
            else:
                queues.failed_payloads.put(job)

        except Queue.Empty:
            continue

def validate_post(queues, traces, args):
    """
    (add description)

    :param queues:
    :param traces:
    :param args:
    :return:
    """

    while not args.graceful_stop.is_set():
        try:
            job = queues.finished_payloads.get(block=True, timeout=1)
        except Queue.Empty:
            continue
        log = logger.getChild(str(job['PandaID']))

        log.debug('adding job report for stageout')
        with open(os.path.join(job['working_dir'], 'jobReport.json')) as data_file:
            job['job_report'] = json.load(data_file)

        queues.data_out.put(job)
<|endoftext|>


<commit_before>import decimal
from django.contrib.gis.geos import fromstr

def get_pt_from_coord(lat_input, long_input):

    lat = None
    long = None
    if lat_input and long_input:
        try:
            lat = decimal.Decimal(lat_input)
        except decimal.InvalidOperation:
            pass

        try:
            long = decimal.Decimal(long_input)
        except decimal.InvalidOperation:
            pass

    if lat and long:
        ref_pnt = fromstr('POINT(%s %s)' % (str(long), str(lat)))
    else:
        ref_pnt =  fromstr('POINT(-87.627778 41.881944)')

    return (ref_pnt, lat, long)

def get_distance(distance="806", distance_unit='m'):

    if not distance_unit in ['km','mi','m','yd','ft',]:
        distance_unit = 'm'

    if distance:
        try:
            distance = decimal.Decimal("%s" % distance)
        except:
            pass

    if not distance:
        distance = "806"
    d = {distance_unit:distance}
    #raise Exception(d)
    return d<commit_msg>fixed default lat long<commit_after>import decimal
from django.contrib.gis.geos import fromstr

def get_pt_from_coord(lat_input, long_input):

    lat = None
    long = None
    if lat_input and long_input:
        try:
            lat = decimal.Decimal(lat_input)
        except decimal.InvalidOperation:
            pass

        try:
            long = decimal.Decimal(long_input)
        except decimal.InvalidOperation:
            pass

    if not lat:
        lat = "41.881944"

    if not long:
        long = "-87.627778"

    ref_pnt = fromstr('POINT(%s %s)' % (str(long), str(lat)))

    return (ref_pnt, lat, long)

def get_distance(distance="806", distance_unit='m'):

    if not distance_unit in ['km','mi','m','yd','ft',]:
        distance_unit = 'm'

    if distance:
        try:
            distance = decimal.Decimal("%s" % distance)
        except:
            pass

    if not distance:
        distance = "806"
    d = {distance_unit:distance}
    #raise Exception(d)
    return d<|endoftext|>


<commit_before>
class ValueTypeBase (object):

    __convert_func = convert_func
    __compare_func = compare_func
    __str_func = str_func

    def     __new__( cls, value ):
        if isinstance( value, ValueType ):
            return value    # already converted type

        return super(ValueType, cls).__new__(cls, value)

    #//-------------------------------------------------------//

    def     __init__( self, value = None ):
        if self is value:
            return self     # no init needed for itself

        base_type = self.__class__.__bases__[0]

        value = self.__convert_func( value )
        assert isinstance( value, base_type )

        base_type.__init__( self, value )

    #//-------------------------------------------------------//

    def     __cmp__(self, other):
        other = self.__class__( other )

        base_type = self.__class__.__bases__[0]

        value1 = base_type( self )      # cast to base type to avoid recursion
        value2 = base_type( other )     # cast to base type to avoid recursion

        return self.__compare_func( value1, value2 )

    #//-------------------------------------------------------//

    def     __str__(self):
        base_type = self.__class__.__bases__[0]
        return base_type.__str__( self )

return ValueType

def     createValueType( base_type, convert_func, compare_func ):

    if convert_func is None:
        def     _convert( value ):
            if value is not None:
                value = base_type(value)
            else:
                value = base_type()
        convert_func = _convert

    if compare_func is None:
        compare_func = base_type.__cmp__

    class ValueType (base_type):

        __convert_func = convert_func
        __compare_func = compare_func
        __str_func = str_func

        def     __new__( cls, value ):
            if isinstance( value, ValueType ):
                return value    # already converted type

            return super(ValueType, cls).__new__(cls, value)

        #//-------------------------------------------------------//

        def     __init__( self, value = None ):
            if self is value:
                return self     # no init needed for itself

            base_type = self.__class__.__bases__[0]

            value = self.__convert_func( value )
            assert isinstance( value, base_type )

            base_type.__init__( self, value )

        #//-------------------------------------------------------//

        def     __cmp__(self, other):
            other = self.__class__( other )

            base_type = self.__class__.__bases__[0]

            value1 = base_type( self )      # cast to base type to avoid recursion
            value2 = base_type( other )     # cast to base type to avoid recursion

            return self.__compare_func( value1, value2 )

        #//-------------------------------------------------------//

        def     __str__(self):
            base_type = self.__class__.__bases__[0]
            return base_type.__str__( self )

    return ValueType

class   Converter:
    """
    A base class to convert a value into base value (string, int, bool, node and etc.)
    """

    #//-------------------------------------------------------//

    def   __init__(self, base_type, compare_func, convert_func )

        assert base_type
        assert convert_func
        assert compare_func

        self.base_type = createBaseType( base_type, self )
        self.compare_func = compare_func
        self.convert_func = convert_func

    #//-------------------------------------------------------//

    def   convert( self, value, isinstance = isinstance ):
        if isinstance( value, self.base_type ):
            return value

        value = self.convert_func( value )

        return self.base_type( value )

    #//-------------------------------------------------------//

    def   compare( self, value1, value2 ):
        value1 = self.convert( value1 )
        value2 = self.convert( value2 )

        value1 = value1.__class__.__bases__[0]( value1 )    # cast to base type to avoid recursion
        value2 = value2.__class__.__bases__[0]( value2 )    # cast to base type to avoid recursion

        return self.compare_func( value1, value2 )

    #//-------------------------------------------------------//

    def   toString( self, value ):
        raise TypeError("Abstract method. Should be implemented in child classes")4

    def   typeHelp( self, value ):
        raise TypeError("Abstract method. Should be implemented in child classes")

    def   valuesHelp( self, value ):
        raise TypeError("Abstract method. Should be implemented in child classes")

#//===========================================================================//

base_type( value )

class   ConverterString (Converter):
    """
    Converts a value into a string value
    """

    #//-------------------------------------------------------//

    def     __init__(self, compare_func = None, convert_func = None )
        if compare_func is None:
            compare_func = cmp

        def     _convert( value ):
            if value is not None:
                value = str(value)
            else:
                value = str()

        if convert_func is None:
            convert_func = _strConverter
        else:
            convert_func = lambda value, convert_func = convert_func, _convert = _convert: convert_func( _convert(value) )

        Converter.__init__( self, str, compare_func, convert_func )

    #//-------------------------------------------------------//

    def   toString( self, value ):
        return self.convert( value )

    def   typeHelp( self, value ):
        return "String"

    def   valuesHelp( self, value ):
        return "String"

class   ConverterEnum (Converter):
    """
    Converts a value into a string value
    """

    #//-------------------------------------------------------//

    class __Value (object):
        __slots__ = ('value', 'aliases', 'compare_func')

        #//-------------------------------------------------------//

        def     __init__(self, value, compare_func ):
            self.value = value
            self.aliases = []
            self.compare_func = compare_func

        #//-------------------------------------------------------//

        def     get(self):
            return self.value

        #//-------------------------------------------------------//

        def     __eq__(self, other):
            cmp_func = self.compare_func

            for value in [self.value] + self.aliases:
                result = cmp_func( value, other )
                if result == 0:
                    return True

            return False

        #//-------------------------------------------------------//

        def     __ne__(self, other):
            return not self.__eq__(other)

        #//-------------------------------------------------------//

        def     __cmp__(self, other):
            if self == other:
                return 0
            return self.compare_func( self.value, other )

        #//-------------------------------------------------------//

        def     addAlias( self, alias ):
            if self != other:
                self.aliases.append( alias )

    #//-------------------------------------------------------//

    def     __init__(self, type_converter )

        self.type_converter = type_converter
        self.allowed_values = {}

        def     _convert( value ):

        convert_func =

        Converter.__init__( self, type_converter.base_type, type_converter.compare_func, convert_func )

        if comparator is None:
            comparator = cmp

        def     _strConverter( value ):
            if value is not None:
                value = str(value)
            else:
                value = str()

        if converter is None:
            converter = _strConverter
        else:
            converter = lambda value, converter = converter, _strConverter = _strConverter: converter( _strConverter(value) )

    #//-------------------------------------------------------//

    def   toString( self, value ):
        return self.convert( value )

    def   typeHelp( self, value ):
        return "String"

    def   valuesHelp( self, value ):
        return "String"

<commit_msg><commit_after>
class ValueTypeBase (object):

    __convert_func = convert_func
    __compare_func = compare_func
    __str_func = str_func

    def     __new__( cls, value ):
        if isinstance( value, ValueType ):
            return value    # already converted type

        return super(ValueType, cls).__new__(cls, value)

    #//-------------------------------------------------------//

    def     __init__( self, value = None ):
        if self is value:
            return self     # no init needed for itself

        base_type = self.__class__.__bases__[0]

        value = self.__convert_func( value )
        assert isinstance( value, base_type )

        base_type.__init__( self, value )

    #//-------------------------------------------------------//

    def     __cmp__(self, other):
        other = self.__class__( other )

        base_type = self.__class__.__bases__[0]

        value1 = base_type( self )      # cast to base type to avoid recursion
        value2 = base_type( other )     # cast to base type to avoid recursion

        return self.__compare_func( value1, value2 )

    #//-------------------------------------------------------//

    def     __str__(self):
        base_type = self.__class__.__bases__[0]
        return base_type.__str__( self )

return ValueType

def     createValueType( base_type, converter, comparator ):

    if converter is None:
        def     _convert( value ):
            if value is not None:
                value = base_type(value)
            else:
                value = base_type()
        convert_func = _convert

    if compare_func is None:
        compare_func = base_type.__cmp__

    class ValueType (base_type):

        converter = converter
        comparator = comparator

        def     __new__( cls, value ):
            if isinstance( value, cls ):
                return value    # already converted type

            return super(ValueType, cls).__new__(cls, value)

        #//-------------------------------------------------------//

        def     __init__( self, value = None ):
            if self is value:
                return self     # no init needed for itself

            cls = self.__class__

            base_type = cls.__bases__[0]

            value = cls.converter( value )
            assert isinstance( value, base_type )

            base_type.__init__( self, value )

        #//-------------------------------------------------------//

        def     __cmp__(self, other):
            other = self.__class__( other )

            base_type = self.__class__.__bases__[0]

            value1 = base_type( self )      # cast to base type to avoid recursion
            value2 = base_type( other )     # cast to base type to avoid recursion

            return self.__compare_func( value1, value2 )

        #//-------------------------------------------------------//

        def     __str__(self):
            base_type = self.__class__.__bases__[0]
            return base_type.__str__( self )

    return ValueType

class   Converter:
    """
    A base class to convert a value into base value (string, int, bool, node and etc.)
    """

    #//-------------------------------------------------------//

    def   __init__(self, base_type, compare_func, convert_func )

        assert base_type
        assert convert_func
        assert compare_func

        self.base_type = createBaseType( base_type, self )
        self.compare_func = compare_func
        self.convert_func = convert_func

    #//-------------------------------------------------------//

    def   convert( self, value, isinstance = isinstance ):
        if isinstance( value, self.base_type ):
            return value

        value = self.convert_func( value )

        return self.base_type( value )

    #//-------------------------------------------------------//

    def   compare( self, value1, value2 ):
        value1 = self.convert( value1 )
        value2 = self.convert( value2 )

        value1 = value1.__class__.__bases__[0]( value1 )    # cast to base type to avoid recursion
        value2 = value2.__class__.__bases__[0]( value2 )    # cast to base type to avoid recursion

        return self.compare_func( value1, value2 )

    #//-------------------------------------------------------//

    def   toString( self, value ):
        raise TypeError("Abstract method. Should be implemented in child classes")4

    def   typeHelp( self, value ):
        raise TypeError("Abstract method. Should be implemented in child classes")

    def   valuesHelp( self, value ):
        raise TypeError("Abstract method. Should be implemented in child classes")

#//===========================================================================//

base_type( value )

class   ConverterString (Converter):
    """
    Converts a value into a string value
    """

    #//-------------------------------------------------------//

    def     __init__(self, compare_func = None, convert_func = None )
        if compare_func is None:
            compare_func = cmp

        def     _convert( value ):
            if value is not None:
                value = str(value)
            else:
                value = str()

        if convert_func is None:
            convert_func = _strConverter
        else:
            convert_func = lambda value, convert_func = convert_func, _convert = _convert: convert_func( _convert(value) )

        Converter.__init__( self, str, compare_func, convert_func )

    #//-------------------------------------------------------//

    def   toString( self, value ):
        return self.convert( value )

    def   typeHelp( self, value ):
        return "String"

    def   valuesHelp( self, value ):
        return "String"

class   ConverterEnum (Converter):
    """
    Converts a value into a string value
    """

    #//-------------------------------------------------------//

    class __Value (object):
        __slots__ = ('value', 'aliases', 'compare_func')

        #//-------------------------------------------------------//

        def     __init__(self, value, compare_func ):
            self.value = value
            self.aliases = []
            self.compare_func = compare_func

        #//-------------------------------------------------------//

        def     get(self):
            return self.value

        #//-------------------------------------------------------//

        def     __eq__(self, other):
            cmp_func = self.compare_func

            for value in [self.value] + self.aliases:
                result = cmp_func( value, other )
                if result == 0:
                    return True

            return False

        #//-------------------------------------------------------//

        def     __ne__(self, other):
            return not self.__eq__(other)

        #//-------------------------------------------------------//

        def     __cmp__(self, other):
            if self == other:
                return 0
            return self.compare_func( self.value, other )

        #//-------------------------------------------------------//

        def     addAlias( self, alias ):
            if self != other:
                self.aliases.append( alias )

    #//-------------------------------------------------------//

    def     __init__(self, type_converter )

        self.type_converter = type_converter
        self.allowed_values = {}

        def     _convert( value ):

        convert_func =

        Converter.__init__( self, type_converter.base_type, type_converter.compare_func, convert_func )

        if comparator is None:
            comparator = cmp

        def     _strConverter( value ):
            if value is not None:
                value = str(value)
            else:
                value = str()

        if converter is None:
            converter = _strConverter
        else:
            converter = lambda value, converter = converter, _strConverter = _strConverter: converter( _strConverter(value) )

    #//-------------------------------------------------------//

    def   toString( self, value ):
        return self.convert( value )

    def   typeHelp( self, value ):
        return "String"

    def   valuesHelp( self, value ):
        return "String"

<|endoftext|>


<commit_before>""" Connection class establishes HTTP connection with server.
    Utilized to send Proto Report Requests.
"""
import threading
import requests

from lightstep.collector_pb2 import ReportResponse

class _HTTPConnection(object):
    """Instances of _Connection are used to establish a connection to the
    server via HTTP protocol.
    """
    def __init__(self, collector_url, timeout_seconds):
        self._collector_url = collector_url
        self._lock = threading.Lock()
        self.ready = True
        self._timeout_seconds = timeout_seconds

    def open(self):
        """Establish HTTP connection to the server.
        """
        pass

    # May throw an Exception on failure.
    def report(self, *args, **kwargs):
        """Report to the server."""
        auth = args[0]
        report = args[1]
        with self._lock:
            try:
                report.auth.access_token = auth.access_token
                headers = {"Content-Type": "application/octet-stream",
                           "Accept": "application/octet-stream",
                           "Lightstep-Access-Token": report.access_token}

                r = requests.post(
                    self._collector_url,
                    headers=headers,
                    data=report.SerializeToString(),
                    timeout=self._timeout_seconds)
                resp = ReportResponse()
                resp.ParseFromString(r.content)
                return resp
            except requests.exceptions.RequestException as err:
                raise err

    def close(self):
        """Close HTTP connection to the server."""
        self.ready = False
        pass
<commit_msg>Add access token from auth not report<commit_after>""" Connection class establishes HTTP connection with server.
    Utilized to send Proto Report Requests.
"""
import threading
import requests

from lightstep.collector_pb2 import ReportResponse

class _HTTPConnection(object):
    """Instances of _Connection are used to establish a connection to the
    server via HTTP protocol.
    """
    def __init__(self, collector_url, timeout_seconds):
        self._collector_url = collector_url
        self._lock = threading.Lock()
        self.ready = True
        self._timeout_seconds = timeout_seconds

    def open(self):
        """Establish HTTP connection to the server.
        """
        pass

    # May throw an Exception on failure.
    def report(self, *args, **kwargs):
        """Report to the server."""
        auth = args[0]
        report = args[1]
        with self._lock:
            try:
                report.auth.access_token = auth.access_token
                headers = {"Content-Type": "application/octet-stream",
                           "Accept": "application/octet-stream",
                           "Lightstep-Access-Token": auth.access_token}

                r = requests.post(
                    self._collector_url,
                    headers=headers,
                    data=report.SerializeToString(),
                    timeout=self._timeout_seconds)
                resp = ReportResponse()
                resp.ParseFromString(r.content)
                return resp
            except requests.exceptions.RequestException as err:
                raise err

    def close(self):
        """Close HTTP connection to the server."""
        self.ready = False
        pass
<|endoftext|>


<commit_before>import datetime
import os

from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.staticfiles.testing import StaticLiveServerTestCase
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.common.by import By
from selenium import webdriver
from webdriver_manager.firefox import GeckoDriverManager

class SessionSecuritySeleniumTestCase(StaticLiveServerTestCase):
    """Integration tests to check the session security package is
    working as expected.
    """

    WARN_AFTER_SECONDS = 5
    EXPIRE_AFTER_SECONDS = 10
    WAIT = 5
    TOLERANCE = 1
    TEST_LOGIN_CREDS = "test"
    TEST_SERVER = "localhost:5000"

    @classmethod
    def setUpClass(cls):
        cls.start_test_broswer()
        os.environ['DJANGO_LIVE_TEST_SERVER_ADDRESS'] = cls.TEST_SERVER
        super(StaticLiveServerTestCase, cls).setUpClass()

    @classmethod
    def tearDownClass(cls):
        cls.browser.quit()
        super(StaticLiveServerTestCase, cls).tearDownClass()

    @classmethod
    def start_test_broswer(cls):
        """Starts up a headless firefox broswer for testing.
        """
        options = webdriver.FirefoxOptions()
        options.add_argument('--headless')
        cls.browser = webdriver.Firefox(executable_path=GeckoDriverManager().install(), options=options)

    def setUp(self):
        super(SessionSecuritySeleniumTestCase, self).setUp()
        settings.SESSION_SECURITY_WARN_AFTER = self.WARN_AFTER_SECONDS
        settings.SESSION_SECURITY_EXPIRE_AFTER = self.EXPIRE_AFTER_SECONDS
        self.setup_admin_user()

    def test_inactivity_session_timeout_procedure(self):
        """Tests that the session secuirty procedure works correctly by warning the user
        after a period of inactivity, then closing the session if there is no activity within a
        further time period.
        """
        self.login()

        # Check the session warning message is hidden and then displayed after the
        # SESSION_SECURITY_WARN_AFTER period.
        self.check_session_warning_displayed()

        # Perform activity in the browser and confirm the warning message is removed.
        self.perform_keyboard_action()
        self.assert_warning_is_hidden()

        # Check the session warning message is hidden and then displayed for a second time
        # after the SESSION_SECURITY_WARN_AFTER period.
        self.check_session_warning_displayed()

        # Check that the session is logged out after the SESSION_SECURITY_EXPIRE_AFTER period
        # when no activity is detected.
        start = datetime.datetime.now()

        expected_time_to_log_out = self.EXPIRE_AFTER_SECONDS - self.WARN_AFTER_SECONDS
        self.assert_logged_out(expected_time_to_log_out + self.TOLERANCE)

        end = datetime.datetime.now()
        delta = end - start

        self.assertGreaterEqual(delta.seconds, expected_time_to_log_out - self.TOLERANCE)

    def test_session_ends_on_browser_close(self):
        """Tests that the session is ended when the broswer is closed.
        """
        # Login to the system and then close the broswer
        self.login()
        self.browser.close()

        # Confirm that on reopening the browser and admin page that the session is
        # not restored.
        self.start_test_broswer()
        self.navigate_to_admin_page()
        self.assert_logged_out(self.WAIT)

    def check_session_warning_displayed(self):
        """Checks that the session security modal goes from hidden to visible within the
        required timeframe.
        """
        start = datetime.datetime.now()

        self.assert_warning_is_hidden()
        self.assert_warning_element_visible(self.WARN_AFTER_SECONDS + self.TOLERANCE)

        end = datetime.datetime.now()
        delta = end - start

        self.assertGreaterEqual(delta.seconds, self.WARN_AFTER_SECONDS)
        self.assertLessEqual(delta.seconds, self.EXPIRE_AFTER_SECONDS)

    def setup_admin_user(self):
        """Creates a test user for the admin portal.
        """
        if not User.objects.filter(username=self.TEST_LOGIN_CREDS).exists():
            test_user = User.objects.create(username=self.TEST_LOGIN_CREDS)
            test_user.set_password(self.TEST_LOGIN_CREDS)
            test_user.is_staff = True
            test_user.save()

    def login(self):
        """Performs admin portal login and confirms the presence of the session cookie.
        """
        self.navigate_to_admin_page()
        self.browser.find_element("id", "id_username").send_keys(self.TEST_LOGIN_CREDS)
        self.browser.find_element("id", "id_password").send_keys(self.TEST_LOGIN_CREDS)
        self.browser.find_element_by_xpath('//input[@value="Log in"]').click()
        self.confirm_logged_in()

    def navigate_to_admin_page(self):
        """Navigates to the admin portal.
        """
        self.browser.get('%s%s' % (self.live_server_url, '/admin/'))

    def perform_keyboard_action(self):
        """Simulates a user pressing the space bar within the browser.
        """
        action = ActionChains(self.browser)
        action.key_down(Keys.SPACE)
        action.key_up(Keys.SPACE)
        action.perform()

    def confirm_logged_in(self):
        """Confirms the broswers is logged into the admin portal by check for the logout button
        and the session cookie.
        """
        try:
            WebDriverWait(self.browser, self.WAIT).until(
                expected_conditions.presence_of_element_located((By.LINK_TEXT, "Log out")))
        except:
            self.fail("Login was unsuccessful")

        self.assertIsNotNone(self.get_session_cookie(), "Session cookie not present after logging in.")

    def assert_warning_element_visible(self, max_wait):
        """Checks that the session security modal is visible within a given time period.

        Args:
            max_wait (float): The number fo second to wait for the element to appear.
        """
        try:
            WebDriverWait(self.browser, max_wait).until(
                expected_conditions.visibility_of_element_located((By.ID, "session_security_warning")))
        except:
            self.fail("Warning message not displayed after %ds or inactivity" % (max_wait))

    def assert_warning_is_hidden(self):
        """Checks that the session security modal is non visible.
        """
        try:
            WebDriverWait(self.browser, self.WAIT).until(
                expected_conditions.invisibility_of_element((By.ID, "session_security_warning")))
        except:
            self.fail("Warning message present during active session")

    def assert_logged_out(self, delay):
        """Checks that the session has been logged out by checking for the presence of the
        login page and that the session cookie has been removed.

        Args:
            delay (float): The number of second to wait for the logout to occur.
        """
        try:
            WebDriverWait(self.browser, delay).until(
                expected_conditions.presence_of_element_located((By.ID, "id_password")))
        except:
            self.fail("Sessions was not logged out after %ds" % (delay))

        self.assertIsNone(self.get_session_cookie(), "Session cookie present after logging out.")

    def get_session_cookie(self):
        """Gets the session cookie from the broswer.

        Returns:
            dict: The session cookie dictionary.
        """
        cookies = (cookie for cookie in self.browser.get_cookies() if cookie["name"] == "sessionid")
        return next(cookies, None)

<commit_msg>LGA-2112: Linting errors fixed.<commit_after>import datetime
import os

from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.staticfiles.testing import StaticLiveServerTestCase
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.common.by import By
from selenium import webdriver
from webdriver_manager.firefox import GeckoDriverManager

class SessionSecuritySeleniumTestCase(StaticLiveServerTestCase):
    """Integration tests to check the session security package is
    working as expected.
    """

    WARN_AFTER_SECONDS = 5
    EXPIRE_AFTER_SECONDS = 10
    WAIT = 5
    TOLERANCE = 1
    TEST_LOGIN_CREDS = "test"
    TEST_SERVER = "localhost:5000"

    @classmethod
    def setUpClass(cls):
        cls.start_test_broswer()
        os.environ['DJANGO_LIVE_TEST_SERVER_ADDRESS'] = cls.TEST_SERVER
        super(StaticLiveServerTestCase, cls).setUpClass()

    @classmethod
    def tearDownClass(cls):
        cls.browser.quit()
        super(StaticLiveServerTestCase, cls).tearDownClass()

    @classmethod
    def start_test_broswer(cls):
        """Starts up a headless firefox broswer for testing.
        """
        options = webdriver.FirefoxOptions()
        options.add_argument('--headless')
        cls.browser = webdriver.Firefox(executable_path=GeckoDriverManager().install(), options=options)

    def setUp(self):
        super(SessionSecuritySeleniumTestCase, self).setUp()
        settings.SESSION_SECURITY_WARN_AFTER = self.WARN_AFTER_SECONDS
        settings.SESSION_SECURITY_EXPIRE_AFTER = self.EXPIRE_AFTER_SECONDS
        self.setup_admin_user()

    def test_inactivity_session_timeout_procedure(self):
        """Tests that the session secuirty procedure works correctly by warning the user
        after a period of inactivity, then closing the session if there is no activity within a
        further time period.
        """
        self.login()

        # Check the session warning message is hidden and then displayed after the
        # SESSION_SECURITY_WARN_AFTER period.
        self.check_session_warning_displayed()

        # Perform activity in the browser and confirm the warning message is removed.
        self.perform_keyboard_action()
        self.assert_warning_is_hidden()

        # Check the session warning message is hidden and then displayed for a second time
        # after the SESSION_SECURITY_WARN_AFTER period.
        self.check_session_warning_displayed()

        # Check that the session is logged out after the SESSION_SECURITY_EXPIRE_AFTER period
        # when no activity is detected.
        start = datetime.datetime.now()

        expected_time_to_log_out = self.EXPIRE_AFTER_SECONDS - self.WARN_AFTER_SECONDS
        self.assert_logged_out(expected_time_to_log_out + self.TOLERANCE)

        end = datetime.datetime.now()
        delta = end - start

        self.assertGreaterEqual(delta.seconds, expected_time_to_log_out - self.TOLERANCE)

    def test_session_ends_on_browser_close(self):
        """Tests that the session is ended when the broswer is closed.
        """
        # Login to the system and then close the broswer
        self.login()
        self.browser.close()

        # Confirm that on reopening the browser and admin page that the session is
        # not restored.
        self.start_test_broswer()
        self.navigate_to_admin_page()
        self.assert_logged_out(self.WAIT)

    def check_session_warning_displayed(self):
        """Checks that the session security modal goes from hidden to visible within the
        required timeframe.
        """
        start = datetime.datetime.now()

        self.assert_warning_is_hidden()
        self.assert_warning_element_visible(self.WARN_AFTER_SECONDS + self.TOLERANCE)

        end = datetime.datetime.now()
        delta = end - start

        self.assertGreaterEqual(delta.seconds, self.WARN_AFTER_SECONDS)
        self.assertLessEqual(delta.seconds, self.EXPIRE_AFTER_SECONDS)

    def setup_admin_user(self):
        """Creates a test user for the admin portal.
        """
        if not User.objects.filter(username=self.TEST_LOGIN_CREDS).exists():
            test_user = User.objects.create(username=self.TEST_LOGIN_CREDS)
            test_user.set_password(self.TEST_LOGIN_CREDS)
            test_user.is_staff = True
            test_user.save()

    def login(self):
        """Performs admin portal login and confirms the presence of the session cookie.
        """
        self.navigate_to_admin_page()
        self.browser.find_element("id", "id_username").send_keys(self.TEST_LOGIN_CREDS)
        self.browser.find_element("id", "id_password").send_keys(self.TEST_LOGIN_CREDS)
        self.browser.find_element_by_xpath('//input[@value="Log in"]').click()
        self.confirm_logged_in()

    def navigate_to_admin_page(self):
        """Navigates to the admin portal.
        """
        self.browser.get('%s%s' % (self.live_server_url, '/admin/'))

    def perform_keyboard_action(self):
        """Simulates a user pressing the space bar within the browser.
        """
        action = ActionChains(self.browser)
        action.key_down(Keys.SPACE)
        action.key_up(Keys.SPACE)
        action.perform()

    def confirm_logged_in(self):
        """Confirms the broswers is logged into the admin portal by check for the logout button
        and the session cookie.
        """
        try:
            WebDriverWait(self.browser, self.WAIT).until(
                expected_conditions.presence_of_element_located((By.LINK_TEXT, "Log out")))
        except Exception:
            self.fail("Login was unsuccessful")

        self.assertIsNotNone(self.get_session_cookie(), "Session cookie not present after logging in.")

    def assert_warning_element_visible(self, max_wait):
        """Checks that the session security modal is visible within a given time period.

        Args:
            max_wait (float): The number fo second to wait for the element to appear.
        """
        try:
            WebDriverWait(self.browser, max_wait).until(
                expected_conditions.visibility_of_element_located((By.ID, "session_security_warning")))
        except Exception:
            self.fail("Warning message not displayed after %ds or inactivity" % (max_wait))

    def assert_warning_is_hidden(self):
        """Checks that the session security modal is non visible.
        """
        try:
            WebDriverWait(self.browser, self.WAIT).until(
                expected_conditions.invisibility_of_element((By.ID, "session_security_warning")))
        except Exception:
            self.fail("Warning message present during active session")

    def assert_logged_out(self, delay):
        """Checks that the session has been logged out by checking for the presence of the
        login page and that the session cookie has been removed.

        Args:
            delay (float): The number of second to wait for the logout to occur.
        """
        try:
            WebDriverWait(self.browser, delay).until(
                expected_conditions.presence_of_element_located((By.ID, "id_password")))
        except Exception:
            self.fail("Sessions was not logged out after %ds" % (delay))

        self.assertIsNone(self.get_session_cookie(), "Session cookie present after logging out.")

    def get_session_cookie(self):
        """Gets the session cookie from the broswer.

        Returns:
            dict: The session cookie dictionary.
        """
        cookies = (cookie for cookie in self.browser.get_cookies() if cookie["name"] == "sessionid")
        return next(cookies, None)
<|endoftext|>


<commit_before>import datetime
from struct import unpack

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from mustaine.protocol import *

# Implementation of Hessian 1.0.2 deserialization
#   see: http://hessian.caucho.com/doc/hessian-1.0-spec.xtp

class ParseError(Exception):
    pass

class Parser(object):
    def parse_string(self, string):
        if isinstance(string, unicode):
            stream = StringIO(string.encode('utf-8'))
        else:
            stream = StringIO(string)

        return self.parse_stream(stream)

    def parse_stream(self, stream):
        self._refs   = []
        self._result = None

        if hasattr(stream, 'read') and hasattr(stream.read, '__call__'):
            self._stream = stream
        else:
            raise TypeError('Stream parser can only handle objects supporting read()')

        while True:
            code = self._read(1)

            if   code == 'c':
                if self._result:
                    raise ParseError('Encountered duplicate type header')

                version = self._read(2)
                if version != '\x01\x00':
                    raise ParseError("Encountered unrecognized call version {0!r}".format(version))

                self._result = Call()
                continue

            elif code == 'r':
                if self._result:
                    raise ParseError('Encountered duplicate type header')

                version = self._read(2)
                if version != '\x01\x00':
                    raise ParseError("Encountered unrecognized reply version {0!r}".format(version))

                self._result = Reply()
                continue

            else:
                if not self._result:
                    raise ParseError("Invalid Hessian message marker: {0!r}".format(code))

                if   code == 'H':
                    key, value = self._read_keyval()
                    self._result.headers[key] = value
                    continue

                elif code == 'm':
                    if not isinstance(self._result, Call):
                        raise ParseError('Encountered illegal method name within reply')

                    if self._result.method:
                        raise ParseError('Encountered duplicate method name definition')

                    self._result.method = self._read(unpack('>H', self._read(2))[0])
                    continue

                elif code == 'f':
                    if not isinstance(self._result, Reply):
                        raise ParseError('Encountered illegal fault within call')

                    if self._result.value:
                        raise ParseError('Encountered illegal extra object within reply')

                    self._result.value = self._read_fault()
                    continue

                elif code == 'z':
                    break

                else:
                    if isinstance(self._result, Call):
                        self._result.args.append(self._read_object(code))
                    else:
                        if self._result.value:
                            raise ParseError('Encountered illegal extra object within reply')

                        self._result.value = self._read_object(code)

        # have to hit a 'z' to land here, TODO derefs?
        return self._result

    def _read(self, n):
        try:
            r = self._stream.read(n)
        except IOError:
            raise ParseError('Encountered unexpected end of stream')
        except:
            raise
        else:
            if len(r) == 0:
                raise ParseError('Encountered unexpected end of stream')

        return r

    def _read_object(self, code):
        if   code == 'N':
            return None
        elif code == 'T':
            return True
        elif code == 'F':
            return False
        elif code == 'I':
            return int(unpack('>l', self._read(4))[0])
        elif code == 'L':
            return long(unpack('>q', self._read(8))[0])
        elif code == 'D':
            return float(unpack('>d', self._read(8))[0])
        elif code == 'd':
            return self._read_date()
        elif code == 's' or code == 'x':
            fragment = self._read_string()
            next     = self._read(1)
            if next.lower() == code:
                return fragment + self._read_object(next)
            else:
                raise ParseError("Expected terminal string segment, got {0!r}".format(next))
        elif code == 'S' or code == 'X':
            return self._read_string()
        elif code == 'b':
            fragment = self._read_binary()
            next     = self._read(1)
            if next.lower() == code:
                return fragment + self._read_object(next)
            else:
                raise ParseError("Expected terminal binary segment, got {0!r}".format(next))
        elif code == 'B':
            return self._read_binary()
        elif code == 'r':
            return self._read_remote()
        elif code == 'R':
            return self._refs[unpack(">L", self._read(4))[0]]
        elif code == 'V':
            return self._read_list()
        elif code == 'M':
            return self._read_map()
        else:
            raise ParseError("Unknown type marker {0!r}".format(code))

    def _read_date(self):
        timestamp = unpack('>q', self._read(8))[0]
        return datetime.datetime.fromtimestamp(timestamp / 1000)

    def _read_string(self):
        len = unpack('>H', self._read(2))[0]

        bytes = []
        while len > 0:
            byte = self._read(1)
            if ord(byte) in range(0x00, 0x7F):
                bytes.append(byte)
            elif ord(byte) in range(0xC2, 0xDF):
                bytes.append(byte + self._read(1))
            elif ord(byte) in range(0xE0, 0xEF):
                bytes.append(byte + self._read(2))
            elif ord(byte) in range(0xF0, 0xF4):
                bytes.append(byte + self._read(3))
            len -= 1

        return ''.join(bytes).decode('utf-8')

    def _read_binary(self):
        len = unpack('>H', self._read(2))[0]
        return Binary(self._read(len))

    def _read_remote(self):
        r    = Remote()
        code = self._read(1)

        if code == 't':
            r.type = self._read(unpack('>H', self._read(2))[0])
            code   = self._read(1)
        else:
            r.type = None

        if code != 's' and code != 'S':
            raise ParseError("Expected string object while parsing Remote object URL")

        r.url = self._read_object(code)
        return r

    def _read_list(self):
        code = self._read(1)

        if code == 't':
            # read and discard list type
            self._read(unpack('>H', self._read(2))[0])
            code = self._read(1)

        if code == 'l':
            # read and discard list length
            self._read(4)
            code = self._read(1)

        result = []
        self._refs.append(result)

        while code != 'z':
            result.append(self._read_object(code))
            code = self._read(1)

        return result

    def _read_map(self):
        code = self._read(1)

        if code == 't':
            # a typed map deserializes to an object
            result = Object(self._read(unpack('>H', self._read(2))[0]))
            code   = self._read(1)
        else:
            # untyped maps deserialize to a dict
            result = {}

        self._refs.append(result)

        fields = {}
        while code != 'z':
            key, value  = self._read_keyval(code)

            if isinstance(result, Object):
                fields[str(key)] = value
            else:
                fields[key] = value

            code = self._read(1)

        if isinstance(result, Object):
            fields['__meta_type'] = result._meta_type
            result.__setstate__(fields)
        else:
            result.update(fields)

        return result

    def _read_fault(self):
        fault = self._read_map()
        return Fault(fault['code'], fault['message'], fault.get('detail'))

    def _read_keyval(self, first=None):
        key   = self._read_object(first or self._read(1))
        value = self._read_object(self._read(1))

        return key, value

<commit_msg>fixed missing type in typed-map<commit_after>import datetime
from struct import unpack

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from mustaine.protocol import *

# Implementation of Hessian 1.0.2 deserialization
#   see: http://hessian.caucho.com/doc/hessian-1.0-spec.xtp

class ParseError(Exception):
    pass

class Parser(object):
    def parse_string(self, string):
        if isinstance(string, unicode):
            stream = StringIO(string.encode('utf-8'))
        else:
            stream = StringIO(string)

        return self.parse_stream(stream)

    def parse_stream(self, stream):
        self._refs   = []
        self._result = None

        if hasattr(stream, 'read') and hasattr(stream.read, '__call__'):
            self._stream = stream
        else:
            raise TypeError('Stream parser can only handle objects supporting read()')

        while True:
            code = self._read(1)

            if   code == 'c':
                if self._result:
                    raise ParseError('Encountered duplicate type header')

                version = self._read(2)
                if version != '\x01\x00':
                    raise ParseError("Encountered unrecognized call version {0!r}".format(version))

                self._result = Call()
                continue

            elif code == 'r':
                if self._result:
                    raise ParseError('Encountered duplicate type header')

                version = self._read(2)
                if version != '\x01\x00':
                    raise ParseError("Encountered unrecognized reply version {0!r}".format(version))

                self._result = Reply()
                continue

            else:
                if not self._result:
                    raise ParseError("Invalid Hessian message marker: {0!r}".format(code))

                if   code == 'H':
                    key, value = self._read_keyval()
                    self._result.headers[key] = value
                    continue

                elif code == 'm':
                    if not isinstance(self._result, Call):
                        raise ParseError('Encountered illegal method name within reply')

                    if self._result.method:
                        raise ParseError('Encountered duplicate method name definition')

                    self._result.method = self._read(unpack('>H', self._read(2))[0])
                    continue

                elif code == 'f':
                    if not isinstance(self._result, Reply):
                        raise ParseError('Encountered illegal fault within call')

                    if self._result.value:
                        raise ParseError('Encountered illegal extra object within reply')

                    self._result.value = self._read_fault()
                    continue

                elif code == 'z':
                    break

                else:
                    if isinstance(self._result, Call):
                        self._result.args.append(self._read_object(code))
                    else:
                        if self._result.value:
                            raise ParseError('Encountered illegal extra object within reply')

                        self._result.value = self._read_object(code)

        # have to hit a 'z' to land here, TODO derefs?
        return self._result

    def _read(self, n):
        try:
            r = self._stream.read(n)
        except IOError:
            raise ParseError('Encountered unexpected end of stream')
        except:
            raise
        else:
            if len(r) == 0:
                raise ParseError('Encountered unexpected end of stream')

        return r

    def _read_object(self, code):
        if   code == 'N':
            return None
        elif code == 'T':
            return True
        elif code == 'F':
            return False
        elif code == 'I':
            return int(unpack('>l', self._read(4))[0])
        elif code == 'L':
            return long(unpack('>q', self._read(8))[0])
        elif code == 'D':
            return float(unpack('>d', self._read(8))[0])
        elif code == 'd':
            return self._read_date()
        elif code == 's' or code == 'x':
            fragment = self._read_string()
            next     = self._read(1)
            if next.lower() == code:
                return fragment + self._read_object(next)
            else:
                raise ParseError("Expected terminal string segment, got {0!r}".format(next))
        elif code == 'S' or code == 'X':
            return self._read_string()
        elif code == 'b':
            fragment = self._read_binary()
            next     = self._read(1)
            if next.lower() == code:
                return fragment + self._read_object(next)
            else:
                raise ParseError("Expected terminal binary segment, got {0!r}".format(next))
        elif code == 'B':
            return self._read_binary()
        elif code == 'r':
            return self._read_remote()
        elif code == 'R':
            return self._refs[unpack(">L", self._read(4))[0]]
        elif code == 'V':
            return self._read_list()
        elif code == 'M':
            return self._read_map()
        else:
            raise ParseError("Unknown type marker {0!r}".format(code))

    def _read_date(self):
        timestamp = unpack('>q', self._read(8))[0]
        return datetime.datetime.fromtimestamp(timestamp / 1000)

    def _read_string(self):
        len = unpack('>H', self._read(2))[0]

        bytes = []
        while len > 0:
            byte = self._read(1)
            if ord(byte) in range(0x00, 0x7F):
                bytes.append(byte)
            elif ord(byte) in range(0xC2, 0xDF):
                bytes.append(byte + self._read(1))
            elif ord(byte) in range(0xE0, 0xEF):
                bytes.append(byte + self._read(2))
            elif ord(byte) in range(0xF0, 0xF4):
                bytes.append(byte + self._read(3))
            len -= 1

        return ''.join(bytes).decode('utf-8')

    def _read_binary(self):
        len = unpack('>H', self._read(2))[0]
        return Binary(self._read(len))

    def _read_remote(self):
        r    = Remote()
        code = self._read(1)

        if code == 't':
            r.type = self._read(unpack('>H', self._read(2))[0])
            code   = self._read(1)
        else:
            r.type = None

        if code != 's' and code != 'S':
            raise ParseError("Expected string object while parsing Remote object URL")

        r.url = self._read_object(code)
        return r

    def _read_list(self):
        code = self._read(1)

        if code == 't':
            # read and discard list type
            self._read(unpack('>H', self._read(2))[0])
            code = self._read(1)

        if code == 'l':
            # read and discard list length
            self._read(4)
            code = self._read(1)

        result = []
        self._refs.append(result)

        while code != 'z':
            result.append(self._read_object(code))
            code = self._read(1)

        return result

    def _read_map(self):
        code = self._read(1)

        if code == 't':
            type_len = unpack('>H', self._read(2))[0]
            if type_len > 0:
                # a typed map deserializes to an object
                result = Object(self._read(type_len))
            else:
                result = {}

            code = self._read(1)
        else:
            # untyped maps deserialize to a dict
            result = {}

        self._refs.append(result)

        fields = {}
        while code != 'z':
            key, value  = self._read_keyval(code)

            if isinstance(result, Object):
                fields[str(key)] = value
            else:
                fields[key] = value

            code = self._read(1)

        if isinstance(result, Object):
            fields['__meta_type'] = result._meta_type
            result.__setstate__(fields)
        else:
            result.update(fields)

        return result

    def _read_fault(self):
        fault = self._read_map()
        return Fault(fault['code'], fault['message'], fault.get('detail'))

    def _read_keyval(self, first=None):
        key   = self._read_object(first or self._read(1))
        value = self._read_object(self._read(1))

        return key, value

<|endoftext|>