<commit_before>import xmlrpclib, socket, time, subprocess, thread, getopt, sys, os, base64, signal, pwd, string, re
from select import select

# Options
global serverUrl, debug, verbose, sleepTime
serverUrl = ""
debug = False
verbose = False
sleepTime = 2
affinity = ""
name = socket.gethostname()

def usage():
	print ("Usage: worker [OPTIONS] SERVER_URL")
	print ("Start a Coalition worker using the server located at SERVER_URL.\n")
	print ("Options:")
	print ("  -d, --debug\t\tRun without the main try/catch")
	print ("  -h, --help\t\tShow this help")
	print ("  -n, --name=NAME\tWorker name (default: "+name+")")
	print ("  -s, --sleep=SLEEPTIME\tSleep time between two heart beats (default: "+str(sleepTime)+"s)")
	print ("  -a, --affinity=AFFINITY\tAffinity words to jobs (default: \"\"")
	print ("  -v, --verbose\t\tIncrease verbosity")
	print ("\nExample : worker -s 30 -v http://localhost:8080")

# Parse the options
try:
	opts, args = getopt.getopt(sys.argv[1:], "a:dhn:s:v", ["affinity=", "debug", "help", "name=", "sleep=", "verbose"])
	if len(args) != 1 :
		usage()
		sys.exit(2)
	serverUrl = args[0]
except getopt.GetoptError, err:
	# print help information and exit:
	print str(err) # will print something like "option -a not recognized"
	usage()
	sys.exit(2)
for o, a in opts:
	if o in ("-d", "--debug"):
		debug = True
	elif o in ("-h", "--help"):
		usage()
		sys.exit(2)
	elif o in ("-n", "--name"):
		name = a
	elif o in ("-a", "--affinity"):
		affinity = a
	elif o in ("-v", "--verbose"):
		verbose = True
	elif o in ("-s", "--sleep"):
		sleepTime = float(a)
	else:
		assert False, "unhandled option " + o

# Log for debugging
def debugOutput (str):
	if verbose:
		print (str)

# Log for debugging
def debugRaw (str):
	if verbose:
		print (str),

# Add to the logs
def info (str):
	global gLog, gLogLock
	gLogLock.acquire()
   	try:
		gLog = gLog + "WORKER: " + str + "\n";
		if verbose:
			print (str)
  	finally:
		gLogLock.release()

server = xmlrpclib.ServerProxy(serverUrl+"/workers")

global working, pid
working = False
pid = 0
global errorCode
errorCode = 0

# Lock for mutual exclusion on the logs
global gLogLock
gLogLock = thread.allocate_lock()

# Current log
global gLog
gLog = ""

#global lock
#lock = thread.allocate_lock ()

# Thread function to execute the job process
def _execProcess (cmd,dir,user):
	global errorCode, gLog, pid

	# Change the user ?
	if user != "":
		debugOutput ("Run the command using login " + user)
		#os.seteuid (pwd.getpwnam(user)[2])
		cmd = "su - " + user + " -c \"" + cmd + "\""

	# Set the working directory
	try:
		os.chdir (dir)
	except OSError:
		info ("Can't change dir to "+dir)

	# Run the job
	info ("exec " + cmd)

	process = subprocess.Popen (cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

	# Get the pid
	pid = int(process.pid)
	while (1):
		# Read some lines of logs
		line = process.stdout.readline()

		# "" means EOF
		if line == "":
			info ("end")
			break

		debugRaw (line)
		gLogLock.acquire()
   		try:
        		gLog = gLog + line
   		finally:
         		gLogLock.release()

	# Get the error code of the job
	errorCode = process.wait ()
	info ("Job returns code: " + str(errorCode))

def execProcess (cmd,dir,user):
	global working, errorCode
	if debug:
		_execProcess (cmd,dir,user)
	else:
		try:
			_execProcess (cmd,dir,user)
		except:
			errorCode = -1
			print ("Fatal error executing the job...")
			time.sleep (sleepTime)

	# Signal to the main process the job is finished
	working = False

### To kill all child process
def killr (pid, sig):
	if sys.platform!="win32":
		names=os.listdir("/proc/")
		for name in names:
			try:
				f = open("/proc/" + name +"/stat","r")
				line = f.readline()
				words =  string.split(line)
				if words[3]==str(pid):
					debugOutput("Found in " + name)
					killr(int(name), sig)
			except IOError:
				pass

	try:
		os.kill(pid,sig)
	except:
		pass

# Safe method to run a command on the server, if retry is true, the function won't return until the message is passed
def run (func, retry):
	while (True):
		try:
			return func ()
		except socket.error:
			pass
		if not retry:
			debugOutput ("Server down, continue...")
			break
		debugOutput ("No server")
		time.sleep (sleepTime)

# Flush the logs to the server
def heartbeat (jobId, retry):
	global gLog, pid, gLogLock
	debugOutput ("Flush logs (" + str(len(gLog)) + " bytes)")
	def func ():
		global gLog
		result = True

		gLogLock.acquire()
   		try:
        		result = server.heartbeat (name, jobId, base64.b64encode(gLog), getloadavg())
			gLog = ""
   		finally:
         		gLogLock.release()

		if not result:
			debugOutput ("Server ask to stop the jod " + str(jobId))

			# Send the kill signal to the process
			if pid != 0:
				debugOutput ("kill "+str(pid)+" "+str(signal.SIGKILL))
				try:
					killr (pid, signal.SIGKILL)
				except OSError:
					debugOutput ("kill failed")
					pass
	run (func, retry)

# LoadAvg
def getloadavg ():
	try:
		return os.getloadavg ()
	except:
		return -1

def evalEnv (_str):
	def _getenv (match):
		result = os.getenv (match.group(1))
		if result == None:
			info ("Environment variable not found : " + match.group(1))
			result = ""
		return result
	return re.sub ('\$\(([^)]*)\)', _getenv, _str)

# Application main loop
def mainLoop ():
	global working, errorCode, gLog, pid

	debugOutput ("Ask for a job")
	# Function to ask a job to the server
	def startFunc ():
		return server.pickjobwithaffinity (name, getloadavg(), affinity)

	# Block until this message to handled by the server
	jobId, cmd, dir, user = run (startFunc, True)

	if jobId != -1:
		debugOutput ("Start jod " + str(jobId) + " : " + cmd)

		# Reset the globals
		working = True
		stop = False
		pid = 0

		# Launch a new thread to run the process
		gLog = ""
		thread.start_new_thread ( execProcess, (evalEnv(cmd),evalEnv(dir),user))

		# Flush the logs
		while (working):
			heartbeat (jobId, False)
			time.sleep (sleepTime)

		# Flush for real for the last time
		heartbeat (jobId, True)

		debugOutput ("Finished jod " + str(jobId) + " (code " + str(errorCode) + ") : " + cmd)

		# Function to end the job
		def endFunc ():
			server.endjob (name, jobId, errorCode)

		# Block until this message to handled by the server
		run (endFunc, True)

	time.sleep (sleepTime)

peuid = os.geteuid ()
while (1):
	os.seteuid (peuid)
	if debug:
		mainLoop ()
	else:
		try:
			mainLoop ()
		except:
			print ("Fatal error, retry...")
			time.sleep (sleepTime)

<commit_msg>Fix : change dir works with ldap <commit_after>import xmlrpclib, socket, time, subprocess, thread, getopt, sys, os, base64, signal, pwd, string, re
from select import select

# Options
global serverUrl, debug, verbose, sleepTime
serverUrl = ""
debug = False
verbose = False
sleepTime = 2
affinity = ""
name = socket.gethostname()

def usage():
	print ("Usage: worker [OPTIONS] SERVER_URL")
	print ("Start a Coalition worker using the server located at SERVER_URL.\n")
	print ("Options:")
	print ("  -d, --debug\t\tRun without the main try/catch")
	print ("  -h, --help\t\tShow this help")
	print ("  -n, --name=NAME\tWorker name (default: "+name+")")
	print ("  -s, --sleep=SLEEPTIME\tSleep time between two heart beats (default: "+str(sleepTime)+"s)")
	print ("  -a, --affinity=AFFINITY\tAffinity words to jobs (default: \"\"")
	print ("  -v, --verbose\t\tIncrease verbosity")
	print ("\nExample : worker -s 30 -v http://localhost:8080")

# Parse the options
try:
	opts, args = getopt.getopt(sys.argv[1:], "a:dhn:s:v", ["affinity=", "debug", "help", "name=", "sleep=", "verbose"])
	if len(args) != 1 :
		usage()
		sys.exit(2)
	serverUrl = args[0]
except getopt.GetoptError, err:
	# print help information and exit:
	print str(err) # will print something like "option -a not recognized"
	usage()
	sys.exit(2)
for o, a in opts:
	if o in ("-d", "--debug"):
		debug = True
	elif o in ("-h", "--help"):
		usage()
		sys.exit(2)
	elif o in ("-n", "--name"):
		name = a
	elif o in ("-a", "--affinity"):
		affinity = a
	elif o in ("-v", "--verbose"):
		verbose = True
	elif o in ("-s", "--sleep"):
		sleepTime = float(a)
	else:
		assert False, "unhandled option " + o

# Log for debugging
def debugOutput (str):
	if verbose:
		print (str)

# Log for debugging
def debugRaw (str):
	if verbose:
		print (str),

# Add to the logs
def info (str):
	global gLog, gLogLock
	gLogLock.acquire()
   	try:
		gLog = gLog + "WORKER: " + str + "\n";
		if verbose:
			print (str)
  	finally:
		gLogLock.release()

server = xmlrpclib.ServerProxy(serverUrl+"/workers")

global working, pid
working = False
pid = 0
global errorCode
errorCode = 0

# Lock for mutual exclusion on the logs
global gLogLock
gLogLock = thread.allocate_lock()

# Current log
global gLog
gLog = ""

#global lock
#lock = thread.allocate_lock ()

# Thread function to execute the job process
def _execProcess (cmd,dir,user):
	global errorCode, gLog, pid

	# Change the user ?
	if user != "":
		debugOutput ("Run the command using login " + user)
		#os.seteuid (pwd.getpwnam(user)[2])
		cmd = "su - " + user + " -c \"" + "cd "+ dir + "; " +cmd + "\""
	else:
		# Set the working directory
		try:
			os.chdir (dir)
		except OSError:
			info ("Can't change dir to "+dir)

	# Run the job
	info ("exec " + cmd)

	process = subprocess.Popen (cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

	# Get the pid
	pid = int(process.pid)
	while (1):
		# Read some lines of logs
		line = process.stdout.readline()

		# "" means EOF
		if line == "":
			info ("end")
			break

		debugRaw (line)
		gLogLock.acquire()
   		try:
        		gLog = gLog + line
   		finally:
         		gLogLock.release()

	# Get the error code of the job
	errorCode = process.wait ()
	info ("Job returns code: " + str(errorCode))

def execProcess (cmd,dir,user):
	global working, errorCode
	if debug:
		_execProcess (cmd,dir,user)
	else:
		try:
			_execProcess (cmd,dir,user)
		except:
			errorCode = -1
			print ("Fatal error executing the job...")
			time.sleep (sleepTime)

	# Signal to the main process the job is finished
	working = False

### To kill all child process
def killr (pid, sig):
	if sys.platform!="win32":
		names=os.listdir("/proc/")
		for name in names:
			try:
				f = open("/proc/" + name +"/stat","r")
				line = f.readline()
				words =  string.split(line)
				if words[3]==str(pid):
					debugOutput("Found in " + name)
					killr(int(name), sig)
			except IOError:
				pass

	try:
		os.kill(pid,sig)
	except:
		pass

# Safe method to run a command on the server, if retry is true, the function won't return until the message is passed
def run (func, retry):
	while (True):
		try:
			return func ()
		except socket.error:
			pass
		if not retry:
			debugOutput ("Server down, continue...")
			break
		debugOutput ("No server")
		time.sleep (sleepTime)

# Flush the logs to the server
def heartbeat (jobId, retry):
	global gLog, pid, gLogLock
	debugOutput ("Flush logs (" + str(len(gLog)) + " bytes)")
	def func ():
		global gLog
		result = True

		gLogLock.acquire()
   		try:
        		result = server.heartbeat (name, jobId, base64.b64encode(gLog), getloadavg())
			gLog = ""
   		finally:
         		gLogLock.release()

		if not result:
			debugOutput ("Server ask to stop the jod " + str(jobId))

			# Send the kill signal to the process
			if pid != 0:
				debugOutput ("kill "+str(pid)+" "+str(signal.SIGKILL))
				try:
					killr (pid, signal.SIGKILL)
				except OSError:
					debugOutput ("kill failed")
					pass
	run (func, retry)

# LoadAvg
def getloadavg ():
	try:
		return os.getloadavg ()
	except:
		return -1

def evalEnv (_str):
	def _getenv (match):
		result = os.getenv (match.group(1))
		if result == None:
			info ("Environment variable not found : " + match.group(1))
			result = ""
		return result
	return re.sub ('\$\(([^)]*)\)', _getenv, _str)

# Application main loop
def mainLoop ():
	global working, errorCode, gLog, pid

	debugOutput ("Ask for a job")
	# Function to ask a job to the server
	def startFunc ():
		return server.pickjobwithaffinity (name, getloadavg(), affinity)

	# Block until this message to handled by the server
	jobId, cmd, dir, user = run (startFunc, True)

	if jobId != -1:
		debugOutput ("Start jod " + str(jobId) + " : " + cmd)

		# Reset the globals
		working = True
		stop = False
		pid = 0

		# Launch a new thread to run the process
		gLog = ""
		thread.start_new_thread ( execProcess, (evalEnv(cmd),evalEnv(dir),user))

		# Flush the logs
		while (working):
			heartbeat (jobId, False)
			time.sleep (sleepTime)

		# Flush for real for the last time
		heartbeat (jobId, True)

		debugOutput ("Finished jod " + str(jobId) + " (code " + str(errorCode) + ") : " + cmd)

		# Function to end the job
		def endFunc ():
			server.endjob (name, jobId, errorCode)

		# Block until this message to handled by the server
		run (endFunc, True)

	time.sleep (sleepTime)

peuid = os.geteuid ()
while (1):
	os.seteuid (peuid)
	if debug:
		mainLoop ()
	else:
		try:
			mainLoop ()
		except:
			print ("Fatal error, retry...")
			time.sleep (sleepTime)

<|endoftext|>


<commit_before># Copyright (c) 2017 Huawei Technologies Co., Ltd.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_policy import policy

from cinder.policies import base

MANAGE_POLICY = "volume_extension:types_manage"
ENCRYPTION_POLICY = "volume_extension:volume_type_encryption"
ENCRYPTION_BASE_POLICY_RULE = 'rule:%s' % ENCRYPTION_POLICY
CREATE_ENCRYPTION_POLICY = "volume_extension:volume_type_encryption:create"
GET_ENCRYPTION_POLICY = "volume_extension:volume_type_encryption:get"
UPDATE_ENCRYPTION_POLICY = "volume_extension:volume_type_encryption:update"
DELETE_ENCRYPTION_POLICY = "volume_extension:volume_type_encryption:delete"
QOS_POLICY = "volume_extension:access_types_qos_specs_id"
EXTRA_SPEC_POLICY = "volume_extension:access_types_extra_specs"
GET_POLICY = "volume_extension:type_get"
GET_ALL_POLICY = "volume_extension:type_get_all"

volume_type_policies = [
    policy.DocumentedRuleDefault(
        name=MANAGE_POLICY,
        check_str=base.RULE_ADMIN_API,
        description="Create, update and delete volume type.",
        operations=[
            {
                'method': 'POST',
                'path': '/types'
            },
            {
                'method': 'PUT',
                'path': '/types'
            },
            {
                'method': 'DELETE',
                'path': '/types'
            }
        ]),
    policy.DocumentedRuleDefault(
        name=GET_POLICY,
        check_str="",
        description="Get one specific volume type.",
        operations=[
            {
                'method': 'GET',
                'path': '/types/{type_id}'
            }
        ]),
    policy.DocumentedRuleDefault(
        name=GET_ALL_POLICY,
        check_str="",
        description="List volume types.",
        operations=[
            {
                'method': 'GET',
                'path': '/types/'
            }
        ]),
    policy.DocumentedRuleDefault(
        name=ENCRYPTION_POLICY,
        check_str=base.RULE_ADMIN_API,
        description="Base policy for all volume type encryption type "
                    "operations.  This can be used to set the policies "
                    "for a volume type's encryption type create, show, "
                    "update, and delete actions in one place, or any of "
                    "those may be set individually using the following "
                    "policy targets for finer grained control.",
        operations=[
            {
                'method': 'POST',
                'path': '/types/{type_id}/encryption'
            },
            {
                'method': 'PUT',
                'path': '/types/{type_id}/encryption/{encryption_id}'
            },
            {
                'method': 'GET',
                'path': '/types/{type_id}/encryption'
            },
            {
                'method': 'GET',
                'path': '/types/{type_id}/encryption/{encryption_id}'
            },
            {
                'method': 'DELETE',
                'path': '/types/{type_id}/encryption/{encryption_id}'
            }
        ]),
    policy.DocumentedRuleDefault(
        name=CREATE_ENCRYPTION_POLICY,
        check_str=ENCRYPTION_BASE_POLICY_RULE,
        description="Create volume type encryption.",
        operations=[
            {
                'method': 'POST',
                'path': '/types/{type_id}/encryption'
            }
        ]),
    policy.DocumentedRuleDefault(
        name=GET_ENCRYPTION_POLICY,
        check_str=ENCRYPTION_BASE_POLICY_RULE,
        description="Show, list volume type encryption.",
        operations=[
            {
                'method': 'GET',
                'path': '/types/{type_id}/encryption/{encryption_id}'
            },
            {
                'method': 'GET',
                'path': '/types/{type_id}/encryption'
            }
        ]),
    policy.DocumentedRuleDefault(
        name=UPDATE_ENCRYPTION_POLICY,
        check_str=ENCRYPTION_BASE_POLICY_RULE,
        description="Update volume type encryption.",
        operations=[
            {
                'method': 'PUT',
                'path': '/types/{type_id}/encryption/{encryption_id}'
            }
        ]),
    policy.DocumentedRuleDefault(
        name=DELETE_ENCRYPTION_POLICY,
        check_str=ENCRYPTION_BASE_POLICY_RULE,
        description="Delete volume type encryption.",
        operations=[
            {
                'method': 'DELETE',
                'path': '/types/{type_id}/encryption/{encryption_id}'
            }
        ]),
    policy.DocumentedRuleDefault(
        name=EXTRA_SPEC_POLICY,
        check_str=base.RULE_ADMIN_API,
        description="List or show volume type with access type extra "
                    "specs attribute.",
        operations=[
            {
                'method': 'GET',
                'path': '/types/{type_id}'
            },
            {
                'method': 'GET',
                'path': '/types'
            }
        ]),
    policy.DocumentedRuleDefault(
        name=QOS_POLICY,
        check_str=base.RULE_ADMIN_API,
        description="List or show volume type with access type qos specs "
                    "id attribute.",
        operations=[
            {
                'method': 'GET',
                'path': '/types/{type_id}'
            },
            {
                'method': 'GET',
                'path': '/types'
            }
        ]),
]

def list_rules():
    return volume_type_policies
<commit_msg>Correct description for encryption-type policies<commit_after># Copyright (c) 2017 Huawei Technologies Co., Ltd.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_policy import policy

from cinder.policies import base

MANAGE_POLICY = "volume_extension:types_manage"
ENCRYPTION_POLICY = "volume_extension:volume_type_encryption"
ENCRYPTION_BASE_POLICY_RULE = 'rule:%s' % ENCRYPTION_POLICY
CREATE_ENCRYPTION_POLICY = "volume_extension:volume_type_encryption:create"
GET_ENCRYPTION_POLICY = "volume_extension:volume_type_encryption:get"
UPDATE_ENCRYPTION_POLICY = "volume_extension:volume_type_encryption:update"
DELETE_ENCRYPTION_POLICY = "volume_extension:volume_type_encryption:delete"
QOS_POLICY = "volume_extension:access_types_qos_specs_id"
EXTRA_SPEC_POLICY = "volume_extension:access_types_extra_specs"
GET_POLICY = "volume_extension:type_get"
GET_ALL_POLICY = "volume_extension:type_get_all"

volume_type_policies = [
    policy.DocumentedRuleDefault(
        name=MANAGE_POLICY,
        check_str=base.RULE_ADMIN_API,
        description="Create, update and delete volume type.",
        operations=[
            {
                'method': 'POST',
                'path': '/types'
            },
            {
                'method': 'PUT',
                'path': '/types'
            },
            {
                'method': 'DELETE',
                'path': '/types'
            }
        ]),
    policy.DocumentedRuleDefault(
        name=GET_POLICY,
        check_str="",
        description="Get one specific volume type.",
        operations=[
            {
                'method': 'GET',
                'path': '/types/{type_id}'
            }
        ]),
    policy.DocumentedRuleDefault(
        name=GET_ALL_POLICY,
        check_str="",
        description="List volume types.",
        operations=[
            {
                'method': 'GET',
                'path': '/types/'
            }
        ]),
    policy.DocumentedRuleDefault(
        name=ENCRYPTION_POLICY,
        check_str=base.RULE_ADMIN_API,
        description="Base policy for all volume type encryption type "
                    "operations.  This can be used to set the policies "
                    "for a volume type's encryption type create, show, "
                    "update, and delete actions in one place, or any of "
                    "those may be set individually using the following "
                    "policy targets for finer grained control.",
        operations=[
            {
                'method': 'POST',
                'path': '/types/{type_id}/encryption'
            },
            {
                'method': 'PUT',
                'path': '/types/{type_id}/encryption/{encryption_id}'
            },
            {
                'method': 'GET',
                'path': '/types/{type_id}/encryption'
            },
            {
                'method': 'GET',
                'path': '/types/{type_id}/encryption/{key}'
            },
            {
                'method': 'DELETE',
                'path': '/types/{type_id}/encryption/{encryption_id}'
            }
        ]),
    policy.DocumentedRuleDefault(
        name=CREATE_ENCRYPTION_POLICY,
        check_str=ENCRYPTION_BASE_POLICY_RULE,
        description="Create volume type encryption.",
        operations=[
            {
                'method': 'POST',
                'path': '/types/{type_id}/encryption'
            }
        ]),
    policy.DocumentedRuleDefault(
        name=GET_ENCRYPTION_POLICY,
        check_str=ENCRYPTION_BASE_POLICY_RULE,
        description="Show a volume type's encryption type, "
                    "show an encryption specs item.",
        operations=[
            {
                'method': 'GET',
                'path': '/types/{type_id}/encryption'
            },
            {
                'method': 'GET',
                'path': '/types/{type_id}/encryption/{key}'
            }
        ]),
    policy.DocumentedRuleDefault(
        name=UPDATE_ENCRYPTION_POLICY,
        check_str=ENCRYPTION_BASE_POLICY_RULE,
        description="Update volume type encryption.",
        operations=[
            {
                'method': 'PUT',
                'path': '/types/{type_id}/encryption/{encryption_id}'
            }
        ]),
    policy.DocumentedRuleDefault(
        name=DELETE_ENCRYPTION_POLICY,
        check_str=ENCRYPTION_BASE_POLICY_RULE,
        description="Delete volume type encryption.",
        operations=[
            {
                'method': 'DELETE',
                'path': '/types/{type_id}/encryption/{encryption_id}'
            }
        ]),
    policy.DocumentedRuleDefault(
        name=EXTRA_SPEC_POLICY,
        check_str=base.RULE_ADMIN_API,
        description="List or show volume type with access type extra "
                    "specs attribute.",
        operations=[
            {
                'method': 'GET',
                'path': '/types/{type_id}'
            },
            {
                'method': 'GET',
                'path': '/types'
            }
        ]),
    policy.DocumentedRuleDefault(
        name=QOS_POLICY,
        check_str=base.RULE_ADMIN_API,
        description="List or show volume type with access type qos specs "
                    "id attribute.",
        operations=[
            {
                'method': 'GET',
                'path': '/types/{type_id}'
            },
            {
                'method': 'GET',
                'path': '/types'
            }
        ]),
]

def list_rules():
    return volume_type_policies
<|endoftext|>


<commit_before>'''
Crstr(e)atstr(e)d on 22/11/2015
@author:
'''

from exceptions import IOError
import re
import os as _os
import sys
from result import Result
from status import TBTAFParsingScriptStatus
from parsing_summary import ParsingSummary
from common.enums.filter_type import TBTAFFilterType
from orchestrator.TBTAFOrchestrator import TBTAFOrchestrator

class TBTAFInterpreter(object):
    '''
        TBTAFInterpreter
    '''

    OrchestratorReference = None
    summary = ParsingSummary()
    urlPattern = urlPattern = "(https?:\/\/)?([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w\.-]*)*\/?"

    ##Params in commands
    TEST_SUITE_PARAM = "testSuite"
    FILE_PATH_PARAM = "filePath"
    TEST_BED_PARAM = "testBed"
    PROJECT_NAME_PARAM = "projectName"
    TAG_LIST_PARAM = "tagList"
    FILTER_PARAM = "filter"
    FORMAT_PARAM = "format"
    URL_LIST_PARAM = "urlList"

    ##Available commands
    CREATE_TEST_BED = "create_test_bed"
    CREATE_TEST_SUITE = "create_test_suite"
    CREATE_NEW_PROJECT = "create_new_project"
    PUBLIS_TEST_PLAN = "publish test_plan"
    EXECUTE = "execute"
    PUBLISH_TEST_RESULT = "publish test_results"
    GET_TEST = "get tests"
    GET_TAGS = "get tags"

    ##Patterns
    TEST_SUITE = "(?P<"+ TEST_SUITE_PARAM +">(\w+))"
    FILE_PATH = "(?P<"+ FILE_PATH_PARAM +">(.*))"
    TEST_BED = "(?P<"+ TEST_BED_PARAM +">(\w+))"
    PROJECT_NAME = "\"(?P<"+ PROJECT_NAME_PARAM +">(.*))\""
    TAG_LIST = "\[(?P<"+ TAG_LIST_PARAM +">((<\w+>)(,<\w+>)*))\]"
    FILTER = "(?P<"+ FILTER_PARAM +">(\w+))"
    FORMAT = "(?P<"+ FORMAT_PARAM +">(\w+))"
    URL_LIST = "(?P<"+ URL_LIST_PARAM +">(("+ urlPattern +")(,"+ urlPattern +")*))"

    ##Message Types
    MSG_ERROR = "Error"
    MSG_WARNING = "Warning"

    FILE_NAME = "fileName"
    FILE_LINE_NUMBER = "lineNumber"

    ##Patterns
    a = "(?P<variable>\w+)\s*=\s*(?P<method>\\create_test_bed\\b)(\("+ URL_LIST +"\))?"
    b = "(?P<variable>\w+)\s*=\s*(?P<method>\\create_test_suite\\b)\(\s*"+ FILE_PATH +"(\,"+ TAG_LIST +")?\)"
    c = "(?P<method>\\create_new_project\\b)\("+ TEST_SUITE +","+ TEST_BED +","+ PROJECT_NAME +"\)"
    d = "(?P<method>\\publish test_plan\\b)\("+ TEST_SUITE +","+ FILE_PATH +","+ FORMAT +"\)"
    e = "(?P<variable>\w+)\s*=\s*(?P<method>\\execute\\b)\("+ TEST_SUITE +","+ TEST_BED +"\)"
    f = "(?P<method>\\publish test_results\\b)\("+ TEST_SUITE +","+ FILE_PATH +","+ FORMAT +"\)"
    g = "(?P<method>\\get tests\\b)\("+ PROJECT_NAME +"((\,"+ TAG_LIST +"),"+ FILTER +")?\)"
    h = "(?P<method>\\get tags\\b)\("+ PROJECT_NAME +"\)"

    def __init__(self):
        '''
            Constructor
        '''

    def _formatMsg(self,fileName, lineNumber, message, _type):
        #message = "{} :     File \"{}}\",line {}}\nDescription: {}}\n"
        return message.format(_type, str(fileName), str(lineNumber), message)
        ##return message.format("","","","")

    def parseScript(self, scriptURL):
        result = Result(TBTAFParsingScriptStatus.SUCCESS, "Success")
        file = None
        try:
            file = self._openFile(scriptURL)
            if (self._parseFile(file, result) == -1):
                file.close()
                print(result.status)
                print(result.message)
                return result
            else:
                if((result.status==TBTAFParsingScriptStatus.SUCCESS) and not(TBTAFInterpreter.OrchestratorReference is None)):
                    self.startExecution(TBTAFInterpreter.OrchestratorReference)
                else:
                    raise ValueError("Reference not set")
        except Exception as e:
            raise InvalidArgumentException(e)
        finally:
            if(file is not None):
                file.close()

        return result

    def _openFile(self,file):
        """
            This method returns object "_file"
        """
        ##________Verify is File exists________
        if (_os.path.isfile(file)):
            pass
        else:
            raise ValueError("File does not exist.")
        ##________Open File and return result________
        try:
            _file = open(file,"r")
            return _file
        except IOError:
            raise ValueError("Cannot open file.")

    ##----------------------------------------------
    ## Main method that will parse tbtaf script    -
    ##----------------------------------------------
    def _parseFile(self,file,result):
        """
        """
        ##________Dict for calling each regular expression________
        mappingPatterns   = {TBTAFInterpreter.CREATE_TEST_BED  :TBTAFInterpreter.a,TBTAFInterpreter.CREATE_TEST_SUITE:TBTAFInterpreter.b ,TBTAFInterpreter.CREATE_NEW_PROJECT  :TBTAFInterpreter.c,
                      TBTAFInterpreter.PUBLIS_TEST_PLAN:TBTAFInterpreter.d,TBTAFInterpreter.EXECUTE          :TBTAFInterpreter.e ,TBTAFInterpreter.PUBLISH_TEST_RESULT:TBTAFInterpreter.f,
                      TBTAFInterpreter.GET_TEST        :TBTAFInterpreter.g,TBTAFInterpreter.GET_TAGS         :TBTAFInterpreter.h}

        ##________Read each line of the file and look for a match of defined methods________
        for i, line in enumerate(file):
            ##________Delete White Spaces from the corners________
            line = line.strip()
            if (line == ""):
                pass
            elif (line.startswith("//")):
                pass

            for command in mappingPatterns:
                if (command in line):
                    m = re.match(mappingPatterns[command],line)
                    fileName = file.name
                    lineNumber = i + 1

                    if (m is None):
                        result.status  = TBTAFParsingScriptStatus.ERROR
                        result.message = self._formatMsg(fileName, lineNumber, "Cannot find command", TBTAFInterpreter.MSG_ERROR)
                    else:
                        m = m.groupdict()
                        ##print("Matched method = " + str(method))
                        ##print(str(m))
                        error = self._addToSummary(m, fileName, lineNumber)
                        if(error != ""):
                            result.status  = TBTAFParsingScriptStatus.ERROR
                            result.message = error
                            return -1

    def _addToSummary(self,m, fileName, lineNumber):
        m[TBTAFInterpreter.FILE_NAME] = fileName
        m[TBTAFInterpreter.FILE_LINE_NUMBER] = lineNumber

        error = '{} is not defined.'
        command = m["method"]

        if (command == TBTAFInterpreter.CREATE_TEST_BED):
            TBTAFInterpreter.summary.createTestBed[m["variable"]] = m

        elif (command == TBTAFInterpreter.CREATE_TEST_SUITE):
            TBTAFInterpreter.summary.createTestSuite[m["variable"]] = m

        elif (command == TBTAFInterpreter.CREATE_NEW_PROJECT):
            if not(m[TBTAFInterpreter.TEST_SUITE_PARAM] in TBTAFInterpreter.summary.createTestSuite):
                error = error.format(m[TBTAFInterpreter.TEST_SUITE_PARAM])
                return formatMsg(fileName, index, error, TBTAFInterpreter.MSG_ERROR)
            if not(m[TBTAFInterpreter.TEST_BED_PARAM] in TBTAFInterpreter.summary.createTestBed):
                error = error.format(m[TBTAFInterpreter.TEST_BED_PARAM])
                return formatMsg(fileName, index, error, TBTAFInterpreter.MSG_ERROR)

            TBTAFInterpreter.summary.createNewProject[m[TBTAFInterpreter.PROJECT_NAME_PARAM]] = m

        elif (command == TBTAFInterpreter.PUBLIS_TEST_PLAN):
            if not(m[TBTAFInterpreter.TEST_SUITE_PARAM] in TBTAFInterpreter.summary.createTestSuite):
                error = error.format(m[TBTAFInterpreter.TEST_SUITE_PARAM])
                return formatMsg(fileName, index, error, TBTAFInterpreter.MSG_ERROR)

            TBTAFInterpreter.summary.publishTestPlan.append(m)

        elif (command == TBTAFInterpreter.EXECUTE):
            if not(m[TBTAFInterpreter.TEST_SUITE_PARAM] in TBTAFInterpreter.summary.createTestSuite):
                error = error.format(m[TBTAFInterpreter.TEST_SUITE_PARAM])
                return formatMsg(fileName, index, error, TBTAFInterpreter.MSG_ERROR)

            TBTAFInterpreter.summary.execute[m["variable"]] = m

        elif (command == TBTAFInterpreter.PUBLISH_TEST_RESULT):
            if not(m[TBTAFInterpreter.TEST_SUITE_PARAM] in TBTAFInterpreter.summary.createTestSuite):
                error = error.format(m[TBTAFInterpreter.TEST_SUITE_PARAM])
                return formatMsg(fileName, index, error, TBTAFInterpreter.MSG_ERROR)

            TBTAFInterpreter.summary.publishTestResults.append(m)

        elif (command == TBTAFInterpreter.GET_TEST):
            if not(m[TBTAFInterpreter.PROJECT_NAME_PARAM] in TBTAFInterpreter.summary.createNewProject):
                error = error.format(m[TBTAFInterpreter.PROJECT_NAME_PARAM])
                return formatMsg(fileName, index, error, TBTAFInterpreter.MSG_ERROR)

            TBTAFInterpreter.summary.getTest.append(m)
        elif (command ==TBTAFInterpreter.GET_TAGS):
            if not(m[TBTAFInterpreter.PROJECT_NAME_PARAM] in TBTAFInterpreter.summary.createNewProject):
                error = error.format(m[TBTAFInterpreter.PROJECT_NAME_PARAM])
                return formatMsg(fileName, index, error, TBTAFInterpreter.MSG_ERROR)

            TBTAFInterpreter.summary.getTags.append(m)
        error = ""
        return error

    def setOrchestratorReference(self, orchestrator):
        TBTAFInterpreter.OrchestratorReference=orchestrator

    def startExecution(self,orchestrator):
        objs = {}
        fileName = ""
        lineNumber = ""

        try:
            #CreateTestBed
            testBeds = TBTAFInterpreter.summary.createTestBed
            for var in testBeds.keys():
                fileName = testBeds[var][TBTAFInterpreter.FILE_NAME]
                lineNumber = testBeds[var][TBTAFInterpreter.FILE_LINE_NUMBER]

                urlList = testBeds[var][TBTAFInterpreter.URL_LIST_PARAM]

                if(urlList is None):
                    objs[var] = orchestrator.createTestBed()
                else:
                    objs[var] = orchestrator.createTestBed(urlList)

            #CreateTestSuite
            testSuites = TBTAFInterpreter.summary.createTestSuite
            for var in testSuites.keys():
                fileName = testSuites[var][TBTAFInterpreter.FILE_NAME]
                lineNumber = testSuites[var][TBTAFInterpreter.FILE_LINE_NUMBER]

                filePath = testSuites[var][TBTAFInterpreter.FILE_PATH_PARAM]
                tagList = testSuites[var][TBTAFInterpreter.TAG_LIST_PARAM]
                if not(tagList is None):
                    tagList = tagList.split(",")

                objs[var] = orchestrator.createTestSuite(filePath, tagList)

            #CreateNewProject
            projects = TBTAFInterpreter.summary.createNewProject
            for var in projects.keys():
                fileName = projects[var][TBTAFInterpreter.FILE_NAME]
                lineNumber = projects[var][TBTAFInterpreter.FILE_LINE_NUMBER]

                testSuite = objs[projects[var][TBTAFInterpreter.TEST_SUITE_PARAM]]
                testBed = objs[projects[var][TBTAFInterpreter.TEST_BED_PARAM]]
                projectName = projects[var][TBTAFInterpreter.PROJECT_NAME_PARAM]

                orchestrator.createNewProject(testSuite, testBed, projectName)

        #except (Exception) as e:
        except (ValueError, IllegalArgumentException, NonSupportedFormatException) as e:
            print("hola")
            print self._formatMsg(fileName, lineNumber, "Fatal Error. Execution cannot continue. " + str(e), TBTAFInterpreter.MSG_ERROR)

        #PublishTestPlan
        try:
            for var in TBTAFInterpreter.summary.publishTestPlan:
                fileName = var[TBTAFInterpreter.FILE_NAME]
                lineNumber = var[TBTAFInterpreter.FILE_LINE_NUMBER]

                testSuite = var[TBTAFInterpreter.TEST_SUITE_PARAM]
                filePath = var[TBTAFInterpreter.FILE_PATH_PARAM]
                format = var[TBTAFInterpreter.FORMAT_PARAM]

                orchestrator.PublishTestPlan(testSuite, filePath, format)

        #except (ValueError) as e:
        except (ValueError, IllegalArgumentException, NonSupportedFormatException) as e:
            print self._formatMsg(fileName, lineNumber, str(e), TBTAFInterpreter.MSG_WARNING)

        #PublishResultReport
        try:
            for var in TBTAFInterpreter.summary.publishTestResults:
                fileName = var[TBTAFInterpreter.FILE_NAME]
                lineNumber = var[TBTAFInterpreter.FILE_LINE_NUMBER]

                testSuite = var[TBTAFInterpreter.TEST_SUITE_PARAM]
                filePath = var[TBTAFInterpreter.FILE_PATH_PARAM]
                format = var[TBTAFInterpreter.FORMAT_PARAM]

                orchestrator.PublishResultReport(testSuite, filePath, format)
        #except (ValueError) as e:
        except (ValueError, IllegalArgumentException, NonSupportedFormatException) as e:
            print self._formatMsg(fileName, lineNumber, str(e), TBTAFInterpreter.MSG_WARNING)

        #ExecuteTests
        try:
            #TBTAFInterpreter.summary.execute = executions
            executions= TBTAFInterpreter.summary.execute
            for var in executions.keys():
                fileName = executions[var][TBTAFInterpreter.FILE_NAME]
                lineNumber = executions[var][TBTAFInterpreter.FILE_LINE_NUMBER]

                testSuite = objs[executions[var][TBTAFInterpreter.TEST_SUITE_PARAM]]
                testBed = objs[executions[var][TBTAFInterpreter.TEST_SUITE_PARAM]]

                objs[var] = orchestrator.executeTests(testSuite, testBed)
        except ValueError as e:
            print self._formatMsg(fileName, lineNumber, str(e), TBTAFInterpreter.MSG_WARNING)

        #GetTests
        try:
            for var in TBTAFInterpreter.summary.getTest:
                fileName = var[TBTAFInterpreter.FILE_NAME]
                lineNumber = var[TBTAFInterpreter.FILE_LINE_NUMBER]

                projectName = var[TBTAFInterpreter.PROJECT_NAME_PARAM]
                tagList = var[TBTAFInterpreter.TAG_LIST_PARAM]
                if not(tagList is None):
                    tagList = tagList.split(",")
                filter = var[TBTAFInterpreter.FILTER_PARAM]
                if(filter == "IN"):
                    filter = TBTAFFilterType.IN
                elif(filter == "OUT"):
                    filter = TBTAFFilterType.OUT
                else:
                    filter = None

                if (tagList is None or _filter is None):
                    orchestrator.getTests(projectName)
                else:
                    orchestrator.getTests(projectName, tagList, _filter)

        except ValueError as e:
            print self._formatMsg(fileName, lineNumber, str(e), TBTAFInterpreter.MSG_WARNING)

        #GetTags
        try:
            for var in TBTAFInterpreter.summary.getTags:
                fileName = var[TBTAFInterpreter.FILE_NAME]
                lineNumber = var[TBTAFInterpreter.FILE_LINE_NUMBER]

                projectName = var[TBTAFInterpreter.PROJECT_NAME_PARAM]

                orchestrator.getTags(projectName)

        except ValueError as e:
            print self._formatMsg(fileName, lineNumber, str(e), TBTAFInterpreter.MSG_WARNING)
<commit_msg>Fixed Orchestrator import, Added Exceptions imports<commit_after>'''
Crstr(e)atstr(e)d on 22/11/2015
@author:
'''

from exceptions import IOError
import re
import os as _os
import sys
from result import Result
from status import TBTAFParsingScriptStatus
from parsing_summary import ParsingSummary
from common.enums.filter_type import TBTAFFilterType
import orchestrator.TBTAFOrchestrator
from common.exception.IllegalArgumentException import IllegalArgumentException
from common.exception.NonSupportedFormatException import NonSupportedFormatException

class TBTAFInterpreter(object):
    '''
        TBTAFInterpreter
    '''

    OrchestratorReference = None
    summary = ParsingSummary()
    urlPattern = urlPattern = "(https?:\/\/)?([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w\.-]*)*\/?"

    ##Params in commands
    TEST_SUITE_PARAM = "testSuite"
    FILE_PATH_PARAM = "filePath"
    TEST_BED_PARAM = "testBed"
    PROJECT_NAME_PARAM = "projectName"
    TAG_LIST_PARAM = "tagList"
    FILTER_PARAM = "filter"
    FORMAT_PARAM = "format"
    URL_LIST_PARAM = "urlList"

    ##Available commands
    CREATE_TEST_BED = "create_test_bed"
    CREATE_TEST_SUITE = "create_test_suite"
    CREATE_NEW_PROJECT = "create_new_project"
    PUBLIS_TEST_PLAN = "publish test_plan"
    EXECUTE = "execute"
    PUBLISH_TEST_RESULT = "publish test_results"
    GET_TEST = "get tests"
    GET_TAGS = "get tags"

    ##Patterns
    TEST_SUITE = "(?P<"+ TEST_SUITE_PARAM +">(\w+))"
    FILE_PATH = "(?P<"+ FILE_PATH_PARAM +">(.*))"
    TEST_BED = "(?P<"+ TEST_BED_PARAM +">(\w+))"
    PROJECT_NAME = "\"(?P<"+ PROJECT_NAME_PARAM +">(.*))\""
    TAG_LIST = "\[(?P<"+ TAG_LIST_PARAM +">((<\w+>)(,<\w+>)*))\]"
    FILTER = "(?P<"+ FILTER_PARAM +">(\w+))"
    FORMAT = "(?P<"+ FORMAT_PARAM +">(\w+))"
    URL_LIST = "(?P<"+ URL_LIST_PARAM +">(("+ urlPattern +")(,"+ urlPattern +")*))"

    ##Message Types
    MSG_ERROR = "Error"
    MSG_WARNING = "Warning"

    FILE_NAME = "fileName"
    FILE_LINE_NUMBER = "lineNumber"

    ##Patterns
    a = "(?P<variable>\w+)\s*=\s*(?P<method>\\create_test_bed\\b)(\("+ URL_LIST +"\))?"
    b = "(?P<variable>\w+)\s*=\s*(?P<method>\\create_test_suite\\b)\(\s*"+ FILE_PATH +"(\,"+ TAG_LIST +")?\)"
    c = "(?P<method>\\create_new_project\\b)\("+ TEST_SUITE +","+ TEST_BED +","+ PROJECT_NAME +"\)"
    d = "(?P<method>\\publish test_plan\\b)\("+ TEST_SUITE +","+ FILE_PATH +","+ FORMAT +"\)"
    e = "(?P<variable>\w+)\s*=\s*(?P<method>\\execute\\b)\("+ TEST_SUITE +","+ TEST_BED +"\)"
    f = "(?P<method>\\publish test_results\\b)\("+ TEST_SUITE +","+ FILE_PATH +","+ FORMAT +"\)"
    g = "(?P<method>\\get tests\\b)\("+ PROJECT_NAME +"((\,"+ TAG_LIST +"),"+ FILTER +")?\)"
    h = "(?P<method>\\get tags\\b)\("+ PROJECT_NAME +"\)"

    def __init__(self):
        '''
            Constructor
        '''

    def _formatMsg(self,fileName, lineNumber, message, _type):
        #message = "{} :     File \"{}}\",line {}}\nDescription: {}}\n"
        return message.format(_type, str(fileName), str(lineNumber), message)
        ##return message.format("","","","")

    def parseScript(self, scriptURL):
        result = Result(TBTAFParsingScriptStatus.SUCCESS, "Success")
        file = None
        try:
            file = self._openFile(scriptURL)
            if (self._parseFile(file, result) == -1):
                file.close()
                print(result.status)
                print(result.message)
                return result
            else:
                if((result.status==TBTAFParsingScriptStatus.SUCCESS) and not(TBTAFInterpreter.OrchestratorReference is None)):
                    self.startExecution(TBTAFInterpreter.OrchestratorReference)
                else:
                    raise ValueError("Reference not set")
        except Exception as e:
            raise InvalidArgumentException(e)
        finally:
            if(file is not None):
                file.close()

        return result

    def _openFile(self,file):
        """
            This method returns object "_file"
        """
        ##________Verify is File exists________
        if (_os.path.isfile(file)):
            pass
        else:
            raise ValueError("File does not exist.")
        ##________Open File and return result________
        try:
            _file = open(file,"r")
            return _file
        except IOError:
            raise ValueError("Cannot open file.")

    ##----------------------------------------------
    ## Main method that will parse tbtaf script    -
    ##----------------------------------------------
    def _parseFile(self,file,result):
        """
        """
        ##________Dict for calling each regular expression________
        mappingPatterns   = {TBTAFInterpreter.CREATE_TEST_BED  :TBTAFInterpreter.a,TBTAFInterpreter.CREATE_TEST_SUITE:TBTAFInterpreter.b ,TBTAFInterpreter.CREATE_NEW_PROJECT  :TBTAFInterpreter.c,
                      TBTAFInterpreter.PUBLIS_TEST_PLAN:TBTAFInterpreter.d,TBTAFInterpreter.EXECUTE          :TBTAFInterpreter.e ,TBTAFInterpreter.PUBLISH_TEST_RESULT:TBTAFInterpreter.f,
                      TBTAFInterpreter.GET_TEST        :TBTAFInterpreter.g,TBTAFInterpreter.GET_TAGS         :TBTAFInterpreter.h}

        ##________Read each line of the file and look for a match of defined methods________
        for i, line in enumerate(file):
            ##________Delete White Spaces from the corners________
            line = line.strip()
            if (line == ""):
                pass
            elif (line.startswith("//")):
                pass

            for command in mappingPatterns:
                if (command in line):
                    m = re.match(mappingPatterns[command],line)
                    fileName = file.name
                    lineNumber = i + 1

                    if (m is None):
                        result.status  = TBTAFParsingScriptStatus.ERROR
                        result.message = self._formatMsg(fileName, lineNumber, "Cannot find command", TBTAFInterpreter.MSG_ERROR)
                    else:
                        m = m.groupdict()
                        ##print("Matched method = " + str(method))
                        ##print(str(m))
                        error = self._addToSummary(m, fileName, lineNumber)
                        if(error != ""):
                            result.status  = TBTAFParsingScriptStatus.ERROR
                            result.message = error
                            return -1

    def _addToSummary(self,m, fileName, lineNumber):
        m[TBTAFInterpreter.FILE_NAME] = fileName
        m[TBTAFInterpreter.FILE_LINE_NUMBER] = lineNumber

        error = '{} is not defined.'
        command = m["method"]

        if (command == TBTAFInterpreter.CREATE_TEST_BED):
            TBTAFInterpreter.summary.createTestBed[m["variable"]] = m

        elif (command == TBTAFInterpreter.CREATE_TEST_SUITE):
            TBTAFInterpreter.summary.createTestSuite[m["variable"]] = m

        elif (command == TBTAFInterpreter.CREATE_NEW_PROJECT):
            if not(m[TBTAFInterpreter.TEST_SUITE_PARAM] in TBTAFInterpreter.summary.createTestSuite):
                error = error.format(m[TBTAFInterpreter.TEST_SUITE_PARAM])
                return formatMsg(fileName, index, error, TBTAFInterpreter.MSG_ERROR)
            if not(m[TBTAFInterpreter.TEST_BED_PARAM] in TBTAFInterpreter.summary.createTestBed):
                error = error.format(m[TBTAFInterpreter.TEST_BED_PARAM])
                return formatMsg(fileName, index, error, TBTAFInterpreter.MSG_ERROR)

            TBTAFInterpreter.summary.createNewProject[m[TBTAFInterpreter.PROJECT_NAME_PARAM]] = m

        elif (command == TBTAFInterpreter.PUBLIS_TEST_PLAN):
            if not(m[TBTAFInterpreter.TEST_SUITE_PARAM] in TBTAFInterpreter.summary.createTestSuite):
                error = error.format(m[TBTAFInterpreter.TEST_SUITE_PARAM])
                return formatMsg(fileName, index, error, TBTAFInterpreter.MSG_ERROR)

            TBTAFInterpreter.summary.publishTestPlan.append(m)

        elif (command == TBTAFInterpreter.EXECUTE):
            if not(m[TBTAFInterpreter.TEST_SUITE_PARAM] in TBTAFInterpreter.summary.createTestSuite):
                error = error.format(m[TBTAFInterpreter.TEST_SUITE_PARAM])
                return formatMsg(fileName, index, error, TBTAFInterpreter.MSG_ERROR)

            TBTAFInterpreter.summary.execute[m["variable"]] = m

        elif (command == TBTAFInterpreter.PUBLISH_TEST_RESULT):
            if not(m[TBTAFInterpreter.TEST_SUITE_PARAM] in TBTAFInterpreter.summary.createTestSuite):
                error = error.format(m[TBTAFInterpreter.TEST_SUITE_PARAM])
                return formatMsg(fileName, index, error, TBTAFInterpreter.MSG_ERROR)

            TBTAFInterpreter.summary.publishTestResults.append(m)

        elif (command == TBTAFInterpreter.GET_TEST):
            if not(m[TBTAFInterpreter.PROJECT_NAME_PARAM] in TBTAFInterpreter.summary.createNewProject):
                error = error.format(m[TBTAFInterpreter.PROJECT_NAME_PARAM])
                return formatMsg(fileName, index, error, TBTAFInterpreter.MSG_ERROR)

            TBTAFInterpreter.summary.getTest.append(m)
        elif (command ==TBTAFInterpreter.GET_TAGS):
            if not(m[TBTAFInterpreter.PROJECT_NAME_PARAM] in TBTAFInterpreter.summary.createNewProject):
                error = error.format(m[TBTAFInterpreter.PROJECT_NAME_PARAM])
                return formatMsg(fileName, index, error, TBTAFInterpreter.MSG_ERROR)

            TBTAFInterpreter.summary.getTags.append(m)
        error = ""
        return error

    def setOrchestratorReference(self, orchestrator):
        TBTAFInterpreter.OrchestratorReference=orchestrator

    def startExecution(self,orchestrator):
        objs = {}
        fileName = ""
        lineNumber = ""

        try:
            #CreateTestBed
            testBeds = TBTAFInterpreter.summary.createTestBed
            for var in testBeds.keys():
                fileName = testBeds[var][TBTAFInterpreter.FILE_NAME]
                lineNumber = testBeds[var][TBTAFInterpreter.FILE_LINE_NUMBER]

                urlList = testBeds[var][TBTAFInterpreter.URL_LIST_PARAM]

                if(urlList is None):
                    objs[var] = orchestrator.createTestBed()
                else:
                    objs[var] = orchestrator.createTestBed(urlList)

            #CreateTestSuite
            testSuites = TBTAFInterpreter.summary.createTestSuite
            for var in testSuites.keys():
                fileName = testSuites[var][TBTAFInterpreter.FILE_NAME]
                lineNumber = testSuites[var][TBTAFInterpreter.FILE_LINE_NUMBER]

                filePath = testSuites[var][TBTAFInterpreter.FILE_PATH_PARAM]
                tagList = testSuites[var][TBTAFInterpreter.TAG_LIST_PARAM]
                if not(tagList is None):
                    tagList = tagList.split(",")

                objs[var] = orchestrator.createTestSuite(filePath, tagList)

            #CreateNewProject
            projects = TBTAFInterpreter.summary.createNewProject
            for var in projects.keys():
                fileName = projects[var][TBTAFInterpreter.FILE_NAME]
                lineNumber = projects[var][TBTAFInterpreter.FILE_LINE_NUMBER]

                testSuite = objs[projects[var][TBTAFInterpreter.TEST_SUITE_PARAM]]
                testBed = objs[projects[var][TBTAFInterpreter.TEST_BED_PARAM]]
                projectName = projects[var][TBTAFInterpreter.PROJECT_NAME_PARAM]

                orchestrator.createNewProject(testSuite, testBed, projectName)

        #except (Exception) as e:
        except (ValueError, IllegalArgumentException, NonSupportedFormatException) as e:
            print("hola")
            print self._formatMsg(fileName, lineNumber, "Fatal Error. Execution cannot continue. " + str(e), TBTAFInterpreter.MSG_ERROR)

        #PublishTestPlan
        try:
            for var in TBTAFInterpreter.summary.publishTestPlan:
                fileName = var[TBTAFInterpreter.FILE_NAME]
                lineNumber = var[TBTAFInterpreter.FILE_LINE_NUMBER]

                testSuite = var[TBTAFInterpreter.TEST_SUITE_PARAM]
                filePath = var[TBTAFInterpreter.FILE_PATH_PARAM]
                format = var[TBTAFInterpreter.FORMAT_PARAM]

                orchestrator.PublishTestPlan(testSuite, filePath, format)

        #except (ValueError) as e:
        except (ValueError, IllegalArgumentException, NonSupportedFormatException) as e:
            print self._formatMsg(fileName, lineNumber, str(e), TBTAFInterpreter.MSG_WARNING)

        #PublishResultReport
        try:
            for var in TBTAFInterpreter.summary.publishTestResults:
                fileName = var[TBTAFInterpreter.FILE_NAME]
                lineNumber = var[TBTAFInterpreter.FILE_LINE_NUMBER]

                testSuite = var[TBTAFInterpreter.TEST_SUITE_PARAM]
                filePath = var[TBTAFInterpreter.FILE_PATH_PARAM]
                format = var[TBTAFInterpreter.FORMAT_PARAM]

                orchestrator.PublishResultReport(testSuite, filePath, format)
        #except (ValueError) as e:
        except (ValueError, IllegalArgumentException, NonSupportedFormatException) as e:
            print self._formatMsg(fileName, lineNumber, str(e), TBTAFInterpreter.MSG_WARNING)

        #ExecuteTests
        try:
            #TBTAFInterpreter.summary.execute = executions
            executions= TBTAFInterpreter.summary.execute
            for var in executions.keys():
                fileName = executions[var][TBTAFInterpreter.FILE_NAME]
                lineNumber = executions[var][TBTAFInterpreter.FILE_LINE_NUMBER]

                testSuite = objs[executions[var][TBTAFInterpreter.TEST_SUITE_PARAM]]
                testBed = objs[executions[var][TBTAFInterpreter.TEST_SUITE_PARAM]]

                objs[var] = orchestrator.executeTests(testSuite, testBed)
        except ValueError as e:
            print self._formatMsg(fileName, lineNumber, str(e), TBTAFInterpreter.MSG_WARNING)

        #GetTests
        try:
            for var in TBTAFInterpreter.summary.getTest:
                fileName = var[TBTAFInterpreter.FILE_NAME]
                lineNumber = var[TBTAFInterpreter.FILE_LINE_NUMBER]

                projectName = var[TBTAFInterpreter.PROJECT_NAME_PARAM]
                tagList = var[TBTAFInterpreter.TAG_LIST_PARAM]
                if not(tagList is None):
                    tagList = tagList.split(",")
                filter = var[TBTAFInterpreter.FILTER_PARAM]
                if(filter == "IN"):
                    filter = TBTAFFilterType.IN
                elif(filter == "OUT"):
                    filter = TBTAFFilterType.OUT
                else:
                    filter = None

                if (tagList is None or _filter is None):
                    orchestrator.getTests(projectName)
                else:
                    orchestrator.getTests(projectName, tagList, _filter)

        except ValueError as e:
            print self._formatMsg(fileName, lineNumber, str(e), TBTAFInterpreter.MSG_WARNING)

        #GetTags
        try:
            for var in TBTAFInterpreter.summary.getTags:
                fileName = var[TBTAFInterpreter.FILE_NAME]
                lineNumber = var[TBTAFInterpreter.FILE_LINE_NUMBER]

                projectName = var[TBTAFInterpreter.PROJECT_NAME_PARAM]

                orchestrator.getTags(projectName)

        except ValueError as e:
            print self._formatMsg(fileName, lineNumber, str(e), TBTAFInterpreter.MSG_WARNING)
<|endoftext|>


<commit_before># This file is part of the Shapy Project.
# Licensing information can be found in the LICENSE file.
# (C) 2015 The Shapy Team. All rights reserved.

import os
import functools
import hashlib
import json

from tornado.web import RequestHandler, HTTPError
from tornado.gen import Return, Task, coroutine
import tornadoredis

from shapy.account import Account

def session(method):
  """Decorates methods to inject user info based on session token."""

  @coroutine
  @functools.wraps(method)
  def wrapper(self, *args, **kwargs):
    # If the session token is not set, omit the user id.
    token = self.get_secure_cookie('session')
    if not token:
      yield method(self, *args, user=None, **kwargs)
      raise Return()

    # Map the session ID to a user & refresh expiration.
    key = 'session:%s' % token
    data = yield Task(self.redis.get, key)
    yield Task(self.redis.expire, key, Account.SESSION_EXPIRE)

    # If user not logged in, pass None.
    if not data:
      yield method(self, *args, user=None, **kwargs)
      raise Return()

    # Initialize the account object.
    data = json.loads(data)
    yield method(self, *args, user=Account(
      data['id'],
      first_name=data['first_name'],
      last_name=data['last_name'],
      email=data['email']
    ), **kwargs)

  return wrapper

class BaseHandler(RequestHandler):
  """Base handler that lazily manages sessions and database connections."""

  @property
  def db(self):
    """Returns a reference to the database pool."""
    return self.application.db

  @property
  def redis(self):
    """Returns a reference to the redis connection."""
    return self.application.redis

  def on_finish(self):
    """Cleanup."""

    if hasattr(self, 'redis_conn'):
      self.redis_conn.disconnect()

  @coroutine
  def login(self, user):
    """Logs the user in, storing a session entry in the database."""
    token = os.urandom(16).encode('hex')
    yield Task(self.redis.set, 'session:%s' % token, json.dumps({
      'id': user[0],
      'first_name': user[1],
      'last_name': user[2],
      'email': user[3]
    }), Account.SESSION_EXPIRE)
    self.set_secure_cookie('session', token)

class APIHandler(BaseHandler):
  """Handles requests to the REST API."""

  def prepare(self):
    """Read request json into arguments dict."""

    if self.request.body:
      for key, value in json.loads(self.request.body).items():
        self.request.arguments[key] = [str(value)]

  def set_default_headers(self):
    """Set the JSON headers."""

    self.set_header('Content-Type', 'application/json')

  def write_error(self, status_code, **kwargs):
    """Handles error messages, outputting a properly formatted JSON message."""

    if 'exc_info' in kwargs:
      _, exception, _ = kwargs['exc_info']
      msg = str(exception)
    else:
      msg = 'Unknown error.'

    self.write(json.dumps({ 'error': msg }))
    self.finish()

<commit_msg>Added missing raise Return<commit_after># This file is part of the Shapy Project.
# Licensing information can be found in the LICENSE file.
# (C) 2015 The Shapy Team. All rights reserved.

import os
import functools
import hashlib
import json

from tornado.web import RequestHandler, HTTPError
from tornado.gen import Return, Task, coroutine
import tornadoredis

from shapy.account import Account

def session(method):
  """Decorates methods to inject user info based on session token."""

  @coroutine
  @functools.wraps(method)
  def wrapper(self, *args, **kwargs):
    # If the session token is not set, omit the user id.
    token = self.get_secure_cookie('session')
    if not token:
      yield method(self, *args, user=None, **kwargs)
      raise Return()

    # Map the session ID to a user & refresh expiration.
    key = 'session:%s' % token
    data = yield Task(self.redis.get, key)
    yield Task(self.redis.expire, key, Account.SESSION_EXPIRE)

    # If user not logged in, pass None.
    if not data:
      yield method(self, *args, user=None, **kwargs)
      raise Return()

    # Initialize the account object.
    data = json.loads(data)
    yield method(self, *args, user=Account(
      data['id'],
      first_name=data['first_name'],
      last_name=data['last_name'],
      email=data['email']
    ), **kwargs)
    raise Return()

  return wrapper

class BaseHandler(RequestHandler):
  """Base handler that lazily manages sessions and database connections."""

  @property
  def db(self):
    """Returns a reference to the database pool."""
    return self.application.db

  @property
  def redis(self):
    """Returns a reference to the redis connection."""
    return self.application.redis

  def on_finish(self):
    """Cleanup."""

    if hasattr(self, 'redis_conn'):
      self.redis_conn.disconnect()

  @coroutine
  def login(self, user):
    """Logs the user in, storing a session entry in the database."""
    token = os.urandom(16).encode('hex')
    yield Task(self.redis.set, 'session:%s' % token, json.dumps({
      'id': user[0],
      'first_name': user[1],
      'last_name': user[2],
      'email': user[3]
    }), Account.SESSION_EXPIRE)
    self.set_secure_cookie('session', token)

class APIHandler(BaseHandler):
  """Handles requests to the REST API."""

  def prepare(self):
    """Read request json into arguments dict."""

    if self.request.body:
      for key, value in json.loads(self.request.body).items():
        self.request.arguments[key] = [str(value)]

  def set_default_headers(self):
    """Set the JSON headers."""

    self.set_header('Content-Type', 'application/json')

  def write_error(self, status_code, **kwargs):
    """Handles error messages, outputting a properly formatted JSON message."""

    if 'exc_info' in kwargs:
      _, exception, _ = kwargs['exc_info']
      msg = str(exception)
    else:
      msg = 'Unknown error.'

    self.write(json.dumps({ 'error': msg }))
    self.finish()

<|endoftext|>


<commit_before>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Tobias Wegner, tobias.wegner@cern.ch, 2017-2018
# - Paul Nilsson, paul.nilsson@cern.ch, 2017

# Reimplemented by Alexey Anisenkov

import os
import logging
import re
from time import time

from .common import resolve_common_transfer_errors, verify_catalog_checksum
from pilot.util.container import execute
from pilot.common.exception import PilotException, ErrorCodes

logger = logging.getLogger(__name__)

require_replicas = True  ## indicate if given copytool requires input replicas to be resolved
allowed_schemas = ['root']  # prioritized list of supported schemas for transfers by given copytool

copy_command = 'xrdcp'

def is_valid_for_copy_in(files):
    return True  ## FIX ME LATER

def is_valid_for_copy_out(files):
    return True  ## FIX ME LATER

def get_timeout(filesize):   ## ISOLATE ME LATER
    """ Get a proper time-out limit based on the file size """

    timeout_max = 3 * 3600  # 3 hours
    timeout_min = 300  # self.timeout

    timeout = timeout_min + int(filesize / 0.5e6)  # approx < 0.5 Mb/sec

    return min(timeout, timeout_max)

def _resolve_checksum_option(setup, **kwargs):

    cmd = "%s --version" % copy_command
    if setup:
        cmd = "source %s; %s" % (setup, cmd)

    logger.info("Execute command (%s) to check xrdcp client version" % cmd)

    rcode, stdout, stderr = execute(cmd, **kwargs)
    logger.info("return code: %s" % rcode)
    logger.info("return output: %s" % (stdout + stderr))

    cmd = "%s -h" % copy_command
    if setup:
        cmd = "source %s; %s" % (setup, cmd)

    logger.info("Execute command (%s) to decide which option should be used to calc/verify file checksum.." % cmd)

    rcode, stdout, stderr = execute(cmd, **kwargs)
    output = stdout + stderr
    logger.info("return code: %s" % rcode)
    logger.debug("return output: %s" % output)

    coption = ""
    checksum_type = 'adler32'  ## consider only adler32 for now

    if rcode:
        logger.error('FAILED to execute command=%s: %s' % (cmd, output))
    else:
        if "--cksum" in output:
            coption = "--cksum %s:print" % checksum_type
        elif "-adler" in output and checksum_type == 'adler32':
            coption = "-adler"
        elif "-md5" in output and checksum_type == 'md5':
            coption = "-md5"

    if coption:
        logger.info("Use %s option to get the checksum for %s command" % (coption, copy_command))

    return coption

def _stagefile(coption, source, destination, filesize, is_stagein, setup=None, **kwargs):
    """
        Stage the file (stagein or stageout)
        :return: destination file details (checksum, checksum_type) in case of success, throw exception in case of failure
        :raise: PilotException in case of controlled error
    """

    cmd = '%s -np -f %s %s %s' % (copy_command, coption, source, destination)
    if setup:
        cmd = "source %s; %s" % (setup, cmd)

    #timeout = get_timeout(filesize)
    #logger.info("Executing command: %s, timeout=%s" % (cmd, timeout))

    rcode, stdout, stderr = execute(cmd, **kwargs)
    logger.info('rcode=%d, stdout=%s, stderr=%s' % (rcode, stdout, stderr))

    if rcode:  ## error occurred
        error = resolve_common_transfer_errors(stdout + stderr, is_stagein=is_stagein)

        #rcode = error.get('rcode')  ## TO BE IMPLEMENTED
        #if not is_stagein and rcode == PilotErrors.ERR_CHKSUMNOTSUP: ## stage-out, on fly checksum verification is not supported .. ignore
        #    logger.info('stage-out: ignore ERR_CHKSUMNOTSUP error .. will explicitly verify uploaded file')
        #    return None, None

        raise PilotException(error.get('error'), code=error.get('rcode'), state=error.get('state'))

    # extract filesize and checksum values from output
    checksum, checksum_type = get_file_checksum_from_output(stdout)

    ## verify transfer by returned checksum or call remote checksum calculation
    ## to be moved at the base level

    is_verified = True   ## TO BE IMPLEMENTED LATER

    if not is_verified:
        rcode = ErrorCodes.GETADMISMATCH if is_stagein else ErrorCodes.PUTADMISMATCH
        raise PilotException("Copy command failed", code=rcode, state='AD_MISMATCH')

def copy_in(files, **kwargs):
    """
        Download given files using xrdcp command.

        :param files: list of `FileSpec` objects
        :raise: PilotException in case of controlled error
    """

    allow_direct_access = kwargs.get('allow_direct_access') or False
    setup = kwargs.pop('copytools', {}).get('xrdcp', {}).get('setup')
    coption = _resolve_checksum_option(setup, **kwargs)
    trace_report = kwargs.get('trace_report')

    localsite = os.environ.get('DQ2_LOCAL_SITE_ID', None)
    for fspec in files:
        # update the trace report
        localsite = localsite if localsite else fspec.ddmendpoint
        trace_report.update(localSite=localsite, remoteSite=fspec.ddmendpoint, filesize=fspec.filesize)
        trace_report.update(filename=fspec.lfn, guid=fspec.guid.replace('-', ''))
        trace_report.update(scope=fspec.scope, dataset=fspec.dataset)

        # continue loop for files that are to be accessed directly
        if fspec.is_directaccess(ensure_replica=False) and allow_direct_access:
            fspec.status_code = 0
            fspec.status = 'remote_io'
            trace_report.update(url=fspec.turl, clientState='FOUND_ROOT', stateReason='direct_access')
            trace_report.send()
            continue

        trace_report.update(catStart=time())

        dst = fspec.workdir or kwargs.get('workdir') or '.'
        destination = os.path.join(dst, fspec.lfn)
        try:
            _stagefile(coption, fspec.turl, destination, fspec.filesize, is_stagein=True, setup=setup, **kwargs)
            fspec.status_code = 0
            fspec.status = 'transferred'
        except Exception as error:
            fspec.status = 'failed'
            fspec.status_code = error.get_error_code() if isinstance(error, PilotException) else ErrorCodes.STAGEINFAILED
            diagnostics = error.get_detail() if isinstance(error, PilotException) else "(consult log)"
            state = 'STAGEIN_ATTEMPT_FAILED'
            trace_report.update(clientState=state, stateReason=diagnostics, timeEnd=time())
            trace_report.send()
            raise PilotException(diagnostics, code=fspec.status_code, state=state)

        trace_report.update(clientState='DONE', stateReason='OK', timeEnd=time())
        trace_report.send()

    return files

def copy_out(files, **kwargs):
    """
        Upload given files using xrdcp command.

        :param files: list of `FileSpec` objects
        :raise: PilotException in case of controlled error
    """

    setup = kwargs.pop('copytools', {}).get('xrdcp', {}).get('setup')
    coption = _resolve_checksum_option(setup, **kwargs)
    trace_report = kwargs.get('trace_report')

    for fspec in files:
        trace_report.update(scope=fspec.scope, dataset=fspec.dataset, url=fspec.surl, filesize=fspec.filesize)
        trace_report.update(catStart=time(), filename=fspec.lfn, guid=fspec.guid.replace('-', ''))

        try:
            _stagefile(coption, fspec.surl, fspec.turl, fspec.filesize, is_stagein=False, setup=setup, **kwargs)
            fspec.status_code = 0
            fspec.status = 'transferred'
            trace_report.update(clientState='DONE', stateReason='OK', timeEnd=time())
            trace_report.send()
        except PilotException as error:
            fspec.status = 'failed'
            fspec.status_code = error.get_error_code() if isinstance(error, PilotException) else ErrorCodes.STAGEOUTFAILED
            trace_report.update(clientState=error.get('state', None) or 'STAGEOUT_ATTEMPT_FAILED',
                                stateReason=error.get('error', 'unknown error'),
                                timeEnd=time())
            trace_report.send()
            raise

    return files

def get_file_checksum_from_output(output):
    """
    Extract checksum value from xrdcp --chksum command output

    :return: (checksum, checksum_type) or (None, None) in case of failure
    """

    if not output:
        return None, None

    if not ("xrootd" in output or "XRootD" in output or "adler32" in output):
        logger.warning("WARNING: Failed to extract checksum: Unexpected output: %s" % output)
        return None, None

    pattern = "(?P<type>md5|adler32): (?P<checksum>[a-zA-Z0-9]+)"
    checksum, checksum_type = None, None

    m = re.search(pattern, output)
    if m:
        checksum_type = m.group('type')
        checksum = m.group('checksum')
        checksum = checksum.zfill(8)  # make it at least 8 chars length (adler32 xrdcp fix)
        #self.log("Copy command returned checksum: %s" % checksum)
    else:
        logger.warning("WARNING: Checksum info not found in output: failed to match pattern=%s in output=%s" % (pattern, output))

    return checksum, checksum_type
<commit_msg>Now extracting filesize from xrdcp output<commit_after>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Tobias Wegner, tobias.wegner@cern.ch, 2017-2018
# - Paul Nilsson, paul.nilsson@cern.ch, 2017

# Reimplemented by Alexey Anisenkov

import os
import logging
import re
from time import time

from .common import resolve_common_transfer_errors, verify_catalog_checksum
from pilot.util.container import execute
from pilot.common.exception import PilotException, ErrorCodes

logger = logging.getLogger(__name__)

require_replicas = True  ## indicate if given copytool requires input replicas to be resolved
allowed_schemas = ['root']  # prioritized list of supported schemas for transfers by given copytool

copy_command = 'xrdcp'

def is_valid_for_copy_in(files):
    return True  ## FIX ME LATER

def is_valid_for_copy_out(files):
    return True  ## FIX ME LATER

def get_timeout(filesize):   ## ISOLATE ME LATER
    """ Get a proper time-out limit based on the file size """

    timeout_max = 3 * 3600  # 3 hours
    timeout_min = 300  # self.timeout

    timeout = timeout_min + int(filesize / 0.5e6)  # approx < 0.5 Mb/sec

    return min(timeout, timeout_max)

def _resolve_checksum_option(setup, **kwargs):

    cmd = "%s --version" % copy_command
    if setup:
        cmd = "source %s; %s" % (setup, cmd)

    logger.info("Execute command (%s) to check xrdcp client version" % cmd)

    rcode, stdout, stderr = execute(cmd, **kwargs)
    logger.info("return code: %s" % rcode)
    logger.info("return output: %s" % (stdout + stderr))

    cmd = "%s -h" % copy_command
    if setup:
        cmd = "source %s; %s" % (setup, cmd)

    logger.info("Execute command (%s) to decide which option should be used to calc/verify file checksum.." % cmd)

    rcode, stdout, stderr = execute(cmd, **kwargs)
    output = stdout + stderr
    logger.info("return code: %s" % rcode)
    logger.debug("return output: %s" % output)

    coption = ""
    checksum_type = 'adler32'  ## consider only adler32 for now

    if rcode:
        logger.error('FAILED to execute command=%s: %s' % (cmd, output))
    else:
        if "--cksum" in output:
            coption = "--cksum %s:print" % checksum_type
        elif "-adler" in output and checksum_type == 'adler32':
            coption = "-adler"
        elif "-md5" in output and checksum_type == 'md5':
            coption = "-md5"

    if coption:
        logger.info("Use %s option to get the checksum for %s command" % (coption, copy_command))

    return coption

def _stagefile(coption, source, destination, filesize, is_stagein, setup=None, **kwargs):
    """
        Stage the file (stagein or stageout)
        :return: destination file details (checksum, checksum_type) in case of success, throw exception in case of failure
        :raise: PilotException in case of controlled error
    """

    cmd = '%s -np -f %s %s %s' % (copy_command, coption, source, destination)
    if setup:
        cmd = "source %s; %s" % (setup, cmd)

    #timeout = get_timeout(filesize)
    #logger.info("Executing command: %s, timeout=%s" % (cmd, timeout))

    rcode, stdout, stderr = execute(cmd, **kwargs)
    logger.info('rcode=%d, stdout=%s, stderr=%s' % (rcode, stdout, stderr))

    if rcode:  ## error occurred
        error = resolve_common_transfer_errors(stdout + stderr, is_stagein=is_stagein)

        #rcode = error.get('rcode')  ## TO BE IMPLEMENTED
        #if not is_stagein and rcode == PilotErrors.ERR_CHKSUMNOTSUP: ## stage-out, on fly checksum verification is not supported .. ignore
        #    logger.info('stage-out: ignore ERR_CHKSUMNOTSUP error .. will explicitly verify uploaded file')
        #    return None, None

        raise PilotException(error.get('error'), code=error.get('rcode'), state=error.get('state'))

    # extract filesize and checksum values from output
    if coption != "":
        filesize_output, checksum_output, checksum_type = get_file_info_from_output(stdout)

    ## verify transfer by returned checksum or call remote checksum calculation
    ## to be moved at the base level

    is_verified = True   ## TO BE IMPLEMENTED LATER

    if not is_verified:
        rcode = ErrorCodes.GETADMISMATCH if is_stagein else ErrorCodes.PUTADMISMATCH
        raise PilotException("Copy command failed", code=rcode, state='AD_MISMATCH')

def copy_in(files, **kwargs):
    """
        Download given files using xrdcp command.

        :param files: list of `FileSpec` objects
        :raise: PilotException in case of controlled error
    """

    allow_direct_access = kwargs.get('allow_direct_access') or False
    setup = kwargs.pop('copytools', {}).get('xrdcp', {}).get('setup')
    coption = _resolve_checksum_option(setup, **kwargs)
    trace_report = kwargs.get('trace_report')

    localsite = os.environ.get('DQ2_LOCAL_SITE_ID', None)
    for fspec in files:
        # update the trace report
        localsite = localsite if localsite else fspec.ddmendpoint
        trace_report.update(localSite=localsite, remoteSite=fspec.ddmendpoint, filesize=fspec.filesize)
        trace_report.update(filename=fspec.lfn, guid=fspec.guid.replace('-', ''))
        trace_report.update(scope=fspec.scope, dataset=fspec.dataset)

        # continue loop for files that are to be accessed directly
        if fspec.is_directaccess(ensure_replica=False) and allow_direct_access:
            fspec.status_code = 0
            fspec.status = 'remote_io'
            trace_report.update(url=fspec.turl, clientState='FOUND_ROOT', stateReason='direct_access')
            trace_report.send()
            continue

        trace_report.update(catStart=time())

        dst = fspec.workdir or kwargs.get('workdir') or '.'
        destination = os.path.join(dst, fspec.lfn)
        try:
            _stagefile(coption, fspec.turl, destination, fspec.filesize, is_stagein=True, setup=setup, **kwargs)
            fspec.status_code = 0
            fspec.status = 'transferred'
        except Exception as error:
            fspec.status = 'failed'
            fspec.status_code = error.get_error_code() if isinstance(error, PilotException) else ErrorCodes.STAGEINFAILED
            diagnostics = error.get_detail() if isinstance(error, PilotException) else "(consult log)"
            state = 'STAGEIN_ATTEMPT_FAILED'
            trace_report.update(clientState=state, stateReason=diagnostics, timeEnd=time())
            trace_report.send()
            raise PilotException(diagnostics, code=fspec.status_code, state=state)

        trace_report.update(clientState='DONE', stateReason='OK', timeEnd=time())
        trace_report.send()

    return files

def copy_out(files, **kwargs):
    """
        Upload given files using xrdcp command.

        :param files: list of `FileSpec` objects
        :raise: PilotException in case of controlled error
    """

    setup = kwargs.pop('copytools', {}).get('xrdcp', {}).get('setup')
    coption = _resolve_checksum_option(setup, **kwargs)
    trace_report = kwargs.get('trace_report')

    for fspec in files:
        trace_report.update(scope=fspec.scope, dataset=fspec.dataset, url=fspec.surl, filesize=fspec.filesize)
        trace_report.update(catStart=time(), filename=fspec.lfn, guid=fspec.guid.replace('-', ''))

        try:
            _stagefile(coption, fspec.surl, fspec.turl, fspec.filesize, is_stagein=False, setup=setup, **kwargs)
            fspec.status_code = 0
            fspec.status = 'transferred'
            trace_report.update(clientState='DONE', stateReason='OK', timeEnd=time())
            trace_report.send()
        except PilotException as error:
            fspec.status = 'failed'
            fspec.status_code = error.get_error_code() if isinstance(error, PilotException) else ErrorCodes.STAGEOUTFAILED
            trace_report.update(clientState=error.get('state', None) or 'STAGEOUT_ATTEMPT_FAILED',
                                stateReason=error.get('error', 'unknown error'),
                                timeEnd=time())
            trace_report.send()
            raise

    return files

def get_file_info_from_output(output):
    """
    Extract file size, checksum value from xrdcp --chksum command output

    :return: (filesize [int/None], checksum, checksum_type) or (None, None, None) in case of failure
    """

    if not output:
        return None, None

    if not ("xrootd" in output or "XRootD" in output or "adler32" in output):
        logger.warning("WARNING: Failed to extract checksum: Unexpected output: %s" % output)
        return None, None

    pattern = "(?P<type>md5|adler32):\ (?P<checksum>[a-zA-Z0-9]+)\ \S+\ (?P<filesize>[0-9]+)"
    filesize, checksum, checksum_type = None, None, None

    m = re.search(pattern, output)
    if m:
        checksum_type = m.group('type')
        checksum = m.group('checksum')
        checksum = checksum.zfill(8)  # make it 8 chars length (adler32 xrdcp fix)
        filesize = m.group('filesize')
        if filesize:
            try:
                filesize = int(filesize)
            except ValueError as e:
                logger.warning('failed to convert filesize to int: %s' % e)
                filesize = None
    else:
        logger.warning("WARNING: Checksum/file size info not found in output: failed to match pattern=%s in output=%s" % (pattern, output))

    return filesize, checksum, checksum_type
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
instaRaider.py

usage: instaRaider.py [-h] -u USER [-c COUNT]

@amirkurtovic

TEST
TEST 1234
TEST 5678

"""
from bs4 import BeautifulSoup
import selenium.webdriver as webdriver
import re
from time import sleep
import urllib
import urllib2
import os
import sys
import argparse

class instaRaider(object):

    def getImageCount(self, url):
        '''
        Given a url to Instagram profile, return number of photos posted
        '''
        response = urllib2.urlopen(url)
        countsCode = re.search(r'counts\":{\"media\":\d+', response.read())
        count = re.findall(r'\d+', countsCode.group())
        return count[0]

    def loadInstagram(self, profileUrl):
        '''
        Using Selenium WebDriver, load Instagram page to get page source

        '''
        count = self.getImageCount(self.profileUrl)
        print self.userName + " has " + str(count) + " photos on Instagram."

        print "Loading Selenium WebDriver..."

        # Load webdriver and scale window down
        driver = webdriver.Firefox()
        driver.set_window_size(40,40)
        driver.set_window_position(20,20)

        # load Instagram profile and wait for PAUSE
        driver.get(self.profileUrl)
        driver.implicitly_wait(self.PAUSE)

        print "Loading Instagram profile..."

        clicks = (int(count)-60)/20+1

        for x in range(3):
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            sys.stdout.write('.')
            sys.stdout.flush()
            sleep(self.PAUSE)

        # Load full Instagram profile if more than initial 60 photos desired
        if (args.count < 61):
            pass
        else:
            # Click on "Load more..." label
            element = driver.find_element_by_xpath(self.loadLabelXPATH)

            for y in range(clicks):
                element.click()
                driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
                sys.stdout.write('.')
                sys.stdout.flush()
                sleep(self.PAUSE)

        # After load all profile photos, retur source to getPhotos()
        source = BeautifulSoup(driver.page_source)

        # close Firefox window
        driver.close()

        return source

    def validUser(self, userName):
        '''
        returns True if Instagram username is valid
        '''
        valid = True
        # check if Instagram username is valid
        req = urllib2.Request(self.profileUrl)

        try:
            urllib2.urlopen(req)
        except:
            valid = False
        # if req doesn't fail, user profile exists
        return valid

    def photoExists(self, url):
        '''
        Returns true if photo exists
        Used when checking which suffix Instagram used for full-res photo
        url: URL to Instagram photo
        '''
        try:
            urllib2.urlopen(url)
        except:
            return False

        return True

    def getPhotos(self, source, userName, count):
        '''
        Given source code for loaded Instagram page,
        extract all hrefs and download full-resolution photos

        source: HTML source code of Instagram profile papge
        '''
        # directory where photos will be saved
        directory = './Images/' + userName + '/'

        # check if directory exists, if not, make it
        if not os.path.exists(directory):
            os.makedirs(directory)

        # logfile to store urls is csv format
        logfile = './Images/' + userName + '/' + userName + '.csv'
        try:
            file = open(logfile, "a")
        except IOError:
            print "\nLog file does not exist."

        # photo number for file names
        photoNumber = 0

        # indexes for progress bar
        photosSaved = 0
        progressBar = 0

        print "\nRaiding Instagram..."
        print "Saving photos to " + directory

        print "------"
        # print progress bar
        print "Photos saved so far:"
        print "---------10--------20--------30--------40--------50"

        for x in source.findAll('li', {'class':'photo'}):

            if (photoNumber >= count):
                break
            else:
                # increment photonumber for next image
                photoNumber += 1

                #extract url to thumbnail from each photo
                rawUrl = re.search(r'url\(\"https?://[^\s<>"]+|www\.[^\s<>"]+', str(x))

                # format thumbnail url to lead to full-resolution photo
                # Instagram full-res URLs end in suffixes stored in fullResSuffixes list
                photoUrl = str(rawUrl.group())[5:-5]
                suffix = ''
                for item in self.fullResSuffixes:
                    url = photoUrl + item
                    if(self.photoExists(url)):
                        suffix = item

                photoUrl = str(rawUrl.group())[5:-5] + suffix
                photoName = directory + userName + "_" + str(photoNumber) + '.jpg'

                # save full-resolution photo
                urllib.urlretrieve(photoUrl, photoName)

                # save filename and url to CSV file
                file.write(photoUrl + "," + photoName + "\n")

                # print hash to progress bar
                if (photosSaved == 50):
                    photosSaved = 1
                    progressBar += 50
                    sys.stdout.write('\n')
                    sys.stdout.write('#')
                    sys.stdout.flush()

                else:
                    # increment progress bar
                    photosSaved += 1
                    sys.stdout.write('#')
                    sys.stdout.flush()

                sleep(self.PAUSE)

        print "\n------"
        print "Saved " + str(photoNumber) + " images to " + directory

        # close logfile
        file.close()
        print "Saved activity in logfile: " + logfile

    def __init__(self, userName):
        self.userName = userName
        self.profileUrl = 'http://instagram.com/' + userName + '/'
        self.fullResSuffixes = ['7.jpg', '8.jpg', 'n.jpg']
        self.PAUSE = 1
        self.loadLabelXPATH = "/html/body/div/div/div/section/div/span/a/span[2]/span/span"

if __name__ == '__main__':

    # parse arguments
    parser = argparse.ArgumentParser(description="InstaRaider")
    parser.add_argument('-u', '--user', help="Instagram username", required=True)
    parser.add_argument('-c', '--count', help="# of photos to download", type=int)
    args = parser.parse_args()

    if (args.user):
        userName = args.user

        raider = instaRaider(userName)
        url = raider.profileUrl

        if not args.count:
            count = raider.getImageCount(url)
        else:
            count = args.count
            if raider.getImageCount(url) < count:
                print "You want to dowload %r photos." % args.count
                print "The user only has %r photo." % raider.getImageCount(url)
                print "Downloading all photos."
                count = raider.getImageCount(url)

        if(raider.validUser(userName)):
            # Get source code from fully loaded Instagram profile page
            source = raider.loadInstagram(url)

            # Download all photos identified on profile page
            raider.getPhotos(source, userName, count)
        else:
            print "Username " + userName + " is not valid."
        <commit_msg>Revesting testing changes<commit_after>#!/usr/bin/env python
"""
instaRaider.py

usage: instaRaider.py [-h] -u USER [-c COUNT]

@amirkurtovic

"""
from bs4 import BeautifulSoup
import selenium.webdriver as webdriver
import re
from time import sleep
import urllib
import urllib2
import os
import sys
import argparse

class instaRaider(object):

    def getImageCount(self, url):
        '''
        Given a url to Instagram profile, return number of photos posted
        '''
        response = urllib2.urlopen(url)
        countsCode = re.search(r'counts\":{\"media\":\d+', response.read())
        count = re.findall(r'\d+', countsCode.group())
        return count[0]

    def loadInstagram(self, profileUrl):
        '''
        Using Selenium WebDriver, load Instagram page to get page source

        '''
        count = self.getImageCount(self.profileUrl)
        print self.userName + " has " + str(count) + " photos on Instagram."

        print "Loading Selenium WebDriver..."

        # Load webdriver and scale window down
        driver = webdriver.Firefox()
        driver.set_window_size(40,40)
        driver.set_window_position(20,20)

        # load Instagram profile and wait for PAUSE
        driver.get(self.profileUrl)
        driver.implicitly_wait(self.PAUSE)

        print "Loading Instagram profile..."

        clicks = (int(count)-60)/20+1

        for x in range(3):
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            sys.stdout.write('.')
            sys.stdout.flush()
            sleep(self.PAUSE)

        # Load full Instagram profile if more than initial 60 photos desired
        if (args.count < 61):
            pass
        else:
            # Click on "Load more..." label
            element = driver.find_element_by_xpath(self.loadLabelXPATH)

            for y in range(clicks):
                element.click()
                driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
                sys.stdout.write('.')
                sys.stdout.flush()
                sleep(self.PAUSE)

        # After load all profile photos, retur source to getPhotos()
        source = BeautifulSoup(driver.page_source)

        # close Firefox window
        driver.close()

        return source

    def validUser(self, userName):
        '''
        returns True if Instagram username is valid
        '''
        valid = True
        # check if Instagram username is valid
        req = urllib2.Request(self.profileUrl)

        try:
            urllib2.urlopen(req)
        except:
            valid = False
        # if req doesn't fail, user profile exists
        return valid

    def photoExists(self, url):
        '''
        Returns true if photo exists
        Used when checking which suffix Instagram used for full-res photo
        url: URL to Instagram photo
        '''
        try:
            urllib2.urlopen(url)
        except:
            return False

        return True

    def getPhotos(self, source, userName, count):
        '''
        Given source code for loaded Instagram page,
        extract all hrefs and download full-resolution photos

        source: HTML source code of Instagram profile papge
        '''
        # directory where photos will be saved
        directory = './Images/' + userName + '/'

        # check if directory exists, if not, make it
        if not os.path.exists(directory):
            os.makedirs(directory)

        # logfile to store urls is csv format
        logfile = './Images/' + userName + '/' + userName + '.csv'
        try:
            file = open(logfile, "a")
        except IOError:
            print "\nLog file does not exist."

        # photo number for file names
        photoNumber = 0

        # indexes for progress bar
        photosSaved = 0
        progressBar = 0

        print "\nRaiding Instagram..."
        print "Saving photos to " + directory

        print "------"
        # print progress bar
        print "Photos saved so far:"
        print "---------10--------20--------30--------40--------50"

        for x in source.findAll('li', {'class':'photo'}):

            if (photoNumber >= count):
                break
            else:
                # increment photonumber for next image
                photoNumber += 1

                #extract url to thumbnail from each photo
                rawUrl = re.search(r'url\(\"https?://[^\s<>"]+|www\.[^\s<>"]+', str(x))

                # format thumbnail url to lead to full-resolution photo
                # Instagram full-res URLs end in suffixes stored in fullResSuffixes list
                photoUrl = str(rawUrl.group())[5:-5]
                suffix = ''
                for item in self.fullResSuffixes:
                    url = photoUrl + item
                    if(self.photoExists(url)):
                        suffix = item

                photoUrl = str(rawUrl.group())[5:-5] + suffix
                photoName = directory + userName + "_" + str(photoNumber) + '.jpg'

                # save full-resolution photo
                urllib.urlretrieve(photoUrl, photoName)

                # save filename and url to CSV file
                file.write(photoUrl + "," + photoName + "\n")

                # print hash to progress bar
                if (photosSaved == 50):
                    photosSaved = 1
                    progressBar += 50
                    sys.stdout.write('\n')
                    sys.stdout.write('#')
                    sys.stdout.flush()

                else:
                    # increment progress bar
                    photosSaved += 1
                    sys.stdout.write('#')
                    sys.stdout.flush()

                sleep(self.PAUSE)

        print "\n------"
        print "Saved " + str(photoNumber) + " images to " + directory

        # close logfile
        file.close()
        print "Saved activity in logfile: " + logfile

    def __init__(self, userName):
        self.userName = userName
        self.profileUrl = 'http://instagram.com/' + userName + '/'
        self.fullResSuffixes = ['7.jpg', '8.jpg', 'n.jpg']
        self.PAUSE = 1
        self.loadLabelXPATH = "/html/body/div/div/div/section/div/span/a/span[2]/span/span"

if __name__ == '__main__':

    # parse arguments
    parser = argparse.ArgumentParser(description="InstaRaider")
    parser.add_argument('-u', '--user', help="Instagram username", required=True)
    parser.add_argument('-c', '--count', help="# of photos to download", type=int)
    args = parser.parse_args()

    if (args.user):
        userName = args.user

        raider = instaRaider(userName)
        url = raider.profileUrl

        if not args.count:
            count = raider.getImageCount(url)
        else:
            count = args.count
            if raider.getImageCount(url) < count:
                print "You want to dowload %r photos." % args.count
                print "The user only has %r photo." % raider.getImageCount(url)
                print "Downloading all photos."
                count = raider.getImageCount(url)

        if(raider.validUser(userName)):
            # Get source code from fully loaded Instagram profile page
            source = raider.loadInstagram(url)

            # Download all photos identified on profile page
            raider.getPhotos(source, userName, count)
        else:
            print "Username " + userName + " is not valid."
        <|endoftext|>