<commit_before># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import unittest
import logging
import sys
import signal
import os.path
import copy
import time
import random
from ConfigParser import ConfigParser

from tashi.services.ttypes import *
from thrift.transport.TSocket import TSocket
from thrift.protocol.TBinaryProtocol import TBinaryProtocol
from thrift.transport.TTransport import TBufferedTransport

from tashi.services import clustermanagerservice
from tashi.messaging.threadpool import synchronized
from tashi.messaging.tashimessaging import TestTashiSubscriber

from tashi.util import getConfig

import tashi.client.client

class ClientConnection(object):
	'''Creates an rpc proxy'''
	def __init__(self, host, port):
	self.host = host
	self.port = port
	self.transport = TBufferedTransport(TSocket(host, int(port)))
	self.protocol = TBinaryProtocol(self.transport)
	self.client = clustermanagerservice.Client(self.protocol)
	self.client._transport = self.transport
	self.client._transport.open()
	def __del__(self):
		self.client._transport.close()

def incrementor(init=0):
	while 1:
		yield init
		init = init + 1

# FIXME: don't duplicate code from clustermanager
# def getConfig(args):
#	 config = ConfigParser()
#	 configFiles = [
#		'/usr/share/tashi/ClusterManagerDefaults.cfg',
#		'/etc/tashi/ClusterManager.cfg',
#		os.path.expanduser('~/.tashi/ClusterManager.cfg')
#		] + ([args[0]] if len(args) > 0 else [])

#	 configFiles = config.read(configFiles)
#	 if len(configFiles) == 0:
#		 print >>sys.stderr, 'Unable to find the configuration file\n'
#		 sys.exit(3)

#	 return config

class TestClient(unittest.TestCase):
	@synchronized()
	def getPortNum(self):
		return self.portnum.next()

	"""macro test cases for single-host tests

	Assumes cwd is 'src/tashi/client/'
	"""
	def setUp(self):
		"""Create a CM and single NM on local host"""
		logging.info('setting up test')

		(self.config, self.configfiles) = getConfig([])

		self.port = 1717		# FIXME: take this (and other things) from config file
		self.portnum = incrementor(self.port)

		self.cwd = os.getcwd()
		self.srcd = os.path.dirname(os.path.dirname(self.cwd))

		self.environ = copy.copy(os.environ)
		self.environ['PYTHONPATH'] = self.srcd
		logging.info('base path = %s' % self.srcd)

		self.nm = os.spawnlpe(os.P_NOWAIT, 'python', 'python',
							  os.path.join(self.srcd, 'tashi', 'nodemanager', 'nodemanager.py'),
							  self.environ)
		self.cm = os.spawnlpe(os.P_WAIT, 'python', 'python',
							 os.path.join(self.srcd,  'tashi', 'clustermanager', 'clustermanager.py'),
							 '--drop', '--create',
							 os.path.expanduser('~/.tashi/ClusterManager.cfg'),
							 self.environ)
		self.cm = os.spawnlpe(os.P_NOWAIT, 'python', 'python',
							 os.path.join(self.srcd,  'tashi', 'clustermanager', 'clustermanager.py'),
							 os.path.expanduser('~/.tashi/ClusterManager.cfg'),
							 self.environ)
		# since we are spawning with P_NOWAIT, we need to sleep to ensure that the CM is listening
		time.sleep(1)
		try:
			self.connection = ClientConnection('localhost', self.config.get('ClusterManagerService', 'port'))
		except Exception, e:
			logging.warning('client connection failed')
			ex = None
			try:
				logging.warning("setUp killing node manager " + str(self.nm))
				os.kill(self.nm, signal.SIGKILL)
			except Exception, e:
				ex = e
				logging.warning('could not kill node manager: '+ str(e))
			try:
				logging.warning('setUp killing cluster manager ' + str(self.cm))
				os.kill(self.cm, signal.SIGKILL)
			except Exception, e:
				ex = e
				logging.warning('could not kill cluster manager: ' + str(e))
			if e != None:
				raise e

		logging.info('node manager PID: %i' % self.nm)
	def tearDown(self):
		'''Kill the CM and NM that were created by setUP'''
		logging.info('tearing down test')
		ex = None
		try:
			logging.debug("killing cluster manager " + str(self.cm))
			os.kill(self.cm, signal.SIGKILL)
		except Exception, e:
			ex = e
			logging.error('Could not kill cluster manager: ' + str(e))

		try:
			logging.debug("killing node manager " + str(self.nm))
			os.kill(self.nm, signal.SIGKILL)
		except Exception, e:
			ex = e
			logging.error('Could not kill node manager: ' + str(e))
		if ex != None:
			raise ex
	def testSetup(self):
		'''empty test to ensure that setUp code works'''
		logging.info('setting up')
	def testHostManagement(self):
		'''test adding/removing/listing hosts

		Right now this just adds a single host: localhost.  Eventually
		it should 1) take a list of hosts from a test configuration
		file, 2) ensure that all were added, 3) remove a random
		subset, 4) ensure that they were correctly removed, 5) remove
		all, 6) ensure that they were correctly removed.'''

		# get empty host list
		hosts = self.connection.client.getHosts()
		self.assertEqual(hosts, [], 'starting host list not empty: ' + str(hosts) )

		# add a host
		host = Host()
		host.hostname = 'localhost'
		host.enabled=True
		self.connection.client.addHost(host)
		hosts = self.connection.client.getHosts()
		self.assertEqual(len(hosts), 1, 'wrong number of hosts %i, should be %i' % (len(hosts), 1) )
		self.assertEqual(hosts[0].hostname, 'localhost', 'wrong hostname: ' + str(hosts[0].hostname) )

		# remove first host
		hid = hosts[0].id
		self.connection.client.removeHost(hid)
		hosts = self.connection.client.getHosts()
		self.assertEqual(hosts, [], 'host list not empty after remove: ' + str(hosts) )

	def testMessaging(self):
		'''test messaging system started by CM

		tests messages published directly, through events in the CM,
		and the log system'''
		# FIXME: add tests for generating events as a side-effect of
		# rpc commands, as well as logging in the CM
		portnum = self.getPortNum()
		self.sub = TestTashiSubscriber(self.config, portnum)
		self.assertEqual(self.sub.messageQueue.qsize(), 0)
		self.pub = tashi.messaging.thriftmessaging.PublisherThrift(self.config.get('MessageBroker', 'host'),
																   int(self.config.get('MessageBroker', 'port')))
		self.pub.publish({'message-type':'text', 'message':'Hello World!'})
		time.sleep(0.5)
		print '*** QSIZE', self.sub.messageQueue.qsize()
		self.assertEqual(self.sub.messageQueue.qsize(), 1)

		self.log = logging.getLogger(__name__)
		messageHandler = tashi.messaging.tashimessaging.TashiLogHandler(self.config)
		self.log.addHandler(messageHandler)
		# FIXME: why can't we log messages with severity below 'warning'?
		self.log.warning('test log message')
		time.sleep(0.5)
		self.assertEqual(self.sub.messageQueue.qsize(), 2)

		# This should generate at least one log message
#		 hosts = self.connection.client.getHosts()
#		 time.sleep(0.5)
#		 if (self.sub.messageQueue.qsize() <= 2):
#			 self.fail()

	def testUserManagement(self):
		'''test adding/removing/listing users

		same as testHostManagement, but with users'''
		usernames = ['sleepy', 'sneezy', 'dopey', 'doc',
					 'grumpy', 'bashful', 'happy']
		# add all users
		for un in usernames:
			user = User()
			user.username = un
			self.connection.client.addUser(user)
		# ensure that all were added
		users = self.connection.client.getUsers()
		self.assertEqual(len(usernames), len(users))
		for user in users:
			usernames.remove(user.username)
		self.assertEqual(0, len(usernames))
		# remove a random subset
		rm = random.sample(users, 4)
		for user in rm:
			self.connection.client.removeUser(user.id)
			users.remove(user)
		newUsers = self.connection.client.getUsers()
		# This ensures that the remaining ones are what we expect:
		for user in newUsers:
			# if there is a user remaining that we asked to be removed,
			# this will throw an exception
			users.remove(user)
		# if a user was removed that we did not intend, this will
		# throw an exception
		self.assertEqual(0, len(users))

#	 def testInstanceConfigurationManagement(self):
#		 '''test adding/removing/listing instance configurations

#		 same as testHostManagement, but with instance configurations'''
#		 self.fail('test not implemented')
	def testHardDiskConfigurationManagement(self):
		'''test adding/removing/listing hard disk configurations

		same as testHostManagement, but with hard disk configurations'''

		user = User(d={'username':'sleepy'})
		self.connection.client.addUser(user)
		users = self.connection.client.getUsers()

		per = PersistentImage()
		per.userId = users[0].id
		per.name = 'sleepy-PersistentImage'
		self.connection.client.addPersistentImage(per)
		pers = self.connection.client.getPersistentImages()

		inst = InstanceConfiguration()
		inst.name = 'sleepy-inst'
		inst.memory = 512
		inst.cores = 1
		self.connection.client.addInstanceConfiguration(inst)
		insts = self.connection.client.getInstanceConfigurations()

		hdc = HardDiskConfiguration()
		hdc.index = 0
		hdc.persistentImageId = pers[0].id
		hdc.persistent = False
		hdc.instanceConfigurationId = insts[0].id

#	 def testCreateDestroyShutdown(self):
#		 '''test creating/destroying/shutting down VMs

#		 not implemented'''
#		 self.fail('test not implemented')
#	 def testSuspendResume(self):
#		 '''test suspending/resuming VMs

#		 not implemented'''
#		 self.fail('test not implemented')
#	 def testMigrate(self):
#		 '''test migration

#		 not implemented'''
#		 self.fail('test not implemented')
#	 def testPauseUnpause(self):
#		 '''test pausing/unpausing VMs

#		 not implemented'''
#		 self.fail('test not implemented')

##############################
# Test Code
##############################
if __name__ == '__main__':
	logging.basicConfig(level=logging.NOTSET,
						format="%(asctime)s %(levelname)s:\t %(message)s",
						stream=sys.stdout)

	suite = unittest.TestLoader().loadTestsFromTestCase(TestClient)
	unittest.TextTestRunner(verbosity=2).run(suite)

<commit_msg>A bug fix from the previous indentation patch (part of TASHI-2).  This fixes incorrect indentation in one file.<commit_after># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import unittest
import logging
import sys
import signal
import os.path
import copy
import time
import random
from ConfigParser import ConfigParser

from tashi.services.ttypes import *
from thrift.transport.TSocket import TSocket
from thrift.protocol.TBinaryProtocol import TBinaryProtocol
from thrift.transport.TTransport import TBufferedTransport

from tashi.services import clustermanagerservice
from tashi.messaging.threadpool import synchronized
from tashi.messaging.tashimessaging import TestTashiSubscriber

from tashi.util import getConfig

import tashi.client.client

class ClientConnection(object):
	'''Creates an rpc proxy'''
	def __init__(self, host, port):
		self.host = host
		self.port = port
		self.transport = TBufferedTransport(TSocket(host, int(port)))
		self.protocol = TBinaryProtocol(self.transport)
		self.client = clustermanagerservice.Client(self.protocol)
		self.client._transport = self.transport
		self.client._transport.open()
	def __del__(self):
		self.client._transport.close()

def incrementor(init=0):
	while 1:
		yield init
		init = init + 1

# FIXME: don't duplicate code from clustermanager
# def getConfig(args):
#	 config = ConfigParser()
#	 configFiles = [
#		'/usr/share/tashi/ClusterManagerDefaults.cfg',
#		'/etc/tashi/ClusterManager.cfg',
#		os.path.expanduser('~/.tashi/ClusterManager.cfg')
#		] + ([args[0]] if len(args) > 0 else [])

#	 configFiles = config.read(configFiles)
#	 if len(configFiles) == 0:
#		 print >>sys.stderr, 'Unable to find the configuration file\n'
#		 sys.exit(3)

#	 return config

class TestClient(unittest.TestCase):
	@synchronized()
	def getPortNum(self):
		return self.portnum.next()

	"""macro test cases for single-host tests

	Assumes cwd is 'src/tashi/client/'
	"""
	def setUp(self):
		"""Create a CM and single NM on local host"""
		logging.info('setting up test')

		(self.config, self.configfiles) = getConfig([])

		self.port = 1717		# FIXME: take this (and other things) from config file
		self.portnum = incrementor(self.port)

		self.cwd = os.getcwd()
		self.srcd = os.path.dirname(os.path.dirname(self.cwd))

		self.environ = copy.copy(os.environ)
		self.environ['PYTHONPATH'] = self.srcd
		logging.info('base path = %s' % self.srcd)

		self.nm = os.spawnlpe(os.P_NOWAIT, 'python', 'python',
							  os.path.join(self.srcd, 'tashi', 'nodemanager', 'nodemanager.py'),
							  self.environ)
		self.cm = os.spawnlpe(os.P_WAIT, 'python', 'python',
							 os.path.join(self.srcd,  'tashi', 'clustermanager', 'clustermanager.py'),
							 '--drop', '--create',
							 os.path.expanduser('~/.tashi/ClusterManager.cfg'),
							 self.environ)
		self.cm = os.spawnlpe(os.P_NOWAIT, 'python', 'python',
							 os.path.join(self.srcd,  'tashi', 'clustermanager', 'clustermanager.py'),
							 os.path.expanduser('~/.tashi/ClusterManager.cfg'),
							 self.environ)
		# since we are spawning with P_NOWAIT, we need to sleep to ensure that the CM is listening
		time.sleep(1)
		try:
			self.connection = ClientConnection('localhost', self.config.get('ClusterManagerService', 'port'))
		except Exception, e:
			logging.warning('client connection failed')
			ex = None
			try:
				logging.warning("setUp killing node manager " + str(self.nm))
				os.kill(self.nm, signal.SIGKILL)
			except Exception, e:
				ex = e
				logging.warning('could not kill node manager: '+ str(e))
			try:
				logging.warning('setUp killing cluster manager ' + str(self.cm))
				os.kill(self.cm, signal.SIGKILL)
			except Exception, e:
				ex = e
				logging.warning('could not kill cluster manager: ' + str(e))
			if e != None:
				raise e

		logging.info('node manager PID: %i' % self.nm)
	def tearDown(self):
		'''Kill the CM and NM that were created by setUP'''
		logging.info('tearing down test')
		ex = None
		try:
			logging.debug("killing cluster manager " + str(self.cm))
			os.kill(self.cm, signal.SIGKILL)
		except Exception, e:
			ex = e
			logging.error('Could not kill cluster manager: ' + str(e))

		try:
			logging.debug("killing node manager " + str(self.nm))
			os.kill(self.nm, signal.SIGKILL)
		except Exception, e:
			ex = e
			logging.error('Could not kill node manager: ' + str(e))
		if ex != None:
			raise ex
	def testSetup(self):
		'''empty test to ensure that setUp code works'''
		logging.info('setting up')
	def testHostManagement(self):
		'''test adding/removing/listing hosts

		Right now this just adds a single host: localhost.  Eventually
		it should 1) take a list of hosts from a test configuration
		file, 2) ensure that all were added, 3) remove a random
		subset, 4) ensure that they were correctly removed, 5) remove
		all, 6) ensure that they were correctly removed.'''

		# get empty host list
		hosts = self.connection.client.getHosts()
		self.assertEqual(hosts, [], 'starting host list not empty: ' + str(hosts) )

		# add a host
		host = Host()
		host.hostname = 'localhost'
		host.enabled=True
		self.connection.client.addHost(host)
		hosts = self.connection.client.getHosts()
		self.assertEqual(len(hosts), 1, 'wrong number of hosts %i, should be %i' % (len(hosts), 1) )
		self.assertEqual(hosts[0].hostname, 'localhost', 'wrong hostname: ' + str(hosts[0].hostname) )

		# remove first host
		hid = hosts[0].id
		self.connection.client.removeHost(hid)
		hosts = self.connection.client.getHosts()
		self.assertEqual(hosts, [], 'host list not empty after remove: ' + str(hosts) )

	def testMessaging(self):
		'''test messaging system started by CM

		tests messages published directly, through events in the CM,
		and the log system'''
		# FIXME: add tests for generating events as a side-effect of
		# rpc commands, as well as logging in the CM
		portnum = self.getPortNum()
		self.sub = TestTashiSubscriber(self.config, portnum)
		self.assertEqual(self.sub.messageQueue.qsize(), 0)
		self.pub = tashi.messaging.thriftmessaging.PublisherThrift(self.config.get('MessageBroker', 'host'),
																   int(self.config.get('MessageBroker', 'port')))
		self.pub.publish({'message-type':'text', 'message':'Hello World!'})
		time.sleep(0.5)
		print '*** QSIZE', self.sub.messageQueue.qsize()
		self.assertEqual(self.sub.messageQueue.qsize(), 1)

		self.log = logging.getLogger(__name__)
		messageHandler = tashi.messaging.tashimessaging.TashiLogHandler(self.config)
		self.log.addHandler(messageHandler)
		# FIXME: why can't we log messages with severity below 'warning'?
		self.log.warning('test log message')
		time.sleep(0.5)
		self.assertEqual(self.sub.messageQueue.qsize(), 2)

		# This should generate at least one log message
#		 hosts = self.connection.client.getHosts()
#		 time.sleep(0.5)
#		 if (self.sub.messageQueue.qsize() <= 2):
#			 self.fail()

	def testUserManagement(self):
		'''test adding/removing/listing users

		same as testHostManagement, but with users'''
		usernames = ['sleepy', 'sneezy', 'dopey', 'doc',
					 'grumpy', 'bashful', 'happy']
		# add all users
		for un in usernames:
			user = User()
			user.username = un
			self.connection.client.addUser(user)
		# ensure that all were added
		users = self.connection.client.getUsers()
		self.assertEqual(len(usernames), len(users))
		for user in users:
			usernames.remove(user.username)
		self.assertEqual(0, len(usernames))
		# remove a random subset
		rm = random.sample(users, 4)
		for user in rm:
			self.connection.client.removeUser(user.id)
			users.remove(user)
		newUsers = self.connection.client.getUsers()
		# This ensures that the remaining ones are what we expect:
		for user in newUsers:
			# if there is a user remaining that we asked to be removed,
			# this will throw an exception
			users.remove(user)
		# if a user was removed that we did not intend, this will
		# throw an exception
		self.assertEqual(0, len(users))

#	 def testInstanceConfigurationManagement(self):
#		 '''test adding/removing/listing instance configurations

#		 same as testHostManagement, but with instance configurations'''
#		 self.fail('test not implemented')
	def testHardDiskConfigurationManagement(self):
		'''test adding/removing/listing hard disk configurations

		same as testHostManagement, but with hard disk configurations'''

		user = User(d={'username':'sleepy'})
		self.connection.client.addUser(user)
		users = self.connection.client.getUsers()

		per = PersistentImage()
		per.userId = users[0].id
		per.name = 'sleepy-PersistentImage'
		self.connection.client.addPersistentImage(per)
		pers = self.connection.client.getPersistentImages()

		inst = InstanceConfiguration()
		inst.name = 'sleepy-inst'
		inst.memory = 512
		inst.cores = 1
		self.connection.client.addInstanceConfiguration(inst)
		insts = self.connection.client.getInstanceConfigurations()

		hdc = HardDiskConfiguration()
		hdc.index = 0
		hdc.persistentImageId = pers[0].id
		hdc.persistent = False
		hdc.instanceConfigurationId = insts[0].id

#	 def testCreateDestroyShutdown(self):
#		 '''test creating/destroying/shutting down VMs

#		 not implemented'''
#		 self.fail('test not implemented')
#	 def testSuspendResume(self):
#		 '''test suspending/resuming VMs

#		 not implemented'''
#		 self.fail('test not implemented')
#	 def testMigrate(self):
#		 '''test migration

#		 not implemented'''
#		 self.fail('test not implemented')
#	 def testPauseUnpause(self):
#		 '''test pausing/unpausing VMs

#		 not implemented'''
#		 self.fail('test not implemented')

##############################
# Test Code
##############################
if __name__ == '__main__':
	logging.basicConfig(level=logging.NOTSET,
						format="%(asctime)s %(levelname)s:\t %(message)s",
						stream=sys.stdout)

	suite = unittest.TestLoader().loadTestsFromTestCase(TestClient)
	unittest.TextTestRunner(verbosity=2).run(suite)

<|endoftext|>


<commit_before>from cloudmesh.config.cm_config import cm_config_server
from cloudmesh.util.logger import LOGGER
from CMUserProviderBaseType import CMUserProviderBaseType
import ldap
import sys

# ----------------------------------------------------------------------
# SETTING UP A LOGGER
# ----------------------------------------------------------------------

log = LOGGER(__file__)

class cm_userLDAP (CMUserProviderBaseType):

    providers = {}
    host = None
    cert = None

    def get_config(self, **kwargs):

        if not kwargs.has_key('host'):  # if kwargs['host'] is None:
            self.host = cm_config_server().get("ldap.hostname")

        if not kwargs.has_key('ldapcert'):  # if kwargs['ldapcert'] is None:
            self.cert = cm_config_server().get("ldap.cert")

    def authenticate(self, userId, password, **kwargs):
        ret = False

        # bug pass **kwargs
        self.get_config()

        # print "'" + userId + "':'" + authProvider + "':'" + authCred + "'"

        # print adminuser, adminpass
        basedn = "ou=People,dc=futuregrid,dc=org"
        userdn = "uid={0},{1}".format(userId,basedn)
        # print userdn
        ldapconn = ldap.initialize("ldap://{0}".format(self.host))
        log.info("Initializing the LDAP connection to server: " + self.host)
        try:
            ldapconn.start_tls_s()
            log.info("tls started...")
            ldapconn.bind_s(userdn, password)
            ret = True
        except ldap.INVALID_CREDENTIALS:
            log.info("Your username or password is incorrect. Cannot bind.")
            ret = False
        except ldap.LDAPError:
            log.info("User '" + userId + "' failed to authenticate due to LDAP error. The user may not exist." + str(sys.exc_info()))
            ret = False
        except:
            ret = False
            log.info("User '" + userId + "' failed to authenticate due to possible password encryption error." + str(sys.exc_info()))
        finally:
            log.info("Unbinding from the LDAP.")
            ldapconn.unbind()
        return ret

    def __init__(self, collection="user"):
        super(cm_userLDAP, self).__init__()

    def refresh(self):
        '''
        refreshes the userdatabase from the user provider
        '''
        self._refresh()
        users = self.list()
        for user in users:
            data = self.get(user)
            self.updates(user, data)

    def connect(self, name, type, **kwargs):
        '''
        registers a provider with som parameters specified in the dict params

        :param name: the name of the provider
        :param type: the type of the provider, overwrites a possibly given type in params
        :param params: a dictionary describing wht needs to be poassed to the service that provides user information
        '''
        # kwargs host=None, ldapcert=None)
        provider = {'type': type,
                    'name': name}
        for k, v in kwargs.iteritems():
            provider[k] = v

        if not kwargs.has_key('host'):  # if kwargs['host'] is None:
            self.host = cm_config_server().get("ldap.hostname")

        if not kwargs.has_key('ldapcert'):  # if kwargs['ldapcert'] is None:
            self.cert = cm_config_server().get("ldap.cert")

        ldap.set_option(ldap.OPT_X_TLS_CACERTFILE, self.cert)
        # BUG IN FINAL VERSION MAKE SURE WE CHECK
        ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT,ldap.OPT_X_TLS_NEVER)

        self.ldapconn = ldap.initialize("ldap://" + self.host)
        self.ldapconn.start_tls_s()
        self.ldapconn.bind_s('', '')
        self.users = {}

    def disconnect(self):
        self.ldapconn.unbind()

    def __del__(self):
        self.disconnect()

    def get(self, username):
        '''
        returns the dict associated with the username
        :param username:
        '''
        return self.users[username]

    def list(self):
        '''
        returns a list with all usernames
        '''
        return self.users.keys()

    def _refresh(self):
        self._getUsers()
        self._getProjects()
        return self.users

    def _getUsers(self):
        # print "LDAP query..."
        try:
            # ldap constant
            ldapbasedn = "dc=futuregrid,dc=org"
            ldapscope = ldap.SCOPE_SUBTREE
            ldapfilter = "(objectclass=inetorgperson)"
            # info to be retrieved from ldap
            ldapattribs = ['uid',
                           'uidNumber',
                           'gidNumber',
                           'mail',
                           'givenName',
                           'sn',
                           'telephoneNumber',
                           'sshPublicKey']

            # retrieve all ldap users
            ldapresults = list(self.ldapconn.search_s(ldapbasedn, ldapscope, ldapfilter, ldapattribs))
            for ldapresult in ldapresults:
                # print ldapresult[1]
                # a result may not have 'mail' attribute, but valid account must have
                if ldapresult[1].has_key('mail'):
                    ldapmail = ldapresult[1]['mail'][0]
                    ldapuid = ldapresult[1]['uid'][0]
                    firstname = ldapresult[1]['givenName'][0]
                    lastname = ldapresult[1]['sn'][0]

                    keys = {}
                    if ldapresult[1].has_key('sshPublicKey'):
                        for sshkey in ldapresult[1]['sshPublicKey']:
                            if sshkey.strip():
                                (keytype, key, nickname) = (
                                    sshkey.strip().split(None, 2) + [''])[0:3]
                                keys[nickname] = "key %s" % sshkey

                    self.users[ldapuid] = {"firstname":firstname,
                                           "lastname":lastname,
                                           "projects":{"active":[]},
                                           "keys":keys }
        except:
            print "WRONG" + str(sys.exc_info())

    def _getProjects(self):
        try:
            ldapbasedn = "dc=futuregrid,dc=org"
            ldapscope = ldap.SCOPE_SUBTREE
            ldapfilter = "(&(objectclass=posixGroup)(cn=fg*))"
            ldapattribs = ['cn', 'memberUid']
            ldapresults = list(self.ldapconn.search_s(ldapbasedn, ldapscope, ldapfilter, ldapattribs))
            for ldapresult in ldapresults:
                if ldapresult[1].has_key('cn'):
                    cn = ldapresult[1]['cn'][0]
                    projid = cn[2:]
                    if ldapresult[1].has_key('memberUid'):
                        for auid in ldapresult[1]['memberUid']:
                            if self.users.has_key(auid):
                                if self.users[auid]["projects"]["active"] is not None:
                                    self.users[auid]["projects"]["active"].append(int(projid))
                                else:
                                    self.users[auid]["projects"]["active"] = [int(projid)]
        except:
            print "WRONG" + str(sys.exc_info())

def main():
    idp = cm_userLDAP (CMUserProviderBaseType)
    idp.connect("fg-ldap", "ldap")
    idp.refresh()
    users = idp.list()

    from pprint import pprint

    pprint(users)

    pprint(idp.users)

if __name__ == "__main__":
    main()
<commit_msg>Add user attributes<commit_after>from cloudmesh.config.cm_config import cm_config_server
from cloudmesh.util.logger import LOGGER
from CMUserProviderBaseType import CMUserProviderBaseType
import ldap
import sys

# ----------------------------------------------------------------------
# SETTING UP A LOGGER
# ----------------------------------------------------------------------

log = LOGGER(__file__)

class cm_userLDAP (CMUserProviderBaseType):

    providers = {}
    host = None
    cert = None

    def get_config(self, **kwargs):

        if not kwargs.has_key('host'):  # if kwargs['host'] is None:
            self.host = cm_config_server().get("ldap.hostname")

        if not kwargs.has_key('ldapcert'):  # if kwargs['ldapcert'] is None:
            self.cert = cm_config_server().get("ldap.cert")

    def authenticate(self, userId, password, **kwargs):
        ret = False

        # bug pass **kwargs
        self.get_config()

        # print "'" + userId + "':'" + authProvider + "':'" + authCred + "'"

        # print adminuser, adminpass
        basedn = "ou=People,dc=futuregrid,dc=org"
        userdn = "uid={0},{1}".format(userId,basedn)
        # print userdn
        ldapconn = ldap.initialize("ldap://{0}".format(self.host))
        log.info("Initializing the LDAP connection to server: " + self.host)
        try:
            ldapconn.start_tls_s()
            log.info("tls started...")
            ldapconn.bind_s(userdn, password)
            ret = True
        except ldap.INVALID_CREDENTIALS:
            log.info("Your username or password is incorrect. Cannot bind.")
            ret = False
        except ldap.LDAPError:
            log.info("User '" + userId + "' failed to authenticate due to LDAP error. The user may not exist." + str(sys.exc_info()))
            ret = False
        except:
            ret = False
            log.info("User '" + userId + "' failed to authenticate due to possible password encryption error." + str(sys.exc_info()))
        finally:
            log.info("Unbinding from the LDAP.")
            ldapconn.unbind()
        return ret

    def __init__(self, collection="user"):
        super(cm_userLDAP, self).__init__()

    def refresh(self):
        '''
        refreshes the userdatabase from the user provider
        '''
        self._refresh()
        users = self.list()
        for user in users:
            data = self.get(user)
            self.updates(user, data)

    def connect(self, name, type, **kwargs):
        '''
        registers a provider with som parameters specified in the dict params

        :param name: the name of the provider
        :param type: the type of the provider, overwrites a possibly given type in params
        :param params: a dictionary describing wht needs to be poassed to the service that provides user information
        '''
        # kwargs host=None, ldapcert=None)
        provider = {'type': type,
                    'name': name}
        for k, v in kwargs.iteritems():
            provider[k] = v

        if not kwargs.has_key('host'):  # if kwargs['host'] is None:
            self.host = cm_config_server().get("ldap.hostname")

        if not kwargs.has_key('ldapcert'):  # if kwargs['ldapcert'] is None:
            self.cert = cm_config_server().get("ldap.cert")

        ldap.set_option(ldap.OPT_X_TLS_CACERTFILE, self.cert)
        # BUG IN FINAL VERSION MAKE SURE WE CHECK
        ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT,ldap.OPT_X_TLS_NEVER)

        self.ldapconn = ldap.initialize("ldap://" + self.host)
        self.ldapconn.start_tls_s()
        self.ldapconn.bind_s('', '')
        self.users = {}

    def disconnect(self):
        self.ldapconn.unbind()

    def __del__(self):
        self.disconnect()

    def get(self, username):
        '''
        returns the dict associated with the username
        :param username:
        '''
        return self.users[username]

    def list(self):
        '''
        returns a list with all usernames
        '''
        return self.users.keys()

    def _refresh(self):
        self._getUsers()
        self._getProjects()
        return self.users

    def _getUsers(self):
        # print "LDAP query..."
        try:
            # ldap constant
            ldapbasedn = "dc=futuregrid,dc=org"
            ldapscope = ldap.SCOPE_SUBTREE
            ldapfilter = "(objectclass=inetorgperson)"
            # info to be retrieved from ldap
            ldapattribs = ['uid',
                           'uidNumber',
                           'gidNumber',
                           'mail',
                           'givenName',
                           'sn',
                           'telephoneNumber',
                           'address',
                           'sshPublicKey']

            # retrieve all ldap users
            ldapresults = list(self.ldapconn.search_s(ldapbasedn, ldapscope, ldapfilter, ldapattribs))
            for ldapresult in ldapresults:
                # print ldapresult[1]
                # a result may not have 'mail' attribute, but valid account must have
                if ldapresult[1].has_key('mail'):
                    ldapmail = ldapresult[1]['mail'][0]
                    ldapuid = ldapresult[1]['uid'][0]
                    ldapuidNumber = ldapresult[1]['uidNumber'][0]
                    ldapgidNumber = ldapresult[1]['gidNumber'][0]
                    firstname = ldapresult[1]['givenName'][0]
                    lastname = ldapresult[1]['sn'][0]
                    phone = ldapresult[1]['phone'][0]
                    address = 'TBD' # not currently in LDAP

                    keys = {}
                    if ldapresult[1].has_key('sshPublicKey'):
                        for sshkey in ldapresult[1]['sshPublicKey']:
                            if sshkey.strip():
                                (keytype, key, nickname) = (
                                    sshkey.strip().split(None, 2) + [''])[0:3]
                                keys[nickname] = "key %s" % sshkey

                    self.users[ldapuid] = {"firstname":firstname,
                                           "lastname":lastname,
                                           "uidNumber":ldapuidNumber,
                                           "gidNumber":ldapgidNumber,
                                           "phone":phone,
                                           "email":ldapmail,
                                           "address":address,
                                           "projects":{"active":[]},
                                           "keys":keys }
        except:
            print "WRONG" + str(sys.exc_info())

    def _getProjects(self):
        try:
            ldapbasedn = "dc=futuregrid,dc=org"
            ldapscope = ldap.SCOPE_SUBTREE
            ldapfilter = "(&(objectclass=posixGroup)(cn=fg*))"
            ldapattribs = ['cn', 'memberUid']
            ldapresults = list(self.ldapconn.search_s(ldapbasedn, ldapscope, ldapfilter, ldapattribs))
            for ldapresult in ldapresults:
                if ldapresult[1].has_key('cn'):
                    cn = ldapresult[1]['cn'][0]
                    projid = cn[2:]
                    if ldapresult[1].has_key('memberUid'):
                        for auid in ldapresult[1]['memberUid']:
                            if self.users.has_key(auid):
                                if self.users[auid]["projects"]["active"] is not None:
                                    self.users[auid]["projects"]["active"].append(int(projid))
                                else:
                                    self.users[auid]["projects"]["active"] = [int(projid)]
        except:
            print "WRONG" + str(sys.exc_info())

def main():
    idp = cm_userLDAP (CMUserProviderBaseType)
    idp.connect("fg-ldap", "ldap")
    idp.refresh()
    users = idp.list()

    from pprint import pprint

    pprint(users)

    pprint(idp.users)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from __future__ import print_function

import os
import numpy as np
import logging
import argparse

from keras.preprocessing import sequence
from keras.utils import np_utils
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, TimeDistributed, TimeDistributedDense
from keras.layers import LSTM
from keras.callbacks import ModelCheckpoint, ProgbarLogger, Callback

BATCH_SIZE = 64
NUM_EPOCHS = 5
INPUT_DIM = 14
INPUT_LENGTH = 20
OUTPUT_DIM = 3
INTER_DIM = (20, 10)
TRAINING_DATA_DIR = "training_data"
USE_ONE_TRAINING_FILE = False
TRAIN_DATA_RATIO = 0.75 # Amount of total data to use for training

np.random.seed(1337)  # for reproducibility

logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO)

class PrintLoss(Callback):
    def __init__(self, pad_ratio):
        self.pad_ratio = pad_ratio

    def on_epoch_end(self, epoch, logs={}):
        logging.info('True Loss: {0}'.format(logs.get('loss') * self.pad_ratio))

# NUM_SAMPLES / NUM_NON_PADDED_SAMPLES
def pad_ratio(y):
    return float(y.shape[0] * y.shape[1]) / np.sum(y, (0,1,2))

def load_training_data():
    logging.info('Loading data...')

    # Only for testing
    if USE_ONE_TRAINING_FILE:
        filename = "training_data/training_0.npz";
        with open(filename) as f:
            data = np.load(f)
            X_train = data["input"]
            y_train = data["output"]
    else:
        Xs = []
        ys = []
        for filename in os.listdir(TRAINING_DATA_DIR):
            full_name = TRAINING_DATA_DIR + "/" + filename;
            with open(full_name) as f:
                data = np.load(f)
                Xs.append(data["input"])
                ys.append(data["output"])
        X_train = np.concatenate(tuple(Xs))
        y_train = np.concatenate(tuple(ys))

    logging.info("Shape of X_train: ")
    logging.info(X_train.shape)
    logging.info("Shape of y_train")
    logging.info(y_train.shape)

    train_test_sep_idx = int(X_train.shape[0] * TRAIN_DATA_RATIO)
    X_test = X_train[train_test_sep_idx:]
    y_test = y_train[train_test_sep_idx:]
    X_train = X_train[:train_test_sep_idx]
    y_train = y_train[:train_test_sep_idx]

    return X_train, y_train, X_test, y_test

def build_model(processor):
    logging.info('Build model...')
    model = Sequential()

    model.add(LSTM(INTER_DIM[0], return_sequences=True, dropout_W=0.2, dropout_U=0.2,
                   input_length=INPUT_LENGTH, input_dim=INPUT_DIM, consume_less=processor))
    model.add(LSTM(INTER_DIM[1], return_sequences=True, dropout_W=0.2, dropout_U=0.2,
                   input_length=INPUT_LENGTH, input_dim=INTER_DIM[0], consume_less=processor))
    model.add(TimeDistributed(Dense(OUTPUT_DIM)))

    model.add(Activation('softmax'))

    # try using different optimizers and different optimizer configs
    model.compile(loss='categorical_crossentropy',
                  optimizer='rmsprop',
                  metrics=['accuracy'])

    # plot(model, to_file='model.png', show_shapes=True, show_layer_names=True)
    return model

def train(model, X_train, y_train, X_test, y_test, start_weights_file=None):
    logging.info('Train...')
    save_weights = ModelCheckpoint('weights.{epoch:02d}.hdf5')
    logger = ProgbarLogger()
    printloss = PrintLoss(pad_ratio(y_test))

    if start_weights_file:
        model.load_weights(start_weights_file)

    model.fit(X_train, y_train, batch_size=BATCH_SIZE, nb_epoch=NUM_EPOCHS,
              validation_data=(X_test, y_test), callbacks=[save_weights, logger, printloss])

    score, acc = model.evaluate(X_test, y_test,
                                batch_size=BATCH_SIZE)
    # logging.info('Test score: {0}'.format(score))
    # logging.info('Test accuracy: {0}'.format(acc))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Train Poker Predictor.')
    parser.add_argument('--file', type=str, help='sum the integers (default: find the max)')
    parser.add_argument('--gpu', const='gpu', default='cpu', nargs="?",
                        help='sum the integers (default: find the max)')

    args = parser.parse_args()

    X_train, y_train, X_test, y_test = load_training_data()
    logging.info("Padding ratio (multiply loss by this): ")
    logging.info(pad_ratio(y_test))

    model = build_model(args.gpu)
    train(model, X_train, y_train, X_test, y_test, start_weights_file=args.file)
<commit_msg>Added random shuffling to production of training and test data.<commit_after>from __future__ import print_function

import os
import numpy as np
import logging
import argparse

from keras.preprocessing import sequence
from keras.utils import np_utils
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, TimeDistributed, TimeDistributedDense
from keras.layers import LSTM
from keras.callbacks import ModelCheckpoint, ProgbarLogger, Callback

BATCH_SIZE = 64
NUM_EPOCHS = 5
INPUT_DIM = 14
INPUT_LENGTH = 20
OUTPUT_DIM = 3
INTER_DIM = (20, 10)
TRAINING_DATA_DIR = "training_data"
USE_ONE_TRAINING_FILE = False
TRAIN_DATA_RATIO = 0.75 # Amount of total data to use for training

np.random.seed(1337)  # for reproducibility

logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO)

class PrintLoss(Callback):
    def __init__(self, pad_ratio):
        self.pad_ratio = pad_ratio

    def on_epoch_end(self, epoch, logs={}):
        logging.info('True Loss: {0}'.format(logs.get('loss') * self.pad_ratio))

# NUM_SAMPLES / NUM_NON_PADDED_SAMPLES
def pad_ratio(y):
    return float(y.shape[0] * y.shape[1]) / np.sum(y, (0,1,2))

def load_training_data():
    logging.info('Loading data...')

    # Only for testing
    if USE_ONE_TRAINING_FILE:
        filename = "training_data/training_0.npz";
        with open(filename) as f:
            data = np.load(f)
            X_train = data["input"]
            y_train = data["output"]
    else:
        Xs = []
        ys = []
        for filename in os.listdir(TRAINING_DATA_DIR):
            full_name = TRAINING_DATA_DIR + "/" + filename;
            with open(full_name) as f:
                data = np.load(f)
                Xs.append(data["input"])
                ys.append(data["output"])
        X_train = np.concatenate(tuple(Xs))
        y_train = np.concatenate(tuple(ys))

    logging.info("Shape of X_train: ")
    logging.info(X_train.shape)
    logging.info("Shape of y_train")
    logging.info(y_train.shape)

    # Randomize hands
    rand_perm = np.random.permutation(X_train.shape[0])
    X_train = X_train[rand_perm]
    y_train = y_train[rand_perm]

    train_test_sep_idx = int(X_train.shape[0] * TRAIN_DATA_RATIO)
    X_test = X_train[train_test_sep_idx:]
    y_test = y_train[train_test_sep_idx:]
    X_train = X_train[:train_test_sep_idx]
    y_train = y_train[:train_test_sep_idx]

    return X_train, y_train, X_test, y_test

def build_model(processor):
    logging.info('Build model...')
    model = Sequential()

    model.add(LSTM(INTER_DIM[0], return_sequences=True, dropout_W=0.2, dropout_U=0.2,
                   input_length=INPUT_LENGTH, input_dim=INPUT_DIM, consume_less=processor))
    model.add(LSTM(INTER_DIM[1], return_sequences=True, dropout_W=0.2, dropout_U=0.2,
                   input_length=INPUT_LENGTH, input_dim=INTER_DIM[0], consume_less=processor))
    model.add(TimeDistributed(Dense(OUTPUT_DIM)))

    model.add(Activation('softmax'))

    # try using different optimizers and different optimizer configs
    model.compile(loss='categorical_crossentropy',
                  optimizer='rmsprop',
                  metrics=['accuracy'])

    # plot(model, to_file='model.png', show_shapes=True, show_layer_names=True)
    return model

def train(model, X_train, y_train, X_test, y_test, start_weights_file=None):
    logging.info('Train...')
    save_weights = ModelCheckpoint('weights.{epoch:02d}.hdf5')
    logger = ProgbarLogger()
    printloss = PrintLoss(pad_ratio(y_test))

    if start_weights_file:
        model.load_weights(start_weights_file)

    model.fit(X_train, y_train, batch_size=BATCH_SIZE, nb_epoch=NUM_EPOCHS,
              validation_data=(X_test, y_test), callbacks=[save_weights, logger, printloss])

    score, acc = model.evaluate(X_test, y_test,
                                batch_size=BATCH_SIZE)
    # logging.info('Test score: {0}'.format(score))
    # logging.info('Test accuracy: {0}'.format(acc))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Train Poker Predictor.')
    parser.add_argument('--file', type=str, help='sum the integers (default: find the max)')
    parser.add_argument('--gpu', const='gpu', default='cpu', nargs="?",
                        help='sum the integers (default: find the max)')

    args = parser.parse_args()

    X_train, y_train, X_test, y_test = load_training_data()
    logging.info("Padding ratio (multiply loss by this): ")
    logging.info(pad_ratio(y_test))

    model = build_model(args.gpu)
    train(model, X_train, y_train, X_test, y_test, start_weights_file=args.file)
<|endoftext|>


<commit_before>import subprocess
import requests
import tempfile
import tarfile
import shutil
import sys
import os

if os.name != "nt":
    print("Error: Can only build pillow-simd on a Windows system")

repo_url = "https://api.github.com/repos/uploadcare/pillow-simd/tags"
tarball_url = requests.get(repo_url).json()[0]["tarball_url"]
r = requests.head(tarball_url, allow_redirects=True)
for x in r.headers["content-disposition"].split(";"):
    if "filename=" in x:
        tarball_fn = os.path.join(tempfile.gettempdir(), x.strip()[9:])

if not os.path.isfile(tarball_fn):
    for fp in os.scandir():
        if "pillow-simd" in fp.name and fp.name.endswith(".tar.gz"):
            print("Removing outdated tarball {}".format(fp.name))
            os.remove(fp.path)
    print("Downloading tarball {}...".format(tarball_fn))
    r = requests.get(tarball_url, stream=True)
    with open(tarball_fn, 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:  # filter out keep-alive new chunks
                f.write(chunk)
else:
    print("Using cached tarball {}".format(tarball_fn))

if os.path.isdir("pillow-simd"):
    shutil.rmtree("pillow-simd")

with tarfile.open(tarball_fn) as pkg_tar:
    for fp in pkg_tar:
        if fp.isfile():
            outp = os.path.join("pillow-simd", *fp.name.split("/")[1:])
            print("{} -> {}".format(fp.name, outp))
            os.makedirs(os.path.dirname(outp), exist_ok=True)
            with open(outp, "wb") as out:
                out.write(pkg_tar.extractfile(fp).read())

# no easy way to specify on the command line to not build default libs
with open("pillow-simd/setup.py") as infile, open("pillow-simd/setup.py.tmp", "w") as outfile:
    for line in infile:
        if line.startswith("        required ="):
            outfile.write("        required = set()\n")
        else:
            outfile.write(line)
shutil.move("pillow-simd/setup.py.tmp", "pillow-simd/setup.py")

owd = os.getcwd()
os.chdir("pillow-simd")
subprocess.run([sys.executable, "setup.py", "build"], check=True)
os.chdir(owd)

for fp in os.scandir("pillow-simd/build"):
    if fp.is_dir() and fp.name.startswith("lib."):
        build_dir = fp.path
        break

bitness = 64 if "amd64" in os.path.basename(build_dir) else 32

with tarfile.open("pillow-simd-{}bit.tar.gz".format(bitness), "w") as out_tar:
    for root, dirs, files in os.walk(build_dir):
        for fp in (os.path.join(root, p) for p in files):
            outp = os.path.relpath(fp, build_dir)
            print("{} -> {}".format(fp, outp))
            out_tar.add(fp, outp)

shutil.rmtree("pillow-simd")
# os.remove(tarball_fn)
<commit_msg>Add correct SSE flag for VC++<commit_after>import subprocess
import requests
import tempfile
import tarfile
import shutil
import sys
import os

if os.name != "nt":
    print("Error: Can only build pillow-simd on a Windows system")

repo_url = "https://api.github.com/repos/uploadcare/pillow-simd/tags"
tarball_url = requests.get(repo_url).json()[0]["tarball_url"]
r = requests.head(tarball_url, allow_redirects=True)
for x in r.headers["content-disposition"].split(";"):
    if "filename=" in x:
        tarball_fn = os.path.join(tempfile.gettempdir(), x.strip()[9:])

if not os.path.isfile(tarball_fn):
    for fp in os.scandir():
        if "pillow-simd" in fp.name and fp.name.endswith(".tar.gz"):
            print("Removing outdated tarball {}".format(fp.name))
            os.remove(fp.path)
    print("Downloading tarball {}...".format(tarball_fn))
    r = requests.get(tarball_url, stream=True)
    with open(tarball_fn, 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:  # filter out keep-alive new chunks
                f.write(chunk)
else:
    print("Using cached tarball {}".format(tarball_fn))

if os.path.isdir("pillow-simd"):
    shutil.rmtree("pillow-simd")

with tarfile.open(tarball_fn) as pkg_tar:
    for fp in pkg_tar:
        if fp.isfile():
            outp = os.path.join("pillow-simd", *fp.name.split("/")[1:])
            print("{} -> {}".format(fp.name, outp))
            os.makedirs(os.path.dirname(outp), exist_ok=True)
            with open(outp, "wb") as out:
                out.write(pkg_tar.extractfile(fp).read())

# no easy way to specify on the command line to not build default libs
with open("pillow-simd/setup.py") as infile, open("pillow-simd/setup.py.tmp", "w") as outfile:
    for line in infile:
        if line.startswith("        required ="):
            outfile.write("        required = set()\n")
        else:
            # visual c++ compiler seems to enable sse4 too with sse2 enabled
            outfile.write(line.replace("-msse4", "/arch:SSE2"))
shutil.move("pillow-simd/setup.py.tmp", "pillow-simd/setup.py")

owd = os.getcwd()
os.chdir("pillow-simd")
subprocess.run([sys.executable, "setup.py", "build"], check=True)
os.chdir(owd)

for fp in os.scandir("pillow-simd/build"):
    if fp.is_dir() and fp.name.startswith("lib."):
        build_dir = fp.path
        break

bitness = 64 if "amd64" in os.path.basename(build_dir) else 32

with tarfile.open("pillow-simd-{}bit.tar.gz".format(bitness), "w") as out_tar:
    for root, dirs, files in os.walk(build_dir):
        for fp in (os.path.join(root, p) for p in files):
            outp = os.path.relpath(fp, build_dir)
            print("{} -> {}".format(fp, outp))
            out_tar.add(fp, outp)

shutil.rmtree("pillow-simd")
<|endoftext|>


<commit_before># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import urllib2
import httplib
import socket
import json
import re
import sys

from telemetry.core import util
from telemetry.core import exceptions
from telemetry.core import user_agent
from telemetry.core import wpr_modes
from telemetry.core import wpr_server
from telemetry.core.chrome import extension_dict_backend
from telemetry.core.chrome import tab_list_backend
from telemetry.core.chrome import tracing_backend
from telemetry.core.chrome import misc_web_contents_backend
from telemetry.test import options_for_unittests

class ExtensionsNotSupportedException(Exception):
  pass

class BrowserBackend(object):
  """A base class for browser backends. Provides basic functionality
  once a remote-debugger port has been established."""

  WEBPAGEREPLAY_HOST = '127.0.0.1'

  def __init__(self, is_content_shell, supports_extensions, options):
    self.browser_type = options.browser_type
    self.is_content_shell = is_content_shell
    self._supports_extensions = supports_extensions
    self.options = options
    self._browser = None
    self._port = None

    self._inspector_protocol_version = 0
    self._chrome_branch_number = 0
    self._webkit_base_revision = 0
    self._tracing_backend = None

    self.webpagereplay_local_http_port = util.GetAvailableLocalPort()
    self.webpagereplay_local_https_port = util.GetAvailableLocalPort()
    self.webpagereplay_remote_http_port = self.webpagereplay_local_http_port
    self.webpagereplay_remote_https_port = self.webpagereplay_local_https_port

    if options.dont_override_profile and not options_for_unittests.AreSet():
      sys.stderr.write('Warning: Not overriding profile. This can cause '
                       'unexpected effects due to profile-specific settings, '
                       'such as about:flags settings, cookies, and '
                       'extensions.\n')
    self._misc_web_contents_backend = (
        misc_web_contents_backend.MiscWebContentsBackend(self))
    self._tab_list_backend = tab_list_backend.TabListBackend(self)
    self._extension_dict_backend = None
    if supports_extensions:
      self._extension_dict_backend = (
          extension_dict_backend.ExtensionDictBackend(self))

  def SetBrowser(self, browser):
    self._browser = browser
    self._tab_list_backend.Init()

  @property
  def browser(self):
    return self._browser

  @property
  def supports_extensions(self):
    """True if this browser backend supports extensions."""
    return self._supports_extensions

  @property
  def misc_web_contents_backend(self):
    """Access to chrome://oobe/login page which is neither a tab nor an
    extension."""
    return self._misc_web_contents_backend

  @property
  def tab_list_backend(self):
    return self._tab_list_backend

  @property
  def extension_dict_backend(self):
    return self._extension_dict_backend

  def GetBrowserStartupArgs(self):
    args = []
    args.extend(self.options.extra_browser_args)
    args.append('--disable-background-networking')
    args.append('--metrics-recording-only')
    args.append('--no-first-run')
    if self.options.wpr_mode != wpr_modes.WPR_OFF:
      args.extend(wpr_server.GetChromeFlags(
          self.WEBPAGEREPLAY_HOST,
          self.webpagereplay_remote_http_port,
          self.webpagereplay_remote_https_port))
    args.extend(user_agent.GetChromeUserAgentArgumentFromType(
        self.options.browser_user_agent_type))

    extensions = [extension.local_path for extension in
                  self.options.extensions_to_load if not extension.is_component]
    extension_str = ','.join(extensions)
    if len(extensions) > 0:
      args.append('--load-extension=%s' % extension_str)

    component_extensions = [extension.local_path for extension in
                  self.options.extensions_to_load if extension.is_component]
    component_extension_str = ','.join(component_extensions)
    if len(component_extensions) > 0:
      args.append('--load-component-extension=%s' % component_extension_str)
    return args

  @property
  def wpr_mode(self):
    return self.options.wpr_mode

  def _WaitForBrowserToComeUp(self, timeout=None):
    def IsBrowserUp():
      try:
        self.Request('', timeout=timeout)
      except (socket.error, httplib.BadStatusLine, urllib2.URLError):
        return False
      else:
        return True
    try:
      util.WaitFor(IsBrowserUp, timeout=30)
    except util.TimeoutException:
      raise exceptions.BrowserGoneException()

    def AllExtensionsLoaded():
      for e in self.options.extensions_to_load:
        if not e.extension_id in self._extension_dict_backend:
          return False
        extension_object = self._extension_dict_backend[e.extension_id]
        extension_object.WaitForDocumentReadyStateToBeInteractiveOrBetter()
      return True
    if self._supports_extensions:
      util.WaitFor(AllExtensionsLoaded, timeout=30)

  def _PostBrowserStartupInitialization(self):
    # Detect version information.
    data = self.Request('version')
    resp = json.loads(data)
    if 'Protocol-Version' in resp:
      self._inspector_protocol_version = resp['Protocol-Version']
      if 'Browser' in resp:
        branch_number_match = re.search('Chrome/\d+\.\d+\.(\d+)\.\d+',
                                        resp['Browser'])
      else:
        branch_number_match = re.search(
            'Chrome/\d+\.\d+\.(\d+)\.\d+ (Mobile )?Safari',
            resp['User-Agent'])
      webkit_version_match = re.search('\((trunk)?\@(\d+)\)',
                                       resp['WebKit-Version'])

      if branch_number_match:
        self._chrome_branch_number = int(branch_number_match.group(1))
      else:
        # Content Shell returns '' for Browser, for now we have to
        # fall-back and assume branch 1025.
        self._chrome_branch_number = 1025

      if webkit_version_match:
        self._webkit_base_revision = int(webkit_version_match.group(2))
      return

    # Detection has failed: assume 18.0.1025.168 ~= Chrome Android.
    self._inspector_protocol_version = 1.0
    self._chrome_branch_number = 1025
    self._webkit_base_revision = 106313

  def Request(self, path, timeout=None):
    url = 'http://localhost:%i/json' % self._port
    if path:
      url += '/' + path
    req = urllib2.urlopen(url, timeout=timeout)
    return req.read()

  @property
  def chrome_branch_number(self):
    return self._chrome_branch_number

  @property
  def supports_tab_control(self):
    return self._chrome_branch_number >= 1303

  @property
  def supports_tracing(self):
    return self.is_content_shell or self._chrome_branch_number >= 1385

  def StartTracing(self):
    if self._tracing_backend is None:
      self._tracing_backend = tracing_backend.TracingBackend(self._port)
    self._tracing_backend.BeginTracing()

  def StopTracing(self):
    self._tracing_backend.EndTracing()

  def GetTraceResultAndReset(self):
    return self._tracing_backend.GetTraceResultAndReset()

  def GetRemotePort(self, _):
    return util.GetAvailableLocalPort()

  def Close(self):
    if self._tracing_backend:
      self._tracing_backend.Close()
      self._tracing_backend = None

  def CreateForwarder(self, *port_pairs):
    raise NotImplementedError()

  def IsBrowserRunning(self):
    raise NotImplementedError()

  def GetStandardOutput(self):
    raise NotImplementedError()

class DoNothingForwarder(object):
  def __init__(self, *port_pairs):
    self._host_port = port_pairs[0].local_port

  @property
  def url(self):
    assert self._host_port
    return 'http://127.0.0.1:%i' % self._host_port

  def Close(self):
    self._host_port = None
<commit_msg>Join load-extension arguments using semicolons instead of commas due to an issue with dbus-send not tokenizing commas correctly<commit_after># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import urllib2
import httplib
import socket
import json
import re
import sys

from telemetry.core import util
from telemetry.core import exceptions
from telemetry.core import user_agent
from telemetry.core import wpr_modes
from telemetry.core import wpr_server
from telemetry.core.chrome import extension_dict_backend
from telemetry.core.chrome import tab_list_backend
from telemetry.core.chrome import tracing_backend
from telemetry.core.chrome import misc_web_contents_backend
from telemetry.test import options_for_unittests

class ExtensionsNotSupportedException(Exception):
  pass

class BrowserBackend(object):
  """A base class for browser backends. Provides basic functionality
  once a remote-debugger port has been established."""

  WEBPAGEREPLAY_HOST = '127.0.0.1'

  def __init__(self, is_content_shell, supports_extensions, options):
    self.browser_type = options.browser_type
    self.is_content_shell = is_content_shell
    self._supports_extensions = supports_extensions
    self.options = options
    self._browser = None
    self._port = None

    self._inspector_protocol_version = 0
    self._chrome_branch_number = 0
    self._webkit_base_revision = 0
    self._tracing_backend = None

    self.webpagereplay_local_http_port = util.GetAvailableLocalPort()
    self.webpagereplay_local_https_port = util.GetAvailableLocalPort()
    self.webpagereplay_remote_http_port = self.webpagereplay_local_http_port
    self.webpagereplay_remote_https_port = self.webpagereplay_local_https_port

    if options.dont_override_profile and not options_for_unittests.AreSet():
      sys.stderr.write('Warning: Not overriding profile. This can cause '
                       'unexpected effects due to profile-specific settings, '
                       'such as about:flags settings, cookies, and '
                       'extensions.\n')
    self._misc_web_contents_backend = (
        misc_web_contents_backend.MiscWebContentsBackend(self))
    self._tab_list_backend = tab_list_backend.TabListBackend(self)
    self._extension_dict_backend = None
    if supports_extensions:
      self._extension_dict_backend = (
          extension_dict_backend.ExtensionDictBackend(self))

  def SetBrowser(self, browser):
    self._browser = browser
    self._tab_list_backend.Init()

  @property
  def browser(self):
    return self._browser

  @property
  def supports_extensions(self):
    """True if this browser backend supports extensions."""
    return self._supports_extensions

  @property
  def misc_web_contents_backend(self):
    """Access to chrome://oobe/login page which is neither a tab nor an
    extension."""
    return self._misc_web_contents_backend

  @property
  def tab_list_backend(self):
    return self._tab_list_backend

  @property
  def extension_dict_backend(self):
    return self._extension_dict_backend

  def GetBrowserStartupArgs(self):
    args = []
    args.extend(self.options.extra_browser_args)
    args.append('--disable-background-networking')
    args.append('--metrics-recording-only')
    args.append('--no-first-run')
    if self.options.wpr_mode != wpr_modes.WPR_OFF:
      args.extend(wpr_server.GetChromeFlags(
          self.WEBPAGEREPLAY_HOST,
          self.webpagereplay_remote_http_port,
          self.webpagereplay_remote_https_port))
    args.extend(user_agent.GetChromeUserAgentArgumentFromType(
        self.options.browser_user_agent_type))

    extensions = [extension.local_path for extension in
                  self.options.extensions_to_load if not extension.is_component]
    # join extension paths with ';' rather than ',' due
    # to a tokenization issue with dbus-send
    extension_str = ';'.join(extensions)
    if len(extensions) > 0:
      args.append('--load-extension=%s' % extension_str)

    component_extensions = [extension.local_path for extension in
                  self.options.extensions_to_load if extension.is_component]
    component_extension_str = ','.join(component_extensions)
    if len(component_extensions) > 0:
      args.append('--load-component-extension=%s' % component_extension_str)
    return args

  @property
  def wpr_mode(self):
    return self.options.wpr_mode

  def _WaitForBrowserToComeUp(self, timeout=None):
    def IsBrowserUp():
      try:
        self.Request('', timeout=timeout)
      except (socket.error, httplib.BadStatusLine, urllib2.URLError):
        return False
      else:
        return True
    try:
      util.WaitFor(IsBrowserUp, timeout=30)
    except util.TimeoutException:
      raise exceptions.BrowserGoneException()

    def AllExtensionsLoaded():
      for e in self.options.extensions_to_load:
        if not e.extension_id in self._extension_dict_backend:
          return False
        extension_object = self._extension_dict_backend[e.extension_id]
        extension_object.WaitForDocumentReadyStateToBeInteractiveOrBetter()
      return True
    if self._supports_extensions:
      util.WaitFor(AllExtensionsLoaded, timeout=30)

  def _PostBrowserStartupInitialization(self):
    # Detect version information.
    data = self.Request('version')
    resp = json.loads(data)
    if 'Protocol-Version' in resp:
      self._inspector_protocol_version = resp['Protocol-Version']
      if 'Browser' in resp:
        branch_number_match = re.search('Chrome/\d+\.\d+\.(\d+)\.\d+',
                                        resp['Browser'])
      else:
        branch_number_match = re.search(
            'Chrome/\d+\.\d+\.(\d+)\.\d+ (Mobile )?Safari',
            resp['User-Agent'])
      webkit_version_match = re.search('\((trunk)?\@(\d+)\)',
                                       resp['WebKit-Version'])

      if branch_number_match:
        self._chrome_branch_number = int(branch_number_match.group(1))
      else:
        # Content Shell returns '' for Browser, for now we have to
        # fall-back and assume branch 1025.
        self._chrome_branch_number = 1025

      if webkit_version_match:
        self._webkit_base_revision = int(webkit_version_match.group(2))
      return

    # Detection has failed: assume 18.0.1025.168 ~= Chrome Android.
    self._inspector_protocol_version = 1.0
    self._chrome_branch_number = 1025
    self._webkit_base_revision = 106313

  def Request(self, path, timeout=None):
    url = 'http://localhost:%i/json' % self._port
    if path:
      url += '/' + path
    req = urllib2.urlopen(url, timeout=timeout)
    return req.read()

  @property
  def chrome_branch_number(self):
    return self._chrome_branch_number

  @property
  def supports_tab_control(self):
    return self._chrome_branch_number >= 1303

  @property
  def supports_tracing(self):
    return self.is_content_shell or self._chrome_branch_number >= 1385

  def StartTracing(self):
    if self._tracing_backend is None:
      self._tracing_backend = tracing_backend.TracingBackend(self._port)
    self._tracing_backend.BeginTracing()

  def StopTracing(self):
    self._tracing_backend.EndTracing()

  def GetTraceResultAndReset(self):
    return self._tracing_backend.GetTraceResultAndReset()

  def GetRemotePort(self, _):
    return util.GetAvailableLocalPort()

  def Close(self):
    if self._tracing_backend:
      self._tracing_backend.Close()
      self._tracing_backend = None

  def CreateForwarder(self, *port_pairs):
    raise NotImplementedError()

  def IsBrowserRunning(self):
    raise NotImplementedError()

  def GetStandardOutput(self):
    raise NotImplementedError()

class DoNothingForwarder(object):
  def __init__(self, *port_pairs):
    self._host_port = port_pairs[0].local_port

  @property
  def url(self):
    assert self._host_port
    return 'http://127.0.0.1:%i' % self._host_port

  def Close(self):
    self._host_port = None
<|endoftext|>


<commit_before>__author__ = 'vivek'
import os
import sys
import radical.pilot

DBURL = "mongodb://ec2-184-72-89-141.compute-1.amazonaws.com:27017"
RCONF  = ["https://raw.github.com/radical-cybertools/radical.pilot/master/configs/xsede.json"]

#------------------------------------------------------------------------------
#
def pilot_state_cb(pilot, state):
    """pilot_state_change_cb() is a callback function. It gets called very
    time a ComputePilot changes its state.
    """
    print "[Callback]: ComputePilot '{0}' state changed to {1}.".format(
        pilot.uid, state)

    if state == radical.pilot.states.FAILED:
        sys.exit(1)

#------------------------------------------------------------------------------
#
def unit_state_change_cb(unit, state):
    """unit_state_change_cb() is a callback function. It gets called very
    time a ComputeUnit changes its state.
    """
    print "[Callback]: ComputeUnit '{0}' state changed to {1}.".format(
        unit.uid, state)
    if state == radical.pilot.states.FAILED:
        print "            Log: %s" % unit.log[-1]

#------------------------------------------------------------------------------
#

if __name__ == '__main__':
    print "Starting 16 core pilot-job..."
    session = radical.pilot.Session(database_url=DBURL)
    print "Session UID: {0} ".format(session.uid)

    # Add an ssh identity to the session.
    cred = radical.pilot.Context('ssh')
    cred.user_id = 'vivek91'
    session.add_context(cred)

    # Add a Pilot Manager. Pilot managers manage one or more ComputePilots.
    pmgr = radical.pilot.PilotManager(session=session)

    pmgr.register_callback(pilot_state_cb)

    # Start a pilot at the remote host as per the configs
    pdesc = radical.pilot.ComputePilotDescription()
    pdesc.resource = 'stampede.tacc.utexas.edu'
    pdesc.runtime = 20
    pdesc.cores = 16

    pilot = pmgr.submit_pilots(pdesc)

    print "Pilot UID       : {0} ".format(pilot.uid )

    umgr = radical.pilot.UnitManager(session=session, scheduler=radical.pilot.SCHED_DIRECT_SUBMISSION)
    # Register our callback with the UnitManager. This callback will get
    # called every time any of the units managed by the UnitManager
    # change their state.
    umgr.register_callback(unit_state_change_cb)

    # Add the previously created ComputePilot to the UnitManager.
    umgr.add_pilots(pilot)

    print "... DONE."

    #------------------------------------------------------------------------------
    #

    #
    # All simulations use the same mdin file.
    #
    dict = {}
    dict['crdfile'] = 'penta.crd'
    dict['topfile'] = 'penta.top'
    dict['mdin']    = 'mdshort.in'
    dict['minin']   = 'min.in'

    # set number of cycles, and number of replicates
    maxcycles = 10
    nreps = 8
    coco_loc = '$HOME/coco/'
    #
    # Main loop begins here:
    #rep00/md0.crd
    for cycle in range(maxcycles):
        dict['cycle'] = str(cycle)
        if cycle == 0:
    #
    # cycle 0: we begin with a single coordinate and topology file.
    # Copy it into the separate rep0? directories ready to run the MD simulations.
    #
            print "creating initial setup"
            #cp = script.Script()
            #for rep in range(nreps):
            #    dict['rep'] = str(rep)
            ##    dict['path'] = os.getcwd()
             #   cp.append('cp {crdfile} {path}/rep0{rep}/md0.crd'.format(**dict))
            #    cp.run('tcsh -f {}')

            cudesc = radical.pilot.ComputeUnitDescription()
            cudesc.executable = '/bin/bash'
            cudesc.cores = 1
            cudesc.pre_exec = ['cd %s/examples' % coco_loc, '( test -d "rep0*" || rm rep0* -rf)' , 'mkdir rep00 rep01 rep02 rep03 rep04 rep05 rep06 rep07' ]
            cudesc.arguments = ['-l','-c','tee rep00/md0.crd rep01/md0.crd rep02/md0.crd rep03/md0.crd rep04/md0.crd rep05/md0.crd rep06/md0.crd rep07/md0.crd < %s > /dev/null' % dict['crdfile']]

            unit = umgr.submit_units(cudesc)

            unit.wait()

        else:

            print 'coco stuff...'

            cudesc = radical.pilot.ComputeUnitDescription()
            cudesc.executable = 'python'
            cudesc.cores = 16
            cudesc.pre_exec = ['module load python','cp postexec.py %s/examples'% coco_loc,'cd %s/examples' % coco_loc,'module load mpi4py']
            cudesc.arguments = ['cocoUi.py', '--grid 5', ' --projs 3 ', ' --frontpoints 8 ', '--cycle %s ' % cycle, '-vvv' ]
            cudesc.input_data = ['postexec.py']
            cudesc.post_exec = ['python postexec.py %s %s' % (nreps,cycle)]
            cudesc.mpi = True
            #cs.append('mpiexec -mca btl ^openib -n 4 python cocoUi.py --grid 5 --projs 3  --frontpoints 8 --cycle {cycle} -vvv'.format(**dict))
            #cs.append('python cocoUi.py --grid 5 --projs 3  --frontpoints 8 --cycle {cycle} -vvv'.format(**dict))

            unit = umgr.submit_units(cudesc)

            unit.wait()

            #try:
            #    cs.run('tcsh -f {}')
            #except IOError as e:
            #    print "I/O error({0}): {1}".format(e.errno, e.strerror)

            #print 'creating new crd files...'
            #for rep in range(nreps):
            #    dict['rep'] = rep
            #    dict['path'] = os.getcwd()
            #    tl = script.Script()
            #    tl.append('source leaprc.ff99SB')
            #    tl.append('x = loadpdb {path}/rep0{rep}/pentaopt{cycle}.pdb'.format(**dict))
            #    tl.append('saveamberparm x {path}/rep0{rep}/delete.me {path}/rep0{rep}/min{cycle}.crd'.format(**dict))
            #    tl.append('quit')
            #    tl.run('tleap -f {}')
#
    # now run the MD
    #
        compute_units = []
        for i in range(nreps):
            print "Submitting new 'pmemd' compute unit"
            dict['rep'] = str(i)
            dict['path'] = coco_loc
            # output files that need to be transferred back: *.mdcrd

            step_1 = 'pmemd -O -i {path}/{minin} -o {path}/rep0{rep}/min{cycle}.out -inf {path}/rep0{rep}/min{cycle}.inf -r {path}/rep0{rep}/md{cycle}.crd -p {path}/{topfile} -c {path}/rep0{rep}/min{cycle}.crd -ref {path}/rep0{rep}/min{cycle}.crd'.format(**dict)
            step_2 = 'pmemd -O -i {path}/{mdin} -o {path}/rep0{rep}/md{cycle}.out -inf {path}/rep0{rep}/md{cycle}.inf -x {path}/rep0{rep}/md{cycle}.mdcrd -r {path}/rep0{rep}/md{cycle}.rst -p {path}/{topfile} -c {path}/rep0{rep}/md{cycle}.crd'.format(**dict)

            cu = radical.pilot.ComputeUnitDescription()
            cu.executable = "/bin/bash"
            cu.cores      = 1
            cu.pre_exec = ['module load TACC ','module load amber','cd %s/examples' % coco_loc]
            cu.arguments  = ['-l', '-c', " %s && %s" % (step_1, step_2)]
            compute_units.append(cu)

        units = umgr.submit_units(compute_units)
        umgr.wait_units()

        #print 'running md cycle {cycle}'.format(**dict)
        #try:
    #    out = md0.run("tcsh -f {}")
        #except IOError as e:
    #    print "I/O error({0}): {1}".format(e.errno, e.strerror)

    session.close()<commit_msg>project id<commit_after>__author__ = 'vivek'
import os
import sys
import radical.pilot

DBURL = "mongodb://ec2-184-72-89-141.compute-1.amazonaws.com:27017"
RCONF  = ["https://raw.github.com/radical-cybertools/radical.pilot/master/configs/xsede.json"]

#------------------------------------------------------------------------------
#
def pilot_state_cb(pilot, state):
    """pilot_state_change_cb() is a callback function. It gets called very
    time a ComputePilot changes its state.
    """
    print "[Callback]: ComputePilot '{0}' state changed to {1}.".format(
        pilot.uid, state)

    if state == radical.pilot.states.FAILED:
        sys.exit(1)

#------------------------------------------------------------------------------
#
def unit_state_change_cb(unit, state):
    """unit_state_change_cb() is a callback function. It gets called very
    time a ComputeUnit changes its state.
    """
    print "[Callback]: ComputeUnit '{0}' state changed to {1}.".format(
        unit.uid, state)
    if state == radical.pilot.states.FAILED:
        print "            Log: %s" % unit.log[-1]

#------------------------------------------------------------------------------
#

if __name__ == '__main__':
    print "Starting 16 core pilot-job..."
    session = radical.pilot.Session(database_url=DBURL)
    print "Session UID: {0} ".format(session.uid)

    # Add an ssh identity to the session.
    cred = radical.pilot.Context('ssh')
    cred.user_id = 'vivek91'
    session.add_context(cred)

    # Add a Pilot Manager. Pilot managers manage one or more ComputePilots.
    pmgr = radical.pilot.PilotManager(session=session)

    pmgr.register_callback(pilot_state_cb)

    # Start a pilot at the remote host as per the configs
    pdesc = radical.pilot.ComputePilotDescription()
    pdesc.resource = 'stampede.tacc.utexas.edu'
    pdesc.runtime = 20
    pdesc.cores = 16
    pdesc.project = os.getenv('ALLOCATION_ID','None')

    pilot = pmgr.submit_pilots(pdesc)

    print "Pilot UID       : {0} ".format(pilot.uid )

    umgr = radical.pilot.UnitManager(session=session, scheduler=radical.pilot.SCHED_DIRECT_SUBMISSION)
    # Register our callback with the UnitManager. This callback will get
    # called every time any of the units managed by the UnitManager
    # change their state.
    umgr.register_callback(unit_state_change_cb)

    # Add the previously created ComputePilot to the UnitManager.
    umgr.add_pilots(pilot)

    print "... DONE."

    #------------------------------------------------------------------------------
    #

    #
    # All simulations use the same mdin file.
    #
    dict = {}
    dict['crdfile'] = 'penta.crd'
    dict['topfile'] = 'penta.top'
    dict['mdin']    = 'mdshort.in'
    dict['minin']   = 'min.in'

    # set number of cycles, and number of replicates
    maxcycles = 10
    nreps = 8
    coco_loc = '$HOME/coco/'
    #
    # Main loop begins here:
    #rep00/md0.crd
    for cycle in range(maxcycles):
        dict['cycle'] = str(cycle)
        if cycle == 0:
    #
    # cycle 0: we begin with a single coordinate and topology file.
    # Copy it into the separate rep0? directories ready to run the MD simulations.
    #
            print "creating initial setup"
            #cp = script.Script()
            #for rep in range(nreps):
            #    dict['rep'] = str(rep)
            ##    dict['path'] = os.getcwd()
             #   cp.append('cp {crdfile} {path}/rep0{rep}/md0.crd'.format(**dict))
            #    cp.run('tcsh -f {}')

            cudesc = radical.pilot.ComputeUnitDescription()
            cudesc.executable = '/bin/bash'
            cudesc.cores = 1
            cudesc.pre_exec = ['cd %s/examples' % coco_loc, '( test -d "rep0*" || rm rep0* -rf)' , 'mkdir rep00 rep01 rep02 rep03 rep04 rep05 rep06 rep07' ]
            cudesc.arguments = ['-l','-c','tee rep00/md0.crd rep01/md0.crd rep02/md0.crd rep03/md0.crd rep04/md0.crd rep05/md0.crd rep06/md0.crd rep07/md0.crd < %s > /dev/null' % dict['crdfile']]

            unit = umgr.submit_units(cudesc)

            unit.wait()

        else:

            print 'coco stuff...'

            cudesc = radical.pilot.ComputeUnitDescription()
            cudesc.executable = 'python'
            cudesc.cores = 16
            cudesc.pre_exec = ['module load python','cp postexec.py %s/examples'% coco_loc,'cd %s/examples' % coco_loc,'module load mpi4py']
            cudesc.arguments = ['cocoUi.py', '--grid 5', ' --projs 3 ', ' --frontpoints 8 ', '--cycle %s ' % cycle, '-vvv' ]
            cudesc.input_data = ['postexec.py']
            cudesc.post_exec = ['python postexec.py %s %s' % (nreps,cycle)]
            cudesc.mpi = True
            #cs.append('mpiexec -mca btl ^openib -n 4 python cocoUi.py --grid 5 --projs 3  --frontpoints 8 --cycle {cycle} -vvv'.format(**dict))
            #cs.append('python cocoUi.py --grid 5 --projs 3  --frontpoints 8 --cycle {cycle} -vvv'.format(**dict))

            unit = umgr.submit_units(cudesc)

            unit.wait()

            #try:
            #    cs.run('tcsh -f {}')
            #except IOError as e:
            #    print "I/O error({0}): {1}".format(e.errno, e.strerror)

            #print 'creating new crd files...'
            #for rep in range(nreps):
            #    dict['rep'] = rep
            #    dict['path'] = os.getcwd()
            #    tl = script.Script()
            #    tl.append('source leaprc.ff99SB')
            #    tl.append('x = loadpdb {path}/rep0{rep}/pentaopt{cycle}.pdb'.format(**dict))
            #    tl.append('saveamberparm x {path}/rep0{rep}/delete.me {path}/rep0{rep}/min{cycle}.crd'.format(**dict))
            #    tl.append('quit')
            #    tl.run('tleap -f {}')
#
    # now run the MD
    #
        compute_units = []
        for i in range(nreps):
            print "Submitting new 'pmemd' compute unit"
            dict['rep'] = str(i)
            dict['path'] = coco_loc
            # output files that need to be transferred back: *.mdcrd

            step_1 = 'pmemd -O -i {path}/{minin} -o {path}/rep0{rep}/min{cycle}.out -inf {path}/rep0{rep}/min{cycle}.inf -r {path}/rep0{rep}/md{cycle}.crd -p {path}/{topfile} -c {path}/rep0{rep}/min{cycle}.crd -ref {path}/rep0{rep}/min{cycle}.crd'.format(**dict)
            step_2 = 'pmemd -O -i {path}/{mdin} -o {path}/rep0{rep}/md{cycle}.out -inf {path}/rep0{rep}/md{cycle}.inf -x {path}/rep0{rep}/md{cycle}.mdcrd -r {path}/rep0{rep}/md{cycle}.rst -p {path}/{topfile} -c {path}/rep0{rep}/md{cycle}.crd'.format(**dict)

            cu = radical.pilot.ComputeUnitDescription()
            cu.executable = "/bin/bash"
            cu.cores      = 1
            cu.pre_exec = ['module load TACC ','module load amber','cd %s/examples' % coco_loc]
            cu.arguments  = ['-l', '-c', " %s && %s" % (step_1, step_2)]
            compute_units.append(cu)

        units = umgr.submit_units(compute_units)
        umgr.wait_units()

        #print 'running md cycle {cycle}'.format(**dict)
        #try:
    #    out = md0.run("tcsh -f {}")
        #except IOError as e:
    #    print "I/O error({0}): {1}".format(e.errno, e.strerror)

    session.close()
<|endoftext|>


<commit_before>import markdown
import itertools
from datetime import datetime

from django.core.urlresolvers import reverse
from django.utils.text import slugify
from django.db import models

class Tag(models.Model):
    name = models.CharField(max_length=50)

    class Meta:
        ordering = ['name']

    def __unicode__(self):
        return self.name

class Category(models.Model):
    name = models.CharField(max_length=100, db_index=True)
    slug = models.CharField(max_length=100, db_index=True, blank=True)
    parent = models.ForeignKey('self', null=True, blank=True,
                               limit_choices_to={'parent': None})

    class Meta:
        verbose_name_plural = 'Categories'
        ordering = ['parent__name', 'name']

    def __unicode__(self):
        if self.parent:
            return "%s (%s)" % (self.name, self.parent)
        else:
            return "%s" % self.name

    def save(self, *args, **kwargs):
        self.slug = slugify(self.name)
        super(Category, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return reverse('blog:category', kwargs=dict(slug=self.slug))

    def children(self):
        return Category.objects.filter(parent=self)

    def descendants(self):
        children = self.children().values('id', 'name', 'slug')
        if children:
            for c in children:
                childs_descendants = Category.objects.get(
                    id=c['id']).descendants()
                if childs_descendants:
                    c['children'] = childs_descendants
                else:
                    c['children'] = None
            return c
        else:
            return None

    @classmethod
    def tree(cls):
        roots = cls.objects.filter(parent=None)
        res = []
        for r in roots:
            res.append({
                'id': r.id,
                'name': r.name,
                'slug': r.slug,
                'children': r.children().values('id', 'name', 'slug')
            })
        return res

class Post(models.Model):
    STATUS_CHOICES = (
        ('p', 'Published'),
        ('d', 'Draft'),
        ('u', 'Unpublished')
    )
    title = models.CharField(max_length=100)
    slug = models.CharField(max_length=100, blank=True)
    body = models.TextField()
    status = models.CharField(max_length=2, choices=STATUS_CHOICES,
                              default='d')
    created = models.DateTimeField(db_index=True, blank=True)
    modified = models.DateTimeField(blank=True)
    tags = models.ManyToManyField(Tag, null=True, blank=True)
    category = models.ForeignKey(Category, null=True, blank=True,
                                 limit_choices_to={'category__parent': None})

    class Meta:
        ordering = ['-created']

    def __unicode__(self):
        return self.title

    def save(self, *args, **kwargs):
        self.slug = slugify(self.title)
        if not self.id:
            self.created = datetime.now()
        self.modified = datetime.now()
        super(Post, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return reverse('blog:post', kwargs=dict(pid=self.id, slug=self.slug))

    def is_published(self):
        return self.status == 'p'

    def preview(self):
        return markdown.markdown(self.body)

    preview.allow_tags = True

    def tagged(self):
        return ", ".join([t['name'] for t in self.tags.values()])

    def related(self):
        tags = self.tags.all()
        return Post.objects.filter(status='p', tags__in=tags).exclude(
            id=self.id).distinct()

    @classmethod
    def archive_tree(cls):
        dates = Post.objects.filter(status='p').values('created')
        d_list = [d['created'].date() for d in dates]

        tree = []
        # Group by year
        tree = [{
            'year': year,
            'months': [{
                'month': g.month,
                'name': g.strftime('%B')
            } for g in group]
        } for year, group in itertools.groupby(d_list, key=lambda x: x.year)]

        # Group by month
        for y in tree:
            y['months'] = [{
                'month': month,
                'name': [g['name'] for g in group][0]
            } for month, group in itertools.groupby(
                y['months'],
                key=lambda x: x['month']
            )]
        return tree
<commit_msg>Add Post methods to publish and unpublish<commit_after>import markdown
import itertools
from datetime import datetime

from django.core.urlresolvers import reverse
from django.utils.text import slugify
from django.db import models

class Tag(models.Model):
    name = models.CharField(max_length=50)

    class Meta:
        ordering = ['name']

    def __unicode__(self):
        return self.name

class Category(models.Model):
    name = models.CharField(max_length=100, db_index=True)
    slug = models.CharField(max_length=100, db_index=True, blank=True)
    parent = models.ForeignKey('self', null=True, blank=True,
                               limit_choices_to={'parent': None})

    class Meta:
        verbose_name_plural = 'Categories'
        ordering = ['parent__name', 'name']

    def __unicode__(self):
        if self.parent:
            return "%s (%s)" % (self.name, self.parent)
        else:
            return "%s" % self.name

    def save(self, *args, **kwargs):
        self.slug = slugify(self.name)
        super(Category, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return reverse('blog:category', kwargs=dict(slug=self.slug))

    def children(self):
        return Category.objects.filter(parent=self)

    def descendants(self):
        children = self.children().values('id', 'name', 'slug')
        if children:
            for c in children:
                childs_descendants = Category.objects.get(
                    id=c['id']).descendants()
                if childs_descendants:
                    c['children'] = childs_descendants
                else:
                    c['children'] = None
            return c
        else:
            return None

    @classmethod
    def tree(cls):
        roots = cls.objects.filter(parent=None)
        res = []
        for r in roots:
            res.append({
                'id': r.id,
                'name': r.name,
                'slug': r.slug,
                'children': r.children().values('id', 'name', 'slug')
            })
        return res

class Post(models.Model):
    STATUS_CHOICES = (
        ('p', 'Published'),
        ('d', 'Draft'),
        ('u', 'Unpublished')
    )
    title = models.CharField(max_length=100)
    slug = models.CharField(max_length=100, blank=True)
    body = models.TextField()
    status = models.CharField(max_length=2, choices=STATUS_CHOICES,
                              default='d')
    created = models.DateTimeField(db_index=True, blank=True)
    modified = models.DateTimeField(blank=True)
    tags = models.ManyToManyField(Tag, null=True, blank=True)
    category = models.ForeignKey(Category, null=True, blank=True,
                                 limit_choices_to={'category__parent': None})

    class Meta:
        ordering = ['-created']

    def __unicode__(self):
        return self.title

    def save(self, *args, **kwargs):
        self.slug = slugify(self.title)
        if not self.id:
            self.created = datetime.now()
        self.modified = datetime.now()
        super(Post, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return reverse('blog:post', kwargs=dict(pid=self.id, slug=self.slug))

    def publish(self):
        self.status = 'p'
        self.save()

    def unpublish(self):
        self.status = 'u'
        self.save()

    def is_published(self):
        return self.status == 'p'

    def preview(self):
        return markdown.markdown(self.body)

    preview.allow_tags = True

    def tagged(self):
        return ", ".join([t['name'] for t in self.tags.values()])

    def related(self):
        tags = self.tags.all()
        return Post.objects.filter(status='p', tags__in=tags).exclude(
            id=self.id).distinct()

    @classmethod
    def archive_tree(cls):
        dates = Post.objects.filter(status='p').values('created')
        d_list = [d['created'].date() for d in dates]

        tree = []
        # Group by year
        tree = [{
            'year': year,
            'months': [{
                'month': g.month,
                'name': g.strftime('%B')
            } for g in group]
        } for year, group in itertools.groupby(d_list, key=lambda x: x.year)]

        # Group by month
        for y in tree:
            y['months'] = [{
                'month': month,
                'name': [g['name'] for g in group][0]
            } for month, group in itertools.groupby(
                y['months'],
                key=lambda x: x['month']
            )]
        return tree
<|endoftext|>