<commit_before>from Song import Song

# http://code.google.com/p/leveldb/
# http://code.google.com/p/py-leveldb/
import leveldb

import appinfo
import utils
from utils import safe_property

# see <https://github.com/albertz/binstruct/> for documentation
import binstruct
def dbRepr(o): return binstruct.varEncode(o).tostring()
def dbUnRepr(s): return binstruct.varDecode(s)

# Structure of the database:
#	There is the main song db songs.db:
#		songId -> song dict
#	songId is any random string, not too long but long enough to avoid >99.999% collisions.
#	song dict can contains (specified in code by global Attribs dict later):
#		artist: str
#		title: str
#		album: str
#		tags: weighted tagmap, dict tag->[0,1]
#		rating: float in [0,1]
#		files: dict filename -> dict with entries:
#			sha1: str
#			metadata: dict
#			fingerprint_AcoustId: str
#			gain: float
#   values should only be stored if they are certain with best accurary

class DB:
	def __init__(self, dir):
		import threading
		self.writelock = threading.Lock()
		self.db = leveldb.LevelDB(appinfo.userdir + "/" + dir, max_open_files=200)

	def __getitem__(self, item):
		return dbUnRepr(self.db.Get(dbRepr(item)))

	def __setitem__(self, key, value):
		self.db.Put(dbRepr(key), dbRepr(value))

	def __delitem__(self, key):
		self.db.Delete(dbRepr(key))

	def setdefault(self, key, value):
		if key in self:
			return self[key]
		else:
			self[key] = value
			return self[key]

	def rangeIter(self, key_from = None, key_to = None, include_value = True):
		def saveDbUnRepr(v):
			try: return dbUnRepr(v)
			except: return None # not/broken binstruct data
		if include_value:
			mapFunc = lambda value: (saveDbUnRepr(value[0]), saveDbUnRepr(value[1]))
		else:
			mapFunc = saveDbUnRepr
		return map(mapFunc, self.db.RangeIter(key_from, key_to, include_value))

def init():
	global songDb
	songDb = DB("songs.db")
	global songHashDb
	songHashDb = DB("songHashs.db")
	global songSearchIndexDb
	songSearchIndexDb = DB("songSearchIndex.db")

def normalizedFilename(fn):
	import os
	fn = os.path.normpath(fn)
	userDir = os.path.expanduser("~")
	if fn.startswith(userDir):
		fn = "~" + fn[len(userDir):]
	if isinstance(fn, unicode): fn = fn.encode("utf-8")
	return fn

def hash(s):
	# I thought about using some more fast Hash like MurmurHash.
	# But this is just simpler now.
	# CRC32 is too less, to much collisions. We want something here
	# which does almost never collide. The whole code in here will
	# mostly ignore collisions (the whole DB is optional, so this
	# shouldn't be a problem in case anyone ever gets a collsion).
	import hashlib
	return hashlib.sha1(s).digest()

HashFileBufferSize = 1024 * 10

def hashFile(f):
	if isinstance(f, (str,unicode)): f = open(f)
	import hashlib
	h = hashlib.sha1()
	while True:
		s = f.read(HashFileBufferSize)
		if not s: break
		h.update(s)
	return h.digest()

# Entries (hash-prefix, attrib, func).
# The function should either return some False value or some non-empty string.
# If an attrib is specified and no func, we just use getattr(song, attrib, None).
SongHashSources = [
	("a", "fingerprint_AcoustId", None),
	("h", "sha1", None),
	("p", None, lambda song: normalizedFilename(song.url)),
]

def mapHash(value):
	if isinstance(value, unicode): value = value.encode("utf-8")
	if len(value) > 32: value = hash(value)
	return value

def getSongHashSources(song):
	for prefix,attrib,func in SongHashSources:
		if not func: func = lambda song: getattr(song, attrib, None)
		value = func(song)
		if value: yield prefix + mapHash(value)

def maybeUpdateHashAfterAttribUpdate(song, attrib, value):
	for prefix,attr,func in SongHashSources:
		if attr == attrib:
			hashDbKey = prefix + mapHash(value)
			try:
				oldId = songHashDb[hashDbKey]
				if oldId != song.id:
					# whoops, we had a wrong song.id before...
					# update it.
					# we might loose some data which was set in the meantime,
					# but i don't really know a good way to solve this...
					song.id = oldId
					updateHashDb(song, song.id)
			except KeyError:
				songHashDb[hashDbKey] = song.id
			return

def getSongId(song):
	for value in getSongHashSources(song):
		try: return songHashDb[value]
		except KeyError: pass
	return None

def updateHashDb(song, songId):
	for value in getSongHashSources(song):
		songHashDb[value] = songId

def calcNewSongId(song):
	"Returns a new unique (in hopefully almost all cases) id for a song."
	"Different files with the same song might return the same id."
	# Originally, I planned to use the hash of the AcoustID fingerprint.
	# However, as this is expensive to calculate in case we have not yet,
	# allow some fallbacks.
	# Just use any available from SongHashSources.
	for value in getSongHashSources(song):
		value = mapHash(value)
		updateHashDb(song, value)
		return value
	assert False # should not happen. if there are such cases later, extend SongHashSources!

class SongFileEntry(object):
	def __init__(self, songEntry, url):
		object.__setattr__(self, "songEntry", songEntry)
		object.__setattr__(self, "url", url)

	@safe_property
	@property
	def _dbDict(self):
		# Note: If this raises an AttributeError for some reason,
		# you will get a *very* strange inf recursion loop in
		# getattr(self, "_dbDict").
		return self.songEntry.files.filesDict.get(self.url, {})

	def __getattr__(self, attr):
		try: return self._dbDict[attr]
		except KeyError: raise AttributeError, "no attrib " + attr

	def update(self, attr, updateFunc, default=None):
		global songDb
		with songDb.writelock:
			d = self.songEntry._dbDict
			fileDict = d.setdefault("files",{}).setdefault(self.url,{})
			value = updateFunc(fileDict.get(attr, default))
			fileDict[attr] = value
			songDb[self.songEntry.id] = d
		return value

	def __setattr__(self, attr, value):
		self.update(attr, lambda _: value)

class SongFilesDict:
	def __init__(self, songEntry):
		self.songEntry = songEntry

	@safe_property
	@property
	def filesDict(self):
		return self.songEntry._dbDict.get("files", {})

	def __getitem__(self, url):
		url = normalizedFilename(url)
		try: self.filesDict[url]
		except: raise
		else: return SongFileEntry(self.songEntry, url)

	def get(self, url):
		url = normalizedFilename(url)
		return SongFileEntry(self.songEntry, url)

class SongEntry(object):
	def __init__(self, song):
		object.__setattr__(self, "songObj", song)

	@safe_property
	@property
	def id(self):
		return self.songObj.id

	@safe_property
	@property
	def files(self):
		return SongFilesDict(self)

	@safe_property
	@property
	def _dbDict(self):
		global songDb
		try: return songDb[self.id]
		except KeyError: return {}

	def __getattr__(self, attr):
		try: return self._dbDict[attr]
		except KeyError: raise AttributeError, "no attrib " + attr

	def update(self, attr, updateFunc, default=None):
		global songDb
		with songDb.writelock:
			d = self._dbDict
			value = updateFunc(d.get(attr, default))
			d[attr] = value
			songDb[self.id] = d
		return value

	def __setattr__(self, attr, value):
		self.update(attr, lambda _: value)

def getSong(song):
	return SongEntry(song)

def getBestSongFileFromDict(filesDict):
	files = filesDict.keys()
	import os
	files = map(os.path.expanduser, files)
	files = filter(os.path.exists, files)
	if not files: return None
	# priority: flac, m4a, ogg
	fsByExt = dict([(os.path.splitext(f)[1].lower(), f) for f in files])
	f = None
	for ext in ["flac","m4a","ogg"]:
		f = fsByExt.get(ext, None)
		if f: break
	if not f: f = files[0] # just take first, whatever that is
	assert f
	return f

def getSongFilenameById(songId):
	global songDb
	try: dbEntry = songDb[songId]
	except KeyError: return None
	filesDict = dbEntry.get("files",{})
	return getBestSongFileFromList(filesDict)

def getSongById(songId):
	f = getSongFilenameById(songId)
	if not f: return None
	song = Song(url=f)
	song.id = songId
	assert song
	return song

def getSongSummaryDictById(songId):
	global songDb
	try: dbEntry = songDb[songId]
	except KeyError: return None
	f = getBestSongFileFromDict(dbEntry.get("files",{}))
	if not f: return None
	fileEntry = dbEntry["files"][normalizedFilename(f)] # Note: this assumes that we never have changed the normalize-func
	# for now, this is just enough for good results in Search.Search.Keys
	return {
		"url": f,
		"id": songId,
		"artist": dbEntry.get("artist", ""),
		"title": dbEntry.get("title", ""),
		"rating": dbEntry.get("rating", 0),
		"duration": fileEntry.get("duration", -1),
	}

class Attrib:
	def __init__(self, fileSpecific=False):
		self.fileSpecific = fileSpecific
	def getObject(self, song):
		if self.fileSpecific:
			return getSong(song).files.get(song.url)
		else:
			return getSong(song)

Attribs = {
	"id": Attrib(), # This is the SongId used here by the DB.
	"artist": Attrib(),
	"title": Attrib(),
	"album": Attrib(),
	"tags": Attrib(),
	"rating": Attrib(),
	"skipCount": Attrib(),
	"completedCount": Attrib(),
	"lastPlayedDate": Attrib(),
	"sha1": Attrib(fileSpecific=True),
	"metadata": Attrib(fileSpecific=True),
	"fingerprint_AcoustId": Attrib(fileSpecific=True),
	"gain": Attrib(fileSpecific=True),
	"duration": Attrib(fileSpecific=True),
# Note that bmpThumbnail is not here. I think it's to heavy
# to be stored for each song in the DB. Let's just calculate it
# on the fly when needed...
# The Song handling code should not assume that all attribs are
# defined here by the DB.
}

def updateSongAttribValue(song, attrib, updateFunc, default=None):
	value = Attribs[attrib].getObject(song).update(attrib, updateFunc, default=default)
	maybeUpdateHashAfterAttribUpdate(song, attrib, value)
	return value

def getSongAttrib(song, attrib):
	value = getattr(Attribs[attrib].getObject(song), attrib)
	assert value is not None, "songdb %r attrib %r is None" % (song, attrib) # if we ever want to allow that, mark it that way in class Attrib
	return value

def insertSearchEntry(song):
	global songSearchIndexDb
	def update(key, updateDict):
		oldDict = songSearchIndexDb[key]
		for dkey,dvalue in updateDict.iteritems():
			oldValue = oldDict.get(dkey, [])
			oldValue = set(oldValue)
			oldValue.update(dvalue)
			oldDict[dkey] = list(oldValue)

	with songSearchIndexDb.writelock:
		tokens = song.artist.split() + song.title.split()
		s = " ".join(tokens).lower()
		def go(startIndex):
			s[startIndex]
			for i in xrange(startIndex, len(s)):
				go(i)
		for i in xrange(len(s)):
			go(i)

def search(query):
	return [{"title": "hey", "artist": query, "url": "/Users/az/README.md"}, {"title": "foo"}]

# Do that right on first import so that all functions here work.
init()

def songdbMain():
	# Later, me might scan through the disc and fill the DB and do updates here.
	# Right now, we don't.
	pass

# For debugging
def dumpDatabases():
	global songDb, songHashDb
	import sys
	from pprint import pprint
	print "Main DB:"
	for key,value in songDb.rangeIter():
		sys.stdout.write("%r: \n" % key)
		pprint(value, indent=2)
	print "\nHashes:"
	for key,value in songHashDb.rangeIter():
		sys.stdout.write("%r: " % key)
		pprint(value, indent=2)

<commit_msg>search index progress (incomplete)<commit_after>from Song import Song

# http://code.google.com/p/leveldb/
# http://code.google.com/p/py-leveldb/
import leveldb

import appinfo
import utils
from utils import safe_property

# see <https://github.com/albertz/binstruct/> for documentation
import binstruct
def dbRepr(o): return binstruct.varEncode(o).tostring()
def dbUnRepr(s): return binstruct.varDecode(s)

# Structure of the database:
#	There is the main song db songs.db:
#		songId -> song dict
#	songId is any random string, not too long but long enough to avoid >99.999% collisions.
#	song dict can contains (specified in code by global Attribs dict later):
#		artist: str
#		title: str
#		album: str
#		tags: weighted tagmap, dict tag->[0,1]
#		rating: float in [0,1]
#		files: dict filename -> dict with entries:
#			sha1: str
#			metadata: dict
#			fingerprint_AcoustId: str
#			gain: float
#   values should only be stored if they are certain with best accurary

class DB:
	def __init__(self, dir):
		import threading
		self.writelock = threading.Lock()
		self.db = leveldb.LevelDB(appinfo.userdir + "/" + dir, max_open_files=200)

	def __getitem__(self, item):
		return dbUnRepr(self.db.Get(dbRepr(item)))

	def __setitem__(self, key, value):
		self.db.Put(dbRepr(key), dbRepr(value))

	def __delitem__(self, key):
		self.db.Delete(dbRepr(key))

	def setdefault(self, key, value):
		if key in self:
			return self[key]
		else:
			self[key] = value
			return self[key]

	def rangeIter(self, key_from = None, key_to = None, include_value = True):
		def saveDbUnRepr(v):
			try: return dbUnRepr(v)
			except: return None # not/broken binstruct data
		if include_value:
			mapFunc = lambda value: (saveDbUnRepr(value[0]), saveDbUnRepr(value[1]))
		else:
			mapFunc = saveDbUnRepr
		return map(mapFunc, self.db.RangeIter(key_from, key_to, include_value))

def init():
	global songDb
	songDb = DB("songs.db")
	global songHashDb
	songHashDb = DB("songHashs.db")
	global songSearchIndexDb
	songSearchIndexDb = DB("songSearchIndex.db")

def normalizedFilename(fn):
	import os
	fn = os.path.normpath(fn)
	userDir = os.path.expanduser("~")
	if fn.startswith(userDir):
		fn = "~" + fn[len(userDir):]
	if isinstance(fn, unicode): fn = fn.encode("utf-8")
	return fn

def hash(s):
	# I thought about using some more fast Hash like MurmurHash.
	# But this is just simpler now.
	# CRC32 is too less, to much collisions. We want something here
	# which does almost never collide. The whole code in here will
	# mostly ignore collisions (the whole DB is optional, so this
	# shouldn't be a problem in case anyone ever gets a collsion).
	import hashlib
	return hashlib.sha1(s).digest()

HashFileBufferSize = 1024 * 10

def hashFile(f):
	if isinstance(f, (str,unicode)): f = open(f)
	import hashlib
	h = hashlib.sha1()
	while True:
		s = f.read(HashFileBufferSize)
		if not s: break
		h.update(s)
	return h.digest()

# Entries (hash-prefix, attrib, func).
# The function should either return some False value or some non-empty string.
# If an attrib is specified and no func, we just use getattr(song, attrib, None).
SongHashSources = [
	("a", "fingerprint_AcoustId", None),
	("h", "sha1", None),
	("p", None, lambda song: normalizedFilename(song.url)),
]

def mapHash(value):
	if isinstance(value, unicode): value = value.encode("utf-8")
	if len(value) > 32: value = hash(value)
	return value

def getSongHashSources(song):
	for prefix,attrib,func in SongHashSources:
		if not func: func = lambda song: getattr(song, attrib, None)
		value = func(song)
		if value: yield prefix + mapHash(value)

def maybeUpdateHashAfterAttribUpdate(song, attrib, value):
	for prefix,attr,func in SongHashSources:
		if attr == attrib:
			hashDbKey = prefix + mapHash(value)
			try:
				oldId = songHashDb[hashDbKey]
				if oldId != song.id:
					# whoops, we had a wrong song.id before...
					# update it.
					# we might loose some data which was set in the meantime,
					# but i don't really know a good way to solve this...
					song.id = oldId
					updateHashDb(song, song.id)
			except KeyError:
				songHashDb[hashDbKey] = song.id
			return

def getSongId(song):
	for value in getSongHashSources(song):
		try: return songHashDb[value]
		except KeyError: pass
	return None

def updateHashDb(song, songId):
	for value in getSongHashSources(song):
		songHashDb[value] = songId

def calcNewSongId(song):
	"Returns a new unique (in hopefully almost all cases) id for a song."
	"Different files with the same song might return the same id."
	# Originally, I planned to use the hash of the AcoustID fingerprint.
	# However, as this is expensive to calculate in case we have not yet,
	# allow some fallbacks.
	# Just use any available from SongHashSources.
	for value in getSongHashSources(song):
		value = mapHash(value)
		updateHashDb(song, value)
		return value
	assert False # should not happen. if there are such cases later, extend SongHashSources!

class SongFileEntry(object):
	def __init__(self, songEntry, url):
		object.__setattr__(self, "songEntry", songEntry)
		object.__setattr__(self, "url", url)

	@safe_property
	@property
	def _dbDict(self):
		# Note: If this raises an AttributeError for some reason,
		# you will get a *very* strange inf recursion loop in
		# getattr(self, "_dbDict").
		return self.songEntry.files.filesDict.get(self.url, {})

	def __getattr__(self, attr):
		try: return self._dbDict[attr]
		except KeyError: raise AttributeError, "no attrib " + attr

	def update(self, attr, updateFunc, default=None):
		global songDb
		with songDb.writelock:
			d = self.songEntry._dbDict
			fileDict = d.setdefault("files",{}).setdefault(self.url,{})
			value = updateFunc(fileDict.get(attr, default))
			fileDict[attr] = value
			songDb[self.songEntry.id] = d
		return value

	def __setattr__(self, attr, value):
		self.update(attr, lambda _: value)

class SongFilesDict:
	def __init__(self, songEntry):
		self.songEntry = songEntry

	@safe_property
	@property
	def filesDict(self):
		return self.songEntry._dbDict.get("files", {})

	def __getitem__(self, url):
		url = normalizedFilename(url)
		try: self.filesDict[url]
		except: raise
		else: return SongFileEntry(self.songEntry, url)

	def get(self, url):
		url = normalizedFilename(url)
		return SongFileEntry(self.songEntry, url)

class SongEntry(object):
	def __init__(self, song):
		object.__setattr__(self, "songObj", song)

	@safe_property
	@property
	def id(self):
		return self.songObj.id

	@safe_property
	@property
	def files(self):
		return SongFilesDict(self)

	@safe_property
	@property
	def _dbDict(self):
		global songDb
		try: return songDb[self.id]
		except KeyError: return {}

	def __getattr__(self, attr):
		try: return self._dbDict[attr]
		except KeyError: raise AttributeError, "no attrib " + attr

	def update(self, attr, updateFunc, default=None):
		global songDb
		with songDb.writelock:
			d = self._dbDict
			value = updateFunc(d.get(attr, default))
			d[attr] = value
			songDb[self.id] = d
		return value

	def __setattr__(self, attr, value):
		self.update(attr, lambda _: value)

def getSong(song):
	return SongEntry(song)

def getBestSongFileFromDict(filesDict):
	files = filesDict.keys()
	import os
	files = map(os.path.expanduser, files)
	files = filter(os.path.exists, files)
	if not files: return None
	# priority: flac, m4a, ogg
	fsByExt = dict([(os.path.splitext(f)[1].lower(), f) for f in files])
	f = None
	for ext in ["flac","m4a","ogg"]:
		f = fsByExt.get(ext, None)
		if f: break
	if not f: f = files[0] # just take first, whatever that is
	assert f
	return f

def getSongFilenameById(songId):
	global songDb
	try: dbEntry = songDb[songId]
	except KeyError: return None
	filesDict = dbEntry.get("files",{})
	return getBestSongFileFromList(filesDict)

def getSongById(songId):
	f = getSongFilenameById(songId)
	if not f: return None
	song = Song(url=f)
	song.id = songId
	assert song
	return song

def getSongSummaryDictById(songId):
	global songDb
	try: dbEntry = songDb[songId]
	except KeyError: return None
	f = getBestSongFileFromDict(dbEntry.get("files",{}))
	if not f: return None
	fileEntry = dbEntry["files"][normalizedFilename(f)] # Note: this assumes that we never have changed the normalize-func
	# for now, this is just enough for good results in Search.Search.Keys
	return {
		"url": f,
		"id": songId,
		"artist": dbEntry.get("artist", ""),
		"title": dbEntry.get("title", ""),
		"rating": dbEntry.get("rating", 0),
		"duration": fileEntry.get("duration", -1),
	}

class Attrib:
	def __init__(self, fileSpecific=False):
		self.fileSpecific = fileSpecific
	def getObject(self, song):
		if self.fileSpecific:
			return getSong(song).files.get(song.url)
		else:
			return getSong(song)

Attribs = {
	"id": Attrib(), # This is the SongId used here by the DB.
	"artist": Attrib(),
	"title": Attrib(),
	"album": Attrib(),
	"tags": Attrib(),
	"rating": Attrib(),
	"skipCount": Attrib(),
	"completedCount": Attrib(),
	"lastPlayedDate": Attrib(),
	"sha1": Attrib(fileSpecific=True),
	"metadata": Attrib(fileSpecific=True),
	"fingerprint_AcoustId": Attrib(fileSpecific=True),
	"gain": Attrib(fileSpecific=True),
	"duration": Attrib(fileSpecific=True),
# Note that bmpThumbnail is not here. I think it's to heavy
# to be stored for each song in the DB. Let's just calculate it
# on the fly when needed...
# The Song handling code should not assume that all attribs are
# defined here by the DB.
}

def updateSongAttribValue(song, attrib, updateFunc, default=None):
	value = Attribs[attrib].getObject(song).update(attrib, updateFunc, default=default)
	maybeUpdateHashAfterAttribUpdate(song, attrib, value)
	return value

def getSongAttrib(song, attrib):
	value = getattr(Attribs[attrib].getObject(song), attrib)
	assert value is not None, "songdb %r attrib %r is None" % (song, attrib) # if we ever want to allow that, mark it that way in class Attrib
	return value

Search_SubtokenLimit = 6
Search_PrefixPostfixStrAttrIndex = 1
Search_PrefixTokenAttrIndex = 2
Search_PostfixTokenAttrIndex = 3
Search_SongAttrIndex = 4

def insertSearchEntry(song):
	global songSearchIndexDb

	# all entries sets. an update merges the sets
	def update(key, updates):
		with songSearchIndexDb.writelock:
			old = songSearchIndexDb[key]
			old = set(old)
			old.update(updates)
			songSearchIndexDb[key] = old

	# subwords of len 1,2,4,8,16,... so we need O(n * log n) subwords for n=wordLen
	def iterSubwords(word):
		wordLen = len(word)
		l = 1
		while l < wordLen:
			for i in range(wordLen - l + 1):
				subWord = word[i:i+l]
				j = max(0, i - l)
				prefix = word[j:i]
				j = min(wordLen, i + 2*l - len(prefix))
				postfix = word[i+l:j]
				yield (subWord, prefix, postfix)
			l **= 2

	# subsequences of len 1,2,..,SubtokenLimit
	import itertools
	def iterSubtokens(tokens):
		for i in range(Search_SubtokenLimit):
			if i > len(tokens): break
			for cmb in itertools.combinations(tokens, i):
				yield cmb

	tokens = song.artist.lower().split() + song.title.lower().split()

	import collections
	localUpdates = collections.defaultdict(set)

	for token in tokens:
		for subWord, prefix, postfix in iterSubwords(token):
			localUpdates[(Search_PrefixPostfixStrAttrIndex, subWord)].add((prefix, postfix),)

	for subTokens in iterSubtokens(tokens):
		localUpdates[(Search_PrefixTokenAttrIndex, subWord)].add((prefix, postfix),)

	for key,value in localUpdates.items():
		update(key, value)

def search(query):
	return [{"title": "hey", "artist": query, "url": "/Users/az/README.md"}, {"title": "foo"}]

# Do that right on first import so that all functions here work.
init()

def songdbMain():
	# Later, me might scan through the disc and fill the DB and do updates here.
	# Right now, we don't.
	pass

# For debugging
def dumpDatabases():
	global songDb, songHashDb
	import sys
	from pprint import pprint
	print "Main DB:"
	for key,value in songDb.rangeIter():
		sys.stdout.write("%r: \n" % key)
		pprint(value, indent=2)
	print "\nHashes:"
	for key,value in songHashDb.rangeIter():
		sys.stdout.write("%r: " % key)
		pprint(value, indent=2)

<|endoftext|>


<commit_before>import sublime, sublime_plugin
import os
import re
from . misc import *

class RBoxSourcePromptCommand(sublime_plugin.TextCommand):

    def run(self, edit):
        view = self.view
        if view.settings().get("auto_match_enabled"):
            view.run_command("insert_snippet", {"contents": "\"${1:$SELECTION}\""})
        else:
            view.run_command("insert", {"characters" : "\""})

        fname = view.file_name()
        if not fname: return

        this_row = view.rowcol(point)[0]
        sel = view.sel()
        if len(sel)!=1: return
        if sel[0].begin() != sel[0].end(): return
        contentb = view.substr(sublime.Region(view.line(point).begin(), point))
        m = re.match(r".*?(source|sourceCpp)\($", contentb)

        if not m: return

        fdir = os.path.dirname(fname)
        def ondone(s):
            s = os.path.relpath(s, fdir)
            view.run_command("insert", {"characters" : escape_dq(s)})
        exts = [".r"] if m.group(1) == "source" else [".cpp", ".c", ".c++"]

        listdir(view, fdir, None, exts, ondone)
<commit_msg>accidentally removed point<commit_after>import sublime, sublime_plugin
import os
import re
from . misc import *

class RBoxSourcePromptCommand(sublime_plugin.TextCommand):

    def run(self, edit):
        view = self.view
        point = view.sel()[0].end() if len(view.sel())>0 else 0

        if view.settings().get("auto_match_enabled"):
            view.run_command("insert_snippet", {"contents": "\"${1:$SELECTION}\""})
        else:
            view.run_command("insert", {"characters" : "\""})

        fname = view.file_name()
        if not fname: return

        this_row = view.rowcol(point)[0]
        sel = view.sel()
        if len(sel)!=1: return
        if sel[0].begin() != sel[0].end(): return
        contentb = view.substr(sublime.Region(view.line(point).begin(), point))
        m = re.match(r".*?(source|sourceCpp)\($", contentb)

        if not m: return

        fdir = os.path.dirname(fname)
        def ondone(s):
            s = os.path.relpath(s, fdir)
            view.run_command("insert", {"characters" : escape_dq(s)})
        exts = [".r"] if m.group(1) == "source" else [".cpp", ".c", ".c++"]

        listdir(view, fdir, None, exts, ondone)
<|endoftext|>


<commit_before># coding=utf-8

import astropy.units
import pytest

from contextlib import nullcontext as does_not_raise
from ibei import SQSolarcell

class TestSQSolarcellConstructorHappyPath():
    """
    Circumstances under which SQSolarcell instance can be instantiated
    """
    def test_args_without_default_values(self, valid_constructor_args):
        """
        SQSolarcell can be instantiated with valid args that don't have defaults
        """
        valid_constructor_args.pop("solar_temperature")

        with does_not_raise():
            solarcell = SQSolarcell(**valid_constructor_args)

    def test_args_with_default_values(self, valid_constructor_args):
        """
        SQSolarcell can be instantiated with valid args incl. ones with defaults
        """
        with does_not_raise():
            solarcell = SQSolarcell(**valid_constructor_args)

    @pytest.mark.parametrize("argname", [
                "bandgap",
            ]
        )
    def test_args_that_can_equal_zero(self, valid_constructor_args, argname):
        """
        SQSolarcell can be instantiated with args not constrained to be nonzero
        """
        valid_constructor_args[argname] = 0

        with does_not_raise():
            solarcell = SQSolarcell(**valid_constructor_args)

@pytest.mark.parametrize("args,method_under_test,expected_output", [
            (
                # Special case.
                {
                    "solar_temperature": 5762,
                    "bandgap": 0.,
                },
                "power_density",
                "0.",
            ),
        ]
    )
def test_methods_regression(args, method_under_test, expected_output):
    """
    Methods' output values should match expected results
    """
    solarcell = SQSolarcell(**args)
    output = getattr(solarcell, method_under_test)()

    assert output == expected_output

@pytest.mark.parametrize("method_under_test,expected_unit,args_mod", [
            ("power_density", "W/m2", {}),
            ("power_density", "W/m2", {"bandgap": 0.}),  # Special case.
            ("efficiency", astropy.units.dimensionless_unscaled, {}),
            ("efficiency", astropy.units.dimensionless_unscaled, {"bandgap": 0.}),  # Special case.
        ]
    )
def test_methods_units(method_under_test, expected_unit, valid_constructor_args, args_mod):
    """
    Units of returned value should match what's documented.
    """
    valid_constructor_args |= args_mod
    solarcell = SQSolarcell(**valid_constructor_args)
    output = getattr(solarcell, method_under_test)()

    assert output.unit.is_equivalent(expected_unit)

# Pytest fixture definitions
# ==========================
@pytest.fixture
def valid_constructor_args():
    args = {
        "solar_temperature": 5762,
        "bandgap": 1.15,
        }

    return args
<commit_msg>Add test and fixture<commit_after># coding=utf-8

import astropy.units
import pytest

from contextlib import nullcontext as does_not_raise
from ibei import SQSolarcell

class TestSQSolarcellConstructorHappyPath():
    """
    Circumstances under which SQSolarcell instance can be instantiated
    """
    def test_args_without_default_values(self, valid_constructor_args):
        """
        SQSolarcell can be instantiated with valid args that don't have defaults
        """
        valid_constructor_args.pop("solar_temperature")

        with does_not_raise():
            solarcell = SQSolarcell(**valid_constructor_args)

    def test_args_with_default_values(self, valid_constructor_args):
        """
        SQSolarcell can be instantiated with valid args incl. ones with defaults
        """
        with does_not_raise():
            solarcell = SQSolarcell(**valid_constructor_args)

    @pytest.mark.parametrize("argname", [
                "bandgap",
            ]
        )
    def test_args_that_can_equal_zero(self, valid_constructor_args, argname):
        """
        SQSolarcell can be instantiated with args not constrained to be nonzero
        """
        valid_constructor_args[argname] = 0

        with does_not_raise():
            solarcell = SQSolarcell(**valid_constructor_args)

    @pytest.mark.parametrize("argname,val", [
                ("solar_temperature", astropy.units.Quantity(5498.85, astropy.units.deg_C)),
                ("bandgap", astropy.units.Quantity(1e-19, astropy.units.J)),
            ]
        )
    def test_quantity_args_compatible_units(self, valid_constructor_quantity_args, argname, val):
        """
        SQSolarcell can be instantiated with args in compatible units
        """
        valid_constructor_quantity_args[argname] = val

        with does_not_raise():
            solarcell = SQSolarcell(**valid_constructor_quantity_args)

@pytest.mark.parametrize("args,method_under_test,expected_output", [
            (
                # Special case.
                {
                    "solar_temperature": 5762,
                    "bandgap": 0.,
                },
                "power_density",
                "0.",
            ),
        ]
    )
def test_methods_regression(args, method_under_test, expected_output):
    """
    Methods' output values should match expected results
    """
    solarcell = SQSolarcell(**args)
    output = getattr(solarcell, method_under_test)()

    assert output == expected_output

@pytest.mark.parametrize("method_under_test,expected_unit,args_mod", [
            ("power_density", "W/m2", {}),
            ("power_density", "W/m2", {"bandgap": 0.}),  # Special case.
            ("efficiency", astropy.units.dimensionless_unscaled, {}),
            ("efficiency", astropy.units.dimensionless_unscaled, {"bandgap": 0.}),  # Special case.
        ]
    )
def test_methods_units(method_under_test, expected_unit, valid_constructor_args, args_mod):
    """
    Units of returned value should match what's documented.
    """
    valid_constructor_args |= args_mod
    solarcell = SQSolarcell(**valid_constructor_args)
    output = getattr(solarcell, method_under_test)()

    assert output.unit.is_equivalent(expected_unit)

# Pytest fixture definitions
# ==========================
@pytest.fixture
def valid_constructor_quantity_args():
    args = {
        "solar_temperature": astropy.units.Quantity(5762., astropy.units.K),
        "bandgap": astropy.units.Quantity(1.15, astropy.units.eV),
        }

    return args

@pytest.fixture(params=[(lambda x: x), (lambda x: getattr(x, "value", x))])
def valid_constructor_args(request, valid_constructor_quantity_args):
    args = {key: request.param(val) for key, val in valid_constructor_quantity_args.items()}

    return args
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from model.group import Contact

def test_test_add_contact(app):
    old_contact = app.contact.get_contact_list()
    app.contact.fill_new(Contact(firstname="natalia", lastname="krasnova", company="jhkjh", address="fewderf", email="email@email.ru", homepage="site.ru"))
    new_contact = app.contact.get_contact_list()
    assert len(old_contact) + 1 == len(new_contact)

def test_add_empty_contact(app):
    old_contact = app.contact.get_contact_list()
    app.contact.fill_new(Contact(firstname="", lastname="", company="", address="", email="", homepage=""))
    new_contact = app.contact.get_contact_list()
    assert len(old_contact) + 1 == len(new_contact)

<commit_msg>добавлены проверки в тест добавления контакта<commit_after># -*- coding: utf-8 -*-

from model.group import Contact

def test_test_add_contact(app):
    old_contact = app.contact.get_contact_list()
    contact = Contact(firstname="natalia", lastname="krasnova", company="jhkjh", address="fewderf", email="email@email.ru", homepage="site.ru")
    app.contact.fill_new(contact)
    new_contact = app.contact.get_contact_list()
    assert len(old_contact) + 1 == len(new_contact)
    old_contact.append (contact)
    assert  sorted (old_contact, key=Contact.id_or_max) == sorted (new_contact, key=Contact.id_or_max)

def test_add_empty_contact(app):
    old_contact = app.contact.get_contact_list()
    contact = Contact(firstname="natalia", lastname="krasnova", company="jhkjh", address="fewderf", email="email@email.ru", homepage="site.ru")
    app.contact.fill_new(contact)
    new_contact = app.contact.get_contact_list()
    assert len(old_contact) + 1 == len(new_contact)
    old_contact.append(contact)
    assert sorted(old_contact, key=Contact.id_or_max) == sorted(new_contact, key=Contact.id_or_max)

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

from sys import exc_info
from dns.resolver import query, NXDOMAIN
from requests import get

class SpamBLError(Exception):
    ''' Base exception class for spambl module '''

class UnknownCodeError(SpamBLError):
    ''' Raise when trying to use an unexpected value of dnsbl return code '''

class DNSBLItem(object):
    ''' Represents a host listed on a DNS blacklist '''

    _classification = None

    def __init__(self, host, source, return_code):
        ''' Create a new instance of DNSBLItem

        :param host: the host value listed on a DNS blacklist, either host name or ip address
        :param source: dnsbl service object
        :param return_code: last octet of ip address returned after querying the source for the host
        '''
        self.host = host
        self.source = source
        self._return_code = return_code

    @property
    def classification(self):
        ''' Classification of this host according to provider of the list from which it has been extracted '''
        if not self._classification:
            self._classification = self.source.get_classification(self._return_code)

        return self._classification

class DNSBLService(object):
    ''' Represents a DNSBL service '''
    def __init__(self, identifier, query_suffix, code_item_class, lists_ips, lists_uris):
        ''' Create new DNSBLService object

        :param identifier: a value designating DNSBL service provider: its name or url address.
        :param query_suffix: a suffix added to DNSBL query address
        :param code_item_class: item classes associated with DNSBL query return codes
        :param lists_ips: information if this object represents an ip blocklist
        :param lists_uris: information if this object represents a domain name blocklist
        '''

        self.identifier = identifier
        self._query_suffix = query_suffix
        self._code_item_class = code_item_class
        self.lists_ips = lists_ips
        self.lists_uris = lists_uris

    def get_classification(self, code):
        ''' Return classification for given code

        :param code: a valid return code extracted from response to DNSBL query
        :raises UnknownCodeError: raised when given code is not specified in self._code_item_class
        :returns: a value associated with a valid return code
        '''

        try:
            return self._code_item_class[code]

        except KeyError:

            msg_template = 'Unexpected code value for dnsbl service {}: {}'
            raise UnknownCodeError(msg_template.format(self.identifier, code)), None, exc_info()[2]

    def query(self, value):
        ''' Query DNSBL service for given value

        :param value: a valid hostname or a valid inverted ip address
        :returns: an integer representing classification code for given value, if it is listed. Otherwise,
        it returns None
        '''
        try:
            response = query(value+'.'+self._query_suffix)

        except NXDOMAIN:
            return None

        else:
            last_octet = response[0].to_text().split('.')[-1]

            return int(last_octet)

class DNSBL(object):
    ''' Represents a DNSBL service provider '''

    def __init__(self, identifier, query_suffix, code_item_class, lists_ips, lists_uris):
        ''' Create new DNSBL object

        :param identifier: a value designating DNSBL service provider: its name or url address.
        :param query_suffix: a suffix added to DNSBL query address
        :param code_item_class: item classes associated with DNSBL query return codes
        :param lists_ips: information if this object represents an ip blocklist
        :param lists_uris: information if this object represents a domain name blocklist
        '''

        self.identifier = identifier
        self._query_suffix = query_suffix
        self._code_item_class = code_item_class
        self.lists_ips = lists_ips
        self.lists_uris = lists_uris

    def get_classification(self, code):
        ''' Return classification for given code

        :param code: a valid return code extracted from response to DNSBL query
        :raises UnknownCodeError: raised when given code is not specified in self._code_item_class
        :returns: a value associated with a valid return code
        '''

        try:
            return self._code_item_class[code]

        except KeyError:

            msg_template = 'Using a code value "{}" unsupported by DNSBL instance representing {}'
            raise UnknownCodeError(msg_template.format(code, self.identifier)), None, exc_info()[2]

    def _get_host_and_query_prefix(self, host_collection):
        ''' Get valid hosts and  query prefixes from hosts given in host_collection

        :param host_collection: a container with valid host values
        :returns: a tuple with original host and a value prepared for lookup
        '''
        if self.lists_ips:
            for ip in host_collection.ips:
                suffix = '.in-addr.arpa' if ip.version == 4 else '.ip6.arpa'
                reverse = ip.reverse_pointer.replace(suffix, '')

                yield ip, reverse

        if self.lists_uris:
            for hostname in host_collection.hostnames:
                hostname = str(hostname)

                yield hostname, hostname.rstrip('.')

    def _query_for(self, host_collection):
        ''' Get hosts that are included both in this blocklist and in host_collection

        :param host_collection: a container with valid host values
        :returns: an item listed on this DNSBL, as an instance of DNSBLItem
        '''
        for host, prefix in self._get_host_and_query_prefix(host_collection):
            try:
                response = query(prefix+'.'+self._query_suffix)

            except NXDOMAIN:
                pass

            else:
                last_octet = response[0].to_text().split('.')[-1]
                yield DNSBLItem(host, self, last_octet)

    def contains_any(self, host_collection):
        ''' Check if given host collection contains any items listed on this DNSBL

        :param host_collection: a container with valid host values
        '''

        return any(self._query_for(host_collection))

    def lookup(self, host_collection):
        ''' Get items from this dns blocklist

        :param host_collection: a container with valud host values
        :returns: all hosts from this dns blocklist that are also included in host_collection
        '''
        return tuple(self._query_for(host_collection))

class HpHostsItem(object):
    ''' Represents a host listed in hpHosts'''

    def __init__(self, host, source, classification):

        self.host = host
        self.source = source
        self.classification = classification

class HpHosts(object):
    ''' hpHosts client '''

    identifier = ' http://www.hosts-file.net/'
    _LISTED = 'Listed'

    def __init__(self, client_name):
        '''
        Constructor

        :param client_name: name of client using the service
        '''

        self.app_id = client_name

    def _query(self, host, classification = False):
        ''' Query the client for data of given host

        :param host: a valid host string
        :param classification: if True: hpHosts is queried also for classification for given host, if listed
        :returns: content of response to GET request to hpHosts for data on the given host
        '''
        url = 'http://verify.hosts-file.net/?v={0}&s={1}'.format(self.app_id, host)
        url = url + '&class=true' if classification else url

        return get(url).content

    def __contains__(self, host):
        ''' Check if given host is present in hpHosts blacklist

        :param host: a valid host string
        :returns: a boolean value True if given host is listed on hpHosts, False otherwise
        '''
        return self._LISTED in self._query(host)

    def lookup(self, host):
        ''' Get an object representing a value for a given host, if listed in hpHosts

        :param host: a valid host string
        :returns: a HpHostItem object, or None if host is not listed
        '''
        data = self._query(host, True)

        if self._LISTED in data:
            elements = data.split(',')
            classification = elements[1] if len(elements) > 1 else None

            return HpHostsItem(host, self.identifier, classification)
        return None

if __name__ == '__main__':
    pass<commit_msg>Add DNSBLClient class to spambl module<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

from sys import exc_info
from dns.resolver import query, NXDOMAIN
from requests import get

class SpamBLError(Exception):
    ''' Base exception class for spambl module '''

class UnknownCodeError(SpamBLError):
    ''' Raise when trying to use an unexpected value of dnsbl return code '''

class DNSBLItem(object):
    ''' Represents a host listed on a DNS blacklist '''

    _classification = None

    def __init__(self, host, source, return_code):
        ''' Create a new instance of DNSBLItem

        :param host: the host value listed on a DNS blacklist, either host name or ip address
        :param source: dnsbl service object
        :param return_code: last octet of ip address returned after querying the source for the host
        '''
        self.host = host
        self.source = source
        self._return_code = return_code

    @property
    def classification(self):
        ''' Classification of this host according to provider of the list from which it has been extracted '''
        if not self._classification:
            self._classification = self.source.get_classification(self._return_code)

        return self._classification

class DNSBLService(object):
    ''' Represents a DNSBL service '''
    def __init__(self, identifier, query_suffix, code_item_class, lists_ips, lists_uris):
        ''' Create new DNSBLService object

        :param identifier: a value designating DNSBL service provider: its name or url address.
        :param query_suffix: a suffix added to DNSBL query address
        :param code_item_class: item classes associated with DNSBL query return codes
        :param lists_ips: information if this object represents an ip blocklist
        :param lists_uris: information if this object represents a domain name blocklist
        '''

        self.identifier = identifier
        self._query_suffix = query_suffix
        self._code_item_class = code_item_class
        self.lists_ips = lists_ips
        self.lists_uris = lists_uris

    def get_classification(self, code):
        ''' Return classification for given code

        :param code: a valid return code extracted from response to DNSBL query
        :raises UnknownCodeError: raised when given code is not specified in self._code_item_class
        :returns: a value associated with a valid return code
        '''

        try:
            return self._code_item_class[code]

        except KeyError:

            msg_template = 'Unexpected code value for dnsbl service {}: {}'
            raise UnknownCodeError(msg_template.format(self.identifier, code)), None, exc_info()[2]

    def query(self, value):
        ''' Query DNSBL service for given value

        :param value: a valid hostname or a valid inverted ip address
        :returns: an integer representing classification code for given value, if it is listed. Otherwise,
        it returns None
        '''
        try:
            response = query(value+'.'+self._query_suffix)

        except NXDOMAIN:
            return None

        else:
            last_octet = response[0].to_text().split('.')[-1]

            return int(last_octet)

class DNSBLClient(object):
    ''' Responsible for querying DNSBL services '''

    def __init__(self, *dnsbl_services):
        ''' Create new DNSBLClitent instance

        :param dnsbl_services: a list of objects representing DNSBL services
        '''
        self.dnsbls = []
        self.uri_dnsbls = []

        for d in dnsbl_services:
            if d.lists_ips:
                self.dnsbls.append(d)

            if d.lists_uris:
                self.uri_dnsbls.append(d)

class DNSBL(object):
    ''' Represents a DNSBL service provider '''

    def __init__(self, identifier, query_suffix, code_item_class, lists_ips, lists_uris):
        ''' Create new DNSBL object

        :param identifier: a value designating DNSBL service provider: its name or url address.
        :param query_suffix: a suffix added to DNSBL query address
        :param code_item_class: item classes associated with DNSBL query return codes
        :param lists_ips: information if this object represents an ip blocklist
        :param lists_uris: information if this object represents a domain name blocklist
        '''

        self.identifier = identifier
        self._query_suffix = query_suffix
        self._code_item_class = code_item_class
        self.lists_ips = lists_ips
        self.lists_uris = lists_uris

    def get_classification(self, code):
        ''' Return classification for given code

        :param code: a valid return code extracted from response to DNSBL query
        :raises UnknownCodeError: raised when given code is not specified in self._code_item_class
        :returns: a value associated with a valid return code
        '''

        try:
            return self._code_item_class[code]

        except KeyError:

            msg_template = 'Using a code value "{}" unsupported by DNSBL instance representing {}'
            raise UnknownCodeError(msg_template.format(code, self.identifier)), None, exc_info()[2]

    def _get_host_and_query_prefix(self, host_collection):
        ''' Get valid hosts and  query prefixes from hosts given in host_collection

        :param host_collection: a container with valid host values
        :returns: a tuple with original host and a value prepared for lookup
        '''
        if self.lists_ips:
            for ip in host_collection.ips:
                suffix = '.in-addr.arpa' if ip.version == 4 else '.ip6.arpa'
                reverse = ip.reverse_pointer.replace(suffix, '')

                yield ip, reverse

        if self.lists_uris:
            for hostname in host_collection.hostnames:
                hostname = str(hostname)

                yield hostname, hostname.rstrip('.')

    def _query_for(self, host_collection):
        ''' Get hosts that are included both in this blocklist and in host_collection

        :param host_collection: a container with valid host values
        :returns: an item listed on this DNSBL, as an instance of DNSBLItem
        '''
        for host, prefix in self._get_host_and_query_prefix(host_collection):
            try:
                response = query(prefix+'.'+self._query_suffix)

            except NXDOMAIN:
                pass

            else:
                last_octet = response[0].to_text().split('.')[-1]
                yield DNSBLItem(host, self, last_octet)

    def contains_any(self, host_collection):
        ''' Check if given host collection contains any items listed on this DNSBL

        :param host_collection: a container with valid host values
        '''

        return any(self._query_for(host_collection))

    def lookup(self, host_collection):
        ''' Get items from this dns blocklist

        :param host_collection: a container with valud host values
        :returns: all hosts from this dns blocklist that are also included in host_collection
        '''
        return tuple(self._query_for(host_collection))

class HpHostsItem(object):
    ''' Represents a host listed in hpHosts'''

    def __init__(self, host, source, classification):

        self.host = host
        self.source = source
        self.classification = classification

class HpHosts(object):
    ''' hpHosts client '''

    identifier = ' http://www.hosts-file.net/'
    _LISTED = 'Listed'

    def __init__(self, client_name):
        '''
        Constructor

        :param client_name: name of client using the service
        '''

        self.app_id = client_name

    def _query(self, host, classification = False):
        ''' Query the client for data of given host

        :param host: a valid host string
        :param classification: if True: hpHosts is queried also for classification for given host, if listed
        :returns: content of response to GET request to hpHosts for data on the given host
        '''
        url = 'http://verify.hosts-file.net/?v={0}&s={1}'.format(self.app_id, host)
        url = url + '&class=true' if classification else url

        return get(url).content

    def __contains__(self, host):
        ''' Check if given host is present in hpHosts blacklist

        :param host: a valid host string
        :returns: a boolean value True if given host is listed on hpHosts, False otherwise
        '''
        return self._LISTED in self._query(host)

    def lookup(self, host):
        ''' Get an object representing a value for a given host, if listed in hpHosts

        :param host: a valid host string
        :returns: a HpHostItem object, or None if host is not listed
        '''
        data = self._query(host, True)

        if self._LISTED in data:
            elements = data.split(',')
            classification = elements[1] if len(elements) > 1 else None

            return HpHostsItem(host, self.identifier, classification)
        return None

if __name__ == '__main__':
    pass<|endoftext|>


<commit_before>from mpi4py import MPI
import mpiunittest as unittest

_basic = [None,
          True, False,
          -7, 0, 7, 2**31,
          -2**63+1, 2**63-1,
          -2.17, 0.0, 3.14,
          1+2j, 2-3j,
          'mpi4py',
          ]
messages = _basic
messages += [ list(_basic),
              tuple(_basic),
              dict([('k%d' % key, val)
                    for key, val in enumerate(_basic)])
              ]
messages = messages + [messages]

def create_topo_comms(comm):
    size = comm.Get_size()
    rank = comm.Get_rank()
    # Cartesian
    n = int(size**1/2.0)
    m = int(size**1/3.0)
    if m*m*m == size:
        dims = [m, m, m]
    elif n*n == size:
        dims = [n, n]
    else:
        dims = [size]
    periods = [True] * len(dims)
    yield comm.Create_cart(dims, periods=periods)
    # Graph
    index, edges = [0], []
    for i in range(size):
        pos = index[-1]
        index.append(pos+2)
        edges.append((i-1)%size)
        edges.append((i+1)%size)
    yield comm.Create_graph(index, edges)
    # Dist Graph
    sources = [(rank-2)%size, (rank-1)%size]
    destinations = [(rank+1)%size, (rank+2)%size]
    yield comm.Create_dist_graph_adjacent(sources, destinations)

def get_neighbors_count(comm):
    topo = comm.Get_topology()
    if topo == MPI.CART:
        ndim = comm.Get_dim()
        return 2*ndim, 2*ndim
    if topo == MPI.GRAPH:
        rank = comm.Get_rank()
        nneighbors = comm.Get_neighbors_count(rank)
        return nneighbors, nneighbors
    if topo == MPI.DIST_GRAPH:
        indeg, outdeg, w = comm.Get_dist_neighbors_count()
        return indeg, outdeg
    return 0, 0

class BaseTestCCONghObj(object):

    COMM = MPI.COMM_NULL

    def testNeighborAllgather(self):
        for comm in create_topo_comms(self.COMM):
            rsize, ssize = get_neighbors_count(comm)
            for smess in messages:
                rmess = comm.neighbor_allgather(smess)
                self.assertEqual(rmess, [smess] * rsize)
            comm.Free()

    def testNeighborAlltoall(self):
        for comm in create_topo_comms(self.COMM):
            rsize, ssize = get_neighbors_count(comm)
            for smess in messages:
                rmess = comm.neighbor_alltoall([smess] * ssize)
                self.assertEqual(rmess, [smess] * rsize)
            comm.Free()

class TestCCONghObjSelf(BaseTestCCONghObj, unittest.TestCase):
    COMM = MPI.COMM_SELF

class TestCCONghObjWorld(BaseTestCCONghObj, unittest.TestCase):
    COMM = MPI.COMM_WORLD

class TestCCONghObjSelfDup(TestCCONghObjSelf):
    def setUp(self):
        self.COMM = MPI.COMM_SELF.Dup()
    def tearDown(self):
        self.COMM.Free()

class TestCCONghObjWorldDup(TestCCONghObjWorld):
    def setUp(self):
        self.COMM = MPI.COMM_WORLD.Dup()
    def tearDown(self):
        self.COMM.Free()

name, version = MPI.get_vendor()
cartcomm = MPI.COMM_SELF.Create_cart([1], periods=[0])
try:
    try:
        cartcomm.neighbor_allgather(None)
    finally:
        cartcomm.Free()
        del cartcomm
    if name == 'Open MPI':
        if version < (1,8,4):
            _create_topo_comms = create_topo_comms
            def create_topo_comms(comm):
                for c in _create_topo_comms(comm):
                    if c.size * 2 < sum(c.degrees):
                        c.Free(); continue
                    yield c
except NotImplementedError:
    del TestCCONghObjSelf
    del TestCCONghObjWorld
    del TestCCONghObjSelfDup
    del TestCCONghObjWorldDup

if __name__ == '__main__':
    unittest.main()
<commit_msg>test: Disable test for Comm.neighbor_allgather() with Open MPI<commit_after>from mpi4py import MPI
import mpiunittest as unittest

_basic = [None,
          True, False,
          -7, 0, 7, 2**31,
          -2**63+1, 2**63-1,
          -2.17, 0.0, 3.14,
          1+2j, 2-3j,
          'mpi4py',
          ]
messages = _basic
messages += [ list(_basic),
              tuple(_basic),
              dict([('k%d' % key, val)
                    for key, val in enumerate(_basic)])
              ]
messages = messages + [messages]

def create_topo_comms(comm):
    size = comm.Get_size()
    rank = comm.Get_rank()
    # Cartesian
    n = int(size**1/2.0)
    m = int(size**1/3.0)
    if m*m*m == size:
        dims = [m, m, m]
    elif n*n == size:
        dims = [n, n]
    else:
        dims = [size]
    periods = [True] * len(dims)
    yield comm.Create_cart(dims, periods=periods)
    # Graph
    index, edges = [0], []
    for i in range(size):
        pos = index[-1]
        index.append(pos+2)
        edges.append((i-1)%size)
        edges.append((i+1)%size)
    yield comm.Create_graph(index, edges)
    # Dist Graph
    sources = [(rank-2)%size, (rank-1)%size]
    destinations = [(rank+1)%size, (rank+2)%size]
    yield comm.Create_dist_graph_adjacent(sources, destinations)

def get_neighbors_count(comm):
    topo = comm.Get_topology()
    if topo == MPI.CART:
        ndim = comm.Get_dim()
        return 2*ndim, 2*ndim
    if topo == MPI.GRAPH:
        rank = comm.Get_rank()
        nneighbors = comm.Get_neighbors_count(rank)
        return nneighbors, nneighbors
    if topo == MPI.DIST_GRAPH:
        indeg, outdeg, w = comm.Get_dist_neighbors_count()
        return indeg, outdeg
    return 0, 0

class BaseTestCCONghObj(object):

    COMM = MPI.COMM_NULL

    def testNeighborAllgather(self):
        for comm in create_topo_comms(self.COMM):
            rsize, ssize = get_neighbors_count(comm)
            for smess in messages:
                rmess = comm.neighbor_allgather(smess)
                self.assertEqual(rmess, [smess] * rsize)
            comm.Free()

    def testNeighborAlltoall(self):
        for comm in create_topo_comms(self.COMM):
            rsize, ssize = get_neighbors_count(comm)
            for smess in messages:
                rmess = comm.neighbor_alltoall([smess] * ssize)
                self.assertEqual(rmess, [smess] * rsize)
            comm.Free()

class TestCCONghObjSelf(BaseTestCCONghObj, unittest.TestCase):
    COMM = MPI.COMM_SELF

class TestCCONghObjWorld(BaseTestCCONghObj, unittest.TestCase):
    COMM = MPI.COMM_WORLD

class TestCCONghObjSelfDup(TestCCONghObjSelf):
    def setUp(self):
        self.COMM = MPI.COMM_SELF.Dup()
    def tearDown(self):
        self.COMM.Free()

class TestCCONghObjWorldDup(TestCCONghObjWorld):
    def setUp(self):
        self.COMM = MPI.COMM_WORLD.Dup()
    def tearDown(self):
        self.COMM.Free()

name, version = MPI.get_vendor()
cartcomm = MPI.COMM_SELF.Create_cart([1], periods=[0])
try:
    try:
        cartcomm.neighbor_allgather(None)
    finally:
        cartcomm.Free()
        del cartcomm
    if name == 'Open MPI':
        if version < (2,1,0):
            del BaseTestCCONghObj.testNeighborAllgather
        if version < (1,8,4):
            _create_topo_comms = create_topo_comms
            def create_topo_comms(comm):
                for c in _create_topo_comms(comm):
                    if c.size * 2 < sum(c.degrees):
                        c.Free(); continue
                    yield c
except NotImplementedError:
    del TestCCONghObjSelf
    del TestCCONghObjWorld
    del TestCCONghObjSelfDup
    del TestCCONghObjWorldDup

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import numpy as np
import hexy as hx

hm = hx.HexMap()
radius = 10
coords = hx.get_spiral(np.array((0, 0, 0)), 1, 10)

def test_axial_to_cube_conversion():
    axial_coords = hx.cube_to_axial(coords)
    cube_coords = hx.axial_to_cube(axial_coords)

    assert np.array_equal(coords, cube_coords)

def test_axial_to_pixel_conversion():
    axial_coords = hx.cube_to_axial(coords)
    pixel_coords = hx.axial_to_pixel(axial_coords, radius)
    pixel_to_axial_coords = hx.pixel_to_axial(pixel_coords, radius)

    assert np.array_equal(axial_coords, pixel_to_axial_coords)

def test_cube_to_pixel_conversion():
    axial_coords = hx.cube_to_axial(coords)
    cube_coords = hx.axial_to_cube(axial_coords)
    pixel_coords = hx.cube_to_pixel(cube_coords, radius)
    pixel_to_cube_coords = hx.pixel_to_cube(pixel_coords, radius)
    pixel_to_cube_to_axial_coords = hx.cube_to_axial(pixel_to_cube_coords)

    assert np.array_equal(cube_coords, pixel_to_cube_coords)

def test_the_converted_coords_and_dataset_coords_retrieve_the_same_data():
    axial_coords = hx.cube_to_axial(coords)
    cube_coords = hx.axial_to_cube(axial_coords)
    pixel_coords = hx.cube_to_pixel(cube_coords, radius)
    pixel_to_cube_coords = hx.pixel_to_cube(pixel_coords, radius)
    pixel_to_cube_to_axial_coords = hx.cube_to_axial(pixel_to_cube_coords)

    # check that we can correctly retrieve hexes after conversions
    hm[axial_coords] = coords
    retrieved = hm[pixel_to_cube_to_axial_coords]

    assert np.array_equal(retrieved, cube_coords)

<commit_msg>Fixed a conversion test<commit_after>import numpy as np
import hexy as hx

hm = hx.HexMap()
radius = 10
coords = hx.get_spiral(np.array((0, 0, 0)), 1, 10)

def test_axial_to_cube_conversion():
    axial_coords = hx.cube_to_axial(coords)
    cube_coords = hx.axial_to_cube(axial_coords)

    assert np.array_equal(coords, cube_coords)

def test_axial_to_pixel_conversion():
    axial_coords = hx.cube_to_axial(coords)
    pixel_coords = hx.axial_to_pixel(axial_coords, radius)
    pixel_to_axial_coords = hx.pixel_to_axial(pixel_coords, radius)

    assert np.array_equal(axial_coords, pixel_to_axial_coords)

def test_cube_to_pixel_conversion():
    axial_coords = hx.cube_to_axial(coords)
    cube_coords = hx.axial_to_cube(axial_coords)
    pixel_coords = hx.cube_to_pixel(cube_coords, radius)
    pixel_to_cube_coords = hx.pixel_to_cube(pixel_coords, radius)
    pixel_to_cube_to_axial_coords = hx.cube_to_axial(pixel_to_cube_coords)

    assert np.array_equal(cube_coords, pixel_to_cube_coords)

def test_the_converted_coords_and_dataset_coords_retrieve_the_same_data():
    axial_coords = hx.cube_to_axial(coords)
    cube_coords = hx.axial_to_cube(axial_coords)
    pixel_coords = hx.cube_to_pixel(cube_coords, radius)
    pixel_to_cube_coords = hx.pixel_to_cube(pixel_coords, radius)
    pixel_to_cube_to_axial_coords = hx.cube_to_axial(pixel_to_cube_coords)

    # check that we can correctly retrieve hexes after conversions
    hm[axial_coords] = coords
    retrieved = hm[pixel_to_cube_to_axial_coords]

    assert np.array_equal(retrieved, coords)

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2015-08-04 23:44:38
# @Author  : Alexa (AlexaZhou@163.com)
# @Link    :
# @Disc    :

import os
import os.path
import resize
import sys
import getopt
from mod_pbxproj import XcodeProject

input_base_dir = "ImageBuilderResources"
output_base_dir = "XcodeImageOutputs"

project_file = ""
xcode_group = ""

def resize_img_in_path(input_path, output_path):

    for parent,dirnames,filenames in os.walk(input_path):

        output_parent = output_base_dir + parent[len(input_base_dir):]

        if os.path.exists(output_parent) == False:
            os.mkdir(output_parent)

        for f in filenames:
            if f.startswith("."):
                continue

            if "@3x" not in f and "@6x" not in f:
                continue

            resize.resize_img( parent+os.sep+f, (output_parent+os.sep+f).replace("@6x","").replace("@3x","") )

def xcode_get_or_create_group_by_path( project_obj, xcode_path):

    group_names = xcode_path.split("/")
    group = None
    for name in group_names:
        name = name.decode("utf-8")

        group = project_obj.get_or_create_group( name, parent= group)

    return group

def xcode_project_sync(file_path, xcode_group):

    if xcode_group == "":
        print("INFO: not specify xcode group, so ignore sync imgs to Xcode project")
        return False

    project_obj = XcodeProject.Load(project_file)
    modifyed = False

    for parent,dirnames,filenames in os.walk(file_path):

        xcode_path = xcode_group + parent[len(file_path):]
        group = xcode_get_or_create_group_by_path( project_obj, xcode_path )

        for f in filenames:
            if f.startswith("."):
                continue

            name = (parent+os.sep+f).decode("utf-8")

            if len(project_obj.get_files_by_name(f.decode("utf-8"), parent = group)) == 0:
                modifyed = True
                print("sync file to Xcode Group:%s"%name.encode("utf-8"))
                project_obj.add_file( name , parent = group )

    if modifyed:
        print("now save project file")
        project_obj.save()
        print("success!\nAnd now use xcproj to retouch project, make it what it was")
        retouch_ret = os.system('xcproj -p "%s" touch'%project_file)
        if retouch_ret == 0:
            print("success!")
        else:
            print("please make sure xcproj tool installed. xcproj can make .pbxproj file look pretty")

        return True
    else:
        return False

def main():
    global input_base_dir, output_base_dir, project_file, xcode_group

    opts, args = getopt.getopt(sys.argv[1:], 'hi:o:g:p:', ["help"])

    for option, value in opts:
        #print("option:%s --> value:%s"%(option, value))

        if option == "-i":
            input_base_dir = value
        elif option == "-o":
            output_base_dir = value
        elif option == "-g":
            xcode_group = value
        elif option == "-p":
            project_file = value
        elif option == "-h" or option == "--help":
            print("use: spider.py -i [image input path] -o [image output path] -g [Xcode group] -p [Xcode .pbxproj file path]")
            sys.exit(1)

    resize_img_in_path( input_base_dir, output_base_dir )
    project_modifyed = xcode_project_sync(output_base_dir, xcode_group)

    if project_modifyed:
        print("images synced to xcode group, please build again")
        sys.exit(1)

if __name__ == '__main__':
    main()
else:
    print ('spider.py had been imported as a module')

<commit_msg>check xproj firsh<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2015-08-04 23:44:38
# @Author  : Alexa (AlexaZhou@163.com)
# @Link    :
# @Disc    :

import os
import os.path
import resize
import sys
import getopt
from mod_pbxproj import XcodeProject

input_base_dir = "ImageBuilderResources"
output_base_dir = "XcodeImageOutput"

project_file = ""
xcode_group = ""

def resize_img_in_path(input_path, output_path):

    for parent,dirnames,filenames in os.walk(input_path):

        output_parent = output_base_dir + parent[len(input_base_dir):]

        if os.path.exists(output_parent) == False:
            os.mkdir(output_parent)

        for f in filenames:
            if f.startswith("."):
                continue

            if "@3x" not in f and "@6x" not in f:
                continue

            resize.resize_img( parent+os.sep+f, (output_parent+os.sep+f).replace("@6x","").replace("@3x","") )

def xcode_get_or_create_group_by_path( project_obj, xcode_path):

    group_names = xcode_path.split("/")
    group = None
    for name in group_names:
        name = name.decode("utf-8")

        group = project_obj.get_or_create_group( name, parent= group)

    return group

def xcode_project_sync(file_path, xcode_group):

    if xcode_group == "":
        print("INFO: not specify xcode group, so ignore sync imgs to Xcode project")
        return False

    ret = os.system("xcproj -h")
    print("ret",ret)

    if ret != 0:
        print("please make sure xcproj tool installed. xcproj can make .pbxproj file look pretty")
        return False

    project_obj = XcodeProject.Load(project_file)
    modifyed = False

    for parent,dirnames,filenames in os.walk(file_path):

        xcode_path = xcode_group + parent[len(file_path):]
        group = xcode_get_or_create_group_by_path( project_obj, xcode_path )

        for f in filenames:
            if f.startswith("."):
                continue

            name = (parent+os.sep+f).decode("utf-8")

            if len(project_obj.get_files_by_name(f.decode("utf-8"), parent = group)) == 0:
                modifyed = True
                print("sync file to Xcode Group:%s"%name.encode("utf-8"))
                project_obj.add_file( name , parent = group )

    if modifyed:
        print("now save project file")
        project_obj.save()
        print("success!\nAnd now use xcproj to retouch project, make it what it was")

        retouch_ret = os.system('xcproj -p "%s" touch'%project_file)
        if retouch_ret == 0:
            print("success!")
        else:
            print("some thing was error, call xproj failed.")

        return True
    else:
        return False

def main():
    global input_base_dir, output_base_dir, project_file, xcode_group

    opts, args = getopt.getopt(sys.argv[1:], 'hi:o:g:p:', ["help"])

    for option, value in opts:
        #print("option:%s --> value:%s"%(option, value))

        if option == "-i":
            input_base_dir = value
        elif option == "-o":
            output_base_dir = value
        elif option == "-g":
            xcode_group = value
        elif option == "-p":
            project_file = value
        elif option == "-h" or option == "--help":
            print("use: spider.py -i [image input path] -o [image output path] -g [Xcode group] -p [Xcode .pbxproj file path]")
            sys.exit(1)

    resize_img_in_path( input_base_dir, output_base_dir )
    project_modifyed = xcode_project_sync(output_base_dir, xcode_group)

    if project_modifyed:
        print("images synced to xcode group, please build again")
        sys.exit(1)

if __name__ == '__main__':
    main()
else:
    print ('spider.py had been imported as a module')

<|endoftext|>


<commit_before>#!/usr/bin/env
# -*- coding: utf-8 -*-
# __author__ = 'QY-Y'

import os
import re
import sqlite3
import time
import sys
import Queue
import getopt
import logging
import threading
import signal
import urllib2
import traceback
import curses
import lxml.html
import hashlib
import gzip
from StringIO import StringIO

IS_EXIT = False
def handler(signum, frame):
	global IS_EXIT
	IS_EXIT = True
# Host: www.baidu.com
# User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:48.0) Gecko/20100101 Firefox/48.0
# Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
# Accept-Language: en-US,en;q=0.5
# Accept-Encoding: gzip, deflate, br
# Cookie: BAIDUID=53782338724485C9E25F994B5C4CF241:FG=1; BIDUPSID=53782338724485C9E25F994B5C4CF241; PSTM=1461994803; BD_UPN=133352; H_PS_PSSID=1462_17942_11598_20847_20857_20837_20771; H_PS_645EC=aceedZSUvnyvX0JXU8siUp6p87RynkqgVFCSJRLh5fsr1mAbFYAE6qGyEr4; BD_CK_SAM=1; BD_HOME=0; BDRCVFR[eHt_ClL0b_s]=mk3SLVN4HKm
# Connection: keep-alive
# Upgrade-Insecure-Requests: 1
# Pragma: no-cache
# Cache-Control: no-cache

def _requestData(url):
	req = urllib2.Request(url)
	req.add_header('User-Agent',
		'Mozilla/5.0 (Windows NT 6.2; rv:16.0) Gecko/20100101 Firefox/48.0')
	req.add_header('Accept-Encoding','gzip')
	req.add_header('Accept-Language','en-US,en;q=0.5')
	try:
		res = urllib2.urlopen(url,timeout = 10)
		content = res.read()
		# try:
			# content = content.decode('gb2312','ignore')
		# except:
			# content = content.decode('utf-8', 'ignore')
		if res.info().get('Content-Encoding') == 'gzip':
			buf = StringIO(content)
			content = gzip.GzipFile(fileobj=buf).read()
		return req.get_type(), req.get_host(), content
	except Exception,e:
		logging.warning(' open['+url+'] failed '+str(e))
		return req.get_type(), req.get_host(), False

def _initDB(dbFile):
	'''init database file create table(if necessary)
	Args:
		dbFile: Name of the Database file
	Returns:
		db: A sqlite3 connection obejct
		c: A cursor from db'''
	exist = False
	ls = os.listdir('.')
	if dbFile in ls:
		exist = True
	db = sqlite3.connect(dbFile, check_same_thread=False)
	c = db.cursor()
	if not exist:
		try:
			c.execute('create table spider(id integer primary key,url text,key text,content text)')
			db.commit()
		except sqlite3.OperationalError:
			logging.cratical(dbFile+'Error in creating table')
	return db, c

def _insert(url, keyWord, content, savedCount):
	'''insert info to defualt database
	Args:
		url: The url to be saved
		keyWord: Key word
		content: Data
	'''
	content = urllib2.quote(str(content))
	try:
		c.execute('insert into spider(url,key,content) values("'
				  +url+'","'+keyWord+'","'+content+'")')
		db.commit()
		savedCount.put(url)
	except sqlite3.OperationalError:
		logging.critical(' insert ['+url+'] error')

class worker(threading.Thread):
	def __init__(self, links, keys, rlock, urlset, md5set, savedCount):
		threading.Thread.__init__(self)
		self.taskQueue = links
		self.keyList = keys
		self.rlock = rlock
		self.link = None
		self.depth = None
		self.setDaemon(True)
		self.start()
		self.exc_traceback = ''
		self.urlset = urlset
		self.md5set = md5set
		self.count = 0
		self.savedCount = savedCount

	def run(self):
		while True:
			try:
				self.link, self.depth = self.taskQueue.get(timeout=2)
			except Exception,e:
				logging.info("get task error")
				break
			try:
				reshost, restype, data = _requestData(self.link)
				if not data:
					self.taskQueue.task_done()
					continue
				self.count += 1
				md5 = hashlib.md5(data).hexdigest()
				if self.depth > 0:
					self.depth -= 1
					subLinks = self.getLinks(data, reshost, restype)
				else:
					subLinks = []
				for subLink in subLinks:
					if not (subLink[0] in self.urlset):
						self.urlset.add((subLink[0]))
						self.taskQueue.put(subLink)
				if not (md5 in self.md5set):
					self.save(data)
					self.md5set.add(md5)
			except Exception,e:
				logging.critical(str(e))
			self.taskQueue.task_done()

	def save(self,data):
		if not data:
			return
		for key in self.keyList:
			if data.find(key)>0:
				logging.info(" ["+key+"] found in ["+self.link+"]")
				self.rlock.acquire()
				_insert(self.link, key, data, self.savedCount)
				self.rlock.release()
				break

	def getLinks(self, data, resType, resHost):
		if not data:
			return []
		host = resType+'://'+resHost
		try:
			data = data.decode('utf8', 'ignore')
			doc = lxml.html.document_fromstring(data)
		except Exception,e:
			logging.critical(str(e))

		tags = ['a', 'iframe', 'frame']
		doc.make_links_absolute(host)
		links = doc.iterlinks()
		newLinks = []
		absoluteLinks = []
		for l in links:
			if l[0].tag in tags:
				newLinks.append(l)
		del links
		for link in newLinks:
			if re.match('http',link[2]):
				absoluteLinks.append((link[2],self.depth))
		return absoluteLinks

class threadPool:
	def __init__(self, num, event, url, depth, keyWords):
		self.num = num
		self.event = event
		self.threads = []
		self.tasks = Queue.Queue()
		self.tasks.put((url, depth))
		self.key = keyWords
		self.url = set()
		self.url.add(url)
		self.urlMD5 = set()
		self.savedCount = Queue.Queue()
		for i in range(self.num):
			newThread = worker(self.tasks, self.key, rlock, self.url, self.urlMD5, self.savedCount)
			self.threads.append(newThread)
		logging.info(" pool init done, " + str(self.num) + " woreker created")

	def supervisor(self):
		global IS_EXIT
		while self.tasks.unfinished_tasks:
			if IS_EXIT:
				try:
					self.tasks.empty()
					self.event.set()
				except Exception,e:
					logging.critical(str(e))
				return
			width = 50
			now = self.tasks.qsize()
			total = len(self.url)
			percent = (float(now)/float(total))
			percent = int((1 - percent)*100)
			printinfo = ('[%%-%ds' % width) % (width * percent / 100 * '=')
			printinfo += '] ' + str(percent) + "%    "
			printinfo += str(len(self.url) - self.tasks.qsize())
			printinfo += '/' + str(len(self.url)) + '\r'
			sys.stdout.write((len(printinfo)+5)*' '+'\r')
  			sys.stdout.flush()
			sys.stdout.write(printinfo)
  			sys.stdout.flush()
			time.sleep(1)

		sys.stdout.write('\n')
		print self.savedCount.qsize(),"pages saved. All tasks done at",time.ctime()
		self.event.set()
		return

def _usage():
	'''print usage'''

	msg = '''usage:
python spider.py -u [URL] -d [Depth] -f [Log File] -l [Level] --thread [Thread Number] \
--dbfile [Database File Name] --key [Key Word]
-u  Begining URL ,must start with http or https
-d  The depth of the spider ,defalut: [1]
-f  Name(path) of the log file ,default: [spider.log]
-l  How detailed the logging should be(1-5), default: [1]
--thread  The capability of the threading pool, default: [10]
--dbfile  Name of the database file, default: [spider.db]
--key (Optional) Key word
--testself (Optional) Test module
-h  Print this page'''
	print msg

def mainHandler(threadNum, url, depth, keyWord, test):
	event = threading.Event()
	event.clear()
	pool = threadPool(num = threadNum, event = event, url = url, depth = depth, keyWords = keyWord)
	pool.supervisor()

if __name__ == '__main__':
	print time.ctime()
	os.system("rm spider.db spider.log")
	signal.signal(signal.SIGINT, handler)
	signal.signal(signal.SIGTERM, handler)
	rlock = threading.RLock()
	url = None
	depth = 1
	logFile = 'spider.log'
	level = 1
	threadNum = 1
	dbFile = 'spider.db'
	# key = ['中国']
	key = ['HTML5']

	optlist, args = getopt.getopt(
		sys.argv[1:],
		'u:d:f:l:h',
		['thread=', 'dbfile=', 'key='])
	for k, v in optlist:
		if k == '-u':
			url = v
		elif k == '-d':
			depth = int(v)
		elif k == '-f':
			logFile = v
		elif k == '-l':
			level = int(v)
		elif k == '--thread':
			threadNum = int(v)
		elif k == '--dbfile':
			dbFile = v
		elif k == '--key':
			key = v
		elif k == '-h':
			_usage()
			exit()

	if (not 0 < level < 6) or (depth < 1) or (threadNum < -100) or (not url):
		print 'input error'
		_usage()
		exit()
	db, c = _initDB(dbFile)
	logLevel = {
		1: logging.CRITICAL,
		2: logging.ERROR,
		3: logging.WARNING,
		4: logging.INFO,
		5: logging.DEBUG,
	}
	logging.basicConfig(filename=logFile, level=logLevel[level],
						format='%(asctime)s-%(levelname)s-%(filename)s-'
						'%(threadName)s-%(message)s', datefmt='[%d/%b/%Y %H:%M:%S]',)
	testself = False
	mainHandler(threadNum, url, depth-1, key, testself)<commit_msg>add self-test<commit_after>#!/usr/bin/env
# -*- coding: utf-8 -*-
# __author__ = 'QY-Y'

import os
import time
import sys
import Queue
import re
import getopt
import logging
import threading
import signal
import sqlite3
import urllib2
import lxml.html
import hashlib
import gzip
from StringIO import StringIO
from progressbar import *

IS_EXIT = False

# Description : chang global variable IS_EXIT.
def handler(signum, frame):
    global IS_EXIT
    IS_EXIT = True

# Description : send request to url,return contentof response.

# Output :
#     req.get_type()
#     req.get_host()
#     content
def request_url(url):
    req = urllib2.Request(url)
    req.add_header(
        'User-Agent',
        'Mozilla/5.0 (Windows NT 6.2; rv:16.0) Gecko/20100101 Firefox/48.0')
    req.add_header('Accept-Encoding', 'gzip')
    req.add_header('Accept-Language', 'en-US,en;q=0.5')
    try:
        res = urllib2.urlopen(url, timeout=10)
        content = res.read()
        if res.info().get('Content-Encoding') == 'gzip':
            buf = StringIO(content)
            content = gzip.GzipFile(fileobj=buf).read()
        return req.get_type(), req.get_host(), content
    except Exception as e:
        logging.warning(' open[' + url + '] failed ' + str(e))
        return req.get_type(), req.get_host(), False
# Description : init database file create table(if necessary)
# Input:
#     dbFile: Name of the Database file
# Output:
#     db: A sqlite3 connection obejct
#     c: A cursor from db
def init_data_base(database):

    exist = False
    files = os.listdir('.')
    if database in files:
        exist = True
    db = sqlite3.connect(database, check_same_thread=False)
    c = db.cursor()
    if not exist:
        try:
            c.execute(
                'create table spider(id integer primary key,url text,key text,content text)')
            db.commit()
        except sqlite3.OperationalError:
            logging.cratical(database + 'Error in creating table')
    return db, c

# Description : insert data to database
def insert_data(url, key_word, content, saved_count):
    content = urllib2.quote(str(content))
    try:
        c.execute('insert into spider(url,key,content) values("'
                  + url + '","' + key_word + '","' + content + '")')
        db.commit()
        saved_count.put(url)
    except sqlite3.OperationalError:
        logging.critical(' insert [' + url + '] error')

# Description : WORKER: search and save pages
class WORKER(threading.Thread):

    def __init__(self, links, keys, rlock, url_set, md5_set, saved_count):
        threading.Thread.__init__(self)
        self.task_queue = links
        self.key_list = keys
        self.rlock = rlock
        self.link = None
        self.depth = None
        self.setDaemon(True)
        self.start()
        self.url_set = url_set
        self.md5_set = md5_set
        self.count = 0
        self.saved_count = saved_count

#    Description : 1.get task from task queue.
#                   2.get sub links
#                   3.save pages
    def run(self):
        while True:
            try:
                self.link, self.depth = self.task_queue.get(timeout=2)
            except Exception as e:
                logging.info("get task error")
                break
            try:
                res_host, res_type, data = request_url(self.link)
                if not data:
                    self.task_queue.task_done()
                    continue
                self.count += 1
                md5 = hashlib.md5(data).hexdigest()
                if self.depth > 0:
                    self.depth -= 1
                    sub_link_list = self.getLinks(data, res_host, res_type)
                else:
                    sub_link_list = []
                for sub_link in sub_link_list:
                    if not (sub_link[0] in self.url_set):
                        self.url_set.add((sub_link[0]))
                        self.task_queue.put(sub_link)
                if not (md5 in self.md5_set):
                    self.save(data)
                    self.md5_set.add(md5)
            except Exception as e:
                logging.critical(str(e))
            self.task_queue.task_done()

#    Description : save pages if key word found
    def save(self, data):
        if not data:
            return
        for key in self.key_list:
            if data.find(key) > 0:
                logging.info(" [" + key + "] found in [" + self.link + "]")
                self.rlock.acquire()
                insert_data(self.link, key, data, self.saved_count)
                self.rlock.release()
                break
#    Description : get sub links,returns a list
    def getLinks(self, data, res_type, res_host):
        if not data:
            return []
        host = res_type + '://' + res_host
        try:
            data = data.decode('utf8', 'ignore')
            doc = lxml.html.document_fromstring(data)
        except Exception as e:
            logging.critical(str(e))
        tags = ['a', 'iframe', 'frame']
        doc.make_links_absolute(host)
        links = doc.iterlinks()
        new_link_list = []
        absolute_Link_list = []
        for l in links:
            if l[0].tag in tags:
                new_link_list.append(l)
        del links
        for link in new_link_list:
            if re.match('http', link[2]):
                absolute_Link_list.append((link[2], self.depth))
        return absolute_Link_list

# Description : manage the task queue and the thread pool,meanwhile show progress
class THREAD_POOL:

    def __init__(self, num, event, url, depth, key_word_list):
        self.num = num
        self.event = event
        self.threads = []
        self.task_queue = Queue.Queue()
        self.task_queue.put((url, depth))
        self.key_list = key_word_list
        self.url_set = set()
        self.url_set.add(url)
        self.md5_set = set()
        self.saved_count = Queue.Queue()
        for i in range(self.num):
            new_thread = WORKER(
                self.task_queue,
                self.key_list,
                rlock,
                self.url_set,
                self.md5_set,
                self.saved_count)
            self.threads.append(new_thread)
        logging.info(" pool init done, " + str(self.num) + " woreker created")
# Description : show progress every 0.5 second and check the global variable IS_EXIT
#           return if IS_EXIT is true or all tasks is done.
    def show_percent(self):
        global IS_EXIT
        while self.task_queue.unfinished_tasks:
            if IS_EXIT:
                try:
                    self.task_queue.empty()
                    self.event.set()
                except Exception as e:
                    logging.critical(str(e))
                return
            width = 50
            now = self.task_queue.qsize()
            total = len(self.url_set)
            percent = (float(now) / float(total))
            percent = int((1 - percent) * 100)
            info = ('[%%-%ds' % width) % (width * percent / 100 * '=')
            info += '] ' + str(percent) + "%    "
            info += str(len(self.url_set) - self.task_queue.qsize())
            info += '/' + str(len(self.url_set)) + '\r'
            sys.stdout.write((len(info) + 5) * ' ' + '\r')
            sys.stdout.flush()
            sys.stdout.write(info)
            sys.stdout.flush()

            time.sleep(0.5)

        sys.stdout.write('\n')
        print self.saved_count.qsize(), "pages saved. All tasks done at", time.ctime()
        self.event.set()
        return

class testSameDB(threading.Thread):

    def __init__(self, cursor, md5, progress):
        threading.Thread.__init__(self)
        self.c = cursor
        self.count = 0
        self.md5 = md5
        self.progress = progress
        self.start()

    def run(self):
        while True:
                # get 10,000 contents each time
            self.c.execute('select content from spider limit %s,%s'
                           % (self.count, self.count + 10000))
            self.count += 10000
            contents = self.c.fetchall()
            if len(contents) == 0:
                break
            for c in contents:
                res = hashlib.md5(c[0].encode('utf8'))
                self.md5.add(res.hexdigest())
                self.progress[0] = self.count

# checke if there are duplicate pages in database(by MD5)
def test(dbFile):
    progress = [0]
    md5 = set()
    # get count of all pages in database.
    db = sqlite3.connect(dbFile, check_same_thread=False)
    c = db.cursor()
    c.execute('select count(*) from spider')
    total_num = c.fetchall()[0][0]
    if total_num is 0:
        logging.warning("0 content in database")
        return
    t = testSameDB(c, md5, progress)
    t.join()
    pBar = ProgressBar(widgets=[Percentage(), Bar()],
                       maxval=total_num).start()
    while progress[0] < total_num:
        pBar.update(progress[0] + 1)
    pBar.finish()
    if len(md5) == total_num:
        print('no duplicate pages found in database')
    else:
        print('duplicate pages found in database')
# Description : print usage
def _usage():

    msg = '''usage:
python spider.py -u [URL] -d [Depth] -f [Log File] -l [Level] --thread [Thread Number] \
--dbfile [Database File Name] --key [Key Word]
-u  Begining URL ,must start with http or https
-d  The depth of the spider ,defalut: [1]
-f  Name(path) of the log file ,default: [spider.log]
-l  How detailed the logging should be(1-5), default: [1]
--thread  The capability of the threading pool, default: [10]
--dbfile  Name of the database file, default: [spider.db]
--key (Optional) Key word
--testself (Optional) Test module
-h  Print this page'''
    print msg

def main_handler(thread_num, url, depth, key_word_list, is_test):
    event = threading.Event()
    event.clear()
    pool = THREAD_POOL(
        num=thread_num,
        event=event,
        url=url,
        depth=depth,
        key_word_list=key_word_list)
    pool.show_percent()
    if is_test:
        test(dbFile)

if __name__ == '__main__':
    print time.ctime()
    signal.signal(signal.SIGINT, handler)
    signal.signal(signal.SIGTERM, handler)
    rlock = threading.RLock()
    url = None
    depth = 1
    logFile = 'spider.log'
    level = 1
    thread_num = 1
    dbFile = 'spider.db'
    key_list = ['HTML5']
    if '--testself' in sys.argv:
        is_test = True
        sys.argv.remove('--testself')
        os.system('rm spider.log spider.db')
    else:
        is_test = False
    optlist, args = getopt.getopt(
        sys.argv[1:],
        'u:d:f:l:h',
        ['thread=', 'dbfile=', 'key=','testself='])
    for k, v in optlist:
        if k == '-u':
            url = v
        elif k == '-d':
            depth = int(v)
        elif k == '-f':
            logFile = v
        elif k == '-l':
            level = int(v)
        elif k == '--thread':
            threadNum = int(v)
        elif k == '--dbfile':
            dbFile = v
        elif k == '--key':
            key_list = v
        elif k == '-h':
            _usage()
            exit()

    if (not 0 < level < 6) or (depth < 1) or (thread_num < -100) or (not url):
        print 'input error'
        _usage()
        exit()
    db, c = init_data_base(dbFile)
    logLevel = {
        1: logging.CRITICAL,
        2: logging.ERROR,
        3: logging.WARNING,
        4: logging.INFO,
        5: logging.DEBUG,
    }
    logging.basicConfig(
        filename=logFile,
        level=logLevel[level],
        format='%(asctime)s-%(levelname)s-%(filename)s-'
        '%(threadName)s-%(message)s',
        datefmt='[%d/%b/%Y %H:%M:%S]',
    )
    main_handler(thread_num, url, depth - 1, key_list, is_test)<|endoftext|>


<commit_before>#! /usr/bin/env python
# Taken from http://www.pygame.org/project-Splash+screen-1186-.html by Rock Achu (rockhachu2)
# and tweaked ;)
import os
import directories
if os.sys.platform == 'linux2':
    os.sys.path.insert(1, os.path.expanduser('~/.local/lib/python2.7/site-packages'))
    os.sys.path.insert(1, os.path.abspath('./lib'))

import pygame
print 'Splash load...'
os.environ['SDL_VIDEO_CENTERED'] = '1'

pygame.init()
pygame.font.init()
no_splash = False
cur_dir = directories.getDataDir()
splash_name = os.path.join(directories.getDataDir(), 'splash')

try:
    # if os.path.exists(splash_name) and len(open(splash_name).read()) > 0:
    #     splash = pygame.image.load(open(splash_name).read().strip())
    # else:
    splash = pygame.image.load(open(os.path.join(directories.getDataDir(), "splash.png"), 'rb'))
    screen = pygame.display.set_mode(splash.get_size(), pygame.NOFRAME)
    screen.blit(splash, (0, 0))
except IOError:
    try:
        font = pygame.font.Font(open(os.path.join(directories.getDataDir(), 'fonts', 'DejaVuSans-Bold.ttf'), 'rb'), 48)
        buf = font.render("MCEDit is loading...", True, (128, 128, 128))
        screen = pygame.display.set_mode((buf.get_width() + 20, buf.get_height() + 20), pygame.NOFRAME)
        screen.blit(buf, (10, 10))
        splash = pygame.display.get_surface()
    except:
        splash = pygame.display.set_mode((1, 1))
    no_splash = True
pygame.display.update()
os.environ['SDL_VIDEO_CENTERED'] = '0'

# Random splash
#
# Uses a 'splash' file to check the state.
# This file contains the name of the splash to be loaded next time MCEdit starts.
# No splash file means it has to be created.
# An empty file means the 'splash.png' file will alwas be used.
#

if not os.path.exists(splash_name):
    open(splash_name, 'w').write('scrap')

if len(open(splash_name).read()) > 0:
    from random import choice
    splashes_folder = os.path.join(directories.getDataDir(), 'splashes')
    if not os.path.exists(splashes_folder):
        splashes_folder = os.path.join(directories.getDataDir(), splashes_folder)
    if os.path.exists(splashes_folder) and os.listdir(splashes_folder):
        new_splash = choice(os.listdir(splashes_folder))
        open(splash_name, 'w').write(os.path.join(directories.getDataDir(), splashes_folder, new_splash))

<commit_msg>Harmonized code and restored random splash screens.<commit_after>#! /usr/bin/env python
# Taken from http://www.pygame.org/project-Splash+screen-1186-.html by Rock Achu (rockhachu2)
# and tweaked ;)
import os
import directories
if os.sys.platform == 'linux2':
    os.sys.path.insert(1, os.path.expanduser('~/.local/lib/python2.7/site-packages'))
    os.sys.path.insert(1, os.path.abspath('./lib'))

import pygame
print 'Splash load...'
os.environ['SDL_VIDEO_CENTERED'] = '1'

pygame.init()
pygame.font.init()
no_splash = False
cur_dir = directories.getDataDir()
splash_name = os.path.join(cur_dir, 'splash')

try:
    if os.path.exists(splash_name) and len(open(splash_name).read()) > 0:
        splash = pygame.image.load(open(splash_name).read().strip())
    else:
        splash = pygame.image.load(open(os.path.join(cur_dir, "splash.png"), 'rb'))
    screen = pygame.display.set_mode(splash.get_size(), pygame.NOFRAME)
    screen.blit(splash, (0, 0))
except IOError:
    try:
        font = pygame.font.Font(open(os.path.join(cur_dir, 'fonts', 'DejaVuSans-Bold.ttf'), 'rb'), 48)
        buf = font.render("MCEDit is loading...", True, (128, 128, 128))
        screen = pygame.display.set_mode((buf.get_width() + 20, buf.get_height() + 20), pygame.NOFRAME)
        screen.blit(buf, (10, 10))
        splash = pygame.display.get_surface()
    except:
        splash = pygame.display.set_mode((1, 1))
    no_splash = True
pygame.display.update()
os.environ['SDL_VIDEO_CENTERED'] = '0'

# Random splash
#
# Uses a 'splash' file to check the state.
# This file contains the name of the splash to be loaded next time MCEdit starts.
# No splash file means it has to be created.
# An empty file means the 'splash.png' file will alwas be used.
#

if not os.path.exists(splash_name):
    open(splash_name, 'w').write('scrap')

if len(open(splash_name).read()) > 0:
    from random import choice
    splashes_folder = os.path.join(cur_dir, 'splashes')
    if not os.path.exists(splashes_folder):
        splashes_folder = os.path.join(cur_dir, splashes_folder)
    if os.path.exists(splashes_folder) and os.listdir(splashes_folder):
        new_splash = choice(os.listdir(splashes_folder))
        open(splash_name, 'w').write(os.path.join(cur_dir, splashes_folder, new_splash))

<|endoftext|>