<commit_before>#!/usr/bin/python

import time
import threading

import SimpleXMLRPCServer,signal,sys,xmlrpclib
import SocketServer
import socket
import logging
import os

try:
	from ssl import *
	HAS_SSL = True
except ImportError:
	HAS_SSL = False

_service={}
_group={}
_res_id=1
_res={}

class ServiceEndPointCall(object):
	def __init__(self,id,method):
		self._id=id
		self._meth=method
	def __call__(self,*args):
		_res[self._id]=self._meth(*args)
		return self._id

class ServiceEndPoint(object):
	def __init__(self, name, id):
		self._id = id
		self._meth={}
		print _service
		s=_service[name]
		for m in s._method:
			self._meth[m]=s._method[m]
	def __getattr__(self, name):
		return ServiceEndPointCall(self._id, self._meth[name])

class Service(object):
	_serviceEndPointID = 0
	def __init__(self, name, audience=''):
		_service[name]=self
		self.__name=name
		self._method={}
		self.exportedMethods=None
		self._response_process=None
		self._response_process_id=None
		self._response=None

	def joinGroup(self,name):
		if not name in _group:
			_group[name]={}
		_group[name][self.__name]=self

	def exportMethod(self, m):
		if callable(m):
			self._method[m.__name__]=m

	def serviceEndPoint(self,s):
		if Service._serviceEndPointID >= 2**16:
			Service._serviceEndPointID = 0
		Service._serviceEndPointID += 1
		return ServiceEndPoint(s, self._serviceEndPointID)

	def conversationId(self):
		return 1

	def processResponse(self,s,id):
		self._response_process, self._response_process_id = s, id

	def processFailure(self,s,id):
		pass

	def resumeResponse(self,s):
		pass

	def cancelResponse(self,s):
		pass

	def suspendResponse(self,s):
		if self._response_process:
			self._response_process(self._response_process_id,
								   _res[self._response_process_id])
		self._response_process=None
		self._response=s(self._response_process_id)

	def abortResponse(self,error, description, origin, details):
		import tools
		if not tools.config['debug_mode']:
			raise Exception("%s -- %s\n\n%s"%(origin,description,details))
		else:
			raise

	def currentFailure(self,s):
		pass

class LocalService(Service):
	def __init__(self, name):
		self.__name=name
		s=_service[name]
		self._service=s
		for m in s._method:
			setattr(self,m,s._method[m])

class ServiceUnavailable(Exception):
	pass

LOG_DEBUG='debug'
LOG_INFO='info'
LOG_WARNING='warn'
LOG_ERROR='error'
LOG_CRITICAL='critical'

def init_logger():
	from tools import config
	import os

	if config['logfile']:
		logf = config['logfile']
		# test if the directories exist, else create them
		try:
			if not os.path.exists(os.path.dirname(logf)):
				os.makedirs(os.path.dirname(logf))
			try:
				fd = open(logf, 'a')
				handler = logging.StreamHandler(fd)
			except IOError:
				sys.stderr.write("ERROR: couldn't open the logfile\n")
				handler = logging.StreamHandler(sys.stdout)
		except OSError:
			sys.stderr.write("ERROR: couldn't create the logfile directory\n")
			handler = logging.StreamHandler(sys.stdout)
	else:
		handler = logging.StreamHandler(sys.stdout)

	# create a format for log messages and dates
	formatter = logging.Formatter('%(asctime)s %(levelname)s:%(name)s:%(message)s', '%a, %d %b %Y %H:%M:%S')

	# tell the handler to use this format
	handler.setFormatter(formatter)

	# add the handler to the root logger
	logging.getLogger().addHandler(handler)
	logging.getLogger().setLevel(logging.INFO)

class Logger(object):
	def notifyChannel(self,name,level,msg):
		log = logging.getLogger(name)
		getattr(log,level)(msg)

class Agent(object):
	_timers = []
	_logger = Logger()

	def setAlarm(self, fn, dt, args=[], kwargs={}):
		wait = dt - time.time()
		if wait > 0:
			self._logger.notifyChannel(
					'timers', LOG_DEBUG,
					"Job scheduled in %s seconds for %s.%s" % (wait,
															   fn.im_class.__name__,
												fn.func_name))
			timer = threading.Timer(wait, fn, args, kwargs)
			timer.start()
			self._timers.append(timer)
		for timer in self._timers[:]:
			if not timer.isAlive():
				self._timers.remove(timer)

	def quit(cls):
		for timer in cls._timers:
			timer.cancel()
	quit=classmethod(quit)

class RpcGateway(object):
	def __init__(self, name):
		self.name=name

class Dispatcher(object):
	def __init__(self):
		pass
	def monitor(self,signal):
		pass
	def run(self):
		pass

class xmlrpc(object):
	class RpcGateway(object):
		def __init__(self, name):
			self.name=name

class GenericXMLRPCRequestHandler:
	def _dispatch(self, method, params):
		import traceback
		try:
			n=self.path.split("/")[-1]
#			print "TERP-CALLING:",n,method,params
			s=LocalService(n)
			m=getattr(s,method)
			s._service._response=None
			r=m(*params)
			res=s._service._response
			if res!=None:
				r=res
			return r
		except Exception,e:
			logger = Logger()
			tb_s = reduce(lambda x, y: x+y, traceback.format_exception(sys.exc_type, sys.exc_value, sys.exc_traceback))
			logger.notifyChannel("web-services", LOG_ERROR, 'Exception in call: ' + tb_s)
			s=str(e)
			import tools
			if tools.config['debug_mode']:
				import pdb
				tb = sys.exc_info()[2]
				pdb.post_mortem(tb)
			raise xmlrpclib.Fault(s, tb_s)

class SimpleXMLRPCRequestHandler(GenericXMLRPCRequestHandler, SimpleXMLRPCServer.SimpleXMLRPCRequestHandler):
	pass

if HAS_SSL:
	class SecureXMLRPCRequestHandler(GenericXMLRPCRequestHandler, SecureXMLRPCServer.SecureXMLRPCRequestHandler):
		pass
else:
	pass

class SimpleThreadedXMLRPCServer(SocketServer.ThreadingMixIn, SimpleXMLRPCServer.SimpleXMLRPCServer):
	def server_bind(self):
		self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
		SimpleXMLRPCServer.SimpleXMLRPCServer.server_bind(self)

if HAS_SSL:
	class SecureThreadedXMLRPCServer(SocketServer.ThreadingMixIn, SecureXMLRPCServer.SecureXMLRPCServer):
		def server_bind(self):
			self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
			SecureXMLRPCServer.SecureXMLRPCServer.server_bind(self)
else:
	pass

class HttpDaemon(threading.Thread):
	def __init__(self, interface,port, secure=False):
		threading.Thread.__init__(self)
		self.__port=port
		self.__interface=interface
		if secure and HAS_SSL:
			self.server = SecureThreadedXMLRPCServer((interface, port), SecureXMLRPCRequestHandler,0)
		else:
			self.server = SimpleThreadedXMLRPCServer((interface, port), SimpleXMLRPCRequestHandler,0)

	def attach(self,path,gw):
		pass

	def stop(self):
		self.running = False
		if hasattr(socket, 'SHUT_RDWR'):
			self.server.socket.shutdown(socket.SHUT_RDWR)
		else:
			self.server.socket.shutdown(2)
		self.server.socket.close()

	def run(self):
		self.server.register_introspection_functions()

		self.running = True
		while self.running:
			self.server.handle_request()
		return True

		# If the server need to be run recursively
		#
		#signal.signal(signal.SIGALRM, self.my_handler)
		#signal.alarm(6)
		#while True:
		#	self.server.handle_request()
		#signal.alarm(0)          # Disable the alarm

import tiny_socket
class TinySocketClientThread(threading.Thread):
	def __init__(self, sock, threads):
		threading.Thread.__init__(self)
		self.sock = sock
		self.threads = threads

	def run(self):
		import traceback
		import time
		import select
		try:
			self.running = True
			ts = tiny_socket.mysocket(self.sock)
			while self.running:
				msg = ts.myreceive()

				try:
					s=LocalService(msg[0])
					m=getattr(s,msg[1])
					s._service._response=None
					r=m(*msg[2:])
					res=s._service._response
					if res!=None:
						r=res
					ts.mysend(r)
				except Exception, e:
					logger = Logger()
					tb_s = reduce(lambda x, y: x+y, traceback.format_exception(sys.exc_type, sys.exc_value, sys.exc_traceback))
					logger.notifyChannel("web-services", LOG_ERROR, 'Exception in call: ' + tb_s)
					s=str(e)
					import tools
					if tools.config['debug_mode']:
						import pdb
						tb = sys.exc_info()[2]
						pdb.post_mortem(tb)
					ts.mysend(e, exception=True, traceback=tb_s)
				self.sock.close()
				self.threads.remove(self)
				return True
		except Exception, e:
			self.sock.close()
			return False
	def stop(self):
		self.running = False

class TinySocketServerThread(threading.Thread):
	def __init__(self, interface, port, secure=False):
		threading.Thread.__init__(self)
		self.__port=port
		self.__interface=interface
		self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
		self.socket.bind((self.__interface, self.__port))
		self.socket.listen(5)
		self.threads = []

	def run(self):
		import select
		try:
			self.running = True
			while self.running:
				(clientsocket, address) = self.socket.accept()
				ct = TinySocketClientThread(clientsocket, self.threads)
				ct.start()
				self.threads.append(ct)
			self.socket.close()
		except Exception, e:
			self.socket.close()
			return False

	def stop(self):
		self.running=False
		for t in self.threads:
			t.stop()
		try:
			if hasattr(socket, 'SHUT_RDWR'):
				self.socket.shutdown(socket.SHUT_RDWR)
			else:
				self.socket.shutdown(2)
			self.socket.close()
		except:
			return False

# vim:noexpandtab:

<commit_msg>KERNEL: fix compatibility for python2.5<commit_after>#!/usr/bin/python

import time
import threading

import SimpleXMLRPCServer,signal,sys,xmlrpclib
import SocketServer
import socket
import logging
import os

try:
	from ssl import *
	HAS_SSL = True
except ImportError:
	HAS_SSL = False

_service={}
_group={}
_res_id=1
_res={}

class ServiceEndPointCall(object):
	def __init__(self,id,method):
		self._id=id
		self._meth=method
	def __call__(self,*args):
		_res[self._id]=self._meth(*args)
		return self._id

class ServiceEndPoint(object):
	def __init__(self, name, id):
		self._id = id
		self._meth={}
		print _service
		s=_service[name]
		for m in s._method:
			self._meth[m]=s._method[m]
	def __getattr__(self, name):
		return ServiceEndPointCall(self._id, self._meth[name])

class Service(object):
	_serviceEndPointID = 0
	def __init__(self, name, audience=''):
		_service[name]=self
		self.__name=name
		self._method={}
		self.exportedMethods=None
		self._response_process=None
		self._response_process_id=None
		self._response=None

	def joinGroup(self,name):
		if not name in _group:
			_group[name]={}
		_group[name][self.__name]=self

	def exportMethod(self, m):
		if callable(m):
			self._method[m.__name__]=m

	def serviceEndPoint(self,s):
		if Service._serviceEndPointID >= 2**16:
			Service._serviceEndPointID = 0
		Service._serviceEndPointID += 1
		return ServiceEndPoint(s, self._serviceEndPointID)

	def conversationId(self):
		return 1

	def processResponse(self,s,id):
		self._response_process, self._response_process_id = s, id

	def processFailure(self,s,id):
		pass

	def resumeResponse(self,s):
		pass

	def cancelResponse(self,s):
		pass

	def suspendResponse(self,s):
		if self._response_process:
			self._response_process(self._response_process_id,
								   _res[self._response_process_id])
		self._response_process=None
		self._response=s(self._response_process_id)

	def abortResponse(self,error, description, origin, details):
		import tools
		if not tools.config['debug_mode']:
			raise Exception("%s -- %s\n\n%s"%(origin,description,details))
		else:
			raise

	def currentFailure(self,s):
		pass

class LocalService(Service):
	def __init__(self, name):
		self.__name=name
		s=_service[name]
		self._service=s
		for m in s._method:
			setattr(self,m,s._method[m])

class ServiceUnavailable(Exception):
	pass

LOG_DEBUG='debug'
LOG_INFO='info'
LOG_WARNING='warn'
LOG_ERROR='error'
LOG_CRITICAL='critical'

def init_logger():
	from tools import config
	import os

	if config['logfile']:
		logf = config['logfile']
		# test if the directories exist, else create them
		try:
			if not os.path.exists(os.path.dirname(logf)):
				os.makedirs(os.path.dirname(logf))
			try:
				fd = open(logf, 'a')
				handler = logging.StreamHandler(fd)
			except IOError:
				sys.stderr.write("ERROR: couldn't open the logfile\n")
				handler = logging.StreamHandler(sys.stdout)
		except OSError:
			sys.stderr.write("ERROR: couldn't create the logfile directory\n")
			handler = logging.StreamHandler(sys.stdout)
	else:
		handler = logging.StreamHandler(sys.stdout)

	# create a format for log messages and dates
	formatter = logging.Formatter('%(asctime)s %(levelname)s:%(name)s:%(message)s', '%a, %d %b %Y %H:%M:%S')

	# tell the handler to use this format
	handler.setFormatter(formatter)

	# add the handler to the root logger
	logging.getLogger().addHandler(handler)
	logging.getLogger().setLevel(logging.INFO)

class Logger(object):
	def notifyChannel(self,name,level,msg):
		log = logging.getLogger(name)
		getattr(log,level)(msg)

class Agent(object):
	_timers = []
	_logger = Logger()

	def setAlarm(self, fn, dt, args=[], kwargs={}):
		wait = dt - time.time()
		if wait > 0:
			self._logger.notifyChannel(
					'timers', LOG_DEBUG,
					"Job scheduled in %s seconds for %s.%s" % (wait,
															   fn.im_class.__name__,
												fn.func_name))
			timer = threading.Timer(wait, fn, args, kwargs)
			timer.start()
			self._timers.append(timer)
		for timer in self._timers[:]:
			if not timer.isAlive():
				self._timers.remove(timer)

	def quit(cls):
		for timer in cls._timers:
			timer.cancel()
	quit=classmethod(quit)

class RpcGateway(object):
	def __init__(self, name):
		self.name=name

class Dispatcher(object):
	def __init__(self):
		pass
	def monitor(self,signal):
		pass
	def run(self):
		pass

class xmlrpc(object):
	class RpcGateway(object):
		def __init__(self, name):
			self.name=name

class GenericXMLRPCRequestHandler:
	def _dispatch(self, method, params):
		import traceback
		try:
			n=self.path.split("/")[-1]
#			print "TERP-CALLING:",n,method,params
			s=LocalService(n)
			m=getattr(s,method)
			s._service._response=None
			r=m(*params)
			res=s._service._response
			if res!=None:
				r=res
			return r
		except Exception,e:
			logger = Logger()
			tb_s = reduce(lambda x, y: x+y, traceback.format_exception(sys.exc_type, sys.exc_value, sys.exc_traceback))
			logger.notifyChannel("web-services", LOG_ERROR, 'Exception in call: ' + tb_s)
			s=str(e)
			import tools
			if tools.config['debug_mode']:
				import pdb
				tb = sys.exc_info()[2]
				pdb.post_mortem(tb)
			raise xmlrpclib.Fault(s, tb_s)

class SimpleXMLRPCRequestHandler(GenericXMLRPCRequestHandler, SimpleXMLRPCServer.SimpleXMLRPCRequestHandler):
	SimpleXMLRPCServer.SimpleXMLRPCRequestHandler.rpc_paths = ('/xmlrpc/db', '/xmlrpc/common', '/xmlrpc/object', '/xmlrpc/report', '/xmlrpc/wizard')

if HAS_SSL:
	class SecureXMLRPCRequestHandler(GenericXMLRPCRequestHandler, SecureXMLRPCServer.SecureXMLRPCRequestHandler):
		SecureXMLRPCServer.SecureXMLRPCRequestHandler.rpc_paths = ('/xmlrpc/db', '/xmlrpc/common', '/xmlrpc/object', '/xmlrpc/report', '/xmlrpc/wizard')

class SimpleThreadedXMLRPCServer(SocketServer.ThreadingMixIn, SimpleXMLRPCServer.SimpleXMLRPCServer):
	def server_bind(self):
		self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
		SimpleXMLRPCServer.SimpleXMLRPCServer.server_bind(self)

if HAS_SSL:
	class SecureThreadedXMLRPCServer(SocketServer.ThreadingMixIn, SecureXMLRPCServer.SecureXMLRPCServer):
		def server_bind(self):
			self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
			SecureXMLRPCServer.SecureXMLRPCServer.server_bind(self)
else:
	pass

class HttpDaemon(threading.Thread):
	def __init__(self, interface,port, secure=False):
		threading.Thread.__init__(self)
		self.__port=port
		self.__interface=interface
		if secure and HAS_SSL:
			self.server = SecureThreadedXMLRPCServer((interface, port), SecureXMLRPCRequestHandler,0)
		else:
			self.server = SimpleThreadedXMLRPCServer((interface, port), SimpleXMLRPCRequestHandler,0)

	def attach(self,path,gw):
		pass

	def stop(self):
		self.running = False
		if os.name <> 'nt':
			if hasattr(socket, 'SHUT_RDWR'):
				self.server.socket.shutdown(socket.SHUT_RDWR)
			else:
				self.server.socket.shutdown(2)
		self.server.socket.close()

	def run(self):
		self.server.register_introspection_functions()

		self.running = True
		while self.running:
			self.server.handle_request()
		return True

		# If the server need to be run recursively
		#
		#signal.signal(signal.SIGALRM, self.my_handler)
		#signal.alarm(6)
		#while True:
		#	self.server.handle_request()
		#signal.alarm(0)          # Disable the alarm

import tiny_socket
class TinySocketClientThread(threading.Thread):
	def __init__(self, sock, threads):
		threading.Thread.__init__(self)
		self.sock = sock
		self.threads = threads

	def run(self):
		import traceback
		import time
		import select
		try:
			self.running = True
			ts = tiny_socket.mysocket(self.sock)
			while self.running:
				msg = ts.myreceive()

				try:
					s=LocalService(msg[0])
					m=getattr(s,msg[1])
					s._service._response=None
					r=m(*msg[2:])
					res=s._service._response
					if res!=None:
						r=res
					ts.mysend(r)
				except Exception, e:
					logger = Logger()
					tb_s = reduce(lambda x, y: x+y, traceback.format_exception(sys.exc_type, sys.exc_value, sys.exc_traceback))
					logger.notifyChannel("web-services", LOG_ERROR, 'Exception in call: ' + tb_s)
					s=str(e)
					import tools
					if tools.config['debug_mode']:
						import pdb
						tb = sys.exc_info()[2]
						pdb.post_mortem(tb)
					ts.mysend(e, exception=True, traceback=tb_s)
				self.sock.close()
				self.threads.remove(self)
				return True
		except Exception, e:
			self.sock.close()
			return False
	def stop(self):
		self.running = False

class TinySocketServerThread(threading.Thread):
	def __init__(self, interface, port, secure=False):
		threading.Thread.__init__(self)
		self.__port=port
		self.__interface=interface
		self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
		self.socket.bind((self.__interface, self.__port))
		self.socket.listen(5)
		self.threads = []

	def run(self):
		import select
		try:
			self.running = True
			while self.running:
				(clientsocket, address) = self.socket.accept()
				ct = TinySocketClientThread(clientsocket, self.threads)
				ct.start()
				self.threads.append(ct)
			self.socket.close()
		except Exception, e:
			self.socket.close()
			return False

	def stop(self):
		self.running=False
		for t in self.threads:
			t.stop()
		try:
			if hasattr(socket, 'SHUT_RDWR'):
				self.socket.shutdown(socket.SHUT_RDWR)
			else:
				self.socket.shutdown(2)
			self.socket.close()
		except:
			return False

# vim:noexpandtab:

<|endoftext|>


<commit_before>#!/usr/local/bin/python

from sprint import sprint as print
import datetime
from io import open
import os
import pprint
import re

import basics
import bfiles
import config
import images
import mbdata
import useful

# pagename = 'biblio'

# -- biblio

map_url = '''https://www.google.com/maps/place/'''
squish_re = re.compile(r'\s\s*')

biblios = {
    'biblio': {
        'tab': 'book',
        'show': ['*Author', '*Title', '*Publisher', '*Year', 'ISBN'],
        'sort': 'author',
        'lastsort': 'title',
        'links': {
            'title': 'pic:pic_id',
            'edit': 'edit:id',
        },
    },
    'bayarea': {
        'tab': 'bayarea',
        'show': ['*Name', 'Address', '*City', 'State', 'Phone'],
        'sort': 'name',
        'lastsort': 'name',
        'links': {
            'name': 'url:url',
            'address': 'map:address,city,state',
            'edit': 'edit:id',
        },
    },
}

# links (both site links and map links) aren't working.
@basics.web_page
def biblio(pif):
    def pic_formatter(x):
        img = pif.render.find_image_file([str(y) for y in x])
        return ('/' + '/'.join(pif.render.find_image_file([str(y) for y in x]))) if ''.join(img) else ''

    row_link_formatters = {
        'map': lambda x: '' if '' in x else (map_url + ','.join(x)).replace(' ', '+'),
        'url': lambda x: x[0],
        'pic': pic_formatter,
    }

    pif.render.hierarchy_append('/', 'Home')
    pif.render.hierarchy_append(pif.request_uri, pif.render.title)
    pif.render.print_html()

    page_info = biblios.get(pif.page_name, {})
    sections = pif.dbh.fetch_sections({'page_id': pif.page_id})
    lsection = sections[0].todict() if sections else dict(note='')
    tab_name = page_info.get('tab', pif.page_name)
    row_links = page_info.get('links', {})
    table = pif.dbh.depref(tab_name, pif.dbh.fetch(tab_name, where='flags&1=0', tag='biblio', verbose=True))

    lrange = dict(entry=list(), note='')
    lsection.update(dict(columns=list(), headers=dict(), range=[lrange]))

    editable = pif.form.get_bool('edit') and pif.is_allowed('a')
    if editable:
        lsection['columns'].append('edit')
        lsection['headers']['edit'] = ' '
        row_link_formatters['edit'] = lambda x: (
            '%s/cgi-bin/editor.cgi?table=%s&id=%s' % (pif.secure_host, tab_name, x[0]))
    for arg in page_info['show']:
        hdr = arg
        key = arg.lower().replace(' ', '_')
        if arg.startswith('*'):
            key = key[1:]
            hdr = '%s' % (pif.page_name, key, hdr[1:])
        lsection['columns'].append(key)
        lsection['headers'][key] = hdr

    sortkey = []
    this_sort = pif.form.get_str('sort', page_info.get('sort'))
    if this_sort:
        if this_sort.isdigit():
            this_sort = max(0, min(len(lsection['columns']) - 1, int(this_sort)))
            this_sort = lsection['columns'][this_sort]
        sortkey.append(this_sort)
    if page_info.get('lastsort'):
        sortkey.append(page_info.get('lastsort'))
    if sortkey:
        table.sort(key=lambda x: [x[y] for y in sortkey if y in lsection['columns']])

    def bib_field(fdict, field):
        cont = str(fdict.get(field, '&nbsp'))
        url = ''
        if field in row_links:
            if field == 'edit':
                cont = str(fdict['id'])
            cmd, arg = row_links[field].split(':')
            arg = [str(fdict.get(x, lsection.get(x, ''))) for x in arg.split(',')]
            if ''.join(arg):
                url = row_link_formatters[cmd](arg)

        edlink = ''
        if field == 'title' and editable:
            if os.path.exists('.' + config.IMG_DIR_BOOK + '/' + fdict['pic_id'] + '.jpg'):
                edlink += (' ' + pif.render.format_link('/cgi-bin/imawidget.cgi?d=.%s&f=%s' % (
                    config.IMG_DIR_BOOK, fdict['pic_id'] + '.jpg'), ''))
            edlink += (' ' + pif.render.format_link('/cgi-bin/upload.cgi?d=.%s&n=%s' % (
                config.IMG_DIR_BOOK, fdict['pic_id'] + '.jpg'), ''))

        return pif.render.format_link(url, cont) + edlink

    lrange['entry'] = [{field: bib_field(fdict, field) for field in lsection['columns']}
                       for fdict in table]
    pif.render.set_button_comment(pif)
    return pif.render.format_template('simplelistix.html', llineup=dict(section=[lsection]))

# -- calendar

def event_type(pif, event):
    return '%s' % pif.render.format_image_icon('i_' + event, event.upper())

def event(pif, ty, llist):
    return [event_type(pif, ty),
            llist.get_arg(' ').replace(';', '
'),
            llist.get_arg(' ').replace(';', '
'),
            llist.get_arg(' ').replace(';', '
')]

@basics.web_page
def calendar(pif):
    pif.render.hierarchy_append('/', 'Home')
    pif.render.hierarchy_append(pif.request_uri, 'Calendar')
    pif.render.print_html()

    dblist = bfiles.SimpleFile(os.path.join(config.SRC_DIR, pif.page_name + '.dat'))

    lrange = None
    lsection = dict(columns=list(), headers=dict(), range=list(), note='')
    llistix = dict(section=[lsection])

    for llist in dblist:

        cmd = llist.get_arg()

        if (cmd == 'h'):
            for iarg in range(1, llist.args()):
                arg = llist.get_arg(' ')
                lsection['columns'].append(arg)
                lsection['headers'][arg] = arg

        elif (cmd == 'm'):
            lrange['entry'].append(dict(zip(lsection['columns'], event(pif, 'meet', llist))))

        elif (cmd == 's'):
            lrange['entry'].append(dict(zip(lsection['columns'], event(pif, 'show', llist))))

        elif (cmd == 'n'):
            if lrange:
                lsection['range'].append(lrange)
            lrange = dict(entry=list(), name=llist.get_arg(' ', 1))

        elif (cmd == 'e'):
            if lrange:
                lsection['range'].append(lrange)
            lrange = None

    if lrange:
        lsection['range'].append(lrange)

    return pif.render.format_template('simplelistix.html', llineup=llistix)

@basics.web_page
def submit_comment(pif):
    pif.render.print_html()
    print(pif.render.format_head())
    # useful.write_message(pif.form)
    ostr = "I am sending this comment for you. "

    mysubject = pif.form.get_str('mysubject')
    mycomment = pif.form.get_str('mycomment')
    myname = pif.form.get_str('myname')
    myemail = pif.form.get_str('myemail')
    credit = pif.form.get_str('credit')
    fname = pif.form.get_str('pic.name')
    fimage = pif.form.get_str('pic')
    pif.form.delete('pic')

    def comment_error(msg):
        return "

ERROR

%s

" % msg

    if myemail and '@' not in myemail:
        return comment_error('Badly formatted email address.  Try again.')

    pif.form.change_key('page', 'page_id')
    fn = "../../comments/comment." + datetime.datetime.now().strftime('%Y%m%d.%H%M%S')

    if any('http://' in mysubject, 'https://' in mysubject,
            'http://' in mycomment, 'https://' in mycomment,
            'http://' in myemail, 'https://' in myemail):
        return comment_error(
            "Whoa there.  This isn't for submitting links.  Please use the SUGGEST A LINK feature from the link list.")

    ostr += "

My Subject

" + mysubject + "
\n"
    ostr += "

My Comment

" + mycomment + "
\n"
    ostr += "
My Name

" + myname + "
\n"
    ostr += "
My Email

" + myemail + "

\n"

    if fimage:
        ostr += "
Relevant File

" + fname + "
\n"
        direc = config.INC_DIR
        descriptions_file = config.LOG_ROOT + '/descr.log'
        dest_filename = images.get_next_upload_filename()
        dest_filename = useful.file_save(direc, dest_filename, fimage)
        images.file_log(direc + '/' + dest_filename, direc)

        cred = who = comment = '-'
        if mycomment:
            comment = squish_re.sub(' ', mycomment)
        if credit:
            cred = squish_re.sub(' ', credit)
        if myname:
            who = squish_re.sub(' ', myname)
        open(descriptions_file, 'a+').write('\t'.join([dest_filename, '-', '-', '-', comment, cred, who]) + '\n')
        ostr = '
Thank you for submitting that file.

\n'

        ostr += "

\n"

    fh = open(fn, "wt")
    fh.write("_POST\n\n" + pprint.pformat(pif.form, indent=2, width=132) + "\n\n")
    fh.write("REMOTE_ADDR=" + os.getenv('REMOTE_ADDR') + "\n")
    ostr += "Thanks for sending that.  Now please use the BACK button on your browser to return to where you were."
    return ostr

@basics.web_page
def counts_main(pif):
    pif.render.title = 'Site Counts'
    pif.render.print_html()
    counts = pif.dbh.fetch_counts()
    v = {c['model_type']: c['count'] for c in counts[1]}
    for c in counts[0]:
        c['name'] = mbdata.model_type_names[c['model_type']]
        c['type'] = mbdata.model_types[c['model_type']]
        c['vars'] = v.get(c['model_type'], 0)

    things = []
    for mt in sorted(set(mbdata.model_types.values())):
        section = {
            'name': mt,
            'columns': ['name', 'count', 'vars'],
            'headers': {'name': 'Model Type', 'count': 'Count', 'vars': 'Variations'},
            'range': [{
                'entry': [x for x in counts[0] if x['type'] == mt],
            }],
        }
        tot = sum([x['count'] for x in section['range'][0]['entry']])
        vc = sum([x['vars'] for x in section['range'][0]['entry']])
        section['range'][0]['entry'].append({'name': 'total', 'count': tot, 'vars': vc})
        if tot:
            things.append(section)
    llineup = {
        'section': things,
    }
    return pif.render.format_template('simplelistix.html', llineup=llineup)
<commit_msg>Fix any.<commit_after>#!/usr/local/bin/python

from sprint import sprint as print
import datetime
from io import open
import os
import pprint
import re

import basics
import bfiles
import config
import images
import mbdata
import useful

# pagename = 'biblio'

# -- biblio

map_url = '''https://www.google.com/maps/place/'''
squish_re = re.compile(r'\s\s*')

biblios = {
    'biblio': {
        'tab': 'book',
        'show': ['*Author', '*Title', '*Publisher', '*Year', 'ISBN'],
        'sort': 'author',
        'lastsort': 'title',
        'links': {
            'title': 'pic:pic_id',
            'edit': 'edit:id',
        },
    },
    'bayarea': {
        'tab': 'bayarea',
        'show': ['*Name', 'Address', '*City', 'State', 'Phone'],
        'sort': 'name',
        'lastsort': 'name',
        'links': {
            'name': 'url:url',
            'address': 'map:address,city,state',
            'edit': 'edit:id',
        },
    },
}

# links (both site links and map links) aren't working.
@basics.web_page
def biblio(pif):
    def pic_formatter(x):
        img = pif.render.find_image_file([str(y) for y in x])
        return ('/' + '/'.join(pif.render.find_image_file([str(y) for y in x]))) if ''.join(img) else ''

    row_link_formatters = {
        'map': lambda x: '' if '' in x else (map_url + ','.join(x)).replace(' ', '+'),
        'url': lambda x: x[0],
        'pic': pic_formatter,
    }

    pif.render.hierarchy_append('/', 'Home')
    pif.render.hierarchy_append(pif.request_uri, pif.render.title)
    pif.render.print_html()

    page_info = biblios.get(pif.page_name, {})
    sections = pif.dbh.fetch_sections({'page_id': pif.page_id})
    lsection = sections[0].todict() if sections else dict(note='')
    tab_name = page_info.get('tab', pif.page_name)
    row_links = page_info.get('links', {})
    table = pif.dbh.depref(tab_name, pif.dbh.fetch(tab_name, where='flags&1=0', tag='biblio', verbose=True))

    lrange = dict(entry=list(), note='')
    lsection.update(dict(columns=list(), headers=dict(), range=[lrange]))

    editable = pif.form.get_bool('edit') and pif.is_allowed('a')
    if editable:
        lsection['columns'].append('edit')
        lsection['headers']['edit'] = ' '
        row_link_formatters['edit'] = lambda x: (
            '%s/cgi-bin/editor.cgi?table=%s&id=%s' % (pif.secure_host, tab_name, x[0]))
    for arg in page_info['show']:
        hdr = arg
        key = arg.lower().replace(' ', '_')
        if arg.startswith('*'):
            key = key[1:]
            hdr = '%s' % (pif.page_name, key, hdr[1:])
        lsection['columns'].append(key)
        lsection['headers'][key] = hdr

    sortkey = []
    this_sort = pif.form.get_str('sort', page_info.get('sort'))
    if this_sort:
        if this_sort.isdigit():
            this_sort = max(0, min(len(lsection['columns']) - 1, int(this_sort)))
            this_sort = lsection['columns'][this_sort]
        sortkey.append(this_sort)
    if page_info.get('lastsort'):
        sortkey.append(page_info.get('lastsort'))
    if sortkey:
        table.sort(key=lambda x: [x[y] for y in sortkey if y in lsection['columns']])

    def bib_field(fdict, field):
        cont = str(fdict.get(field, '&nbsp'))
        url = ''
        if field in row_links:
            if field == 'edit':
                cont = str(fdict['id'])
            cmd, arg = row_links[field].split(':')
            arg = [str(fdict.get(x, lsection.get(x, ''))) for x in arg.split(',')]
            if ''.join(arg):
                url = row_link_formatters[cmd](arg)

        edlink = ''
        if field == 'title' and editable:
            if os.path.exists('.' + config.IMG_DIR_BOOK + '/' + fdict['pic_id'] + '.jpg'):
                edlink += (' ' + pif.render.format_link('/cgi-bin/imawidget.cgi?d=.%s&f=%s' % (
                    config.IMG_DIR_BOOK, fdict['pic_id'] + '.jpg'), ''))
            edlink += (' ' + pif.render.format_link('/cgi-bin/upload.cgi?d=.%s&n=%s' % (
                config.IMG_DIR_BOOK, fdict['pic_id'] + '.jpg'), ''))

        return pif.render.format_link(url, cont) + edlink

    lrange['entry'] = [{field: bib_field(fdict, field) for field in lsection['columns']}
                       for fdict in table]
    pif.render.set_button_comment(pif)
    return pif.render.format_template('simplelistix.html', llineup=dict(section=[lsection]))

# -- calendar

def event_type(pif, event):
    return '%s' % pif.render.format_image_icon('i_' + event, event.upper())

def event(pif, ty, llist):
    return [event_type(pif, ty),
            llist.get_arg(' ').replace(';', '
'),
            llist.get_arg(' ').replace(';', '
'),
            llist.get_arg(' ').replace(';', '
')]

@basics.web_page
def calendar(pif):
    pif.render.hierarchy_append('/', 'Home')
    pif.render.hierarchy_append(pif.request_uri, 'Calendar')
    pif.render.print_html()

    dblist = bfiles.SimpleFile(os.path.join(config.SRC_DIR, pif.page_name + '.dat'))

    lrange = None
    lsection = dict(columns=list(), headers=dict(), range=list(), note='')
    llistix = dict(section=[lsection])

    for llist in dblist:

        cmd = llist.get_arg()

        if (cmd == 'h'):
            for iarg in range(1, llist.args()):
                arg = llist.get_arg(' ')
                lsection['columns'].append(arg)
                lsection['headers'][arg] = arg

        elif (cmd == 'm'):
            lrange['entry'].append(dict(zip(lsection['columns'], event(pif, 'meet', llist))))

        elif (cmd == 's'):
            lrange['entry'].append(dict(zip(lsection['columns'], event(pif, 'show', llist))))

        elif (cmd == 'n'):
            if lrange:
                lsection['range'].append(lrange)
            lrange = dict(entry=list(), name=llist.get_arg(' ', 1))

        elif (cmd == 'e'):
            if lrange:
                lsection['range'].append(lrange)
            lrange = None

    if lrange:
        lsection['range'].append(lrange)

    return pif.render.format_template('simplelistix.html', llineup=llistix)

@basics.web_page
def submit_comment(pif):
    pif.render.print_html()
    print(pif.render.format_head())
    # useful.write_message(pif.form)
    ostr = "I am sending this comment for you. "

    mysubject = pif.form.get_str('mysubject')
    mycomment = pif.form.get_str('mycomment')
    myname = pif.form.get_str('myname')
    myemail = pif.form.get_str('myemail')
    credit = pif.form.get_str('credit')
    fname = pif.form.get_str('pic.name')
    fimage = pif.form.get_str('pic')
    pif.form.delete('pic')

    def comment_error(msg):
        return "

ERROR

%s

" % msg

    if myemail and '@' not in myemail:
        return comment_error('Badly formatted email address.  Try again.')

    pif.form.change_key('page', 'page_id')
    fn = "../../comments/comment." + datetime.datetime.now().strftime('%Y%m%d.%H%M%S')

    if any(['http://' in mysubject, 'https://' in mysubject,
            'http://' in mycomment, 'https://' in mycomment,
            'http://' in myemail, 'https://' in myemail]):
        return comment_error(
            "Whoa there.  This isn't for submitting links.  Please use the SUGGEST A LINK feature from the link list.")

    ostr += "

My Subject

" + mysubject + "
\n"
    ostr += "

My Comment

" + mycomment + "
\n"
    ostr += "
My Name

" + myname + "
\n"
    ostr += "
My Email

" + myemail + "

\n"

    if fimage:
        ostr += "
Relevant File

" + fname + "
\n"
        direc = config.INC_DIR
        descriptions_file = config.LOG_ROOT + '/descr.log'
        dest_filename = images.get_next_upload_filename()
        dest_filename = useful.file_save(direc, dest_filename, fimage)
        images.file_log(direc + '/' + dest_filename, direc)

        cred = who = comment = '-'
        if mycomment:
            comment = squish_re.sub(' ', mycomment)
        if credit:
            cred = squish_re.sub(' ', credit)
        if myname:
            who = squish_re.sub(' ', myname)
        open(descriptions_file, 'a+').write('\t'.join([dest_filename, '-', '-', '-', comment, cred, who]) + '\n')
        ostr = '
Thank you for submitting that file.

\n'

        ostr += "

\n"

    fh = open(fn, "wt")
    fh.write("_POST\n\n" + pprint.pformat(pif.form, indent=2, width=132) + "\n\n")
    fh.write("REMOTE_ADDR=" + os.getenv('REMOTE_ADDR') + "\n")
    ostr += "Thanks for sending that.  Now please use the BACK button on your browser to return to where you were."
    return ostr

@basics.web_page
def counts_main(pif):
    pif.render.title = 'Site Counts'
    pif.render.print_html()
    counts = pif.dbh.fetch_counts()
    v = {c['model_type']: c['count'] for c in counts[1]}
    for c in counts[0]:
        c['name'] = mbdata.model_type_names[c['model_type']]
        c['type'] = mbdata.model_types[c['model_type']]
        c['vars'] = v.get(c['model_type'], 0)

    things = []
    for mt in sorted(set(mbdata.model_types.values())):
        section = {
            'name': mt,
            'columns': ['name', 'count', 'vars'],
            'headers': {'name': 'Model Type', 'count': 'Count', 'vars': 'Variations'},
            'range': [{
                'entry': [x for x in counts[0] if x['type'] == mt],
            }],
        }
        tot = sum([x['count'] for x in section['range'][0]['entry']])
        vc = sum([x['vars'] for x in section['range'][0]['entry']])
        section['range'][0]['entry'].append({'name': 'total', 'count': tot, 'vars': vc})
        if tot:
            things.append(section)
    llineup = {
        'section': things,
    }
    return pif.render.format_template('simplelistix.html', llineup=llineup)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

#   Copyright (c) 2010-2014, MIT Probabilistic Computing Project
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""Main bayeslite API.

The focus of the bayeslite API is the *BayesDB*, a handle for a
database in memory or on disk.  To obtain a BayesDB handle, either
creating one in memory or opening or creating one on disk, use
:func:`bayesdb_open`::

   import bayeslite

   memdb = bayeslite.bayesdb_open()
   filedb = bayeslite.bayesdb_open(pathname='foo.bdb')

When done, close it with the :meth:`~BayesDB.close` method::

   filedb.close()
   memdb.close()

The ``filedb`` will read data and any saved generators from the given
database file, and save any modifications durably in that same file.
The ``memdb`` is an initially-empty in-memory database whose contents
will be forgotten when it is closed or when the Python process exists.

You can query the probable (according to the analyses stored in
the database) implications of the data by passing BQL queries
to the :meth:`~BayesDB.execute` method::

   for x in bdb.execute('estimate pairwise dependence probablity from foo_gen'):
       print x

You can also execute normal SQL on a BayesDB handle `bdb` with the
:meth:`~BayesDB.sql_execute` method::

   bdb.sql_execute('create table t(x int, y text, z real)')
   bdb.sql_execute("insert into t values(1, 'xyz', 42.5)")
   bdb.sql_execute("insert into t values(1, 'pqr', 83.7)")
   bdb.sql_execute("insert into t values(2, 'xyz', 1000)")

(BQL does not yet support CREATE TABLE and INSERT directly, so you
must use :meth:`~BayesDB.sql_execute` for those.)

When imported, the :mod:`bayeslite` module will notify the MIT
Probabilistic Computing Project over the internet of the software
version you are using, and warn if it is out-of-date.  To disable
this, set the environment variable ``BAYESDB_DISABLE_VERSION_CHECK``
before import, such as with::

   import os
   os.environ['BAYESDB_DISABLE_VERSION_CHECK'] = '1'
   import bayeslite

If you would like to analyze your own data with BayesDB, please
contact bayesdb@mit.edu to participate in our research project.
"""

from bayeslite.bayesdb import BayesDB
from bayeslite.bayesdb import bayesdb_open
from bayeslite.codebook import bayesdb_load_codebook_csv_file
from bayeslite.exception import BayesDBException
from bayeslite.exception import BQLError
from bayeslite.legacy_models import bayesdb_load_legacy_models
from bayeslite.metamodel import IBayesDBMetamodel
from bayeslite.metamodel import bayesdb_builtin_metamodel
from bayeslite.metamodel import bayesdb_deregister_metamodel
from bayeslite.metamodel import bayesdb_register_metamodel
from bayeslite.parse import BQLParseError
from bayeslite.read_csv import bayesdb_read_csv
from bayeslite.read_csv import bayesdb_read_csv_file
from bayeslite.txn import BayesDBTxnError
from bayeslite.version import __version__

__all__ = [
    'BQLError',
    'BQLParseError',
    'BayesDB',
    'BayesDBException',
    'BayesDBTxnError',
    'bayesdb_deregister_metamodel',
    'bayesdb_load_codebook_csv_file',
    'bayesdb_load_legacy_models',
    'bayesdb_open',
    'bayesdb_read_csv',
    'bayesdb_read_csv_file',
    'bayesdb_register_metamodel',
    'IBayesDBMetamodel',
    '__version__',
]

from bayeslite.metamodels.crosscat import CrosscatMetamodel
from crosscat.LocalEngine import LocalEngine as CrosscatLocalEngine

bayesdb_builtin_metamodel(CrosscatMetamodel(CrosscatLocalEngine(seed=0)))

import bayeslite.remote
import os
if not 'BAYESDB_DISABLE_VERSION_CHECK' in os.environ:
    bayeslite.remote.version_check()

# Notebooks should contain comment lines documenting this behavior and
# offering a solution, like so:
# Please keep BayesDB up to date. To disable remote version checking:
# import os; os.environ['BAYESDB_DISABLE_VERSION_CHECK'] = '1'
<commit_msg>Define and export bql_quote_name.<commit_after># -*- coding: utf-8 -*-

#   Copyright (c) 2010-2014, MIT Probabilistic Computing Project
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""Main bayeslite API.

The focus of the bayeslite API is the *BayesDB*, a handle for a
database in memory or on disk.  To obtain a BayesDB handle, either
creating one in memory or opening or creating one on disk, use
:func:`bayesdb_open`::

   import bayeslite

   memdb = bayeslite.bayesdb_open()
   filedb = bayeslite.bayesdb_open(pathname='foo.bdb')

When done, close it with the :meth:`~BayesDB.close` method::

   filedb.close()
   memdb.close()

The ``filedb`` will read data and any saved generators from the given
database file, and save any modifications durably in that same file.
The ``memdb`` is an initially-empty in-memory database whose contents
will be forgotten when it is closed or when the Python process exists.

You can query the probable (according to the analyses stored in
the database) implications of the data by passing BQL queries
to the :meth:`~BayesDB.execute` method::

   for x in bdb.execute('estimate pairwise dependence probablity from foo_gen'):
       print x

You can also execute normal SQL on a BayesDB handle `bdb` with the
:meth:`~BayesDB.sql_execute` method::

   bdb.sql_execute('create table t(x int, y text, z real)')
   bdb.sql_execute("insert into t values(1, 'xyz', 42.5)")
   bdb.sql_execute("insert into t values(1, 'pqr', 83.7)")
   bdb.sql_execute("insert into t values(2, 'xyz', 1000)")

(BQL does not yet support CREATE TABLE and INSERT directly, so you
must use :meth:`~BayesDB.sql_execute` for those.)

When imported, the :mod:`bayeslite` module will notify the MIT
Probabilistic Computing Project over the internet of the software
version you are using, and warn if it is out-of-date.  To disable
this, set the environment variable ``BAYESDB_DISABLE_VERSION_CHECK``
before import, such as with::

   import os
   os.environ['BAYESDB_DISABLE_VERSION_CHECK'] = '1'
   import bayeslite

If you would like to analyze your own data with BayesDB, please
contact bayesdb@mit.edu to participate in our research project.
"""

from bayeslite.bayesdb import BayesDB
from bayeslite.bayesdb import bayesdb_open
from bayeslite.codebook import bayesdb_load_codebook_csv_file
from bayeslite.exception import BayesDBException
from bayeslite.exception import BQLError
from bayeslite.legacy_models import bayesdb_load_legacy_models
from bayeslite.metamodel import IBayesDBMetamodel
from bayeslite.metamodel import bayesdb_builtin_metamodel
from bayeslite.metamodel import bayesdb_deregister_metamodel
from bayeslite.metamodel import bayesdb_register_metamodel
from bayeslite.parse import BQLParseError
from bayeslite.read_csv import bayesdb_read_csv
from bayeslite.read_csv import bayesdb_read_csv_file
from bayeslite.sqlite3_util import sqlite3_quote_name
from bayeslite.txn import BayesDBTxnError
from bayeslite.version import __version__

# XXX This is not a good place for me.  Find me a better home, please!
def bql_quote_name(name):
    """Quote `name` as a BQL identifier, e.g. a table or column name.

    Do NOT use this for strings, e.g. inserting data into a table.
    Use query parameters instead.
    """
    return sqlite3_quote_name(name)

__all__ = [
    'BQLError',
    'BQLParseError',
    'BayesDB',
    'BayesDBException',
    'BayesDBTxnError',
    'bayesdb_deregister_metamodel',
    'bayesdb_load_codebook_csv_file',
    'bayesdb_load_legacy_models',
    'bayesdb_open',
    'bayesdb_read_csv',
    'bayesdb_read_csv_file',
    'bayesdb_register_metamodel',
    'bql_quote_name',
    'IBayesDBMetamodel',
    '__version__',
]

from bayeslite.metamodels.crosscat import CrosscatMetamodel
from crosscat.LocalEngine import LocalEngine as CrosscatLocalEngine

bayesdb_builtin_metamodel(CrosscatMetamodel(CrosscatLocalEngine(seed=0)))

import bayeslite.remote
import os
if not 'BAYESDB_DISABLE_VERSION_CHECK' in os.environ:
    bayeslite.remote.version_check()

# Notebooks should contain comment lines documenting this behavior and
# offering a solution, like so:
# Please keep BayesDB up to date. To disable remote version checking:
# import os; os.environ['BAYESDB_DISABLE_VERSION_CHECK'] = '1'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# This module is part of the GeoTag-X project validator tool.
#
# Author: Jeremy Othieno (j.othieno@gmail.com)
#
# Copyright (c) 2016-2017 UNITAR/UNOSAT
#
# The MIT License (MIT)
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
# OR OTHER DEALINGS IN THE SOFTWARE.
__version__ = "0.1.0"
<commit_msg>Bump the version to 0.1.1<commit_after># -*- coding: utf-8 -*-
#
# This module is part of the GeoTag-X project validator tool.
#
# Author: Jeremy Othieno (j.othieno@gmail.com)
#
# Copyright (c) 2016-2017 UNITAR/UNOSAT
#
# The MIT License (MIT)
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
# OR OTHER DEALINGS IN THE SOFTWARE.
__version__ = "0.1.1"
<|endoftext|>


<commit_before># coding: utf-8
""" качаем свинок """

import os
from page_processing import download

DIR = os.environ.get('PWD') + '/download'
MAIN_URL = os.environ.get('URL', 'http://comicsia.ru/collections/swine')

if __name__ == '__main__':
    download(MAIN_URL, DIR)
<commit_msg>Теперь можно переопределять папку для загрузки<commit_after># coding: utf-8
""" качаем свинок """

import os
from page_processing import download

DIR = os.environ.get('PWD') + '/' + os.environ.get('DST', 'download')
MAIN_URL = os.environ.get('URL', 'http://comicsia.ru/collections/swine')

if __name__ == '__main__':
    download(MAIN_URL, DIR)
<|endoftext|>


<commit_before># -*- coding: iso-8859-1 -*-
# -----------------------------------------------------------------------------
# Imlib2.py - Imlib2 wrapper for Python
# -----------------------------------------------------------------------------
# $Id$
# -----------------------------------------------------------------------------
# kaa.imlib2 - An imlib2 wrapper for Python
# Copyright (C) 2004-2006 Dirk Meyer, Jason Tackaberry
#
# First Edition: Jason Tackaberry <tack@urandom.ca>
# Maintainer:    Jason Tackaberry <tack@urandom.ca>
#
# Please see the file AUTHORS for a complete list of authors.
#
# This library is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version
# 2.1 as published by the Free Software Foundation.
#
# This library is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301 USA
#
# -----------------------------------------------------------------------------

import types
import math
import os
import glob
import md5

from version import VERSION

# imlib2 wrapper
import _Imlib2

# image and font wrapper
from image import *
from font import *

# Implement a crude image cache.
#
# Imlib2 maintains its own cache, but I don't think it caches
# the raw image data, since this ends up being faster.

_image_cache = {
    'images': {},
    'order': [],
    'size': 0,
    'max-size': 16*1024   # 16 MB
}

def open_without_cache(file):
    """
    Create a new image object from the file 'file' without using the
    internal cache.
    """
    return Image(file, False)

def open(file):
    """
    Create a new image object from the file 'file'.
    """
    if file in _image_cache['images']:
        return _image_cache['images'][file].copy()

    image = Image(file)
    _image_cache['images'][file] = image
    _image_cache['order'].insert(0, file)
    _image_cache['size'] += image.width * image.height * 4 / 1024

    while _image_cache['size'] > _image_cache['max-size']:
        file = _image_cache['order'].pop()
        expired = _image_cache['images'][file]
        del _image_cache['images'][file]
        _image_cache['size'] -= expired.width * expired.height * 4 / 1024

    return image

def open_from_memory(buf):
    """
    Create a new image object from a memory buffer.
    """
    if type(buf) == str:
        buf = buffer(buf)
    img = _Imlib2.open_from_memory(buf)
    return Image(img)

def new(size, bytes = None, from_format = 'BGRA', copy = True):
    """
    Generates a new Image of size 'size', which is a tuple holding the width
    and height.  If 'bytes' is specified, the image is initialized from the
    raw BGRA data.  If 'copy' is False, 'bytes' must be either a write buffer
    or an integer pointing to a location in memory that will be used to hold
    the image.  (In this caes, from_format must be BGRA.)
    """
    if 0 in size:
        raise ValueError, 'Invalid image size %s' % repr(size)
    for val in size:
        if not isinstance(val, int):
            raise ValueError, 'Invalid image size %s' % repr(size)
    if bytes:
        if False in map(lambda x: x in 'RGBA', list(from_format)):
            raise ValueError, 'Converting from unsupported format: ' +  from_format
        if type(bytes) != int and len(bytes) < size[0]*size[1]*len(from_format):
            raise ValueError, 'Not enough bytes for converted format: expected %d, got %d' % \
                              (size[0]*size[1]*len(from_format), len(bytes))
        return Image(_Imlib2.create(size, bytes, from_format, copy))
    else:
        return Image(_Imlib2.create(size))

def add_font_path(path):
    """
    Add the given path to the list of paths to scan when loading fonts.
    """
    _Imlib2.add_font_path(path)

def load_font(font, size):
    """
    Return a Font object from the given font specified in the form
    'FontName/Size', such as 'Arial/16'
    """
    return Font(font + '/' + str(size))

def get_font_style_geometry(style):
    """
    Return the additional pixel the font needs for the style. This function
    will return left, top, right, bottom as number of pixels the text will
    start to the left/top and the number of pixels it needs more at the
    right/bottom. To avoid extra calculations the function will also return
    the additional width and height needed for the style.
    """
    return TEXT_STYLE_GEOMETRY[style]
<commit_msg>remove unneeded import<commit_after># -*- coding: iso-8859-1 -*-
# -----------------------------------------------------------------------------
# Imlib2.py - Imlib2 wrapper for Python
# -----------------------------------------------------------------------------
# $Id$
# -----------------------------------------------------------------------------
# kaa.imlib2 - An imlib2 wrapper for Python
# Copyright (C) 2004-2006 Dirk Meyer, Jason Tackaberry
#
# First Edition: Jason Tackaberry <tack@urandom.ca>
# Maintainer:    Jason Tackaberry <tack@urandom.ca>
#
# Please see the file AUTHORS for a complete list of authors.
#
# This library is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version
# 2.1 as published by the Free Software Foundation.
#
# This library is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301 USA
#
# -----------------------------------------------------------------------------

import types
import math
import os
import glob

from version import VERSION

# imlib2 wrapper
import _Imlib2

# image and font wrapper
from image import *
from font import *

# Implement a crude image cache.
#
# Imlib2 maintains its own cache, but I don't think it caches
# the raw image data, since this ends up being faster.

_image_cache = {
    'images': {},
    'order': [],
    'size': 0,
    'max-size': 16*1024   # 16 MB
}

def open_without_cache(file):
    """
    Create a new image object from the file 'file' without using the
    internal cache.
    """
    return Image(file, False)

def open(file):
    """
    Create a new image object from the file 'file'.
    """
    if file in _image_cache['images']:
        return _image_cache['images'][file].copy()

    image = Image(file)
    _image_cache['images'][file] = image
    _image_cache['order'].insert(0, file)
    _image_cache['size'] += image.width * image.height * 4 / 1024

    while _image_cache['size'] > _image_cache['max-size']:
        file = _image_cache['order'].pop()
        expired = _image_cache['images'][file]
        del _image_cache['images'][file]
        _image_cache['size'] -= expired.width * expired.height * 4 / 1024

    return image

def open_from_memory(buf):
    """
    Create a new image object from a memory buffer.
    """
    if type(buf) == str:
        buf = buffer(buf)
    img = _Imlib2.open_from_memory(buf)
    return Image(img)

def new(size, bytes = None, from_format = 'BGRA', copy = True):
    """
    Generates a new Image of size 'size', which is a tuple holding the width
    and height.  If 'bytes' is specified, the image is initialized from the
    raw BGRA data.  If 'copy' is False, 'bytes' must be either a write buffer
    or an integer pointing to a location in memory that will be used to hold
    the image.  (In this caes, from_format must be BGRA.)
    """
    if 0 in size:
        raise ValueError, 'Invalid image size %s' % repr(size)
    for val in size:
        if not isinstance(val, int):
            raise ValueError, 'Invalid image size %s' % repr(size)
    if bytes:
        if False in map(lambda x: x in 'RGBA', list(from_format)):
            raise ValueError, 'Converting from unsupported format: ' +  from_format
        if type(bytes) != int and len(bytes) < size[0]*size[1]*len(from_format):
            raise ValueError, 'Not enough bytes for converted format: expected %d, got %d' % \
                              (size[0]*size[1]*len(from_format), len(bytes))
        return Image(_Imlib2.create(size, bytes, from_format, copy))
    else:
        return Image(_Imlib2.create(size))

def add_font_path(path):
    """
    Add the given path to the list of paths to scan when loading fonts.
    """
    _Imlib2.add_font_path(path)

def load_font(font, size):
    """
    Return a Font object from the given font specified in the form
    'FontName/Size', such as 'Arial/16'
    """
    return Font(font + '/' + str(size))

def get_font_style_geometry(style):
    """
    Return the additional pixel the font needs for the style. This function
    will return left, top, right, bottom as number of pixels the text will
    start to the left/top and the number of pixels it needs more at the
    right/bottom. To avoid extra calculations the function will also return
    the additional width and height needed for the style.
    """
    return TEXT_STYLE_GEOMETRY[style]
<|endoftext|>


<commit_before># spongemock __main__.py
# author: Noah Krim
# email: nkrim62@gmail.com

from __future__ import print_function

import argparse
import re

from sys import stderr

try:
	from Tkinter import Tk
except ImportError:
	try:
		from tkinter import Tk
	except ImportError:
		pass

from .spongemock import mock

def eprint(*args, **kwargs):
	print(*args, file=stderr, **kwargs)

def main():
	parser = init_parser()
	args = parser.parse_args()
	try:
		out = mock(' '.join(args.text), args.bias, args.seed or args.strseed or None)
	except Exception as e:
		eprint('Error: '+sys.argv[0]+': '+str(e))
		exit(1)
	if args.copy:
		try:
			copy(out)
		except NameError:
			eprint('Warning: '+sys.argv[0]+': could not copy to clipboard. '
				+'Please make sure Tkinter is installed (more info https://tkinter.unpythonic.net/wiki/How_to_install_Tkinter).')
		except Exception:
			eprint('Warning: '+sys.argv[0]+': could not copy to clipboard because of an unexpected error.')
	print(out)

def init_parser():
	parser = argparse.ArgumentParser(description='Mock some text like spongebob would. mOCk SoMe TexT lIKe SpONGebOb wOuLd.')
	parser.add_argument('text', nargs='+', help='the text to mock. ThE tExT tO mOCk.')
	parser.add_argument('-c', '--copy', action='store_true', help='Mocked text will be copied to the clipboard.')
	parser.add_argument('-b', '--bias', type=float, default=0.5,
		help='This bias is used to succesively increase the chance of swapping from the previously-mocked case. '
			+'A value of `0` will ensure the chance is always 50/50, '
			+'and a value of `1` will ensure that after the first random choice the capitalization perfectly oscilates. '
			+'Default is `0.5`.')
	seed_group = parser.add_mutually_exclusive_group()
	seed_group.add_argument('-s', '--seed', type=parsable_seed, help='Seed for random number generator. Can be any number or string (numbers are parsed).')
	seed_group.add_argument('-S', '--strseed', help='Seed for random number generator. Does not attempt to parse the string to a number.')
	return parser

def parsable_seed(str_seed):
	# Try int parse
	if re.match(r'^-?\d+$', str_seed):
		return int(float(str_seed))
	# Try float parse
	try:
		return float(str_seed)
	except Exception:
		pass
	return str_seed

def copy(text):
	r = Tk()
	r.withdraw()
	r.clipboard_clear()
	r.clipboard_append(text)
	r.destroy()

if __name__ == '__main__':
	main()<commit_msg>v2.0 stuff reverted back from tkinter and just fixed some pyperclip error messaging.<commit_after>#!/usr/bin/env python
# spongemock __main__.py
# author: Noah Krim
# email: nkrim62@gmail.com

from __future__ import print_function

import argparse
import re
from pyperclip import copy
import sys

def eprint(*args, **kwargs):
	print(*args, file=sys.stderr, **kwargs)

def main():
	parser = init_parser()
	args = parser.parse_args()
	try:
		out = mock(' '.join(args.text), args.bias, args.seed or args.strseed or None)
	except Exception as e:
		eprint('Error: '+sys.argv[0]+': '+str(e))
		exit(1)
	if args.copy:
		try:
			copy(out)
		except Exception:
			eprint('Warning: '+sys.argv[0]+': could not copy the output to the clipboard because of an unexpected error. '
				+'If using Linux, pleaes make sure you have all the proper modules installed for pyperclip '
				+'(more info: https://tkinter.unpythonic.net/wiki/How_to_install_Tkinter).')
	print(out)

def init_parser():
	parser = argparse.ArgumentParser(description='Mock some text like spongebob would. mOCk SoMe TexT lIKe SpONGebOb wOuLd.')
	parser.add_argument('text', nargs='+', help='the text to mock. ThE tExT tO mOCk.')
	parser.add_argument('-c', '--copy', action='store_true', help='Mocked text will be copied to the clipboard.')
	parser.add_argument('-b', '--bias', type=float, default=0.5,
		help='This bias is used to succesively increase the chance of swapping from the previously-mocked case. '
			+'A value of `0` will ensure the chance is always 50/50, '
			+'and a value of `1` will ensure that after the first random choice the capitalization perfectly oscilates. '
			+'Default is `0.5`.')
	seed_group = parser.add_mutually_exclusive_group()
	seed_group.add_argument('-s', '--seed', type=parsable_seed, help='Seed for random number generator. Can be any number or string (numbers are parsed).')
	seed_group.add_argument('-S', '--strseed', help='Seed for random number generator. Does not attempt to parse the string to a number.')
	return parser

def parsable_seed(str_seed):
	# Try int parse
	if re.match(r'^-?\d+$', str_seed):
		return int(float(str_seed))
	# Try float parse
	try:
		return float(str_seed)
	except Exception:
		pass
	return str_seed

if __name__ == '__main__':
	if __package__ is None:
		from os import path
		sys.path.append( path.dirname(path.abspath(__file__) ) )
		from spongemock import mock
	else:
		from .spongemock import mock
	main()<|endoftext|>


<commit_before>#! /usr/bin/env python

########################################################################
#                                                                      #
# This script was written by Thomas Heavey in 2016.                    #
#        theavey@bu.edu     thomasjheavey@gmail.com                    #
#                                                                      #
# Copyright 2016 Thomas J. Heavey IV                                   #
#                                                                      #
# Licensed under the Apache License, Version 2.0 (the "License");      #
# you may not use this file except in compliance with the License.     #
# You may obtain a copy of the License at                              #
#                                                                      #
#    http://www.apache.org/licenses/LICENSE-2.0                        #
#                                                                      #
# Unless required by applicable law or agreed to in writing, software  #
# distributed under the License is distributed on an "AS IS" BASIS,    #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or      #
# implied.                                                             #
# See the License for the specific language governing permissions and  #
# limitations under the License.                                       #
#                                                                      #
########################################################################

import glob
import gromacs.formats
import gromacs.tools
import re
import matplotlib.pyplot as plt
import os
import numpy as np

__version__ = '0.0.2'

# todo define a set of run data as a class
# possibly as a class of mdanalysis universes, but not sure how to work with
# replicas vs. walkers there.

def find_energies():
    """find_energies() is a function that finds all files in the current
    directory that end in a numeral followed by '.edr'. For each of these
    files, it checks if a file named energy(same number).xvg exists, and if
    not, creates it by calling the GROMACS tool gmx energy where input='13'
    corresponds to the total energy at each time."""
    energy_files = glob.glob('*[0-9].edr')
    output_files = []
    for file_name in energy_files:
        output_name = ('energy' + re.search('[0-9]*(?=\.edr)',
                                            file_name).group(0) + '.xvg')
        if not os.path.isfile(output_name):
            gromacs.tools.Energy(f=file_name, o=output_name, input="13")()
        output_files += [output_name]
    output_files.sort()
    output_files.sort(key=len)
    return output_files

def import_energies(output_files):
    """import_energies(file_list) takes a list of .xvg files in the current
    directory, imports their second columns (likely a list of energies at
    consecutive time steps), and returns that as a list of arrays."""
    imported_data = []
    for file_name in output_files:
        xvg_file = gromacs.formats.XVG(filename=file_name)
        imported_data += [xvg_file.array[1]]
    return imported_data

# Run this only if called from the command line
if __name__ == "__main__":
    import argparse
    # todo add argument and code for way to save to a file instead of viewing
    # todo take arguments to change the plotting
    parser = argparse.ArgumentParser(description='A script to plot energy '
                                                 'histograms from a GROMACS '
                                                 'parallel tempering simulation.')
    parser.add_argument('--version', action='version',
                        version='%(prog)s v{}'.format(__version__))
    args = parser.parse_args()

    out_files = find_energies()
    all_data = import_energies(out_files)

    plt.hist(all_data, 50, histtype='stepfilled')

    plt.show()

def combine_energy_files(basename='energy', files=False):
    """combine_energy_files(basename='energy', files=False) is a function that
    combines a set of .xvg files writes a combined .xvg file.
    'basename' is the first part of the name for the xvg files to be combined and
    should differentiate these from any other *.xvg files in the folder.
    Alternatively, the list of files (in the desired order) can be passed in
    with the keyword 'files'."""
    if not files:
        files = glob.glob(basename + '*.xvg')
        files.sort()
        files.sort(key=len)
    data = [gromacs.formats.XVG(filename=files[0]).array[0]]
    data += import_energies(files)
    data = np.array(data)
    gromacs.formats.XVG(array=data).write(filename=basename+'_comb.xvg')

def deconvolve_energies(energyfile='energy_comb.xvg',
                        indexfile='replica_temp.xvg'):
    """deconvolve_energies(energyfile='energy_comb.xvg',
    indexfile='replica_temp.xvg') is a function that takes an xvg files that is
    has n columns of energies likely from a replica exchange simulation where each
    replica remains at a constant temperature (as GROMACS does) and using the n
    data columns of an index xvg file returns an array of the energies where each
    row is now from one 'walker' (continuous coordinates taken by sampling various
    temperatures or other replica conditions)."""
    energies_indexed = gromacs.formats.XVG(filename=energyfile).array
    indices_indexed = gromacs.formats.XVG(filename=indexfile).array.astype(int)
    # todo check for relative start/end points automatically
    length_e = energies_indexed.shape[1]
    length_i = indices_indexed.shape[1]
    ratio = float(length_e) / float(length_i)
    approx_ratio = int(round(ratio))
    if ratio == 1.0:
        deconvolved_energies = energies_indexed[1:][
            indices_indexed[1:],
            np.arange(length_i)]
    elif ratio > 1:
        if approx_ratio == ratio:
            extra_e = 0
            extra_i = 0
        elif approx_ratio > ratio:
            extra_i = length_i - length_e / approx_ratio
            extra_e = np.mod(length_e, length_i-extra_i)
        elif approx_ratio < ratio:
            extra_e = np.mod(length_e, length_i)
            extra_i = 0
        else:
            raise ImportError('ratio: {}, approx ratio: {}'.format(ratio, approx_ratio))
        deconvolved_energies = energies_indexed[1:, :length_e-extra_e:approx_ratio][
            indices_indexed[1:, :length_i-extra_i],
            np.arange((length_i-extra_i))]
        e_times = [energies_indexed[0, :length_e-extra_e:approx_ratio][0],
                   energies_indexed[0, :length_e-extra_e:approx_ratio][-1]]
        i_times = [indices_indexed[0, :length_i-extra_i][0],
                   indices_indexed[0, :length_i-extra_i][-1]]
    elif ratio < 1:
        print('likely undersampling energies because energy / indices ratio is '
              '{}'.format(ratio))
        ratio = 1 / ratio
        approx_ratio = int(round(ratio))
        if approx_ratio == ratio:
            extra_e = 0
            extra_i = 0
        # Not so sure about this...
        elif approx_ratio > ratio:
            extra_e = length_e - length_i / approx_ratio
            extra_i = np.mod(length_i, length_e-extra_e)
        elif approx_ratio < ratio:
            extra_i = np.mod(length_i, length_e)
            extra_e = 0
        else:
            raise ImportError('ratio: {}, approx ratio: {}'.format(ratio, approx_ratio))
        deconvolved_energies = energies_indexed[1:, :length_e-extra_e][
            indices_indexed[1:, :length_i-extra_i:approx_ratio],
            np.arange((length_i-extra_i)/approx_ratio)]
        e_times = [energies_indexed[0, :length_e-extra_e][0],
                   energies_indexed[0, :length_e-extra_e][-1]]
        i_times = [indices_indexed[0, :length_i-extra_i:approx_ratio][0],
                   indices_indexed[0, :length_i-extra_i:approx_ratio][-1]]
    else:
        print('length of energy file is {}'.format(length_e))
        print('length of index file is {}'.format(length_i))
        raise ImportError('Not sure how to handle those values')
    # todo maybe error check is a more pythonic manner with a try/except loop
    # if length_e != length_i:
    #     print("length of energies, {}, != length of indices, "
    #           "{}!".format(length_e, length_i))
    #     raise IndexError('lengths not equals')
    print('energies start: {}; end: {}'.format(e_times[0], e_times[1]))
    print('indices start: {}; end: {}'.format(i_times[0], i_times[1]))
    print('These values should be about the same if this is working properly')
    return deconvolved_energies

def plot_array(array, index_offset=0, num_replicas=False, n_rows=False, n_cols=False):
    """plot_array(array, index_offset=0, num_replicas=16, n_rows=False, n_cols=False)
    will put each column of array in a different axes of a figure and then return
    the figure."""
    if not num_replicas:
        num_replicas = array.shape[0] - index_offset
    from math import sqrt, ceil
    if n_rows == n_cols == False:
        n_rows = int(ceil(sqrt(float(num_replicas))))
        n_cols = n_rows
    fig, axes = plt.subplots(n_rows, n_cols, sharex=True, sharey=True)
    for i in range(num_replicas):
        ax = axes.flat[i]
        ax.plot(array[i+index_offset])
    return fig

def hist_array(array, index_offset=0, num_replicas=False, n_rows=False, n_cols=False,
               n_bins=10):
    """hist_array(array, index_offset=0, num_replicas=16, n_rows=False, n_cols=False,
    n_bins=10) will put each column of array in a different axes of a figure and
    then return the figure."""
    if not num_replicas:
        num_replicas = array.shape[0] - index_offset
    from math import sqrt, ceil
    if n_rows == n_cols == False:
        n_rows = int(ceil(sqrt(float(num_replicas))))
        n_cols = n_rows
    fig, axes = plt.subplots(n_rows, n_cols, sharex=True, sharey=True)
    for i in range(num_replicas):
        ax = axes.flat[i]
        ax.hist(array[i+index_offset], n_bins)
    return fig

def solute_trr(trr_base_name='npt_PT_out', tpr_base_name='TOPO/npt',
               output_base_name='solute', index='index.ndx', demux=True):
    """solute_trr takes file base names as input, creates a separate trr file for each
    trajectory that only includes the solutes, and then returns a list of the names of
    the created files.
    If demux=True, it will first use trjcat to deconvolve the walker trajectories (to
    get continuous coordinate files as opposed to continuous temperature).
    This uses gromacswrapper to call trjconv and possibly trjcat."""
    trr_files = glob.glob(trr_base_name + '*.trr')
    trr_files.sort()
    trr_files.sort(key=len)
    tpr_files = glob.glob(tpr_base_name + '*.tpr')
    tpr_files.sort()
    tpr_files.sort(key=len)
    output_files = []
    if demux:
        d_trr_base_name = 'deconv' + trr_base_name
        prev_deconv_files = glob.glob(d_trr_base_name+'*.trr')
        if len(prev_deconv_files) == len(trr_files):
            print('Likely already deconvolved trajectories, skipping that step')
        else:
            gromacs.tools.Trjcat(f=trr_files, o='demuxed.trr', n='index.ndx',
                                 demux='replica_index.xvg', input='CHR')()
            trr_files = glob.glob('*demuxed.trr')
            trr_files.sort()
            trr_files.sort(key=len)
            for (i, trr_file) in enumerate(trr_files):
                number = trr_file.split('_')[0]
                new_name = d_trr_base_name + number + '.trr'
                os.rename(trr_file, new_name)
                trr_files[i] = new_name
        trr_base_name = d_trr_base_name
    if len(trr_files) != len(tpr_files):
        raise IndexError('Number of trr and tpr files not equal: '
                         '{} and {}'.format(len(trr_files), len(tpr_files)))
    for (i, trr_name) in enumerate(trr_files):
        number_match = re.search('(?:'+trr_base_name+')(\d+)(?:\.trr)', trr_name)
        number = number_match.group(1)
        out_file = output_base_name + number + '.trr'
        output_files.append(out_file)
        gromacs.tools.Trjconv(s=tpr_files[i], pbc='mol', f=trr_name, o=out_file,
                              n=index, center=True, input=('CHR', 'CHR'))()
    return output_files

def radii_of_gyration(basename='solute', atom_selection=False, resname='TAD'):
    """A function to find the radius of gyration for all timesteps for a set of
    REMD trajectories.
    The basename is the name before the numbers for the trajectory files.
    resname is the name of the desired residue to be selected.
    A more general atom_selection can be given, in which case resname will
    be ignored.
    Assumes all trajectories are the same length.
    Returns the values as a numpy array."""
    u_solutes = []
    files = glob.glob(basename+'*.trr')
    num_files = len(files)
    for file_name in files:
        u_solutes.append(MDAnalysis.Universe('geom-solutes.gro', file_name))
    rgs = np.zeros((num_files, len(u_solutes[0].trajectory)))
    if not atom_selection:
        selection = 'resname ' + resname
    else:
        selection = atom_selection
    for (i, u) in enumerate(u_solutes):
        tad = u.select_atoms(selection)
        for fr in u.trajectory:
            rgs[i, fr.frame] = tad.radius_of_gyration()
    return rgs
<commit_msg>Updated imports<commit_after>#! /usr/bin/env python

########################################################################
#                                                                      #
# This script was written by Thomas Heavey in 2016.                    #
#        theavey@bu.edu     thomasjheavey@gmail.com                    #
#                                                                      #
# Copyright 2016 Thomas J. Heavey IV                                   #
#                                                                      #
# Licensed under the Apache License, Version 2.0 (the "License");      #
# you may not use this file except in compliance with the License.     #
# You may obtain a copy of the License at                              #
#                                                                      #
#    http://www.apache.org/licenses/LICENSE-2.0                        #
#                                                                      #
# Unless required by applicable law or agreed to in writing, software  #
# distributed under the License is distributed on an "AS IS" BASIS,    #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or      #
# implied.                                                             #
# See the License for the specific language governing permissions and  #
# limitations under the License.                                       #
#                                                                      #
########################################################################

import glob
import os
import re

import gromacs.formats
import gromacs.tools
import matplotlib.pyplot as plt
import numpy as np
import MDAnalysis

__version__ = '0.0.2'

# todo define a set of run data as a class
# possibly as a class of mdanalysis universes, but not sure how to work with
# replicas vs. walkers there.

def find_energies():
    """find_energies() is a function that finds all files in the current
    directory that end in a numeral followed by '.edr'. For each of these
    files, it checks if a file named energy(same number).xvg exists, and if
    not, creates it by calling the GROMACS tool gmx energy where input='13'
    corresponds to the total energy at each time."""
    energy_files = glob.glob('*[0-9].edr')
    output_files = []
    for file_name in energy_files:
        output_name = ('energy' + re.search('[0-9]*(?=\.edr)',
                                            file_name).group(0) + '.xvg')
        if not os.path.isfile(output_name):
            gromacs.tools.Energy(f=file_name, o=output_name, input="13")()
        output_files += [output_name]
    output_files.sort()
    output_files.sort(key=len)
    return output_files

def import_energies(output_files):
    """import_energies(file_list) takes a list of .xvg files in the current
    directory, imports their second columns (likely a list of energies at
    consecutive time steps), and returns that as a list of arrays."""
    imported_data = []
    for file_name in output_files:
        xvg_file = gromacs.formats.XVG(filename=file_name)
        imported_data += [xvg_file.array[1]]
    return imported_data

# Run this only if called from the command line
if __name__ == "__main__":
    import argparse
    # todo add argument and code for way to save to a file instead of viewing
    # todo take arguments to change the plotting
    parser = argparse.ArgumentParser(description='A script to plot energy '
                                                 'histograms from a GROMACS '
                                                 'parallel tempering simulation.')
    parser.add_argument('--version', action='version',
                        version='%(prog)s v{}'.format(__version__))
    args = parser.parse_args()

    out_files = find_energies()
    all_data = import_energies(out_files)

    plt.hist(all_data, 50, histtype='stepfilled')

    plt.show()

def combine_energy_files(basename='energy', files=False):
    """combine_energy_files(basename='energy', files=False) is a function that
    combines a set of .xvg files writes a combined .xvg file.
    'basename' is the first part of the name for the xvg files to be combined and
    should differentiate these from any other *.xvg files in the folder.
    Alternatively, the list of files (in the desired order) can be passed in
    with the keyword 'files'."""
    if not files:
        files = glob.glob(basename + '*.xvg')
        files.sort()
        files.sort(key=len)
    data = [gromacs.formats.XVG(filename=files[0]).array[0]]
    data += import_energies(files)
    data = np.array(data)
    gromacs.formats.XVG(array=data).write(filename=basename+'_comb.xvg')

def deconvolve_energies(energyfile='energy_comb.xvg',
                        indexfile='replica_temp.xvg'):
    """deconvolve_energies(energyfile='energy_comb.xvg',
    indexfile='replica_temp.xvg') is a function that takes an xvg files that is
    has n columns of energies likely from a replica exchange simulation where each
    replica remains at a constant temperature (as GROMACS does) and using the n
    data columns of an index xvg file returns an array of the energies where each
    row is now from one 'walker' (continuous coordinates taken by sampling various
    temperatures or other replica conditions)."""
    energies_indexed = gromacs.formats.XVG(filename=energyfile).array
    indices_indexed = gromacs.formats.XVG(filename=indexfile).array.astype(int)
    # todo check for relative start/end points automatically
    length_e = energies_indexed.shape[1]
    length_i = indices_indexed.shape[1]
    ratio = float(length_e) / float(length_i)
    approx_ratio = int(round(ratio))
    if ratio == 1.0:
        deconvolved_energies = energies_indexed[1:][
            indices_indexed[1:],
            np.arange(length_i)]
    elif ratio > 1:
        if approx_ratio == ratio:
            extra_e = 0
            extra_i = 0
        elif approx_ratio > ratio:
            extra_i = length_i - length_e / approx_ratio
            extra_e = np.mod(length_e, length_i-extra_i)
        elif approx_ratio < ratio:
            extra_e = np.mod(length_e, length_i)
            extra_i = 0
        else:
            raise ImportError('ratio: {}, approx ratio: {}'.format(ratio, approx_ratio))
        deconvolved_energies = energies_indexed[1:, :length_e-extra_e:approx_ratio][
            indices_indexed[1:, :length_i-extra_i],
            np.arange((length_i-extra_i))]
        e_times = [energies_indexed[0, :length_e-extra_e:approx_ratio][0],
                   energies_indexed[0, :length_e-extra_e:approx_ratio][-1]]
        i_times = [indices_indexed[0, :length_i-extra_i][0],
                   indices_indexed[0, :length_i-extra_i][-1]]
    elif ratio < 1:
        print('likely undersampling energies because energy / indices ratio is '
              '{}'.format(ratio))
        ratio = 1 / ratio
        approx_ratio = int(round(ratio))
        if approx_ratio == ratio:
            extra_e = 0
            extra_i = 0
        # Not so sure about this...
        elif approx_ratio > ratio:
            extra_e = length_e - length_i / approx_ratio
            extra_i = np.mod(length_i, length_e-extra_e)
        elif approx_ratio < ratio:
            extra_i = np.mod(length_i, length_e)
            extra_e = 0
        else:
            raise ImportError('ratio: {}, approx ratio: {}'.format(ratio, approx_ratio))
        deconvolved_energies = energies_indexed[1:, :length_e-extra_e][
            indices_indexed[1:, :length_i-extra_i:approx_ratio],
            np.arange((length_i-extra_i)/approx_ratio)]
        e_times = [energies_indexed[0, :length_e-extra_e][0],
                   energies_indexed[0, :length_e-extra_e][-1]]
        i_times = [indices_indexed[0, :length_i-extra_i:approx_ratio][0],
                   indices_indexed[0, :length_i-extra_i:approx_ratio][-1]]
    else:
        print('length of energy file is {}'.format(length_e))
        print('length of index file is {}'.format(length_i))
        raise ImportError('Not sure how to handle those values')
    # todo maybe error check is a more pythonic manner with a try/except loop
    # if length_e != length_i:
    #     print("length of energies, {}, != length of indices, "
    #           "{}!".format(length_e, length_i))
    #     raise IndexError('lengths not equals')
    print('energies start: {}; end: {}'.format(e_times[0], e_times[1]))
    print('indices start: {}; end: {}'.format(i_times[0], i_times[1]))
    print('These values should be about the same if this is working properly')
    return deconvolved_energies

def plot_array(array, index_offset=0, num_replicas=False, n_rows=False, n_cols=False):
    """plot_array(array, index_offset=0, num_replicas=16, n_rows=False, n_cols=False)
    will put each column of array in a different axes of a figure and then return
    the figure."""
    if not num_replicas:
        num_replicas = array.shape[0] - index_offset
    from math import sqrt, ceil
    if n_rows == n_cols == False:
        n_rows = int(ceil(sqrt(float(num_replicas))))
        n_cols = n_rows
    fig, axes = plt.subplots(n_rows, n_cols, sharex=True, sharey=True)
    for i in range(num_replicas):
        ax = axes.flat[i]
        ax.plot(array[i+index_offset])
    return fig

def hist_array(array, index_offset=0, num_replicas=False, n_rows=False, n_cols=False,
               n_bins=10):
    """hist_array(array, index_offset=0, num_replicas=16, n_rows=False, n_cols=False,
    n_bins=10) will put each column of array in a different axes of a figure and
    then return the figure."""
    if not num_replicas:
        num_replicas = array.shape[0] - index_offset
    from math import sqrt, ceil
    if n_rows == n_cols == False:
        n_rows = int(ceil(sqrt(float(num_replicas))))
        n_cols = n_rows
    fig, axes = plt.subplots(n_rows, n_cols, sharex=True, sharey=True)
    for i in range(num_replicas):
        ax = axes.flat[i]
        ax.hist(array[i+index_offset], n_bins)
    return fig

def solute_trr(trr_base_name='npt_PT_out', tpr_base_name='TOPO/npt',
               output_base_name='solute', index='index.ndx', demux=True):
    """solute_trr takes file base names as input, creates a separate trr file for each
    trajectory that only includes the solutes, and then returns a list of the names of
    the created files.
    If demux=True, it will first use trjcat to deconvolve the walker trajectories (to
    get continuous coordinate files as opposed to continuous temperature).
    This uses gromacswrapper to call trjconv and possibly trjcat."""
    trr_files = glob.glob(trr_base_name + '*.trr')
    trr_files.sort()
    trr_files.sort(key=len)
    tpr_files = glob.glob(tpr_base_name + '*.tpr')
    tpr_files.sort()
    tpr_files.sort(key=len)
    output_files = []
    if demux:
        d_trr_base_name = 'deconv' + trr_base_name
        prev_deconv_files = glob.glob(d_trr_base_name+'*.trr')
        if len(prev_deconv_files) == len(trr_files):
            print('Likely already deconvolved trajectories, skipping that step')
        else:
            gromacs.tools.Trjcat(f=trr_files, o='demuxed.trr', n='index.ndx',
                                 demux='replica_index.xvg', input='CHR')()
            trr_files = glob.glob('*demuxed.trr')
            trr_files.sort()
            trr_files.sort(key=len)
            for (i, trr_file) in enumerate(trr_files):
                number = trr_file.split('_')[0]
                new_name = d_trr_base_name + number + '.trr'
                os.rename(trr_file, new_name)
                trr_files[i] = new_name
        trr_base_name = d_trr_base_name
    if len(trr_files) != len(tpr_files):
        raise IndexError('Number of trr and tpr files not equal: '
                         '{} and {}'.format(len(trr_files), len(tpr_files)))
    for (i, trr_name) in enumerate(trr_files):
        number_match = re.search('(?:'+trr_base_name+')(\d+)(?:\.trr)', trr_name)
        number = number_match.group(1)
        out_file = output_base_name + number + '.trr'
        output_files.append(out_file)
        gromacs.tools.Trjconv(s=tpr_files[i], pbc='mol', f=trr_name, o=out_file,
                              n=index, center=True, input=('CHR', 'CHR'))()
    return output_files

def radii_of_gyration(basename='solute', atom_selection=False, resname='TAD'):
    """A function to find the radius of gyration for all timesteps for a set of
    REMD trajectories.
    The basename is the name before the numbers for the trajectory files.
    resname is the name of the desired residue to be selected.
    A more general atom_selection can be given, in which case resname will
    be ignored.
    Assumes all trajectories are the same length.
    Returns the values as a numpy array."""
    u_solutes = []
    files = glob.glob(basename+'*.trr')
    num_files = len(files)
    for file_name in files:
        u_solutes.append(MDAnalysis.Universe('geom-solutes.gro', file_name))
    rgs = np.zeros((num_files, len(u_solutes[0].trajectory)))
    if not atom_selection:
        selection = 'resname ' + resname
    else:
        selection = atom_selection
    for (i, u) in enumerate(u_solutes):
        tad = u.select_atoms(selection)
        for fr in u.trajectory:
            rgs[i, fr.frame] = tad.radius_of_gyration()
    return rgs
<|endoftext|>