<commit_before>#@title Imports
import os
import numpy as np
import time

import graph_tool

def gini(array):
    array = array.astype(np.float32)
    array += np.finfo(np.float32).eps
    array = np.sort(array)
    n = array.shape[0]
    index = np.arange(1, n + 1)
    return np.sum((2 * index - n  - 1) * array) / (n * np.sum(array))

def diameter_lb(graph, n_tries=100):
  max_diameter = -1
  nodes = np.random.choice(list(graph.vertices()), n_tries)
  for node in nodes:
    diameter = graph_tool.topology.pseudo_diameter(graph, node)[0]
    if diameter > max_diameter:
      max_diameter = diameter
  return max_diameter

def bfs_sp(graph, source, others):
  others = set(others)
  for edge in graph_tool.search.bfs_iterator(graph, source):
    if edge.target() in others:
      return graph_tool.topology.shortest_distance(graph, source, edge.target())
  return -1

def average_cc_sp_length(G_comp):
  sp_lengths = 0
  sp_distances = graph_tool.topology.shortest_distance(G_comp, directed=False)
  for vtx in G_comp.vertices():
    sp_lengths += np.sum(sp_distances[vtx].a)
  return sp_lengths / G_comp.num_vertices() / (G_comp.num_vertices() - 1)

def matrix_row_norm(X):
  return X / np.linalg.norm(X, axis=1)[:, None]

def GraphMetrics(G):
  result = {}
  result['n_nodes'] = G.num_vertices()
  result['n_edges'] = G.num_edges()
  result['edge_density'] = G.num_edges() / G.num_vertices() / (G.num_vertices() - 1)
  result['avg_in_degree'] = np.mean(G.get_in_degrees(G.get_vertices()))
  result['edge_reciprocity'] = graph_tool.topology.edge_reciprocity(G)
  result['avg_undirected_degree'] = np.mean(G.get_out_degrees(G.get_vertices())) / 2
  result['degree_gini'] = gini(G.get_out_degrees(G.get_vertices()))
  result['pseudo_diameter'] = diameter_lb(G)
  coreness = np.array(graph_tool.topology.kcore_decomposition(G).a)
  result['coreness_eq_1'] = np.sum(coreness == 1) / G.num_vertices()
  result['coreness_geq_2'] = np.sum(coreness >= 2) / G.num_vertices()
  result['coreness_geq_5'] = np.sum(coreness >= 5) / G.num_vertices()
  result['coreness_geq_10'] = np.sum(coreness >= 10) / G.num_vertices()
  result['coreness_gini'] = gini(coreness)
  result['avg_local_cc'] = np.mean(np.array(graph_tool.clustering.local_clustering(G).a))
  result['global_cc'] = graph_tool.clustering.global_clustering(G)[0]
  comp = graph_tool.topology.label_largest_component(G)
  result['cc_size'] = int(comp.a.sum()) / G.num_vertices()
  return result

def edge_homogeneity(graph, labels):
  count_in = 0
  count_total = 0
  for edge in graph.edges():
    count_total += 1
    count_in += (
        labels[int(edge.source())] == labels[int(edge.target())])
  return count_in / count_total

def sum_angular_distance_matrix_nan(X, Y, batch_size=100):
  nx = X.shape[0]
  ny = Y.shape[0]

  total_sum = 0.0

  pos1 = 0
  while pos1 < nx:
    if pos1 % batch_size == 0:
      print('----now doing pos1 = %d' % pos1)
    end1 = min(pos1 + batch_size, nx)
    vec1 = X[pos1:end1, :]

    pos2 = 0
    curr_sum = 0.0
    while pos2 < ny:
      # Get data
      end2 = min(pos2 + batch_size, ny)
      vec2 = Y[pos2:end2, :]

      vecsims = np.matmul(vec1, vec2.T)
      vecsims = np.clip(vecsims, -1, 1)
      vecsims = 1.0 - np.arccos(vecsims) / np.pi
      vecsims[np.where(np.isnan(vecsims))] = 1.0
      total_sum += np.sum(vecsims)

      pos2 += batch_size

    pos1 += batch_size
  return total_sum

def feature_homogeneity(normed_features, labels):
  all_labels = sorted(list(set(labels)))
  n_labels = len(all_labels)
  sum_mat = np.zeros((n_labels, n_labels))
  count_mat = np.zeros((n_labels, n_labels))
  for label_idx, i in enumerate(all_labels):
    idx_i = np.where(labels == i)[0]
    vecs_i = normed_features[idx_i, :]
    print('processing i label %d w/%d examples' % (i, vecs_i.shape[0]))
    for j in all_labels[label_idx:]:
      start_time = time.time()
      idx_j = np.where(labels == j)[0]
      vecs_j = normed_features[idx_j, :]
      print('-processing j label %d w/%d examples' % (j, vecs_j.shape[0]))
      the_sum = sum_angular_distance_matrix_nan(vecs_i, vecs_j)
      the_count = len(idx_j) * len(idx_i)
      if i == j:
        the_sum -= float(len(idx_j))
        the_sum /= 2.0
        the_count -= float(len(idx_j))
        the_count /= 2
      sum_mat[i, j] = the_sum
      count_mat[i, j] = the_count
      elapsed = time.time() - start_time
      print('-label %d took %0.2f minutes' % (j, elapsed / 60.0))
  out_avg = np.sum(sum_mat[np.triu_indices(sum_mat.shape[0])]) / (
    np.sum(count_mat[np.triu_indices(count_mat.shape[0])]))
  in_avg = np.sum(np.diag(sum_mat)) / np.sum(np.diag(count_mat))
  return in_avg, out_avg

def NodeLabelMetrics(graph, labels, features):
  metrics = {'edge_homogeneity': edge_homogeneity(graph, labels)}
  normed_features = matrix_row_norm(features)
  in_avg, out_avg = feature_homogeneity(normed_features, labels)
  metrics.update({'avg_in_feature_angular_distance' : in_avg,
                  'avg_out_feature_angular_distance': out_avg})
  return metrics<commit_msg>Add "SNR" for feature angulars<commit_after>#@title Imports
import os
import numpy as np
import time

import graph_tool

def gini(array):
    array = array.astype(np.float32)
    array += np.finfo(np.float32).eps
    array = np.sort(array)
    n = array.shape[0]
    index = np.arange(1, n + 1)
    return np.sum((2 * index - n  - 1) * array) / (n * np.sum(array))

def diameter_lb(graph, n_tries=100):
  max_diameter = -1
  nodes = np.random.choice(list(graph.vertices()), n_tries)
  for node in nodes:
    diameter = graph_tool.topology.pseudo_diameter(graph, node)[0]
    if diameter > max_diameter:
      max_diameter = diameter
  return max_diameter

def bfs_sp(graph, source, others):
  others = set(others)
  for edge in graph_tool.search.bfs_iterator(graph, source):
    if edge.target() in others:
      return graph_tool.topology.shortest_distance(graph, source, edge.target())
  return -1

def average_cc_sp_length(G_comp):
  sp_lengths = 0
  sp_distances = graph_tool.topology.shortest_distance(G_comp, directed=False)
  for vtx in G_comp.vertices():
    sp_lengths += np.sum(sp_distances[vtx].a)
  return sp_lengths / G_comp.num_vertices() / (G_comp.num_vertices() - 1)

def matrix_row_norm(X):
  return X / np.linalg.norm(X, axis=1)[:, None]

def GraphMetrics(G):
  result = {}
  result['n_nodes'] = G.num_vertices()
  result['n_edges'] = G.num_edges()
  result['edge_density'] = G.num_edges() / G.num_vertices() / (G.num_vertices() - 1)
  result['avg_in_degree'] = np.mean(G.get_in_degrees(G.get_vertices()))
  result['edge_reciprocity'] = graph_tool.topology.edge_reciprocity(G)
  result['avg_undirected_degree'] = np.mean(G.get_out_degrees(G.get_vertices())) / 2
  result['degree_gini'] = gini(G.get_out_degrees(G.get_vertices()))
  result['pseudo_diameter'] = diameter_lb(G)
  coreness = np.array(graph_tool.topology.kcore_decomposition(G).a)
  result['coreness_eq_1'] = np.sum(coreness == 1) / G.num_vertices()
  result['coreness_geq_2'] = np.sum(coreness >= 2) / G.num_vertices()
  result['coreness_geq_5'] = np.sum(coreness >= 5) / G.num_vertices()
  result['coreness_geq_10'] = np.sum(coreness >= 10) / G.num_vertices()
  result['coreness_gini'] = gini(coreness)
  result['avg_local_cc'] = np.mean(np.array(graph_tool.clustering.local_clustering(G).a))
  result['global_cc'] = graph_tool.clustering.global_clustering(G)[0]
  comp = graph_tool.topology.label_largest_component(G)
  result['cc_size'] = int(comp.a.sum()) / G.num_vertices()
  return result

def edge_homogeneity(graph, labels):
  count_in = 0
  count_total = 0
  for edge in graph.edges():
    count_total += 1
    count_in += (
        labels[int(edge.source())] == labels[int(edge.target())])
  return count_in / count_total

def sum_angular_distance_matrix_nan(X, Y, batch_size=100):
  nx = X.shape[0]
  ny = Y.shape[0]

  total_sum = 0.0

  pos1 = 0
  while pos1 < nx:
    if pos1 % batch_size == 0:
      print('----now doing pos1 = %d' % pos1)
    end1 = min(pos1 + batch_size, nx)
    vec1 = X[pos1:end1, :]

    pos2 = 0
    curr_sum = 0.0
    while pos2 < ny:
      # Get data
      end2 = min(pos2 + batch_size, ny)
      vec2 = Y[pos2:end2, :]

      vecsims = np.matmul(vec1, vec2.T)
      vecsims = np.clip(vecsims, -1, 1)
      vecsims = 1.0 - np.arccos(vecsims) / np.pi
      vecsims[np.where(np.isnan(vecsims))] = 1.0
      total_sum += np.sum(vecsims)

      pos2 += batch_size

    pos1 += batch_size
  return total_sum

def feature_homogeneity(normed_features, labels):
  all_labels = sorted(list(set(labels)))
  n_labels = len(all_labels)
  sum_mat = np.zeros((n_labels, n_labels))
  count_mat = np.zeros((n_labels, n_labels))
  for label_idx, i in enumerate(all_labels):
    idx_i = np.where(labels == i)[0]
    vecs_i = normed_features[idx_i, :]
    print('processing i label %d w/%d examples' % (i, vecs_i.shape[0]))
    for j in all_labels[label_idx:]:
      start_time = time.time()
      idx_j = np.where(labels == j)[0]
      vecs_j = normed_features[idx_j, :]
      print('-processing j label %d w/%d examples' % (j, vecs_j.shape[0]))
      the_sum = sum_angular_distance_matrix_nan(vecs_i, vecs_j)
      the_count = len(idx_j) * len(idx_i)
      if i == j:
        the_sum -= float(len(idx_j))
        the_sum /= 2.0
        the_count -= float(len(idx_j))
        the_count /= 2
      sum_mat[i, j] = the_sum
      count_mat[i, j] = the_count
      elapsed = time.time() - start_time
      print('-label %d took %0.2f minutes' % (j, elapsed / 60.0))
  out_avg = np.sum(sum_mat[np.triu_indices(sum_mat.shape[0])]) / (
    np.sum(count_mat[np.triu_indices(count_mat.shape[0])]))
  in_avg = np.sum(np.diag(sum_mat)) / np.sum(np.diag(count_mat))
  return in_avg, out_avg

def NodeLabelMetrics(graph, labels, features):
  metrics = {'edge_homogeneity': edge_homogeneity(graph, labels)}
  normed_features = matrix_row_norm(features)
  in_avg, out_avg = feature_homogeneity(normed_features, labels)
  metrics.update({'avg_in_feature_angular_distance': in_avg,
                  'avg_out_feature_angular_distance': out_avg,
                  'feature_angular_snr': in_avg/out_avg})
  return metrics<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
# Thomas Nagy, 2005-2010 (ita)

"""
logging, colors, terminal width and pretty-print
"""

import os, re, traceback, sys
from waflib import Utils, ansiterm

if not os.environ.get('NOSYNC', False):
	# synchronized output is nearly mandatory to prevent garbled output
	if sys.stdout.isatty() and id(sys.stdout) == id(sys.__stdout__):
		sys.stdout = ansiterm.AnsiTerm(sys.stdout)
	if sys.stderr.isatty() and id(sys.stderr) == id(sys.__stderr__):
		sys.stderr = ansiterm.AnsiTerm(sys.stderr)

# import the logging module after since it holds a reference on sys.stderr
# in case someone uses the root logger
import logging

LOG_FORMAT = os.environ.get('WAF_LOG_FORMAT', '%(asctime)s %(c1)s%(zone)s%(c2)s %(message)s')
HOUR_FORMAT = os.environ.get('WAF_HOUR_FORMAT', '%H:%M:%S')

zones = ''
verbose = 0

colors_lst = {
'USE' : True,
'BOLD'  :'\x1b[01;1m',
'RED'   :'\x1b[01;31m',
'GREEN' :'\x1b[32m',
'YELLOW':'\x1b[33m',
'PINK'  :'\x1b[35m',
'BLUE'  :'\x1b[01;34m',
'CYAN'  :'\x1b[36m',
'GREY'  :'\x1b[37m',
'NORMAL':'\x1b[0m',
'cursor_on'  :'\x1b[?25h',
'cursor_off' :'\x1b[?25l',
}

indicator = '\r\x1b[K%s%s%s'

try:
	unicode
except NameError:
	unicode = None

def enable_colors(use):
	if use == 1:
		if not (sys.stderr.isatty() or sys.stdout.isatty()):
			use = 0
		if Utils.is_win32 and os.name != 'java':
			term = os.environ.get('TERM', '') # has ansiterm
		else:
			term = os.environ.get('TERM', 'dumb')

		if term in ('dumb', 'emacs'):
			use = 0

	if use >= 1:
		os.environ['TERM'] = 'vt100'

	colors_lst['USE'] = use

# If console packages are available, replace the dummy function with a real
# implementation
try:
	get_term_cols = ansiterm.get_term_cols
except AttributeError:
	def get_term_cols():
		return 80

get_term_cols.__doc__ = """
	Get the console width in characters.

	:return: the number of characters per line
	:rtype: int
	"""

def get_color(cl):
	if not colors_lst['USE']: return ''
	return colors_lst.get(cl, '')

class color_dict(object):
	"""attribute-based color access, eg: colors.PINK"""
	def __getattr__(self, a):
		return get_color(a)
	def __call__(self, a):
		return get_color(a)

colors = color_dict()

re_log = re.compile(r'(\w+): (.*)', re.M)
class log_filter(logging.Filter):
	"""
	The waf logs are of the form 'name: message', and can be filtered by 'waf --zones=name'.
	For example, the following::

		from waflib import Logs
		Logs.debug('test: here is a message')

	Will be displayed only when executing::

		$ waf --zones=test
	"""
	def __init__(self, name=None):
		pass

	def filter(self, rec):
		"""
		filter a record, adding the colors automatically

		* error: red
		* warning: yellow

		:param rec: message to record
		"""
		rec.zone = rec.module
		if rec.levelno >= logging.INFO:
			return True

		m = re_log.match(rec.msg)
		if m:
			rec.zone = m.group(1)
			rec.msg = m.group(2)

		if zones:
			return getattr(rec, 'zone', '') in zones or '*' in zones
		elif not verbose > 2:
			return False
		return True

class log_handler(logging.StreamHandler):
	"""Dispatches messages to stderr/stdout depending on the severity level"""
	def emit(self, record):
		# default implementation
		try:
			try:
				self.stream = record.stream
			except AttributeError:
				if record.levelno >= logging.WARNING:
					record.stream = self.stream = sys.stderr
				else:
					record.stream = self.stream = sys.stdout
			self.emit_override(record)
			self.flush()
		except (KeyboardInterrupt, SystemExit):
			raise
		except: # from the python library -_-
			self.handleError(record)

	def emit_override(self, record, **kw):
		self.terminator = getattr(record, 'terminator', '\n')
		stream = self.stream
		if unicode:
			# python2
			msg = self.formatter.format(record)
			fs = '%s' + self.terminator
			try:
				if (isinstance(msg, unicode) and getattr(stream, 'encoding', None)):
					fs = fs.decode(stream.encoding)
					try:
						stream.write(fs % msg)
					except UnicodeEncodeError:
						stream.write((fs % msg).encode(stream.encoding))
				else:
					stream.write(fs % msg)
			except UnicodeError:
				stream.write((fs % msg).encode('utf-8'))
		else:
			logging.StreamHandler.emit(self, record)

class formatter(logging.Formatter):
	"""Simple log formatter which handles colors"""
	def __init__(self):
		logging.Formatter.__init__(self, LOG_FORMAT, HOUR_FORMAT)

	def format(self, rec):
		"""Messages in warning, error or info mode are displayed in color by default"""
		try:
			msg = rec.msg.decode('utf-8')
		except Exception:
			msg = rec.msg

		use = colors_lst['USE']
		if (use == 1 and rec.stream.isatty()) or use == 2:

			c1 = getattr(rec, 'c1', None)
			if c1 is None:
				c1 = ''
				if rec.levelno >= logging.ERROR:
					c1 = colors.RED
				elif rec.levelno >= logging.WARNING:
					c1 = colors.YELLOW
				elif rec.levelno >= logging.INFO:
					c1 = colors.GREEN
			c2 = getattr(rec, 'c2', colors.NORMAL)
			msg = '%s%s%s' % (c1, msg, c2)
		else:
			# remove single \r that make long lines in text files
			# and other terminal commands
			msg = re.sub(r'\r(?!\n)|\x1B\[(K|.*?(m|h|l))', '', msg)

		if rec.levelno >= logging.INFO: # ??
			return msg

		rec.msg = msg
		rec.c1 = colors.PINK
		rec.c2 = colors.NORMAL
		return logging.Formatter.format(self, rec)

log = None
"""global logger for Logs.debug, Logs.error, etc"""

def debug(*k, **kw):
	"""
	Wrap logging.debug, the output is filtered for performance reasons
	"""
	if verbose:
		k = list(k)
		k[0] = k[0].replace('\n', ' ')
		global log
		log.debug(*k, **kw)

def error(*k, **kw):
	"""
	Wrap logging.errors, display the origin of the message when '-vv' is set
	"""
	global log
	log.error(*k, **kw)
	if verbose > 2:
		st = traceback.extract_stack()
		if st:
			st = st[:-1]
			buf = []
			for filename, lineno, name, line in st:
				buf.append('  File "%s", line %d, in %s' % (filename, lineno, name))
				if line:
					buf.append('	%s' % line.strip())
			if buf: log.error("\n".join(buf))

def warn(*k, **kw):
	"""
	Wrap logging.warn
	"""
	global log
	log.warn(*k, **kw)

def info(*k, **kw):
	"""
	Wrap logging.info
	"""
	global log
	log.info(*k, **kw)

def init_log():
	"""
	Initialize the loggers globally
	"""
	global log
	log = logging.getLogger('waflib')
	log.handlers = []
	log.filters = []
	hdlr = log_handler()
	hdlr.setFormatter(formatter())
	log.addHandler(hdlr)
	log.addFilter(log_filter())
	log.setLevel(logging.DEBUG)

def make_logger(path, name):
	"""
	Create a simple logger, which is often used to redirect the context command output::

		from waflib import Logs
		bld.logger = Logs.make_logger('test.log', 'build')
		bld.check(header_name='sadlib.h', features='cxx cprogram', mandatory=False)

		# have the file closed immediately
		Logs.free_logger(bld.logger)

		# stop logging
		bld.logger = None

	The method finalize() of the command will try to free the logger, if any

	:param path: file name to write the log output to
	:type path: string
	:param name: logger name (loggers are reused)
	:type name: string
	"""
	logger = logging.getLogger(name)
	hdlr = logging.FileHandler(path, 'w')
	formatter = logging.Formatter('%(message)s')
	hdlr.setFormatter(formatter)
	logger.addHandler(hdlr)
	logger.setLevel(logging.DEBUG)
	return logger

def make_mem_logger(name, to_log, size=8192):
	"""
	Create a memory logger to avoid writing concurrently to the main logger
	"""
	from logging.handlers import MemoryHandler
	logger = logging.getLogger(name)
	hdlr = MemoryHandler(size, target=to_log)
	formatter = logging.Formatter('%(message)s')
	hdlr.setFormatter(formatter)
	logger.addHandler(hdlr)
	logger.memhandler = hdlr
	logger.setLevel(logging.DEBUG)
	return logger

def free_logger(logger):
	"""
	Free the resources held by the loggers created through make_logger or make_mem_logger.
	This is used for file cleanup and for handler removal (logger objects are re-used).
	"""
	try:
		for x in logger.handlers:
			x.close()
			logger.removeHandler(x)
	except Exception:
		pass

def pprint(col, msg, label='', sep='\n'):
	"""
	Print messages in color immediately on stderr::

		from waflib import Logs
		Logs.pprint('RED', 'Something bad just happened')

	:param col: color name to use in :py:const:`Logs.colors_lst`
	:type col: string
	:param msg: message to display
	:type msg: string or a value that can be printed by %s
	:param label: a message to add after the colored output
	:type label: string
	:param sep: a string to append at the end (line separator)
	:type sep: string
	"""
	info("%s%s%s %s" % (colors(col), msg, colors.NORMAL, label), extra={'terminator':sep})

<commit_msg>Subclasses should not provide incompatible constructors<commit_after>#!/usr/bin/env python
# encoding: utf-8
# Thomas Nagy, 2005-2010 (ita)

"""
logging, colors, terminal width and pretty-print
"""

import os, re, traceback, sys
from waflib import Utils, ansiterm

if not os.environ.get('NOSYNC', False):
	# synchronized output is nearly mandatory to prevent garbled output
	if sys.stdout.isatty() and id(sys.stdout) == id(sys.__stdout__):
		sys.stdout = ansiterm.AnsiTerm(sys.stdout)
	if sys.stderr.isatty() and id(sys.stderr) == id(sys.__stderr__):
		sys.stderr = ansiterm.AnsiTerm(sys.stderr)

# import the logging module after since it holds a reference on sys.stderr
# in case someone uses the root logger
import logging

LOG_FORMAT = os.environ.get('WAF_LOG_FORMAT', '%(asctime)s %(c1)s%(zone)s%(c2)s %(message)s')
HOUR_FORMAT = os.environ.get('WAF_HOUR_FORMAT', '%H:%M:%S')

zones = ''
verbose = 0

colors_lst = {
'USE' : True,
'BOLD'  :'\x1b[01;1m',
'RED'   :'\x1b[01;31m',
'GREEN' :'\x1b[32m',
'YELLOW':'\x1b[33m',
'PINK'  :'\x1b[35m',
'BLUE'  :'\x1b[01;34m',
'CYAN'  :'\x1b[36m',
'GREY'  :'\x1b[37m',
'NORMAL':'\x1b[0m',
'cursor_on'  :'\x1b[?25h',
'cursor_off' :'\x1b[?25l',
}

indicator = '\r\x1b[K%s%s%s'

try:
	unicode
except NameError:
	unicode = None

def enable_colors(use):
	if use == 1:
		if not (sys.stderr.isatty() or sys.stdout.isatty()):
			use = 0
		if Utils.is_win32 and os.name != 'java':
			term = os.environ.get('TERM', '') # has ansiterm
		else:
			term = os.environ.get('TERM', 'dumb')

		if term in ('dumb', 'emacs'):
			use = 0

	if use >= 1:
		os.environ['TERM'] = 'vt100'

	colors_lst['USE'] = use

# If console packages are available, replace the dummy function with a real
# implementation
try:
	get_term_cols = ansiterm.get_term_cols
except AttributeError:
	def get_term_cols():
		return 80

get_term_cols.__doc__ = """
	Get the console width in characters.

	:return: the number of characters per line
	:rtype: int
	"""

def get_color(cl):
	if not colors_lst['USE']: return ''
	return colors_lst.get(cl, '')

class color_dict(object):
	"""attribute-based color access, eg: colors.PINK"""
	def __getattr__(self, a):
		return get_color(a)
	def __call__(self, a):
		return get_color(a)

colors = color_dict()

re_log = re.compile(r'(\w+): (.*)', re.M)
class log_filter(logging.Filter):
	"""
	The waf logs are of the form 'name: message', and can be filtered by 'waf --zones=name'.
	For example, the following::

		from waflib import Logs
		Logs.debug('test: here is a message')

	Will be displayed only when executing::

		$ waf --zones=test
	"""
	def __init__(self, name=''):
		logging.Filter.__init__(self, name)

	def filter(self, rec):
		"""
		filter a record, adding the colors automatically

		* error: red
		* warning: yellow

		:param rec: message to record
		"""
		rec.zone = rec.module
		if rec.levelno >= logging.INFO:
			return True

		m = re_log.match(rec.msg)
		if m:
			rec.zone = m.group(1)
			rec.msg = m.group(2)

		if zones:
			return getattr(rec, 'zone', '') in zones or '*' in zones
		elif not verbose > 2:
			return False
		return True

class log_handler(logging.StreamHandler):
	"""Dispatches messages to stderr/stdout depending on the severity level"""
	def emit(self, record):
		# default implementation
		try:
			try:
				self.stream = record.stream
			except AttributeError:
				if record.levelno >= logging.WARNING:
					record.stream = self.stream = sys.stderr
				else:
					record.stream = self.stream = sys.stdout
			self.emit_override(record)
			self.flush()
		except (KeyboardInterrupt, SystemExit):
			raise
		except: # from the python library -_-
			self.handleError(record)

	def emit_override(self, record, **kw):
		self.terminator = getattr(record, 'terminator', '\n')
		stream = self.stream
		if unicode:
			# python2
			msg = self.formatter.format(record)
			fs = '%s' + self.terminator
			try:
				if (isinstance(msg, unicode) and getattr(stream, 'encoding', None)):
					fs = fs.decode(stream.encoding)
					try:
						stream.write(fs % msg)
					except UnicodeEncodeError:
						stream.write((fs % msg).encode(stream.encoding))
				else:
					stream.write(fs % msg)
			except UnicodeError:
				stream.write((fs % msg).encode('utf-8'))
		else:
			logging.StreamHandler.emit(self, record)

class formatter(logging.Formatter):
	"""Simple log formatter which handles colors"""
	def __init__(self):
		logging.Formatter.__init__(self, LOG_FORMAT, HOUR_FORMAT)

	def format(self, rec):
		"""Messages in warning, error or info mode are displayed in color by default"""
		try:
			msg = rec.msg.decode('utf-8')
		except Exception:
			msg = rec.msg

		use = colors_lst['USE']
		if (use == 1 and rec.stream.isatty()) or use == 2:

			c1 = getattr(rec, 'c1', None)
			if c1 is None:
				c1 = ''
				if rec.levelno >= logging.ERROR:
					c1 = colors.RED
				elif rec.levelno >= logging.WARNING:
					c1 = colors.YELLOW
				elif rec.levelno >= logging.INFO:
					c1 = colors.GREEN
			c2 = getattr(rec, 'c2', colors.NORMAL)
			msg = '%s%s%s' % (c1, msg, c2)
		else:
			# remove single \r that make long lines in text files
			# and other terminal commands
			msg = re.sub(r'\r(?!\n)|\x1B\[(K|.*?(m|h|l))', '', msg)

		if rec.levelno >= logging.INFO: # ??
			return msg

		rec.msg = msg
		rec.c1 = colors.PINK
		rec.c2 = colors.NORMAL
		return logging.Formatter.format(self, rec)

log = None
"""global logger for Logs.debug, Logs.error, etc"""

def debug(*k, **kw):
	"""
	Wrap logging.debug, the output is filtered for performance reasons
	"""
	if verbose:
		k = list(k)
		k[0] = k[0].replace('\n', ' ')
		global log
		log.debug(*k, **kw)

def error(*k, **kw):
	"""
	Wrap logging.errors, display the origin of the message when '-vv' is set
	"""
	global log
	log.error(*k, **kw)
	if verbose > 2:
		st = traceback.extract_stack()
		if st:
			st = st[:-1]
			buf = []
			for filename, lineno, name, line in st:
				buf.append('  File "%s", line %d, in %s' % (filename, lineno, name))
				if line:
					buf.append('	%s' % line.strip())
			if buf: log.error("\n".join(buf))

def warn(*k, **kw):
	"""
	Wrap logging.warn
	"""
	global log
	log.warn(*k, **kw)

def info(*k, **kw):
	"""
	Wrap logging.info
	"""
	global log
	log.info(*k, **kw)

def init_log():
	"""
	Initialize the loggers globally
	"""
	global log
	log = logging.getLogger('waflib')
	log.handlers = []
	log.filters = []
	hdlr = log_handler()
	hdlr.setFormatter(formatter())
	log.addHandler(hdlr)
	log.addFilter(log_filter())
	log.setLevel(logging.DEBUG)

def make_logger(path, name):
	"""
	Create a simple logger, which is often used to redirect the context command output::

		from waflib import Logs
		bld.logger = Logs.make_logger('test.log', 'build')
		bld.check(header_name='sadlib.h', features='cxx cprogram', mandatory=False)

		# have the file closed immediately
		Logs.free_logger(bld.logger)

		# stop logging
		bld.logger = None

	The method finalize() of the command will try to free the logger, if any

	:param path: file name to write the log output to
	:type path: string
	:param name: logger name (loggers are reused)
	:type name: string
	"""
	logger = logging.getLogger(name)
	hdlr = logging.FileHandler(path, 'w')
	formatter = logging.Formatter('%(message)s')
	hdlr.setFormatter(formatter)
	logger.addHandler(hdlr)
	logger.setLevel(logging.DEBUG)
	return logger

def make_mem_logger(name, to_log, size=8192):
	"""
	Create a memory logger to avoid writing concurrently to the main logger
	"""
	from logging.handlers import MemoryHandler
	logger = logging.getLogger(name)
	hdlr = MemoryHandler(size, target=to_log)
	formatter = logging.Formatter('%(message)s')
	hdlr.setFormatter(formatter)
	logger.addHandler(hdlr)
	logger.memhandler = hdlr
	logger.setLevel(logging.DEBUG)
	return logger

def free_logger(logger):
	"""
	Free the resources held by the loggers created through make_logger or make_mem_logger.
	This is used for file cleanup and for handler removal (logger objects are re-used).
	"""
	try:
		for x in logger.handlers:
			x.close()
			logger.removeHandler(x)
	except Exception:
		pass

def pprint(col, msg, label='', sep='\n'):
	"""
	Print messages in color immediately on stderr::

		from waflib import Logs
		Logs.pprint('RED', 'Something bad just happened')

	:param col: color name to use in :py:const:`Logs.colors_lst`
	:type col: string
	:param msg: message to display
	:type msg: string or a value that can be printed by %s
	:param label: a message to add after the colored output
	:type label: string
	:param sep: a string to append at the end (line separator)
	:type sep: string
	"""
	info("%s%s%s %s" % (colors(col), msg, colors.NORMAL, label), extra={'terminator':sep})

<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Utilities for handling subprocesses.

Mostly necessary only because of http://bugs.python.org/issue1652.

"""

import copy
import errno
import fcntl
import gevent.socket
import os
import signal
import subprocess
import sys

from subprocess import PIPE
from cStringIO import StringIO

class NonBlockPipeFileWrap(object):
    def __init__(self, fp):
        # Make the file nonblocking
        fcntl.fcntl(fp, fcntl.F_SETFL, os.O_NONBLOCK)
        self._fp = fp

    def read(self, size=None):
        # Some adaptation from gevent's examples/processes.py
        accum = StringIO()

        while size is None or accum.tell() < size:
            try:
                if size is None:
                    max_read = 4096
                else:
                    max_read = min(4096, size - accum.tell())

                chunk = self._fp.read(max_read)

                # End of the stream: leave the loop
                if not chunk:
                    break
                accum.write(chunk)
            except IOError, ex:
                if ex[0] != errno.EAGAIN:
                    raise
                sys.exc_clear()
            gevent.socket.wait_read(self._fp.fileno())

        return accum.getvalue()

    def write(self, data):
        # Some adaptation from gevent's examples/processes.py
        buf = StringIO(data)
        bytes_total = len(data)
        bytes_written = 0
        while bytes_written < bytes_total:
            try:
                # self._fp.write() doesn't return anything, so use
                # os.write.
                bytes_written += os.write(self._fp.fileno(), buf.read(4096))
            except IOError, ex:
                if ex[0] != errno.EAGAIN:
                    raise
                sys.exc_clear()
            gevent.socket.wait_write(self._fp.fileno())

    def fileno(self):
        return self._fp.fileno()

    def close(self):
        return self._fp.close()

    def flush(self):
        return self._fp.flush()

def subprocess_setup(f=None):
    """
    SIGPIPE reset for subprocess workaround

    Python installs a SIGPIPE handler by default. This is usually not
    what non-Python subprocesses expect.

    Calls an optional "f" first in case other code wants a preexec_fn,
    then restores SIGPIPE to what most Unix processes expect.

    http://bugs.python.org/issue1652
    http://www.chiark.greenend.org.uk/ucgi/~cjwatson/blosxom/2009-07-02-python-sigpipe.html

    """

    def wrapper(*args, **kwargs):
        if f is not None:
            f(*args, **kwargs)

        signal.signal(signal.SIGPIPE, signal.SIG_DFL)

    return wrapper

def popen_sp(*args, **kwargs):
    """
    Same as subprocess.Popen, but restores SIGPIPE

    This bug is documented (See subprocess_setup) but did not make it
    to standard library.  Could also be resolved by using the
    python-subprocess32 backport and using it appropriately (See
    'restore_signals' keyword argument to Popen)

    """

    kwargs['preexec_fn'] = subprocess_setup(kwargs.get('preexec_fn'))
    proc = subprocess.Popen(*args, **kwargs)

    # Patch up the process object to use non-blocking I/O that yields
    # to the gevent hub.
    for fp_symbol in ['stdin', 'stdout', 'stderr']:
        value = getattr(proc, fp_symbol)
        if value is not None:
            setattr(proc, fp_symbol, NonBlockPipeFileWrap(value))

    return proc

def pipe(*args):
    """
    Takes as parameters several dicts, each with the same
    parameters passed to popen.

    Runs the various processes in a pipeline, connecting
    the stdout of every process except the last with the
    stdin of the next process.

    Adapted from http://www.enricozini.org/2009/debian/python-pipes/

    """
    if len(args) < 2:
        raise ValueError, "pipe needs at least 2 processes"
    # Set stdout=PIPE in every subprocess except the last
    for i in args[:-1]:
        i["stdout"] = subprocess.PIPE

    # Runs all subprocesses connecting stdins and stdouts to create the
    # pipeline. Closes stdouts to avoid deadlocks.
    popens = [popen_sp(**args[0])]
    for i in range(1, len(args)):
        args[i]["stdin"] = popens[i - 1].stdout
        popens.append(popen_sp(**args[i]))
        popens[i - 1].stdout.close()

    # Returns the array of subprocesses just created
    return popens

def pipe_wait(popens):
    """
    Given an array of Popen objects returned by the
    pipe method, wait for all processes to terminate
    and return the array with their return values.

    Taken from http://www.enricozini.org/2009/debian/python-pipes/

    """
    # Avoid mutating the passed copy
    popens = copy.copy(popens)
    results = [0] * len(popens)
    while popens:
        last = popens.pop(-1)
        results[len(popens)] = last.wait()
    return results
<commit_msg>Add closed property to NonBlockPipeFileWrap<commit_after>#!/usr/bin/env python
"""
Utilities for handling subprocesses.

Mostly necessary only because of http://bugs.python.org/issue1652.

"""

import copy
import errno
import fcntl
import gevent.socket
import os
import signal
import subprocess
import sys

from subprocess import PIPE
from cStringIO import StringIO

class NonBlockPipeFileWrap(object):
    def __init__(self, fp):
        # Make the file nonblocking
        fcntl.fcntl(fp, fcntl.F_SETFL, os.O_NONBLOCK)
        self._fp = fp

    def read(self, size=None):
        # Some adaptation from gevent's examples/processes.py
        accum = StringIO()

        while size is None or accum.tell() < size:
            try:
                if size is None:
                    max_read = 4096
                else:
                    max_read = min(4096, size - accum.tell())

                chunk = self._fp.read(max_read)

                # End of the stream: leave the loop
                if not chunk:
                    break
                accum.write(chunk)
            except IOError, ex:
                if ex[0] != errno.EAGAIN:
                    raise
                sys.exc_clear()
            gevent.socket.wait_read(self._fp.fileno())

        return accum.getvalue()

    def write(self, data):
        # Some adaptation from gevent's examples/processes.py
        buf = StringIO(data)
        bytes_total = len(data)
        bytes_written = 0
        while bytes_written < bytes_total:
            try:
                # self._fp.write() doesn't return anything, so use
                # os.write.
                bytes_written += os.write(self._fp.fileno(), buf.read(4096))
            except IOError, ex:
                if ex[0] != errno.EAGAIN:
                    raise
                sys.exc_clear()
            gevent.socket.wait_write(self._fp.fileno())

    def fileno(self):
        return self._fp.fileno()

    def close(self):
        return self._fp.close()

    def flush(self):
        return self._fp.flush()

    @property
    def closed(self):
        return self._fp.closed

def subprocess_setup(f=None):
    """
    SIGPIPE reset for subprocess workaround

    Python installs a SIGPIPE handler by default. This is usually not
    what non-Python subprocesses expect.

    Calls an optional "f" first in case other code wants a preexec_fn,
    then restores SIGPIPE to what most Unix processes expect.

    http://bugs.python.org/issue1652
    http://www.chiark.greenend.org.uk/ucgi/~cjwatson/blosxom/2009-07-02-python-sigpipe.html

    """

    def wrapper(*args, **kwargs):
        if f is not None:
            f(*args, **kwargs)

        signal.signal(signal.SIGPIPE, signal.SIG_DFL)

    return wrapper

def popen_sp(*args, **kwargs):
    """
    Same as subprocess.Popen, but restores SIGPIPE

    This bug is documented (See subprocess_setup) but did not make it
    to standard library.  Could also be resolved by using the
    python-subprocess32 backport and using it appropriately (See
    'restore_signals' keyword argument to Popen)

    """

    kwargs['preexec_fn'] = subprocess_setup(kwargs.get('preexec_fn'))
    proc = subprocess.Popen(*args, **kwargs)

    # Patch up the process object to use non-blocking I/O that yields
    # to the gevent hub.
    for fp_symbol in ['stdin', 'stdout', 'stderr']:
        value = getattr(proc, fp_symbol)
        if value is not None:
            setattr(proc, fp_symbol, NonBlockPipeFileWrap(value))

    return proc

def pipe(*args):
    """
    Takes as parameters several dicts, each with the same
    parameters passed to popen.

    Runs the various processes in a pipeline, connecting
    the stdout of every process except the last with the
    stdin of the next process.

    Adapted from http://www.enricozini.org/2009/debian/python-pipes/

    """
    if len(args) < 2:
        raise ValueError, "pipe needs at least 2 processes"
    # Set stdout=PIPE in every subprocess except the last
    for i in args[:-1]:
        i["stdout"] = subprocess.PIPE

    # Runs all subprocesses connecting stdins and stdouts to create the
    # pipeline. Closes stdouts to avoid deadlocks.
    popens = [popen_sp(**args[0])]
    for i in range(1, len(args)):
        args[i]["stdin"] = popens[i - 1].stdout
        popens.append(popen_sp(**args[i]))
        popens[i - 1].stdout.close()

    # Returns the array of subprocesses just created
    return popens

def pipe_wait(popens):
    """
    Given an array of Popen objects returned by the
    pipe method, wait for all processes to terminate
    and return the array with their return values.

    Taken from http://www.enricozini.org/2009/debian/python-pipes/

    """
    # Avoid mutating the passed copy
    popens = copy.copy(popens)
    results = [0] * len(popens)
    while popens:
        last = popens.pop(-1)
        results[len(popens)] = last.wait()
    return results
<|endoftext|>


<commit_before>from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.ext.mutable import MutableDict
from sqlalchemy.exc import IntegrityError

from app import db

from models import product  # needed for sqla i think
from models.orcid import OrcidProfile
from models.product import make_product
from models.product import NoDoiException
from models.orcid import make_and_populate_orcid_profile

import jwt
import twitter
import os
import shortuuid
import requests
import json
import re
import datetime
import logging
import operator
import threading
from util import elapsed
from time import time
from collections import defaultdict

def make_person_from_google(person_dict):
    print "\n\nmaking new person with person_dict: ", person_dict, "\n\n"
    new_person = Person(
        email=person_dict["email"],
        given_name=person_dict["given_name"],
        family_name=person_dict["family_name"],
        picture=person_dict["picture"],
        oauth_source='google',
        oauth_api_raw=person_dict
    )

    db.session.add(new_person)
    db.session.commit()

    return new_person

def add_or_overwrite_person_from_orcid_id(orcid_id,
                                          campaign_email=None,
                                          campaign=None,
                                          high_priority=False):

    # if one already there, use it and overwrite.  else make a new one.
    my_profile = Person.query.filter_by(orcid_id=orcid_id).first()
    if my_profile:
        db.session.merge(my_profile)
        print u"using already made person for {}".format(orcid_id)
    else:
        # make a person with this orcid_id
        my_profile = Person(orcid_id=orcid_id)
        db.session.add(my_profile)
        print u"made new person for {}".format(orcid_id)

    # set the campaign name and email it came in with (if any)
    my_profile.campaign = campaign
    my_profile.campaign_email = campaign_email

    my_profile.refresh(high_priority=high_priority)

    # now write to the db
    db.session.commit()
    return my_profile

class Person(db.Model):
    id = db.Column(db.Text, primary_key=True)
    orcid_id = db.Column(db.Text, unique=True)
    email = db.Column(db.Text)
    given_name = db.Column(db.Text)
    family_name = db.Column(db.Text)
    picture = db.Column(db.Text)

    oauth_source = db.Column(db.Text)
    oauth_api_raw = db.Column(JSONB)

    given_names_orcid = db.Column(db.Text)
    family_name_orcid = db.Column(db.Text)
    api_raw = db.Column(db.Text)

    t_index = db.Column(db.Integer)
    num_products = db.Column(db.Integer)

    metric_sums = db.Column(MutableDict.as_mutable(JSONB))
    num_with_metrics = db.Column(MutableDict.as_mutable(JSONB))
    num_sources = db.Column(db.Integer)

    altmetric_score = db.Column(db.Float)
    monthly_event_count = db.Column(db.Float)

    created = db.Column(db.DateTime)
    updated = db.Column(db.DateTime)

    error = db.Column(db.Text)

    campaign = db.Column(db.Text)
    campaign_email = db.Column(db.Text)

    products = db.relationship(
        'Product',
        lazy='subquery',
        cascade="all, delete-orphan",
        backref=db.backref("person", lazy="subquery"),
        foreign_keys="Product.orcid_id"
    )

    def __init__(self, **kwargs):
        shortuuid.set_alphabet('abcdefghijklmnopqrstuvwxyz1234567890')
        self.id = shortuuid.uuid()[0:10]
        self.created = datetime.datetime.utcnow().isoformat()
        super(Person, self).__init__(**kwargs)

    def calculate_profile_summary_numbers(self):
        self.set_altmetric_score()
        # self.set_t_index()
        # self.set_metric_sums()
        # self.set_num_sources()
        # self.set_num_with_metrics()

    def make_badges(self):
        pass

    # doesn't throw errors; sets error column if error
    def refresh(self, high_priority=False):

        print u"refreshing {}".format(self.orcid_id)
        self.error = None

        # call orcid api.  includes error handling.
        try:
            self.set_attributes_and_works_from_orcid()
        except (KeyboardInterrupt, SystemExit):
            # let these ones through, don't save anything to db
            raise
        except requests.Timeout:
            self.error = "timeout orcid data error"
            print self.error
        except Exception:
            logging.exception("orcid data error")
            self.error = "orcid data error"

        # now call altmetric.com api. includes error handling and rate limiting.
        # blocks, so might sleep for a long time if waiting out API rate limiting
        # error handling done inside called function so it can be specific to the work
        self.set_data_from_altmetric(high_priority)

        self.calculate_profile_summary_numbers()
        self.make_badges()

        self.updated = datetime.datetime.utcnow().isoformat()

        print u"updated metrics for all {num} products for {orcid_id}".format(
            orcid_id=self.orcid_id,
            num=len(self.products))

        if self.error:
            print u"ERROR refreshing profile {id}: {msg}".format(
                id=self.id,
                msg=self.error)

    def add_product(self, product_to_add):
        if product_to_add.doi in [p.doi for p in self.products]:
            return False
        else:
            self.products.append(product_to_add)
            return True

    def set_attributes_and_works_from_orcid(self):
        # look up profile in orcid and set/overwrite our attributes
        orcid_data = make_and_populate_orcid_profile(self.orcid_id)

        self.given_names_orcid = orcid_data.given_names
        self.family_name_orcid = orcid_data.family_name
        self.api_raw = json.dumps(orcid_data.api_raw_profile)

        # now walk through all the orcid works and save the most recent ones in our db
        all_products = []
        for work in orcid_data.works:
            try:
                # add product if DOI not all ready there
                # dedup the DOIs here so we get 100 deduped ones below
                my_product = make_product(work)
                if my_product.doi not in [p.doi for p in all_products]:
                    all_products.append(my_product)
            except NoDoiException:
                # just ignore this work, it's not a product for our purposes.
                pass

        # set number of products to be the number of deduped DOIs, before taking most recent
        self.num_products = len(all_products)

        # sort all products by most recent year first
        all_products.sort(key=operator.attrgetter('year_int'), reverse=True)

        # then keep only most recent N DOIs
        for my_product in all_products[:100]:
            self.add_product(my_product)

    def set_data_from_altmetric(self, high_priority=False):
        threads = []

        # start a thread for each work
        # threads may block for a while sleeping if run out of API calls
        for work in self.products:
            process = threading.Thread(target=work.set_altmetric_api_raw, args=[high_priority])
            process.start()
            threads.append(process)

        # wait till all work is done
        for process in threads:
            process.join()

        # now go see if any of them had errors
        # need to do it this way because can't catch thread failures; have to check
        # object afterwards instead to see if they logged failures
        for work in self.products:
            if work.error:
                # don't print out doi here because that could cause another bug
                print u"setting person error; {} for product {}".format(work.error, work.id)
                self.error = work.error

    def set_t_index(self):
        my_products = self.products

        tweet_counts = []
        for p in my_products:
            try:
                int(tweet_counts.append(p.altmetric_counts["tweeters"]))
            except (KeyError, TypeError):
                tweet_counts.append(0)

        self.t_index = h_index(tweet_counts)

        print u"t-index={t_index} based on {tweeted_count} tweeted products ({total} total)".format(
            t_index=self.t_index,
            tweeted_count=len([x for x in tweet_counts if x]),
            total=len(my_products)
        )

    def set_monthly_event_count(self):
        self.monthly_event_count = 0
        counter = defaultdict(int)

        for product in self.products:
            if product.event_dates:
                for event_date in product.event_dates:
                    for month_string in ["2015-10", "2015-11", "2015-12"]:
                        if event_date.startswith(month_string):
                            counter[month_string] += 1

        try:
            self.monthly_event_count = min(counter.values())
        except ValueError:
            pass # no events

        print "setting events in last 3 months as {}".format(self.monthly_event_count)

    def set_altmetric_score(self):
        self.altmetric_score = 0
        for p in self.products:
            if p.altmetric_score:
                self.altmetric_score += p.altmetric_score
        print u"total altmetric score: {}".format(self.altmetric_score)

    def set_metric_sums(self):
        if self.metric_sums is None:
            self.metric_sums = {}

        for p in self.products:
            for metric, count in p.altmetric_counts.iteritems():
                try:
                    self.metric_sums[metric] += int(count)
                except KeyError:
                    self.metric_sums[metric] = int(count)

        print "setting metric_sums", self.metric_sums

    def set_num_sources(self):
        if self.metric_sums is None:
            self.metric_sums = {}

        self.num_sources = len(self.metric_sums.keys())

    def set_num_with_metrics(self):
        if self.num_with_metrics is None:
            self.num_with_metrics = {}

        for p in self.products:
            for metric, count in p.altmetric_counts.iteritems():
                try:
                    self.num_with_metrics[metric] += 1
                except KeyError:
                    self.num_with_metrics[metric] = 1

        print "setting num_with_metrics", self.num_with_metrics

    def get_token(self):
        payload = {
            'sub': self.email,
            'iat': datetime.datetime.utcnow(),
            'exp': datetime.datetime.utcnow() + datetime.timedelta(days=999),
            'picture': self.picture
        }
        token = jwt.encode(payload, os.getenv("JWT_KEY"))
        return token.decode('unicode_escape')

    def __repr__(self):
        return u'<Person ({id}, {orcid_id}) "{given_names} {family_name}" >'.format(
            id=self.id,
            orcid_id=self.orcid_id,
            given_names=self.given_names_orcid,
            family_name=self.family_name_orcid
        )

    def to_dict_orcid(self):
        return {
            "id": self.id,
            "orcid_id": self.orcid_id,
            "given_names": self.given_names_orcid,
            "family_name": self.family_name_orcid,
            "metric_sums": self.metric_sums,
            "monthly_event_count": self.monthly_event_count,
            "products": [p.to_dict() for p in self.products]
        }

    def to_dict(self):
        return {
            "email": self.email,
            "given_names": self.given_name,
            "family_name": self.family_name,
            "picture": self.picture,
            "orcid_id": self.orcid_id
        }

def h_index(citations):
    # from http://www.rainatian.com/2015/09/05/leetcode-python-h-index/

    citations.sort(reverse=True)

    i=0
    while (i<len(citations) and i+1 <= citations[i]):
        i += 1

    return i
<commit_msg>add spaces<commit_after>from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.ext.mutable import MutableDict
from sqlalchemy.exc import IntegrityError

from app import db

from models import product  # needed for sqla i think
from models.orcid import OrcidProfile
from models.product import make_product
from models.product import NoDoiException
from models.orcid import make_and_populate_orcid_profile

import jwt
import twitter
import os
import shortuuid
import requests
import json
import re
import datetime
import logging
import operator
import threading
from util import elapsed
from time import time
from collections import defaultdict

def make_person_from_google(person_dict):
    print "\n\nmaking new person with person_dict: ", person_dict, "\n\n"
    new_person = Person(
        email=person_dict["email"],
        given_name=person_dict["given_name"],
        family_name=person_dict["family_name"],
        picture=person_dict["picture"],
        oauth_source='google',
        oauth_api_raw=person_dict
    )

    db.session.add(new_person)
    db.session.commit()

    return new_person

def add_or_overwrite_person_from_orcid_id(orcid_id,
                                          campaign_email=None,
                                          campaign=None,
                                          high_priority=False):

    # if one already there, use it and overwrite.  else make a new one.
    my_profile = Person.query.filter_by(orcid_id=orcid_id).first()
    if my_profile:
        db.session.merge(my_profile)
        print u"\nusing already made person for {}".format(orcid_id)
    else:
        # make a person with this orcid_id
        my_profile = Person(orcid_id=orcid_id)
        db.session.add(my_profile)
        print u"\nmade new person for {}".format(orcid_id)

    # set the campaign name and email it came in with (if any)
    my_profile.campaign = campaign
    my_profile.campaign_email = campaign_email

    my_profile.refresh(high_priority=high_priority)

    # now write to the db
    db.session.commit()
    return my_profile

class Person(db.Model):
    id = db.Column(db.Text, primary_key=True)
    orcid_id = db.Column(db.Text, unique=True)
    email = db.Column(db.Text)
    given_name = db.Column(db.Text)
    family_name = db.Column(db.Text)
    picture = db.Column(db.Text)

    oauth_source = db.Column(db.Text)
    oauth_api_raw = db.Column(JSONB)

    given_names_orcid = db.Column(db.Text)
    family_name_orcid = db.Column(db.Text)
    api_raw = db.Column(db.Text)

    t_index = db.Column(db.Integer)
    num_products = db.Column(db.Integer)

    metric_sums = db.Column(MutableDict.as_mutable(JSONB))
    num_with_metrics = db.Column(MutableDict.as_mutable(JSONB))
    num_sources = db.Column(db.Integer)

    altmetric_score = db.Column(db.Float)
    monthly_event_count = db.Column(db.Float)

    created = db.Column(db.DateTime)
    updated = db.Column(db.DateTime)

    error = db.Column(db.Text)

    campaign = db.Column(db.Text)
    campaign_email = db.Column(db.Text)

    products = db.relationship(
        'Product',
        lazy='subquery',
        cascade="all, delete-orphan",
        backref=db.backref("person", lazy="subquery"),
        foreign_keys="Product.orcid_id"
    )

    def __init__(self, **kwargs):
        shortuuid.set_alphabet('abcdefghijklmnopqrstuvwxyz1234567890')
        self.id = shortuuid.uuid()[0:10]
        self.created = datetime.datetime.utcnow().isoformat()
        super(Person, self).__init__(**kwargs)

    def calculate_profile_summary_numbers(self):
        self.set_altmetric_score()
        # self.set_t_index()
        # self.set_metric_sums()
        # self.set_num_sources()
        # self.set_num_with_metrics()

    def make_badges(self):
        pass

    # doesn't throw errors; sets error column if error
    def refresh(self, high_priority=False):

        print u"refreshing {}".format(self.orcid_id)
        self.error = None

        # call orcid api.  includes error handling.
        try:
            self.set_attributes_and_works_from_orcid()
        except (KeyboardInterrupt, SystemExit):
            # let these ones through, don't save anything to db
            raise
        except requests.Timeout:
            self.error = "timeout orcid data error"
            print self.error
        except Exception:
            logging.exception("orcid data error")
            self.error = "orcid data error"

        # now call altmetric.com api. includes error handling and rate limiting.
        # blocks, so might sleep for a long time if waiting out API rate limiting
        # error handling done inside called function so it can be specific to the work
        self.set_data_from_altmetric(high_priority)

        self.calculate_profile_summary_numbers()
        self.make_badges()

        self.updated = datetime.datetime.utcnow().isoformat()

        print u"updated metrics for all {num} products for {orcid_id}".format(
            orcid_id=self.orcid_id,
            num=len(self.products))

        if self.error:
            print u"ERROR refreshing profile {id}: {msg}".format(
                id=self.id,
                msg=self.error)

    def add_product(self, product_to_add):
        if product_to_add.doi in [p.doi for p in self.products]:
            return False
        else:
            self.products.append(product_to_add)
            return True

    def set_attributes_and_works_from_orcid(self):
        # look up profile in orcid and set/overwrite our attributes
        orcid_data = make_and_populate_orcid_profile(self.orcid_id)

        self.given_names_orcid = orcid_data.given_names
        self.family_name_orcid = orcid_data.family_name
        self.api_raw = json.dumps(orcid_data.api_raw_profile)

        # now walk through all the orcid works and save the most recent ones in our db
        all_products = []
        for work in orcid_data.works:
            try:
                # add product if DOI not all ready there
                # dedup the DOIs here so we get 100 deduped ones below
                my_product = make_product(work)
                if my_product.doi not in [p.doi for p in all_products]:
                    all_products.append(my_product)
            except NoDoiException:
                # just ignore this work, it's not a product for our purposes.
                pass

        # set number of products to be the number of deduped DOIs, before taking most recent
        self.num_products = len(all_products)

        # sort all products by most recent year first
        all_products.sort(key=operator.attrgetter('year_int'), reverse=True)

        # then keep only most recent N DOIs
        for my_product in all_products[:100]:
            self.add_product(my_product)

    def set_data_from_altmetric(self, high_priority=False):
        threads = []

        # start a thread for each work
        # threads may block for a while sleeping if run out of API calls
        for work in self.products:
            process = threading.Thread(target=work.set_altmetric_api_raw, args=[high_priority])
            process.start()
            threads.append(process)

        # wait till all work is done
        for process in threads:
            process.join()

        # now go see if any of them had errors
        # need to do it this way because can't catch thread failures; have to check
        # object afterwards instead to see if they logged failures
        for work in self.products:
            if work.error:
                # don't print out doi here because that could cause another bug
                print u"setting person error; {} for product {}".format(work.error, work.id)
                self.error = work.error

    def set_t_index(self):
        my_products = self.products

        tweet_counts = []
        for p in my_products:
            try:
                int(tweet_counts.append(p.altmetric_counts["tweeters"]))
            except (KeyError, TypeError):
                tweet_counts.append(0)

        self.t_index = h_index(tweet_counts)

        print u"t-index={t_index} based on {tweeted_count} tweeted products ({total} total)".format(
            t_index=self.t_index,
            tweeted_count=len([x for x in tweet_counts if x]),
            total=len(my_products)
        )

    def set_monthly_event_count(self):
        self.monthly_event_count = 0
        counter = defaultdict(int)

        for product in self.products:
            if product.event_dates:
                for event_date in product.event_dates:
                    for month_string in ["2015-10", "2015-11", "2015-12"]:
                        if event_date.startswith(month_string):
                            counter[month_string] += 1

        try:
            self.monthly_event_count = min(counter.values())
        except ValueError:
            pass # no events

        print "setting events in last 3 months as {}".format(self.monthly_event_count)

    def set_altmetric_score(self):
        self.altmetric_score = 0
        for p in self.products:
            if p.altmetric_score:
                self.altmetric_score += p.altmetric_score
        print u"total altmetric score: {}".format(self.altmetric_score)

    def set_metric_sums(self):
        if self.metric_sums is None:
            self.metric_sums = {}

        for p in self.products:
            for metric, count in p.altmetric_counts.iteritems():
                try:
                    self.metric_sums[metric] += int(count)
                except KeyError:
                    self.metric_sums[metric] = int(count)

        print "setting metric_sums", self.metric_sums

    def set_num_sources(self):
        if self.metric_sums is None:
            self.metric_sums = {}

        self.num_sources = len(self.metric_sums.keys())

    def set_num_with_metrics(self):
        if self.num_with_metrics is None:
            self.num_with_metrics = {}

        for p in self.products:
            for metric, count in p.altmetric_counts.iteritems():
                try:
                    self.num_with_metrics[metric] += 1
                except KeyError:
                    self.num_with_metrics[metric] = 1

        print "setting num_with_metrics", self.num_with_metrics

    def get_token(self):
        payload = {
            'sub': self.email,
            'iat': datetime.datetime.utcnow(),
            'exp': datetime.datetime.utcnow() + datetime.timedelta(days=999),
            'picture': self.picture
        }
        token = jwt.encode(payload, os.getenv("JWT_KEY"))
        return token.decode('unicode_escape')

    def __repr__(self):
        return u'<Person ({id}, {orcid_id}) "{given_names} {family_name}" >'.format(
            id=self.id,
            orcid_id=self.orcid_id,
            given_names=self.given_names_orcid,
            family_name=self.family_name_orcid
        )

    def to_dict_orcid(self):
        return {
            "id": self.id,
            "orcid_id": self.orcid_id,
            "given_names": self.given_names_orcid,
            "family_name": self.family_name_orcid,
            "metric_sums": self.metric_sums,
            "monthly_event_count": self.monthly_event_count,
            "products": [p.to_dict() for p in self.products]
        }

    def to_dict(self):
        return {
            "email": self.email,
            "given_names": self.given_name,
            "family_name": self.family_name,
            "picture": self.picture,
            "orcid_id": self.orcid_id
        }

def h_index(citations):
    # from http://www.rainatian.com/2015/09/05/leetcode-python-h-index/

    citations.sort(reverse=True)

    i=0
    while (i<len(citations) and i+1 <= citations[i]):
        i += 1

    return i
<|endoftext|>


<commit_before>import sys, time, os, wget, stat, shutil, subprocess
from functools import partial as p

neededfiles = {'adafruit-pitft-touch-cal': 'https://raw.githubusercontent.com/adafruit/Adafruit-PiTFT-Helper/master/',
			   'adafruit-pitft.sh': 'https://raw.githubusercontent.com/adafruit/Raspberry-Pi-Installer-Scripts/master/'}

gitselector = {'stable': 'currentrelease', 'personal': 'homerelease', 'beta': 'currentbeta'}
gitprefix = 'https://raw.githubusercontent.com/kevinkahn/softconsole/'
installscripts = {'vncserverpi.service': 'scripts/', 'lxterminal.conf': 'scripts/', 'githubutil.py': ''}

def GetScripts(vers, save=''):
	if save != '':
		for s in installscripts:
			os.rename(s, '.consoleinstallleftovers/' + s + '.' + save)
	for nm, floc in installscripts.items():
		wget.download(gitprefix + gitselector[vers] + '/' + floc + nm, nm, bar=None)
	shutil.chown('lxterminal.conf', user='pi', group='pi')

def GetYN(prompt, Allownum=False):
	while True:
		answer = input(prompt + ' ')
		if answer in ('Y', 'y'):
			return True
		elif answer in ('N', 'n'):
			return False
		elif Allownum:
			return answer
		else:
			print('Answer Y or N')

def GetVal(prompt, allowed=None):
	while True:
		answer = input(prompt + ' ')
		if allowed is None:
			return answer
		else:
			if answer in allowed:
				return answer
			else:
				print('Choices are {}'.format(allowed))

# noinspection PyBroadException
def GetInt(prompt, allowed=None):
	while True:
		try:
			answer = int(input(prompt + ' '))
			if allowed is None:
				return answer
			else:
				if answer in allowed:
					return answer
				else:
					print('Choices are {}'.format(allowed))
		except:
			print('Bad input - choices are: {}'.format(allowed))

scriptvars = []

def AddToScript(varset, value):
	global scriptvars
	if isinstance(value, bool):
		setting = ('N', 'Y')[value]
	elif isinstance(value, str):
		setting = value
	else:
		setting = str(value)
	scriptvars.append(varset + '=' + setting + '\n')

def adafruit(scr, rotation):
	fin = open('/home/pi/adafruit-pitft.sh')
	displays = []
	while not displays:
		line = fin.readline()
		if line.startswith('PITFT_TYPES='):
			displays = line[line.index('(') + 1: line.index(')')].replace('"', '').split(' ')
		else:
			pass
	fin.close()

	# noinspection PyBroadException
	try:
		dnum = displays.index(scr) + 1
	except:
		print('Screen name not found in Adafruit scripts - report to developer!')
		sys.exit(1)
	fout = open('adafinput', 'w')
	fout.write(str(dnum) + '\n')
	fout.write(str(rotation) + '\n')
	fout.write('Y\n')  # pi console to pitft
	fout.write('N\n')  # don't reboot
	fout.close()
	return ["echo Adafruit {} screen\n".format(scr),
			"./adafruit-pitft.sh < adafinput\n",
			"raspi-config nonint do_boot_behaviour B4 # set boot to desktop already logged in\n"]

# noinspection PyUnusedLocal
def doflip(scr):
	global baseorientation
	print("If you are not using a Pi4 you can use hardware to flip the base orientation")
	print("of the display so that the power connector is on the top.")
	print("If you are on a Pi4 use the soft rotation option that will get asked for next")
	print("Also use the soft rotation option for other orientations.")
	flip = GetYN("Flip 7 inch screen so power at top using hardware option? (Y/N)")
	if flip:
		baseorientation['pi7'] = 'power at top'
		return ['echo Flip 7 inch screen\n', 'echo "lcd_rotate=2" >> /boot/config.txt\n']
	else:
		return ['echo Nornmal 7 inch screen\n', ]

def noscreen(scr):
	return ['echo User setup screen with type: {}\n'.format(scr), ]

# Start of script

shutil.rmtree('.consoleinstallleftovers', ignore_errors=True)
os.mkdir('.consoleinstallleftovers')

for n, loc in neededfiles.items():
	# noinspection PyBroadException
	try:
		os.remove(n)
	except Exception:
		pass
	wget.download(loc + n, n, bar=None)
	os.chmod(n, stat.S_IXUSR)

print("**************************************************************", flush=True)
print("**************************************************************", flush=True)
print(" Will now collect all necessary configuration information", flush=True)
print(" After this the install can run unattended to completion", flush=True)
print("**************************************************************", flush=True)
print("**************************************************************", flush=True)
time.sleep(3)
with open('/etc/issue') as f:
	sysver = f.readline()
	if "Linux 10" in sysver:
		Buster = True
AddToScript('Buster', 'Y' if Buster else 'N')
AddToScript('NodeName', GetVal("What name for this system?"))
AddToScript('VNCstdPort', GetYN("Install VNC on standard port (Y/N/alt port number)?", Allownum=True))
personal = GetYN("Is this the developer personal system (Y/N) (bit risky to say Y if it not)?")
if personal:
	with open('homesystem', 'w') as f:
		f.write('homesystem\n')
AddToScript('Personal', personal)
beta = GetYN("Download current beta as well as stable? (usually waste of time)")
AddToScript('InstallBeta', beta)
AddToScript('AutoConsole', GetYN("Autostart console (Y/N)?"))
AddToScript('Reboot', GetYN("Automatically reboot to clean system after install?"))

GetScripts('personal' if personal else 'stable')
if beta: GetScripts('beta', save=('personal' if personal else 'stable'))

screentype = '--'
supportedscreens = ('28r', '28c', '35r', 'pi7')
# adafruit script rotations {'28r': 4, '28c': 2, '35r': 4}
screeninstallcode = {'28r': p(adafruit, '28r', 4), '28c': p(adafruit, '28c', 2), '35r': p(adafruit, '35r', 4),
					 'pi7': p(doflip, 'pi7'), '--': p(noscreen, '--')}
baseorientation = {'28c': 'power on left',
				   '35r': 'power on left',
				   'pi7': 'power at bottom'}

doscreen = GetYN("Do you want to install a known screen (Alternative is to install any screen drivers yourself)?")
if doscreen:
	screentype = GetVal("What type screen ({})?".format(supportedscreens), supportedscreens)
	installsrc = screeninstallcode[screentype]()
else:
	screentype = GetVal("Enter name of screen for console reference:")
	installscr = screeninstallcode['noscreen']()

rot = 0
if screentype in baseorientation:
	print(" You can choose to rotate the display from its base orientation")
	print(" Base orientation for {} screen is with {}".format(screentype, baseorientation[screentype]))
	print(" Rotation options(power connection for reference:")
	print("     0: use base orientation")
	print("     1: 90 degrees counterclockwise")
	print("     2: 180 degrees counterclockwise (vertical flip)")
	print("     3: 270 degrees counterclockwise")
	rot = GetInt('Rotation option:', (0, 1, 2, 3, 4))
	if rot != 0: installsrc.append('echo Soft rotation code {}\n'.format(rot))

screentp = screentype + 'B' if Buster else screentype

with open('.Screentype', 'w') as f:
	if rot == 0:
		f.write(screentp + '\n')
	else:
		f.write(screentp + ',' + str(rot) + '\n')

with open('installvals', 'w') as f:
	f.writelines(scriptvars)
with open('installscreencode', 'w') as f:
	f.writelines(installsrc)

ISYname = ""
ans = ""
ISYIP = ""
ISYUSER = ""
ISYPWD = ""
exswitch = ""
MinExamp = GetYN("Set up minimal example system?")

if MinExamp:
	go = False
	while not go:
		ISYname = input("Name to use for the ISY hub (defaults to ISY): ")
		if ISYname == "":
			ISYname = "ISY"
		ISYIP = input("full URL to access ISY: ")
		if ISYIP.endswith('/'):
			ISYIP = ISYIP[0:-1]
		ISYUSER = input("ISY user name: ")
		ISYPWD = input("ISY password: ")
		exswitch = input("Example switch to use (ISY name): ")
		print("ISY Name: " + ISYname)
		print("IP:       " + ISYIP)
		print("USER:     " + ISYUSER)
		print("PASSWORD: " + ISYPWD)
		print("SWITCH:   " + "[[" + exswitch + "]]")
		go = GetYN("OK? (y/n)")

print("Set up directory environment for console")

with open('versionselector', 'w') as f:
	f.write('stable\n')

dirs = ['Console', 'consolestable', 'consolebeta', 'consolerem', 'consoledev', 'Console', 'Console/cfglib']
if personal: dirs.append('consolecur')
for pdir in dirs:
	# noinspection PyBroadException
	try:
		os.mkdir(pdir)
		print("Created: " + str(pdir))
	except:
		print("Already present: " + str(pdir))
	shutil.chown(pdir, user='pi', group='pi')

if MinExamp:
	with open('/home/pi/Console/cfglib/auth.cfg', "w") as f:
		cfg = ("[" + ISYname + "]",
			   "type = ISY",
			   "address = " + ISYIP,
			   "user = " + ISYUSER,
			   "password = " + ISYPWD,
			   "\n")
		f.write("\n".join(cfg))
	with open('/home/pi/Console/config.txt', 'w') as f:
		cfg = ('cfglib = cfglib',
			   'include = auth.cfg, myclock.cfg',
			   'DefaultHub = ' + ISYname,
			   'HomeScreenName = test',
			   'PersistTO = 30',
			   'DimLevel = 5',
			   'DimTO = 15',
			   'DimIdleListNames = MyClock,',
			   'DimIdleListTimes = 20,',
			   'MainChain = test, MyClock',
			   '[test]',
			   'type = Keypad',
			   'label = My, Test',
			   '[[' + exswitch + ']]',
			   "\n")
		f.write("\n".join(cfg))

print("\n\nSoftconsole install paramters:")
for l in scriptvars:
	print('    ' + l.replace('\n', ''))
if MinExamp:
	print("    Create minimal example configuration")
else:
	print("    Skip minimal example configuration")
print('---------------------', flush=True)

import githubutil as U

print('Download console code', flush=True)

if personal:
	# personal system
	U.StageVersion('consolestable', 'homerelease', 'Initial Install')
	print("Stage homerelease as stable")
else:
	U.StageVersion('consolestable', 'currentrelease', 'Initial Install')
	print("Stage standard stable release")

U.InstallStagedVersion('consolestable')
print('Installed stable version', flush=True)

if beta:
	U.StageVersion('consolebeta', 'currentbeta', 'Initial Install')
	print('Stage beta also')
	U.InstallStagedVersion('consolebeta')
	print('Intalled staged beta')

if os.path.exists('/boot/auth'):
	shutil.rmtree('Console/local', ignore_errors=True)
	shutil.move('/boot/auth', 'Console/local')

subprocess.call("cp -r /home/pi/consolestable/'example configs'/* /home/pi/Console", shell=True)

print("****************************************************************", flush=True)
print("****************************************************************", flush=True)
print(" System will now update/upgrade Raspbian", flush=True)
print(" THIS COULD TAKE 10-15 MINUTES DEPENDING ON OUTSTANDING UPDATES", flush=True)
print(" AND THE SPEED OF YOUR SD CARD!", flush=True)
print(" After that it will install the softconsole using your input", flush=True)
print("****************************************************************", flush=True)
print("****************************************************************", flush=True)

if screentype == '28c':
	print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
	print(" NOTE!!! NOTE!!! NOTE!!!")
	print(" If you are using the 28c screen, the settings from Adafruit that")
	print(" sets up are likely wrong.  Look at /boot/config.txt")
	print(" Next to last line should be: dtoverlay=pitft28-capacitive,rotate=180")
	print(" It may well show as rotate=90 after install due to bugs in their scripts")
	print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
	time.sleep(10)
time.sleep(3)
<commit_msg>Bug in getting params for no screen install<commit_after>import sys, time, os, wget, stat, shutil, subprocess
from functools import partial as p

neededfiles = {'adafruit-pitft-touch-cal': 'https://raw.githubusercontent.com/adafruit/Adafruit-PiTFT-Helper/master/',
			   'adafruit-pitft.sh': 'https://raw.githubusercontent.com/adafruit/Raspberry-Pi-Installer-Scripts/master/'}

gitselector = {'stable': 'currentrelease', 'personal': 'homerelease', 'beta': 'currentbeta'}
gitprefix = 'https://raw.githubusercontent.com/kevinkahn/softconsole/'
installscripts = {'vncserverpi.service': 'scripts/', 'lxterminal.conf': 'scripts/', 'githubutil.py': ''}

def GetScripts(vers, save=''):
	if save != '':
		for s in installscripts:
			os.rename(s, '.consoleinstallleftovers/' + s + '.' + save)
	for nm, floc in installscripts.items():
		wget.download(gitprefix + gitselector[vers] + '/' + floc + nm, nm, bar=None)
	shutil.chown('lxterminal.conf', user='pi', group='pi')

def GetYN(prompt, Allownum=False):
	while True:
		answer = input(prompt + ' ')
		if answer in ('Y', 'y'):
			return True
		elif answer in ('N', 'n'):
			return False
		elif Allownum:
			return answer
		else:
			print('Answer Y or N')

def GetVal(prompt, allowed=None):
	while True:
		answer = input(prompt + ' ')
		if allowed is None:
			return answer
		else:
			if answer in allowed:
				return answer
			else:
				print('Choices are {}'.format(allowed))

# noinspection PyBroadException
def GetInt(prompt, allowed=None):
	while True:
		try:
			answer = int(input(prompt + ' '))
			if allowed is None:
				return answer
			else:
				if answer in allowed:
					return answer
				else:
					print('Choices are {}'.format(allowed))
		except:
			print('Bad input - choices are: {}'.format(allowed))

scriptvars = []

def AddToScript(varset, value):
	global scriptvars
	if isinstance(value, bool):
		setting = ('N', 'Y')[value]
	elif isinstance(value, str):
		setting = value
	else:
		setting = str(value)
	scriptvars.append(varset + '=' + setting + '\n')

def adafruit(scr, rotation):
	fin = open('/home/pi/adafruit-pitft.sh')
	displays = []
	while not displays:
		line = fin.readline()
		if line.startswith('PITFT_TYPES='):
			displays = line[line.index('(') + 1: line.index(')')].replace('"', '').split(' ')
		else:
			pass
	fin.close()

	# noinspection PyBroadException
	try:
		dnum = displays.index(scr) + 1
	except:
		print('Screen name not found in Adafruit scripts - report to developer!')
		sys.exit(1)
	fout = open('adafinput', 'w')
	fout.write(str(dnum) + '\n')
	fout.write(str(rotation) + '\n')
	fout.write('Y\n')  # pi console to pitft
	fout.write('N\n')  # don't reboot
	fout.close()
	return ["echo Adafruit {} screen\n".format(scr),
			"./adafruit-pitft.sh < adafinput\n",
			"raspi-config nonint do_boot_behaviour B4 # set boot to desktop already logged in\n"]

# noinspection PyUnusedLocal
def doflip(scr):
	global baseorientation
	print("If you are not using a Pi4 you can use hardware to flip the base orientation")
	print("of the display so that the power connector is on the top.")
	print("If you are on a Pi4 use the soft rotation option that will get asked for next")
	print("Also use the soft rotation option for other orientations.")
	flip = GetYN("Flip 7 inch screen so power at top using hardware option? (Y/N)")
	if flip:
		baseorientation['pi7'] = 'power at top'
		return ['echo Flip 7 inch screen\n', 'echo "lcd_rotate=2" >> /boot/config.txt\n']
	else:
		return ['echo Nornmal 7 inch screen\n', ]

def noscreen(scr):
	return ['echo User setup screen with type: {}\n'.format(scr), ]

# Start of script

shutil.rmtree('.consoleinstallleftovers', ignore_errors=True)
os.mkdir('.consoleinstallleftovers')

for n, loc in neededfiles.items():
	# noinspection PyBroadException
	try:
		os.remove(n)
	except Exception:
		pass
	wget.download(loc + n, n, bar=None)
	os.chmod(n, stat.S_IXUSR)

print("**************************************************************", flush=True)
print("**************************************************************", flush=True)
print(" Will now collect all necessary configuration information", flush=True)
print(" After this the install can run unattended to completion", flush=True)
print("**************************************************************", flush=True)
print("**************************************************************", flush=True)
time.sleep(3)
with open('/etc/issue') as f:
	sysver = f.readline()
	if "Linux 10" in sysver:
		Buster = True
AddToScript('Buster', 'Y' if Buster else 'N')
AddToScript('NodeName', GetVal("What name for this system?"))
AddToScript('VNCstdPort', GetYN("Install VNC on standard port (Y/N/alt port number)?", Allownum=True))
personal = GetYN("Is this the developer personal system (Y/N) (bit risky to say Y if it not)?")
if personal:
	with open('homesystem', 'w') as f:
		f.write('homesystem\n')
AddToScript('Personal', personal)
beta = GetYN("Download current beta as well as stable? (usually waste of time)")
AddToScript('InstallBeta', beta)
AddToScript('AutoConsole', GetYN("Autostart console (Y/N)?"))
AddToScript('Reboot', GetYN("Automatically reboot to clean system after install?"))

GetScripts('personal' if personal else 'stable')
if beta: GetScripts('beta', save=('personal' if personal else 'stable'))

screentype = '--'
supportedscreens = ('28r', '28c', '35r', 'pi7')
# adafruit script rotations {'28r': 4, '28c': 2, '35r': 4}
screeninstallcode = {'28r': p(adafruit, '28r', 4), '28c': p(adafruit, '28c', 2), '35r': p(adafruit, '35r', 4),
					 'pi7': p(doflip, 'pi7'), '--': p(noscreen, '--')}
baseorientation = {'28c': 'power on left',
				   '35r': 'power on left',
				   'pi7': 'power at bottom'}

doscreen = GetYN("Do you want to install a known screen (Alternative is to install any screen drivers yourself)?")
if doscreen:
	screentype = GetVal("What type screen ({})?".format(supportedscreens), supportedscreens)
	installsrc = screeninstallcode[screentype]()
else:
	screentype = GetVal("Enter name of screen for console reference:")
	installscr = screeninstallcode['--']()

rot = 0
if screentype in baseorientation:
	print(" You can choose to rotate the display from its base orientation")
	print(" Base orientation for {} screen is with {}".format(screentype, baseorientation[screentype]))
	print(" Rotation options(power connection for reference:")
	print("     0: use base orientation")
	print("     1: 90 degrees counterclockwise")
	print("     2: 180 degrees counterclockwise (vertical flip)")
	print("     3: 270 degrees counterclockwise")
	rot = GetInt('Rotation option:', (0, 1, 2, 3, 4))
	if rot != 0: installsrc.append('echo Soft rotation code {}\n'.format(rot))

screentp = screentype + 'B' if Buster else screentype

with open('.Screentype', 'w') as f:
	if rot == 0:
		f.write(screentp + '\n')
	else:
		f.write(screentp + ',' + str(rot) + '\n')

with open('installvals', 'w') as f:
	f.writelines(scriptvars)
with open('installscreencode', 'w') as f:
	f.writelines(installsrc)

ISYname = ""
ans = ""
ISYIP = ""
ISYUSER = ""
ISYPWD = ""
exswitch = ""
MinExamp = GetYN("Set up minimal example system?")

if MinExamp:
	go = False
	while not go:
		ISYname = input("Name to use for the ISY hub (defaults to ISY): ")
		if ISYname == "":
			ISYname = "ISY"
		ISYIP = input("full URL to access ISY: ")
		if ISYIP.endswith('/'):
			ISYIP = ISYIP[0:-1]
		ISYUSER = input("ISY user name: ")
		ISYPWD = input("ISY password: ")
		exswitch = input("Example switch to use (ISY name): ")
		print("ISY Name: " + ISYname)
		print("IP:       " + ISYIP)
		print("USER:     " + ISYUSER)
		print("PASSWORD: " + ISYPWD)
		print("SWITCH:   " + "[[" + exswitch + "]]")
		go = GetYN("OK? (y/n)")

print("Set up directory environment for console")

with open('versionselector', 'w') as f:
	f.write('stable\n')

dirs = ['Console', 'consolestable', 'consolebeta', 'consolerem', 'consoledev', 'Console', 'Console/cfglib']
if personal: dirs.append('consolecur')
for pdir in dirs:
	# noinspection PyBroadException
	try:
		os.mkdir(pdir)
		print("Created: " + str(pdir))
	except:
		print("Already present: " + str(pdir))
	shutil.chown(pdir, user='pi', group='pi')

if MinExamp:
	with open('/home/pi/Console/cfglib/auth.cfg', "w") as f:
		cfg = ("[" + ISYname + "]",
			   "type = ISY",
			   "address = " + ISYIP,
			   "user = " + ISYUSER,
			   "password = " + ISYPWD,
			   "\n")
		f.write("\n".join(cfg))
	with open('/home/pi/Console/config.txt', 'w') as f:
		cfg = ('cfglib = cfglib',
			   'include = auth.cfg, myclock.cfg',
			   'DefaultHub = ' + ISYname,
			   'HomeScreenName = test',
			   'PersistTO = 30',
			   'DimLevel = 5',
			   'DimTO = 15',
			   'DimIdleListNames = MyClock,',
			   'DimIdleListTimes = 20,',
			   'MainChain = test, MyClock',
			   '[test]',
			   'type = Keypad',
			   'label = My, Test',
			   '[[' + exswitch + ']]',
			   "\n")
		f.write("\n".join(cfg))

print("\n\nSoftconsole install paramters:")
for l in scriptvars:
	print('    ' + l.replace('\n', ''))
if MinExamp:
	print("    Create minimal example configuration")
else:
	print("    Skip minimal example configuration")
print('---------------------', flush=True)

import githubutil as U

print('Download console code', flush=True)

if personal:
	# personal system
	U.StageVersion('consolestable', 'homerelease', 'Initial Install')
	print("Stage homerelease as stable")
else:
	U.StageVersion('consolestable', 'currentrelease', 'Initial Install')
	print("Stage standard stable release")

U.InstallStagedVersion('consolestable')
print('Installed stable version', flush=True)

if beta:
	U.StageVersion('consolebeta', 'currentbeta', 'Initial Install')
	print('Stage beta also')
	U.InstallStagedVersion('consolebeta')
	print('Intalled staged beta')

if os.path.exists('/boot/auth'):
	shutil.rmtree('Console/local', ignore_errors=True)
	shutil.move('/boot/auth', 'Console/local')

subprocess.call("cp -r /home/pi/consolestable/'example configs'/* /home/pi/Console", shell=True)

print("****************************************************************", flush=True)
print("****************************************************************", flush=True)
print(" System will now update/upgrade Raspbian", flush=True)
print(" THIS COULD TAKE 10-15 MINUTES DEPENDING ON OUTSTANDING UPDATES", flush=True)
print(" AND THE SPEED OF YOUR SD CARD!", flush=True)
print(" After that it will install the softconsole using your input", flush=True)
print("****************************************************************", flush=True)
print("****************************************************************", flush=True)

if screentype == '28c':
	print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
	print(" NOTE!!! NOTE!!! NOTE!!!")
	print(" If you are using the 28c screen, the settings from Adafruit that")
	print(" sets up are likely wrong.  Look at /boot/config.txt")
	print(" Next to last line should be: dtoverlay=pitft28-capacitive,rotate=180")
	print(" It may well show as rotate=90 after install due to bugs in their scripts")
	print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
	time.sleep(10)
time.sleep(3)
<|endoftext|>