<commit_before># -*- coding: utf-8 -*-

import os
import shutil
import subprocess
import dropbox
import pyqrcode

from datetime import datetime

import ConfigParser
config = ConfigParser.ConfigParser()
config.readfp(open('config.cfg'))

APP_KEY = config.get('Dropbox','app_key')
APP_SECRET = config.get('Dropbox','app_secret')
APP_TOKEN = config.get('Dropbox','app_token')

REC_FPS = config.getint('Recorder','rec_fps')

ROOT_FOLDER = config.get('System','root_folder')
REC_FOLDER = config.get('System','rec_folder')

def clearRecFolder(folder, rec_timestamps, rec_fps=REC_FPS):
    ms_per_frame = 1000/rec_fps

    rec_timestamps = [int(round(x/ms_per_frame))*(ms_per_frame) for x in rec_timestamps] # rounds each timestamp to closest 20
    rec_timestamps = [os.path.join(folder,'%06d.jpg' % x) for x in rec_timestamps] # create filenames from timestamps

    all_recorded_files = [os.path.join(folder,f) for f in sorted(os.listdir(folder))]

    deleted = 0
    for f in all_recorded_files:
        if f not in rec_timestamps:
            deleted += 1
            os.remove(f)
    # print('deleted', deleted, 'files of', len(all_recorded_files), 'remaining', (len(all_recorded_files) - deleted))
    print('{} deleted {} files of {}, remaining {}'.format(datetime.now(), deleted, len(all_recorded_files), (len(all_recorded_files) - deleted)))

def buildVideo(folder, video_path, rec_timestamps, rec_fps=REC_FPS):
    clearRecFolder(folder, rec_timestamps)

    outputframes = [os.path.join(folder,f) for f in sorted(os.listdir(folder))]

    # rename outputframes
    for i in range(len(outputframes)):
        os.rename(outputframes[i], (os.path.join(folder,'%06d.jpg'%(30+i+1))))

    # copy trailer frames
    trailer_folder = os.path.join(ROOT_FOLDER,'img', 'trailer')
    trailer_frames = [os.path.join(trailer_folder,f) for f in sorted(os.listdir(trailer_folder))]
    for f in trailer_frames:
        shutil.copy(f, REC_FOLDER)

    # create mp4
    subprocess.call([
        '/usr/local/bin/ffmpeg',
        '-v', '16',
        '-y',
        '-framerate', '24',
        '-i', '/var/tmp/rec/%06d.jpg',
        '-c:v', 'libx264',
        '-preset', 'faster',
        '-pix_fmt', 'yuv420p',
        video_path
    ])

# TODO: implement abstract upload class
def uploadToDropbox(local_path, app_key=APP_KEY, app_secret=APP_SECRET, app_token=APP_TOKEN):

    flow = dropbox.client.DropboxOAuth2FlowNoRedirect(app_key, app_secret)
    client = dropbox.client.DropboxClient(app_token)

    f = open(local_path, 'rb')
    filename = 'videos/cdmkk_%s.mp4' % (datetime.now().strftime('%Y%m%d_%H%M%S'),)
    client.put_file(filename, f)
    response = client.share(filename)
    return response['url']

def generateQrCode(url, folder):
    # creating qrcode from url
    url = pyqrcode.create(url)
    qr_code_path = os.path.join(folder,'code.png')
    url.png(qr_code_path, scale=1) #scale=1 means: 1 little square is 1px
    return qr_code_path
<commit_msg>removed warning, improved logging<commit_after># -*- coding: utf-8 -*-

import os
import shutil
import subprocess
import dropbox
import pyqrcode

from datetime import datetime

# removing InsecurePlatformWarning
import urllib3
urllib3.disable_warnings()

import ConfigParser
config = ConfigParser.ConfigParser()
config.readfp(open('config.cfg'))

APP_KEY = config.get('Dropbox','app_key')
APP_SECRET = config.get('Dropbox','app_secret')
APP_TOKEN = config.get('Dropbox','app_token')

REC_FPS = config.getint('Recorder','rec_fps')

ROOT_FOLDER = config.get('System','root_folder')
REC_FOLDER = config.get('System','rec_folder')

def clearRecFolder(folder, rec_timestamps, rec_fps=REC_FPS):
    ms_per_frame = 1000/rec_fps

    rec_timestamps = [int(round(x/ms_per_frame))*(ms_per_frame) for x in rec_timestamps] # rounds each timestamp to closest 20
    rec_timestamps = [os.path.join(folder,'%06d.jpg' % x) for x in rec_timestamps] # create filenames from timestamps

    all_recorded_files = [os.path.join(folder,f) for f in sorted(os.listdir(folder))]

    deleted = 0
    for f in all_recorded_files:
        if f not in rec_timestamps:
            deleted += 1
            os.remove(f)
    # print('deleted', deleted, 'files of', len(all_recorded_files), 'remaining', (len(all_recorded_files) - deleted))
    print('{} - deleted {} files of {}, remaining {}'.format(datetime.now(), deleted, len(all_recorded_files), (len(all_recorded_files) - deleted)))

def buildVideo(folder, video_path, rec_timestamps, rec_fps=REC_FPS):
    clearRecFolder(folder, rec_timestamps)

    outputframes = [os.path.join(folder,f) for f in sorted(os.listdir(folder))]

    # rename outputframes
    for i in range(len(outputframes)):
        os.rename(outputframes[i], (os.path.join(folder,'%06d.jpg'%(30+i+1))))

    # copy trailer frames
    trailer_folder = os.path.join(ROOT_FOLDER,'img', 'trailer')
    trailer_frames = [os.path.join(trailer_folder,f) for f in sorted(os.listdir(trailer_folder))]
    for f in trailer_frames:
        shutil.copy(f, REC_FOLDER)

    # create mp4
    subprocess.call([
        '/usr/local/bin/ffmpeg',
        '-v', '16',
        '-y',
        '-framerate', '24',
        '-i', '/var/tmp/rec/%06d.jpg',
        '-c:v', 'libx264',
        '-preset', 'faster',
        '-pix_fmt', 'yuv420p',
        video_path
    ])

# TODO: implement abstract upload class
def uploadToDropbox(local_path, app_key=APP_KEY, app_secret=APP_SECRET, app_token=APP_TOKEN):

    flow = dropbox.client.DropboxOAuth2FlowNoRedirect(app_key, app_secret)
    client = dropbox.client.DropboxClient(app_token)

    f = open(local_path, 'rb')
    filename = 'videos/cdmkk_%s.mp4' % (datetime.now().strftime('%Y%m%d_%H%M%S'),)
    client.put_file(filename, f)
    response = client.share(filename)
    return response['url']

def generateQrCode(url, folder):
    # creating qrcode from url
    url = pyqrcode.create(url)
    qr_code_path = os.path.join(folder,'code.png')
    url.png(qr_code_path, scale=1) #scale=1 means: 1 little square is 1px
    return qr_code_path
<|endoftext|>


<commit_before>""" The aim of this file is to contain all the function
and the main which have to do with emotion recognition, especially
with the Kanade database."""

import glob
import argparse
# import DimensionalityReduction
import cPickle as pickle
from sklearn import cross_validation

import os
import fnmatch
import matplotlib.image as io
import matplotlib.pyplot as plt
import numpy as np
from skimage.transform import resize
from skimage import io
from skimage import color
import os

import facedetection

import deepbelief as db
import restrictedBoltzmannMachine as rbm
from common import *

parser = argparse.ArgumentParser(description='digit recognition')
parser.add_argument('--rbmnesterov', dest='rbmnesterov',action='store_true', default=False,
                    help=("if true, rbms are trained using nesterov momentum"))
parser.add_argument('--save',dest='save',action='store_true', default=False,
                    help="if true, the network is serialized and saved")
parser.add_argument('--train',dest='train',action='store_true', default=False,
                    help=("if true, the network is trained from scratch from the"
                          "traning data"))
parser.add_argument('--rbm', dest='rbm',action='store_true', default=False,
                    help=("if true, the code for traning an rbm on the data is run"))
parser.add_argument('--db', dest='db',action='store_true', default=False,
                    help=("if true, the code for traning a deepbelief net on the"
                          "data is run"))
parser.add_argument('--trainSize', type=int, default=10000,
                    help='the number of tranining cases to be considered')
parser.add_argument('--testSize', type=int, default=1000,
                    help='the number of testing cases to be considered')
parser.add_argument('netFile', help="file where the serialized network should be saved")
parser.add_argument('--nesterov', dest='nesterov',action='store_true', default=False,
                    help=("if true, the deep belief net is trained using nesterov momentum"))
parser.add_argument('--rmsprop', dest='rmsprop',action='store_true', default=False,
                    help=("if true, rmsprop is used when training the deep belief net."))
parser.add_argument('--rbmrmsprop', dest='rbmrmsprop',action='store_true', default=False,
                    help=("if true, rmsprop is used when training the rbms."))
parser.add_argument('--cv', dest='cv',action='store_true', default=False,
                    help=("if true, do cross validation"))
parser.add_argument('--facedetection', dest='facedetection',action='store_true', default=False,
                    help=("if true, do face detection"))
parser.add_argument('--maxEpochs', type=int, default=1000,
                    help='the maximum number of supervised epochs')
parser.add_argument('--validation',dest='validation',action='store_true', default=False,
                    help="if true, the network is trained using a validation set")

# DEBUG mode?
parser.add_argument('--debug', dest='debug',action='store_false', default=False,
                    help=("if true, the deep belief net is ran in DEBUG mode"))

# Get the arguments of the program
args = parser.parse_args()

# Set the debug mode in the deep belief net
db.DEBUG = args.debug

SMALL_SIZE = ((40, 30))

def rbmEmotions(big=False, reconstructRandom=False):
  data, labels = readKanade(big)
  print "data.shape"
  print data.shape

  trainData = data[0:-1, :]

  # Train the network
  if args.train:
    # The number of hidden units is taken from a deep learning tutorial
    # The data are the values of the images have to be normalized before being
    # presented to the network
    nrVisible = len(data[0])
    nrHidden = 800
    # use 1 dropout to test the rbm for now
    net = rbm.RBM(nrVisible, nrHidden, 0.01, 1, 1, rmsprop=args.rbmrmsprop,
                  nesterov=args.rbmnesterov)
    net.train(trainData)
    t = visualizeWeights(net.weights.T, SMALL_SIZE, (10,10))
  else:
    # Take the saved network and use that for reconstructions
    f = open(args.netFile, "rb")
    t = pickle.load(f)
    net = pickle.load(f)
    f.close()

  # get a random image and see it looks like
  # if reconstructRandom:
  #   test = np.random.random_sample(test.shape)

  # Show the initial image first
  test = data[-1, :]
  print "test.shape"
  print test.shape

  plt.imshow(vectorToImage(test, SMALL_SIZE), cmap=plt.cm.gray)
  plt.axis('off')
  plt.savefig('initialface.png', transparent=True)

  recon = net.reconstruct(test.reshape(1, test.shape[0]))
  print recon.shape

  plt.imshow(vectorToImage(recon, SMALL_SIZE), cmap=plt.cm.gray)
  plt.axis('off')
  plt.savefig('reconstructface.png', transparent=True)

  # Show the weights and their form in a tile fashion
  # Plot the weights
  plt.imshow(t, cmap=plt.cm.gray)
  plt.axis('off')
  if args.rbmrmsprop:
    st='rmsprop'
  else:
    st = 'simple'
  plt.savefig('weights' + st + '.png', transparent=True)
  print "done"

  if args.save:
    f = open(args.netFile, "wb")
    pickle.dump(t, f)
    pickle.dump(net, f)

"""
  Arguments:
    big: should the big or small images be used?
"""
def deepBeliefKanadeCV(big=False):
  data, labels = readKanade(big)

  data, labels = shuffle(data, labels)

  print "data.shape"
  print data.shape
  print "labels.shape"
  print labels.shape

  # TODO: try boosting for CV in order to increase the number of folds
  params =[(0.1, 0.1) , (0.1, 0.05), (0.05, 0.01), (0.05, 0.05)]
  unsupervisedData = buildUnsupervisedDataSet()

  kf = cross_validation.KFold(n=len(data), k=len(params))
  bestCorrect = 0
  bestProbs = 0

  fold = 0
  for train, test in kf:

    trainData = data[train]
    trainLabels = labels[train]

    # TODO: this might require more thought
    net = db.DBN(5, [1200, 1500, 1500, 1500, 7],
               unsupervisedLearningRate=params[fold][0],
               supervisedLearningRate=params[fold][1],
               nesterovMomentum=args.nesterov,
               rbmNesterovMomentum=args.rbmnesterov,
               rmsprop=args.rmsprop,
               hiddenDropout=0.5,
               rbmHiddenDropout=0.5,
               visibleDropout=0.8,
               rbmVisibleDropout=1)

    net.train(trainData, trainLabels,
              maxEpochs=args.maxEpochs,
              validation=args.validation,
              unsupervisedData=unsupervisedData)

    probs, predicted = net.classify(data[test])

    actualLabels = labels[test]
    correct = 0
    errorCases = []

    for i in xrange(len(test)):
      print "predicted"
      print "probs"
      print probs[i]
      print predicted[i]
      print "actual"
      actual = actualLabels[i]
      print np.argmax(actual)
      if predicted[i] == np.argmax(actual):
        correct += 1
      else:
        errorCases.append(i)

    print "correct for " + str(params[fold])
    print correct

    if bestCorrect < correct:
      bestCorrect = correct
      bestParam = params[fold]
      bestProbs = correct * 1.0 / len(test)

    fold += 1

  print "bestParam"
  print bestParam

  print "bestProbs"
  print bestProbs

# TODO: shuffle training data for minibatches
def deepBeliefKanade(big=False):
  data, labels = readKanade(big, None)

  print "data.shape"
  print data.shape
  print "labels.shape"
  print labels.shape

  # Random data for training and testing
  kf = cross_validation.KFold(n=len(data), n_folds=5)
  for train, test in kf:
    break

  trainData = data[train]
  trainLabels = labels[train]

  # TODO: this might require more thought
  net = db.DBN(5, [1200, 1500, 1500, 1500, 7],
             unsupervisedLearningRate=0.05,
             # is this not a bad learning rate?
             supervisedLearningRate=0.01,
             nesterovMomentum=args.nesterov,
             rbmNesterovMomentum=args.rbmnesterov,
             rmsprop=args.rmsprop,
             hiddenDropout=0.5,
             rbmHiddenDropout=0.5,
             visibleDropout=0.8,
             rbmVisibleDropout=1)

  unsupervisedData = buildUnsupervisedDataSet()

  net.train(trainData, trainLabels, maxEpochs=args.maxEpochs,
            validation=args.validation, unsupervisedData=unsupervisedData)

  probs, predicted = net.classify(data[test])

  actualLabels = labels[test]
  correct = 0
  errorCases = []

  for i in xrange(len(test)):
    print "predicted"
    print "probs"
    print probs[i]
    print predicted[i]
    print "actual"
    actual = actualLabels[i]
    print np.argmax(actual)
    if predicted[i] == np.argmax(actual):
      correct += 1
    else:
      errorCases.append(i)

  print "correct"
  print correct

  print "percentage correct"
  print correct  * 1.0/ len(test)

def buildUnsupervisedDataSet():
  return np.vstack((
    # readCroppedYale(),
    readAttData(),
    readJaffe(),
    # readNottingham(),
    readAberdeen()))

def readKanade(big=False, folds=None):
  if big:
    files = glob.glob('kanade_150*.pickle')
  else:
    files = glob.glob('kanade_f*.pickle')

  if not folds:
    folds = range(1, 6)

  # Read the data from them. Sort out the files that do not have
  # the folds that we want
  # TODO: do this better (with regex in the file name)
  # DO not reply on the order returned
  files = [ files[x -1] for x in folds]

  data = np.array([])
  labels = np.array([])
  # TODO: do LDA on the training data

  # TODO: do proper CV in which you use 4 folds for training and one for testing
  # at that time
  dataFolds = []
  labelFolds = []
  for filename in files:
    with open(filename, "rb") as  f:
      # Sort out the labels from the data
      # TODO: run the readKanade again tomorrow and change these idnices here
      dataAndLabels = pickle.load(f)
      foldData = dataAndLabels[:, 0:-1]
      print "foldData.shape"
      print foldData.shape
      foldLabels = dataAndLabels[:,-1]
      dataFolds.append(foldData)
      foldLabels = np.array(map(int, foldLabels))

      vectorLabels = labelsToVectors(foldLabels -1, 7)
      labelFolds.append(vectorLabels)

      print "foldLabels.shape"
      print vectorLabels.shape

  data =  np.vstack(tuple(dataFolds))
  labels = np.vstack(tuple(labelFolds))
  return data, labels

# TODO: get big, small as argument in order to be able to fit the resizing
def readCroppedYale():
  # PATH = "/data/mcr10/yaleb/CroppedYale"
  PATH = "/home/aela/uni/project/CroppedYale"

  imageFiles = [os.path.join(dirpath, f)
    for dirpath, dirnames, files in os.walk(PATH)
    for f in fnmatch.filter(files, '*.pgm')]

  # Filter out the ones that containt "amyes bient"
  imageFiles = [ x for x in imageFiles if not "Ambient" in x]

  images = []
  for f in imageFiles:
    img = io.imread(f)
    img = resize(img, SMALL_SIZE)

    images += [img.reshape(-1)]

  return np.array(images)

def readAttData():
  PATH = "/data/mcr10/att"
  # PATH = "/home/aela/uni/project/code/pics/cambrdige_pics"

  imageFiles = [os.path.join(dirpath, f)
    for dirpath, dirnames, files in os.walk(PATH)
    for f in fnmatch.filter(files, '*.pgm')]

  images = []
  for f in imageFiles:
    img = io.imread(f)
    img = resize(img, SMALL_SIZE)
    images += [img.reshape(-1)]

  return np.array(images)

def readAndCrop(path, extension, doRecognition, isColoured=False):
  dirforres = "detection-cropped"
  pathForCropped = os.path.join(path, dirforres)

  if doRecognition:
    if not os.path.exists(pathForCropped):
      os.makedirs(pathForCropped)

    imageFiles = [(os.path.join(dirpath, f), f)
      for dirpath, dirnames, files in os.walk(path)
      for f in fnmatch.filter(files, '*.' + extension)]

    images = []

    for fullPath, shortPath in imageFiles:
      # Do not do this for already cropped images
      if pathForCropped in fullPath:
        continue

      print fullPath
      img = io.imread(fullPath)
      if isColoured:
        img = color.rgb2gray(img)
        img = np.array(img * 255, dtype='uint8')

      face = facedetection.cropFace(img)
      if not face == None:
        # Only do the resizing once you are done with the cropping of the faces
        face = resize(face, SMALL_SIZE)
        # Check that you are always saving them in the right format
        print "face.min"
        print face.min()

        print "face.max"
        print face.max()

        assert face.min() >=0 and face.max() <=1
        images += [face.reshape(-1)]

        # Save faces as files
        croppedFileName = os.path.join(pathForCropped, shortPath)
        io.imsave(croppedFileName, face)

  else:
    images = []
    imageFiles = [os.path.join(dirpath, f)
      for dirpath, dirnames, files in os.walk(pathForCropped)
      for f in fnmatch.filter(files, '*.' + extension)]

    for f in imageFiles:
      img = io.imread(f)
      if type(img[0,0]) == np.uint8:
        print "rescaling unit"
      # if not (img.min() >=0.0 and img.max()<=1.000008):
        img = img / 255.0
      images += [img.reshape(-1)]

  print len(images)
  return np.array(images)

# This needs some thought: remove the cropped folder from path?

def readJaffe():
  PATH = "/data/mcr10/jaffe"
  # PATH = "/home/aela/uni/project/jaffe"
  return readAndCrop(PATH , "tiff", args.facedetection, isColoured=False)

def readNottingham():
  PATH = "/home/aela/uni/project/nottingham"
  # PATH = "/data/mcr10/nottingham"
  return readAndCrop(PATH, "gif", args.facedetection, isColoured=False)

def readAberdeen():
  PATH = "/data/mcr10/Aberdeen"
  # PATH = "/home/aela/uni/project/Aberdeen"
  return readAndCrop(PATH, "jpg", args.facedetection, isColoured=True)

def main():
  # deepBeliefKanade()

  # readNottingham()
  # readCroppedYale()
  # readJaffe()
  # readAttData()
  # readAberdeen()
  if args.rbm:
    rbmEmotions()
  elif args.cv:
    deepBeliefKanadeCV()
  elif args.db:
    deepBeliefKanade()

# You can also group the emotions into positive and negative to see
# if you can get better results (probably yes)
if __name__ == '__main__':
  import random
  print "FIXING RANDOMNESS"
  random.seed(6)
  np.random.seed(6)
  main()<commit_msg>passing mini batch size to emotions as well<commit_after>""" The aim of this file is to contain all the function
and the main which have to do with emotion recognition, especially
with the Kanade database."""

import glob
import argparse
# import DimensionalityReduction
import cPickle as pickle
from sklearn import cross_validation

import os
import fnmatch
import matplotlib.image as io
import matplotlib.pyplot as plt
import numpy as np
from skimage.transform import resize
from skimage import io
from skimage import color
import os

import facedetection

import deepbelief as db
import restrictedBoltzmannMachine as rbm
from common import *

parser = argparse.ArgumentParser(description='digit recognition')
parser.add_argument('--rbmnesterov', dest='rbmnesterov',action='store_true', default=False,
                    help=("if true, rbms are trained using nesterov momentum"))
parser.add_argument('--save',dest='save',action='store_true', default=False,
                    help="if true, the network is serialized and saved")
parser.add_argument('--train',dest='train',action='store_true', default=False,
                    help=("if true, the network is trained from scratch from the"
                          "traning data"))
parser.add_argument('--rbm', dest='rbm',action='store_true', default=False,
                    help=("if true, the code for traning an rbm on the data is run"))
parser.add_argument('--db', dest='db',action='store_true', default=False,
                    help=("if true, the code for traning a deepbelief net on the"
                          "data is run"))
parser.add_argument('--trainSize', type=int, default=10000,
                    help='the number of tranining cases to be considered')
parser.add_argument('--testSize', type=int, default=1000,
                    help='the number of testing cases to be considered')
parser.add_argument('netFile', help="file where the serialized network should be saved")
parser.add_argument('--nesterov', dest='nesterov',action='store_true', default=False,
                    help=("if true, the deep belief net is trained using nesterov momentum"))
parser.add_argument('--rmsprop', dest='rmsprop',action='store_true', default=False,
                    help=("if true, rmsprop is used when training the deep belief net."))
parser.add_argument('--rbmrmsprop', dest='rbmrmsprop',action='store_true', default=False,
                    help=("if true, rmsprop is used when training the rbms."))
parser.add_argument('--cv', dest='cv',action='store_true', default=False,
                    help=("if true, do cross validation"))
parser.add_argument('--facedetection', dest='facedetection',action='store_true', default=False,
                    help=("if true, do face detection"))
parser.add_argument('--maxEpochs', type=int, default=1000,
                    help='the maximum number of supervised epochs')
parser.add_argument('--miniBatchSize', type=int, default=10,
                    help='the number of training points in a mini batch')
parser.add_argument('--validation',dest='validation',action='store_true', default=False,
                    help="if true, the network is trained using a validation set")

# DEBUG mode?
parser.add_argument('--debug', dest='debug',action='store_false', default=False,
                    help=("if true, the deep belief net is ran in DEBUG mode"))

# Get the arguments of the program
args = parser.parse_args()

# Set the debug mode in the deep belief net
db.DEBUG = args.debug

SMALL_SIZE = ((40, 30))

def rbmEmotions(big=False, reconstructRandom=False):
  data, labels = readKanade(big)
  print "data.shape"
  print data.shape

  trainData = data[0:-1, :]

  # Train the network
  if args.train:
    # The number of hidden units is taken from a deep learning tutorial
    # The data are the values of the images have to be normalized before being
    # presented to the network
    nrVisible = len(data[0])
    nrHidden = 800
    # use 1 dropout to test the rbm for now
    net = rbm.RBM(nrVisible, nrHidden, 0.01, 1, 1, rmsprop=args.rbmrmsprop,
                  nesterov=args.rbmnesterov)
    net.train(trainData)
    t = visualizeWeights(net.weights.T, SMALL_SIZE, (10,10))
  else:
    # Take the saved network and use that for reconstructions
    f = open(args.netFile, "rb")
    t = pickle.load(f)
    net = pickle.load(f)
    f.close()

  # get a random image and see it looks like
  # if reconstructRandom:
  #   test = np.random.random_sample(test.shape)

  # Show the initial image first
  test = data[-1, :]
  print "test.shape"
  print test.shape

  plt.imshow(vectorToImage(test, SMALL_SIZE), cmap=plt.cm.gray)
  plt.axis('off')
  plt.savefig('initialface.png', transparent=True)

  recon = net.reconstruct(test.reshape(1, test.shape[0]))
  print recon.shape

  plt.imshow(vectorToImage(recon, SMALL_SIZE), cmap=plt.cm.gray)
  plt.axis('off')
  plt.savefig('reconstructface.png', transparent=True)

  # Show the weights and their form in a tile fashion
  # Plot the weights
  plt.imshow(t, cmap=plt.cm.gray)
  plt.axis('off')
  if args.rbmrmsprop:
    st='rmsprop'
  else:
    st = 'simple'
  plt.savefig('weights' + st + '.png', transparent=True)
  print "done"

  if args.save:
    f = open(args.netFile, "wb")
    pickle.dump(t, f)
    pickle.dump(net, f)

"""
  Arguments:
    big: should the big or small images be used?
"""
def deepBeliefKanadeCV(big=False):
  data, labels = readKanade(big)

  data, labels = shuffle(data, labels)

  print "data.shape"
  print data.shape
  print "labels.shape"
  print labels.shape

  # TODO: try boosting for CV in order to increase the number of folds
  params =[(0.1, 0.1) , (0.1, 0.05), (0.05, 0.01), (0.05, 0.05)]
  unsupervisedData = buildUnsupervisedDataSet()

  kf = cross_validation.KFold(n=len(data), k=len(params))
  bestCorrect = 0
  bestProbs = 0

  fold = 0
  for train, test in kf:

    trainData = data[train]
    trainLabels = labels[train]

    # TODO: this might require more thought
    net = db.DBN(5, [1200, 1500, 1500, 1500, 7],
               unsupervisedLearningRate=params[fold][0],
               supervisedLearningRate=params[fold][1],
               nesterovMomentum=args.nesterov,
               rbmNesterovMomentum=args.rbmnesterov,
               rmsprop=args.rmsprop,
               miniBatchSize=args.miniBatchSize,
               hiddenDropout=0.5,
               rbmHiddenDropout=0.5,
               visibleDropout=0.8,
               rbmVisibleDropout=1)

    net.train(trainData, trainLabels,
              maxEpochs=args.maxEpochs,
              validation=args.validation,
              unsupervisedData=unsupervisedData)

    probs, predicted = net.classify(data[test])

    actualLabels = labels[test]
    correct = 0
    errorCases = []

    for i in xrange(len(test)):
      print "predicted"
      print "probs"
      print probs[i]
      print predicted[i]
      print "actual"
      actual = actualLabels[i]
      print np.argmax(actual)
      if predicted[i] == np.argmax(actual):
        correct += 1
      else:
        errorCases.append(i)

    print "correct for " + str(params[fold])
    print correct

    if bestCorrect < correct:
      bestCorrect = correct
      bestParam = params[fold]
      bestProbs = correct * 1.0 / len(test)

    fold += 1

  print "bestParam"
  print bestParam

  print "bestProbs"
  print bestProbs

# TODO: shuffle training data for minibatches
def deepBeliefKanade(big=False):
  data, labels = readKanade(big, None)

  print "data.shape"
  print data.shape
  print "labels.shape"
  print labels.shape

  # Random data for training and testing
  kf = cross_validation.KFold(n=len(data), n_folds=5)
  for train, test in kf:
    break

  trainData = data[train]
  trainLabels = labels[train]

  # TODO: this might require more thought
  net = db.DBN(5, [1200, 1500, 1500, 1500, 7],
             unsupervisedLearningRate=0.05,
             # is this not a bad learning rate?
             supervisedLearningRate=0.01,
             nesterovMomentum=args.nesterov,
             rbmNesterovMomentum=args.rbmnesterov,
             rmsprop=args.rmsprop,
             miniBatchSize=args.miniBatchSize,
             hiddenDropout=0.5,
             rbmHiddenDropout=0.5,
             visibleDropout=0.8,
             rbmVisibleDropout=1)

  unsupervisedData = buildUnsupervisedDataSet()

  net.train(trainData, trainLabels, maxEpochs=args.maxEpochs,
            validation=args.validation, unsupervisedData=unsupervisedData)

  probs, predicted = net.classify(data[test])

  actualLabels = labels[test]
  correct = 0
  errorCases = []

  for i in xrange(len(test)):
    print "predicted"
    print "probs"
    print probs[i]
    print predicted[i]
    print "actual"
    actual = actualLabels[i]
    print np.argmax(actual)
    if predicted[i] == np.argmax(actual):
      correct += 1
    else:
      errorCases.append(i)

  print "correct"
  print correct

  print "percentage correct"
  print correct  * 1.0/ len(test)

def buildUnsupervisedDataSet():
  return np.vstack((
    # readCroppedYale(),
    readAttData(),
    readJaffe(),
    # readNottingham(),
    readAberdeen()))

def readKanade(big=False, folds=None):
  if big:
    files = glob.glob('kanade_150*.pickle')
  else:
    files = glob.glob('kanade_f*.pickle')

  if not folds:
    folds = range(1, 6)

  # Read the data from them. Sort out the files that do not have
  # the folds that we want
  # TODO: do this better (with regex in the file name)
  # DO not reply on the order returned
  files = [ files[x -1] for x in folds]

  data = np.array([])
  labels = np.array([])
  # TODO: do LDA on the training data

  # TODO: do proper CV in which you use 4 folds for training and one for testing
  # at that time
  dataFolds = []
  labelFolds = []
  for filename in files:
    with open(filename, "rb") as  f:
      # Sort out the labels from the data
      # TODO: run the readKanade again tomorrow and change these idnices here
      dataAndLabels = pickle.load(f)
      foldData = dataAndLabels[:, 0:-1]
      print "foldData.shape"
      print foldData.shape
      foldLabels = dataAndLabels[:,-1]
      dataFolds.append(foldData)
      foldLabels = np.array(map(int, foldLabels))

      vectorLabels = labelsToVectors(foldLabels -1, 7)
      labelFolds.append(vectorLabels)

      print "foldLabels.shape"
      print vectorLabels.shape

  data =  np.vstack(tuple(dataFolds))
  labels = np.vstack(tuple(labelFolds))
  return data, labels

# TODO: get big, small as argument in order to be able to fit the resizing
def readCroppedYale():
  # PATH = "/data/mcr10/yaleb/CroppedYale"
  PATH = "/home/aela/uni/project/CroppedYale"

  imageFiles = [os.path.join(dirpath, f)
    for dirpath, dirnames, files in os.walk(PATH)
    for f in fnmatch.filter(files, '*.pgm')]

  # Filter out the ones that containt "amyes bient"
  imageFiles = [ x for x in imageFiles if not "Ambient" in x]

  images = []
  for f in imageFiles:
    img = io.imread(f)
    img = resize(img, SMALL_SIZE)

    images += [img.reshape(-1)]

  return np.array(images)

def readAttData():
  PATH = "/data/mcr10/att"
  # PATH = "/home/aela/uni/project/code/pics/cambrdige_pics"

  imageFiles = [os.path.join(dirpath, f)
    for dirpath, dirnames, files in os.walk(PATH)
    for f in fnmatch.filter(files, '*.pgm')]

  images = []
  for f in imageFiles:
    img = io.imread(f)
    img = resize(img, SMALL_SIZE)
    images += [img.reshape(-1)]

  return np.array(images)

def readAndCrop(path, extension, doRecognition, isColoured=False):
  dirforres = "detection-cropped"
  pathForCropped = os.path.join(path, dirforres)

  if doRecognition:
    if not os.path.exists(pathForCropped):
      os.makedirs(pathForCropped)

    imageFiles = [(os.path.join(dirpath, f), f)
      for dirpath, dirnames, files in os.walk(path)
      for f in fnmatch.filter(files, '*.' + extension)]

    images = []

    for fullPath, shortPath in imageFiles:
      # Do not do this for already cropped images
      if pathForCropped in fullPath:
        continue

      print fullPath
      img = io.imread(fullPath)
      if isColoured:
        img = color.rgb2gray(img)
        img = np.array(img * 255, dtype='uint8')

      face = facedetection.cropFace(img)
      if not face == None:
        # Only do the resizing once you are done with the cropping of the faces
        face = resize(face, SMALL_SIZE)
        # Check that you are always saving them in the right format
        print "face.min"
        print face.min()

        print "face.max"
        print face.max()

        assert face.min() >=0 and face.max() <=1
        images += [face.reshape(-1)]

        # Save faces as files
        croppedFileName = os.path.join(pathForCropped, shortPath)
        io.imsave(croppedFileName, face)

  else:
    images = []
    imageFiles = [os.path.join(dirpath, f)
      for dirpath, dirnames, files in os.walk(pathForCropped)
      for f in fnmatch.filter(files, '*.' + extension)]

    for f in imageFiles:
      img = io.imread(f)
      if type(img[0,0]) == np.uint8:
        print "rescaling unit"
      # if not (img.min() >=0.0 and img.max()<=1.000008):
        img = img / 255.0
      images += [img.reshape(-1)]

  print len(images)
  return np.array(images)

# This needs some thought: remove the cropped folder from path?

def readJaffe():
  PATH = "/data/mcr10/jaffe"
  # PATH = "/home/aela/uni/project/jaffe"
  return readAndCrop(PATH , "tiff", args.facedetection, isColoured=False)

def readNottingham():
  PATH = "/home/aela/uni/project/nottingham"
  # PATH = "/data/mcr10/nottingham"
  return readAndCrop(PATH, "gif", args.facedetection, isColoured=False)

def readAberdeen():
  PATH = "/data/mcr10/Aberdeen"
  # PATH = "/home/aela/uni/project/Aberdeen"
  return readAndCrop(PATH, "jpg", args.facedetection, isColoured=True)

def main():
  # deepBeliefKanade()

  # readNottingham()
  # readCroppedYale()
  # readJaffe()
  # readAttData()
  # readAberdeen()
  if args.rbm:
    rbmEmotions()
  elif args.cv:
    deepBeliefKanadeCV()
  elif args.db:
    deepBeliefKanade()

# You can also group the emotions into positive and negative to see
# if you can get better results (probably yes)
if __name__ == '__main__':
  import random
  print "FIXING RANDOMNESS"
  random.seed(6)
  np.random.seed(6)
  main()<|endoftext|>


<commit_before># coding: utf-8
# Copyright (c) Pymatgen Development Team.
# Distributed under the terms of the MIT License.
"""
Provides a class for interacting with KPath classes to
generate high-symmetry k-paths using different conventions.
"""

import itertools
from warnings import warn

import networkx as nx
import numpy as np

from pymatgen.symmetry.kpath import (
    KPathBase,
    KPathLatimerMunro,
    KPathSeek,
    KPathSetyawanCurtarolo,
)

from pymatgen.electronic_structure.bandstructure import BandStructureSymmLine
from pymatgen.electronic_structure.core import Spin

__author__ = "Jason Munro"
__copyright__ = "Copyright 2020, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Jason Munro"
__email__ = "jmunro@lbl.gov"
__status__ = "Development"
__date__ = "March 2020"

class HighSymmKpath(KPathBase):
    """
    This class generates path along high symmetry lines in the
    Brillouin zone according to different conventions.
    The class is designed to be used with a specific primitive
    cell setting. The definitions for the primitive cell
    used can be found in: Computational Materials Science,
    49(2), 299-312. doi:10.1016/j.commatsci.2010.05.010.
    The space group analyzer can be used to produce the correct
    primitive structure
    (method get_primitive_standard_structure(international_monoclinic=False)).
    Ensure input structure is correct before 'get_kpoints()' method is used.
    See individual KPath classes for details on specific conventions.
    """

    def __init__(
        self,
        structure,
        has_magmoms=False,
        magmom_axis=None,
        path_type="sc",
        symprec=0.01,
        angle_tolerance=5,
        atol=1e-5,
    ):
        """
        Args:
            structure (Structure): Structure object
            has_magmoms (boolean): Whether the input structure contains
                magnetic moments as site properties with the key 'magmom.'
                Values may be in the form of 3-component vectors given in
                the basis of the input lattice vectors, in
                which case the spin axis will default to a_3, the third
                real-space lattice vector (this triggers a warning).
            magmom_axis (list or numpy array): 3-component vector specifying
                direction along which magnetic moments given as scalars
                should point. If all magnetic moments are provided as
                vectors then this argument is not used.
            path_type (string): Chooses which convention to use to generate
                the high symmetry path. Options are: 'setyawan_curtarolo', 'hinuma',
                'latimer_munro' for the Setyawan & Curtarolo, Hinuma et al., and
                Latimer & Munro conventions. Choosing 'all' will generate one path
                with points from all three conventions. Equivalent labels between
                each will also be generated. Order will always be Latimer & Munro,
                Setyawan & Curtarolo, and Hinuma et al. Lengths for each of the paths
                will also be generated and output as a list. Note for 'all' the user
                will have to alter the labels on their own for plotting.
            symprec (float): Tolerance for symmetry finding
            angle_tolerance (float): Angle tolerance for symmetry finding.
            atol (float): Absolute tolerance used to determine symmetric
                equivalence of points and lines on the BZ.
        """

        super().__init__(structure, symprec=symprec, angle_tolerance=angle_tolerance, atol=atol)

        self._path_type = path_type

        self._equiv_labels = None
        self._path_lengths = None
        self._label_index = None

        if path_type != "all":

            if path_type == "latimer_munro":
                self._kpath = self._get_lm_kpath(has_magmoms, magmom_axis, symprec, angle_tolerance, atol).kpath
            elif path_type == "setyawan_curtarolo":
                self._kpath = self._get_sc_kpath(symprec, angle_tolerance, atol).kpath
            elif path_type == "hinuma":
                hin_dat = self._get_hin_kpath(symprec, angle_tolerance, atol, not has_magmoms)
                self._kpath = hin_dat.kpath
                self._hin_tmat = hin_dat._tmat

        else:

            if has_magmoms:
                raise ValueError("Cannot select 'all' with non-zero magmoms.")

            lm_bs = self._get_lm_kpath(has_magmoms, magmom_axis, symprec, angle_tolerance, atol)
            rpg = lm_bs._rpg

            sc_bs = self._get_sc_kpath(symprec, angle_tolerance, atol)
            hin_bs = self._get_hin_kpath(symprec, angle_tolerance, atol, not has_magmoms)

            index = 0
            cat_points = {}
            label_index = {}
            num_path = []
            self._path_lengths = []

            for bs in [lm_bs, sc_bs, hin_bs]:
                for key, value in enumerate(bs.kpath["kpoints"]):
                    cat_points[index] = bs.kpath["kpoints"][value]
                    label_index[index] = value
                    index += 1

                total_points_path = 0
                for seg in bs.kpath["path"]:
                    total_points_path += len(seg)

                for block in bs.kpath["path"]:
                    new_block = []
                    for label in block:
                        for ind in range(
                            len(label_index) - len(bs.kpath["kpoints"]),
                            len(label_index),
                        ):
                            if label_index[ind] == label:
                                new_block.append(ind)

                    num_path.append(new_block)

                self._path_lengths.append(total_points_path)

            self._label_index = label_index

            self._kpath = {"kpoints": cat_points, "path": num_path}

            self._equiv_labels = self._get_klabels(lm_bs, sc_bs, hin_bs, rpg)

    @property
    def path_type(self):
        """
        Returns:
        The type of kpath chosen
        """
        return self._path_type

    @property
    def label_index(self):
        """
        Returns:
        The correspondance between numbers and kpoint symbols for the
        combined kpath generated when path_type = 'all'. None otherwise.
        """
        return self._label_index

    @property
    def equiv_labels(self):
        """
        Returns:
        The correspondance between the kpoint symbols in the Latimer and
        Munro convention, Setyawan and Curtarolo, and Hinuma
        conventions respectively. Only generated when path_type = 'all'.
        """
        return self._equiv_labels

    @property
    def path_lengths(self):
        """
        Returns:
        List of lengths of the Latimer and Munro, Setyawan and Curtarolo, and Hinuma
        conventions in the combined HighSymmKpath object when path_type = 'all' respectively.
        None otherwise.
        """
        return self._path_lengths

    def _get_lm_kpath(self, has_magmoms, magmom_axis, symprec, angle_tolerance, atol):
        """
        Returns:
        Latimer and Munro k-path with labels.
        """

        return KPathLatimerMunro(self._structure, has_magmoms, magmom_axis, symprec, angle_tolerance, atol)

    def _get_sc_kpath(self, symprec, angle_tolerance, atol):
        """
        Returns:
        Setyawan and Curtarolo k-path with labels.
        """
        kpath = KPathSetyawanCurtarolo(self._structure, symprec, angle_tolerance, atol)

        self.prim = kpath.prim
        self.conventional = kpath.conventional
        self.prim_rec = kpath.prim_rec
        self._rec_lattice = self.prim_rec

        return kpath

    def _get_hin_kpath(self, symprec, angle_tolerance, atol, tri):
        """
        Returns:
        Hinuma et al. k-path with labels.
        """
        bs = KPathSeek(self._structure, symprec, angle_tolerance, atol, tri)

        kpoints = bs.kpath["kpoints"]
        tmat = bs._tmat
        for key in kpoints:
            kpoints[key] = np.dot(np.transpose(np.linalg.inv(tmat)), kpoints[key])

        bs.kpath["kpoints"] = kpoints
        self._rec_lattice = self._structure.lattice.reciprocal_lattice

        warn(
            "K-path from the Hinuma et al. convention has been transformed to the basis of the reciprocal lattice \
of the input structure. Use `KPathSeek` for the path in the original author-intended basis."
        )

        return bs

    def _get_klabels(self, lm_bs, sc_bs, hin_bs, rpg):
        """
        Returns:
        labels (dict): Dictionary of equivalent labels for paths if 'all' is chosen.
            If an exact kpoint match cannot be found, symmetric equivalency will be
            searched for and indicated with an asterisk in the equivalent label.
            If an equivalent label can still not be found, or the point is not in
            the explicit kpath, its equivalent label will be set to itself in the output.
        """

        lm_path = lm_bs.kpath
        sc_path = sc_bs.kpath
        hin_path = hin_bs.kpath

        n_op = len(rpg)

        pairs = itertools.permutations([{"setyawan_curtarolo": sc_path}, {"latimer_munro": lm_path}, {"hinuma": hin_path}], r=2)
        labels = {"setyawan_curtarolo": {}, "latimer_munro": {}, "hinuma": {}}

        for (a, b) in pairs:
            [(a_type, a_path)] = list(a.items())
            [(b_type, b_path)] = list(b.items())

            sc_count = np.zeros(n_op)

            for o_num in range(0, n_op):
                a_tr_coord = []

                for (label_a, coord_a) in a_path["kpoints"].items():
                    a_tr_coord.append(np.dot(rpg[o_num], coord_a))

                for coord_a in a_tr_coord:
                    for key, value in b_path["kpoints"].items():
                        if np.allclose(value, coord_a, atol=self._atol):
                            sc_count[o_num] += 1
                            break

            a_to_b_labels = {}
            unlabeled = {}

            for (label_a, coord_a) in a_path["kpoints"].items():
                coord_a_t = np.dot(rpg[np.argmax(sc_count)], coord_a)
                assigned = False

                for (label_b, coord_b) in b_path["kpoints"].items():
                    if np.allclose(coord_b, coord_a_t, atol=self._atol):
                        a_to_b_labels[label_a] = label_b
                        assigned = True
                        break

                if not assigned:
                    unlabeled[label_a] = coord_a

            for (label_a, coord_a) in unlabeled.items():
                for op in rpg:
                    coord_a_t = np.dot(op, coord_a)
                    key = [
                        key
                        for key, value in b_path["kpoints"].items()
                        if np.allclose(value, coord_a_t, atol=self._atol)
                    ]

                    if key != []:
                        a_to_b_labels[label_a] = key[0][0] + "^{*}"
                        break

                if key == []:
                    a_to_b_labels[label_a] = label_a

            labels[a_type][b_type] = a_to_b_labels

        return labels

    @staticmethod
    def get_continuous_path(bandstructure):
        """
        Obtain a continous version of an inputted path using graph theory.
        This routine will attempt to add connections between nodes of
        odd-degree to ensure a Eulerian path can be formed. Initial
        k-path must be able to be converted to a connected graph. See
        npj Comput Mater 6, 112 (2020). 10.1038/s41524-020-00383-7
        for more details.

        Args:
        bandstructure (BandstructureSymmLine): BandstructureSymmLine object.

        Returns:
        bandstructure (BandstructureSymmLine): New BandstructureSymmLine object with continous path.
        """

        G = nx.Graph()

        labels = []
        for point in bandstructure.kpoints:
            if point.label is not None:
                labels.append(point.label)

        plot_axis = []
        for i in range(int(len(labels) / 2)):
            G.add_edges_from([(labels[2 * i], labels[(2 * i) + 1])])
            plot_axis.append((labels[2 * i], labels[(2 * i) + 1]))

        G_euler = nx.algorithms.euler.eulerize(G)

        G_euler_circuit = nx.algorithms.euler.eulerian_circuit(G_euler)

        distances_map = []
        kpath_euler = []

        for edge_euler in G_euler_circuit:
            kpath_euler.append(edge_euler)
            for edge_reg in plot_axis:
                if edge_euler == edge_reg:
                    distances_map.append((plot_axis.index(edge_reg), False))
                elif edge_euler[::-1] == edge_reg:
                    distances_map.append((plot_axis.index(edge_reg), True))

        if bandstructure.is_spin_polarized:
            spins = [Spin.up, Spin.down]
        else:
            spins = [Spin.up]

        new_kpoints = []
        new_bands = {spin: [np.array([]) for _ in range(bandstructure.nb_bands)] for spin in spins}
        new_projections = {spin: [[] for _ in range(bandstructure.nb_bands)] for spin in spins}

        for entry in distances_map:
            if not entry[1]:
                branch = bandstructure.branches[entry[0]]
                start = branch["start_index"]
                stop = branch["end_index"] + 1
                step = 1

            else:
                branch = bandstructure.branches[entry[0]]
                start = branch["end_index"]
                stop = branch["start_index"] - 1
                step = -1

            # kpoints
            new_kpoints += [point.frac_coords for point in bandstructure.kpoints[start:stop:step]]

            # eigenvals
            for spin in spins:
                for n, band in enumerate(bandstructure.bands[spin]):

                    new_bands[spin][n] = np.concatenate((new_bands[spin][n], band[start:stop:step]))

            # projections
            for spin in spins:
                for n, band in enumerate(bandstructure.projections[spin]):

                    new_projections[spin][n] += band[start:stop:step].tolist()

        for spin in spins:
            new_projections[spin] = np.array(new_projections[spin])

        new_labels_dict = {label: point.frac_coords for label, point in bandstructure.labels_dict.items()}

        new_bandstructure = BandStructureSymmLine(
            kpoints=new_kpoints,
            eigenvals=new_bands,
            lattice=bandstructure.lattice_rec,
            efermi=bandstructure.efermi,
            labels_dict=new_labels_dict,
            structure=bandstructure.structure,
            projections=new_projections,
        )

        return new_bandstructure
<commit_msg>Default path set to setyawan_curtarolo<commit_after># coding: utf-8
# Copyright (c) Pymatgen Development Team.
# Distributed under the terms of the MIT License.
"""
Provides a class for interacting with KPath classes to
generate high-symmetry k-paths using different conventions.
"""

import itertools
from warnings import warn

import networkx as nx
import numpy as np

from pymatgen.symmetry.kpath import (
    KPathBase,
    KPathLatimerMunro,
    KPathSeek,
    KPathSetyawanCurtarolo,
)

from pymatgen.electronic_structure.bandstructure import BandStructureSymmLine
from pymatgen.electronic_structure.core import Spin

__author__ = "Jason Munro"
__copyright__ = "Copyright 2020, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Jason Munro"
__email__ = "jmunro@lbl.gov"
__status__ = "Development"
__date__ = "March 2020"

class HighSymmKpath(KPathBase):
    """
    This class generates path along high symmetry lines in the
    Brillouin zone according to different conventions.
    The class is designed to be used with a specific primitive
    cell setting. The definitions for the primitive cell
    used can be found in: Computational Materials Science,
    49(2), 299-312. doi:10.1016/j.commatsci.2010.05.010.
    The space group analyzer can be used to produce the correct
    primitive structure
    (method get_primitive_standard_structure(international_monoclinic=False)).
    Ensure input structure is correct before 'get_kpoints()' method is used.
    See individual KPath classes for details on specific conventions.
    """

    def __init__(
        self,
        structure,
        has_magmoms=False,
        magmom_axis=None,
        path_type="setyawan_curtarolo",
        symprec=0.01,
        angle_tolerance=5,
        atol=1e-5,
    ):
        """
        Args:
            structure (Structure): Structure object
            has_magmoms (boolean): Whether the input structure contains
                magnetic moments as site properties with the key 'magmom.'
                Values may be in the form of 3-component vectors given in
                the basis of the input lattice vectors, in
                which case the spin axis will default to a_3, the third
                real-space lattice vector (this triggers a warning).
            magmom_axis (list or numpy array): 3-component vector specifying
                direction along which magnetic moments given as scalars
                should point. If all magnetic moments are provided as
                vectors then this argument is not used.
            path_type (string): Chooses which convention to use to generate
                the high symmetry path. Options are: 'setyawan_curtarolo', 'hinuma',
                'latimer_munro' for the Setyawan & Curtarolo, Hinuma et al., and
                Latimer & Munro conventions. Choosing 'all' will generate one path
                with points from all three conventions. Equivalent labels between
                each will also be generated. Order will always be Latimer & Munro,
                Setyawan & Curtarolo, and Hinuma et al. Lengths for each of the paths
                will also be generated and output as a list. Note for 'all' the user
                will have to alter the labels on their own for plotting.
            symprec (float): Tolerance for symmetry finding
            angle_tolerance (float): Angle tolerance for symmetry finding.
            atol (float): Absolute tolerance used to determine symmetric
                equivalence of points and lines on the BZ.
        """

        super().__init__(structure, symprec=symprec, angle_tolerance=angle_tolerance, atol=atol)

        self._path_type = path_type

        self._equiv_labels = None
        self._path_lengths = None
        self._label_index = None

        if path_type != "all":

            if path_type == "latimer_munro":
                self._kpath = self._get_lm_kpath(has_magmoms, magmom_axis, symprec, angle_tolerance, atol).kpath
            elif path_type == "setyawan_curtarolo":
                self._kpath = self._get_sc_kpath(symprec, angle_tolerance, atol).kpath
            elif path_type == "hinuma":
                hin_dat = self._get_hin_kpath(symprec, angle_tolerance, atol, not has_magmoms)
                self._kpath = hin_dat.kpath
                self._hin_tmat = hin_dat._tmat

        else:

            if has_magmoms:
                raise ValueError("Cannot select 'all' with non-zero magmoms.")

            lm_bs = self._get_lm_kpath(has_magmoms, magmom_axis, symprec, angle_tolerance, atol)
            rpg = lm_bs._rpg

            sc_bs = self._get_sc_kpath(symprec, angle_tolerance, atol)
            hin_bs = self._get_hin_kpath(symprec, angle_tolerance, atol, not has_magmoms)

            index = 0
            cat_points = {}
            label_index = {}
            num_path = []
            self._path_lengths = []

            for bs in [lm_bs, sc_bs, hin_bs]:
                for key, value in enumerate(bs.kpath["kpoints"]):
                    cat_points[index] = bs.kpath["kpoints"][value]
                    label_index[index] = value
                    index += 1

                total_points_path = 0
                for seg in bs.kpath["path"]:
                    total_points_path += len(seg)

                for block in bs.kpath["path"]:
                    new_block = []
                    for label in block:
                        for ind in range(
                            len(label_index) - len(bs.kpath["kpoints"]),
                            len(label_index),
                        ):
                            if label_index[ind] == label:
                                new_block.append(ind)

                    num_path.append(new_block)

                self._path_lengths.append(total_points_path)

            self._label_index = label_index

            self._kpath = {"kpoints": cat_points, "path": num_path}

            self._equiv_labels = self._get_klabels(lm_bs, sc_bs, hin_bs, rpg)

    @property
    def path_type(self):
        """
        Returns:
        The type of kpath chosen
        """
        return self._path_type

    @property
    def label_index(self):
        """
        Returns:
        The correspondance between numbers and kpoint symbols for the
        combined kpath generated when path_type = 'all'. None otherwise.
        """
        return self._label_index

    @property
    def equiv_labels(self):
        """
        Returns:
        The correspondance between the kpoint symbols in the Latimer and
        Munro convention, Setyawan and Curtarolo, and Hinuma
        conventions respectively. Only generated when path_type = 'all'.
        """
        return self._equiv_labels

    @property
    def path_lengths(self):
        """
        Returns:
        List of lengths of the Latimer and Munro, Setyawan and Curtarolo, and Hinuma
        conventions in the combined HighSymmKpath object when path_type = 'all' respectively.
        None otherwise.
        """
        return self._path_lengths

    def _get_lm_kpath(self, has_magmoms, magmom_axis, symprec, angle_tolerance, atol):
        """
        Returns:
        Latimer and Munro k-path with labels.
        """

        return KPathLatimerMunro(self._structure, has_magmoms, magmom_axis, symprec, angle_tolerance, atol)

    def _get_sc_kpath(self, symprec, angle_tolerance, atol):
        """
        Returns:
        Setyawan and Curtarolo k-path with labels.
        """
        kpath = KPathSetyawanCurtarolo(self._structure, symprec, angle_tolerance, atol)

        self.prim = kpath.prim
        self.conventional = kpath.conventional
        self.prim_rec = kpath.prim_rec
        self._rec_lattice = self.prim_rec

        return kpath

    def _get_hin_kpath(self, symprec, angle_tolerance, atol, tri):
        """
        Returns:
        Hinuma et al. k-path with labels.
        """
        bs = KPathSeek(self._structure, symprec, angle_tolerance, atol, tri)

        kpoints = bs.kpath["kpoints"]
        tmat = bs._tmat
        for key in kpoints:
            kpoints[key] = np.dot(np.transpose(np.linalg.inv(tmat)), kpoints[key])

        bs.kpath["kpoints"] = kpoints
        self._rec_lattice = self._structure.lattice.reciprocal_lattice

        warn(
            "K-path from the Hinuma et al. convention has been transformed to the basis of the reciprocal lattice \
of the input structure. Use `KPathSeek` for the path in the original author-intended basis."
        )

        return bs

    def _get_klabels(self, lm_bs, sc_bs, hin_bs, rpg):
        """
        Returns:
        labels (dict): Dictionary of equivalent labels for paths if 'all' is chosen.
            If an exact kpoint match cannot be found, symmetric equivalency will be
            searched for and indicated with an asterisk in the equivalent label.
            If an equivalent label can still not be found, or the point is not in
            the explicit kpath, its equivalent label will be set to itself in the output.
        """

        lm_path = lm_bs.kpath
        sc_path = sc_bs.kpath
        hin_path = hin_bs.kpath

        n_op = len(rpg)

        pairs = itertools.permutations([{"setyawan_curtarolo": sc_path}, {"latimer_munro": lm_path}, {"hinuma": hin_path}], r=2)
        labels = {"setyawan_curtarolo": {}, "latimer_munro": {}, "hinuma": {}}

        for (a, b) in pairs:
            [(a_type, a_path)] = list(a.items())
            [(b_type, b_path)] = list(b.items())

            sc_count = np.zeros(n_op)

            for o_num in range(0, n_op):
                a_tr_coord = []

                for (label_a, coord_a) in a_path["kpoints"].items():
                    a_tr_coord.append(np.dot(rpg[o_num], coord_a))

                for coord_a in a_tr_coord:
                    for key, value in b_path["kpoints"].items():
                        if np.allclose(value, coord_a, atol=self._atol):
                            sc_count[o_num] += 1
                            break

            a_to_b_labels = {}
            unlabeled = {}

            for (label_a, coord_a) in a_path["kpoints"].items():
                coord_a_t = np.dot(rpg[np.argmax(sc_count)], coord_a)
                assigned = False

                for (label_b, coord_b) in b_path["kpoints"].items():
                    if np.allclose(coord_b, coord_a_t, atol=self._atol):
                        a_to_b_labels[label_a] = label_b
                        assigned = True
                        break

                if not assigned:
                    unlabeled[label_a] = coord_a

            for (label_a, coord_a) in unlabeled.items():
                for op in rpg:
                    coord_a_t = np.dot(op, coord_a)
                    key = [
                        key
                        for key, value in b_path["kpoints"].items()
                        if np.allclose(value, coord_a_t, atol=self._atol)
                    ]

                    if key != []:
                        a_to_b_labels[label_a] = key[0][0] + "^{*}"
                        break

                if key == []:
                    a_to_b_labels[label_a] = label_a

            labels[a_type][b_type] = a_to_b_labels

        return labels

    @staticmethod
    def get_continuous_path(bandstructure):
        """
        Obtain a continous version of an inputted path using graph theory.
        This routine will attempt to add connections between nodes of
        odd-degree to ensure a Eulerian path can be formed. Initial
        k-path must be able to be converted to a connected graph. See
        npj Comput Mater 6, 112 (2020). 10.1038/s41524-020-00383-7
        for more details.

        Args:
        bandstructure (BandstructureSymmLine): BandstructureSymmLine object.

        Returns:
        bandstructure (BandstructureSymmLine): New BandstructureSymmLine object with continous path.
        """

        G = nx.Graph()

        labels = []
        for point in bandstructure.kpoints:
            if point.label is not None:
                labels.append(point.label)

        plot_axis = []
        for i in range(int(len(labels) / 2)):
            G.add_edges_from([(labels[2 * i], labels[(2 * i) + 1])])
            plot_axis.append((labels[2 * i], labels[(2 * i) + 1]))

        G_euler = nx.algorithms.euler.eulerize(G)

        G_euler_circuit = nx.algorithms.euler.eulerian_circuit(G_euler)

        distances_map = []
        kpath_euler = []

        for edge_euler in G_euler_circuit:
            kpath_euler.append(edge_euler)
            for edge_reg in plot_axis:
                if edge_euler == edge_reg:
                    distances_map.append((plot_axis.index(edge_reg), False))
                elif edge_euler[::-1] == edge_reg:
                    distances_map.append((plot_axis.index(edge_reg), True))

        if bandstructure.is_spin_polarized:
            spins = [Spin.up, Spin.down]
        else:
            spins = [Spin.up]

        new_kpoints = []
        new_bands = {spin: [np.array([]) for _ in range(bandstructure.nb_bands)] for spin in spins}
        new_projections = {spin: [[] for _ in range(bandstructure.nb_bands)] for spin in spins}

        for entry in distances_map:
            if not entry[1]:
                branch = bandstructure.branches[entry[0]]
                start = branch["start_index"]
                stop = branch["end_index"] + 1
                step = 1

            else:
                branch = bandstructure.branches[entry[0]]
                start = branch["end_index"]
                stop = branch["start_index"] - 1
                step = -1

            # kpoints
            new_kpoints += [point.frac_coords for point in bandstructure.kpoints[start:stop:step]]

            # eigenvals
            for spin in spins:
                for n, band in enumerate(bandstructure.bands[spin]):

                    new_bands[spin][n] = np.concatenate((new_bands[spin][n], band[start:stop:step]))

            # projections
            for spin in spins:
                for n, band in enumerate(bandstructure.projections[spin]):

                    new_projections[spin][n] += band[start:stop:step].tolist()

        for spin in spins:
            new_projections[spin] = np.array(new_projections[spin])

        new_labels_dict = {label: point.frac_coords for label, point in bandstructure.labels_dict.items()}

        new_bandstructure = BandStructureSymmLine(
            kpoints=new_kpoints,
            eigenvals=new_bands,
            lattice=bandstructure.lattice_rec,
            efermi=bandstructure.efermi,
            labels_dict=new_labels_dict,
            structure=bandstructure.structure,
            projections=new_projections,
        )

        return new_bandstructure
<|endoftext|>


<commit_before>from openerp import fields, models, api

class Benefit(models.Model):
    _name = 'mqo.benefit'

    name = fields.Char(string="Benefit")
    category =  fields.Selection([('individual', 'Individual'),
                               ('team', 'Team'),
                               ('leader', 'Leader'),
                               ('organisation', 'Organisation')],
                               string='Category')
    program_links = fields.One2many('mqo.benefit_program_link', 'benefit', string="program_links")

class BenefitProgramLink(models.Model):
    _name = 'mqo.benefit_program_link'

    program = fields.Many2one('mqo.program', string="Program")
    benefit = fields.Many2one('mqo.benefit', string="Benefit")
    sequence = fields.Integer(string="Sequence")

<commit_msg>[FIX] Correct sequence of benefit links.<commit_after>from openerp import fields, models, api

class Benefit(models.Model):
    _name = 'mqo.benefit'

    name = fields.Char(string="Benefit")
    category =  fields.Selection([('individual', 'Individual'),
                               ('team', 'Team'),
                               ('leader', 'Leader'),
                               ('organisation', 'Organisation')],
                               string='Category')
    program_links = fields.One2many('mqo.benefit_program_link', 'benefit', string="program_links")

class BenefitProgramLink(models.Model):
    _name = 'mqo.benefit_program_link'

    program = fields.Many2one('mqo.program', string="Program")
    benefit = fields.Many2one('mqo.benefit', string="Benefit")
    sequence = fields.Integer(string="Sequence")

    _order = "program, sequence"
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
This package contains the python code editor widget
"""
import sys
from pyqode.core.api import ColorScheme
from pyqode.python.backend import server
from pyqode.qt import QtCore, QtGui
from pyqode.core import api
from pyqode.core import modes
from pyqode.core import panels
from pyqode.python import managers as pymanagers
from pyqode.python import modes as pymodes
from pyqode.python import panels as pypanels
from pyqode.python.folding import PythonFoldDetector

class PyCodeEditBase(api.CodeEdit):
    """
    Base class for creating a python code editor widget. The base class
    takes care of setting up the syntax highlighter.

    .. note:: This code editor widget use PEP 0263 to detect file encoding.
              If the opened file does not respects the PEP 0263,
              :py:func:`locale.getpreferredencoding` is used as the default
              encoding.
    """

    def __init__(self, parent=None, create_default_actions=True,
                 color_scheme='qt'):
        super(PyCodeEditBase, self).__init__(parent, create_default_actions)
        self.file = pymanagers.PyFileManager(self)
        self.modes.append(pymodes.PythonSH(
            self.document(), color_scheme=ColorScheme(color_scheme)))

    def setPlainText(self, txt, mimetype='text/x-python', encoding='utf-8'):
        """
        Extends QCodeEdit.setPlainText to allow user to setPlainText without
        mimetype (since the python syntax highlighter does not use it).
        """
        self.syntax_highlighter.docstrings[:] = []
        self.syntax_highlighter.import_statements[:] = []
        super(PyCodeEditBase, self).setPlainText(txt, mimetype, encoding)

class PyCodeEdit(PyCodeEditBase):
    """
    Extends PyCodeEditBase with a set of hardcoded modes and panels specifics
    to a python code editor widget.
    """
    DARK_STYLE = 0
    LIGHT_STYLE = 1

    mimetypes = ['text/x-python']

    def __init__(self, parent=None, server_script=server.__file__,
                 interpreter=sys.executable, args=None,
                 create_default_actions=True, color_scheme='qt'):
        super(PyCodeEdit, self).__init__(
            parent=parent, create_default_actions=create_default_actions,
            color_scheme=color_scheme)
        self.backend.start(server_script, interpreter, args)
        self.setLineWrapMode(self.NoWrap)
        self.setWindowTitle("pyQode - Python Editor")

        # install those modes first as they are required by other modes/panels
        self.modes.append(pymodes.DocumentAnalyserMode())

        # panels
        self.panels.append(panels.FoldingPanel())
        self.panels.append(panels.LineNumberPanel())
        self.panels.append(panels.CheckerPanel())
        self.panels.append(panels.GlobalCheckerPanel(),
                           panels.GlobalCheckerPanel.Position.RIGHT)
        self.panels.append(panels.SearchAndReplacePanel(),
                           panels.SearchAndReplacePanel.Position.BOTTOM)
        self.panels.append(panels.EncodingPanel(), api.Panel.Position.TOP)
        self.add_separator()
        self.panels.append(pypanels.QuickDocPanel(), api.Panel.Position.BOTTOM)

        # modes
        # generic
        self.modes.append(modes.CaretLineHighlighterMode())
        self.modes.append(modes.FileWatcherMode())
        self.modes.append(modes.RightMarginMode())
        self.modes.append(modes.ZoomMode())
        self.modes.append(modes.SymbolMatcherMode())
        self.modes.append(modes.CodeCompletionMode())
        self.modes.append(modes.OccurrencesHighlighterMode())
        self.modes.append(modes.SmartBackSpaceMode())
        self.modes.append(modes.ExtendedSelectionMode())
        # python specifics
        self.modes.append(pymodes.PyAutoIndentMode())
        self.modes.append(pymodes.PyAutoCompleteMode())
        self.modes.append(pymodes.FrostedCheckerMode())
        self.modes.append(pymodes.PEP8CheckerMode())
        self.modes.append(pymodes.CalltipsMode())
        self.modes.append(pymodes.PyIndenterMode())
        self.modes.append(pymodes.GoToAssignmentsMode())
        self.modes.append(pymodes.CommentsMode())
        self.syntax_highlighter.fold_detector = PythonFoldDetector()

    def clone(self):
        clone = self.__class__(
            parent=self.parent(), server_script=self.backend.server_script,
            interpreter=self.backend.interpreter, args=self.backend.args,
            color_scheme=self.syntax_highlighter.color_scheme.name)
        return clone

    def __repr__(self):
        return 'PyCodeEdit(path=%r)' % self.file.path
<commit_msg>Fix PythonSH install order in order to fix caret line color<commit_after># -*- coding: utf-8 -*-
"""
This package contains the python code editor widget
"""
import sys
from pyqode.core.api import ColorScheme
from pyqode.python.backend import server
from pyqode.qt import QtCore, QtGui
from pyqode.core import api
from pyqode.core import modes
from pyqode.core import panels
from pyqode.python import managers as pymanagers
from pyqode.python import modes as pymodes
from pyqode.python import panels as pypanels
from pyqode.python.folding import PythonFoldDetector

class PyCodeEditBase(api.CodeEdit):
    """
    Base class for creating a python code editor widget. The base class
    takes care of setting up the syntax highlighter.

    .. note:: This code editor widget use PEP 0263 to detect file encoding.
              If the opened file does not respects the PEP 0263,
              :py:func:`locale.getpreferredencoding` is used as the default
              encoding.
    """

    def __init__(self, parent=None, create_default_actions=True,
                 color_scheme='qt'):
        super(PyCodeEditBase, self).__init__(parent, create_default_actions)
        self.file = pymanagers.PyFileManager(self)

    def setPlainText(self, txt, mimetype='text/x-python', encoding='utf-8'):
        """
        Extends QCodeEdit.setPlainText to allow user to setPlainText without
        mimetype (since the python syntax highlighter does not use it).
        """
        self.syntax_highlighter.docstrings[:] = []
        self.syntax_highlighter.import_statements[:] = []
        super(PyCodeEditBase, self).setPlainText(txt, mimetype, encoding)

class PyCodeEdit(PyCodeEditBase):
    """
    Extends PyCodeEditBase with a set of hardcoded modes and panels specifics
    to a python code editor widget.
    """
    DARK_STYLE = 0
    LIGHT_STYLE = 1

    mimetypes = ['text/x-python']

    def __init__(self, parent=None, server_script=server.__file__,
                 interpreter=sys.executable, args=None,
                 create_default_actions=True, color_scheme='qt'):
        super(PyCodeEdit, self).__init__(
            parent=parent, create_default_actions=create_default_actions,
            color_scheme=color_scheme)
        self.backend.start(server_script, interpreter, args)
        self.setLineWrapMode(self.NoWrap)
        self.setWindowTitle("pyQode - Python Editor")

        # install those modes first as they are required by other modes/panels
        self.modes.append(pymodes.DocumentAnalyserMode())

        # panels
        self.panels.append(panels.FoldingPanel())
        self.panels.append(panels.LineNumberPanel())
        self.panels.append(panels.CheckerPanel())
        self.panels.append(panels.GlobalCheckerPanel(),
                           panels.GlobalCheckerPanel.Position.RIGHT)
        self.panels.append(panels.SearchAndReplacePanel(),
                           panels.SearchAndReplacePanel.Position.BOTTOM)
        self.panels.append(panels.EncodingPanel(), api.Panel.Position.TOP)
        self.add_separator()
        self.panels.append(pypanels.QuickDocPanel(), api.Panel.Position.BOTTOM)

        # modes
        # generic
        self.modes.append(modes.CaretLineHighlighterMode())
        self.modes.append(modes.FileWatcherMode())
        self.modes.append(modes.RightMarginMode())
        self.modes.append(modes.ZoomMode())
        self.modes.append(modes.SymbolMatcherMode())
        self.modes.append(modes.CodeCompletionMode())
        self.modes.append(modes.OccurrencesHighlighterMode())
        self.modes.append(modes.SmartBackSpaceMode())
        self.modes.append(modes.ExtendedSelectionMode())
        # python specifics
        self.modes.append(pymodes.PyAutoIndentMode())
        self.modes.append(pymodes.PyAutoCompleteMode())
        self.modes.append(pymodes.FrostedCheckerMode())
        self.modes.append(pymodes.PEP8CheckerMode())
        self.modes.append(pymodes.CalltipsMode())
        self.modes.append(pymodes.PyIndenterMode())
        self.modes.append(pymodes.GoToAssignmentsMode())
        self.modes.append(pymodes.CommentsMode())
        self.modes.append(pymodes.PythonSH(
            self.document(), color_scheme=ColorScheme(color_scheme)))
        self.syntax_highlighter.fold_detector = PythonFoldDetector()

    def clone(self):
        clone = self.__class__(
            parent=self.parent(), server_script=self.backend.server_script,
            interpreter=self.backend.interpreter, args=self.backend.args,
            color_scheme=self.syntax_highlighter.color_scheme.name)
        return clone

    def __repr__(self):
        return 'PyCodeEdit(path=%r)' % self.file.path
<|endoftext|>


<commit_before># Copyright 2020 DeepMind Technologies Limited. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Delaying object constructions."""

import abc
from typing import Callable, Generic, TypeVar, Union

import tree

T = TypeVar('T')

class _Uninitialized:
  pass

class Dereferenceable(Generic[T], abc.ABC):

  @abc.abstractmethod
  def dereference(self) -> T:
    """Dereferences itself."""

def maybe_dereference(obj: Union[T, Dereferenceable[T]]) -> T:
  if isinstance(obj, Dereferenceable):
    return obj.dereference()
  return obj

_EXCEPTION_MESSAGE = ('Error occurred when evaluating Deferred defined at:\n'
                      '{init_stack}\n')

class Deferred(Dereferenceable[T], Generic[T]):
  """Delays object construction to achieve serializability.

  Assuming we want to pass a non-serializable Python object, say an environment
  object, to an Actor, the following will lead to a serialization error:

      program.add_node(lp.CourierNode(Actor, envloader.load_from_settings(
          platform='Atari',
          settings={
              'levelName': 'pong',
              'env_loader.version': requested_version,
              'zero_indexed_actions': True,
              'interleaved_pixels': True,
          })))

  This helper class delays the object construction and fixes this error. The
  object is constructed when the Actor is actually instantiated remotely, where
  the Actor constructed will receive an actual environment object (just like how
  handles are dereferenced automatically):

      program.add_node(lp.CourierNode(Actor, lp.Deferred(
          envloader.load_from_settings,
          platform='Atari',
          settings={
              'levelName': 'pong',
              'env_loader.version': requested_version,
              'zero_indexed_actions': True,
              'interleaved_pixels': True,
          })))
  """

  def __init__(self, constructor: Callable[..., T], *args, **kwargs) -> None:
    self._constructor = constructor
    self._args = args
    self._kwargs = kwargs
    self._init_stack = 'Stack trace missing'
    self._deferred_object = _Uninitialized()

  def dereference(self) -> T:
    if isinstance(self._deferred_object, _Uninitialized):
      args, kwargs = tree.map_structure(maybe_dereference,
                                        (self._args, self._kwargs))
      try:
        self._deferred_object = self._constructor(*args, **kwargs)
      except:
        new_message = _EXCEPTION_MESSAGE.format(
            init_stack=''.join(self._init_stack))
        raise RuntimeError(new_message)

    return self._deferred_object

  def _apply_to_args(self, fn):
    return tree.map_structure(fn, (self._args, self._kwargs))
<commit_msg>Chain runtime errors with errors raised from constructing the deferred object.<commit_after># Copyright 2020 DeepMind Technologies Limited. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Delaying object constructions."""

import abc
from typing import Callable, Generic, TypeVar, Union

import tree

T = TypeVar('T')

class _Uninitialized:
  pass

class Dereferenceable(Generic[T], abc.ABC):

  @abc.abstractmethod
  def dereference(self) -> T:
    """Dereferences itself."""

def maybe_dereference(obj: Union[T, Dereferenceable[T]]) -> T:
  if isinstance(obj, Dereferenceable):
    return obj.dereference()
  return obj

_EXCEPTION_MESSAGE = ('Error ({error_msg}) occurred when evaluating Deferred '
                      'defined at:\n{init_stack}\n')

class Deferred(Dereferenceable[T], Generic[T]):
  """Delays object construction to achieve serializability.

  Assuming we want to pass a non-serializable Python object, say an environment
  object, to an Actor, the following will lead to a serialization error:

      program.add_node(lp.CourierNode(Actor, envloader.load_from_settings(
          platform='Atari',
          settings={
              'levelName': 'pong',
              'env_loader.version': requested_version,
              'zero_indexed_actions': True,
              'interleaved_pixels': True,
          })))

  This helper class delays the object construction and fixes this error. The
  object is constructed when the Actor is actually instantiated remotely, where
  the Actor constructed will receive an actual environment object (just like how
  handles are dereferenced automatically):

      program.add_node(lp.CourierNode(Actor, lp.Deferred(
          envloader.load_from_settings,
          platform='Atari',
          settings={
              'levelName': 'pong',
              'env_loader.version': requested_version,
              'zero_indexed_actions': True,
              'interleaved_pixels': True,
          })))
  """

  def __init__(self, constructor: Callable[..., T], *args, **kwargs) -> None:
    self._constructor = constructor
    self._args = args
    self._kwargs = kwargs
    self._init_stack = 'Stack trace missing'
    self._deferred_object = _Uninitialized()

  def dereference(self) -> T:
    if isinstance(self._deferred_object, _Uninitialized):
      args, kwargs = tree.map_structure(maybe_dereference,
                                        (self._args, self._kwargs))
      try:
        self._deferred_object = self._constructor(*args, **kwargs)
      except Exception as e:
        new_message = _EXCEPTION_MESSAGE.format(
            init_stack=''.join(self._init_stack),
            # For clarity during pdb, we also inline the internal error message.
            error_msg=str(e))
        raise RuntimeError(new_message) from e

    return self._deferred_object

  def _apply_to_args(self, fn):
    return tree.map_structure(fn, (self._args, self._kwargs))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import warnings
from pyramid.exceptions import ConfigurationError

__version__ = '0.1'

def get_category(settings, key):
    category = settings.get(key, 'Warning')
    warnings_cls = {'Warning': Warning,
                    'UserWarning': UserWarning,
                    'DeprecationWarning': DeprecationWarning,
                    'SyntaxWarning': SyntaxWarning,
                    'RuntimeWarning': RuntimeWarning,
                    'FutureWarning': FutureWarning,
                    'PendingDeprecationWarning': PendingDeprecationWarning,
                    'ImportWarning': ImportWarning,
                    'UnicodeWarning': UnicodeWarning,
                    }
    try:
        return warnings_cls[category]
    except KeyError:
        raise ConfigurationError('Invalid configuration value '
                                 'for "{0}"'.format(key))

def includeme(config):

    settings = config.registry.settings
    key = 'filterwarnings.category'
    warnings.filterwarnings(settings.get('filterwarnings.action', 'ignore'),
                            settings.get('filterwarnings.message', ''),
                            get_category(settings, key),
                            settings.get('filterwarnings.module', '')
                            )

    index = 1
    while True:
        prefix = 'filterwarnings.{0}'.format(index)
        action = settings.get('{0}.action'.format(prefix))
        if not action:
            break
        key = '{0}.category'.format(prefix)
        warnings.filterwarnings(action,
                                settings.get('{0}.message'.format(prefix), ''),
                                get_category(settings, key),
                                settings.get('{0}.module'.format(prefix), ''),
                                append=True
                                )
        index += 1
<commit_msg>Bump new development version number<commit_after># -*- coding: utf-8 -*-
import warnings
from pyramid.exceptions import ConfigurationError

__version__ = '0.2~dev'

def get_category(settings, key):
    category = settings.get(key, 'Warning')
    warnings_cls = {'Warning': Warning,
                    'UserWarning': UserWarning,
                    'DeprecationWarning': DeprecationWarning,
                    'SyntaxWarning': SyntaxWarning,
                    'RuntimeWarning': RuntimeWarning,
                    'FutureWarning': FutureWarning,
                    'PendingDeprecationWarning': PendingDeprecationWarning,
                    'ImportWarning': ImportWarning,
                    'UnicodeWarning': UnicodeWarning,
                    }
    try:
        return warnings_cls[category]
    except KeyError:
        raise ConfigurationError('Invalid configuration value '
                                 'for "{0}"'.format(key))

def includeme(config):

    settings = config.registry.settings
    key = 'filterwarnings.category'
    warnings.filterwarnings(settings.get('filterwarnings.action', 'ignore'),
                            settings.get('filterwarnings.message', ''),
                            get_category(settings, key),
                            settings.get('filterwarnings.module', '')
                            )

    index = 1
    while True:
        prefix = 'filterwarnings.{0}'.format(index)
        action = settings.get('{0}.action'.format(prefix))
        if not action:
            break
        key = '{0}.category'.format(prefix)
        warnings.filterwarnings(action,
                                settings.get('{0}.message'.format(prefix), ''),
                                get_category(settings, key),
                                settings.get('{0}.module'.format(prefix), ''),
                                append=True
                                )
        index += 1
<|endoftext|>


<commit_before>'''
Created on 27 may 2014

@author: Scondo
'''
import unittest
import pngsuite
from PIL import Image
from io import BytesIO

from PIL import PngImagePlugin as pilpng
from png import PngImagePlugin as purepng

try:
    reload
except NameError:
    from imp import reload

class PilImageToPyPngAdapter:
    def __init__(self, im):
        self.im = im

    def __len__(self):
        return self.im.size[1]

    def __getitem__(self, row):
        out = []
        for col in range(self.im.size[0]):
            px = self.im.getpixel((col, row))
            if hasattr(px, '__iter__'):
                #Multi-channel image
                out.extend(px)
            else:
                #Single channel image
                out.append(px)
        return out

class BaseTest(unittest.TestCase):
    test_ = None

    def compareImages(self, im1, im2):
        if 'gamma' in im1.info:
            self.assertEqual(im1.info['gamma'], im2.info.get('gamma'))
        if im1.mode != im2.mode or im1.mode == 'P':
            im1 = im1.convert('RGBA')
            im2 = im2.convert('RGBA')
        pix1 = list(PilImageToPyPngAdapter(im1))
        pix2 = list(PilImageToPyPngAdapter(im2))
        if im1.mode == 'RGBA':
            self.assertEqual(pix1[0][3::4], pix2[0][3::4])  # alpha fast check
        self.assertEqual(pix1[0], pix2[0])  # fast check
        self.assertEqual(pix1, pix2)  # complete check

class ReadTest(BaseTest):
    def runTest(self):
        if self.test_ is None:
            return
        test_file = BytesIO(self.test_)
        # Load via PurePNG
        reload(purepng)
        im_pure = Image.open(test_file)
        im_pure.load()
        test_file.seek(0)
        # Load via PIL default plugin
        reload(pilpng)
        im_pil = Image.open(test_file)
        self.compareImages(im_pil, im_pure)

class WriteTest(BaseTest):
    def runTest(self):
        if self.test_ is None:
            return
        test_file = BytesIO(self.test_)
        # Load via PIL default plugin
        reload(pilpng)
        im_orig = Image.open(test_file)
        # Save via PurePNG
        reload(purepng)
        pure_file = BytesIO()
        im_orig.save(pure_file, 'PNG')
        # Load again, plugin unimportant after read test
        pure_file.seek(0)
        im_new = Image.open(pure_file)
        self.compareImages(im_orig, im_new)

# Generate tests for each suite file
# Except known cases when fail caused by PIL
testsuite = pngsuite.png
del testsuite['tbbn1g04']
# grayscale + transparency does not provide alpha
del testsuite['Basn0g03']
# PIL ignore sBIT on 4bit greyscale, PurePNG provide more accuracy
for tname_, test_ in (testsuite.items()):
    globals()[tname_ + '_rtest'] = type(tname_ + '_rtest', (ReadTest,),
                                       {'test_': test_})
    globals()[tname_ + '_wtest'] = type(tname_ + '_wtest', (WriteTest,),
                                       {'test_': test_})

if __name__ == "__main__":
    unittest.main()
<commit_msg>Rework testing iterator to iterator protocol in addition to getitem<commit_after>'''
Created on 27 may 2014

@author: Scondo
'''
import unittest
import pngsuite
from PIL import Image
from io import BytesIO

from PIL import PngImagePlugin as pilpng
from png import PngImagePlugin as purepng

try:
    reload
except NameError:
    from imp import reload

class PilImageToPyPngAdapter:
    def __init__(self, im):
        self.im = im
        self.nowrow = 0

    def __len__(self):
        return self.im.size[1]

    def next(self):
        if self.nowrow >= self.__len__():
            raise StopIteration()
        else:
            self.nowrow += 1
            return self.__getitem__(self.nowrow)

    def __iter__(self):
        return self

    def __getitem__(self, row):
        out = []
        for col in range(self.im.size[0]):
            px = self.im.getpixel((col, row))
            if hasattr(px, '__iter__'):
                # Multi-channel image
                out.extend(px)
            else:
                # Single channel image
                out.append(px)
        return out

class BaseTest(unittest.TestCase):
    test_ = None

    def compareImages(self, im1, im2):
        if 'gamma' in im1.info:
            self.assertEqual(im1.info['gamma'], im2.info.get('gamma'))
        if im1.mode != im2.mode or im1.mode == 'P':
            im1 = im1.convert('RGBA')
            im2 = im2.convert('RGBA')
        pix1 = PilImageToPyPngAdapter(im1)
        pix2 = PilImageToPyPngAdapter(im2)
        if im1.mode == 'RGBA':
            self.assertEqual(pix1[0][3::4], pix2[0][3::4])  # alpha fast check
        self.assertEqual(pix1[0], pix2[0])  # fast check
        self.assertEqual(list(pix1), list(pix2))  # complete check

class ReadTest(BaseTest):
    def runTest(self):
        if self.test_ is None:
            return
        test_file = BytesIO(self.test_)
        # Load via PurePNG
        reload(purepng)
        im_pure = Image.open(test_file)
        im_pure.load()
        test_file.seek(0)
        # Load via PIL default plugin
        reload(pilpng)
        im_pil = Image.open(test_file)
        self.compareImages(im_pil, im_pure)

class WriteTest(BaseTest):
    def runTest(self):
        if self.test_ is None:
            return
        test_file = BytesIO(self.test_)
        # Load via PIL default plugin
        reload(pilpng)
        im_orig = Image.open(test_file)
        # Save via PurePNG
        reload(purepng)
        pure_file = BytesIO()
        im_orig.save(pure_file, 'PNG')
        # Load again, plugin unimportant after read test
        pure_file.seek(0)
        im_new = Image.open(pure_file)
        self.compareImages(im_orig, im_new)

# Generate tests for each suite file
# Except known cases when fail caused by PIL
testsuite = pngsuite.png
del testsuite['tbbn1g04']
# grayscale + transparency does not provide alpha
del testsuite['Basn0g03']
# PIL ignore sBIT on 4bit greyscale, PurePNG provide more accuracy
for tname_, test_ in (testsuite.items()):
    globals()[tname_ + '_rtest'] = type(tname_ + '_rtest', (ReadTest,),
                                       {'test_': test_})
    globals()[tname_ + '_wtest'] = type(tname_ + '_wtest', (WriteTest,),
                                       {'test_': test_})

if __name__ == "__main__":
    unittest.main()
<|endoftext|>