<commit_before>from sklearn import cross_validation
from readfacedatabases import *
import copy

# TODO: move to common?
def splitTrainTest(data1, data2, labels1, labels2, ratio):
  assert len(data1) == len(data2)
  assert len(labels1) == len(labels2)
  assert len(labels1) == len(data1)
  assert len(labels1) == len(labels2)

  data1, data2, labels1, labels2 = shuffle(data1, data2, labels1, labels2)

  # Random data for training and testing
  kf = cross_validation.KFold(n=len(data1), n_folds=ratio)
  for train, test in kf:
    break

  return (data1[train], data1[test], data2[train], data2[test],
          labels1[train], labels1[test], labels2[train], labels2[test])

def splitShuffling(shuffling, labelsShuffling):

  shuffling, labelsShuffling = shuffle(shuffling, labelsShuffling)

  labels = np.unique(labelsShuffling)

  # TODO: we already had this? maybe not remake it
  labelsToData = {}
  for label in labels:
    labelsToData[label] = list(shuffling[labels == label])
    print len(labelsToData[label])

  shuffledData1 = []
  shuffledData2 = []
  labelsData1 = []
  labelsData2 = []

  currentLabels = list(labels)

  while len(shuffledData1) + len(shuffledData2) <= len(shuffling):
    print "len(shuffledData1) + len(shuffledData2)"
    print len(shuffledData1) + len(shuffledData2)
    print "len(shuffling)"
    print len(shuffling)

    print "currentLabels"
    print currentLabels

    label1 = np.random.choice(currentLabels, 1, replace=False)
    label1 = label1[0]
    print "label1"
    print label1
    copyCurrentLabels = list(currentLabels)
    copyCurrentLabels.remove(label1)

    if not copyCurrentLabels:
      break

    label2 = np.random.choice(np.array(copyCurrentLabels), 1, replace=False)
    label2 = label2[0]
    print "label2"
    print label2

    print label1
    print label2

    if label1 == label2:
      print " in first if"
      continue

    # print "labelsToData"
    # print labelsToData

    dataLabel1 = labelsToData[label1]
    dataLabel2 = labelsToData[label2]
    if len(dataLabel1) == 0:
      print "in second if"
      currentLabels.remove(label1)
      continue
    if len(dataLabel2) == 0:
      print "in second if"
      currentLabels.remove(label2)
      continue

    # data1LabelIndex = np.random.choice(len(labelsToData[label1]))
    # data2LabelIndex = np.random.choice(len(labelsToData[label2]))
    shuffledData1 += [dataLabel1[0]]
    shuffledData2 += [dataLabel2[0]]
    labelsData1 += [label1]
    labelsData2 += [label2]

    del labelsToData[label1][0]
    del labelsToData[label2][0]

    # print "labelsToData"
    # print labelsToData

  shuffledData1 = np.vstack(shuffledData1)
  shuffledData2 = np.vstack(shuffledData2)

  labelsData1 = np.hstack(labelsData1)
  labelsData2 = np.hstack(labelsData2)

  # remaing = list(shuffling)
  # remaininLabels = list(labelsShuffling)

  # print "shuffling size"
  # print len(shuffling)

  # shuffledData1 = []
  # shuffledData2 = []
  # labelsData1 = []
  # labelsData2 = []

  # for label in labels:
  #   print "label"
  #   print label

  #   nrRemainingData = len(remaing)

  #   if nrRemainingData == 0:
  #     break

  #   labelIndices = np.array(remaininLabels) == label
  #   concreteIndices = np.arange(nrRemainingData)[labelIndices]

  #   # If nothing of this label is left, just continue
  #   if len(concreteIndices) == 0:
  #     continue

  #   otherIndices = np.arange(nrRemainingData)[np.invert(labelIndices)]

  #   if len(otherIndices) == 0:
  #     continue

  #   indicesToTake = min(len(concreteIndices), len(otherIndices))

  #   otherIndices = np.random.choice(otherIndices, indicesToTake, replace=False)

  #   # concreteData = np.array(remaing)[concreteIndices]

  #   shuffledData1 += [np.array(remaing)[concreteIndices]]
  #   labelsData1 += [np.array(remaininLabels)[concreteIndices]]

  #   shuffledData2 += [np.array(remaing)[otherIndices]]
  #   labelsData2 += [np.array(remaininLabels)[otherIndices]]

  #   indicesToRemove = np.hstack((otherIndices, concreteIndices))
  #   remaing = [v for i, v in enumerate(remaing) if i not in indicesToRemove]
  #   remaininLabels = [v for i, v in enumerate(remaininLabels) if i not in indicesToRemove]

  #   assert len(remaing) == len(remaininLabels)

  # shuffledData1 = np.vstack(shuffledData1)
  # shuffledData2 = np.vstack(shuffledData2)

  # labelsData1 = np.hstack(labelsData1)
  # labelsData2 = np.hstack(labelsData2)

  # print shuffledData1.shape
  # print shuffledData2.shape

  # assert len(shuffledData1) == len(shuffledData2)
  # assert len(labelsData1) == len(labelsData2)

  # assert len(shuffledData1) <= len(shuffling) / 2

  """  STOP NEW METHOD """
  # shuffledData1 = shuffling[0: len(shuffling) / 2]
  # shuffledData2 = shuffling[len(shuffling)/2 :]

  # labelsData1 = labelsShuffling[0: len(shuffling) /2]
  # labelsData2 = labelsShuffling[len(shuffling)/2:]

  # shuffledData1 = np.array(shuffledData1)
  # shuffledData1 = np.array(shuffledData2)
  # labelsData1 = np.array(labelsData1)
  # labelsData2 = np.array(labelsData2)

  return shuffledData1, shuffledData2, labelsData1, labelsData2

# TODO: I think this can be written easier with the code similar to the Emotions one
# you can create more tuples than just one per image
# you can put each image in 5 tuples and that will probably owrk better
# it might be useful to also give the same image twice
def splitDataMultiPIESubject(imgsPerLabel=None):
  subjectsToImgs = readMultiPIESubjects()

  data1, data2, subjects1, subjects2, shuffling, subjectsShuffling = splitDataInPairsWithLabels(subjectsToImgs, imgsPerLabel, None)

  trainData1, testData1, trainData2, testData2, trainSubjects1, testSubjects1,\
        trainSubjects2, testSubjects2 = splitTrainTest(data1, data2, subjects1, subjects2, 5)

  print "trainData1.shape"
  print trainData1.shape

  shuffledData1, shuffledData2, subjectsData1, subjectsData2 = splitShuffling(shuffling, subjectsShuffling)

  print len(shuffledData1)
  print len(shuffledData2)

  trainShuffedData1, testShuffedData1, trainShuffedData2, testShuffedData2,\
    trainShuffledSubjects1, testShuffledSubjects1, trainShuffledSubjects2, testShuffledSubjects2 =\
        splitTrainTest(shuffledData1, shuffledData2,
                      subjectsData1, subjectsData2, 5)

  trainData1 = np.vstack((trainData1, trainShuffedData1))

  print "trainData1.shape"
  print trainData1.shape

  trainData2 = np.vstack((trainData2, trainShuffedData2))

  testData1 = np.vstack((testData1, testShuffedData1))
  testData2 = np.vstack((testData2, testShuffedData2))

  trainSubjects1 = np.hstack((trainSubjects1, trainShuffledSubjects1))
  testSubjects1 = np.hstack((testSubjects1, testShuffledSubjects1))

  trainSubjects2 = np.hstack((trainSubjects2, trainShuffledSubjects2))
  testSubjects2 = np.hstack((testSubjects2, testShuffledSubjects2))

  assert len(subjects1) == len(subjects2)
  assert len(trainSubjects1) == len(trainSubjects1)
  assert len(testSubjects1) == len(testSubjects2)

  similaritiesTrain = similarityDifferentLabels(trainSubjects1, trainSubjects2)
  similaritiesTest = similarityDifferentLabels(testSubjects1, testSubjects2)

  print "trainSubjects1.shape"
  print trainSubjects1.shape

  print "similaritiesTrain.shape"
  print similaritiesTrain.shape
  # print similaritiesTrain

  assert len(trainData1) == len(trainData2)
  assert len(testData1) == len(testData2)

  trainData1, trainData2, similaritiesTrain = shuffle3(trainData1, trainData2, similaritiesTrain)
  testData1, testData2, similaritiesTest = shuffle3(testData1, testData2, similaritiesTest)

  return trainData1, trainData2, testData1, testData2, similaritiesTrain, similaritiesTest

def splitDataInPairsWithLabels(labelsToImages, imgsPerLabel, labelsToTake=None):
  data1 = []
  data2 = []

  shuffling = []
  labelsShuffling = []
  labels1 = []
  labels2 = []

  for label, images in labelsToImages.iteritems():
    if labelsToTake is not None and label not in labelsToTake:
      print "skipping subject"
      continue

    # The database might contain the labels in similar
    # poses, and illumination conditions, so shuffle before
    np.random.shuffle(images)

    if imgsPerLabel is not None:
      images = images[:imgsPerLabel]

    delta = len(images) / 4 + label % 2
    last2Index = 2 * delta
    data1 += images[0: delta]
    data2 += images[delta: last2Index]

    labels1 += [label] * delta
    labels2 += [label] * delta

    imagesForShuffling = images[last2Index : ]
    shuffling += imagesForShuffling
    labelsShuffling += [label] * len(imagesForShuffling)

  print "len(labelsShuffling)"
  print len(labelsShuffling)

  print "shuffling"
  print len(shuffling)

  assert len(shuffling) == len(labelsShuffling)
  shuffling, labelsShuffling = shuffleList(shuffling, labelsShuffling)

  print len(data1)
  print len(data2)
  assert len(data1) == len(data2)

  data1 = np.array(data1)
  data2 = np.array(data2)
  labels1 = np.array(labels1)
  labels2 = np.array(labels2)
  shuffling = np.array(shuffling)
  labelsShuffling = np.array(labelsShuffling)

  return data1, data2, labels1, labels2, shuffling, labelsShuffling

def splitDataAccordingToLabels(labelsToImages, labelsToTake, imgsPerLabel=None):
  data1, data2, labels1, labels2, shuffling, labelsShuffling  = splitDataInPairsWithLabels(labelsToImages, imgsPerLabel, labelsToTake=labelsToTake)

  shuffledData1, shuffledData2, labelsData1, labelsData2 = splitShuffling(shuffling, labelsShuffling)

  data1 = np.vstack((data1, shuffledData1))
  data2 = np.vstack((data2, shuffledData2))

  labels1 = np.hstack((labels1, labelsData1))
  labels2 = np.hstack((labels2, labelsData2))

  return data1, data2, labels1, labels2

def similarityDifferentLabels(labels1, labels2):
  assert len(labels1) == len(labels2)
  return labels1 == labels2

def splitSimilarityYale():
  subjectsToImgs = readCroppedYaleSubjects()

  # Get all subjects
  data1, data2, subjects1, subjects2 = splitDataAccordingToLabels(subjectsToImgs,
                                          None, imgsPerLabel=None)

  return data1, data2, similarityDifferentLabels(subjects1, subjects2)

def splitSimilaritiesPIEEmotions():
  emotionToImages = readMultiPIEEmotions()
  # Get all emotions
  data1, data2, emotions1, emotions2 = splitDataAccordingToLabels(emotionToImages,
                                          None, imgsPerLabel=None)
  labels = similarityDifferentLabels(emotions1, emotions2)

  data1, data2, labels = shuffle3(data1, data2, labels)

  kf = cross_validation.KFold(n=len(data1), n_folds=5)
  for train, test in kf:
    break

  return (data1[train], data2[train], labels[train],
          data1[test], data2[test], labels[test])

def splitEmotionsMultiPieKeepSubjects():
  subjectToEmotions = readMultiPIEEmotionsPerSubject()

  totalData1 = []
  totalData2 = []
  totalLabels1 = []
  totalLabels2 = []
  for subject, emotionToImages in subjectToEmotions.iteritems():
    data1, data2, labels1, labels2 = splitDataAccordingToLabels(emotionToImages, None, None)
    totalData1 += [data1]
    totalData2 += [data2]
    totalLabels1 += [labels1]
    totalLabels2 += [labels2]

  totalData1 = np.vstack(totalData1)
  totalData1 = np.vstack(totalData2)
  totalLabels1 = np.hstack(totalLabels1)
  totalLabels2 = np.hstack(totalLabels2)
  return totalData1, totalData2, totalLabels1, totalLabels2

def splitEmotionsMultiPieKeepSubjectsTestTrain():
  kf = cross_validation.KFold(n=len(totalData1), n_folds=5)
  for train, test in kf:
    break

  labels = similarityDifferentLabels(totalLabels1, totalLabels2)

  return (totalData1[train], totalData2[train], labels[train],
          totalData1[test], totalData2[test], labels[test])

def testShuffling():
  shuffling = [1,2,3, 4]
  labelsShuffling = [1,2,3, 4]
  a, b, c, d  = splitShuffling(shuffling, labelsShuffling)
  assert not c[0] ==  d[0]
  assert not c[1] ==  d[1]
  assert sorted(list(a) + list(b)) == [1,2,3,4]

  shuffling = [1,2,3]
  labelsShuffling = [1,2,3]
  a, b, c, d  = splitShuffling(shuffling, labelsShuffling)
  assert not c[0] ==  d[0]
  assert not a[0] == b[0]

  shuffling = [1,2,4,5]
  labelsShuffling = [1,2,1,2]
  a, b, c, d  = splitShuffling(shuffling, labelsShuffling)
  assert not c[0] ==  d[0]
  assert not c[1] == d[1]

  fst = sorted(list(a[0]))
  snd = sorted(list(b[0]))

  assert fst == [1,4] or fst == [2,5]
  assert snd == [2,5] or snd == [1,4]

  if fst == [1,4]:
    assert c[0] == 1
  else:
    assert c[0] == 2

  shuffling = [ np.array([1,1]), np.array([2,2]), np.array([4,4]), np.array([5,5]),
                np.array([6, 6]), np.array([7, 7]) ]
  labelsShuffling = [1, 2, 3, 2, 1, 3]
  a, b, c, d  = splitShuffling(shuffling, labelsShuffling)
  assert not c[0] ==  d[0]
  assert not c[1] == d[1]
  assert not c[2] == d[2]

  print zip(a, c)
  print zip(b, d)

if __name__ == '__main__':
  # print shuffleList([1,2], [3,4])
  print len(shuffle([1,2], [3,4]))
  testShuffling()
<commit_msg>Print<commit_after>from sklearn import cross_validation
from readfacedatabases import *
import copy

# TODO: move to common?
def splitTrainTest(data1, data2, labels1, labels2, ratio):
  assert len(data1) == len(data2)
  assert len(labels1) == len(labels2)
  assert len(labels1) == len(data1)
  assert len(labels1) == len(labels2)

  data1, data2, labels1, labels2 = shuffle(data1, data2, labels1, labels2)

  # Random data for training and testing
  kf = cross_validation.KFold(n=len(data1), n_folds=ratio)
  for train, test in kf:
    break

  return (data1[train], data1[test], data2[train], data2[test],
          labels1[train], labels1[test], labels2[train], labels2[test])

def splitShuffling(shuffling, labelsShuffling):

  shuffling, labelsShuffling = shuffle(shuffling, labelsShuffling)

  labels = np.unique(labelsShuffling)

  # TODO: we already had this? maybe not remake it
  labelsToData = {}
  for label in labels:
    print shuffling[labels == label]
    labelsToData[label] = list(shuffling[labels == label])
    print len(labelsToData[label])

  if True:
    raise Exception("test")

  shuffledData1 = []
  shuffledData2 = []
  labelsData1 = []
  labelsData2 = []

  currentLabels = list(labels)

  while len(shuffledData1) + len(shuffledData2) <= len(shuffling):
    print "len(shuffledData1) + len(shuffledData2)"
    print len(shuffledData1) + len(shuffledData2)
    print "len(shuffling)"
    print len(shuffling)

    print "currentLabels"
    print currentLabels

    label1 = np.random.choice(currentLabels, 1, replace=False)
    label1 = label1[0]
    print "label1"
    print label1
    copyCurrentLabels = list(currentLabels)
    copyCurrentLabels.remove(label1)

    if not copyCurrentLabels:
      break

    label2 = np.random.choice(np.array(copyCurrentLabels), 1, replace=False)
    label2 = label2[0]
    print "label2"
    print label2

    print label1
    print label2

    if label1 == label2:
      print " in first if"
      continue

    # print "labelsToData"
    # print labelsToData

    dataLabel1 = labelsToData[label1]
    dataLabel2 = labelsToData[label2]
    if len(dataLabel1) == 0:
      print "in second if"
      currentLabels.remove(label1)
      continue
    if len(dataLabel2) == 0:
      print "in second if"
      currentLabels.remove(label2)
      continue

    # data1LabelIndex = np.random.choice(len(labelsToData[label1]))
    # data2LabelIndex = np.random.choice(len(labelsToData[label2]))
    shuffledData1 += [dataLabel1[0]]
    shuffledData2 += [dataLabel2[0]]
    labelsData1 += [label1]
    labelsData2 += [label2]

    del labelsToData[label1][0]
    del labelsToData[label2][0]

    # print "labelsToData"
    # print labelsToData

  shuffledData1 = np.vstack(shuffledData1)
  shuffledData2 = np.vstack(shuffledData2)

  labelsData1 = np.hstack(labelsData1)
  labelsData2 = np.hstack(labelsData2)

  # remaing = list(shuffling)
  # remaininLabels = list(labelsShuffling)

  # print "shuffling size"
  # print len(shuffling)

  # shuffledData1 = []
  # shuffledData2 = []
  # labelsData1 = []
  # labelsData2 = []

  # for label in labels:
  #   print "label"
  #   print label

  #   nrRemainingData = len(remaing)

  #   if nrRemainingData == 0:
  #     break

  #   labelIndices = np.array(remaininLabels) == label
  #   concreteIndices = np.arange(nrRemainingData)[labelIndices]

  #   # If nothing of this label is left, just continue
  #   if len(concreteIndices) == 0:
  #     continue

  #   otherIndices = np.arange(nrRemainingData)[np.invert(labelIndices)]

  #   if len(otherIndices) == 0:
  #     continue

  #   indicesToTake = min(len(concreteIndices), len(otherIndices))

  #   otherIndices = np.random.choice(otherIndices, indicesToTake, replace=False)

  #   # concreteData = np.array(remaing)[concreteIndices]

  #   shuffledData1 += [np.array(remaing)[concreteIndices]]
  #   labelsData1 += [np.array(remaininLabels)[concreteIndices]]

  #   shuffledData2 += [np.array(remaing)[otherIndices]]
  #   labelsData2 += [np.array(remaininLabels)[otherIndices]]

  #   indicesToRemove = np.hstack((otherIndices, concreteIndices))
  #   remaing = [v for i, v in enumerate(remaing) if i not in indicesToRemove]
  #   remaininLabels = [v for i, v in enumerate(remaininLabels) if i not in indicesToRemove]

  #   assert len(remaing) == len(remaininLabels)

  # shuffledData1 = np.vstack(shuffledData1)
  # shuffledData2 = np.vstack(shuffledData2)

  # labelsData1 = np.hstack(labelsData1)
  # labelsData2 = np.hstack(labelsData2)

  # print shuffledData1.shape
  # print shuffledData2.shape

  # assert len(shuffledData1) == len(shuffledData2)
  # assert len(labelsData1) == len(labelsData2)

  # assert len(shuffledData1) <= len(shuffling) / 2

  """  STOP NEW METHOD """
  # shuffledData1 = shuffling[0: len(shuffling) / 2]
  # shuffledData2 = shuffling[len(shuffling)/2 :]

  # labelsData1 = labelsShuffling[0: len(shuffling) /2]
  # labelsData2 = labelsShuffling[len(shuffling)/2:]

  # shuffledData1 = np.array(shuffledData1)
  # shuffledData1 = np.array(shuffledData2)
  # labelsData1 = np.array(labelsData1)
  # labelsData2 = np.array(labelsData2)

  return shuffledData1, shuffledData2, labelsData1, labelsData2

# TODO: I think this can be written easier with the code similar to the Emotions one
# you can create more tuples than just one per image
# you can put each image in 5 tuples and that will probably owrk better
# it might be useful to also give the same image twice
def splitDataMultiPIESubject(imgsPerLabel=None):
  subjectsToImgs = readMultiPIESubjects()

  data1, data2, subjects1, subjects2, shuffling, subjectsShuffling = splitDataInPairsWithLabels(subjectsToImgs, imgsPerLabel, None)

  trainData1, testData1, trainData2, testData2, trainSubjects1, testSubjects1,\
        trainSubjects2, testSubjects2 = splitTrainTest(data1, data2, subjects1, subjects2, 5)

  print "trainData1.shape"
  print trainData1.shape

  shuffledData1, shuffledData2, subjectsData1, subjectsData2 = splitShuffling(shuffling, subjectsShuffling)

  print len(shuffledData1)
  print len(shuffledData2)

  trainShuffedData1, testShuffedData1, trainShuffedData2, testShuffedData2,\
    trainShuffledSubjects1, testShuffledSubjects1, trainShuffledSubjects2, testShuffledSubjects2 =\
        splitTrainTest(shuffledData1, shuffledData2,
                      subjectsData1, subjectsData2, 5)

  trainData1 = np.vstack((trainData1, trainShuffedData1))

  print "trainData1.shape"
  print trainData1.shape

  trainData2 = np.vstack((trainData2, trainShuffedData2))

  testData1 = np.vstack((testData1, testShuffedData1))
  testData2 = np.vstack((testData2, testShuffedData2))

  trainSubjects1 = np.hstack((trainSubjects1, trainShuffledSubjects1))
  testSubjects1 = np.hstack((testSubjects1, testShuffledSubjects1))

  trainSubjects2 = np.hstack((trainSubjects2, trainShuffledSubjects2))
  testSubjects2 = np.hstack((testSubjects2, testShuffledSubjects2))

  assert len(subjects1) == len(subjects2)
  assert len(trainSubjects1) == len(trainSubjects1)
  assert len(testSubjects1) == len(testSubjects2)

  similaritiesTrain = similarityDifferentLabels(trainSubjects1, trainSubjects2)
  similaritiesTest = similarityDifferentLabels(testSubjects1, testSubjects2)

  print "trainSubjects1.shape"
  print trainSubjects1.shape

  print "similaritiesTrain.shape"
  print similaritiesTrain.shape
  # print similaritiesTrain

  assert len(trainData1) == len(trainData2)
  assert len(testData1) == len(testData2)

  trainData1, trainData2, similaritiesTrain = shuffle3(trainData1, trainData2, similaritiesTrain)
  testData1, testData2, similaritiesTest = shuffle3(testData1, testData2, similaritiesTest)

  return trainData1, trainData2, testData1, testData2, similaritiesTrain, similaritiesTest

def splitDataInPairsWithLabels(labelsToImages, imgsPerLabel, labelsToTake=None):
  data1 = []
  data2 = []

  shuffling = []
  labelsShuffling = []
  labels1 = []
  labels2 = []

  for label, images in labelsToImages.iteritems():
    if labelsToTake is not None and label not in labelsToTake:
      print "skipping subject"
      continue

    # The database might contain the labels in similar
    # poses, and illumination conditions, so shuffle before
    np.random.shuffle(images)

    if imgsPerLabel is not None:
      images = images[:imgsPerLabel]

    delta = len(images) / 4 + label % 2
    last2Index = 2 * delta
    data1 += images[0: delta]
    data2 += images[delta: last2Index]

    labels1 += [label] * delta
    labels2 += [label] * delta

    imagesForShuffling = images[last2Index : ]
    shuffling += imagesForShuffling
    labelsShuffling += [label] * len(imagesForShuffling)

  print "len(labelsShuffling)"
  print len(labelsShuffling)

  print "shuffling"
  print len(shuffling)

  assert len(shuffling) == len(labelsShuffling)
  shuffling, labelsShuffling = shuffleList(shuffling, labelsShuffling)

  print len(data1)
  print len(data2)
  assert len(data1) == len(data2)

  data1 = np.array(data1)
  data2 = np.array(data2)
  labels1 = np.array(labels1)
  labels2 = np.array(labels2)
  shuffling = np.array(shuffling)
  labelsShuffling = np.array(labelsShuffling)

  return data1, data2, labels1, labels2, shuffling, labelsShuffling

def splitDataAccordingToLabels(labelsToImages, labelsToTake, imgsPerLabel=None):
  data1, data2, labels1, labels2, shuffling, labelsShuffling  = splitDataInPairsWithLabels(labelsToImages, imgsPerLabel, labelsToTake=labelsToTake)

  shuffledData1, shuffledData2, labelsData1, labelsData2 = splitShuffling(shuffling, labelsShuffling)

  data1 = np.vstack((data1, shuffledData1))
  data2 = np.vstack((data2, shuffledData2))

  labels1 = np.hstack((labels1, labelsData1))
  labels2 = np.hstack((labels2, labelsData2))

  return data1, data2, labels1, labels2

def similarityDifferentLabels(labels1, labels2):
  assert len(labels1) == len(labels2)
  return labels1 == labels2

def splitSimilarityYale():
  subjectsToImgs = readCroppedYaleSubjects()

  # Get all subjects
  data1, data2, subjects1, subjects2 = splitDataAccordingToLabels(subjectsToImgs,
                                          None, imgsPerLabel=None)

  return data1, data2, similarityDifferentLabels(subjects1, subjects2)

def splitSimilaritiesPIEEmotions():
  emotionToImages = readMultiPIEEmotions()
  # Get all emotions
  data1, data2, emotions1, emotions2 = splitDataAccordingToLabels(emotionToImages,
                                          None, imgsPerLabel=None)
  labels = similarityDifferentLabels(emotions1, emotions2)

  data1, data2, labels = shuffle3(data1, data2, labels)

  kf = cross_validation.KFold(n=len(data1), n_folds=5)
  for train, test in kf:
    break

  return (data1[train], data2[train], labels[train],
          data1[test], data2[test], labels[test])

def splitEmotionsMultiPieKeepSubjects():
  subjectToEmotions = readMultiPIEEmotionsPerSubject()

  totalData1 = []
  totalData2 = []
  totalLabels1 = []
  totalLabels2 = []
  for subject, emotionToImages in subjectToEmotions.iteritems():
    data1, data2, labels1, labels2 = splitDataAccordingToLabels(emotionToImages, None, None)
    totalData1 += [data1]
    totalData2 += [data2]
    totalLabels1 += [labels1]
    totalLabels2 += [labels2]

  totalData1 = np.vstack(totalData1)
  totalData1 = np.vstack(totalData2)
  totalLabels1 = np.hstack(totalLabels1)
  totalLabels2 = np.hstack(totalLabels2)
  return totalData1, totalData2, totalLabels1, totalLabels2

def splitEmotionsMultiPieKeepSubjectsTestTrain():
  kf = cross_validation.KFold(n=len(totalData1), n_folds=5)
  for train, test in kf:
    break

  labels = similarityDifferentLabels(totalLabels1, totalLabels2)

  return (totalData1[train], totalData2[train], labels[train],
          totalData1[test], totalData2[test], labels[test])

def testShuffling():
  shuffling = [1,2,3, 4]
  labelsShuffling = [1,2,3, 4]
  a, b, c, d  = splitShuffling(shuffling, labelsShuffling)
  assert not c[0] ==  d[0]
  assert not c[1] ==  d[1]
  assert sorted(list(a) + list(b)) == [1,2,3,4]

  shuffling = [1,2,3]
  labelsShuffling = [1,2,3]
  a, b, c, d  = splitShuffling(shuffling, labelsShuffling)
  assert not c[0] ==  d[0]
  assert not a[0] == b[0]

  shuffling = [1,2,4,5]
  labelsShuffling = [1,2,1,2]
  a, b, c, d  = splitShuffling(shuffling, labelsShuffling)
  assert not c[0] ==  d[0]
  assert not c[1] == d[1]

  fst = sorted(list(a[0]))
  snd = sorted(list(b[0]))

  assert fst == [1,4] or fst == [2,5]
  assert snd == [2,5] or snd == [1,4]

  if fst == [1,4]:
    assert c[0] == 1
  else:
    assert c[0] == 2

  shuffling = [ np.array([1,1]), np.array([2,2]), np.array([4,4]), np.array([5,5]),
                np.array([6, 6]), np.array([7, 7]) ]
  labelsShuffling = [1, 2, 3, 2, 1, 3]
  a, b, c, d  = splitShuffling(shuffling, labelsShuffling)
  assert not c[0] ==  d[0]
  assert not c[1] == d[1]
  assert not c[2] == d[2]

  print zip(a, c)
  print zip(b, d)

if __name__ == '__main__':
  # print shuffleList([1,2], [3,4])
  print len(shuffle([1,2], [3,4]))
  testShuffling()
<|endoftext|>


<commit_before>"""Tests for controlflow."""

import unittest
from unittest.mock import patch

import controlflow

class ControlFlowTest(unittest.TestCase):

  def test_system_error_exit_raises_systemexit_error(self):
    with self.assertRaises(SystemExit):
      controlflow.system_error_exit(1, 'exit message')

  def test_system_error_exit_raises_systemexit_with_return_code(self):
    with self.assertRaises(SystemExit) as context_manager:
      controlflow.system_error_exit(100, 'exit message')
    self.assertEqual(context_manager.exception.code, 100)

  @patch.object(controlflow.display, 'print_error')
  def test_system_error_exit_prints_error_before_exiting(self, mock_print_err):
    with self.assertRaises(SystemExit):
      controlflow.system_error_exit(100, 'exit message')
    self.assertIn('exit message', mock_print_err.call_args[0][0])

  def test_csv_field_error_exit_raises_systemexit_error(self):
    with self.assertRaises(SystemExit):
      controlflow.csv_field_error_exit('aField',
                                       ['unusedField1', 'unusedField2'])

  def test_csv_field_error_exit_exits_code_2(self):
    with self.assertRaises(SystemExit) as context_manager:
      controlflow.csv_field_error_exit('aField',
                                       ['unusedField1', 'unusedField2'])
    self.assertEqual(context_manager.exception.code, 2)

  @patch.object(controlflow.display, 'print_error')
  def test_csv_field_error_exit_prints_error_details(self, mock_print_err):
    with self.assertRaises(SystemExit):
      controlflow.csv_field_error_exit('aField',
                                       ['unusedField1', 'unusedField2'])
    printed_message = mock_print_err.call_args[0][0]
    self.assertIn('aField', printed_message)
    self.assertIn('unusedField1', printed_message)
    self.assertIn('unusedField2', printed_message)

  def test_invalid_json_exit_raises_systemexit_error(self):
    with self.assertRaises(SystemExit):
      controlflow.invalid_json_exit('filename')

  def test_invalid_json_exit_exit_exits_code_17(self):
    with self.assertRaises(SystemExit) as context_manager:
      controlflow.invalid_json_exit('filename')
    self.assertEqual(context_manager.exception.code, 17)

  @patch.object(controlflow.display, 'print_error')
  def test_invalid_json_exit_prints_error_details(self, mock_print_err):
    with self.assertRaises(SystemExit):
      controlflow.invalid_json_exit('filename')
    printed_message = mock_print_err.call_args[0][0]
    self.assertIn('filename', printed_message)

  @patch.object(controlflow.time, 'sleep')
  def test_wait_on_failure_waits_exponentially(self, mock_sleep):
    controlflow.wait_on_failure(1, 5, 'Backoff attempt #1')
    controlflow.wait_on_failure(2, 5, 'Backoff attempt #2')
    controlflow.wait_on_failure(3, 5, 'Backoff attempt #3')

    sleep_calls = mock_sleep.call_args_list
    self.assertGreaterEqual(sleep_calls[0][0][0], 2**1)
    self.assertGreaterEqual(sleep_calls[1][0][0], 2**2)
    self.assertGreaterEqual(sleep_calls[2][0][0], 2**3)

  @patch.object(controlflow.time, 'sleep')
  def test_wait_on_failure_does_not_exceed_60_secs_wait(self, mock_sleep):
    total_attempts = 20
    for attempt in range(1, total_attempts + 1):
      controlflow.wait_on_failure(
          attempt,
          total_attempts,
          'Attempt #%s' % attempt,
          # Suppress messages while we make a lot of attempts.
          error_print_threshold=total_attempts + 1)
      # Wait time may be between 60 and 61 secs, due to rand addition.
      self.assertLess(mock_sleep.call_args[0][0], 61)

  # Prevent the system from actually sleeping and thus slowing down the test.
  @patch.object(controlflow.time, 'sleep')
  @patch.object(controlflow.sys.stderr, 'write')
  def test_wait_on_failure_prints_errors(self, mock_stderr_write,
                                         unused_mock_sleep):
    message = 'An error message to display'
    controlflow.wait_on_failure(1, 5, message, error_print_threshold=0)
    self.assertIn(message, mock_stderr_write.call_args[0][0])

  @patch.object(controlflow.time, 'sleep')
  @patch.object(controlflow.sys.stderr, 'write')
  def test_wait_on_failure_only_prints_after_threshold(self, mock_stderr_write,
                                                       unused_mock_sleep):
    total_attempts = 5
    threshold = 3
    for attempt in range(1, total_attempts + 1):
      controlflow.wait_on_failure(
          attempt,
          total_attempts,
          'Attempt #%s' % attempt,
          error_print_threshold=threshold)
    self.assertEqual(total_attempts - threshold, mock_stderr_write.call_count)
<commit_msg>fix retry test<commit_after>"""Tests for controlflow."""

import unittest
from unittest.mock import patch

import controlflow

class ControlFlowTest(unittest.TestCase):

  def test_system_error_exit_raises_systemexit_error(self):
    with self.assertRaises(SystemExit):
      controlflow.system_error_exit(1, 'exit message')

  def test_system_error_exit_raises_systemexit_with_return_code(self):
    with self.assertRaises(SystemExit) as context_manager:
      controlflow.system_error_exit(100, 'exit message')
    self.assertEqual(context_manager.exception.code, 100)

  @patch.object(controlflow.display, 'print_error')
  def test_system_error_exit_prints_error_before_exiting(self, mock_print_err):
    with self.assertRaises(SystemExit):
      controlflow.system_error_exit(100, 'exit message')
    self.assertIn('exit message', mock_print_err.call_args[0][0])

  def test_csv_field_error_exit_raises_systemexit_error(self):
    with self.assertRaises(SystemExit):
      controlflow.csv_field_error_exit('aField',
                                       ['unusedField1', 'unusedField2'])

  def test_csv_field_error_exit_exits_code_2(self):
    with self.assertRaises(SystemExit) as context_manager:
      controlflow.csv_field_error_exit('aField',
                                       ['unusedField1', 'unusedField2'])
    self.assertEqual(context_manager.exception.code, 2)

  @patch.object(controlflow.display, 'print_error')
  def test_csv_field_error_exit_prints_error_details(self, mock_print_err):
    with self.assertRaises(SystemExit):
      controlflow.csv_field_error_exit('aField',
                                       ['unusedField1', 'unusedField2'])
    printed_message = mock_print_err.call_args[0][0]
    self.assertIn('aField', printed_message)
    self.assertIn('unusedField1', printed_message)
    self.assertIn('unusedField2', printed_message)

  def test_invalid_json_exit_raises_systemexit_error(self):
    with self.assertRaises(SystemExit):
      controlflow.invalid_json_exit('filename')

  def test_invalid_json_exit_exit_exits_code_17(self):
    with self.assertRaises(SystemExit) as context_manager:
      controlflow.invalid_json_exit('filename')
    self.assertEqual(context_manager.exception.code, 17)

  @patch.object(controlflow.display, 'print_error')
  def test_invalid_json_exit_prints_error_details(self, mock_print_err):
    with self.assertRaises(SystemExit):
      controlflow.invalid_json_exit('filename')
    printed_message = mock_print_err.call_args[0][0]
    self.assertIn('filename', printed_message)

  @patch.object(controlflow.time, 'sleep')
  def test_wait_on_failure_waits_exponentially(self, mock_sleep):
    controlflow.wait_on_failure(1, 5, 'Backoff attempt #1')
    controlflow.wait_on_failure(2, 5, 'Backoff attempt #2')
    controlflow.wait_on_failure(3, 5, 'Backoff attempt #3')

    sleep_calls = mock_sleep.call_args_list
    self.assertGreaterEqual(sleep_calls[0][0][0], 2**1)
    self.assertGreaterEqual(sleep_calls[1][0][0], 2**2)
    self.assertGreaterEqual(sleep_calls[2][0][0], 2**3)

  @patch.object(controlflow.time, 'sleep')
  def test_wait_on_failure_does_not_exceed_60_secs_wait(self, mock_sleep):
    total_attempts = 20
    for attempt in range(1, total_attempts + 1):
      controlflow.wait_on_failure(
          attempt,
          total_attempts,
          'Attempt #%s' % attempt,
          # Suppress messages while we make a lot of attempts.
          error_print_threshold=total_attempts + 1)
      # Wait time may be between 60 and 61 secs, due to rand addition.
      self.assertLessEqual(mock_sleep.call_args[0][0], 61)

  # Prevent the system from actually sleeping and thus slowing down the test.
  @patch.object(controlflow.time, 'sleep')
  @patch.object(controlflow.sys.stderr, 'write')
  def test_wait_on_failure_prints_errors(self, mock_stderr_write,
                                         unused_mock_sleep):
    message = 'An error message to display'
    controlflow.wait_on_failure(1, 5, message, error_print_threshold=0)
    self.assertIn(message, mock_stderr_write.call_args[0][0])

  @patch.object(controlflow.time, 'sleep')
  @patch.object(controlflow.sys.stderr, 'write')
  def test_wait_on_failure_only_prints_after_threshold(self, mock_stderr_write,
                                                       unused_mock_sleep):
    total_attempts = 5
    threshold = 3
    for attempt in range(1, total_attempts + 1):
      controlflow.wait_on_failure(
          attempt,
          total_attempts,
          'Attempt #%s' % attempt,
          error_print_threshold=threshold)
    self.assertEqual(total_attempts - threshold, mock_stderr_write.call_count)
<|endoftext|>


<commit_before># coding: utf-8
import subprocess
import asyncio
import aiohttp
from parkworker import const
from parkworker.monits.base import CheckResult
from parkworker3.monits.base import Monit

class PingMonit(Monit):
    name = 'general.ping'
    description = 'Ping host checking.'

    async def async_check(self, host: str, **kwargs):
        # print('Start ping', host)
        process = await asyncio.create_subprocess_exec(
            'ping', host, '-c1',
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
        )

        await process.wait()

        if process.returncode == 0:
            level = const.LEVEL_OK
        else:
            level = const.LEVEL_FAIL

        stdout = await process.stdout.read()
        check_result = CheckResult(
            level=level,
            extra={'stdout': stdout.decode('utf-8')},
        )

        # import random
        # await asyncio.sleep(random.randint(5, 10))
        # print('End ping', host)
        return check_result

class HttpMonit(Monit):
    name = 'general.http'
    description = 'Http checking.'

    async def async_check(self, host: str, **kwargs):
        protocol = kwargs.get('protocol', 'http')
        port = kwargs.get('port', '80')
        path = kwargs.get('path', '/')
        url = '%s://%s:%s%s' % (protocol, host, port, path)
        print('Start http', host, url)

        extra = {'url': url}
        level = const.LEVEL_OK
        async with aiohttp.get(url) as response:
            extra['is_http_status_ok'] = response.status == kwargs.get('http_status', 200)
            if not extra['is_http_status_ok']:
                extra['http_status'] = response.status
                level = const.LEVEL_FAIL

            if kwargs.get('contains'):
                content = await response.read()
                content = content.decode('utf-8')
                extra['is_contains_ok'] = kwargs['contains'] in content
                if not extra['is_contains_ok']:
                    extra['content'] = content
                    level = const.LEVEL_FAIL

        check_result = CheckResult(
            level=level,
            extra=extra,
        )

        return check_result
<commit_msg>add registration monit description<commit_after># coding: utf-8
import subprocess
import asyncio
import aiohttp
from parkworker import const
from parkworker.monits.base import CheckResult
from parkworker3.monits.base import Monit

class PingMonit(Monit):
    name = 'general.ping'
    description = 'Ping host checking.'

    async def async_check(self, host: str, **kwargs):
        # print('Start ping', host)
        process = await asyncio.create_subprocess_exec(
            'ping', host, '-c1',
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
        )

        await process.wait()

        if process.returncode == 0:
            level = const.LEVEL_OK
        else:
            level = const.LEVEL_FAIL

        stdout = await process.stdout.read()
        check_result = CheckResult(
            level=level,
            extra={'stdout': stdout.decode('utf-8')},
        )

        # import random
        # await asyncio.sleep(random.randint(5, 10))
        # print('End ping', host)
        return check_result

class HttpMonit(Monit):
    name = 'general.http'
    description = 'Http checking. Get options: \n' \
                  ' - protocol. Default "http". \n' \
                  ' - port. Default 80. \n' \
                  ' - path. Default "/". \n' \
                  ' - http_status. Default 200. \n' \
                  ' - contains. Optional. If defined, returned content will be checked on containing text.'

    async def async_check(self, host: str, **kwargs):
        protocol = kwargs.get('protocol', 'http')
        port = kwargs.get('port', '80')
        path = kwargs.get('path', '/')
        url = '%s://%s:%s%s' % (protocol, host, port, path)
        print('Start http', host, url)

        extra = {'url': url}
        level = const.LEVEL_OK
        async with aiohttp.get(url) as response:
            extra['is_http_status_ok'] = response.status == kwargs.get('http_status', 200)
            if not extra['is_http_status_ok']:
                extra['http_status'] = response.status
                level = const.LEVEL_FAIL

            if kwargs.get('contains'):
                content = await response.read()
                content = content.decode('utf-8')
                extra['is_contains_ok'] = kwargs['contains'] in content
                if not extra['is_contains_ok']:
                    extra['content'] = content
                    level = const.LEVEL_FAIL

        check_result = CheckResult(
            level=level,
            extra=extra,
        )

        return check_result
<|endoftext|>


<commit_before>import os
import fnmatch
import stat

try:
    from . import msg, shared as G, utils
    assert G and msg and utils
except ImportError:
    import msg
    import shared as G

IGNORE_FILES = ['.gitignore', '.hgignore', '.flignore', '.flooignore']
# TODO: make this configurable
HIDDEN_WHITELIST = ['.floo'] + IGNORE_FILES
# TODO: grab global git ignores:
# gitconfig_file = popen("git config -z --get core.excludesfile", "r");
DEFAULT_IGNORES = ['extern', 'node_modules', 'tmp', 'vendor']
MAX_FILE_SIZE = 1024 * 1024 * 5

def create_flooignore(path):
    flooignore = os.path.join(path, '.flooignore')
    # A very short race condition, but whatever.
    if os.path.exists(flooignore):
        return
    try:
        with open(flooignore, 'wb') as fd:
            fd.write('\n'.join(DEFAULT_IGNORES).encode('utf-8'))
    except Exception as e:
        msg.error('Error creating default .flooignore: %s' % str(e))

class Ignore(object):
    def __init__(self, parent, path):
        self.parent = parent
        self.size = 0
        self.children = []
        self.files = []
        self.ignores = {
            '/TOO_BIG/': []
        }
        self.path = utils.unfuck_path(path)

        msg.log('Initializing ignores for %s' % path)
        for ignore_file in IGNORE_FILES:
            try:
                self.load(ignore_file)
            except:
                pass

        try:
            paths = os.listdir(self.path)
        except Exception as e:
            msg.error('Error listing path %s: %s' % (path, unicode(e)))
            return
        for p in paths:
            p_path = os.path.join(path, p)
            if p[0] == '.' and p not in HIDDEN_WHITELIST:
                msg.log('Ignoring hidden path %s' % p_path)
                continue
            is_ignored = self.is_ignored(p_path)
            if is_ignored:
                msg.log(is_ignored)
                continue
            try:
                s = os.stat(p_path)
            except Exception as e:
                msg.error('Error lstat()ing path %s: %s' % (p_path, unicode(e)))
                continue
            if stat.S_ISDIR(s.st_mode):
                ig = Ignore(self, p_path)
                self.children.append(ig)
                self.size += ig.size
                continue
            elif stat.S_ISREG(s.st_mode):
                if s.st_size > (MAX_FILE_SIZE):
                    self.ignores['/TOO_BIG/'].append(p)
                    msg.log(self.is_ignored_message(p_path, p, '/TOO_BIG/'))
                else:
                    self.size += s.st_size
                    self.files.append(p)

    def load(self, ignore_file):
        with open(os.path.join(self.path, ignore_file), 'rb') as fd:
            ignores = fd.read().decode('utf-8')
        self.ignores[ignore_file] = []
        for ignore in ignores.split('\n'):
            ignore = ignore.strip()
            if len(ignore) == 0:
                continue
            if ignore[0] == '#':
                continue
            msg.debug('Adding %s to ignore patterns' % ignore)
            self.ignores[ignore_file].append(ignore)

    def list_paths(self):
        for f in self.files:
            yield os.path.join(self.path, f)
        for c in self.children:
            for p in c.list_paths():
                yield p

    def is_ignored_message(self, path, pattern, ignore_file):
        if ignore_file == '/TOO_BIG/':
            return '%s ignored because it is too big (more than %s bytes)' % (path, MAX_FILE_SIZE)
        return '%s ignored by pattern %s in %s' % (path, pattern, os.path.join(self.path, ignore_file))

    def is_ignored(self, path):
        rel_path = os.path.relpath(path, self.path)
        for ignore_file, patterns in self.ignores.items():
            for pattern in patterns:
                base_path, file_name = os.path.split(rel_path)
                if pattern[0] == '/':
                    if utils.unfuck_path(base_path) == self.path and fnmatch.fnmatch(file_name, pattern[1:]):
                        return self.is_ignored_message(path, pattern, ignore_file)
                else:
                    if fnmatch.fnmatch(file_name, pattern):
                        return self.is_ignored_message(path, pattern, ignore_file)
                    if fnmatch.fnmatch(rel_path, pattern):
                        return self.is_ignored_message(path, pattern, ignore_file)
        if self.parent:
            return self.parent.is_ignored(path)
        return False

def build_ignores(path):
    current_ignore = Ignore(None, G.PROJECT_PATH)
    current_path = G.PROJECT_PATH
    starting = os.path.relpath(path, G.PROJECT_PATH)
    for p in starting.split(os.sep):
        current_path = os.path.join(current_path, p)
        if p == '..':
            raise ValueError('%s is not in project path %s' % (current_path, G.PROJECT_PATH))
        current_ignore = Ignore(current_ignore, current_path)
    return current_ignore
<commit_msg>fixed bug with unicode in ignore<commit_after>import os
import fnmatch
import stat

try:
    from . import msg, shared as G, utils
    assert G and msg and utils
except ImportError:
    import msg
    import shared as G

try:
    unicode()
except NameError:
    unicode = str

IGNORE_FILES = ['.gitignore', '.hgignore', '.flignore', '.flooignore']
# TODO: make this configurable
HIDDEN_WHITELIST = ['.floo'] + IGNORE_FILES
# TODO: grab global git ignores:
# gitconfig_file = popen("git config -z --get core.excludesfile", "r");
DEFAULT_IGNORES = ['extern', 'node_modules', 'tmp', 'vendor']
MAX_FILE_SIZE = 1024 * 1024 * 5

def create_flooignore(path):
    flooignore = os.path.join(path, '.flooignore')
    # A very short race condition, but whatever.
    if os.path.exists(flooignore):
        return
    try:
        with open(flooignore, 'wb') as fd:
            fd.write('\n'.join(DEFAULT_IGNORES).encode('utf-8'))
    except Exception as e:
        msg.error('Error creating default .flooignore: %s' % str(e))

class Ignore(object):
    def __init__(self, parent, path):
        self.parent = parent
        self.size = 0
        self.children = []
        self.files = []
        self.ignores = {
            '/TOO_BIG/': []
        }
        self.path = utils.unfuck_path(path)

        msg.log('Initializing ignores for %s' % path)
        for ignore_file in IGNORE_FILES:
            try:
                self.load(ignore_file)
            except:
                pass

        try:
            paths = os.listdir(self.path)
        except Exception as e:
            msg.error('Error listing path %s: %s' % (path, unicode(e)))
            return
        for p in paths:
            p_path = os.path.join(path, p)
            if p[0] == '.' and p not in HIDDEN_WHITELIST:
                msg.log('Ignoring hidden path %s' % p_path)
                continue
            is_ignored = self.is_ignored(p_path)
            if is_ignored:
                msg.log(is_ignored)
                continue
            try:
                s = os.stat(p_path)
            except Exception as e:
                msg.error('Error lstat()ing path %s: %s' % (p_path, unicode(e)))
                continue
            if stat.S_ISDIR(s.st_mode):
                ig = Ignore(self, p_path)
                self.children.append(ig)
                self.size += ig.size
                continue
            elif stat.S_ISREG(s.st_mode):
                if s.st_size > (MAX_FILE_SIZE):
                    self.ignores['/TOO_BIG/'].append(p)
                    msg.log(self.is_ignored_message(p_path, p, '/TOO_BIG/'))
                else:
                    self.size += s.st_size
                    self.files.append(p)

    def load(self, ignore_file):
        with open(os.path.join(self.path, ignore_file), 'rb') as fd:
            ignores = fd.read().decode('utf-8')
        self.ignores[ignore_file] = []
        for ignore in ignores.split('\n'):
            ignore = ignore.strip()
            if len(ignore) == 0:
                continue
            if ignore[0] == '#':
                continue
            msg.debug('Adding %s to ignore patterns' % ignore)
            self.ignores[ignore_file].append(ignore)

    def list_paths(self):
        for f in self.files:
            yield os.path.join(self.path, f)
        for c in self.children:
            for p in c.list_paths():
                yield p

    def is_ignored_message(self, path, pattern, ignore_file):
        if ignore_file == '/TOO_BIG/':
            return '%s ignored because it is too big (more than %s bytes)' % (path, MAX_FILE_SIZE)
        return '%s ignored by pattern %s in %s' % (path, pattern, os.path.join(self.path, ignore_file))

    def is_ignored(self, path):
        rel_path = os.path.relpath(path, self.path)
        for ignore_file, patterns in self.ignores.items():
            for pattern in patterns:
                base_path, file_name = os.path.split(rel_path)
                if pattern[0] == '/':
                    if utils.unfuck_path(base_path) == self.path and fnmatch.fnmatch(file_name, pattern[1:]):
                        return self.is_ignored_message(path, pattern, ignore_file)
                else:
                    if fnmatch.fnmatch(file_name, pattern):
                        return self.is_ignored_message(path, pattern, ignore_file)
                    if fnmatch.fnmatch(rel_path, pattern):
                        return self.is_ignored_message(path, pattern, ignore_file)
        if self.parent:
            return self.parent.is_ignored(path)
        return False

def build_ignores(path):
    current_ignore = Ignore(None, G.PROJECT_PATH)
    current_path = G.PROJECT_PATH
    starting = os.path.relpath(path, G.PROJECT_PATH)
    for p in starting.split(os.sep):
        current_path = os.path.join(current_path, p)
        if p == '..':
            raise ValueError('%s is not in project path %s' % (current_path, G.PROJECT_PATH))
        current_ignore = Ignore(current_ignore, current_path)
    return current_ignore
<|endoftext|>


<commit_before>from sklearn import cross_validation
from readfacedatabases import *
import copy

# TODO: move to common?
def splitTrainTest(data1, data2, labels1, labels2, ratio):
  assert len(data1) == len(data2)
  assert len(labels1) == len(labels2)
  assert len(labels1) == len(data1)
  assert len(labels1) == len(labels2)

  data1, data2, labels1, labels2 = shuffle(data1, data2, labels1, labels2)

  # Random data for training and testing
  kf = cross_validation.KFold(n=len(data1), n_folds=ratio)
  for train, test in kf:
    break

  return (data1[train], data1[test], data2[train], data2[test],
          labels1[train], labels1[test], labels2[train], labels2[test])

def splitShuffling(shuffling, labelsShuffling):

  shuffling, labelsShuffling = shuffle(shuffling, labelsShuffling)

  labels = np.unique(labelsShuffling)

  # TODO: we already had this? maybe not remake it
  labelsToData = {}
  for label in labels:
    labelsToData[label] = list(shuffling[labels == label])

  shuffledData1 = []
  shuffledData2 = []
  labelsData1 = []
  labelsData2 = []

  currentLabels = list(labels)

  while len(labels) >= 2:
    chosenLabels = np.random.choice(labels, 2)
    label1 = chosenLabels[0]
    label2 = chosenLabels[1]

    if label1 == label2:
      continue

    dataLabel1 = labelsToData[label1]
    dataLabel2 = labelsToData[label2]
    if len(dataLabel1) == 0 or len(dataLabel2) == 0:
      continue

    # data1LabelIndex = np.random.choice(len(labelsToData[label1]))
    # data2LabelIndex = np.random.choice(len(labelsToData[label2]))
    shuffledData1 += [dataLabel1[0]]
    shuffledData2 += [dataLabel2[0]]
    labelsData1 += [label1]
    labelsData2 += [label2]

    currentLabels.remove(label1)
    currentLabels.remove(label2)

    del labelsToData[label1][0]
    del labelsToData[label2][0]

  shuffledData1 = np.vstack(shuffledData1)
  shuffledData2 = np.vstack(shuffledData2)

  labelsData1 = np.hstack(labelsData1)
  labelsData2 = np.hstack(labelsData2)

  # remaing = list(shuffling)
  # remaininLabels = list(labelsShuffling)

  # print "shuffling size"
  # print len(shuffling)

  # shuffledData1 = []
  # shuffledData2 = []
  # labelsData1 = []
  # labelsData2 = []

  # for label in labels:
  #   print "label"
  #   print label

  #   nrRemainingData = len(remaing)

  #   if nrRemainingData == 0:
  #     break

  #   labelIndices = np.array(remaininLabels) == label
  #   concreteIndices = np.arange(nrRemainingData)[labelIndices]

  #   # If nothing of this label is left, just continue
  #   if len(concreteIndices) == 0:
  #     continue

  #   otherIndices = np.arange(nrRemainingData)[np.invert(labelIndices)]

  #   if len(otherIndices) == 0:
  #     continue

  #   indicesToTake = min(len(concreteIndices), len(otherIndices))

  #   otherIndices = np.random.choice(otherIndices, indicesToTake, replace=False)

  #   # concreteData = np.array(remaing)[concreteIndices]

  #   shuffledData1 += [np.array(remaing)[concreteIndices]]
  #   labelsData1 += [np.array(remaininLabels)[concreteIndices]]

  #   shuffledData2 += [np.array(remaing)[otherIndices]]
  #   labelsData2 += [np.array(remaininLabels)[otherIndices]]

  #   indicesToRemove = np.hstack((otherIndices, concreteIndices))
  #   remaing = [v for i, v in enumerate(remaing) if i not in indicesToRemove]
  #   remaininLabels = [v for i, v in enumerate(remaininLabels) if i not in indicesToRemove]

  #   assert len(remaing) == len(remaininLabels)

  # shuffledData1 = np.vstack(shuffledData1)
  # shuffledData2 = np.vstack(shuffledData2)

  # labelsData1 = np.hstack(labelsData1)
  # labelsData2 = np.hstack(labelsData2)

  # print shuffledData1.shape
  # print shuffledData2.shape

  # assert len(shuffledData1) == len(shuffledData2)
  # assert len(labelsData1) == len(labelsData2)

  # assert len(shuffledData1) <= len(shuffling) / 2

  """  STOP NEW METHOD """
  # shuffledData1 = shuffling[0: len(shuffling) / 2]
  # shuffledData2 = shuffling[len(shuffling)/2 :]

  # labelsData1 = labelsShuffling[0: len(shuffling) /2]
  # labelsData2 = labelsShuffling[len(shuffling)/2:]

  # shuffledData1 = np.array(shuffledData1)
  # shuffledData1 = np.array(shuffledData2)
  # labelsData1 = np.array(labelsData1)
  # labelsData2 = np.array(labelsData2)

  return shuffledData1, shuffledData2, labelsData1, labelsData2

# TODO: I think this can be written easier with the code similar to the Emotions one
# you can create more tuples than just one per image
# you can put each image in 5 tuples and that will probably owrk better
# it might be useful to also give the same image twice
def splitDataMultiPIESubject(imgsPerLabel=None):
  subjectsToImgs = readMultiPIESubjects()

  data1, data2, subjects1, subjects2, shuffling, subjectsShuffling = splitDataInPairsWithLabels(subjectsToImgs, imgsPerLabel, None)

  trainData1, testData1, trainData2, testData2, trainSubjects1, testSubjects1,\
        trainSubjects2, testSubjects2 = splitTrainTest(data1, data2, subjects1, subjects2, 5)

  print "trainData1.shape"
  print trainData1.shape

  shuffledData1, shuffledData2, subjectsData1, subjectsData2 = splitShuffling(shuffling, subjectsShuffling)

  print len(shuffledData1)
  print len(shuffledData2)

  trainShuffedData1, testShuffedData1, trainShuffedData2, testShuffedData2,\
    trainShuffledSubjects1, testShuffledSubjects1, trainShuffledSubjects2, testShuffledSubjects2 =\
        splitTrainTest(shuffledData1, shuffledData2,
                      subjectsData1, subjectsData2, 5)

  trainData1 = np.vstack((trainData1, trainShuffedData1))

  print "trainData1.shape"
  print trainData1.shape

  trainData2 = np.vstack((trainData2, trainShuffedData2))

  testData1 = np.vstack((testData1, testShuffedData1))
  testData2 = np.vstack((testData2, testShuffedData2))

  trainSubjects1 = np.hstack((trainSubjects1, trainShuffledSubjects1))
  testSubjects1 = np.hstack((testSubjects1, testShuffledSubjects1))

  trainSubjects2 = np.hstack((trainSubjects2, trainShuffledSubjects2))
  testSubjects2 = np.hstack((testSubjects2, testShuffledSubjects2))

  assert len(subjects1) == len(subjects2)
  assert len(trainSubjects1) == len(trainSubjects1)
  assert len(testSubjects1) == len(testSubjects2)

  similaritiesTrain = similarityDifferentLabels(trainSubjects1, trainSubjects2)
  similaritiesTest = similarityDifferentLabels(testSubjects1, testSubjects2)

  print "trainSubjects1.shape"
  print trainSubjects1.shape

  print "similaritiesTrain.shape"
  print similaritiesTrain.shape
  # print similaritiesTrain

  assert len(trainData1) == len(trainData2)
  assert len(testData1) == len(testData2)

  trainData1, trainData2, similaritiesTrain = shuffle3(trainData1, trainData2, similaritiesTrain)
  testData1, testData2, similaritiesTest = shuffle3(testData1, testData2, similaritiesTest)

  return trainData1, trainData2, testData1, testData2, similaritiesTrain, similaritiesTest

def splitDataInPairsWithLabels(labelsToImages, imgsPerLabel, labelsToTake=None):
  data1 = []
  data2 = []

  shuffling = []
  labelsShuffling = []
  labels1 = []
  labels2 = []

  for label, images in labelsToImages.iteritems():
    if labelsToTake is not None and label not in labelsToTake:
      print "skipping subject"
      continue

    # The database might contain the labels in similar
    # poses, and illumination conditions, so shuffle before
    np.random.shuffle(images)

    if imgsPerLabel is not None:
      images = images[:imgsPerLabel]

    delta = len(images) / 4 + label % 2
    last2Index = 2 * delta
    data1 += images[0: delta]
    data2 += images[delta: last2Index]

    labels1 += [label] * delta
    labels2 += [label] * delta

    imagesForShuffling = images[last2Index : ]
    shuffling += imagesForShuffling
    labelsShuffling += [label] * len(imagesForShuffling)

  print "len(labelsShuffling)"
  print len(labelsShuffling)

  print "shuffling"
  print len(shuffling)

  assert len(shuffling) == len(labelsShuffling)
  shuffling, labelsShuffling = shuffleList(shuffling, labelsShuffling)

  print len(data1)
  print len(data2)
  assert len(data1) == len(data2)

  data1 = np.array(data1)
  data2 = np.array(data2)
  labels1 = np.array(labels1)
  labels2 = np.array(labels2)
  shuffling = np.array(shuffling)
  labelsShuffling = np.array(labelsShuffling)

  return data1, data2, labels1, labels2, shuffling, labelsShuffling

def splitDataAccordingToLabels(labelsToImages, labelsToTake, imgsPerLabel=None):
  data1, data2, labels1, labels2, shuffling, labelsShuffling  = splitDataInPairsWithLabels(labelsToImages, imgsPerLabel, labelsToTake=labelsToTake)

  shuffledData1, shuffledData2, labelsData1, labelsData2 = splitShuffling(shuffling, labelsShuffling)

  data1 = np.vstack((data1, shuffledData1))
  data2 = np.vstack((data2, shuffledData2))

  labels1 = np.hstack((labels1, labelsData1))
  labels2 = np.hstack((labels2, labelsData2))

  return data1, data2, labels1, labels2

def similarityDifferentLabels(labels1, labels2):
  assert len(labels1) == len(labels2)
  return labels1 == labels2

def splitSimilarityYale():
  subjectsToImgs = readCroppedYaleSubjects()

  # Get all subjects
  data1, data2, subjects1, subjects2 = splitDataAccordingToLabels(subjectsToImgs,
                                          None, imgsPerLabel=None)

  return data1, data2, similarityDifferentLabels(subjects1, subjects2)

def splitSimilaritiesPIEEmotions():
  emotionToImages = readMultiPIEEmotions()
  # Get all emotions
  data1, data2, emotions1, emotions2 = splitDataAccordingToLabels(emotionToImages,
                                          None, imgsPerLabel=None)
  labels = similarityDifferentLabels(emotions1, emotions2)

  data1, data2, labels = shuffle3(data1, data2, labels)

  kf = cross_validation.KFold(n=len(data1), n_folds=5)
  for train, test in kf:
    break

  return (data1[train], data2[train], labels[train],
          data1[test], data2[test], labels[test])

def splitEmotionsMultiPieKeepSubjects():
  subjectToEmotions = readMultiPIEEmotionsPerSubject()

  totalData1 = []
  totalData2 = []
  totalLabels1 = []
  totalLabels2 = []
  for subject, emotionToImages in subjectToEmotions.iteritems():
    data1, data2, labels1, labels2 = splitDataAccordingToLabels(emotionToImages, None, None)
    totalData1 += [data1]
    totalData2 += [data2]
    totalLabels1 += [labels1]
    totalLabels2 += [labels2]

  totalData1 = np.vstack(totalData1)
  totalData1 = np.vstack(totalData2)
  totalLabels1 = np.hstack(totalLabels1)
  totalLabels2 = np.hstack(totalLabels2)
  return totalData1, totalData2, totalLabels1, totalLabels2

def splitEmotionsMultiPieKeepSubjectsTestTrain():
  kf = cross_validation.KFold(n=len(totalData1), n_folds=5)
  for train, test in kf:
    break

  labels = similarityDifferentLabels(totalLabels1, totalLabels2)

  return (totalData1[train], totalData2[train], labels[train],
          totalData1[test], totalData2[test], labels[test])

def testShuffling():
  shuffling = [1,2,3, 4]
  labelsShuffling = [1,2,3, 4]
  a, b, c, d  = splitShuffling(shuffling, labelsShuffling)
  assert not c[0] ==  d[0]
  assert not c[1] ==  d[1]
  assert sorted(list(a) + list(b)) == [1,2,3,4]

  shuffling = [1,2,3]
  labelsShuffling = [1,2,3]
  a, b, c, d  = splitShuffling(shuffling, labelsShuffling)
  assert not c[0] ==  d[0]
  assert not a[0] == b[0]

  shuffling = [1,2,4,5]
  labelsShuffling = [1,2,1,2]
  a, b, c, d  = splitShuffling(shuffling, labelsShuffling)
  assert not c[0] ==  d[0]
  assert not c[1] == d[1]

  fst = sorted(list(a[0]))
  snd = sorted(list(b[0]))

  assert fst == [1,4] or fst == [2,5]
  assert snd == [2,5] or snd == [1,4]

  if fst == [1,4]:
    assert c[0] == 1
  else:
    assert c[0] == 2

  shuffling = [ np.array([1,1]), np.array([2,2]), np.array([4,4]), np.array([5,5]),
                np.array([6, 6]), np.array([7, 7]) ]
  labelsShuffling = [1, 2, 3, 2, 1, 3]
  a, b, c, d  = splitShuffling(shuffling, labelsShuffling)
  assert not c[0] ==  d[0]
  assert not c[1] == d[1]
  assert not c[2] == d[2]

  print zip(a, c)
  print zip(b, d)

if __name__ == '__main__':
  # print shuffleList([1,2], [3,4])
  print len(shuffle([1,2], [3,4]))
  testShuffling()
<commit_msg>fixed infinite loop<commit_after>from sklearn import cross_validation
from readfacedatabases import *
import copy

# TODO: move to common?
def splitTrainTest(data1, data2, labels1, labels2, ratio):
  assert len(data1) == len(data2)
  assert len(labels1) == len(labels2)
  assert len(labels1) == len(data1)
  assert len(labels1) == len(labels2)

  data1, data2, labels1, labels2 = shuffle(data1, data2, labels1, labels2)

  # Random data for training and testing
  kf = cross_validation.KFold(n=len(data1), n_folds=ratio)
  for train, test in kf:
    break

  return (data1[train], data1[test], data2[train], data2[test],
          labels1[train], labels1[test], labels2[train], labels2[test])

def splitShuffling(shuffling, labelsShuffling):

  shuffling, labelsShuffling = shuffle(shuffling, labelsShuffling)

  labels = np.unique(labelsShuffling)

  # TODO: we already had this? maybe not remake it
  labelsToData = {}
  for label in labels:
    labelsToData[label] = list(shuffling[labels == label])

  shuffledData1 = []
  shuffledData2 = []
  labelsData1 = []
  labelsData2 = []

  currentLabels = list(labels)

  while len(currentLabels) >= 2:
    chosenLabels = np.random.choice(currentLabels, 2)
    label1 = chosenLabels[0]
    label2 = chosenLabels[1]

    if label1 == label2:
      continue

    dataLabel1 = labelsToData[label1]
    dataLabel2 = labelsToData[label2]
    if len(dataLabel1) == 0 or len(dataLabel2) == 0:
      continue

    # data1LabelIndex = np.random.choice(len(labelsToData[label1]))
    # data2LabelIndex = np.random.choice(len(labelsToData[label2]))
    shuffledData1 += [dataLabel1[0]]
    shuffledData2 += [dataLabel2[0]]
    labelsData1 += [label1]
    labelsData2 += [label2]

    currentLabels.remove(label1)
    currentLabels.remove(label2)

    del labelsToData[label1][0]
    del labelsToData[label2][0]

  shuffledData1 = np.vstack(shuffledData1)
  shuffledData2 = np.vstack(shuffledData2)

  labelsData1 = np.hstack(labelsData1)
  labelsData2 = np.hstack(labelsData2)

  # remaing = list(shuffling)
  # remaininLabels = list(labelsShuffling)

  # print "shuffling size"
  # print len(shuffling)

  # shuffledData1 = []
  # shuffledData2 = []
  # labelsData1 = []
  # labelsData2 = []

  # for label in labels:
  #   print "label"
  #   print label

  #   nrRemainingData = len(remaing)

  #   if nrRemainingData == 0:
  #     break

  #   labelIndices = np.array(remaininLabels) == label
  #   concreteIndices = np.arange(nrRemainingData)[labelIndices]

  #   # If nothing of this label is left, just continue
  #   if len(concreteIndices) == 0:
  #     continue

  #   otherIndices = np.arange(nrRemainingData)[np.invert(labelIndices)]

  #   if len(otherIndices) == 0:
  #     continue

  #   indicesToTake = min(len(concreteIndices), len(otherIndices))

  #   otherIndices = np.random.choice(otherIndices, indicesToTake, replace=False)

  #   # concreteData = np.array(remaing)[concreteIndices]

  #   shuffledData1 += [np.array(remaing)[concreteIndices]]
  #   labelsData1 += [np.array(remaininLabels)[concreteIndices]]

  #   shuffledData2 += [np.array(remaing)[otherIndices]]
  #   labelsData2 += [np.array(remaininLabels)[otherIndices]]

  #   indicesToRemove = np.hstack((otherIndices, concreteIndices))
  #   remaing = [v for i, v in enumerate(remaing) if i not in indicesToRemove]
  #   remaininLabels = [v for i, v in enumerate(remaininLabels) if i not in indicesToRemove]

  #   assert len(remaing) == len(remaininLabels)

  # shuffledData1 = np.vstack(shuffledData1)
  # shuffledData2 = np.vstack(shuffledData2)

  # labelsData1 = np.hstack(labelsData1)
  # labelsData2 = np.hstack(labelsData2)

  # print shuffledData1.shape
  # print shuffledData2.shape

  # assert len(shuffledData1) == len(shuffledData2)
  # assert len(labelsData1) == len(labelsData2)

  # assert len(shuffledData1) <= len(shuffling) / 2

  """  STOP NEW METHOD """
  # shuffledData1 = shuffling[0: len(shuffling) / 2]
  # shuffledData2 = shuffling[len(shuffling)/2 :]

  # labelsData1 = labelsShuffling[0: len(shuffling) /2]
  # labelsData2 = labelsShuffling[len(shuffling)/2:]

  # shuffledData1 = np.array(shuffledData1)
  # shuffledData1 = np.array(shuffledData2)
  # labelsData1 = np.array(labelsData1)
  # labelsData2 = np.array(labelsData2)

  return shuffledData1, shuffledData2, labelsData1, labelsData2

# TODO: I think this can be written easier with the code similar to the Emotions one
# you can create more tuples than just one per image
# you can put each image in 5 tuples and that will probably owrk better
# it might be useful to also give the same image twice
def splitDataMultiPIESubject(imgsPerLabel=None):
  subjectsToImgs = readMultiPIESubjects()

  data1, data2, subjects1, subjects2, shuffling, subjectsShuffling = splitDataInPairsWithLabels(subjectsToImgs, imgsPerLabel, None)

  trainData1, testData1, trainData2, testData2, trainSubjects1, testSubjects1,\
        trainSubjects2, testSubjects2 = splitTrainTest(data1, data2, subjects1, subjects2, 5)

  print "trainData1.shape"
  print trainData1.shape

  shuffledData1, shuffledData2, subjectsData1, subjectsData2 = splitShuffling(shuffling, subjectsShuffling)

  print len(shuffledData1)
  print len(shuffledData2)

  trainShuffedData1, testShuffedData1, trainShuffedData2, testShuffedData2,\
    trainShuffledSubjects1, testShuffledSubjects1, trainShuffledSubjects2, testShuffledSubjects2 =\
        splitTrainTest(shuffledData1, shuffledData2,
                      subjectsData1, subjectsData2, 5)

  trainData1 = np.vstack((trainData1, trainShuffedData1))

  print "trainData1.shape"
  print trainData1.shape

  trainData2 = np.vstack((trainData2, trainShuffedData2))

  testData1 = np.vstack((testData1, testShuffedData1))
  testData2 = np.vstack((testData2, testShuffedData2))

  trainSubjects1 = np.hstack((trainSubjects1, trainShuffledSubjects1))
  testSubjects1 = np.hstack((testSubjects1, testShuffledSubjects1))

  trainSubjects2 = np.hstack((trainSubjects2, trainShuffledSubjects2))
  testSubjects2 = np.hstack((testSubjects2, testShuffledSubjects2))

  assert len(subjects1) == len(subjects2)
  assert len(trainSubjects1) == len(trainSubjects1)
  assert len(testSubjects1) == len(testSubjects2)

  similaritiesTrain = similarityDifferentLabels(trainSubjects1, trainSubjects2)
  similaritiesTest = similarityDifferentLabels(testSubjects1, testSubjects2)

  print "trainSubjects1.shape"
  print trainSubjects1.shape

  print "similaritiesTrain.shape"
  print similaritiesTrain.shape
  # print similaritiesTrain

  assert len(trainData1) == len(trainData2)
  assert len(testData1) == len(testData2)

  trainData1, trainData2, similaritiesTrain = shuffle3(trainData1, trainData2, similaritiesTrain)
  testData1, testData2, similaritiesTest = shuffle3(testData1, testData2, similaritiesTest)

  return trainData1, trainData2, testData1, testData2, similaritiesTrain, similaritiesTest

def splitDataInPairsWithLabels(labelsToImages, imgsPerLabel, labelsToTake=None):
  data1 = []
  data2 = []

  shuffling = []
  labelsShuffling = []
  labels1 = []
  labels2 = []

  for label, images in labelsToImages.iteritems():
    if labelsToTake is not None and label not in labelsToTake:
      print "skipping subject"
      continue

    # The database might contain the labels in similar
    # poses, and illumination conditions, so shuffle before
    np.random.shuffle(images)

    if imgsPerLabel is not None:
      images = images[:imgsPerLabel]

    delta = len(images) / 4 + label % 2
    last2Index = 2 * delta
    data1 += images[0: delta]
    data2 += images[delta: last2Index]

    labels1 += [label] * delta
    labels2 += [label] * delta

    imagesForShuffling = images[last2Index : ]
    shuffling += imagesForShuffling
    labelsShuffling += [label] * len(imagesForShuffling)

  print "len(labelsShuffling)"
  print len(labelsShuffling)

  print "shuffling"
  print len(shuffling)

  assert len(shuffling) == len(labelsShuffling)
  shuffling, labelsShuffling = shuffleList(shuffling, labelsShuffling)

  print len(data1)
  print len(data2)
  assert len(data1) == len(data2)

  data1 = np.array(data1)
  data2 = np.array(data2)
  labels1 = np.array(labels1)
  labels2 = np.array(labels2)
  shuffling = np.array(shuffling)
  labelsShuffling = np.array(labelsShuffling)

  return data1, data2, labels1, labels2, shuffling, labelsShuffling

def splitDataAccordingToLabels(labelsToImages, labelsToTake, imgsPerLabel=None):
  data1, data2, labels1, labels2, shuffling, labelsShuffling  = splitDataInPairsWithLabels(labelsToImages, imgsPerLabel, labelsToTake=labelsToTake)

  shuffledData1, shuffledData2, labelsData1, labelsData2 = splitShuffling(shuffling, labelsShuffling)

  data1 = np.vstack((data1, shuffledData1))
  data2 = np.vstack((data2, shuffledData2))

  labels1 = np.hstack((labels1, labelsData1))
  labels2 = np.hstack((labels2, labelsData2))

  return data1, data2, labels1, labels2

def similarityDifferentLabels(labels1, labels2):
  assert len(labels1) == len(labels2)
  return labels1 == labels2

def splitSimilarityYale():
  subjectsToImgs = readCroppedYaleSubjects()

  # Get all subjects
  data1, data2, subjects1, subjects2 = splitDataAccordingToLabels(subjectsToImgs,
                                          None, imgsPerLabel=None)

  return data1, data2, similarityDifferentLabels(subjects1, subjects2)

def splitSimilaritiesPIEEmotions():
  emotionToImages = readMultiPIEEmotions()
  # Get all emotions
  data1, data2, emotions1, emotions2 = splitDataAccordingToLabels(emotionToImages,
                                          None, imgsPerLabel=None)
  labels = similarityDifferentLabels(emotions1, emotions2)

  data1, data2, labels = shuffle3(data1, data2, labels)

  kf = cross_validation.KFold(n=len(data1), n_folds=5)
  for train, test in kf:
    break

  return (data1[train], data2[train], labels[train],
          data1[test], data2[test], labels[test])

def splitEmotionsMultiPieKeepSubjects():
  subjectToEmotions = readMultiPIEEmotionsPerSubject()

  totalData1 = []
  totalData2 = []
  totalLabels1 = []
  totalLabels2 = []
  for subject, emotionToImages in subjectToEmotions.iteritems():
    data1, data2, labels1, labels2 = splitDataAccordingToLabels(emotionToImages, None, None)
    totalData1 += [data1]
    totalData2 += [data2]
    totalLabels1 += [labels1]
    totalLabels2 += [labels2]

  totalData1 = np.vstack(totalData1)
  totalData1 = np.vstack(totalData2)
  totalLabels1 = np.hstack(totalLabels1)
  totalLabels2 = np.hstack(totalLabels2)
  return totalData1, totalData2, totalLabels1, totalLabels2

def splitEmotionsMultiPieKeepSubjectsTestTrain():
  kf = cross_validation.KFold(n=len(totalData1), n_folds=5)
  for train, test in kf:
    break

  labels = similarityDifferentLabels(totalLabels1, totalLabels2)

  return (totalData1[train], totalData2[train], labels[train],
          totalData1[test], totalData2[test], labels[test])

def testShuffling():
  shuffling = [1,2,3, 4]
  labelsShuffling = [1,2,3, 4]
  a, b, c, d  = splitShuffling(shuffling, labelsShuffling)
  assert not c[0] ==  d[0]
  assert not c[1] ==  d[1]
  assert sorted(list(a) + list(b)) == [1,2,3,4]

  shuffling = [1,2,3]
  labelsShuffling = [1,2,3]
  a, b, c, d  = splitShuffling(shuffling, labelsShuffling)
  assert not c[0] ==  d[0]
  assert not a[0] == b[0]

  shuffling = [1,2,4,5]
  labelsShuffling = [1,2,1,2]
  a, b, c, d  = splitShuffling(shuffling, labelsShuffling)
  assert not c[0] ==  d[0]
  assert not c[1] == d[1]

  fst = sorted(list(a[0]))
  snd = sorted(list(b[0]))

  assert fst == [1,4] or fst == [2,5]
  assert snd == [2,5] or snd == [1,4]

  if fst == [1,4]:
    assert c[0] == 1
  else:
    assert c[0] == 2

  shuffling = [ np.array([1,1]), np.array([2,2]), np.array([4,4]), np.array([5,5]),
                np.array([6, 6]), np.array([7, 7]) ]
  labelsShuffling = [1, 2, 3, 2, 1, 3]
  a, b, c, d  = splitShuffling(shuffling, labelsShuffling)
  assert not c[0] ==  d[0]
  assert not c[1] == d[1]
  assert not c[2] == d[2]

  print zip(a, c)
  print zip(b, d)

if __name__ == '__main__':
  # print shuffleList([1,2], [3,4])
  print len(shuffle([1,2], [3,4]))
  testShuffling()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Yannick Vaucher
#    Copyright 2013 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from openerp.osv import orm

from postlogistics.web_service import PostlogisticsWebService

class stock_picking(orm.Model):
    _inherit = 'stock.picking'

    def _generate_postlogistics_label(self, cr, uid, picking,
                                      webservice_class=None, context=None):
        """ Generate labels and write tracking numbers received """
        user_obj = self.pool.get('res.users')
        user = user_obj.browse(cr, uid, uid, context=context)
        company = user.company_id
        if webservice_class is None:
            webservice_class = PostlogisticsWebService

        web_service = webservice_class(company)
        res = web_service.generate_label(picking, user.lang)

        if 'errors' in res:
            raise orm.except_orm('Error', '\n'.join(res['errors']))

        trackings = set([line.tracking_id for line in picking.move_lines])

        labels = []
        # if there are no pack defined, write tracking_number on picking
        # otherwise, write it on serial field of each pack
        for track in trackings:
            if not track:
                # ignore lines without tracking when there is tracking
                # in a picking
                if len(trackings) > 1:
                    continue
                label = res['value'][0]
                tracking_number = label['tracking_number']
                self.write(cr, uid, picking.id,
                           {'carrier_tracking_ref': tracking_number},
                           context=context)
            else:
                label = None
                for search_label in res['value']:
                    if track.name in search_label['item_id'].split('+')[-1]:
                        label = search_label
                        tracking_number = label['tracking_number']
                        track.write({'serial': tracking_number})
                        break
            labels.append({'tracking_id': track and track.id or False,
                           'file': label['binary'].decode('base64'),
                           'file_type': label['file_type'],
                           'name': tracking_number,
                           })

        return labels

    def generate_shipping_labels(self, cr, uid, ids, context=None):
        """ Add label generation for Postlogistics """
        if isinstance(ids, (long, int)):
            ids = [ids]
        assert len(ids) == 1
        picking = self.browse(cr, uid, ids[0], context=context)
        if picking.carrier_id.type == 'postlogistics':
            return self._generate_postlogistics_label(cr, uid, picking,
                                                      context=context)
        return super(stock_picking, self
                     ).generate_shipping_labels(cr, uid, ids, context=context)

class ShippingLabel(orm.Model):
    """ Child class of ir attachment to identify which are labels """
    _inherit = 'shipping.label'

    def _get_file_type_selection(self, cr, uid, context=None):
        """ Return a concatenated list of extensions of label file format
        plus file format from super

        This will be filtered and sorted in __get_file_type_selection

        :return: list of tuple (code, name)

        """
        file_types = super(ShippingLabel, self
                           )._get_file_type_selection(cr, uid, context=context)
        new_types = [('eps', 'EPS'),
                     ('gif', 'GIF'),
                     ('jpg', 'JPG'),
                     ('png', 'PNG'),
                     ('pdf', 'PDF'),
                     ('spdf', 'sPDF'), # sPDF is a pdf without integrated font
                     ('zpl2', 'ZPL2')]
        file_types.extend(new_types)
        return file_types
<commit_msg>add extension to file name<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Yannick Vaucher
#    Copyright 2013 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from openerp.osv import orm

from postlogistics.web_service import PostlogisticsWebService

class stock_picking(orm.Model):
    _inherit = 'stock.picking'

    def _generate_postlogistics_label(self, cr, uid, picking,
                                      webservice_class=None, context=None):
        """ Generate labels and write tracking numbers received """
        user_obj = self.pool.get('res.users')
        user = user_obj.browse(cr, uid, uid, context=context)
        company = user.company_id
        if webservice_class is None:
            webservice_class = PostlogisticsWebService

        web_service = webservice_class(company)
        res = web_service.generate_label(picking, user.lang)

        if 'errors' in res:
            raise orm.except_orm('Error', '\n'.join(res['errors']))

        trackings = set([line.tracking_id for line in picking.move_lines])

        labels = []
        # if there are no pack defined, write tracking_number on picking
        # otherwise, write it on serial field of each pack
        for track in trackings:
            if not track:
                # ignore lines without tracking when there is tracking
                # in a picking
                if len(trackings) > 1:
                    continue
                label = res['value'][0]
                tracking_number = label['tracking_number']
                self.write(cr, uid, picking.id,
                           {'carrier_tracking_ref': tracking_number},
                           context=context)
            else:
                label = None
                for search_label in res['value']:
                    if track.name in search_label['item_id'].split('+')[-1]:
                        label = search_label
                        tracking_number = label['tracking_number']
                        track.write({'serial': tracking_number})
                        break
            labels.append({'tracking_id': track and track.id or False,
                           'file': label['binary'].decode('base64'),
                           'file_type': label['file_type'],
                           'name': tracking_number + '.' + label['file_type'],
                           })

        return labels

    def generate_shipping_labels(self, cr, uid, ids, context=None):
        """ Add label generation for Postlogistics """
        if isinstance(ids, (long, int)):
            ids = [ids]
        assert len(ids) == 1
        picking = self.browse(cr, uid, ids[0], context=context)
        if picking.carrier_id.type == 'postlogistics':
            return self._generate_postlogistics_label(cr, uid, picking,
                                                      context=context)
        return super(stock_picking, self
                     ).generate_shipping_labels(cr, uid, ids, context=context)

class ShippingLabel(orm.Model):
    """ Child class of ir attachment to identify which are labels """
    _inherit = 'shipping.label'

    def _get_file_type_selection(self, cr, uid, context=None):
        """ Return a concatenated list of extensions of label file format
        plus file format from super

        This will be filtered and sorted in __get_file_type_selection

        :return: list of tuple (code, name)

        """
        file_types = super(ShippingLabel, self
                           )._get_file_type_selection(cr, uid, context=context)
        new_types = [('eps', 'EPS'),
                     ('gif', 'GIF'),
                     ('jpg', 'JPG'),
                     ('png', 'PNG'),
                     ('pdf', 'PDF'),
                     ('spdf', 'sPDF'), # sPDF is a pdf without integrated font
                     ('zpl2', 'ZPL2')]
        file_types.extend(new_types)
        return file_types
<|endoftext|>


<commit_before>from Settings import DefineManager
from Utils import LogManager

def CalculateProcess(fieldData):
    LogManager.PrintLog("FieldHelper",
                        "CalculateProcess", "player field data accepted",
                        DefineManager.LOG_LEVEL_INFO)
    BestCardSwap(fieldData, 0)
    BestCardSwap(fieldData, 1)
    return

def BestCardSwap(fieldData, playerNumber):
    LogManager.PrintLog("FieldHelper",
                        "BestCardSwap", "calculate best card swap player : " + playerNumber,
                        DefineManager.LOG_LEVEL_INFO)
    return<commit_msg>Core/FieldHelper module : fix string bug<commit_after>from Settings import DefineManager
from Utils import LogManager

def CalculateProcess(fieldData):
    LogManager.PrintLog("FieldHelper",
                        "CalculateProcess", "player field data accepted",
                        DefineManager.LOG_LEVEL_INFO)
    BestCardSwap(fieldData, 0)
    BestCardSwap(fieldData, 1)
    return

def BestCardSwap(fieldData, playerNumber):
    LogManager.PrintLog("FieldHelper",
                        "BestCardSwap", "calculate best card swap player : " + str(playerNumber),
                        DefineManager.LOG_LEVEL_INFO)
    return<|endoftext|>


<commit_before>from django.db import models
from django.utils import timezone
from django.db.models import Min

from django.dispatch import receiver
from django.db.models.signals import post_delete

import datetime

# Create your models here.
class Album(models.Model):
    name = models.CharField(max_length = 200)

    created = models.DateTimeField(auto_now_add=True, default=datetime.datetime(2000,1,1))

    def __str__(self):
        return self.name

class Photo(models.Model):
    # constants

    # Compressed Image Resolutions
    THUMBSIZE = (65,65)
    PREVIEWSIZE = (300,200)
    CAROUSELSIZE = (2000,1200)

    name = models.CharField(max_length = 200, blank = True)
    description = models.TextField(blank = True)

    imgOrig = models.ImageField()

    imgPreview = models.ImageField(editable=False)
    imgThumb = models.ImageField(editable=False)
    imgCarousel = models.ImageField(editable=False)

    album = models.ForeignKey(Album)

    added = models.DateTimeField(auto_now_add=True, default = timezone.now())

    # override SAVE to create thumbs and preview image files
    def save(self):
        from PIL import Image, ImageOps
        from io import BytesIO
        from django.core.files.uploadedfile import SimpleUploadedFile
        import os

        def has_changed(instance, field):
            if not instance.pk:
                return False
            old_value = instance.__class__._default_manager.filter(pk=instance.pk).values(field).get()[field]
            return not getattr(instance, field) == old_value

        # Only create new thumbnails if Image file changed or this is a completly new database entry
        if ((self.pk is None) or has_changed(self, 'imgOrig')):
            DJANGO_TYPE = self.imgOrig.file.content_type

            # Possible Image Types
            IMGTYPE = {
                'PNG': {
                        "PIL_TYPE":"png",
                        "EXTENSION":"png",
                        "MIME":"image/png",
                    },
                'JPEG': {
                        "PIL_TYPE":"jpeg",
                        "EXTENSION":"jpg",
                        "MIME":"image/jpeg",
                    }
                }

            # which types to use for which image
            PREVIEW_TYPE = IMGTYPE["PNG"]
            THUMB_TYPE = IMGTYPE["PNG"]
            CAROUSEL_TYPE = IMGTYPE["JPEG"]

            # Open Original photo
            imgPilOrig = Image.open(BytesIO(self.imgOrig.read()))

            # convert original to RGB
            if imgPilOrig.mode not in ('L', 'RGB'):
                imgPilOrig = imgPilOrig.convert('RGB')

            # Create a Copy of the original photo to be resized to every needed size
            imgPilPreview = imgPilOrig.copy()
            imgPilCarousel = imgPilOrig.copy()

            # create Thumbnails (Peview + Carousel) and Icons
            imgPilThumb = ImageOps.fit(imgPilPreview, self.THUMBSIZE, Image.ANTIALIAS)
            imgPilPreview.thumbnail(self.PREVIEWSIZE, Image.ANTIALIAS)
            imgPilCarousel.thumbnail(self.CAROUSELSIZE, Image.ANTIALIAS)

            # IO Stream
            previewHandle = BytesIO()
            thumbHandle = BytesIO()
            carouselHandle  = BytesIO()

            # save the Image in the expected file format
            imgPilPreview.save(previewHandle, PREVIEW_TYPE["PIL_TYPE"])
            imgPilThumb.save(thumbHandle, THUMB_TYPE["PIL_TYPE"])
            imgPilCarousel.save(carouselHandle, CAROUSEL_TYPE["PIL_TYPE"])

            previewHandle.seek(0)
            thumbHandle.seek(0)
            carouselHandle.seek(0)

            #make Django know the newl created files
            previewSuf = SimpleUploadedFile(os.path.split(self.imgOrig.name)[-1], previewHandle.read(), content_type=PREVIEW_TYPE["MIME"])
            thumbSuf = SimpleUploadedFile(os.path.split(self.imgOrig.name)[-1], thumbHandle.read(), content_type=THUMB_TYPE["MIME"])
            carouselSuf = SimpleUploadedFile(os.path.split(self.imgOrig.name)[-1], carouselHandle.read(), content_type=CAROUSEL_TYPE["MIME"])

            # save them
            self.imgThumb.save('%s_thumb.%s'%(os.path.splitext(thumbSuf.name)[0],THUMB_TYPE["EXTENSION"]), thumbSuf, save=False)
            self.imgPreview.save('%s_preview.%s'%(os.path.splitext(previewSuf.name)[0],PREVIEW_TYPE["EXTENSION"]), previewSuf, save=False)
            self.imgCarousel.save('%s_carousel.%s'%(os.path.splitext(carouselSuf.name)[0],CAROUSEL_TYPE["EXTENSION"]), carouselSuf, save=False)

        super(Photo, self).save()

    def __str__(self):
        return self.name + " (in " + self.album.name +")"

    def admin_image(self):
        return '' % self.imgPreview.url

    admin_image.short_description = 'Preview'
    admin_image.allow_tags = True

@receiver(post_delete, sender=Photo)
def delete_photofile(sender, instance, **kwargs):
    imgs = { instance.imgOrig, instance.imgPreview, instance.imgThumb, instance.imgCarousel }
    for img in imgs:
        img.delete(False)<commit_msg>deleted default to get django migrate working<commit_after>from django.db import models
from django.utils import timezone
from django.db.models import Min

from django.dispatch import receiver
from django.db.models.signals import post_delete

import datetime

# Create your models here.
class Album(models.Model):
    name = models.CharField(max_length = 200)

    created = models.DateTimeField(auto_now_add=True)

    def __str__(self):
        return self.name

class Photo(models.Model):
    # constants

    # Compressed Image Resolutions
    THUMBSIZE = (65,65)
    PREVIEWSIZE = (300,200)
    CAROUSELSIZE = (2000,1200)

    name = models.CharField(max_length = 200, blank = True)
    description = models.TextField(blank = True)

    imgOrig = models.ImageField()

    imgPreview = models.ImageField(editable=False)
    imgThumb = models.ImageField(editable=False)
    imgCarousel = models.ImageField(editable=False)

    album = models.ForeignKey(Album)

    added = models.DateTimeField(auto_now_add=True)

    # override SAVE to create thumbs and preview image files
    def save(self):
        from PIL import Image, ImageOps
        from io import BytesIO
        from django.core.files.uploadedfile import SimpleUploadedFile
        import os

        def has_changed(instance, field):
            if not instance.pk:
                return False
            old_value = instance.__class__._default_manager.filter(pk=instance.pk).values(field).get()[field]
            return not getattr(instance, field) == old_value

        # Only create new thumbnails if Image file changed or this is a completly new database entry
        if ((self.pk is None) or has_changed(self, 'imgOrig')):
            DJANGO_TYPE = self.imgOrig.file.content_type

            # Possible Image Types
            IMGTYPE = {
                'PNG': {
                        "PIL_TYPE":"png",
                        "EXTENSION":"png",
                        "MIME":"image/png",
                    },
                'JPEG': {
                        "PIL_TYPE":"jpeg",
                        "EXTENSION":"jpg",
                        "MIME":"image/jpeg",
                    }
                }

            # which types to use for which image
            PREVIEW_TYPE = IMGTYPE["PNG"]
            THUMB_TYPE = IMGTYPE["PNG"]
            CAROUSEL_TYPE = IMGTYPE["JPEG"]

            # Open Original photo
            imgPilOrig = Image.open(BytesIO(self.imgOrig.read()))

            # convert original to RGB
            if imgPilOrig.mode not in ('L', 'RGB'):
                imgPilOrig = imgPilOrig.convert('RGB')

            # Create a Copy of the original photo to be resized to every needed size
            imgPilPreview = imgPilOrig.copy()
            imgPilCarousel = imgPilOrig.copy()

            # create Thumbnails (Peview + Carousel) and Icons
            imgPilThumb = ImageOps.fit(imgPilPreview, self.THUMBSIZE, Image.ANTIALIAS)
            imgPilPreview.thumbnail(self.PREVIEWSIZE, Image.ANTIALIAS)
            imgPilCarousel.thumbnail(self.CAROUSELSIZE, Image.ANTIALIAS)

            # IO Stream
            previewHandle = BytesIO()
            thumbHandle = BytesIO()
            carouselHandle  = BytesIO()

            # save the Image in the expected file format
            imgPilPreview.save(previewHandle, PREVIEW_TYPE["PIL_TYPE"])
            imgPilThumb.save(thumbHandle, THUMB_TYPE["PIL_TYPE"])
            imgPilCarousel.save(carouselHandle, CAROUSEL_TYPE["PIL_TYPE"])

            previewHandle.seek(0)
            thumbHandle.seek(0)
            carouselHandle.seek(0)

            #make Django know the newl created files
            previewSuf = SimpleUploadedFile(os.path.split(self.imgOrig.name)[-1], previewHandle.read(), content_type=PREVIEW_TYPE["MIME"])
            thumbSuf = SimpleUploadedFile(os.path.split(self.imgOrig.name)[-1], thumbHandle.read(), content_type=THUMB_TYPE["MIME"])
            carouselSuf = SimpleUploadedFile(os.path.split(self.imgOrig.name)[-1], carouselHandle.read(), content_type=CAROUSEL_TYPE["MIME"])

            # save them
            self.imgThumb.save('%s_thumb.%s'%(os.path.splitext(thumbSuf.name)[0],THUMB_TYPE["EXTENSION"]), thumbSuf, save=False)
            self.imgPreview.save('%s_preview.%s'%(os.path.splitext(previewSuf.name)[0],PREVIEW_TYPE["EXTENSION"]), previewSuf, save=False)
            self.imgCarousel.save('%s_carousel.%s'%(os.path.splitext(carouselSuf.name)[0],CAROUSEL_TYPE["EXTENSION"]), carouselSuf, save=False)

        super(Photo, self).save()

    def __str__(self):
        return self.name + " (in " + self.album.name +")"

    def admin_image(self):
        return '' % self.imgPreview.url

    admin_image.short_description = 'Preview'
    admin_image.allow_tags = True

@receiver(post_delete, sender=Photo)
def delete_photofile(sender, instance, **kwargs):
    imgs = { instance.imgOrig, instance.imgPreview, instance.imgThumb, instance.imgCarousel }
    for img in imgs:
        img.delete(False)<|endoftext|>