<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

from __future__ import division
from pylab import *
import os
import glob
from scipy import interpolate
import time

FigureSize = [8,9]
Spectrapath = '/afs/psi.ch/user/h/haberthuer/EssentialMed/Images/12-GOTTHARD_and_TIS/GOTTHARD'
Spectra = sort(glob.glob(os.path.join(Spectrapath, '*.txt')))

FileName = [os.path.basename(item) for item in Spectra]
Data = [np.loadtxt(item) for item in Spectra]
DataName = [open(item).readlines()[0].split()[0][1:-2] for item in Spectra]

# Get Filenames of Spectra and split it up into the desired values like kV, mAs
# and exposure time with some basic string handling.
Modality = [item.split('_')[0] for item in FileName]
Energy = [int(item.split('_')[1][:-2]) for item in FileName]
Current = [int(item.split('_')[2][:-2]) for item in FileName]
mAs = [float(item.split('_')[3][:-3]) for item in FileName]
ExposureTime = [int(item.split('_')[4][:-6]) for item in FileName]
# Generate Labels which we can use later-on
Label = [Modality[i] + ': ' + str(Energy[i]) + 'kV, ' + str(mAs[i]) + 'mAs, ' + str(ExposureTime[i]) + 'ms' for i in range(len(Spectra))]

Frames = [open(item).readlines()[0].split()[1] for item in Spectra]
BinCenter = [open(item).readlines()[1].split()[0] for item in Spectra]
Photons = [open(item).readlines()[1].split()[1] for item in Spectra]
PhotonsPerFrame = [open(item).readlines()[1].split()[2] for item in Spectra]

# Calculate attenuation in 320 um of Silicon
SiliconAttenuation = np.loadtxt('Si_Attenuation.dat')
SiliconDensity = 2.329  # g/cm³
SiliconThickness = 320  # um

# Plot transmission
plt.figure(figsize=FigureSize)
plt.plot(SiliconAttenuation[:, 0]*1000,
         (np.exp(- (SiliconAttenuation[:, 1] * SiliconDensity * SiliconThickness / 10000 ))),
         'ro', label='Si Transmission')
plt.xlabel('Photon Energy [keV]')
plt.ylabel('Tranmission')
plt.ylim([0,1])
plt.savefig('1_Si_Transmission.pdf', transparent=True)

# Plot transmission
plt.figure(figsize=FigureSize)
plt.loglog(SiliconAttenuation[:, 0], SiliconAttenuation[:, 1],
         'r', label='Si Transmission')
plt.xlabel('Photon Energy [keV]')
plt.ylabel('Tranmission')
plt.savefig('1_Si_Transmission.pdf', transparent=True)

plt.show()
sys.exit()

# Plot transmission-zoom
plt.figure(figsize=FigureSize)
for i in reversed(range(5000,20250,500)):
    plt.plot(SiliconAttenuation[:, 0]*1000,
            (np.exp(- (SiliconAttenuation[:, 1] * SiliconDensity * SiliconThickness / 10000 ))),
            'ro', label='Si Transmission')
    plt.xlabel('Photon Energy [keV]')
    plt.ylabel('Tranmission')
    plt.ylim([0,1])
    plt.xlim(xmin=0)
    plt.xlim(xmax=i)
    print '%05i' % i
    plt.draw()
    plt.savefig('anim' + str('%05i' % (20000 - i)) + '.png', transparent=True)

for i in reversed(range(1000,5000,200)):
    plt.plot(SiliconAttenuation[:, 0]*1000,
            (np.exp(- (SiliconAttenuation[:, 1] * SiliconDensity * SiliconThickness / 10000 ))),
            'ro', label='Si Transmission')
    plt.xlabel('Photon Energy [keV]')
    plt.ylabel('Tranmission')
    plt.ylim([0,1])
    plt.xlim(xmin=0)
    plt.xlim(xmax=i)
    print '%05i' % i
    plt.draw()
    plt.savefig('anim' + str('%05i' % (20000 - i)) + '.png', transparent=True)

for i in reversed(range(120,1000,50)):
    plt.plot(SiliconAttenuation[:, 0]*1000,
            (np.exp(- (SiliconAttenuation[:, 1] * SiliconDensity * SiliconThickness / 10000 ))),
            'ro', label='Si Transmission')
    plt.xlabel('Photon Energy [keV]')
    plt.ylabel('Tranmission')
    plt.ylim([0,1])
    plt.xlim(xmin=0)
    plt.xlim(xmax=i)
    print '%05i' % i
    plt.draw()
    plt.savefig('anim' + str('%05i' % (20000 - i)) + '.png', transparent=True)

# Plot transmission with the limits that are interesting for us
plt.figure(figsize=FigureSize)
plt.plot(SiliconAttenuation[:, 0]*1000,
         (np.exp(- (SiliconAttenuation[:, 1] * SiliconDensity * SiliconThickness / 10000 ))),
         'ro', label='Si Transmission')
plt.xlabel('Photon Energy [keV]')
plt.ylabel('Tranmission')
plt.xlim([0,120])
plt.ylim([0,1])
plt.savefig('2_Si_Transmission_limits.pdf', transparent=True)
#~ plt.show()

# Plot interpolated transmission
x = SiliconAttenuation[:, 0] * 1000
y = (np.exp(- (SiliconAttenuation[:, 1] * SiliconDensity * SiliconThickness / 10000 )))
interpolated = interpolate.interp1d(x, y, kind='cubic')
xnew = np.linspace(1,120,1000)  # get 1000 steps from 1 to 120

plt.figure(figsize=FigureSize)
plt.plot(SiliconAttenuation[:, 0]*1000,
         (np.exp(- (SiliconAttenuation[:, 1] * SiliconDensity * SiliconThickness / 10000 ))),
         'ro', label='Si Transmission')
plt.plot(xnew,interpolated(xnew),'r',label='Interpolated values')
plt.xlabel('Photon Energy [keV]')
plt.ylabel('Tranmission')
plt.xlim([0,120])
plt.ylim([0,1])
plt.savefig('3_Si_Transmission_limits_interpolated.pdf', transparent=True)

print 'Plotting uncorrected spectra'
for i in range(int(len(Spectra)/2)):
    plt.figure(figsize=FigureSize)
    k = i + int(len(Spectra)/2)
    plt.plot(Data[i][:, 0], Data[i][:, 1], label=Label[i], color='k')
    plt.plot(Data[k][:, 0], Data[k][:, 1], label=Label[k], color='g')
    plt.legend(loc=1)
    plt.xlim([0,5000])
    plt.ylim(ymin=0)
    plt.savefig('4_' + DataName[i] + '.pdf', transparent=True)

print 'Plotting corrected spectra'
for i in range(int(len(Spectra)/2)):
    plt.figure(figsize=FigureSize)
    k = i + int(len(Spectra)/2)
    plt.plot(Data[i][:, 0], Data[i][:, 1], color='k', alpha=0.125)
    plt.plot(Data[k][:, 0], Data[k][:, 1], color='g', alpha=0.125)
    plt.plot(xnew * 5000 / 120, interpolated(xnew) * Data[i][:, 1], color='k',
             label=Label[i])
    plt.plot(xnew * 5000 / 120, interpolated(xnew) * Data[k][:, 1], color='g',
             label=Label[k])
    plt.legend(loc=1)
    plt.xlim([0,5000])
    plt.ylim(ymin=0)
    plt.savefig('5_' + DataName[i] + 'corrected.pdf', transparent=True)

print 'Plotting corrected log-spectra'
for i in range(int(len(Spectra)/2)):
    plt.figure(figsize=FigureSize)
    k = i + int(len(Spectra)/2)
    plt.semilogy(Data[i][:, 0], Data[i][:, 1], color='k', alpha=0.125)
    plt.semilogy(Data[k][:, 0], Data[k][:, 1], color='g', alpha=0.125)
    plt.semilogy(xnew * 5000 / 120, interpolated(xnew) * Data[i][:, 1], color='k',
             label=Label[i])
    plt.semilogy(xnew * 5000 / 120, interpolated(xnew) * Data[k][:, 1], color='g',
             label=Label[k])
    plt.legend(loc=1)
    plt.xlim([0,5000])
    plt.ylim(ymin=1)
    plt.savefig('6_' + DataName[i] + 'log.pdf', transparent=True)

sys.exit()

print 'Plotting Spectra, Logplot and Difference for'
for i in range(int(len(Spectra)/2)):
    plt.figure(figsize=FigureSize)
    k = i + int(len(Spectra)/2)
    print '    * ' + DataName[i], 'vs.', DataName[k]
    print '        * for', DataName[i], 'we recorded',\
        '%.3e' % int(np.sum(Data[i][:, 1])), 'photons.'
    print '        * for', DataName[k], 'we recorded',\
        '%.3e' % int(np.sum(Data[k][:, 1])), 'photons.'
    print '        * the difference is',\
        '%.3e' % int(np.sum(Data[i][:, 1]) - np.sum(Data[k][:, 1])), 'photons'
    plt.subplot(1, 3, 1)
    plt.plot(Data[i][:, 0], Data[i][:, 1], label=DataName[i], color='k')
    plt.plot(Data[k][:, 0], Data[k][:, 1], label=DataName[k], color='g')
    plt.xlabel('BinCenter [adc]')
    plt.ylabel('Photons [count]')
    plt.xlim(xmin=0)
    # Plotting left, right and then middle, so we have the legend on top
    plt.subplot(1, 3, 3)
    plt.plot(Data[i][:, 0], Data[i][:, 1], color='k',
             label=' '.join([Modality[i] + ',', str(Energy[i]) + 'kV,',
                             str(mAs[i]) + 'mAs']))
    plt.plot(Data[k][:, 0], Data[k][:, 1], color='g',
             label=' '.join([Modality[k][:2] + ' ' + Modality[k][13:] + ',',
             str(Energy[k]) + 'kV,', str(mAs[k]) + 'mAs']))
    plt.plot(Data[i][:, 0], (Data[i] - Data[k])[:, 1],
             label='Difference', color='r')
    plt.legend(loc='center', bbox_to_anchor=(0.5, 0.1))
    plt.xlabel('BinCenter [adc]')
    plt.ylabel('Photons [count]')
    plt.title('Difference')
    plt.xlim(xmin=0)
    plt.subplot(1, 3, 2)
    plt.semilogy(Data[i][:, 0], Data[i][:, 1], label=DataName[i], color='k')
    plt.semilogy(Data[k][:, 0], Data[k][:, 1], label=DataName[k], color='g')
    plt.xlabel('BinCenter [adc]')
    plt.ylabel('Photons [log count]')
    plt.legend(loc='center', bbox_to_anchor=(0.5, 0.9))
    plt.xlim(xmin=0)
    plt.savefig(os.path.join('img', 'Full_' + DataName[i] + '.png'),
                transparent=True)
    plt.savefig(os.path.join('img', 'Full_' + DataName[i] + '.pdf'),
                transparent=True)
    plt.show()

exit()

print
print 'Plotting'
for i in range(int(len(Spectra)/2)):
    plt.figure(figsize=FigureSize)
    k = i + int(len(Spectra)/2)
    print '    * ' + DataName[i] + '/' + DataName[k]
    plt.plot(Data[i][:, 0], Data[i][:, 1], color='k',
                 label=' '.join([Modality[i] + ',',
                                 str(Energy[i]) + 'kV,',
                                 str(mAs[i]) + 'mAs,',
                                 str(ExposureTime[i]) + 'ms']))
    plt.plot(Data[k][:, 0], Data[k][:, 1], color='g',
                 label=' '.join([Modality[k] + ',',
                                 str(Energy[k]) + 'kV,',
                                 str(mAs[k]) + 'mAs,',
                                 str(ExposureTime[k]) + 'ms']))
    plt.plot(Data[i][:, 0], (Data[i] - Data[k])[:, 1],
             label='Difference', color='r')
    plt.legend(loc='best')
    plt.xlabel('BinCenter [adc]')
    plt.xlim(xmin=0)
    plt.savefig(os.path.join('img', 'Photons_' + DataName[i] + '.png'),
                transparent=True)
    plt.savefig(os.path.join('img', 'Photons_' + DataName[i] + '.pdf'),
                transparent=True)
    #~ plt.show()

print
print 'Plotting Logplot for every modality'
for i in range(int(len(Spectra)/2)):
    plt.figure(figsize=FigureSize)
    k = i + int(len(Spectra)/2)
    print '    * ' + DataName[i] + '/' + DataName[k]
    plt.semilogy(Data[i][:, 0], Data[i][:, 1], color='k',
                 label=' '.join([Modality[i] + ',',
                                 str(Energy[i]) + 'kV,',
                                 str(mAs[i]) + 'mAs,',
                                 str(ExposureTime[i]) + 'ms']))
    plt.semilogy(Data[k][:, 0], Data[k][:, 1], color='g',
                 label=' '.join([Modality[k] + ',',
                                 str(Energy[k]) + 'kV,',
                                 str(mAs[k]) + 'mAs,',
                                 str(ExposureTime[k]) + 'ms']))
    plt.semilogy(Data[i][:, 0], (Data[i] - Data[k])[:, 1],
                 label='Difference', color='r')
    plt.legend(loc='best')
    plt.xlabel('BinCenter [adc]')
    plt.xlim(xmin=0)
    plt.savefig(os.path.join('img', 'Log_Photons_' + DataName[i] + '.png'),
                transparent=True)
    plt.savefig(os.path.join('img', 'Log_Photons_' + DataName[i] + '.pdf'),
                transparent=True)
    #~ plt.show()
<commit_msg>Version used for plotting the ProgressReport stuff<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

from __future__ import division
from pylab import *
import os
import glob
from scipy import interpolate
import time

FigureSize = [8,9]
Spectrapath = '/afs/psi.ch/user/h/haberthuer/EssentialMed/Images/12-GOTTHARD_and_TIS/GOTTHARD'
Spectra = sort(glob.glob(os.path.join(Spectrapath, '*.txt')))

FileName = [os.path.basename(item) for item in Spectra]
Data = [np.loadtxt(item) for item in Spectra]
DataName = [open(item).readlines()[0].split()[0][1:-2] for item in Spectra]

# Get Filenames of Spectra and split it up into the desired values like kV, mAs
# and exposure time with some basic string handling.
Modality = [item.split('_')[0] for item in FileName]
Energy = [int(item.split('_')[1][:-2]) for item in FileName]
Current = [int(item.split('_')[2][:-2]) for item in FileName]
mAs = [float(item.split('_')[3][:-3]) for item in FileName]
ExposureTime = [int(item.split('_')[4][:-6]) for item in FileName]
# Generate Labels which we can use later-on
Label = [Modality[i] + ': ' + str(Energy[i]) + 'kV, ' + str(mAs[i]) + 'mAs, ' + str(ExposureTime[i]) + 'ms' for i in range(len(Spectra))]

Frames = [open(item).readlines()[0].split()[1] for item in Spectra]
BinCenter = [open(item).readlines()[1].split()[0] for item in Spectra]
Photons = [open(item).readlines()[1].split()[1] for item in Spectra]
PhotonsPerFrame = [open(item).readlines()[1].split()[2] for item in Spectra]

# Calculate attenuation in 320 um of Silicon
SiliconAttenuation = np.loadtxt('Si_Attenuation.dat')
SiliconDensity = 2.329  # g/cm³
SiliconThickness = 320  # um

# Plot transmission
plt.figure(figsize=FigureSize)
plt.plot(SiliconAttenuation[:, 0]*1000,
         (np.exp(- (SiliconAttenuation[:, 1] * SiliconDensity * SiliconThickness / 10000 ))),
         'ro', label='Si Transmission')
plt.xlabel('Photon Energy [keV]')
plt.ylabel('Tranmission')
plt.ylim([0,1])
plt.savefig('1_Si_Transmission.pdf', transparent=True)

# Plot transmission
plt.figure(figsize=FigureSize)
plt.loglog(SiliconAttenuation[:, 0], SiliconAttenuation[:, 1],
         'r', label='Si Transmission')
plt.xlabel('Photon Energy [keV]')
plt.ylabel('Tranmission')
plt.savefig('1_Si_Transmission.pdf', transparent=True)

# Plot transmission-zoom
plt.figure(figsize=FigureSize)
for i in reversed(range(5000,20250,500)):
    plt.plot(SiliconAttenuation[:, 0]*1000,
            (np.exp(- (SiliconAttenuation[:, 1] * SiliconDensity * SiliconThickness / 10000 ))),
            'ro', label='Si Transmission')
    plt.xlabel('Photon Energy [keV]')
    plt.ylabel('Tranmission')
    plt.ylim([0,1])
    plt.xlim(xmin=0)
    plt.xlim(xmax=i)
    print '%05i' % i
    plt.draw()
    plt.savefig('anim' + str('%05i' % (20000 - i)) + '.png', transparent=True)

for i in reversed(range(1000,5000,200)):
    plt.plot(SiliconAttenuation[:, 0]*1000,
            (np.exp(- (SiliconAttenuation[:, 1] * SiliconDensity * SiliconThickness / 10000 ))),
            'ro', label='Si Transmission')
    plt.xlabel('Photon Energy [keV]')
    plt.ylabel('Tranmission')
    plt.ylim([0,1])
    plt.xlim(xmin=0)
    plt.xlim(xmax=i)
    print '%05i' % i
    plt.draw()
    plt.savefig('anim' + str('%05i' % (20000 - i)) + '.png', transparent=True)

for i in reversed(range(120,1000,50)):
    plt.plot(SiliconAttenuation[:, 0]*1000,
            (np.exp(- (SiliconAttenuation[:, 1] * SiliconDensity * SiliconThickness / 10000 ))),
            'ro', label='Si Transmission')
    plt.xlabel('Photon Energy [keV]')
    plt.ylabel('Tranmission')
    plt.ylim([0,1])
    plt.xlim(xmin=0)
    plt.xlim(xmax=i)
    print '%05i' % i
    plt.draw()
    plt.savefig('anim' + str('%05i' % (20000 - i)) + '.png', transparent=True)

# Plot transmission with the limits that are interesting for us
plt.figure(figsize=FigureSize)
plt.plot(SiliconAttenuation[:, 0]*1000,
         (np.exp(- (SiliconAttenuation[:, 1] * SiliconDensity * SiliconThickness / 10000 ))),
         'ro', label='Si Transmission')
plt.xlabel('Photon Energy [keV]')
plt.ylabel('Tranmission')
plt.xlim([0,120])
plt.ylim([0,1])
plt.savefig('2_Si_Transmission_limits.pdf', transparent=True)
plt.show()

# Plot interpolated transmission
x = SiliconAttenuation[:, 0] * 1000
y = (np.exp(- (SiliconAttenuation[:, 1] * SiliconDensity * SiliconThickness / 10000 )))
interpolated = interpolate.interp1d(x, y, kind='cubic')
xnew = np.linspace(1,120,1000)  # get 1000 steps from 1 to 120

plt.figure(figsize=FigureSize)
plt.plot(SiliconAttenuation[:, 0]*1000,
         (np.exp(- (SiliconAttenuation[:, 1] * SiliconDensity * SiliconThickness / 10000 ))),
         'ro', label='Si Transmission')
plt.plot(xnew,interpolated(xnew),'r',label='Interpolated values')
plt.xlabel('Photon Energy [keV]')
plt.ylabel('Tranmission')
plt.xlim([0,120])
plt.ylim([0,1])
plt.savefig('3_Si_Transmission_limits_interpolated.pdf', transparent=True)

print 'Plotting uncorrected spectra'
for i in range(int(len(Spectra)/2)):
    plt.figure(figsize=FigureSize)
    k = i + int(len(Spectra)/2)
    plt.plot(Data[i][:, 0], Data[i][:, 1], label=Label[i], color='k')
    plt.plot(Data[k][:, 0], Data[k][:, 1], label=Label[k], color='g')
    plt.legend(loc=1)
    plt.xlabel('BinCenter [adc]')
    plt.ylabel('Photons [count]')
    plt.xlim([0,5000])
    plt.ylim(ymin=0)
    plt.savefig('4_' + DataName[i] + '.pdf', transparent=True)

print 'Plotting corrected spectra'
for i in range(int(len(Spectra)/2)):
    plt.figure(figsize=FigureSize)
    k = i + int(len(Spectra)/2)
    plt.plot(Data[i][:, 0], Data[i][:, 1], color='k', alpha=0.125)
    plt.plot(Data[k][:, 0], Data[k][:, 1], color='g', alpha=0.125)
    plt.plot(xnew * 5000 / 120, interpolated(xnew) * Data[i][:, 1], color='k',
             label=Label[i])
    plt.plot(xnew * 5000 / 120, interpolated(xnew) * Data[k][:, 1], color='g',
             label=Label[k])
    plt.legend(loc=1)
    plt.xlabel('BinCenter [adc]')
    plt.ylabel('Photons [count]')
    plt.xlim([0,5000])
    plt.ylim(ymin=0)
    plt.savefig('5_' + DataName[i] + 'corrected.pdf', transparent=True)

print 'Plotting corrected log-spectra'
for i in range(int(len(Spectra)/2)):
    plt.figure(figsize=FigureSize)
    k = i + int(len(Spectra)/2)
    plt.semilogy(Data[i][:, 0], Data[i][:, 1], color='k', alpha=0.125)
    plt.semilogy(Data[k][:, 0], Data[k][:, 1], color='g', alpha=0.125)
    plt.semilogy(xnew * 5000 / 120, interpolated(xnew) * Data[i][:, 1], color='k',
             label=Label[i])
    plt.semilogy(xnew * 5000 / 120, interpolated(xnew) * Data[k][:, 1], color='g',
             label=Label[k])
    plt.xlabel('BinCenter [adc]')
    plt.ylabel('Photons [count]')
    plt.legend(loc=1)
    plt.xlim([0,5000])
    plt.ylim(ymin=1)
    plt.savefig('6_' + DataName[i] + 'log.pdf', transparent=True)

sys.exit()

print 'Plotting Spectra, Logplot and Difference for'
for i in range(int(len(Spectra)/2)):
    plt.figure(figsize=FigureSize)
    k = i + int(len(Spectra)/2)
    print '    * ' + DataName[i], 'vs.', DataName[k]
    print '        * for', DataName[i], 'we recorded',\
        '%.3e' % int(np.sum(Data[i][:, 1])), 'photons.'
    print '        * for', DataName[k], 'we recorded',\
        '%.3e' % int(np.sum(Data[k][:, 1])), 'photons.'
    print '        * the difference is',\
        '%.3e' % int(np.sum(Data[i][:, 1]) - np.sum(Data[k][:, 1])), 'photons'
    plt.subplot(1, 3, 1)
    plt.plot(Data[i][:, 0], Data[i][:, 1], label=DataName[i], color='k')
    plt.plot(Data[k][:, 0], Data[k][:, 1], label=DataName[k], color='g')
    plt.xlabel('BinCenter [adc]')
    plt.ylabel('Photons [count]')
    plt.xlim(xmin=0)
    # Plotting left, right and then middle, so we have the legend on top
    plt.subplot(1, 3, 3)
    plt.plot(Data[i][:, 0], Data[i][:, 1], color='k',
             label=' '.join([Modality[i] + ',', str(Energy[i]) + 'kV,',
                             str(mAs[i]) + 'mAs']))
    plt.plot(Data[k][:, 0], Data[k][:, 1], color='g',
             label=' '.join([Modality[k][:2] + ' ' + Modality[k][13:] + ',',
             str(Energy[k]) + 'kV,', str(mAs[k]) + 'mAs']))
    plt.plot(Data[i][:, 0], (Data[i] - Data[k])[:, 1],
             label='Difference', color='r')
    plt.legend(loc='center', bbox_to_anchor=(0.5, 0.1))
    plt.xlabel('BinCenter [adc]')
    plt.ylabel('Photons [count]')
    plt.title('Difference')
    plt.xlim(xmin=0)
    plt.subplot(1, 3, 2)
    plt.semilogy(Data[i][:, 0], Data[i][:, 1], label=DataName[i], color='k')
    plt.semilogy(Data[k][:, 0], Data[k][:, 1], label=DataName[k], color='g')
    plt.xlabel('BinCenter [adc]')
    plt.ylabel('Photons [log count]')
    plt.legend(loc='center', bbox_to_anchor=(0.5, 0.9))
    plt.xlim(xmin=0)
    plt.savefig(os.path.join('img', 'Full_' + DataName[i] + '.png'),
                transparent=True)
    plt.savefig(os.path.join('img', 'Full_' + DataName[i] + '.pdf'),
                transparent=True)
    # plt.show()

exit()

print
print 'Plotting'
for i in range(int(len(Spectra)/2)):
    plt.figure(figsize=FigureSize)
    k = i + int(len(Spectra)/2)
    print '    * ' + DataName[i] + '/' + DataName[k]
    plt.plot(Data[i][:, 0], Data[i][:, 1], color='k',
                 label=' '.join([Modality[i] + ',',
                                 str(Energy[i]) + 'kV,',
                                 str(mAs[i]) + 'mAs,',
                                 str(ExposureTime[i]) + 'ms']))
    plt.plot(Data[k][:, 0], Data[k][:, 1], color='g',
                 label=' '.join([Modality[k] + ',',
                                 str(Energy[k]) + 'kV,',
                                 str(mAs[k]) + 'mAs,',
                                 str(ExposureTime[k]) + 'ms']))
    plt.plot(Data[i][:, 0], (Data[i] - Data[k])[:, 1],
             label='Difference', color='r')
    plt.legend(loc='best')
    plt.xlabel('BinCenter [adc]')
    plt.xlim(xmin=0)
    plt.savefig(os.path.join('img', 'Photons_' + DataName[i] + '.png'),
                transparent=True)
    plt.savefig(os.path.join('img', 'Photons_' + DataName[i] + '.pdf'),
                transparent=True)
    #~ plt.show()

print
print 'Plotting Logplot for every modality'
for i in range(int(len(Spectra)/2)):
    plt.figure(figsize=FigureSize)
    k = i + int(len(Spectra)/2)
    print '    * ' + DataName[i] + '/' + DataName[k]
    plt.semilogy(Data[i][:, 0], Data[i][:, 1], color='k',
                 label=' '.join([Modality[i] + ',',
                                 str(Energy[i]) + 'kV,',
                                 str(mAs[i]) + 'mAs,',
                                 str(ExposureTime[i]) + 'ms']))
    plt.semilogy(Data[k][:, 0], Data[k][:, 1], color='g',
                 label=' '.join([Modality[k] + ',',
                                 str(Energy[k]) + 'kV,',
                                 str(mAs[k]) + 'mAs,',
                                 str(ExposureTime[k]) + 'ms']))
    plt.semilogy(Data[i][:, 0], (Data[i] - Data[k])[:, 1],
                 label='Difference', color='r')
    plt.legend(loc='best')
    plt.xlabel('BinCenter [adc]')
    plt.xlim(xmin=0)
    plt.savefig(os.path.join('img', 'Log_Photons_' + DataName[i] + '.png'),
                transparent=True)
    plt.savefig(os.path.join('img', 'Log_Photons_' + DataName[i] + '.pdf'),
                transparent=True)
    #~ plt.show()
<|endoftext|>


<commit_before>from acme.tf import utils as tf2_utils
from acme.agents.tf import actors
import tensorflow as tf

def generate_train_data(policy_net, environment, n_samples):
    sample_count = 0
    actor = actors.FeedForwardActor(policy_network=policy_net)
    timestep = environment.reset()
    actor.observe_first(timestep)

    current_obs_list = []
    action_list = []
    next_obs_list = []
    reward_list = []
    discount_list = []
    while sample_count < n_samples:
        current_obs = timestep.observation
        action = actor.select_action(current_obs)
        timestep = environment.step(action)
        actor.observe(action, next_timestep=timestep)
        next_obs = timestep.observation
        reward = timestep.reward
        discount = 1.0

        current_obs_list.append(tf2_utils.add_batch_dim(current_obs))
        action_list.append(tf2_utils.add_batch_dim(action))
        reward_list.append(tf2_utils.add_batch_dim(reward))
        discount_list.append(tf2_utils.add_batch_dim(discount))
        next_obs_list.append(tf2_utils.add_batch_dim(next_obs))

        if timestep.last():
            timestep = environment.reset()
            actor.observe_first(timestep)

        sample_count += 1

    current_obs_data = tf.concat(current_obs_list, axis=0)
    action_data = tf.concat(action_list, axis=0)
    next_obs_data = tf.concat(next_obs_list, axis=0)
    reward_data = tf.concat(reward_list, axis=0)
    discount_data = tf.concat(discount_list, axis=0)

    return tf.data.Dataset.from_tensor_slices((current_obs_data,
                                               action_data,
                                               reward_data,
                                               discount_data,
                                               next_obs_data,))
<commit_msg>bug fix<commit_after>from acme.tf import utils as tf2_utils
from acme.agents.tf import actors
import tensorflow as tf

def generate_train_data(policy_net, environment, n_samples):
    sample_count = 0
    actor = actors.FeedForwardActor(policy_network=policy_net)
    timestep = environment.reset()
    actor.observe_first(timestep)

    current_obs_list = []
    action_list = []
    next_obs_list = []
    reward_list = []
    discount_list = []
    while sample_count < n_samples:
        current_obs = timestep.observation
        action = actor.select_action(current_obs)
        timestep = environment.step(action)
        actor.observe(action, next_timestep=timestep)
        next_obs = timestep.observation
        reward = timestep.reward
        discount = 1.0

        current_obs_list.append(tf2_utils.add_batch_dim(current_obs))
        action_list.append(tf2_utils.add_batch_dim(action))
        reward_list.append(tf2_utils.add_batch_dim(reward))
        discount_list.append(tf2_utils.add_batch_dim(discount))
        next_obs_list.append(tf2_utils.add_batch_dim(next_obs))

        if timestep.last():
            timestep = environment.reset()
            actor.observe_first(timestep)

        sample_count += 1

    current_obs_data = tf.concat(current_obs_list, axis=0)
    action_data = tf.concat(action_list, axis=0)
    next_obs_data = tf.concat(next_obs_list, axis=0)
    reward_data = tf.concat(reward_list, axis=0)
    discount_data = tf.concat(discount_list, axis=0)

    return tf.data.Dataset.from_tensor_slices((current_obs_data,
                                               action_data,
                                               reward_data,
                                               discount_data,
                                               next_obs_data,
                                               action_data))
<|endoftext|>


<commit_before>#!/usr/bin/python3

import socket
import threading
import socketserver
import time
import random
import pickle
import requests
import re
from lxml import html
import ff1
import sqlite3

sleep = time.sleep

def do_c(job):
    page = requests.get(job[1] + "1")
    if page.status_code == 204:  # Network stuff
        sleep(120)
        return 1

    t = html.fromstring(page.content)
    t.xpath('//div[@class="search-result__r1"]/div/a/@href')
    t_cars = t.xpath(
        '//h1[@class="search-form__count js-results-count"]/text()'
    )[0]
    t_cars = re.search('[0-9,]+', t_cars).group(0)
    t_cars = int(t_cars.replace(",", ""))
    pageno = t_cars // 10 + 1

    for i in range(pageno):
        add_jlist(("b", job[1] + str(i + 1)))  # c jobs produce b jobs

    print("C " + str(pageno))
    return 0

def updater():
    global jlist, jlock
    with sqlite3.connect('test.db') as conn:
        c = conn.cursor()
        now = int(time.time())
        cutoff = now - 60 * 60 * 24
        c.execute('SELECT url FROM cars WHERE last_seen < ?'
                  'AND first_gone IS NULL', [cutoff])
        alist = c.fetchall()

    # how this next line works is beyond me
    from_database = [item for sublist in alist for item in sublist]
    from_jlist = []
    clist = jlist

    for i in clist:
        if i[0] == 'a':
            from_jlist.append(i[1])

    to_add = list(set(from_database) - set(from_jlist))

    for i in to_add:
        add_jlist(['a', i])

    return len(to_add)

def do_b(job):
    count = 0
    page = requests.get(job[1])

    if page.status_code == 204:  # Network stuff
        sleep(120)
        return 1

    t = html.fromstring(page.content)
    v = t.xpath('//div[@class="search-result__r1"]/div/a/@href')

    for a in v:
        ID = re.search('(?<=classified/advert/)[0-9]+', a)
        if ID:
            ID = int(ID.group(0))
            add_jlist(("a", ID))  # b jobs produce a jobs
            add_rlist([ID] + [None] * 17 + [int(time.time())] + [None])
            count += 1

    print("B " + str(count))
    return 0

def do_a(job):
    result = ff1.scraper(job[1])

    if result[1] == 'AUTOTRADER_FAILED':
        print("Server failed")
        sleep(60)
        return 1

    add_rlist(result)
    print("A")
    return 0

def sql_writer():
    global STAHP
    global rlist
    global rlock
    with sqlite3.connect('test.db') as conn:
        c = conn.cursor()
        c.execute('CREATE TABLE IF NOT EXISTS cars ('
                  'url INTEGER PRIMARY KEY,'
                  ' make TEXT,'
                  ' model TEXT,'
                  ' engine_size NUM,'
                  ' year INT,'
                  ' body_type TEXT,'
                  ' mileage NUM,'
                  ' transmission TEXT,'
                  ' fuel TEXT,'
                  ' co2_emissions NUM,'
                  ' doors INT,'
                  ' seats INT,'
                  ' category TEXT,'
                  ' engine_power NUM,'
                  ' drive_train TEXT,'
                  ' price NUM,'
                  ' seller TEXT,'
                  ' full_desc TEXT,'
                  ' last_seen NUM,'
                  ' first_gone NUM);')

        while not STAHP:
            while(rlist):
                job = rlist[0]
                c.execute("SELECT * FROM cars WHERE url=?", [job[0]])
                match = list(c.fetchall())
                if(match):
                    final = combine_lines(match, job)
                else:
                    final = job
                c.execute("DELETE FROM cars WHERE url=?", [job[0]])
                c.execute(
                    'INSERT INTO cars VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?'
                    ',?,?,?,?,?,?)', final)
                while rlock:
                    sleep(random.uniform(0.01, 0.1))
                rlock = True
                rlist.remove(job)
                rlock = False
            conn.commit()
            sleep(5)

# This combines lines (one line in newline, multiple in olds)
def combine_lines(olds, newline):
    final = newline
    for old in olds:
        final = combine_two(final, old)
    return final

# The bigger entries are kept for each field
# except for the last field, in which the smallest non-None entry is kept
def combine_two(newline, oldline):
    final = newline
    for i in range(len(final) - 1):
        final[i] = my_max(final[i], oldline[i])
    if not final[-1]:
        final[-1] = oldline[-1]
    elif oldline[-1]:
        final[-1] = min(final[-1], oldline[-1])
    return final

def my_max(one, two):
    if not one:
        return two
    if not two:
        return one
    if type(one) is str or type(two) is str:
        return(max(str(one), str(two)))
    else:
        return(max(one, two))

def jparse(data):
    if(data[0] == "a"):
        ret = [None] * 2
        ret[0] = "a"
        try:
            ret[1] = int(data[1:])
        # FIXME: Undefined exception
        except Exception:
            return 0
        return ret
    elif(data[0] == "b"):
        ret = [None] * 2
        ret[0] = "b"
        ret[1] = data[1:]
        return ret
    elif(data[0] == "c"):
        ret = [None] * 2
        ret[0] = "c"
        ret[1] = data[1:]
        return ret
    else:
        return 0

class ThreadedTCPRequestHandler(socketserver.BaseRequestHandler):

    def handle(self):
        global STAHP
        data = self.request.recv(1024).decode('UTF-8')
        if(data == 'shutdown'):
            STAHP = True
            self.request.sendall(b'shutting down')
            while(self.request.recv(1024)):
                pass
            return 0
        if data == 'updater':
            print('U ' + str(updater()))
            self.request.sendall(b'updatr')
            while(self.request.recv(1024)):
                pass
            return 0
        job = jparse(data)
        if job:
            status = add_jlist(job)
            if(status == 0):
                self.request.sendall(b'added')
            else:
                self.request.sendall(bytes('failed to add (%s)' % (status),
                                           'UTF-8'))
        else:
            self.request.sendall(b'not a correct command')
        while(self.request.recv(1024)):
            pass
        print("*")

class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
    pass

jlock = False   # Lock to jlist
jlist = []      # Job list
rlock = False   # Lock to rlist
rlist = []      # Result list
HOST, PORT = "localhost", 9500
STAHP = False   # How the main loop knows when to STAHP

server = ThreadedTCPServer((HOST, PORT), ThreadedTCPRequestHandler)

ip, port = server.server_address

# Start a thread with the server -- that thread will then start one
# more thread for each request
server_thread = threading.Thread(target=server.serve_forever)
# Exit the server thread when the main thread terminates
server_thread.daemon = True
server_thread.start()

# This thread to move things from rlist to a sqlite3 database file
# Always running to avoid starting a new connection for each entry
sql_thread = threading.Thread(target=sql_writer)
sql_thread.daemon = True
sql_thread.start()

# Pay attention, this is the MAIN LOOP:
while not STAHP:
    sleep(random.gammavariate(.1, 1))
    if(jlist):
        job = random.choice(jlist)
        if(job[0] == "a"):
            status = do_a(job)
        elif(job[0] == "b"):
            status = do_b(job)
        elif(job[0] == "c"):
            status = do_c(job)

        if(status == 0):
            while jlock:
                sleep(random.uniform(0.01, 0.1))
            jlock = True
            jlist.remove(job)
            jlock = False
        else:
            print("job failed:")
            print(job)
            print("status: " + str(status))

# If we have unfinished jobs in jlist, that's pickled for later
# surely SOMEONE should import them at startup then
if(jlist):
    print('unfinished jobs in jlist', len(jlist))
    with open(str("jlist.txt"), "wb") as fl:
        pickle.dump(jlist, fl)

print(len(rlist))

server.shutdown()
server.server_close()
<commit_msg>server works without locks<commit_after>#!/usr/bin/python3

import socket
import threading
import socketserver
import time
import random
import pickle
import requests
import re
from lxml import html
import ff1
import sqlite3

sleep = time.sleep

def sql_writer():
    global STAHP
    global rlist
    with sqlite3.connect('test.db') as conn:
        c = conn.cursor()
        c.execute('CREATE TABLE IF NOT EXISTS cars ('
                  'url INTEGER PRIMARY KEY,'
                  ' make TEXT,'
                  ' model TEXT,'
                  ' engine_size NUM,'
                  ' year INT,'
                  ' body_type TEXT,'
                  ' mileage NUM,'
                  ' transmission TEXT,'
                  ' fuel TEXT,'
                  ' co2_emissions NUM,'
                  ' doors INT,'
                  ' seats INT,'
                  ' category TEXT,'
                  ' engine_power NUM,'
                  ' drive_train TEXT,'
                  ' price NUM,'
                  ' seller TEXT,'
                  ' full_desc TEXT,'
                  ' last_seen NUM,'
                  ' first_gone NUM);')

        while not STAHP:
            while(rlist):
                job = rlist[0]
                c.execute("SELECT * FROM cars WHERE url=?", [job[0]])
                match = list(c.fetchall())
                if(match):
                    final = combine_lines(match, job)
                else:
                    final = job
                c.execute("DELETE FROM cars WHERE url=?", [job[0]])
                c.execute(
                    'INSERT INTO cars VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?'
                    ',?,?,?,?,?,?)', final)
                rlist.remove(job)
            conn.commit()
            sleep(5)

def do_c(job):
    page = requests.get(job[1] + "1")
    if page.status_code == 204:  # Network stuff
        sleep(120)
        return 1

    t = html.fromstring(page.content)
    t.xpath('//div[@class="search-result__r1"]/div/a/@href')
    t_cars = t.xpath(
        '//h1[@class="search-form__count js-results-count"]/text()'
    )[0]
    t_cars = re.search('[0-9,]+', t_cars).group(0)
    t_cars = int(t_cars.replace(",", ""))
    pageno = t_cars // 10 + 1

    for i in range(pageno):
        jlist.append(("b", job[1] + str(i + 1)))  # c jobs produce b jobs

    print("C " + str(pageno))
    return 0

def updater():
    global jlist
    with sqlite3.connect('test.db') as conn:
        c = conn.cursor()
        now = int(time.time())
        cutoff = now - 60 * 60 * 24
        c.execute('SELECT url FROM cars WHERE last_seen < ?'
                  'AND first_gone IS NULL', [cutoff])
        alist = c.fetchall()

    # how this next line works is beyond me
    from_database = [item for sublist in alist for item in sublist]
    from_jlist = []
    clist = jlist

    for i in clist:
        if i[0] == 'a':
            from_jlist.append(i[1])

    to_add = list(set(from_database) - set(from_jlist))

    for i in to_add:
        jlist.append(['a', i])

    return len(to_add)

def do_b(job):
    count = 0
    page = requests.get(job[1])

    if page.status_code == 204:  # Network stuff
        sleep(120)
        return 1

    t = html.fromstring(page.content)
    v = t.xpath('//div[@class="search-result__r1"]/div/a/@href')

    for a in v:
        ID = re.search('(?<=classified/advert/)[0-9]+', a)
        if ID:
            ID = int(ID.group(0))
            jlist.append(("a", ID))  # b jobs produce a jobs
            rlist.append([ID] + [None] * 17 + [int(time.time())] + [None])
            count += 1

    print("B " + str(count))
    return 0

def do_a(job):
    result = ff1.scraper(job[1])

    if result[1] == 'AUTOTRADER_FAILED':
        print("Server failed")
        sleep(60)
        return 1

    rlist.append(result)
    print("A")
    return 0

def combine_lines(olds, newline):
    """This combines lines (one line in newline, multiple in olds)"""
    final = newline
    for old in olds:
        final = combine_two(final, old)
    return final

def combine_two(newline, oldline):
    """ The bigger entries are kept for each field, except """
    """ for the last field, in which the smallest non-None entry is kept"""
    final = newline
    for i in range(len(final) - 1):
        final[i] = my_max(final[i], oldline[i])
    if not final[-1]:
        final[-1] = oldline[-1]
    elif oldline[-1]:
        final[-1] = min(final[-1], oldline[-1])
    return final

def my_max(one, two):
    if not one:
        return two
    if not two:
        return one
    if type(one) is str or type(two) is str:
        return(max(str(one), str(two)))
    else:
        return(max(one, two))

def jparse(data):
    if(data[0] == "a"):
        ret = [None] * 2
        ret[0] = "a"
        try:
            ret[1] = int(data[1:])
        # FIXME: Undefined exception
        except Exception:
            return 0
        return ret
    elif(data[0] == "b"):
        ret = [None] * 2
        ret[0] = "b"
        ret[1] = data[1:]
        return ret
    elif(data[0] == "c"):
        ret = [None] * 2
        ret[0] = "c"
        ret[1] = data[1:]
        return ret
    else:
        return 0

class ThreadedTCPRequestHandler(socketserver.BaseRequestHandler):

    def handle(self):
        global STAHP
        data = self.request.recv(1024).decode('UTF-8')
        if(data == 'shutdown'):
            STAHP = True
            self.request.sendall(b'shutting down')
            while(self.request.recv(1024)):
                pass
            return 0
        if data == 'updater':
            print('U ' + str(updater()))
            self.request.sendall(b'updatr')
            while(self.request.recv(1024)):
                pass
            return 0
        job = jparse(data)
        if job:
            jlist.append(job)
            self.request.sendall(b'added')
        else:
            self.request.sendall(b'not a correct command')
        while(self.request.recv(1024)):
            pass
        print("*")

class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
    pass

jlist = []      # Job list
rlist = []      # Result list
HOST, PORT = "localhost", 9500
STAHP = False   # How the main loop knows when to STAHP

server = ThreadedTCPServer((HOST, PORT), ThreadedTCPRequestHandler)

ip, port = server.server_address

# Start a thread with the server -- that thread will then start one
# more thread for each request
server_thread = threading.Thread(target=server.serve_forever)

# Exit the server thread when the main thread terminates
server_thread.daemon = True
server_thread.start()

# This thread to move things from rlist to a sqlite3 database file
# Always running to avoid starting a new connection for each entry
sql_thread = threading.Thread(target=sql_writer)
sql_thread.daemon = True
sql_thread.start()

# Pay attention, this is the MAIN LOOP:
while not STAHP:
    sleep(random.gammavariate(.1, 1))
    if(jlist):
        job = random.choice(jlist)
        if(job[0] == "a"):
            status = do_a(job)
        elif(job[0] == "b"):
            status = do_b(job)
        elif(job[0] == "c"):
            status = do_c(job)

        if(status == 0):
            jlist.remove(job)
        else:
            print("job failed:")
            print(job)
            print("status: " + str(status))

# If we have unfinished jobs in jlist, that's pickled for later
# surely SOMEONE should import them at startup then
if(jlist):
    print('unfinished jobs in jlist', len(jlist))
    with open(str("jlist.txt"), "wb") as fl:
        pickle.dump(jlist, fl)

print(len(rlist))

server.shutdown()
server.server_close()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Program: SSH from Python
Programmer: Michael Fryar, Research Fellow, EPoD
Date created: August 28, 2017

Purpose: Establish SSH tunnel using python rather than bash
"""

import os
import subprocess

from write_to_gsheets_AGG import write_to_sheet

# Change to directory containing configuration files
os.chdir("C:/Users/mifryar/Documents/epodx")

# Establish ssh tunnel in background that auto-closes
# -f "fork into background", -F "use configuration file"
# -o ExistOnForwardFailure=yes "wait until connection and port
#   forwardings are set up before placing in background"
# sleep 10 "give python script 60 seconds to start using tunnel and
#   close tunnel after python script stops using it"
# Ref 1: https://www.g-loaded.eu/2006/11/24/auto-closing-ssh-tunnels/
# Ref 2: https://gist.github.com/scy/6781836

config = "-F ./ssh-config epodx-analytics-api"
option = "-o ExitOnForwardFailure=yes"

ssh = subprocess.run("ssh -f config option sleep 60")

# Change to directory containing script to write to Google Sheets
os.chdir("C:/Users/mifryar/Documents/epodx-dashboards")
write_to_sheet()
<commit_msg>Manipulate pathnames in way that allows compatibility with Linux and Windows<commit_after>#!/usr/bin/env python

"""Program: SSH from Python
Programmer: Michael Fryar, Research Fellow, EPoD
Date created: August 28, 2017

Purpose: Establish SSH tunnel using python rather than bash
"""

import os
import subprocess

from write_to_gsheets_AGG import write_to_sheet

# Change to directory containing configuration files
home_dir = os.path.expanduser('~')
epodx_dir = os.path.join(home_dir, 'Documents/epodx')
dashboards_dir = os.path.join(home_dir, 'Documents/epodx-dashboards')
os.chdir(epodx_dir)

# Establish ssh tunnel in background that auto-closes
# -f "fork into background", -F "use configuration file"
# -o ExistOnForwardFailure=yes "wait until connection and port
#   forwardings are set up before placing in background"
# sleep 10 "give python script 60 seconds to start using tunnel and
#   close tunnel after python script stops using it"
# Ref 1: https://www.g-loaded.eu/2006/11/24/auto-closing-ssh-tunnels/
# Ref 2: https://gist.github.com/scy/6781836

config = "-F ./ssh-config epodx-analytics-api"
option = "-o ExitOnForwardFailure=yes"

ssh = subprocess.run("ssh -f config option sleep 60")

# Change to directory containing script to write to Google Sheets
os.chdir("C:/Users/mifryar/Documents/epodx-dashboards")
write_to_sheet()
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-

from flask_script import Command, Option, Server

import flask

import rdflib
from nanopub import Nanopublication
import tempfile

from whyis.namespace import np

class LoadNanopub(Command):
    '''Add a nanopublication to the knowledge graph.'''

    def get_options(self):
        return [
            Option('--input', '-i', dest='input_file',
                   type=str),
            Option('--format', '-f', dest='file_format',
                   type=str),
            Option('--revises', '-r', dest='was_revision_of',
                   type=str),
        ]

    def run(self, input_file, file_format="trig", was_revision_of=None):
        flask.current_app.managed = True
        if was_revision_of is not None:
            wasRevisionOf = set(flask.current_app.db.objects(predicate=np.hasAssertion,
                                                             subject=rdflib.URIRef(was_revision_of)))
            if len(wasRevisionOf) == 0:
                print("Could not find active nanopublication to revise:", was_revision_of)
                return
            was_revision_of = wasRevisionOf
        g = rdflib.ConjunctiveGraph(identifier=rdflib.BNode().skolemize(), store="Sleepycat")
        graph_tempdir = tempfile.mkdtemp()
        g.store.open(graph_tempdir, True)
        # g = rdflib.ConjunctiveGraph(identifier=rdflib.BNode().skolemize())

        g1 = g.parse(location=input_file, format=file_format, publicID=flask.current_app.NS.local)
        if len(list(g.subjects(rdflib.RDF.type, np.Nanopublication))) == 0:
            print("Could not find existing nanopublications.", len(g1), len(g))
            new_np = Nanopublication(store=g1.store)
            new_np.add((new_np.identifier, rdflib.RDF.type, np.Nanopublication))
            new_np.add((new_np.identifier, np.hasAssertion, g1.identifier))
            new_np.add((g1.identifier, rdflib.RDF.type, np.Assertion))

        nanopub_prepare_graph = rdflib.ConjunctiveGraph(store="Sleepycat")
        nanopub_prepare_graph_tempdir = tempfile.mkdtemp()
        nanopub_prepare_graph.store.open(nanopub_prepare_graph_tempdir, True)
        nanopubs = []
        for npub in flask.current_app.nanopub_manager.prepare(g, store=nanopub_prepare_graph.store):
            if was_revision_of is not None:
                for r in was_revision_of:
                    print("Marking as revision of", r)
                    npub.pubinfo.add((npub.assertion.identifier, flask.current_app.NS.prov.wasRevisionOf, r))
            print('Prepared', npub.identifier)
            nanopubs.append(npub)
        flask.current_app.nanopub_manager.publish(*nanopubs)
        print("Published", npub.identifier)
<commit_msg>load_nanopub: --temp-store, delete temp dirs when using Sleepycat<commit_after># -*- coding:utf-8 -*-
import shutil

from flask_script import Command, Option, Server

import flask

import rdflib
from nanopub import Nanopublication
import tempfile

from whyis.namespace import np

class LoadNanopub(Command):
    '''Add a nanopublication to the knowledge graph.'''

    _TEMP_STORE_DEFAULT = "Sleepycat"

    def get_options(self):
        return [
            Option('--input', '-i', dest='input_file',
                   type=str),
            Option('--format', '-f', dest='file_format',
                   type=str),
            Option('--revises', '-r', dest='was_revision_of',
                   type=str),
            Option("--temp-store", dest="temp_store", type=str, default=self._TEMP_STORE_DEFAULT, help="backing store type to use for temporary graphs")
        ]

    def run(self, input_file, file_format="trig", temp_store=_TEMP_STORE_DEFAULT, was_revision_of=None):
        flask.current_app.managed = True
        if was_revision_of is not None:
            wasRevisionOf = set(flask.current_app.db.objects(predicate=np.hasAssertion,
                                                             subject=rdflib.URIRef(was_revision_of)))
            if len(wasRevisionOf) == 0:
                print("Could not find active nanopublication to revise:", was_revision_of)
                return
            was_revision_of = wasRevisionOf
        g = rdflib.ConjunctiveGraph(identifier=rdflib.BNode().skolemize(), store=temp_store)
        if temp_store == "Sleepycat":
            g_store_tempdir = tempfile.mkdtemp()
            g.store.open(g_store_tempdir, True)
        else:
            g_store_tempdir = None
        # g = rdflib.ConjunctiveGraph(identifier=rdflib.BNode().skolemize())

        try:
            g1 = g.parse(location=input_file, format=file_format, publicID=flask.current_app.NS.local)
            if len(list(g.subjects(rdflib.RDF.type, np.Nanopublication))) == 0:
                print("Could not find existing nanopublications.", len(g1), len(g))
                new_np = Nanopublication(store=g1.store)
                new_np.add((new_np.identifier, rdflib.RDF.type, np.Nanopublication))
                new_np.add((new_np.identifier, np.hasAssertion, g1.identifier))
                new_np.add((g1.identifier, rdflib.RDF.type, np.Assertion))

            nanopub_prepare_graph = rdflib.ConjunctiveGraph(store=temp_store)
            if temp_store == "Sleepycat":
                nanopub_prepare_graph_store_tempdir = tempfile.mkdtemp()
                nanopub_prepare_graph.store.open(nanopub_prepare_graph_store_tempdir, True)
            else:
                nanopub_prepare_graph_store_tempdir = None

            try:
                nanopubs = []
                for npub in flask.current_app.nanopub_manager.prepare(g, store=nanopub_prepare_graph.store):
                    if was_revision_of is not None:
                        for r in was_revision_of:
                            print("Marking as revision of", r)
                            npub.pubinfo.add((npub.assertion.identifier, flask.current_app.NS.prov.wasRevisionOf, r))
                    print('Prepared', npub.identifier)
                    nanopubs.append(npub)
                flask.current_app.nanopub_manager.publish(*nanopubs)
                print("Published", npub.identifier)
            finally:
                if nanopub_prepare_graph_store_tempdir is not None:
                    shutil.rmtree(nanopub_prepare_graph_store_tempdir)
        finally:
            if g_store_tempdir is not None:
                shutil.rmtree(g_store_tempdir)
<|endoftext|>


<commit_before>import itertools

from whylog.config import Clue
from whylog.constraints import IdenticalConstraint, TimeConstraint
from whylog.constraints.exceptions import UnsupportedConstraintTypeError
from whylog.front import FrontInput

class ConstraintRegistry(object):
    CONSTRAINTS = {
        'identical': IdenticalConstraint([], {}),
        'time': TimeConstraint(
            [], {}
        )
        # register your constraint here
    }

    @classmethod
    def get_constraint(cls, constraint_data):
        if constraint_data['name'] in cls.CONSTRAINTS:
            return cls.CONSTRAINTS[constraint_data['name']]
        raise UnsupportedConstraintTypeError(constraint_data)

class ConstraintManager(object):
    """
    there should be one such object per rule being verified
    """

    def __init__(self):
        self._actual_constraints = {}

    def __getitem__(self, constraint_data):
        constraint = self._actual_constraints.get(constraint_data['name'])
        if constraint is None:
            constraint_verifier = ConstraintRegistry.get_constraint(constraint_data)
            self._actual_constraints[constraint_data['name']] = constraint_verifier
        return self._actual_constraints[constraint_data['name']]

class Verifier(object):
    UNMATCHED = Clue(None, None, None, None)

    @classmethod
    def _create_investigation_result(cls, clues_combination, constraints):
        """
        basing on clues combination and constraints,
        returns appropriate InvestigationResult object
        which collects information about lines
        (FrontInput objects) instead of Clues
        """
        return InvestigationResult(
            [FrontInput.from_clue(clue) for clue in clues_combination], constraints
        )

    @classmethod
    def _verify_constraint(cls, combination, effect, constraint, constraint_manager):
        """
        checks if specified clues (which represents parsers: 1,2,.. for some rule) and
        effect (which represents parser 0 from this rule) satisfy one given constraint.
        returns True if so, or False otherwise
        """
        constraint_verifier = constraint_manager[constraint]
        groups = []
        for group_info in constraint['clues_groups']:
            parser_num, group_num = group_info
            if parser_num == 0:
                groups.append(effect.regex_parameters[group_num - 1])
            else:
                if combination[parser_num - 1] == Verifier.UNMATCHED:
                    return False
                groups.append(combination[parser_num - 1].regex_parameters[group_num - 1])
        return constraint_verifier.verify(groups, constraint['params'])

    @classmethod
    def _clues_combinations(cls, clues_tuples, collected_subset=[]):
        """
        recursive generator that returns all permutations according to schema:
        from first pair (list, number) of clues_tuples,
        produces permutations with size 'number' from 'list's elements
        and concatenates it with _clues_combinations invoked on the rest of clues_tuples.
        example:
        >>> xs = [([1, 2, 3], 2), ([4, 5], 1)]
        >>> for l in Verifier._clues_combinations(xs):
        >>>     print l
        [1, 2, 4]
        [1, 2, 5]
        [1, 3, 4]
        [1, 3, 5]
        [2, 1, 4]
        [2, 1, 5]
        [2, 3, 4]
        [2, 3, 5]
        [3, 1, 4]
        [3, 1, 5]
        [3, 2, 4]
        [3, 2, 5]
        it always should be called with empty accumulator,
        that is collected_subset=[]
        """
        if len(clues_tuples) != 0:
            first_list, repetitions_number = clues_tuples[0]
            for clues in itertools.permutations(first_list, repetitions_number):
                for subset in cls._clues_combinations(
                    clues_tuples[1:], collected_subset + list(clues)
                ):
                    yield subset
        else:
            yield collected_subset

    @classmethod
    def _construct_proper_clues_lists(cls, original_clues_lists):
        clues_lists = []
        for clues, occurrences in original_clues_lists:
            if clues:
                clues_lists.append((clues, occurrences))
            else:
                clues_lists.append(([Verifier.UNMATCHED], occurrences))
        return clues_lists

    @classmethod
    def constraints_and(cls, clues_lists, effect, constraints, constraint_manager):
        """
        for each combination of clues (they are generated by _clues_combinations)
        checks if for all given constraints their requirements are satisfied
        and for each such combination produces InvestigationResult object.
        returns list of all produced InvestigationResults
        """
        clues_lists = cls._construct_proper_clues_lists(clues_lists)
        causes = []
        for combination in cls._clues_combinations(clues_lists):
            if all(
                cls._verify_constraint(combination, effect, constraint, constraint_manager)
                for constraint in constraints
            ):
                causes.append(cls._create_investigation_result(combination, constraints))
        return causes

    @classmethod
    def constraints_or(cls, clues_lists, effect, constraints, constraint_manager):
        """
        for each combination of clues (they are generated by _clues_combinations)
        checks if for any of given constraints their requirements are satisfied
        and for each such combination produces InvestigationResult object.
        returns list of all produced InvestigationResults
        """
        clues_lists = cls._construct_proper_clues_lists(clues_lists)
        causes = []
        for combination in cls._clues_combinations(clues_lists):
            verified_constraints = [
                constraint
                for constraint in constraints
                if cls._verify_constraint(combination, effect, constraint, constraint_manager)
            ]
            if len(verified_constraints) > 0:
                causes.append(
                    cls._create_investigation_result(
                        [
                            clue for clue in combination if not clue == Verifier.UNMATCHED
                        ], verified_constraints
                    )
                )
        return causes

class InvestigationResult(object):
    def __init__(self, lines, constraints):
        self.lines = lines
        self.constraints = constraints

    def __repr__(self):
        return "\n(\n    result lines: %s;\n    due to constraints: %s\n)" % (
            self.lines, self.constraints
        )

    def __eq__(self, other):
        return all([
            self.lines == other.lines,
            self.constraints == other.constraints
        ])  # yapf: disable
<commit_msg>Quick import fix<commit_after>import itertools

from whylog.config.investigation_plan import Clue
from whylog.constraints import IdenticalConstraint, TimeConstraint
from whylog.constraints.exceptions import UnsupportedConstraintTypeError
from whylog.front import FrontInput

class ConstraintRegistry(object):
    CONSTRAINTS = {
        'identical': IdenticalConstraint([], {}),
        'time': TimeConstraint(
            [], {}
        )
        # register your constraint here
    }

    @classmethod
    def get_constraint(cls, constraint_data):
        if constraint_data['name'] in cls.CONSTRAINTS:
            return cls.CONSTRAINTS[constraint_data['name']]
        raise UnsupportedConstraintTypeError(constraint_data)

class ConstraintManager(object):
    """
    there should be one such object per rule being verified
    """

    def __init__(self):
        self._actual_constraints = {}

    def __getitem__(self, constraint_data):
        constraint = self._actual_constraints.get(constraint_data['name'])
        if constraint is None:
            constraint_verifier = ConstraintRegistry.get_constraint(constraint_data)
            self._actual_constraints[constraint_data['name']] = constraint_verifier
        return self._actual_constraints[constraint_data['name']]

class Verifier(object):
    UNMATCHED = Clue(None, None, None, None)

    @classmethod
    def _create_investigation_result(cls, clues_combination, constraints):
        """
        basing on clues combination and constraints,
        returns appropriate InvestigationResult object
        which collects information about lines
        (FrontInput objects) instead of Clues
        """
        return InvestigationResult(
            [FrontInput.from_clue(clue) for clue in clues_combination], constraints
        )

    @classmethod
    def _verify_constraint(cls, combination, effect, constraint, constraint_manager):
        """
        checks if specified clues (which represents parsers: 1,2,.. for some rule) and
        effect (which represents parser 0 from this rule) satisfy one given constraint.
        returns True if so, or False otherwise
        """
        constraint_verifier = constraint_manager[constraint]
        groups = []
        for group_info in constraint['clues_groups']:
            parser_num, group_num = group_info
            if parser_num == 0:
                groups.append(effect.regex_parameters[group_num - 1])
            else:
                if combination[parser_num - 1] == Verifier.UNMATCHED:
                    return False
                groups.append(combination[parser_num - 1].regex_parameters[group_num - 1])
        return constraint_verifier.verify(groups, constraint['params'])

    @classmethod
    def _clues_combinations(cls, clues_tuples, collected_subset=[]):
        """
        recursive generator that returns all permutations according to schema:
        from first pair (list, number) of clues_tuples,
        produces permutations with size 'number' from 'list's elements
        and concatenates it with _clues_combinations invoked on the rest of clues_tuples.
        example:
        >>> xs = [([1, 2, 3], 2), ([4, 5], 1)]
        >>> for l in Verifier._clues_combinations(xs):
        >>>     print l
        [1, 2, 4]
        [1, 2, 5]
        [1, 3, 4]
        [1, 3, 5]
        [2, 1, 4]
        [2, 1, 5]
        [2, 3, 4]
        [2, 3, 5]
        [3, 1, 4]
        [3, 1, 5]
        [3, 2, 4]
        [3, 2, 5]
        it always should be called with empty accumulator,
        that is collected_subset=[]
        """
        if len(clues_tuples) != 0:
            first_list, repetitions_number = clues_tuples[0]
            for clues in itertools.permutations(first_list, repetitions_number):
                for subset in cls._clues_combinations(
                    clues_tuples[1:], collected_subset + list(clues)
                ):
                    yield subset
        else:
            yield collected_subset

    @classmethod
    def _construct_proper_clues_lists(cls, original_clues_lists):
        clues_lists = []
        for clues, occurrences in original_clues_lists:
            if clues:
                clues_lists.append((clues, occurrences))
            else:
                clues_lists.append(([Verifier.UNMATCHED], occurrences))
        return clues_lists

    @classmethod
    def constraints_and(cls, clues_lists, effect, constraints, constraint_manager):
        """
        for each combination of clues (they are generated by _clues_combinations)
        checks if for all given constraints their requirements are satisfied
        and for each such combination produces InvestigationResult object.
        returns list of all produced InvestigationResults
        """
        clues_lists = cls._construct_proper_clues_lists(clues_lists)
        causes = []
        for combination in cls._clues_combinations(clues_lists):
            if all(
                cls._verify_constraint(combination, effect, constraint, constraint_manager)
                for constraint in constraints
            ):
                causes.append(cls._create_investigation_result(combination, constraints))
        return causes

    @classmethod
    def constraints_or(cls, clues_lists, effect, constraints, constraint_manager):
        """
        for each combination of clues (they are generated by _clues_combinations)
        checks if for any of given constraints their requirements are satisfied
        and for each such combination produces InvestigationResult object.
        returns list of all produced InvestigationResults
        """
        clues_lists = cls._construct_proper_clues_lists(clues_lists)
        causes = []
        for combination in cls._clues_combinations(clues_lists):
            verified_constraints = [
                constraint
                for constraint in constraints
                if cls._verify_constraint(combination, effect, constraint, constraint_manager)
            ]
            if len(verified_constraints) > 0:
                causes.append(
                    cls._create_investigation_result(
                        [
                            clue for clue in combination if not clue == Verifier.UNMATCHED
                        ], verified_constraints
                    )
                )
        return causes

class InvestigationResult(object):
    def __init__(self, lines, constraints):
        self.lines = lines
        self.constraints = constraints

    def __repr__(self):
        return "\n(\n    result lines: %s;\n    due to constraints: %s\n)" % (
            self.lines, self.constraints
        )

    def __eq__(self, other):
        return all([
            self.lines == other.lines,
            self.constraints == other.constraints
        ])  # yapf: disable
<|endoftext|>


<commit_before>#!/usr/bin/env python2
#
# svnshell.py : a Python-based shell interface for cruising 'round in
#               the filesystem.
#
######################################################################
#
# Copyright (c) 2000-2001 CollabNet.  All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.  The terms
# are also available at http://subversion.tigris.org/license-1.html.
# If newer versions of this license are posted there, you may use a
# newer version instead, at your option.
#
######################################################################
#

import sys
import string
import time
import re
from random import randint
from svn import fs, util, repos

class SVNShell:
  def __init__(self, pool, path):
    """initialize an SVNShell object"""
    self.pool = pool
    self.taskpool = util.svn_pool_create(pool)
    self.fs_ptr = repos.svn_repos_fs(repos.svn_repos_open(path, pool))
    self.is_rev = 1
    self.rev = fs.youngest_rev(self.fs_ptr, pool)
    self.txn = None
    self.root = fs.revision_root(self.fs_ptr, self.rev, pool)
    self.path = "/"
    self._do_prompt()

  def cmd_help(self, *args):
    """print shell help"""
    print "Available commands:"
    print "  cd DIR       : change the current working directory to DIR"
    print "  exit         : exit the shell"
    print "  ls           : list the contents of the current directory"
    print "  lstxns       : list the transactions available for browsing"
    print "  setrev REV   : set the current revision to browse"
    print "  settxn TXN   : set the current transaction to browse"
    print "  youngest     : list the youngest browsable revision number"

  def cmd_cd(self, *args):
    """change directory"""
    args = args[0]
    if len(args) < 1:
      return
    newpath = string.join(filter(None, string.split(args[0], '/')), '/')
    if args[0][0] == '/' or self.path == '/':
      newpath = '/' + newpath
    else:
      newpath = self.path + '/' + newpath

    # cleanup '.' and '..'
    parts = filter(None, string.split(newpath, '/'))
    finalparts = []
    for part in parts:
      if part == '.':
        pass
      elif part == '..':
        if len(finalparts) == 0:
          return
        else:
          finalparts.pop(-1)
      else:
        finalparts.append(part)
    newpath = '/' + string.join(finalparts, '/')

    # make sure that path actually exists in the filesystem as a directory
    kind = fs.check_path(self.root, newpath, self.taskpool)
    if kind != util.svn_node_dir:
      print "Path '" + newpath + "' is not a valid filesystem directory."
      return
    self.path = newpath
    util.svn_pool_clear(self.taskpool)

  def cmd_ls(self, *args):
    """list the contents of the current directory"""
    entries = fs.dir_entries(self.root, self.path, self.taskpool)
    keys = entries.keys()
    keys.sort()
    print "   REV   AUTHOR  NODE-REV-ID     SIZE         DATE NAME"
    print "----------------------------------------------------------------------------"
    for entry in keys:
      fullpath = self.path + '/' + entry
      size = ''
      is_dir = fs.is_dir(self.root, fullpath, self.taskpool)
      if is_dir:
        name = entry + '/'
      else:
        size = str(fs.file_length(self.root, fullpath, self.taskpool))
        name = entry
      node_id = fs.unparse_id(fs.dirent_t_id_get(entries[entry]),
                              self.taskpool)
      created_rev = fs.node_created_rev(self.root, fullpath, self.taskpool)
      author = fs.revision_prop(self.fs_ptr, created_rev,
                                util.SVN_PROP_REVISION_AUTHOR, self.taskpool)
      date = fs.revision_prop(self.fs_ptr, created_rev,
                              util.SVN_PROP_REVISION_DATE, self.taskpool)
      date = self._format_date(date, self.taskpool)

      print "%6s %8s <%10s> %8s %12s %s" % (created_rev, author,
                                            node_id, size, date, name)
    util.svn_pool_clear(self.taskpool)

  def cmd_lstxns(self, *args):
    """list the transactions available for browsing"""
    txns = fs.list_transactions(self.fs_ptr, self.taskpool)
    txns.sort()
    counter = 0
    for txn in txns:
      counter = counter + 1
      print "%8s  " % txn,
      if counter == 6:
        print ""
        counter = 0
    print ""
    util.svn_pool_clear(self.taskpool)

  def cmd_setrev(self, *args):
    """set the current revision to view"""
    args = args[0]
    try:
      rev = int(args[0])
      newroot = fs.revision_root(self.fs_ptr, rev, self.pool)
    except:
      print "Error setting the revision to '" + str(rev) + "'"
      return
    fs.close_root(self.root)
    self.root = newroot
    self.rev = rev
    self.is_rev = 1
    self._do_path_landing()

  def cmd_settxn(self, *args):
    """set the current transaction to view"""
    args = args[0]
    txn = args[0]
    try:
      txnobj = fs.open_txn(self.fs_ptr, txn, self.pool)
      newroot = fs.txn_root(txnobj, self.pool)
    except:
      print "Error setting the transaction to '" + txn + "'"
      return
    fs.close_root(self.root)
    self.root = newroot
    self.txn = txn
    self.is_rev = 0
    self._do_path_landing()

  def cmd_youngest(self, *args):
    """list the youngest revision available for browsing"""
    rev = fs.youngest_rev(self.fs_ptr, self.taskpool)
    print rev
    util.svn_pool_clear(self.taskpool)

  def _format_date(self, date, pool):
    date = util.svn_time_from_nts(date, pool)
    date = time.asctime(time.localtime(date / 1000000))
    return date[4:-8]

  def _do_path_landing(self):
    """try to land on self.path as a directory in root, failing up to '/'"""
    not_found = 1
    newpath = self.path
    while not_found:
      kind = fs.check_path(self.root, newpath, self.taskpool)
      if kind == util.svn_node_dir:
        not_found = 0
      else:
        parts = filter(None,string.split(newpath, '/'))
        parts.pop(-1)
        newpath = '/' + string.join(parts, '/')
    self.path = newpath
    util.svn_pool_clear(self.taskpool)

  _errors = ["Huh?",
             "Whatchoo talkin' 'bout, Willis?",
             "Say what?",
             "Nope.  Not gonna do it.",
             "Ehh...I don't think so, chief."]
  def _do_prompt(self):
    """present the prompt and handle the user's input"""
    if self.is_rev:
      prompt = "<rev: " + str(self.rev)
    else:
      prompt = "<txn: " + self.txn
    prompt += " " + self.path + ">$ "
    try:
      input = raw_input(prompt)
    except EOFError:
      return

    ### This will currently screw up when the arguments to the
    ### commands have spaces in them, like 'cd "My Dir"'
    args = filter(None, string.split(input, ' '))
    if len(args) == 0:
      pass
    elif args[0] == 'exit':
      return
    elif not hasattr(self, 'cmd_' + args[0]):
      msg = self._errors[randint(0, len(self._errors) - 1)]
      print msg
    else:
      getattr(self, 'cmd_' + args[0])(args[1:])
    self._do_prompt()

def _basename(path):
  "Return the basename for a '/'-separated path."
  idx = string.rfind(path, '/')
  if idx == -1:
    return path
  return path[idx+1:]

def usage(exit):
  if exit:
    output = sys.stderr
  else:
    output = sys.stdout
  output.write(
    "usage: %s REPOS_PATH\n"
    "\n"
    "Once the program has started, type 'help' at the prompt for hints on\n"
    "using the shell.\n" % sys.argv[0])
  sys.exit(exit)

def main():
  if len(sys.argv) < 2:
    usage(1)
  util.run_app(SVNShell, sys.argv[1])

if __name__ == '__main__':
  main()
<commit_msg>* subversion/tools/examples/svnshell.py   (main): Use the termios module, if available, to setup terminal     attributes (this makes the 'backspace' key work on my machine, hope     it helps others).<commit_after>#!/usr/bin/env python2
#
# svnshell.py : a Python-based shell interface for cruising 'round in
#               the filesystem.
#
######################################################################
#
# Copyright (c) 2000-2001 CollabNet.  All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.  The terms
# are also available at http://subversion.tigris.org/license-1.html.
# If newer versions of this license are posted there, you may use a
# newer version instead, at your option.
#
######################################################################
#

import sys
import string
import time
import re
from random import randint
from svn import fs, util, repos

class SVNShell:
  def __init__(self, pool, path):
    """initialize an SVNShell object"""
    self.pool = pool
    self.taskpool = util.svn_pool_create(pool)
    self.fs_ptr = repos.svn_repos_fs(repos.svn_repos_open(path, pool))
    self.is_rev = 1
    self.rev = fs.youngest_rev(self.fs_ptr, pool)
    self.txn = None
    self.root = fs.revision_root(self.fs_ptr, self.rev, pool)
    self.path = "/"
    self._do_prompt()

  def cmd_help(self, *args):
    """print shell help"""
    print "Available commands:"
    print "  cd DIR       : change the current working directory to DIR"
    print "  exit         : exit the shell"
    print "  ls           : list the contents of the current directory"
    print "  lstxns       : list the transactions available for browsing"
    print "  setrev REV   : set the current revision to browse"
    print "  settxn TXN   : set the current transaction to browse"
    print "  youngest     : list the youngest browsable revision number"

  def cmd_cd(self, *args):
    """change directory"""
    args = args[0]
    if len(args) < 1:
      return
    newpath = string.join(filter(None, string.split(args[0], '/')), '/')
    if args[0][0] == '/' or self.path == '/':
      newpath = '/' + newpath
    else:
      newpath = self.path + '/' + newpath

    # cleanup '.' and '..'
    parts = filter(None, string.split(newpath, '/'))
    finalparts = []
    for part in parts:
      if part == '.':
        pass
      elif part == '..':
        if len(finalparts) == 0:
          return
        else:
          finalparts.pop(-1)
      else:
        finalparts.append(part)
    newpath = '/' + string.join(finalparts, '/')

    # make sure that path actually exists in the filesystem as a directory
    kind = fs.check_path(self.root, newpath, self.taskpool)
    if kind != util.svn_node_dir:
      print "Path '" + newpath + "' is not a valid filesystem directory."
      return
    self.path = newpath
    util.svn_pool_clear(self.taskpool)

  def cmd_ls(self, *args):
    """list the contents of the current directory"""
    entries = fs.dir_entries(self.root, self.path, self.taskpool)
    keys = entries.keys()
    keys.sort()
    print "   REV   AUTHOR  NODE-REV-ID     SIZE         DATE NAME"
    print "----------------------------------------------------------------------------"
    for entry in keys:
      fullpath = self.path + '/' + entry
      size = ''
      is_dir = fs.is_dir(self.root, fullpath, self.taskpool)
      if is_dir:
        name = entry + '/'
      else:
        size = str(fs.file_length(self.root, fullpath, self.taskpool))
        name = entry
      node_id = fs.unparse_id(fs.dirent_t_id_get(entries[entry]),
                              self.taskpool)
      created_rev = fs.node_created_rev(self.root, fullpath, self.taskpool)
      author = fs.revision_prop(self.fs_ptr, created_rev,
                                util.SVN_PROP_REVISION_AUTHOR, self.taskpool)
      date = fs.revision_prop(self.fs_ptr, created_rev,
                              util.SVN_PROP_REVISION_DATE, self.taskpool)
      date = self._format_date(date, self.taskpool)

      print "%6s %8s <%10s> %8s %12s %s" % (created_rev, author,
                                            node_id, size, date, name)
    util.svn_pool_clear(self.taskpool)

  def cmd_lstxns(self, *args):
    """list the transactions available for browsing"""
    txns = fs.list_transactions(self.fs_ptr, self.taskpool)
    txns.sort()
    counter = 0
    for txn in txns:
      counter = counter + 1
      print "%8s  " % txn,
      if counter == 6:
        print ""
        counter = 0
    print ""
    util.svn_pool_clear(self.taskpool)

  def cmd_setrev(self, *args):
    """set the current revision to view"""
    args = args[0]
    try:
      rev = int(args[0])
      newroot = fs.revision_root(self.fs_ptr, rev, self.pool)
    except:
      print "Error setting the revision to '" + str(rev) + "'"
      return
    fs.close_root(self.root)
    self.root = newroot
    self.rev = rev
    self.is_rev = 1
    self._do_path_landing()

  def cmd_settxn(self, *args):
    """set the current transaction to view"""
    args = args[0]
    txn = args[0]
    try:
      txnobj = fs.open_txn(self.fs_ptr, txn, self.pool)
      newroot = fs.txn_root(txnobj, self.pool)
    except:
      print "Error setting the transaction to '" + txn + "'"
      return
    fs.close_root(self.root)
    self.root = newroot
    self.txn = txn
    self.is_rev = 0
    self._do_path_landing()

  def cmd_youngest(self, *args):
    """list the youngest revision available for browsing"""
    rev = fs.youngest_rev(self.fs_ptr, self.taskpool)
    print rev
    util.svn_pool_clear(self.taskpool)

  def _format_date(self, date, pool):
    date = util.svn_time_from_nts(date, pool)
    date = time.asctime(time.localtime(date / 1000000))
    return date[4:-8]

  def _do_path_landing(self):
    """try to land on self.path as a directory in root, failing up to '/'"""
    not_found = 1
    newpath = self.path
    while not_found:
      kind = fs.check_path(self.root, newpath, self.taskpool)
      if kind == util.svn_node_dir:
        not_found = 0
      else:
        parts = filter(None,string.split(newpath, '/'))
        parts.pop(-1)
        newpath = '/' + string.join(parts, '/')
    self.path = newpath
    util.svn_pool_clear(self.taskpool)

  _errors = ["Huh?",
             "Whatchoo talkin' 'bout, Willis?",
             "Say what?",
             "Nope.  Not gonna do it.",
             "Ehh...I don't think so, chief."]
  def _do_prompt(self):
    """present the prompt and handle the user's input"""
    if self.is_rev:
      prompt = "<rev: " + str(self.rev)
    else:
      prompt = "<txn: " + self.txn
    prompt += " " + self.path + ">$ "
    try:
      input = raw_input(prompt)
    except EOFError:
      return

    ### This will currently screw up when the arguments to the
    ### commands have spaces in them, like 'cd "My Dir"'
    args = filter(None, string.split(input, ' '))
    if len(args) == 0:
      pass
    elif args[0] == 'exit':
      return
    elif not hasattr(self, 'cmd_' + args[0]):
      msg = self._errors[randint(0, len(self._errors) - 1)]
      print msg
    else:
      getattr(self, 'cmd_' + args[0])(args[1:])
    self._do_prompt()

def _basename(path):
  "Return the basename for a '/'-separated path."
  idx = string.rfind(path, '/')
  if idx == -1:
    return path
  return path[idx+1:]

def usage(exit):
  if exit:
    output = sys.stderr
  else:
    output = sys.stdout
  output.write(
    "usage: %s REPOS_PATH\n"
    "\n"
    "Once the program has started, type 'help' at the prompt for hints on\n"
    "using the shell.\n" % sys.argv[0])
  sys.exit(exit)

def main():
  if len(sys.argv) < 2:
    usage(1)

  try:
    import termios
    attrs = termios.tcgetattr(sys.stdin)
    termios.tcsetattr(sys.stdin, termios.TCSANOW, attrs)
  except:
    pass

  util.run_app(SVNShell, sys.argv[1])

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before>import numpy as np
import cv2

R_MEAN = 123
G_MEAN = 117
B_MEAN = 104

def substract_mean(image_array):
    image_array = image_array.astype(np.float32)
    image_array[:, :, 0] -= R_MEAN
    image_array[:, :, 1] -= G_MEAN
    image_array[:, :, 2] -= B_MEAN
    image_array = image_array[:, :, ::-1]
    return image_array

def load_image(image_path, target_size=None, return_original_shape=False):
    image_array = cv2.imread(image_path)
    image_array = cv2.cvtColor(image_array, cv2.COLOR_BGR2RGB)
    height, width = image_array.shape[:2]
    if target_size is not None:
        image_array = cv2.resize(image_array, target_size)
    if return_original_shape:
        return image_array, (height, width)
    else:
        return image_array

def get_image_size(image_path):
    image_array = cv2.imread(image_path)
    height, width = image_array.shape[:2]
    return (height, width)
<commit_msg>Added option to load image using RGB or BGR format<commit_after>import numpy as np
import cv2

R_MEAN = 123
G_MEAN = 117
B_MEAN = 104

def substract_mean(image_array):
    image_array = image_array.astype(np.float32)
    image_array[:, :, 0] -= R_MEAN
    image_array[:, :, 1] -= G_MEAN
    image_array[:, :, 2] -= B_MEAN
    image_array = image_array[:, :, ::-1]
    return image_array

def load_image(image_path, target_size=None,
               return_original_shape=False, RGB=True):
    image_array = cv2.imread(image_path)
    if RGB:
        image_array = cv2.cvtColor(image_array, cv2.COLOR_BGR2RGB)
    height, width = image_array.shape[:2]
    if target_size is not None:
        image_array = cv2.resize(image_array, target_size)
    if return_original_shape:
        return image_array, (height, width)
    else:
        return image_array

def get_image_size(image_path):
    image_array = cv2.imread(image_path)
    height, width = image_array.shape[:2]
    return (height, width)
<|endoftext|>