<commit_before>#!/usr/bin/env python
from data_explorer.plot_utils import plot_1d, plot_2d, plot_3d
import numpy as np
from matplotlib import pyplot
from sys import stdin
from optparse import OptionParser
from collections import deque

def parse_args():
  op = OptionParser()
  op.add_option('-3', action='store_true', default=False,
                help='plot in 3D', dest='three_d')
  op.add_option('-t','--transpose', action='store_true',
                default=False, help='transpose data')
  op.add_option('-y', action='store_true', default=False,
                help='use a 1:n range for x values')
  op.add_option('-x', action='store_true', default=False,
                help='use first column for x values')
  op.add_option('--log', action='store_true',
                default=False, help='plot with log scale y')
  op.add_option('--marker', type=str, default='-',
                help='line style/marker flags')
  op.add_option('--legend', type=str, default='', help='CSV legend labels')
  op.add_option('--xlabel', type=str, default='', help='X axis label')
  op.add_option('--ylabel', type=str, default='', help='Y axis label')
  op.add_option('--delim', type=str, default=None,
                help='Column delimiter (default: whitespace)')
  op.add_option('-s', type=int, default=1,
                help='smoothing value (default 1,no smoothing)')
  op.add_option('-r', '--rolling', type=int,
                help='animated rolling graph buffer size')
  op.add_option('-d', '--downsample', type=float,
                help='sampling rate, as a ratio of total # samples')
  op.add_option('--hist', type=int, default=0,
                help='When >0, plots a histogram with n buckets')
  return op.parse_args()

def preprocess(data, opts):
  if opts.transpose:
    data = data.T
  if opts.s > 2:
    window = np.ones(opts.s)/float(opts.s)
    if len(data.shape) == 1:
      data = np.convolve(window,data,mode='valid')
    else:
      if len(data.shape) > 1:
        assert opts.y, 'Can only convolve 1-d sequences'
      for i in xrange(data.shape[1]):
        data[:,i] = np.convolve(window,data[:,i],mode='same')
      pad = opts.s//2
      data = data[pad:-pad]
  if opts.downsample:
    assert len(data.shape) == 1, 'Multiple line downsampling is NYI'
    new_len = int(len(data) * opts.downsample)
    old_xs = np.arange(len(data))
    new_xs = np.unique(np.linspace(0,len(data),new_len).astype(int))
    data = np.interp(new_xs, old_xs, data)
  return data

def plot(data, opts):
  if opts.three_d:
    plot_3d(data,opts.marker)
  elif len(data.shape) == 1 or data.shape[1] == 1:
    plot_1d(data,opts.marker,log=opts.log)
  elif opts.y:
    plot_1d(data,opts.marker,log=opts.log)
  else:
    plot_2d(data,(not opts.x),opts.marker,log=opts.log)

def decorate(opts):
  if opts.legend:
    pyplot.legend(opts.legend.split(','),loc='best')
  if opts.xlabel:
    pyplot.xlabel(opts.xlabel)
  if opts.ylabel:
    pyplot.ylabel(opts.ylabel)

def static_plot(opts, fh):
  data = np.loadtxt(fh, delimiter=opts.delim)
  data = preprocess(data, opts)
  if opts.hist > 0:
    pyplot.hist(data, opts.hist)
  else:
    plot(data, opts)
  decorate(opts)
  pyplot.show()

def rolling_plot(opts, fh):
  disallowed_options = [
      ('-t',opts.transpose), ('-3',opts.three_d), ('-x',opts.x), ('-y',opts.y),
      ('-s > 1', opts.s > 1), ('-d', opts.downsample), ('--hist', opts.hist)
  ]
  for name,check in disallowed_options:
    if check:
      print "Option %s is not supported for rolling plots." % name
      return
  pyplot.ion()
  data = np.zeros(opts.rolling)
  ax = pyplot.gca()
  ax.set_autoscale_on(True)
  line2d, = ax.plot(data)
  buf = deque(maxlen=opts.rolling)
  delim = opts.delim if opts.delim is not None else ' '
  line = fh.readline()
  while line:
    buf.append(np.fromstring(line, sep=delim))
    data[:len(buf)] = np.array(buf).ravel()
    line2d.set_ydata(data)
    ax.relim()
    ax.autoscale_view(True,True,True)
    pyplot.draw()
    pyplot.pause(0.0001)
    line = fh.readline()

if __name__ == '__main__':
  opts, args = parse_args()
  fh = open(args[0]) if args else stdin
  if opts.rolling:
    rolling_plot(opts, fh)
  else:
    static_plot(opts, fh)
<commit_msg>add support for axis labels and log plots<commit_after>#!/usr/bin/env python
from data_explorer.plot_utils import plot_1d, plot_2d, plot_3d
import numpy as np
from matplotlib import pyplot
from sys import stdin
from optparse import OptionParser
from collections import deque

def parse_args():
  op = OptionParser()
  op.add_option('-3', action='store_true', default=False,
                help='plot in 3D', dest='three_d')
  op.add_option('-t','--transpose', action='store_true',
                default=False, help='transpose data')
  op.add_option('-y', action='store_true', default=False,
                help='use a 1:n range for x values')
  op.add_option('-x', action='store_true', default=False,
                help='use first column for x values')
  op.add_option('--log', action='store_true',
                default=False, help='plot with log scale y')
  op.add_option('--marker', type=str, default='-',
                help='line style/marker flags')
  op.add_option('--legend', type=str, default='', help='CSV legend labels')
  op.add_option('--xlabel', type=str, default='', help='X axis label')
  op.add_option('--ylabel', type=str, default='', help='Y axis label')
  op.add_option('--delim', type=str, default=None,
                help='Column delimiter (default: whitespace)')
  op.add_option('-s', type=int, default=1,
                help='smoothing value (default 1,no smoothing)')
  op.add_option('-r', '--rolling', type=int,
                help='animated rolling graph buffer size')
  op.add_option('-d', '--downsample', type=float,
                help='sampling rate, as a ratio of total # samples')
  op.add_option('--hist', type=int, default=0,
                help='When >0, plots a histogram with n buckets')
  return op.parse_args()

def preprocess(data, opts):
  if opts.transpose:
    data = data.T
  if opts.s > 2:
    window = np.ones(opts.s)/float(opts.s)
    if len(data.shape) == 1:
      data = np.convolve(window,data,mode='valid')
    else:
      if len(data.shape) > 1:
        assert opts.y, 'Can only convolve 1-d sequences'
      for i in xrange(data.shape[1]):
        data[:,i] = np.convolve(window,data[:,i],mode='same')
      pad = opts.s//2
      data = data[pad:-pad]
  if opts.downsample:
    assert len(data.shape) == 1, 'Multiple line downsampling is NYI'
    new_len = int(len(data) * opts.downsample)
    old_xs = np.arange(len(data))
    new_xs = np.unique(np.linspace(0,len(data),new_len).astype(int))
    data = np.interp(new_xs, old_xs, data)
  return data

def plot(data, opts):
  if opts.three_d:
    plot_3d(data,opts.marker)
  elif len(data.shape) == 1 or data.shape[1] == 1:
    plot_1d(data,opts.marker,log=opts.log)
  elif opts.y:
    plot_1d(data,opts.marker,log=opts.log)
  else:
    plot_2d(data,(not opts.x),opts.marker,log=opts.log)

def decorate(opts):
  if opts.legend:
    pyplot.legend(opts.legend.split(','),loc='best')
  if opts.xlabel:
    pyplot.xlabel(opts.xlabel)
  if opts.ylabel:
    pyplot.ylabel(opts.ylabel)

def static_plot(opts, fh):
  data = np.loadtxt(fh, delimiter=opts.delim)
  data = preprocess(data, opts)
  if opts.hist > 0:
    pyplot.hist(data, opts.hist)
  else:
    plot(data, opts)
  decorate(opts)
  pyplot.show()

def rolling_plot(opts, fh):
  disallowed_options = [
      ('-t',opts.transpose), ('-3',opts.three_d), ('-x',opts.x), ('-y',opts.y),
      ('-s > 1', opts.s > 1), ('-d', opts.downsample), ('--hist', opts.hist)
  ]
  for name,check in disallowed_options:
    if check:
      print "Option %s is not supported for rolling plots." % name
      return
  pyplot.ion()
  data = np.zeros(opts.rolling)
  ax = pyplot.gca()
  ax.set_autoscale_on(True)
  ax.set_xlabel(opts.xlabel)
  ax.set_ylabel(opts.ylabel)
  if opts.log:
    line2d, = ax.semilogy(data+1, opts.marker)
  else:
    line2d, = ax.plot(data, opts.marker)
  buf = deque(maxlen=opts.rolling)
  delim = opts.delim if opts.delim is not None else ' '
  line = fh.readline()
  while line:
    buf.append(np.fromstring(line, sep=delim))
    data[:len(buf)] = np.array(buf).ravel()
    line2d.set_ydata(data)
    ax.relim()
    ax.autoscale_view(True,True,True)
    pyplot.draw()
    pyplot.pause(0.0001)
    line = fh.readline()

if __name__ == '__main__':
  opts, args = parse_args()
  fh = open(args[0]) if args else stdin
  if opts.rolling:
    rolling_plot(opts, fh)
  else:
    static_plot(opts, fh)
<|endoftext|>


<commit_before>import netCDF4
from netCDF4 import Dataset
from mpl_toolkits.basemap import Basemap  #map stuff
import numpy as np
import matplotlib.pyplot as plt
import os
import glob
from datetime import datetime, timedelta
import pandas as pd
get_ipython().magic(u'matplotlib inline')

###########
# Below are helper functions for the satellite
###########

def plot_satellite_image(filename):
	'''takes in a file and outputs a plot of satellite'''
	rootgrp = Dataset(filename, "a", format="NETCDF4")
	lons = rootgrp.variables['lon'][:]
	lats = rootgrp.variables['lat'][:]
	data = rootgrp.variables['data'][:]
	rootgrp.close() #need to close before you can open again

	# Get some parameters for the Stereographic Projection
	m = Basemap(width=800000,height=800000, #create a basemap object with these parameters
	            resolution='l',projection='stere',\
	            lat_ts=40,lat_0=39.5,lon_0=-104.5)

	xi, yi = m(lons, lats) #map onton x and y for plotting
	plt.figure(figsize=(10,10)) # Plot Data
	cs = m.pcolor(xi,yi,np.squeeze(data)) #data is 1 x 14 x 36, squeeze makes it 14 x 36

	m.drawparallels(np.arange(-80., 81., 1.), labels=[1,0,0,0], fontsize=10) # Add Grid Lines
	m.drawmeridians(np.arange(-180., 181., 1.), labels=[0,0,0,1], fontsize=10) # Add Grid Lines
	m.drawstates(linewidth=3) # Add state boundaries

	cbar = m.colorbar(cs, location='bottom', pad="10%") # Add Colorbar
	plt.title('GOES 15 - Sensor 1') # Add Title
	plt.show()

def return_satellite_data(filename, filefolder):
	''' Input:  filename
		Output: lons, lats, data'''
	rootgrp = Dataset(filefolder + filename, "a", format="NETCDF4") #generic name for data
	lons = rootgrp.variables['lon'][:] #extract lons ...
	lats = rootgrp.variables['lat'][:] #...lats...
	data = rootgrp.variables['data'][:] #...and data from netCDF file
	rootgrp.close() #need to close before you can open again
	return (lons, lats, np.squeeze(data))

def find_file_details(filefolder, filetype = 'nc'): #match 2 or 3 letters
    '''Takes in a filefolder with satellite data and returns list of files,
        list of file details, list of dates, and list of channels
        Example usage:  filefolder = "data/satellite/colorado/summer6months/data"
						list_of_files, list_of_files_details, \
						list_of_dates, list_of_channels = find_file_details(filefolder)'''

    data_files = os.listdir(filefolder) #list files in directory

    list_of_files = [] #contains all the files
    list_of_files_details = [] #contains information collected from filename
    for myfile in data_files:
        if (myfile[-2:] == filetype or myfile[-3:] == filetype): #only get netCDF by default
            list_of_files.append(myfile)
            list_of_files_details.append(myfile.split('.'))

    list_of_dates = [] #contains datetime data for files
    list_of_channels = [] #contains channel data
    for i,_ in enumerate(list_of_files_details):
        mytime = list_of_files_details[i][1]+" "+list_of_files_details[i][2]+" "+list_of_files_details[i][3]
        mydatetime = datetime.strptime(mytime, '%Y %j %H%M%S')
        list_of_dates.append(mydatetime)
        list_of_channels.append(list_of_files_details[i][4])
    return (list_of_files, list_of_files_details, list_of_dates, list_of_channels)

def find_closest_date(desired_datetime, filefolder):
	'''Input: desired datetime, Output: datetime of closest file(s)
	Example usage:   desired_datetime = datetime(2014, 5, 5, 19)
	closest_datetime = find_closest_date(desired_datetime)'''
	list_of_files, list_of_files_details, \
	list_of_dates, list_of_channels = find_file_details(filefolder)
	time_differences = []
	for i,time in enumerate(list_of_dates):
		time_difference = list_of_dates[i] - desired_datetime
		time_differences.append(abs(time_difference).total_seconds())

	if min(time_differences) < 10800: #return datetime only if < 3 hours
		return list_of_dates[np.argmin(time_differences)]
	else:
		print "No file with this datetime within 3 hours!"+" Desired datetime = "+ str(desired_datetime)

def find_filename(desired_datetime, desired_channel, filefolder):
	'''return filename with desired features
	Example usage:  desired_channel = 'BAND_01'
	desired_datetime = datetime(2014, 4, 2, 12)
	print find_filename(desired_datetime, desired_channel)'''
	list_of_files, list_of_files_details, \
	list_of_dates, list_of_channels = find_file_details(filefolder)
	closest_datetime = find_closest_date(desired_datetime, filefolder)
	list_of_channel_matches = []
	for i,channel in enumerate(list_of_channels):
		if channel == desired_channel:
			list_of_channel_matches.append(i)

	list_of_date_matches = []
	for i,date in enumerate(list_of_dates):
		if date == closest_datetime:
			list_of_date_matches.append(i)
	#returns interested file (should just be one)
	return list_of_files[list(set(list_of_channel_matches) & set(list_of_date_matches))[0]]

##################################################################
# Above are functions for satellite data.  Below are functions for sensor
# and PV Output data
##################################################################

# This works for both sensor and pvoutput data:

def find_file_from_date(desired_date, filefolder, filetype = 'csv'):
    '''Input: datetime, folder, filetype; Output: file
    Usage: filefolder = "data/pvoutput/pvoutput6months/"
	desired_date = datetime(2014, 5, 5) #year, month, day [, hour, minute, second]
	find_file_from_date(desired_date, filefolder)'''
    data_files = os.listdir(filefolder)

    list_of_files = [] #contains all the files
    list_of_files_details = [] #contains information collected from filename
    for myfile in data_files:
        if (myfile[-2:] == filetype or myfile[-3:] == filetype): #only get netCDF by default
            list_of_files.append(myfile)
            list_of_files_details.append(myfile.split('.'))

    for i,val in enumerate(list_of_files_details):
        try:
            if datetime.strptime(list_of_files_details[i][0], '%Y%m%d') == desired_date:
                return list_of_files[i]
        except:
            pass

def pad(x):
    time = x.astype(str).zfill(4)
    return time[:2] + ':' + time[2:]

def return_sensor_data(filename, filefolder):
    '''Input: desired file and folder
    Output: sensor data pandas dataframe
    Usage: filefolder = 'data/sensor_data/colorado6months/'
    filename = '20140401.csv'
    return_sensor_data(myfile, filefolder)'''

    df_header = pd.read_csv(filefolder + 'header.csv')
    headers = df_header.columns
    df_sensor = pd.read_csv(filefolder + filename,header=None) #sensors
    df_sensor.columns = headers
    df_sensor['MST'] = df_sensor['MST'].map(pad)
    #make a datetime index:
    df_sensor['datetime'] = pd.to_datetime(df_sensor['Year'].astype(str)+
                                           df_sensor['DOY'].astype(str)+
                                           df_sensor['MST'].astype(str),
                                           format='%Y%j%H:%M')
    #convert to UTC time, website shows that they disregard daylight savings time
    df_sensor['datetime'] = df_sensor['datetime'] + pd.Timedelta(hours=7)
    df_sensor.set_index(['datetime'],inplace=True) #set created column as index ...
    df_sensor = df_sensor.resample('H')# ... so that we can resample it (hourly)
    return df_sensor

def return_pvoutput_data(filename, filefolder):
    '''Input: Desired file and folder
    Output: PV Output dataframe
    Usage: filefolder = 'data/pvoutput/pvoutput6months/'
    filename = '20140401.csv'
    return_pvoutput_data(filename, filefolder)'''
    df_output = pd.read_csv(filefolder + filename) #pvoutput
    #df_output['datetime'] = df_output['datetime'] + pd.Timedelta(hours=6) #convert to utc time
    df_output['datetime'] = df_output['datetime'].apply(pd.to_datetime)
    df_output['datetime'] = df_output['datetime'] + pd.Timedelta(hours=6) #convert to utc time
    df_output.set_index(['datetime'],inplace=True) #set created column as index ...
    df_output = df_output.resample('H') # ...so that we can resample it (hourly)
    return df_output[['Power']]

################### Below are functions for querying the data to build models ##########
def make_time(mytime, sunlight = 15, days = 182):
    '''Input: mytime (startime datetime) [,sunlight (hours), days]
    Output: a list of daylight times'''
    times = []
    for j in range(0,days): #days
        for i in range(0,sunlight): #sunlight time
            times.append(mytime)
            mytime += timedelta(hours = 1)
        mytime += timedelta(hours = (24-sunlight))    #night time, skip this
    return times

<commit_msg>wed2 1pm mac<commit_after>import netCDF4
from netCDF4 import Dataset
from mpl_toolkits.basemap import Basemap  #map stuff
import numpy as np
import matplotlib.pyplot as plt
import os
import glob
from datetime import datetime, timedelta
import pandas as pd
get_ipython().magic(u'matplotlib inline')

###########
# Below are helper functions for the satellite
###########

def plot_satellite_image(filename):
	'''takes in a file and outputs a plot of satellite'''
	rootgrp = Dataset(filename, "a", format="NETCDF4")
	lons = rootgrp.variables['lon'][:]
	lats = rootgrp.variables['lat'][:]
	data = rootgrp.variables['data'][:]
	rootgrp.close() #need to close before you can open again

	# Get some parameters for the Stereographic Projection
	m = Basemap(width=800000,height=800000, #create a basemap object with these parameters
	            resolution='l',projection='stere',\
	            lat_ts=40,lat_0=39.5,lon_0=-104.5)

	xi, yi = m(lons, lats) #map onton x and y for plotting
	plt.figure(figsize=(10,10)) # Plot Data
	cs = m.pcolor(xi,yi,np.squeeze(data)) #data is 1 x 14 x 36, squeeze makes it 14 x 36

	m.drawparallels(np.arange(-80., 81., 1.), labels=[1,0,0,0], fontsize=10) # Add Grid Lines
	m.drawmeridians(np.arange(-180., 181., 1.), labels=[0,0,0,1], fontsize=10) # Add Grid Lines
	m.drawstates(linewidth=3) # Add state boundaries

	cbar = m.colorbar(cs, location='bottom', pad="10%") # Add Colorbar
	plt.title('GOES 15 - Sensor 1') # Add Title
	plt.show()

def return_satellite_data(filename, filefolder):
	''' Input:  filename
		Output: lons, lats, data'''
	rootgrp = Dataset(filefolder + filename, "a", format="NETCDF4") #generic name for data
	lons = rootgrp.variables['lon'][:] #extract lons ...
	lats = rootgrp.variables['lat'][:] #...lats...
	data = rootgrp.variables['data'][:] #...and data from netCDF file
	rootgrp.close() #need to close before you can open again
	return (lons, lats, np.squeeze(data))

def find_file_details(filefolder, filetype = 'nc'): #match 2 or 3 letters
    '''Takes in a filefolder with satellite data and returns list of files,
        list of file details, list of dates, and list of channels
        Example usage:  filefolder = "data/satellite/colorado/summer6months/data"
						list_of_files, list_of_files_details, \
						list_of_dates, list_of_channels = find_file_details(filefolder)'''

    data_files = os.listdir(filefolder) #list files in directory

    list_of_files = [] #contains all the files
    list_of_files_details = [] #contains information collected from filename
    for myfile in data_files:
        if (myfile[-2:] == filetype or myfile[-3:] == filetype): #only get netCDF by default
            list_of_files.append(myfile)
            list_of_files_details.append(myfile.split('.'))

    list_of_dates = [] #contains datetime data for files
    list_of_channels = [] #contains channel data
    for i,_ in enumerate(list_of_files_details):
        mytime = list_of_files_details[i][1]+" "+list_of_files_details[i][2]+" "+list_of_files_details[i][3]
        mydatetime = datetime.strptime(mytime, '%Y %j %H%M%S')
        list_of_dates.append(mydatetime)
        list_of_channels.append(list_of_files_details[i][4])
    return (list_of_files, list_of_files_details, list_of_dates, list_of_channels)

def find_closest_date(desired_datetime, filefolder):
	'''Input: desired datetime, Output: datetime of closest file(s)
	Example usage:   desired_datetime = datetime(2014, 5, 5, 19)
	closest_datetime = find_closest_date(desired_datetime)'''
	list_of_files, list_of_files_details, \
	list_of_dates, list_of_channels = find_file_details(filefolder)
	time_differences = []
	for i,time in enumerate(list_of_dates):
		time_difference = list_of_dates[i] - desired_datetime
		time_differences.append(abs(time_difference).total_seconds())

	if min(time_differences) < 10800: #return datetime only if < 3 hours
		return list_of_dates[np.argmin(time_differences)]
	else:
		print "No file with this datetime within 3 hours!"+" Desired datetime = "+ str(desired_datetime)

def find_filename(desired_datetime, desired_channel, filefolder):
	'''return filename with desired features
	Example usage:  desired_channel = 'BAND_01'
	desired_datetime = datetime(2014, 4, 2, 12)
	print find_filename(desired_datetime, desired_channel)'''
	list_of_files, list_of_files_details, \
	list_of_dates, list_of_channels = find_file_details(filefolder)
	closest_datetime = find_closest_date(desired_datetime, filefolder)
	list_of_channel_matches = []
	for i,channel in enumerate(list_of_channels):
		if channel == desired_channel:
			list_of_channel_matches.append(i)

	list_of_date_matches = []
	for i,date in enumerate(list_of_dates):
		if date == closest_datetime:
			list_of_date_matches.append(i)
	#returns interested file (should just be one)
	return list_of_files[list(set(list_of_channel_matches) & set(list_of_date_matches))[0]]

##################################################################
# Above are functions for satellite data.  Below are functions for sensor
# and PV Output data
##################################################################

# This works for both sensor and pvoutput data:

def find_file_from_date(desired_date, filefolder, filetype = 'csv'):
    '''Input: datetime, folder, filetype; Output: file
    Usage: filefolder = "data/pvoutput/pvoutput6months/"
	desired_date = datetime(2014, 5, 5) #year, month, day [, hour, minute, second]
	find_file_from_date(desired_date, filefolder)'''
    data_files = os.listdir(filefolder)

    list_of_files = [] #contains all the files
    list_of_files_details = [] #contains information collected from filename
    for myfile in data_files:
        if (myfile[-2:] == filetype or myfile[-3:] == filetype): #only get netCDF by default
            list_of_files.append(myfile)
            list_of_files_details.append(myfile.split('.'))

    for i,val in enumerate(list_of_files_details):
        try:
            if datetime.strptime(list_of_files_details[i][0], '%Y%m%d') == desired_date:
                return list_of_files[i]
        except:
            pass

def pad(x):
    time = x.astype(str).zfill(4)
    return time[:2] + ':' + time[2:]

def return_sensor_data(filename, filefolder):
    '''Input: desired file and folder
    Output: sensor data pandas dataframe
    Usage: filefolder = 'data/sensor_data/colorado6months/'
    filename = '20140401.csv'
    return_sensor_data(myfile, filefolder)'''

    df_header = pd.read_csv(filefolder + 'header.csv')
    headers = df_header.columns
    df_sensor = pd.read_csv(filefolder + filename,header=None) #sensors
    df_sensor.columns = headers
    df_sensor['MST'] = df_sensor['MST'].map(pad)
    #make a datetime index:
    df_sensor['datetime'] = pd.to_datetime(df_sensor['Year'].astype(str)+
                                           df_sensor['DOY'].astype(str)+
                                           df_sensor['MST'].astype(str),
                                           format='%Y%j%H:%M')
    #convert to UTC time, website shows that they disregard daylight savings time
    df_sensor['datetime'] = df_sensor['datetime'] + pd.Timedelta(hours=7)
    df_sensor.set_index(['datetime'],inplace=True) #set created column as index ...
    df_sensor = df_sensor.resample('H')# ... so that we can resample it (hourly)
    return df_sensor

def return_pvoutput_data(filename, filefolder):
    '''Input: Desired file and folder
    Output: PV Output dataframe
    Usage: filefolder = 'data/pvoutput/pvoutput6months/'
    filename = '20140401.csv'
    return_pvoutput_data(filename, filefolder)'''
    df_output = pd.read_csv(filefolder + filename) #pvoutput
    #df_output['datetime'] = df_output['datetime'] + pd.Timedelta(hours=6) #convert to utc time
    df_output['datetime'] = df_output['datetime'].apply(pd.to_datetime)
    df_output['datetime'] = df_output['datetime'] + pd.Timedelta(hours=6) #convert to utc time
    df_output.set_index(['datetime'],inplace=True) #set created column as index ...
    df_output = df_output.resample('H') # ...so that we can resample it (hourly)
    return df_output[['Power']]

################### Below are functions for querying the data to build models ##########
def make_time(mytime, sunlight = 15, days = 182):
    '''Input: mytime (startime datetime) [,sunlight (hours), days]
    Output: a list of daylight times'''
    times = []
    for j in range(0,days): #days
        for i in range(0,sunlight): #sunlight time
            times.append(mytime)
            mytime += timedelta(hours = 1)
        mytime += timedelta(hours = (24-sunlight))    #night time, skip this
    return times

def datetime_from_date(desired_datetime):
    '''Input: datetime, Output: date in datetime format
    Info: Subtracts 6 hours from datetime to make sure in correct folder'''
    from datetime import datetime, timedelta, time
    desired_date = (desired_datetime - timedelta(hours=6)).date() #make sure correct date
    desired_date = datetime.combine(desired_date, time.min) #get into datetime format
    return desired_date

<|endoftext|>


<commit_before>from django.http import HttpResponse
from django.contrib.gis.geos import GEOSGeometry
from django.contrib.gis.geos import WKTWriter
from django.utils import simplejson as json
import types

def geojson_to_csv(request):
    """
    This function should recieve a post request with
    a featurecollection.

    From this featurecollection this function will make a csv
    string and return it as a csv file for download.
    """

    json_string = request.raw_post_data

    feature_collection = json.loads(json_string)

    crs = getattr(feature_collection,
                  'crs',
                  {
                    'type': 'name',
                    'properties': {
                        'name': 'urn:ogc:def:crs:EPSG::4979'
                        }})

    features = feature_collection['features']

    csv_header_set = set()
    for feature in features:
        csv_header_set.update(create_csv_header_set(feature['properties']))

    csv_header_list = list(csv_header_set)
    csv_header_list.sort()
    csv_header_list.insert(0, 'wkt')
    csv_header_list.insert(0, 'id')

    #create the csv
    csv_string = u""

    #write the header
    for header in csv_header_list:
        csv_string = u"%s%s;" % (csv_string, header)

    csv_string = u"%s\n" % (csv_string,)
    csv_row = []
    csv_rows = []
    print csv_string

    for feature in features:

        geometry = feature['geometry']
        properties = feature['properties']

        id = None
        if feature.has_key('id'):
            id = feature['id']

        properties['id'] = id

        #make the geometry
        properties['wkt'] = GEOSGeometry(json.dumps(geometry)).wkt

        value_list = get_value_list(properties, csv_header_list)

        for value in value_list:

            #remove harmful characterers
            if type(value) == types.UnicodeType or type(value) == types.StringType:
                value = value.replace(";", "")
                value = value.replace("\n", " ")
                value = value.replace("\r", " ")

            csv_row.append("%s;" % value)

        csv_rows.append(''.join(csv_row))

    csv_string = "%s%s" % (csv_string, '\n'.join(csv_rows))

    return HttpResponse(csv_string,
                        content_type='text/csv')

def json_to_csv(request):
    """
    This function takes a array of json objects
    and returns a csv file for download.
    """

    json_string = request.raw_post_data

    json_array = json.loads(json_string)

    if type(json_array) == types.DictType:
        json_array = [json_array]

    #this part has to check for nested objects
    csv_header_set = set()
    for json_dict in json_array:
        csv_header_set.update(create_csv_header_set(json_dict))

    csv_header_list = list(csv_header_set)
    csv_header_list.sort()

    #create the csv
    csv_string = u""

    #write the header
    for header in csv_header_list:
        csv_string = u"%s%s;" % (csv_string, header)

    csv_string = u"%s\n" % (csv_string,)
    csv_row = []
    csv_rows = []

    for json_dict in json_array:
        value_list = get_value_list(json_dict, csv_header_list)

        for value in value_list:

            #remove harmful characterers
            if type(value) == types.UnicodeType or type(value) == types.StringType:
                value = value.replace(";", "")
                value = value.replace("\n", " ")
                value = value.replace("\r", " ")

            #modify value to json string
            value = json.dumps(value)

            csv_row.append("%s;" % value)

        csv_rows.append(''.join(csv_row))

    csv_string = "%s%s" % (csv_string, '\n'.join(csv_rows))

    return HttpResponse(csv_string,
                        mimetype='text/csv')

def create_csv_header_set(json_dict):

    csv_header_set = set()

    for key, item in json_dict.items():
        if type(item) == types.DictType:
            temp_set = create_csv_header_set(item)
            for header in temp_set:
                add_header = "%s.%s" % (key, header)
                csv_header_set.add(add_header)
        else:
            csv_header_set.add(key)

    return csv_header_set

def get_value_list(json_dict, key_list):

    value_list = []

    for key in key_list:

        keys = key.split(".")

        if len(keys) == 1:
            if json_dict.has_key(key):
                value_list.append(json_dict[key])
            else:
                value_list.append("")
        else:
            temp_key = ""

            for k in keys[1:]:
                if temp_key != "":
                    temp_key = temp_key + "." + k
                else:
                    temp_key = k

            if json_dict.has_key(keys[0]):
                temp_value_list = get_value_list(json_dict[keys[0]],
                                                 [temp_key])
            else:
                temp_value_list = get_value_list({},
                                                 [temp_key])

            value_list.extend(temp_value_list)

    return value_list
<commit_msg>fixed a bug with not emptying rows between rows<commit_after>from django.http import HttpResponse
from django.contrib.gis.geos import GEOSGeometry
from django.contrib.gis.geos import WKTWriter
from django.utils import simplejson as json
import types

def geojson_to_csv(request):
    """
    This function should recieve a post request with
    a featurecollection.

    From this featurecollection this function will make a csv
    string and return it as a csv file for download.
    """

    json_string = request.raw_post_data

    feature_collection = json.loads(json_string)

    crs = getattr(feature_collection,
                  'crs',
                  {
                    'type': 'name',
                    'properties': {
                        'name': 'urn:ogc:def:crs:EPSG::4979'
                        }})

    features = feature_collection['features']

    csv_header_set = set()
    for feature in features:
        csv_header_set.update(create_csv_header_set(feature['properties']))

    csv_header_list = list(csv_header_set)
    csv_header_list.sort()
    csv_header_list.insert(0, 'wkt')
    csv_header_list.insert(0, 'id')

    #create the csv
    csv_string = u""

    #write the header
    for header in csv_header_list:
        csv_string = u"%s%s;" % (csv_string, header)

    csv_string = u"%s\n" % (csv_string,)
    csv_row = []
    csv_rows = []
    print csv_string

    for feature in features:

        geometry = feature['geometry']
        properties = feature['properties']

        id = None
        if feature.has_key('id'):
            id = feature['id']

        properties['id'] = id

        #make the geometry
        properties['wkt'] = GEOSGeometry(json.dumps(geometry)).wkt

        value_list = get_value_list(properties, csv_header_list)

        for value in value_list:

            #remove harmful characterers
            if type(value) == types.UnicodeType or type(value) == types.StringType:
                value = value.replace(";", "")
                value = value.replace("\n", " ")
                value = value.replace("\r", " ")

            csv_row.append("%s;" % value)

        csv_rows.append(''.join(csv_row))
        csv_row = []

    csv_string = "%s%s" % (csv_string, '\n'.join(csv_rows))

    return HttpResponse(csv_string,
                        content_type='text/csv')

def json_to_csv(request):
    """
    This function takes a array of json objects
    and returns a csv file for download.
    """

    json_string = request.raw_post_data

    json_array = json.loads(json_string)

    if type(json_array) == types.DictType:
        json_array = [json_array]

    #this part has to check for nested objects
    csv_header_set = set()
    for json_dict in json_array:
        csv_header_set.update(create_csv_header_set(json_dict))

    csv_header_list = list(csv_header_set)
    csv_header_list.sort()

    #create the csv
    csv_string = u""

    #write the header
    for header in csv_header_list:
        csv_string = u"%s%s;" % (csv_string, header)

    csv_string = u"%s\n" % (csv_string,)
    csv_row = []
    csv_rows = []

    for json_dict in json_array:
        value_list = get_value_list(json_dict, csv_header_list)

        for value in value_list:

            #remove harmful characterers
            if type(value) == types.UnicodeType or type(value) == types.StringType:
                value = value.replace(";", "")
                value = value.replace("\n", " ")
                value = value.replace("\r", " ")

            #modify value to json string
            value = json.dumps(value)

            csv_row.append("%s;" % value)

        csv_rows.append(''.join(csv_row))
        csv_row = []

    csv_string = "%s%s" % (csv_string, '\n'.join(csv_rows))

    return HttpResponse(csv_string,
                        mimetype='text/csv')

def create_csv_header_set(json_dict):

    csv_header_set = set()

    for key, item in json_dict.items():
        if type(item) == types.DictType:
            temp_set = create_csv_header_set(item)
            for header in temp_set:
                add_header = "%s.%s" % (key, header)
                csv_header_set.add(add_header)
        else:
            csv_header_set.add(key)

    return csv_header_set

def get_value_list(json_dict, key_list):

    value_list = []

    for key in key_list:

        keys = key.split(".")

        if len(keys) == 1:
            if json_dict.has_key(key):
                value_list.append(json_dict[key])
            else:
                value_list.append("")
        else:
            temp_key = ""

            for k in keys[1:]:
                if temp_key != "":
                    temp_key = temp_key + "." + k
                else:
                    temp_key = k

            if json_dict.has_key(keys[0]):
                temp_value_list = get_value_list(json_dict[keys[0]],
                                                 [temp_key])
            else:
                temp_value_list = get_value_list({},
                                                 [temp_key])

            value_list.extend(temp_value_list)

    return value_list
<|endoftext|>


<commit_before>#!/usr/bin/env python
# xfail-license

import sys, fileinput, subprocess, re
from licenseck import *
import snapshot

err=0
cols=100

# Be careful to support Python 2.4, 2.6, and 3.x here!
config_proc=subprocess.Popen([ "git", "config", "core.autocrlf" ],
                             stdout=subprocess.PIPE)
result=config_proc.communicate()[0]

true="true".encode('utf8')
autocrlf=result.strip() == true if result is not None else False

def report_error_name_no(name, no, s):
    global err
    print("%s:%d: %s" % (name, no, s))
    err=1

def report_err(s):
    report_error_name_no(fileinput.filename(), fileinput.filelineno(), s)

def report_warn(s):
    print("%s:%d: %s" % (fileinput.filename(),
                         fileinput.filelineno(),
                         s))

def do_license_check(name, contents):
    if not check_license(name, contents):
        report_error_name_no(name, 1, "incorrect license")

file_names = [s for s in sys.argv[1:] if (not s.endswith("_gen.rs"))
                                     and (not ".#" in s)]

current_name = ""
current_contents = ""

try:
    for line in fileinput.input(file_names,
                                openhook=fileinput.hook_encoded("utf-8")):

        if fileinput.filename().find("tidy.py") == -1:
            if line.find("FIXME") != -1:
                if re.search("FIXME.*#\d+", line) == None:
                    report_err("FIXME without issue number")
            if line.find("TODO") != -1:
                report_err("TODO is deprecated; use FIXME")
            match = re.match(r'^.*//\s*(NOTE.*)$', line)
            if match:
                m = match.group(1)
                if "snap" in m.lower():
                    report_warn(match.group(1))
            match = re.match(r'^.*//\s*SNAP\s+(\w+)', line)
            if match:
                hsh = match.group(1)
                a, b, c, phash = snapshot.determine_curr_snapshot_info()
                if not phash.startswith(hsh):
                    report_err("Snapshot out of date: " + line)
            else:
                if "SNAP" in line:
                    report_warn("Unmatched SNAP line: " + line)

        if (line.find('\t') != -1 and
            fileinput.filename().find("Makefile") == -1):
            report_err("tab character")
        if not autocrlf and line.find('\r') != -1:
            report_err("CR character")
        if line.endswith(" \n") or line.endswith("\t\n"):
            report_err("trailing whitespace")
        line_len = len(line)-2 if autocrlf else len(line)-1

        if line_len > cols:
            report_err("line longer than %d chars" % cols)

        if fileinput.isfirstline() and current_name != "":
            do_license_check(current_name, current_contents)

        if fileinput.isfirstline():
            current_name = fileinput.filename()
            current_contents = ""

        current_contents += line

    if current_name != "":
        do_license_check(current_name, current_contents)

except UnicodeDecodeError, e:
    report_err("UTF-8 decoding error " + str(e))

sys.exit(err)
<commit_msg>Changed tidy script to remove restriction on FIXME and to warn against XXX instead<commit_after>#!/usr/bin/env python
# xfail-license

import sys, fileinput, subprocess, re
from licenseck import *
import snapshot

err=0
cols=100

# Be careful to support Python 2.4, 2.6, and 3.x here!
config_proc=subprocess.Popen([ "git", "config", "core.autocrlf" ],
                             stdout=subprocess.PIPE)
result=config_proc.communicate()[0]

true="true".encode('utf8')
autocrlf=result.strip() == true if result is not None else False

def report_error_name_no(name, no, s):
    global err
    print("%s:%d: %s" % (name, no, s))
    err=1

def report_err(s):
    report_error_name_no(fileinput.filename(), fileinput.filelineno(), s)

def report_warn(s):
    print("%s:%d: %s" % (fileinput.filename(),
                         fileinput.filelineno(),
                         s))

def do_license_check(name, contents):
    if not check_license(name, contents):
        report_error_name_no(name, 1, "incorrect license")

file_names = [s for s in sys.argv[1:] if (not s.endswith("_gen.rs"))
                                     and (not ".#" in s)]

current_name = ""
current_contents = ""

try:
    for line in fileinput.input(file_names,
                                openhook=fileinput.hook_encoded("utf-8")):

        if fileinput.filename().find("tidy.py") == -1:
            if line.find("// XXX") != -1:
                report_err("XXX is no longer necessary, use FIXME")
            if line.find("TODO") != -1:
                report_err("TODO is deprecated; use FIXME")
            match = re.match(r'^.*//\s*(NOTE.*)$', line)
            if match:
                m = match.group(1)
                if "snap" in m.lower():
                    report_warn(match.group(1))
            match = re.match(r'^.*//\s*SNAP\s+(\w+)', line)
            if match:
                hsh = match.group(1)
                a, b, c, phash = snapshot.determine_curr_snapshot_info()
                if not phash.startswith(hsh):
                    report_err("Snapshot out of date: " + line)
            else:
                if "SNAP" in line:
                    report_warn("Unmatched SNAP line: " + line)

        if (line.find('\t') != -1 and
            fileinput.filename().find("Makefile") == -1):
            report_err("tab character")
        if not autocrlf and line.find('\r') != -1:
            report_err("CR character")
        if line.endswith(" \n") or line.endswith("\t\n"):
            report_err("trailing whitespace")
        line_len = len(line)-2 if autocrlf else len(line)-1

        if line_len > cols:
            report_err("line longer than %d chars" % cols)

        if fileinput.isfirstline() and current_name != "":
            do_license_check(current_name, current_contents)

        if fileinput.isfirstline():
            current_name = fileinput.filename()
            current_contents = ""

        current_contents += line

    if current_name != "":
        do_license_check(current_name, current_contents)

except UnicodeDecodeError, e:
    report_err("UTF-8 decoding error " + str(e))

sys.exit(err)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import sys
import shutil
import datetime
import subprocess
import tarfile
import drdump

from django.core.management import call_command
from django.conf import settings

def get_base_path():
    if hasattr(settings, 'DATA_DOWNLOADER_PATH'):
        base_path = settings.DATA_DOWNLOADER_PATH
    else:
        base_path = os.path.abspath(os.path.join(settings.BASE_DIR, 'project/protected_medias/datas'))
    return base_path

class Dump(object):
    def __init__(self, data_type, base_path=None, database_dumper=None):
        assert data_type in ('data', 'db', 'media')
        self.data_type = data_type
        self.base_path = base_path or get_base_path()
        self.database_dumper = database_dumper or get_default_dumper()

    def get_metadata(self):
        try:
            infos = os.stat(self.path)
            date = datetime.datetime.fromtimestamp(int(infos.st_mtime))
            return {
                'date': date,
                'size': infos.st_size
            }
        except OSError:
            return {
                'date': None,
                'size': None
            }

    @property
    def project_name(self):
        return os.path.basename(settings.BASE_DIR)

    @property
    def archive_name(self):
        return "%s_%s.tar.gz" % (self.project_name, self.data_type)

    @property
    def mimetype(self):
        return "application/x-gzip"

    @property
    def path(self):
        return os.path.join(self.base_path, self.archive_name)

    def _dump_media(self):
        return [settings.MEDIA_ROOT.replace("%s/" % os.getcwd(), ''), ]

    def _clean_dumps_path(self):
        dumps_path = os.path.join(settings.BASE_DIR, 'dumps')
        if os.path.exists(dumps_path):
            shutil.rmtree(dumps_path)
        os.mkdir(dumps_path)

    def _ensure_base_path(self):
        try:
            os.makedirs(self.base_path)
        except OSError:
            pass

    def create(self):
        folders = []
        if self.data_type == 'db':
            folders.extend(self.database_dumper())
        elif self.data_type == 'media':
            folders.extend(self._dump_media())
        elif self.data_type == 'data':
            folders.extend(self._dump_media())
            folders.extend(self.database_dumper())

        self._ensure_base_path()
        with tarfile.open(self.path, "w:gz") as tar:
            for folder in folders:
                tar.add(folder)

    def destroy(self):
        os.remove(self.path)

class OldDrDump(object):
    @classmethod
    def default(cls):
        if hasattr(settings, 'DATADUMP_BIN_PATH'):
            return cls(settings.DATADUMP_BIN_PATH)
        if hasattr(sys, 'real_prefix'):
            # Running in a virtual env
            candidate = os.path.join(sys.prefix, 'bin/datadump')
            if os.path.exists(candidate):
                return cls(candidate)

        # Try a globally installed instance
        return cls('datadump')

    def __init__(self, bin_path):
        self.bin_path = bin_path

    def __call__(self):
        self._clean_dumps_path()
        subprocess.check_output(self.bin_path)
        dump_path = os.path.join(settings.BASE_DIR, 'dumps')
        return [
            dump_path.replace("%s/" % os.getcwd(), ''),
        ]

class DrDump(object):
    def __call__(self):
        call_command('dr_dump')
        dump_path = os.path.join(settings.BASE_DIR, 'dumps')
        return [
            dump_path.replace("%s/" % os.getcwd(), ''),
        ]

if drdump.__version__.startswith('0.'):
    get_default_dumper = OldDrDump.default
else:
    get_default_dumper = DrDump
<commit_msg>Revert "use relatif path"<commit_after># -*- coding: utf-8 -*-

import os
import sys
import shutil
import datetime
import subprocess
import tarfile
import drdump

from django.core.management import call_command
from django.conf import settings

def get_base_path():
    if hasattr(settings, 'DATA_DOWNLOADER_PATH'):
        base_path = settings.DATA_DOWNLOADER_PATH
    else:
        base_path = os.path.abspath(os.path.join(settings.BASE_DIR, 'project/protected_medias/datas'))
    return base_path

class Dump(object):
    def __init__(self, data_type, base_path=None, database_dumper=None):
        assert data_type in ('data', 'db', 'media')
        self.data_type = data_type
        self.base_path = base_path or get_base_path()
        self.database_dumper = database_dumper or get_default_dumper()

    def get_metadata(self):
        try:
            infos = os.stat(self.path)
            date = datetime.datetime.fromtimestamp(int(infos.st_mtime))
            return {
                'date': date,
                'size': infos.st_size
            }
        except OSError:
            return {
                'date': None,
                'size': None
            }

    @property
    def project_name(self):
        return os.path.basename(settings.BASE_DIR)

    @property
    def archive_name(self):
        return "%s_%s.tar.gz" % (self.project_name, self.data_type)

    @property
    def mimetype(self):
        return "application/x-gzip"

    @property
    def path(self):
        return os.path.join(self.base_path, self.archive_name)

    def _dump_media(self):
        return [
            settings.MEDIA_ROOT,
        ]

    def _clean_dumps_path(self):
        dumps_path = os.path.join(settings.BASE_DIR, 'dumps')
        if os.path.exists(dumps_path):
            shutil.rmtree(dumps_path)
        os.mkdir(dumps_path)

    def _ensure_base_path(self):
        try:
            os.makedirs(self.base_path)
        except OSError:
            pass

    def create(self):
        folders = []
        if self.data_type == 'db':
            folders.extend(self.database_dumper())
        elif self.data_type == 'media':
            folders.extend(self._dump_media())
        elif self.data_type == 'data':
            folders.extend(self._dump_media())
            folders.extend(self.database_dumper())

        self._ensure_base_path()
        with tarfile.open(self.path, "w:gz") as tar:
            for folder in folders:
                tar.add(folder)

    def destroy(self):
        os.remove(self.path)

class OldDrDump(object):
    @classmethod
    def default(cls):
        if hasattr(settings, 'DATADUMP_BIN_PATH'):
            return cls(settings.DATADUMP_BIN_PATH)
        if hasattr(sys, 'real_prefix'):
            # Running in a virtual env
            candidate = os.path.join(sys.prefix, 'bin/datadump')
            if os.path.exists(candidate):
                return cls(candidate)

        # Try a globally installed instance
        return cls('datadump')

    def __init__(self, bin_path):
        self.bin_path = bin_path

    def __call__(self):
        self._clean_dumps_path()
        subprocess.check_output(self.bin_path)
        dump_path = os.path.join(settings.BASE_DIR, 'dumps')
        return [
            dump_path,
        ]

class DrDump(object):
    def __call__(self):
        call_command('dr_dump')
        dump_path = os.path.join(settings.BASE_DIR, 'dumps')
        return [
            dump_path.replace("%s/" % os.getcwd(), ''),
        ]

if drdump.__version__.startswith('0.'):
    get_default_dumper = OldDrDump.default
else:
    get_default_dumper = DrDump
<|endoftext|>


<commit_before>
import math

def enter_point() :
    x = float(input("Enter an x value: "))
    y = float(input("Enter a y value: "))
    z = float(input("Enter a z value: "))
    return (x, y, z)

def three_points() :
    '''Returns a list of 3 3d pts'''
    points = []
    for i in range(3):
        points.append(enter_point())
    return points

def two_vectors(points) :
    '''Inputs a list of 2 pts and returns a list of vectors from 1st pt'''
    vectors = []
    for i in range(2) :
        x = points[i+1][0]-points[0][0]
        y = points[i+1][1]-points[0][1]
        z = points[i+1][2]-points[0][2]
        vectors.append((x,y,z))
    return vectors

def xprod(vectors) :
    '''Inputs a list of 2 vector and returns A, B, C, of plane equation'''
    A = vectors[0][1]*vectors[1][2]-vectors[0][2]*vectors[1][1]
    B = -1*(vectors[0][0]*vectors[1][2]-vectors[0][2]*vectors[1][0])
    C = vectors[0][0]*vectors[1][1]-vectors[0][1]*vectors[1][0]
    return (A, B, C)

def solve_tri(vectors) :
    '''Inputs a list of 2 vectors and returns 2 magnitudes and included angle'''
    dot = vectors[0][0]*vectors[1][0]-vectors[0][1]*vectors[1][1]+vectors[0][2]*vectors[1][2]
    mag1 = math.sqrt(vectors[0][0]**2 + vectors[0][1]**2 + vectors[0][2]**2)
    mag2 =

    <commit_msg>Updated from Brython Server: 2/12/2016 2:43:57 PM<commit_after>
import math

def enter_point() :
    x = float(input("Enter an x value: "))
    y = float(input("Enter a y value: "))
    z = float(input("Enter a z value: "))
    return (x, y, z)

def three_points() :
    '''Returns a list of 3 3d pts'''
    points = []
    for i in range(3):
        points.append(enter_point())
    return points

def two_vectors(points) :
    '''Inputs a list of 2 pts and returns a list of vectors from 1st pt'''
    vectors = []
    for i in range(2) :
        x = points[i+1][0]-points[0][0]
        y = points[i+1][1]-points[0][1]
        z = points[i+1][2]-points[0][2]
        vectors.append((x,y,z))
    return vectors

def xprod(vectors) :
    '''Inputs a list of 2 vector and returns A, B, C, of plane equation'''
    A = vectors[0][1]*vectors[1][2]-vectors[0][2]*vectors[1][1]
    B = -1*(vectors[0][0]*vectors[1][2]-vectors[0][2]*vectors[1][0])
    C = vectors[0][0]*vectors[1][1]-vectors[0][1]*vectors[1][0]
    return (A, B, C)

def solve_tri(vectors) :
    '''Inputs a list of 2 vectors and returns 2 magnitudes and included angle'''
    dot = vectors[0][0]*vectors[1][0]-vectors[0][1]*vectors[1][1]+vectors[0][2]*vectors[1][2]
    mag1 = math.sqrt(vectors[0][0]**2 + vectors[0][1]**2 + vectors[0][2]**2)
    mag2 = math.sqrt(vectors[1][0]**2 + vectors[1][1]**2 + vectors[1][2]**2)

    <|endoftext|>


<commit_before># Natural Language Toolkit: Naive Bayes Classifiers
#
# Copyright (C) 2001 University of Pennsylvania
# Author: Edward Loper <edloper@gradient.cis.upenn.edu>
# URL: <http://nltk.sf.net>
# For license information, see LICENSE.TXT
#
# $Id$

"""

                     P(class) * P(features|class)
P(class|features) = ------------------------------
                           P(features)

"""

from nltk.classifier import *
from nltk.feature import *
from nltk.probability import *
from nltk.token import Token
from nltk.tokenizer import WhitespaceTokenizer
from nltk.corpus import brown
from nltk.chktype import chktype as _chktype
import types

##//////////////////////////////////////////////////////
##  Naive Bayes Classifier
##//////////////////////////////////////////////////////

class NaiveBayesClassifier(ClassifierI, PropertyIndirectionMixIn):
    def __init__(self, class_probdist, feature_probdist, **property_names):
        """
        @param class_probdist: P(cls)
        @param feature_probdist: P(fval|cls,fid)
        """
        PropertyIndirectionMixIn.__init__(self, **property_names)
        self._class_probdist = class_probdist
        self._feature_probdist = feature_probdist
        self._classes = class_probdist.samples()

    def classes(self):
        return self._classes

    def classify(self, token):
        # Get the token's features.
        FEATURES = self.property('FEATURES')
        CLASS_PROBS = self.property('CLASS_PROBS')
        CLASS = self.property('CLASS')

        # Find the log probabilty of each class, given the features.
        logprob_dict = {}
        for cls in self._classes:
            logprob_dict[cls] = self._class_probdist.logprob(cls)
        for (cls, fname) in self._feature_probdist.conditions():
            probdist = self._feature_probdist[cls, fname]
            if fname not in token[FEATURES]: continue # [XX] is this right?
            fval = token[FEATURES][fname]
            logprob_dict[cls] += probdist.logprob(token[FEATURES][fname])

        probs = DictionaryProbDist(logprob_dict, normalize=True, log=True)
        token[CLASS_PROBS] = probs
        token[CLASS] = probs.max()

##//////////////////////////////////////////////////////
##  Naive Bayes Classifier Trainer
##//////////////////////////////////////////////////////

class NaiveBayesClassifierTrainer(ClassifierTrainerI,
                                  PropertyIndirectionMixIn):

    def train(self, tokens):
        FEATURES = self.property('FEATURES')
        CLASS = self.property('CLASS')

        class_freqdist = FreqDist()
        feature_freqdist = ConditionalFreqDist()
        feature_values = {}

        for token in tokens:
            cls = token[CLASS]
            class_freqdist.inc(cls)
            for fname, fval in token[FEATURES].items():
                # Recod the value in the freq(fval|cls, fname) distribution.
                feature_freqdist[cls, fname].inc(fval)
                # Record that fname can take the value fval.
                if fname not in feature_values: feature_values[fname] = Set()
                feature_values[fname].add(fval)

        # Create the P(cls) distribution
        class_probdist = ELEProbDist(class_freqdist)

        # Create the P(fval|cls, fname) distribution
        def make_probdist(freqdist, (cls, fname)):
            return ELEProbDist(freqdist, bins=len(feature_values[fname]))
        feature_probdist = ConditionalProbDist(feature_freqdist, make_probdist, True)

        # [xx] property indirection?
        return NaiveBayesClassifier(class_probdist, feature_probdist)

##//////////////////////////////////////////////////////
##  Multinomial play
##//////////////////////////////////////////////////////

# Features can fire multiple times.  Probability that a feature fires
# n times is p^n, where p is the probability that it fires once.  So
# we already know E(u)/n.  That's the freq dist we made.  Well,
# kinda...  Not really.  But to do that, we just need to inc by val,
# not by 1, in NaiveBayesClassifierFreqDist.

# So then we have E(fid)/n.  That is, p(fid).  So when we're asked to
# estimate P(id=val), take P(id=1)^val...

# Doing this to the freq-dist streight is playing a little fast &
# loose, though..  But doing it in a wrapper pdf seems fine.
# Hm.. still might be tricky, though.  I'll have to think about it.

#from math import exp
#class _MultinomialNaiveBayesClassifierProbDist(ProbDistI):
#    """
#    This class is not currently functional.
#    """
#    def __init__(self, fd_list, labeled_tokens, l=0.5):
#        fcounts = zeros(len(fd_list), 'd')
#        self._N = 0
#        fcounts += l
#        self._N += l*len(fcounts)
#
#        for tok in labeled_tokens:
#            self._N += 1
#            for (fid, val) in fd_list.detect(tok.type()).assignments():
#                fcounts[fid] += val
#
#        # Expected value for each feature
#        self._exp = fcounts / self._N
#
#    def prob(self, event):
#        # Inherit docs from FreqDistI
#        _chktype('NaiveBayesClassifierFreqDist.freq', 1, event, (AssignmentEvent,))
#        (fid, value) = event.assignment()
#
#        # valfact = value!
#        valfact = reduce( lambda x,y:x*y, range(2,value+1), 1.0 )
#
#        multinomial = ((self._exp[fid] ** value) / valfact)
#        poisson = ((self._exp[fid] ** value) * exp(-self._exp[fid]) /
#                   valfact)
#        return poisson
#
#class _MultinomialNaiveBayesClassifierTrainer(ClassifierTrainerI):
#    """
#    This class is not currently functional.
#    """
#    def __init__(self, fd_list):
#        self._fd_list = fd_list
#
#    def train(self, labeled_tokens, **kwargs):
#        labels = None
#        for (key, val) in kwargs.items():
#            if key == 'labels': labels = val
#            else: raise TypeError('Unknown keyword arg %s' % key)
#        if labels is None:
#            labels = find_labels(labeled_tokens)
#
#        probdist = MultinomialNaiveBayesClassifierProbDist(self._fd_list,
#                                                   labeled_tokens)
#        return NaiveBayesClassifier(self._fd_list, labels, probdist)
#
#    def __repr__(self):
#        return '<NaiveBayesClassifierTrainer: %d features>' % len(self._fd_list)

##//////////////////////////////////////////////////////
##  Test code
##//////////////////////////////////////////////////////

from nltk.feature import *
from nltk.feature.word import *

def demo():
    import nltk.corpus

    # Load the training data, and split it into test & train.
    print 'reading data...'
    toks = []
    for item in nltk.corpus.brown.items()[:30]:
        text = nltk.corpus.brown.tokenize(item, addcontexts=True)
        toks += text['SUBTOKENS']
    toks = toks
    split = len(toks)-20
    train, test = toks[:split], toks[split:]

    # We're using TAG as our CLASS
    for tok in toks:
        cls = tok['TAG']
        if '-' in cls and cls != '--': cls = cls.split('-')[0]
        if '+' in cls: cls = cls.split('+')[0]
        tok['CLASS'] = cls

    # Create the feature detector.
    detector = MergedFeatureDetector(
        TextFeatureDetector(),                 # word's text
        ContextWordFeatureDetector(offset=-1), # previous word's text
        ContextWordFeatureDetector(offset=1),  # next word's text
        )

    # Run feature detection on the training data.
    print 'feature detection...'
    for tok in train: detector.detect_features(tok)

    # Train a new classifier
    print 'training...'
    classifier = NaiveBayesClassifierTrainer().train(train)

    # Use it to classify the test words.
    print 'classifying...'
    print
    print 'correct? |     token     | cls | class distribution'
    print '---------+---------------+-----+-------------------------------------'
    for tok in test:
        s = '%22s' % tok.exclude('CONTEXT', 'CLASS')
        c = tok['CLASS']
        detector.detect_features(tok)
        classifier.classify(tok)
        if c == tok['CLASS']: s = '   '+s
        else: s = '[X]' + s
        s += ' %-4s  ' % tok['CLASS']
        pdist = tok['CLASS_PROBS']
        probs = [(pdist.prob(val),val) for val in pdist.samples()]

        probs.sort(); probs.reverse()
        for prob,val in probs[:3]:
            s += '%5s=%.3f' % (val,prob)
        print s + ' ...'

if __name__ == '__main__': demo()
<commit_msg>- Removed unused import<commit_after># Natural Language Toolkit: Naive Bayes Classifiers
#
# Copyright (C) 2001 University of Pennsylvania
# Author: Edward Loper <edloper@gradient.cis.upenn.edu>
# URL: <http://nltk.sf.net>
# For license information, see LICENSE.TXT
#
# $Id$

"""

                     P(class) * P(features|class)
P(class|features) = ------------------------------
                           P(features)

"""

from nltk.classifier import *
from nltk.feature import *
from nltk.probability import *
from nltk.token import Token
from nltk.corpus import brown
from nltk.chktype import chktype as _chktype
import types

##//////////////////////////////////////////////////////
##  Naive Bayes Classifier
##//////////////////////////////////////////////////////

class NaiveBayesClassifier(ClassifierI, PropertyIndirectionMixIn):
    def __init__(self, class_probdist, feature_probdist, **property_names):
        """
        @param class_probdist: P(cls)
        @param feature_probdist: P(fval|cls,fid)
        """
        PropertyIndirectionMixIn.__init__(self, **property_names)
        self._class_probdist = class_probdist
        self._feature_probdist = feature_probdist
        self._classes = class_probdist.samples()

    def classes(self):
        return self._classes

    def classify(self, token):
        # Get the token's features.
        FEATURES = self.property('FEATURES')
        CLASS_PROBS = self.property('CLASS_PROBS')
        CLASS = self.property('CLASS')

        # Find the log probabilty of each class, given the features.
        logprob_dict = {}
        for cls in self._classes:
            logprob_dict[cls] = self._class_probdist.logprob(cls)
        for (cls, fname) in self._feature_probdist.conditions():
            probdist = self._feature_probdist[cls, fname]
            if fname not in token[FEATURES]: continue # [XX] is this right?
            fval = token[FEATURES][fname]
            logprob_dict[cls] += probdist.logprob(token[FEATURES][fname])

        probs = DictionaryProbDist(logprob_dict, normalize=True, log=True)
        token[CLASS_PROBS] = probs
        token[CLASS] = probs.max()

##//////////////////////////////////////////////////////
##  Naive Bayes Classifier Trainer
##//////////////////////////////////////////////////////

class NaiveBayesClassifierTrainer(ClassifierTrainerI,
                                  PropertyIndirectionMixIn):

    def train(self, tokens):
        FEATURES = self.property('FEATURES')
        CLASS = self.property('CLASS')

        class_freqdist = FreqDist()
        feature_freqdist = ConditionalFreqDist()
        feature_values = {}

        for token in tokens:
            cls = token[CLASS]
            class_freqdist.inc(cls)
            for fname, fval in token[FEATURES].items():
                # Recod the value in the freq(fval|cls, fname) distribution.
                feature_freqdist[cls, fname].inc(fval)
                # Record that fname can take the value fval.
                if fname not in feature_values: feature_values[fname] = Set()
                feature_values[fname].add(fval)

        # Create the P(cls) distribution
        class_probdist = ELEProbDist(class_freqdist)

        # Create the P(fval|cls, fname) distribution
        def make_probdist(freqdist, (cls, fname)):
            return ELEProbDist(freqdist, bins=len(feature_values[fname]))
        feature_probdist = ConditionalProbDist(feature_freqdist, make_probdist, True)

        # [xx] property indirection?
        return NaiveBayesClassifier(class_probdist, feature_probdist)

##//////////////////////////////////////////////////////
##  Multinomial play
##//////////////////////////////////////////////////////

# Features can fire multiple times.  Probability that a feature fires
# n times is p^n, where p is the probability that it fires once.  So
# we already know E(u)/n.  That's the freq dist we made.  Well,
# kinda...  Not really.  But to do that, we just need to inc by val,
# not by 1, in NaiveBayesClassifierFreqDist.

# So then we have E(fid)/n.  That is, p(fid).  So when we're asked to
# estimate P(id=val), take P(id=1)^val...

# Doing this to the freq-dist streight is playing a little fast &
# loose, though..  But doing it in a wrapper pdf seems fine.
# Hm.. still might be tricky, though.  I'll have to think about it.

#from math import exp
#class _MultinomialNaiveBayesClassifierProbDist(ProbDistI):
#    """
#    This class is not currently functional.
#    """
#    def __init__(self, fd_list, labeled_tokens, l=0.5):
#        fcounts = zeros(len(fd_list), 'd')
#        self._N = 0
#        fcounts += l
#        self._N += l*len(fcounts)
#
#        for tok in labeled_tokens:
#            self._N += 1
#            for (fid, val) in fd_list.detect(tok.type()).assignments():
#                fcounts[fid] += val
#
#        # Expected value for each feature
#        self._exp = fcounts / self._N
#
#    def prob(self, event):
#        # Inherit docs from FreqDistI
#        _chktype('NaiveBayesClassifierFreqDist.freq', 1, event, (AssignmentEvent,))
#        (fid, value) = event.assignment()
#
#        # valfact = value!
#        valfact = reduce( lambda x,y:x*y, range(2,value+1), 1.0 )
#
#        multinomial = ((self._exp[fid] ** value) / valfact)
#        poisson = ((self._exp[fid] ** value) * exp(-self._exp[fid]) /
#                   valfact)
#        return poisson
#
#class _MultinomialNaiveBayesClassifierTrainer(ClassifierTrainerI):
#    """
#    This class is not currently functional.
#    """
#    def __init__(self, fd_list):
#        self._fd_list = fd_list
#
#    def train(self, labeled_tokens, **kwargs):
#        labels = None
#        for (key, val) in kwargs.items():
#            if key == 'labels': labels = val
#            else: raise TypeError('Unknown keyword arg %s' % key)
#        if labels is None:
#            labels = find_labels(labeled_tokens)
#
#        probdist = MultinomialNaiveBayesClassifierProbDist(self._fd_list,
#                                                   labeled_tokens)
#        return NaiveBayesClassifier(self._fd_list, labels, probdist)
#
#    def __repr__(self):
#        return '<NaiveBayesClassifierTrainer: %d features>' % len(self._fd_list)

##//////////////////////////////////////////////////////
##  Test code
##//////////////////////////////////////////////////////

from nltk.feature import *
from nltk.feature.word import *

def demo():
    import nltk.corpus

    # Load the training data, and split it into test & train.
    print 'reading data...'
    toks = []
    for item in nltk.corpus.brown.items()[:30]:
        text = nltk.corpus.brown.tokenize(item, addcontexts=True)
        toks += text['SUBTOKENS']
    toks = toks
    split = len(toks)-20
    train, test = toks[:split], toks[split:]

    # We're using TAG as our CLASS
    for tok in toks:
        cls = tok['TAG']
        if '-' in cls and cls != '--': cls = cls.split('-')[0]
        if '+' in cls: cls = cls.split('+')[0]
        tok['CLASS'] = cls

    # Create the feature detector.
    detector = MergedFeatureDetector(
        TextFeatureDetector(),                 # word's text
        ContextWordFeatureDetector(offset=-1), # previous word's text
        ContextWordFeatureDetector(offset=1),  # next word's text
        )

    # Run feature detection on the training data.
    print 'feature detection...'
    for tok in train: detector.detect_features(tok)

    # Train a new classifier
    print 'training...'
    classifier = NaiveBayesClassifierTrainer().train(train)

    # Use it to classify the test words.
    print 'classifying...'
    print
    print 'correct? |     token     | cls | class distribution'
    print '---------+---------------+-----+-------------------------------------'
    for tok in test:
        s = '%22s' % tok.exclude('CONTEXT', 'CLASS')
        c = tok['CLASS']
        detector.detect_features(tok)
        classifier.classify(tok)
        if c == tok['CLASS']: s = '   '+s
        else: s = '[X]' + s
        s += ' %-4s  ' % tok['CLASS']
        pdist = tok['CLASS_PROBS']
        probs = [(pdist.prob(val),val) for val in pdist.samples()]

        probs.sort(); probs.reverse()
        for prob,val in probs[:3]:
            s += '%5s=%.3f' % (val,prob)
        print s + ' ...'

if __name__ == '__main__': demo()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
                AllSky340

Base class and utilities for operating an SBIG AllSky340 allsky camera.
Tested and used with the monochrome model only.  Color version will require
some modifications. The camera is based on the SG-4 autoguider so this could
easily be adapted to that as well.  The information used here was gleaned from:

http://sbig.impulse.net/pdffiles/SBIG%20SG-4%20&%20AllSky-340(C)%20Serial%20Interface%20Specification.pdf

Currently only supporting 1x1 binning and either full (640x480) or cropped
(512x480).

Author                     Version             Date
--------------------------------------------------------
TE Pickering                 0.1             20130114

TODO
--------------------------------------------------------
- add support for SG-4 and color version.  split into basic class for SG-4
  that the AllSky340 classes can inherit from.

- figure out how to do simultaneous expose and readout...

Updates
--------------------------------------------------------

"""

import sys
import os
import serial
import struct
import pyfits
import time
import numpy as np
import logging

def emit_colored_ansi(fn):
    def new(*args):
        levelno = args[1].levelno
        if(levelno >= 50):
            color = '\x1b[31m'  # red
        elif(levelno >= 40):
            color = '\x1b[31m'  # red
        elif(levelno >= 30):
            color = '\x1b[33m'  # yellow
        elif(levelno >= 20):
            color = '\x1b[32m'  # green
        elif(levelno >= 10):
            color = '\x1b[35m'  # pink
        else:
            color = '\x1b[0m'  # normal
        args[1].levelname = color + args[1].levelname + '\x1b[0m'  # normal
        return fn(*args)
    return new

# Initialize logger
logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO)
cam_log = logging.getLogger()
fh = logging.FileHandler("skycam.log")
fh.setLevel(logging.DEBUG)
fh.setFormatter(logging.Formatter("%(asctime)s: %(levelname)s - %(message)s"))
cam_log.addHandler(fh)
ch = logging.StreamHandler()
f = logging.Formatter("%(levelname)s: %(message)s")
ch.setFormatter(f)
logging.StreamHandler.emit = emit_colored_ansi(logging.StreamHandler.emit)

class AllSky340:
    def __init__(self, port="/dev/tty.usbserial",
                 baudrate=460800, timeout=1):
        self.ser = serial.Serial()
        self.ser.port = port
        self.ser.baudrate = baudrate
        self.ser.timeout = timeout
        self.ser.open()
        cam_log.info("Camera opened on port %s." % port)

    def log_info(self, msg):
        cam_log.info(msg)

    def log_warn(self, msg):
        cam_log.warn(msg)

    def log_err(self, msg):
        cam_log.error(msg)

    def get_baudrate(self):
        return self.ser.baudrate

    def checksum(self, c):
        inv = ~ord(c[0]) & 0xFF
        mask = ~(1 << 7)
        checksum = inv & mask
        if len(c) > 1:
            for i in range(1, len(c)):
                inv = ~ord(c[i]) & 0xFF
                xor = checksum ^ inv
                checksum = xor & mask
        return checksum

    def command(self, cmd, nbytes):
        to_send = cmd + struct.pack("B", self.checksum(cmd))
        self.ser.write(to_send)
        if nbytes > 0:
            return self.ser.read(nbytes)
        else:
            return True

    def ping(self):
        resp = self.command("E", 2)
        cam_log.info("Pinged camera and received response of %s" % resp)
        if resp[1] == "O":
            cam_log.info("Camera on-line and communicating properly.")
        else:
            cam_log.warn("Comm problem. Did not receive correct response from camera.")
        return resp[1]

    def open_shutter(self):
        cam_log.info("Opening camera shutter")
        return self.command("O", 0)

    def close_shutter(self):
        cam_log.info("Closing camera shutter")
        return self.command("C", 0)

    def shutter_off(self):
        cam_log.info("De-energizing camera shutter")
        return self.command("K", 0)

    def abort_image(self):
        cam_log.info("Aborting current exposure")
        return self.command("A", 0)

    def firmware(self):
        resp = self.command("V", 2)
        minor = ord(resp[1])
        major = ord(resp[0])
        if major >= 8:
            ver = "T"
            major -= 8
        else:
            ver = "V"
        version = "%s%i.%i" % (ver, major, minor)
        cam_log.info("Camera firmware: %s (raw) %s (formatted)" %
                     (resp, version))
        return version

    def set_baudrate(self, baud):
        old_baud = self.ser.baudrate
        cam_log.info("Setting camera baudrate to %d" % baud)
        cmds = {9600:   "B0",
                19200:  "B1",
                38400:  "B2",
                57600:  "B3",
                115200: "B4",
                230400: "B5",
                460800: "B6"}
        if baud not in cmds.keys():
            cam_log.error("NOT A VALID BAUDRATE!")
            return False
        else:
            cmd = cmds[baud]
            self.command(cmd, 0)
            self.ser.baudrate = baud
            resp = self.ser.read(2)
            cam_log.info("Baudrate set, got response of %s" % resp)
            resp = self.command("Test", 7)
            cam_log.info("Tested new baudrate and got response of %s" % resp)
            if resp[:-1] == "TestOk":
                cam_log.info("Camera accepted new baudrate.")
                self.command("k", 0)
                return True
            else:
                cam_log.warn("Camera did not accept new baudrate. Resetting to previous rate.")
                self.ser.baudrate = old_baud
                return False

    def heater_on(self):
        """
        per communication with SBIG, the heater is wired to the X+ guide relay
        on the camera.  use the 'force guide relays' command to toggle it
        """
        cmd = 'g' + struct.pack("B", 1)
        cam_log.info("Turning camera heater on")
        self.command(cmd, 0)
        return True

    def heater_off(self):
        cmd = 'g' + struct.pack("B", 0)
        cam_log.info("Turning camera heater off")
        self.command(cmd, 0)
        return True

    def shutter_chop_on(self):
        """
        this is an undocumented command that was provided by SBIG via
        email.  it's simply U plus 0x01 (on) or 0x00 (off).
        """
        cmd = 'U' + struct.pack("B", 1)
        cam_log.info("Enabling chopping shutter mode.")
        self.command(cmd, 0)
        return True

    def shutter_chop_off(self):
        cmd = 'U' + struct.pack("B", 0)
        cam_log.info("Disabling chopping shutter mode.")
        self.command(cmd, 0)
        return True

    def block_read(self, npix, ntries=0):
        """
        cheeky recursive function to read block of pixels and ask for repeat
        transmissions if checksums don't match
        """
        max_tries = 5
        block = self.ser.read(npix * 2)
        lrc_byte = self.ser.read(1)
        if len(lrc_byte) > 0:
            cam_lrc = ord(lrc_byte)
        else:
            cam_lrc = 0
        lrc = 0
        for b in block:
            lrc ^= ord(b)
        if lrc == cam_lrc:
            return block
        else:
            if ntries < max_tries:
                cam_log.warn("Camera read-out error. Re-transmitting block (try #%d)..."
                             % ntries)
                self.command('R', 0)
                # not sure why this is needed. not mentioned in document...
                f = self.ser.read(1)
                return self.block_read(npix, ntries=ntries+1)
            else:
                cam_log.error("Camera read_out failed. Stopping transfer.")
                self.command('S', 0)
                return False

    def getImage(self, exptime, light=False, cropped=False):
        eres = 1.0e-4
        e = "%06x" % int(exptime / eres)
        if light:
            e = e + "00" + "01"
        else:
            e = e + "00" + "00"

        etime = e.decode("hex")
        command = "T" + etime
        self.command(command, 0)

        if cropped:
            crop = "cropped"
        else:
            crop = "full"

        if light:
            imtype = "light"
        else:
            imtype = "dark"

        cam_log.info("Exposing %s %s image for %f seconds...." %
                     (crop, imtype, exptime))

        out = "E"
        s = ""
        while out != "D":
            out = self.ser.read(1)

        cam_log.info("Transferring image from camera....")
        self.command('X', 0)
        imag_str = bytearray()

        # not sure why this is needed. not mentioned in document...
        f = self.ser.read(1)

        # we only use 1x1 binning and either full or cropped
        if cropped:
            nbytes = 245760
        else:
            nbytes = 307200

        # this is true for 1x1 binning.  it would be 1024 for 2x2 binning
        # or the length of one line in sub-frame mode.
        npix = 4096
        for i in range(nbytes / npix):
            block_str = self.block_read(npix)
            # if the read completely fails, just replace with zeros
            if block_str == False:
                l = npix*[0]
                block_str = struct.pack('B' * len(l), *l)
            imag_str.extend(block_str)
            self.command('K', 0)

        # pull these extra bytes out.  also not sure why...
        cam = self.ser.read(1)
        cam = self.ser.read(1)
        cam = self.ser.read(1)
        cam = self.ser.read(1)
        cam = self.ser.read(1)

        # this is ugly and there has to be a better way, but it works.
        earr = [z for z in imag_str[::2]]
        oarr = [z for z in imag_str[1::2]]
        arr = []

        for i in range(len(earr)):
            s = struct.pack("BB", earr[i], oarr[i])
            n = struct.unpack("H", s)[0]
            arr.append(n)

        if cropped:
            imag = np.array(arr, dtype=np.int32).reshape(480, 512)
        else:
            imag = np.array(arr, dtype=np.int32).reshape(480, 640)
        return np.flipud(imag)

if __name__ == '__main__':
    c = AllSky340()
    method = sys.argv[1].lower()
    args = ", ".join(sys.argv[2:])
    eval("c.%s(%s)" % (method, args))
<commit_msg>use ping to return true or false if camera is there or not<commit_after>#!/usr/bin/env python
"""
                AllSky340

Base class and utilities for operating an SBIG AllSky340 allsky camera.
Tested and used with the monochrome model only.  Color version will require
some modifications. The camera is based on the SG-4 autoguider so this could
easily be adapted to that as well.  The information used here was gleaned from:

http://sbig.impulse.net/pdffiles/SBIG%20SG-4%20&%20AllSky-340(C)%20Serial%20Interface%20Specification.pdf

Currently only supporting 1x1 binning and either full (640x480) or cropped
(512x480).

Author                     Version             Date
--------------------------------------------------------
TE Pickering                 0.1             20130114

TODO
--------------------------------------------------------
- add support for SG-4 and color version.  split into basic class for SG-4
  that the AllSky340 classes can inherit from.

- figure out how to do simultaneous expose and readout...

Updates
--------------------------------------------------------

"""

import sys
import os
import serial
import struct
import pyfits
import time
import numpy as np
import logging

def emit_colored_ansi(fn):
    def new(*args):
        levelno = args[1].levelno
        if(levelno >= 50):
            color = '\x1b[31m'  # red
        elif(levelno >= 40):
            color = '\x1b[31m'  # red
        elif(levelno >= 30):
            color = '\x1b[33m'  # yellow
        elif(levelno >= 20):
            color = '\x1b[32m'  # green
        elif(levelno >= 10):
            color = '\x1b[35m'  # pink
        else:
            color = '\x1b[0m'  # normal
        args[1].levelname = color + args[1].levelname + '\x1b[0m'  # normal
        return fn(*args)
    return new

# Initialize logger
logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO)
cam_log = logging.getLogger()
fh = logging.FileHandler("skycam.log")
fh.setLevel(logging.DEBUG)
fh.setFormatter(logging.Formatter("%(asctime)s: %(levelname)s - %(message)s"))
cam_log.addHandler(fh)
ch = logging.StreamHandler()
f = logging.Formatter("%(levelname)s: %(message)s")
ch.setFormatter(f)
logging.StreamHandler.emit = emit_colored_ansi(logging.StreamHandler.emit)

class AllSky340:
    def __init__(self, port="/dev/tty.usbserial",
                 baudrate=460800, timeout=1):
        self.ser = serial.Serial()
        self.ser.port = port
        self.ser.baudrate = baudrate
        self.ser.timeout = timeout
        self.ser.open()
        cam_log.info("Camera opened on port %s." % port)

    def log_info(self, msg):
        cam_log.info(msg)

    def log_warn(self, msg):
        cam_log.warn(msg)

    def log_err(self, msg):
        cam_log.error(msg)

    def get_baudrate(self):
        return self.ser.baudrate

    def checksum(self, c):
        inv = ~ord(c[0]) & 0xFF
        mask = ~(1 << 7)
        checksum = inv & mask
        if len(c) > 1:
            for i in range(1, len(c)):
                inv = ~ord(c[i]) & 0xFF
                xor = checksum ^ inv
                checksum = xor & mask
        return checksum

    def command(self, cmd, nbytes):
        to_send = cmd + struct.pack("B", self.checksum(cmd))
        self.ser.write(to_send)
        if nbytes > 0:
            return self.ser.read(nbytes)
        else:
            return True

    def ping(self):
        resp = self.command("E", 2)
        cam_log.info("Pinged camera and received response of %s" % resp)
        if resp[1] == "O":
            cam_log.info("Camera on-line and communicating properly.")
            return True
        else:
            cam_log.warn("Comm problem. Did not receive correct response from camera.")
            return False

    def open_shutter(self):
        cam_log.info("Opening camera shutter")
        return self.command("O", 0)

    def close_shutter(self):
        cam_log.info("Closing camera shutter")
        return self.command("C", 0)

    def shutter_off(self):
        cam_log.info("De-energizing camera shutter")
        return self.command("K", 0)

    def abort_image(self):
        cam_log.info("Aborting current exposure")
        return self.command("A", 0)

    def firmware(self):
        resp = self.command("V", 2)
        minor = ord(resp[1])
        major = ord(resp[0])
        if major >= 8:
            ver = "T"
            major -= 8
        else:
            ver = "V"
        version = "%s%i.%i" % (ver, major, minor)
        cam_log.info("Camera firmware: %s (raw) %s (formatted)" %
                     (resp, version))
        return version

    def set_baudrate(self, baud):
        old_baud = self.ser.baudrate
        cam_log.info("Setting camera baudrate to %d" % baud)
        cmds = {9600:   "B0",
                19200:  "B1",
                38400:  "B2",
                57600:  "B3",
                115200: "B4",
                230400: "B5",
                460800: "B6"}
        if baud not in cmds.keys():
            cam_log.error("NOT A VALID BAUDRATE!")
            return False
        else:
            cmd = cmds[baud]
            self.command(cmd, 0)
            self.ser.baudrate = baud
            resp = self.ser.read(2)
            cam_log.info("Baudrate set, got response of %s" % resp)
            resp = self.command("Test", 7)
            cam_log.info("Tested new baudrate and got response of %s" % resp)
            if resp[:-1] == "TestOk":
                cam_log.info("Camera accepted new baudrate.")
                self.command("k", 0)
                return True
            else:
                cam_log.warn("Camera did not accept new baudrate. Resetting to previous rate.")
                self.ser.baudrate = old_baud
                return False

    def heater_on(self):
        """
        per communication with SBIG, the heater is wired to the X+ guide relay
        on the camera.  use the 'force guide relays' command to toggle it
        """
        cmd = 'g' + struct.pack("B", 1)
        cam_log.info("Turning camera heater on")
        self.command(cmd, 0)
        return True

    def heater_off(self):
        cmd = 'g' + struct.pack("B", 0)
        cam_log.info("Turning camera heater off")
        self.command(cmd, 0)
        return True

    def shutter_chop_on(self):
        """
        this is an undocumented command that was provided by SBIG via
        email.  it's simply U plus 0x01 (on) or 0x00 (off).
        """
        cmd = 'U' + struct.pack("B", 1)
        cam_log.info("Enabling chopping shutter mode.")
        self.command(cmd, 0)
        return True

    def shutter_chop_off(self):
        cmd = 'U' + struct.pack("B", 0)
        cam_log.info("Disabling chopping shutter mode.")
        self.command(cmd, 0)
        return True

    def block_read(self, npix, ntries=0):
        """
        cheeky recursive function to read block of pixels and ask for repeat
        transmissions if checksums don't match
        """
        max_tries = 5
        block = self.ser.read(npix * 2)
        lrc_byte = self.ser.read(1)
        if len(lrc_byte) > 0:
            cam_lrc = ord(lrc_byte)
        else:
            cam_lrc = 0
        lrc = 0
        for b in block:
            lrc ^= ord(b)
        if lrc == cam_lrc:
            return block
        else:
            if ntries < max_tries:
                cam_log.warn("Camera read-out error. Re-transmitting block (try #%d)..."
                             % ntries)
                self.command('R', 0)
                # not sure why this is needed. not mentioned in document...
                f = self.ser.read(1)
                return self.block_read(npix, ntries=ntries+1)
            else:
                cam_log.error("Camera read_out failed. Stopping transfer.")
                self.command('S', 0)
                return False

    def getImage(self, exptime, light=False, cropped=False):
        eres = 1.0e-4
        e = "%06x" % int(exptime / eres)
        if light:
            e = e + "00" + "01"
        else:
            e = e + "00" + "00"

        etime = e.decode("hex")
        command = "T" + etime
        self.command(command, 0)

        if cropped:
            crop = "cropped"
        else:
            crop = "full"

        if light:
            imtype = "light"
        else:
            imtype = "dark"

        cam_log.info("Exposing %s %s image for %f seconds...." %
                     (crop, imtype, exptime))

        out = "E"
        s = ""
        while out != "D":
            out = self.ser.read(1)

        cam_log.info("Transferring image from camera....")
        self.command('X', 0)
        imag_str = bytearray()

        # not sure why this is needed. not mentioned in document...
        f = self.ser.read(1)

        # we only use 1x1 binning and either full or cropped
        if cropped:
            nbytes = 245760
        else:
            nbytes = 307200

        # this is true for 1x1 binning.  it would be 1024 for 2x2 binning
        # or the length of one line in sub-frame mode.
        npix = 4096
        for i in range(nbytes / npix):
            block_str = self.block_read(npix)
            # if the read completely fails, just replace with zeros
            if block_str == False:
                l = npix*[0]
                block_str = struct.pack('B' * len(l), *l)
            imag_str.extend(block_str)
            self.command('K', 0)

        # pull these extra bytes out.  also not sure why...
        cam = self.ser.read(1)
        cam = self.ser.read(1)
        cam = self.ser.read(1)
        cam = self.ser.read(1)
        cam = self.ser.read(1)

        # this is ugly and there has to be a better way, but it works.
        earr = [z for z in imag_str[::2]]
        oarr = [z for z in imag_str[1::2]]
        arr = []

        for i in range(len(earr)):
            s = struct.pack("BB", earr[i], oarr[i])
            n = struct.unpack("H", s)[0]
            arr.append(n)

        if cropped:
            imag = np.array(arr, dtype=np.int32).reshape(480, 512)
        else:
            imag = np.array(arr, dtype=np.int32).reshape(480, 640)
        return np.flipud(imag)

if __name__ == '__main__':
    c = AllSky340()
    method = sys.argv[1].lower()
    args = ", ".join(sys.argv[2:])
    eval("c.%s(%s)" % (method, args))
<|endoftext|>