<commit_before>import sys
sys.path.append('/home/jwalker/dynamics/python/atmos-tools')
sys.path.append('/home/jwalker/dynamics/python/atmos-read')

import xray
import numpy as np
from datetime import datetime
import matplotlib.pyplot as plt
from matplotlib import animation
import collections
import pandas as pd
import atmos as atm
import precipdat
import merra
import indices
from utils import daily_rel2onset, comp_days_centered, composite

# ----------------------------------------------------------------------
years = range(1979, 2015)
datadir = atm.homedir() + 'datastore/merra/daily/'
savedir = 'mp4/'
onsetfile = datadir + 'merra_vimt_ps-300mb_apr-sep_1979-2014.nc'
ensofile = atm.homedir() + 'dynamics/calc/ENSO/enso_oni.csv'
enso_ssn = 'JJA'

remove_tricky = False
years_tricky = [2002, 2004, 2007, 2009, 2010]

# Lat-lon box for MFC / precip
lon1, lon2 = 60, 100
lat1, lat2 = 10, 30

# ----------------------------------------------------------------------
# Monsoon onset day and index timeseries
onset_nm = 'HOWI'
maxbreak = 10
npts = 100
with xray.open_dataset(onsetfile) as ds:
    uq_int = ds['uq_int'].load()
    vq_int = ds['vq_int'].load()
    howi, _ = indices.onset_HOWI(uq_int, vq_int, npts, maxbreak=maxbreak)
    howi.attrs['title'] = 'HOWI (N=%d)' % npts

# Array of onset days
onset = howi['onset']

# Tile the climatology to each year
tseries_clim = howi['tseries_clim']
vals = atm.biggify(tseries_clim.values, howi['tseries'].values, tile=True)
_, _, coords, dims = atm.meta(howi['tseries'])
tseries_clim = xray.DataArray(vals, name=tseries_clim.name, coords=coords,
                              dims=dims)

# Daily timeseries for each year
tseries = xray.Dataset()
tseries[onset_nm] = howi['tseries']
tseries[onset_nm + '_clim'] = tseries_clim

# ----------------------------------------------------------------------
# MFC over SASM region
print('Calculating MFC')
mfc = atm.moisture_flux_conv(ds['uq_int'], ds['vq_int'], already_int=True)
mfcbar = atm.mean_over_geobox(mfc, lat1, lat2, lon1, lon2)

nroll = 7
mfcbar = atm.rolling_mean(mfcbar, nroll, axis=-1, center=True)
tseries['MFC'] = mfcbar

# ----------------------------------------------------------------------
# ENSO
enso = pd.read_csv(ensofile, index_col=0)
enso = enso[enso_ssn].loc[years]
enso = xray.DataArray(enso).rename({'Year' : 'year'})

# ======================================================================
# PLOTS
# ======================================================================

# ----------------------------------------------------------------------
# Daily timeseries plots together (day of year)
style = {onset_nm : 'k', onset_nm + '_clim' : 'k--', 'MFC' : 'b'}
onset_style = {onset_nm : 'k'}
d_onset = {onset_nm : onset.values}
indices.plot_tseries_together(tseries, onset=d_onset, data_style=style,
                              onset_style=onset_style, show_days=True)

# ----------------------------------------------------------------------
# Daily timeseries composites relative to onset day
keys = [onset_nm, 'MFC']
npre, npost = 30, 90
tseries_rel = xray.Dataset()
for key in keys:
    tseries_rel[key] = daily_rel2onset(tseries[key], onset, npre, npost,
                                       yearnm='year', daynm='day')
dayrel = tseries_rel['dayrel'].values

offset, factor = {}, {}
for key in keys:
    offset[key] = -np.nanmean(tseries[key].values.ravel())
    factor[key] = np.nanstd(tseries[key].values.ravel())

def plot_tseries(dayrel, ind, std, clr, key, xlabel, ylabel):
    plt.plot(dayrel, ind, clr, label=key)
    plt.fill_between(dayrel, ind-std, ind+std, color=clr, alpha=0.2)
    plt.legend(loc='lower right')
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    plt.grid(True)
    plt.autoscale(tight=True)

clrs = {onset_nm : 'b', 'MFC' : 'g'}
plt.figure(figsize=(8, 10))
for i, key in enumerate(keys):
    ind = tseries_rel[key].mean(dim='year')
    std = tseries_rel[key].std(dim='year')

    # Individual timeseries
    plt.subplot(3, 1, i + 1)
    plot_tseries(dayrel, ind, std, clrs[key], key, '', 'Timeseries')
    if i == 0:
        plt.title('1979-2014 Climatological Composites')

    # Standardized timeseries together
    ind = (ind + offset[key]) / factor[key]
    std = std / factor[key]
    plt.subplot(3, 1, 3)
    xlabel = 'Day of year relative to ' + onset_nm + ' onset day'
    ylabel = 'Standardized Timeseries'
    plot_tseries(dayrel, ind, std, clrs[key], key, xlabel, ylabel)

# ----------------------------------------------------------------------
# Composite timeseries

nyrs = 5
comp_ind = {}
comp_ts = {}

# Earliest / latest onset years
onset_sorted = onset.to_series()
onset_sorted.sort()
comp_ind['Early'] = onset_sorted[:nyrs]
comp_ind['Late'] = onset_sorted[-1:-nyrs-1:-1]

# Nino / Nina years
enso_sorted = enso.to_series()
enso_sorted.sort()
comp_ind['Nina'] = enso_sorted[:nyrs]
comp_ind['Nino'] = enso_sorted[-1:-nyrs-1:-1]

comp_yrs = {key : comp_ind[key].index.values for key in comp_ind}

for key in comp_yrs:
    print(key)
    print(comp_ind[key])

# Composite timeseries
for key in comp_yrs:
    comp_ts[key] = atm.subset(tseries[onset_nm], 'year', comp_yrs[key])

# Plot composites and individual years
def ts_subplot(comp_ts, key, onset, enso, clim_ts):
    ts = comp_ts[key]
    days = ts['day'].values
    yrs = ts['year'].values
    for y, year in enumerate(yrs):
        onset_ind = onset.loc[year].values
        enso_ind = enso.loc[year].values
        label = '%d %d %.1f' % (year, onset_ind, enso_ind)
        plt.plot(days, ts[y], label=label)
    label = 'comp %.1f %.1f' % (onset.loc[yrs].mean().values,
                               enso.loc[yrs].mean().values)
    plt.plot(days, ts.mean(dim='year'), linewidth=2, color='k', label=label)
    label = 'clim %.1f %.1f' % (onset.mean().values, enso.mean().values)
    plt.plot(days, clim_ts, 'k--', linewidth=2, label=label)
    plt.xticks(range(75, 275, 25))
    plt.xlim(days.min(), days.max())
    plt.ylim(-1, 2)
    plt.grid(True)
    plt.legend(loc='upper left', fontsize=10)
    plt.title(key)

nrow, ncol = 2, 2
fig, axes = plt.subplots(nrow, ncol, figsize=(14, 10), sharex=True, sharey=True)
plt.subplots_adjust(left=0.08, right=0.97, wspace=0.1, hspace=0.2)
for i, key in enumerate(['Early', 'Nina', 'Late', 'Nino']):
    plt.subplot(nrow, ncol, i + 1)
    ts_subplot(comp_ts, key, onset, enso, tseries[onset_nm + '_clim'][0])
    row, col = atm.subplot_index(nrow, ncol, i + 1)
    if row == nrow:
        plt.xlabel('Day of Year')
    if col == 1:
        plt.ylabel(onset_nm)

# ----------------------------------------------------------------------
# Correlations between onset day and ENSO

df = pd.DataFrame()
df['onset'] = onset.to_series()
df['enso'] = enso.to_series()

def corr_matrix(df, incl_index=False):
    if incl_index:
        df[df.index.name] = df.index

    cols = df.columns
    n = len(cols)
    r = np.nan * np.ones((n, n))
    p = np.nan * np.ones((n, n))

    for i in range(n):
        for j in range(i + 1):
            r[i, j], p[i, j] = scipy.stats.pearsonr(df[cols[i]], df[cols[j]])
            r[j, i], p[j, i] = r[i, j], p[i, j]

    corr = {}
    corr['r'] = pd.DataFrame(r, index=cols, columns=cols)
    corr['p'] = pd.DataFrame(p, index=cols, columns=cols)

    return corr
<commit_msg>Moved corr_matrix() to atmos-tools<commit_after>import sys
sys.path.append('/home/jwalker/dynamics/python/atmos-tools')
sys.path.append('/home/jwalker/dynamics/python/atmos-read')

import xray
import numpy as np
from datetime import datetime
import matplotlib.pyplot as plt
from matplotlib import animation
import collections
import pandas as pd
import atmos as atm
import precipdat
import merra
import indices
from utils import daily_rel2onset, comp_days_centered, composite

# ----------------------------------------------------------------------
years = range(1979, 2015)
datadir = atm.homedir() + 'datastore/merra/daily/'
savedir = 'mp4/'
onsetfile = datadir + 'merra_vimt_ps-300mb_apr-sep_1979-2014.nc'
ensofile = atm.homedir() + 'dynamics/calc/ENSO/enso_oni.csv'
enso_ssn = 'JJA'

remove_tricky = False
years_tricky = [2002, 2004, 2007, 2009, 2010]

# Lat-lon box for MFC / precip
lon1, lon2 = 60, 100
lat1, lat2 = 10, 30

# ----------------------------------------------------------------------
# Monsoon onset day and index timeseries
onset_nm = 'HOWI'
maxbreak = 10
npts = 100
with xray.open_dataset(onsetfile) as ds:
    uq_int = ds['uq_int'].load()
    vq_int = ds['vq_int'].load()
    howi, _ = indices.onset_HOWI(uq_int, vq_int, npts, maxbreak=maxbreak)
    howi.attrs['title'] = 'HOWI (N=%d)' % npts

# Array of onset days
onset = howi['onset']

# Tile the climatology to each year
tseries_clim = howi['tseries_clim']
vals = atm.biggify(tseries_clim.values, howi['tseries'].values, tile=True)
_, _, coords, dims = atm.meta(howi['tseries'])
tseries_clim = xray.DataArray(vals, name=tseries_clim.name, coords=coords,
                              dims=dims)

# Daily timeseries for each year
tseries = xray.Dataset()
tseries[onset_nm] = howi['tseries']
tseries[onset_nm + '_clim'] = tseries_clim

# ----------------------------------------------------------------------
# MFC over SASM region
print('Calculating MFC')
mfc = atm.moisture_flux_conv(ds['uq_int'], ds['vq_int'], already_int=True)
mfcbar = atm.mean_over_geobox(mfc, lat1, lat2, lon1, lon2)

nroll = 7
mfcbar = atm.rolling_mean(mfcbar, nroll, axis=-1, center=True)
tseries['MFC'] = mfcbar

# ----------------------------------------------------------------------
# ENSO
enso = pd.read_csv(ensofile, index_col=0)
enso = enso[enso_ssn].loc[years]
enso = xray.DataArray(enso).rename({'Year' : 'year'})

# ======================================================================
# PLOTS
# ======================================================================

# ----------------------------------------------------------------------
# Daily timeseries plots together (day of year)
style = {onset_nm : 'k', onset_nm + '_clim' : 'k--', 'MFC' : 'b'}
onset_style = {onset_nm : 'k'}
d_onset = {onset_nm : onset.values}
indices.plot_tseries_together(tseries, onset=d_onset, data_style=style,
                              onset_style=onset_style, show_days=True)

# ----------------------------------------------------------------------
# Daily timeseries composites relative to onset day
keys = [onset_nm, 'MFC']
npre, npost = 30, 90
tseries_rel = xray.Dataset()
for key in keys:
    tseries_rel[key] = daily_rel2onset(tseries[key], onset, npre, npost,
                                       yearnm='year', daynm='day')
dayrel = tseries_rel['dayrel'].values

offset, factor = {}, {}
for key in keys:
    offset[key] = -np.nanmean(tseries[key].values.ravel())
    factor[key] = np.nanstd(tseries[key].values.ravel())

def plot_tseries(dayrel, ind, std, clr, key, xlabel, ylabel):
    plt.plot(dayrel, ind, clr, label=key)
    plt.fill_between(dayrel, ind-std, ind+std, color=clr, alpha=0.2)
    plt.legend(loc='lower right')
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    plt.grid(True)
    plt.autoscale(tight=True)

clrs = {onset_nm : 'b', 'MFC' : 'g'}
plt.figure(figsize=(8, 10))
for i, key in enumerate(keys):
    ind = tseries_rel[key].mean(dim='year')
    std = tseries_rel[key].std(dim='year')

    # Individual timeseries
    plt.subplot(3, 1, i + 1)
    plot_tseries(dayrel, ind, std, clrs[key], key, '', 'Timeseries')
    if i == 0:
        plt.title('1979-2014 Climatological Composites')

    # Standardized timeseries together
    ind = (ind + offset[key]) / factor[key]
    std = std / factor[key]
    plt.subplot(3, 1, 3)
    xlabel = 'Day of year relative to ' + onset_nm + ' onset day'
    ylabel = 'Standardized Timeseries'
    plot_tseries(dayrel, ind, std, clrs[key], key, xlabel, ylabel)

# ----------------------------------------------------------------------
# Composite timeseries

nyrs = 5
comp_ind = {}
comp_ts = {}

# Earliest / latest onset years
onset_sorted = onset.to_series()
onset_sorted.sort()
comp_ind['Early'] = onset_sorted[:nyrs]
comp_ind['Late'] = onset_sorted[-1:-nyrs-1:-1]

# Nino / Nina years
enso_sorted = enso.to_series()
enso_sorted.sort()
comp_ind['Nina'] = enso_sorted[:nyrs]
comp_ind['Nino'] = enso_sorted[-1:-nyrs-1:-1]

comp_yrs = {key : comp_ind[key].index.values for key in comp_ind}

for key in comp_yrs:
    print(key)
    print(comp_ind[key])

# Composite timeseries
for key in comp_yrs:
    comp_ts[key] = atm.subset(tseries[onset_nm], 'year', comp_yrs[key])

# Plot composites and individual years
def ts_subplot(comp_ts, key, onset, enso, clim_ts):
    ts = comp_ts[key]
    days = ts['day'].values
    yrs = ts['year'].values
    for y, year in enumerate(yrs):
        onset_ind = onset.loc[year].values
        enso_ind = enso.loc[year].values
        label = '%d %d %.1f' % (year, onset_ind, enso_ind)
        plt.plot(days, ts[y], label=label)
    label = 'comp %.1f %.1f' % (onset.loc[yrs].mean().values,
                               enso.loc[yrs].mean().values)
    plt.plot(days, ts.mean(dim='year'), linewidth=2, color='k', label=label)
    label = 'clim %.1f %.1f' % (onset.mean().values, enso.mean().values)
    plt.plot(days, clim_ts, 'k--', linewidth=2, label=label)
    plt.xticks(range(75, 275, 25))
    plt.xlim(days.min(), days.max())
    plt.ylim(-1, 2)
    plt.grid(True)
    plt.legend(loc='upper left', fontsize=10)
    plt.title(key)

nrow, ncol = 2, 2
fig, axes = plt.subplots(nrow, ncol, figsize=(14, 10), sharex=True, sharey=True)
plt.subplots_adjust(left=0.08, right=0.97, wspace=0.1, hspace=0.2)
for i, key in enumerate(['Early', 'Nina', 'Late', 'Nino']):
    plt.subplot(nrow, ncol, i + 1)
    ts_subplot(comp_ts, key, onset, enso, tseries[onset_nm + '_clim'][0])
    row, col = atm.subplot_index(nrow, ncol, i + 1)
    if row == nrow:
        plt.xlabel('Day of Year')
    if col == 1:
        plt.ylabel(onset_nm)

# ----------------------------------------------------------------------
# Correlations between onset day and ENSO

df = pd.DataFrame()
df['onset'] = onset.to_series()
df['enso'] = enso.to_series()
<|endoftext|>


<commit_before>#!/usr/bin/env python
import roslib
roslib.load_manifest('ledcalibrator')
import sys
import rospy
import cv2
from std_msgs.msg import String
from sensor_msgs.msg import Image
from cv_bridge import CvBridge, CvBridgeError

import numpy as np

sys.path.append('../../pyledstrip')
import pyledstrip

current_ros_image = None
current_image = np.array([])

image_pub = rospy.Publisher("image_topic_output",Image)
image_pub2 = rospy.Publisher("image_topic_bg_image",Image)
image_pub3 = rospy.Publisher("image_topic_fg_image",Image)
bridge = CvBridge()

strip_pixel_coordinates = None

DISTANCE_BETWEEN_TWO_LEDS = 100/60

class image_converter:

  def __init__(self):

    self.bridge = CvBridge()
    self.image_sub = rospy.Subscriber("/usb_cam/image_raw",Image,self.callback)

  def callback(self,data):
    global current_image, current_ros_image

    try:
      cv_image = self.bridge.imgmsg_to_cv2(data, "bgr8")
    except CvBridgeError as e:
      print(e)

    # save in buffer
    current_ros_image = data
    current_image = cv_image

def run_calibration_sweep(total_ids = 300, strip_resolution = 5, accumulated_frames = 3):
  '''runs one sweep on the lightstrip and returns the detected coordinates'''
  global current_image, bridge, image_pub, image_pub2, image_pub3

  rospy.loginfo("running calibration sweep ...")

  coordinates = {}

  if current_image.size == 0:
    rospy.logwarn("no image in the buffer. aborting.")
    return

  for i in range(0, total_ids, strip_resolution):

    # capture background and foreground image

    #setAllOff()
    pyledstrip.clear()
    pyledstrip.transmit()

    rospy.sleep(.5)
    bg_image = current_image.astype(int)

    #setLEDXOn()
    pyledstrip.set_rgb(i,255,255,255)
    pyledstrip.transmit()

    ACCUMULATE_FRAMES = accumulated_frames

    fg_image = np.array([])

    rospy.sleep(.5)
    fg_image = current_image.astype(int)

    for j in range(1,ACCUMULATE_FRAMES):
      rospy.sleep(.5)
      fg_image = fg_image + current_image.astype(int)

    fg_image = fg_image / ACCUMULATE_FRAMES

    # second background image
    pyledstrip.clear()
    pyledstrip.transmit()

    rospy.sleep(.5)
    bg_image2 = current_image.astype(int)

    # calculate diff image

    diff = np.maximum(fg_image - bg_image - bg_image2, 0)

    diff2 = diff.astype(np.uint8)

    gray_image = cv2.cvtColor(diff2, cv2.COLOR_BGR2GRAY)

    output = cv2.cvtColor(gray_image, cv2.COLOR_GRAY2BGR)

    # find maximum location

    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(gray_image)

    # report on console and image

    coordinates[i] = max_loc

    print("{0}: {2}, ".format(i, max_loc[0], max_loc[1]))
    rospy.loginfo("%d: %d, ", i, max_loc[1])

    cv2.circle(output, max_loc, 10, 255);

    try:
      pass
      image_pub.publish(bridge.cv2_to_imgmsg(output, "bgr8"))
      image_pub2.publish(bridge.cv2_to_imgmsg(bg_image.astype(np.uint8), "bgr8"))
      image_pub3.publish(bridge.cv2_to_imgmsg(fg_image.astype(np.uint8), "bgr8"))
    except CvBridgeError as e:
      print(e)

  return coordinates

def estimate_metric_coordinates(pixel_coordinates):
  '''estimate metric coordinates from pixel coordinates using a distance assumption between two LEDs'''

  rospy.logwarn("estimate_metric_coordinates not implemented yet. returning pixel coordinates.")
  return pixel_coordinates

  '''TODO'''

def verify_coordinates(coordinates):
  '''verifies the coordinates by calculating the min and max distance between points'''
  return False

def visualize_coordinates(coordinates):
  '''visualizes the coordinates using ROS markers'''
  return False

def detect_led_on(led_id, bg_image, image):
  '''detects if led x is on using the led coordinates.
     we currently run the full picture background substraction and assume detection if distance is lower than threshold.
     We should use a region of interest and a light detection at the known position instead'''
  global strip_pixel_coordinates

  led_coordinate = strip_pixel_coordinates[led_id]

  # calculate diff image
  diff = np.maximum(image - bg_image, 0)

  diff2 = diff.astype(np.uint8)

  gray_image = cv2.cvtColor(diff2, cv2.COLOR_BGR2GRAY)

  output = cv2.cvtColor(gray_image, cv2.COLOR_GRAY2BGR)

  # find maximum location

  min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(gray_image)

  distance_to_led = np.linalg.norm(np.array(led_coordinate) - np.array(max_loc))

  rospy.loginfo("Distance to LED: %d", distance_to_led)

  cv2.circle(output, max_loc, 10, 255);

  try:
    pass
    image_pub.publish(bridge.cv2_to_imgmsg(output, "bgr8"))
    image_pub2.publish(bridge.cv2_to_imgmsg(bg_image.astype(np.uint8), "bgr8"))
    image_pub3.publish(bridge.cv2_to_imgmsg(image.astype(np.uint8), "bgr8"))
  except CvBridgeError as e:
    print(e)

  '''dirty: hard-coded threshold'''
  return (distance_to_led < 10)

def estimate_delay(led_id):
  '''estimates the time delay from sending an led activation to the feedback at the camera'''
  global current_ros_image

  rospy.loginfo('estimating delay ...')

  pyledstrip.clear()
  pyledstrip.transmit()

  rospy.sleep(.5)
  bg_image = current_image.astype(int)
  old_seq = current_ros_image.header.seq

  #setLEDXOn()
  pyledstrip.set_rgb(led_id,255,255,255)

  send_time = rospy.get_rostime()
  pyledstrip.transmit()

  rospy.loginfo("Sent led on at %i %i", send_time.secs, send_time.nsecs)

  # hacky image-subscription
  rcv_time = 0
  for i in range(0,100):
    ros_image = current_ros_image
    if ros_image.header.seq == old_seq:
      rospy.sleep(0.01)
      continue

    led_on = detect_led_on(led_id, bg_image, current_image)
    if led_on:
      rcv_time = ros_image.header.stamp
      rospy.loginfo("Rcv led on at %i %i", rcv_time.secs, rcv_time.nsecs)
      break

    old_seq = ros_image.header.seq

  if not led_on:
    rospy.logerr("no LED detected")
    return False

  delay = rcv_time - send_time

  milliseconds_delay = delay.secs * 1000 + delay.nsecs / 1000000.

  rospy.loginfo("Delay is %d ms", milliseconds_delay)

  return delay

def my_callback(event):
  global current_image, bridge, image_pub, image_pub2, image_pub3, strip_pixel_coordinates

  print 'Timer called at ' + str(event.current_real)

  coordinates = run_calibration_sweep(20,5,1)

  strip_pixel_coordinates = coordinates

  metric_coordinates = estimate_metric_coordinates(coordinates)

  print(coordinates)
  print(metric_coordinates)

  estimate_delay(10)

def main(args):
  ic = image_converter()
  rospy.init_node('image_converter', anonymous=True)
  rospy.Timer(rospy.Duration(1), my_callback)
  try:
    rospy.spin()
  except KeyboardInterrupt:
    print("Shutting down")
  cv2.destroyAllWindows()

if __name__ == '__main__':
    main(sys.argv)
<commit_msg>update delay in next calibration sweep<commit_after>#!/usr/bin/env python
import roslib
roslib.load_manifest('ledcalibrator')
import sys
import rospy
import cv2
from std_msgs.msg import String
from sensor_msgs.msg import Image
from cv_bridge import CvBridge, CvBridgeError

import numpy as np

sys.path.append('../../pyledstrip')
import pyledstrip

current_ros_image = None
current_image = np.array([])

image_pub = rospy.Publisher("image_topic_output",Image)
image_pub2 = rospy.Publisher("image_topic_bg_image",Image)
image_pub3 = rospy.Publisher("image_topic_fg_image",Image)
bridge = CvBridge()

strip_pixel_coordinates = None

DISTANCE_BETWEEN_TWO_LEDS = 100/60

delay = 1.0

class image_converter:

  def __init__(self):

    self.bridge = CvBridge()
    self.image_sub = rospy.Subscriber("/usb_cam/image_raw",Image,self.callback)

  def callback(self,data):
    global current_image, current_ros_image

    try:
      cv_image = self.bridge.imgmsg_to_cv2(data, "bgr8")
    except CvBridgeError as e:
      print(e)

    # save in buffer
    current_ros_image = data
    current_image = cv_image

def run_calibration_sweep(total_ids = 300, strip_resolution = 5, accumulated_frames = 3, delay = 0.5):
  '''runs one sweep on the lightstrip and returns the detected coordinates'''
  global current_image, bridge, image_pub, image_pub2, image_pub3

  rospy.loginfo("running calibration sweep ...")

  coordinates = {}

  if current_image.size == 0:
    rospy.logwarn("no image in the buffer. aborting.")
    return

  for i in range(0, total_ids, strip_resolution):

    # capture background and foreground image

    #setAllOff()
    pyledstrip.clear()
    pyledstrip.transmit()

    rospy.sleep(delay)
    bg_image = current_image.astype(int)

    #setLEDXOn()
    pyledstrip.set_rgb(i,255,255,255)
    pyledstrip.transmit()

    ACCUMULATE_FRAMES = accumulated_frames

    fg_image = np.array([])

    rospy.sleep(delay)
    fg_image = current_image.astype(int)

    for j in range(1,ACCUMULATE_FRAMES):
      rospy.sleep(delay)
      fg_image = fg_image + current_image.astype(int)

    fg_image = fg_image / ACCUMULATE_FRAMES

    # second background image
    pyledstrip.clear()
    pyledstrip.transmit()

    rospy.sleep(delay)
    bg_image2 = current_image.astype(int)

    # calculate diff image

    diff = np.maximum(fg_image - bg_image - bg_image2, 0)

    diff2 = diff.astype(np.uint8)

    gray_image = cv2.cvtColor(diff2, cv2.COLOR_BGR2GRAY)

    output = cv2.cvtColor(gray_image, cv2.COLOR_GRAY2BGR)

    # find maximum location

    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(gray_image)

    # report on console and image

    coordinates[i] = max_loc

    print("{0}: {2}, ".format(i, max_loc[0], max_loc[1]))
    rospy.loginfo("%d: %d, ", i, max_loc[1])

    cv2.circle(output, max_loc, 10, 255);

    try:
      pass
      image_pub.publish(bridge.cv2_to_imgmsg(output, "bgr8"))
      image_pub2.publish(bridge.cv2_to_imgmsg(bg_image.astype(np.uint8), "bgr8"))
      image_pub3.publish(bridge.cv2_to_imgmsg(fg_image.astype(np.uint8), "bgr8"))
    except CvBridgeError as e:
      print(e)

  return coordinates

def estimate_metric_coordinates(pixel_coordinates):
  '''estimate metric coordinates from pixel coordinates using a distance assumption between two LEDs'''

  rospy.logwarn("estimate_metric_coordinates not implemented yet. returning pixel coordinates.")
  return pixel_coordinates

  '''TODO'''

def verify_coordinates(coordinates):
  '''verifies the coordinates by calculating the min and max distance between points'''
  return False

def visualize_coordinates(coordinates):
  '''visualizes the coordinates using ROS markers'''
  return False

def detect_led_on(led_id, bg_image, image):
  '''detects if led x is on using the led coordinates.
     we currently run the full picture background substraction and assume detection if distance is lower than threshold.
     We should use a region of interest and a light detection at the known position instead'''
  global strip_pixel_coordinates

  led_coordinate = strip_pixel_coordinates[led_id]

  # calculate diff image
  diff = np.maximum(image - bg_image, 0)

  diff2 = diff.astype(np.uint8)

  gray_image = cv2.cvtColor(diff2, cv2.COLOR_BGR2GRAY)

  output = cv2.cvtColor(gray_image, cv2.COLOR_GRAY2BGR)

  # find maximum location

  min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(gray_image)

  distance_to_led = np.linalg.norm(np.array(led_coordinate) - np.array(max_loc))

  rospy.loginfo("Distance to LED: %d", distance_to_led)

  cv2.circle(output, max_loc, 10, 255);

  try:
    pass
    image_pub.publish(bridge.cv2_to_imgmsg(output, "bgr8"))
    image_pub2.publish(bridge.cv2_to_imgmsg(bg_image.astype(np.uint8), "bgr8"))
    image_pub3.publish(bridge.cv2_to_imgmsg(image.astype(np.uint8), "bgr8"))
  except CvBridgeError as e:
    print(e)

  '''dirty: hard-coded threshold'''
  return (distance_to_led < 10)

def estimate_delay(led_id):
  '''estimates the time delay from sending an led activation to the feedback at the camera'''
  global current_ros_image

  rospy.loginfo('estimating delay ...')

  pyledstrip.clear()
  pyledstrip.transmit()

  rospy.sleep(.5)
  bg_image = current_image.astype(int)
  old_seq = current_ros_image.header.seq

  #setLEDXOn()
  pyledstrip.set_rgb(led_id,255,255,255)

  send_time = rospy.get_rostime()
  pyledstrip.transmit()

  rospy.loginfo("Sent led on at %i %i", send_time.secs, send_time.nsecs)

  # hacky image-subscription
  rcv_time = 0
  for i in range(0,100):
    ros_image = current_ros_image
    if ros_image.header.seq == old_seq:
      rospy.sleep(0.01)
      continue

    led_on = detect_led_on(led_id, bg_image, current_image)
    if led_on:
      rcv_time = ros_image.header.stamp
      rospy.loginfo("Rcv led on at %i %i", rcv_time.secs, rcv_time.nsecs)
      break

    old_seq = ros_image.header.seq

  if not led_on:
    rospy.logerr("no LED detected")
    return False, 0

  delay = rcv_time - send_time

  seconds_delay = float(delay.secs) + float(delay.nsecs) / 1000000000.

  rospy.loginfo("Delay is %f s", seconds_delay)

  return True, seconds_delay

def my_callback(event):
  global current_image, bridge, image_pub, image_pub2, image_pub3, strip_pixel_coordinates, delay

  print 'Timer called at ' + str(event.current_real)

  coordinates = run_calibration_sweep(20,5,1,delay)

  strip_pixel_coordinates = coordinates

  metric_coordinates = estimate_metric_coordinates(coordinates)

  print(coordinates)
  print(metric_coordinates)

  success, new_delay = estimate_delay(10)

  if success: delay = new_delay
  else: delay = delay + 0.1

def main(args):
  ic = image_converter()
  rospy.init_node('image_converter', anonymous=True)
  rospy.Timer(rospy.Duration(1), my_callback)
  try:
    rospy.spin()
  except KeyboardInterrupt:
    print("Shutting down")
  cv2.destroyAllWindows()

if __name__ == '__main__':
    main(sys.argv)
<|endoftext|>


<commit_before>import os
import sys
import logfetch_base
import requests
import time
import threading
from grep import grep_command
from termcolor import colored

TAIL_LOG_FORMAT = '{0}/sandbox/{1}/read'
READ_INTERVAL = 5
THREAD_TIMEOUT = 100000

def start_tail(args):
  if args.requestId:
    sys.stderr.write('Fetching tasks\n')
    tasks = [str(t) for t in logfetch_base.tasks_for_requests(args)]
  else:
    tasks = [args.taskId]
  if args.verbose:
    sys.stderr.write(colored('Tailing logs for tasks:\n', 'green'))
    for t in tasks:
      sys.stderr.write(colored('{0}\n'.format(t), 'yellow'))
  sys.stderr.write(colored('ctrl+c to exit\n', 'cyan'))
  try:
    threads = []
    for task in tasks:
      thread = LogStreamer(args, task)
      threads.append(thread)
      thread.start()
    for t in threads:
      t.join(THREAD_TIMEOUT) #Need a timeout otherwise can't be killed by ctrl+c
      if not t.isAlive:
        break
  except KeyboardInterrupt:
    sys.stderr.write(colored('Stopping tail', 'magenta') + '\n')
    sys.exit(0)

class LogStreamer(threading.Thread):
  def __init__(self, args, task):
    threading.Thread.__init__(self)
    self.daemon = True
    self.Args = args
    self.Task = task

  def run(self):
    self.stream_log_for_task(self.Args, self.Task)

  def stream_log_for_task(self, args, task):
    uri = TAIL_LOG_FORMAT.format(logfetch_base.base_uri(args), task)
    path = '{0}/{1}'.format(task, args.logfile)
    keep_trying = True
    try:
      offset = self.get_initial_offset(uri, path, args)
    except ValueError:
      sys.stderr.write(colored('Could not tail logs for task {0}, check that the task is still active and that the slave it runs on has not been decommissioned\n'.format(task), 'red'))
      keep_trying = False
    while keep_trying:
      try:
        offset = self.fetch_new_log_data(uri, path, offset, args, task)
        time.sleep(5)
      except ValueError:
        sys.stderr.write(colored('Could not tail logs for task {0}, check that the task is still active and that the slave it runs on has not been decommissioned\n'.format(task), 'red'))
        keep_trying = False

  def get_initial_offset(self, uri, path, args):
    params = {"path" : path}
    return long(requests.get(uri, params=params, headers=args.headers).json()['offset'])

  def fetch_new_log_data(self, uri, path, offset, args, task):
    params = {
      "path" : path,
      "offset" : offset
    }
    response = requests.get(uri, params=params, headers=args.headers).json()
    prefix = '({0}) =>\n'.format(task) if args.verbose else ''
    if len(response['data'].encode('utf-8')) > 0:
      if args.grep:
        filename = '{0}/.grep{1}'.format(args.dest, self.Task)
        self.create_grep_file(args, filename, response['data'])
        output = os.popen(grep_command(args, filename)).read()
        sys.stdout.write('{0}{1}'.format(colored(prefix, 'cyan'), output))
        self.remove_grep_file(filename)
      else:
        sys.stdout.write('{0}{1}'.format(colored(prefix, 'cyan'), response['data'].encode('utf-8')))
      return offset + len(response['data'].encode('utf-8'))
    else:
      return offset

  def create_grep_file(self, args, filename, content):
    grep_file = open(filename, 'wb')
    grep_file.write(content.encode('utf-8'))
    grep_file.close()

  def remove_grep_file(self, grep_file):
    if os.path.isfile(grep_file):
      os.remove(grep_file)
<commit_msg>better error when log file is not found<commit_after>import os
import sys
import logfetch_base
import requests
import time
import threading
from grep import grep_command
from termcolor import colored

TAIL_LOG_FORMAT = '{0}/sandbox/{1}/read'
READ_INTERVAL = 5
THREAD_TIMEOUT = 100000

def start_tail(args):
  if args.requestId:
    sys.stderr.write('Fetching tasks\n')
    tasks = [str(t) for t in logfetch_base.tasks_for_requests(args)]
  else:
    tasks = [args.taskId]
  if args.verbose:
    sys.stderr.write(colored('Tailing logs for tasks:\n', 'green'))
    for t in tasks:
      sys.stderr.write(colored('{0}\n'.format(t), 'yellow'))
  sys.stderr.write(colored('ctrl+c to exit\n', 'cyan'))
  try:
    threads = []
    for task in tasks:
      thread = LogStreamer(args, task)
      threads.append(thread)
      thread.start()
    for t in threads:
      t.join(THREAD_TIMEOUT) #Need a timeout otherwise can't be killed by ctrl+c
      if not t.isAlive:
        break
  except KeyboardInterrupt:
    sys.stderr.write(colored('Stopping tail', 'magenta') + '\n')
    sys.exit(0)

class LogStreamer(threading.Thread):
  def __init__(self, args, task):
    threading.Thread.__init__(self)
    self.daemon = True
    self.Args = args
    self.Task = task

  def run(self):
    self.stream_log_for_task(self.Args, self.Task)

  def stream_log_for_task(self, args, task):
    uri = TAIL_LOG_FORMAT.format(logfetch_base.base_uri(args), task)
    path = '{0}/{1}'.format(task, args.logfile)
    keep_trying = True
    try:
      params = {"path" : path}
      logfile_response = requests.get(uri, params=params, headers=args.headers)
      logfile_response.raise_for_status()
      offset = long(logfile_response.json()['offset'])
    except ValueError:
      sys.stderr.write(colored('Could get initial offset for log in task {0}, check that the task is still active and that the slave it runs on has not been decommissioned\n'.format(task), 'red'))
      keep_trying = False
    except:
      sys.stderr.write(colored('Could not find log file at path {1} for task {0}, check your -l arg and try again\n'.format(args.logfile, task), 'red'))
      keep_trying = False
    while keep_trying:
      try:
        offset = self.fetch_new_log_data(uri, path, offset, args, task)
        time.sleep(5)
      except ValueError:
        sys.stderr.write(colored('Could not tail logs for task {0}, check that the task is still active and that the slave it runs on has not been decommissioned\n'.format(task), 'red'))
        keep_trying = False

  def get_initial_offset(self, uri, path, args):
    params = {"path" : path}

    return long(requests.get(uri, params=params, headers=args.headers).json()['offset'])

  def fetch_new_log_data(self, uri, path, offset, args, task):
    params = {
      "path" : path,
      "offset" : offset
    }
    response = requests.get(uri, params=params, headers=args.headers).json()
    prefix = '({0}) =>\n'.format(task) if args.verbose else ''
    if len(response['data'].encode('utf-8')) > 0:
      if args.grep:
        filename = '{0}/.grep{1}'.format(args.dest, self.Task)
        self.create_grep_file(args, filename, response['data'])
        output = os.popen(grep_command(args, filename)).read()
        sys.stdout.write('{0}{1}'.format(colored(prefix, 'cyan'), output))
        self.remove_grep_file(filename)
      else:
        sys.stdout.write('{0}{1}'.format(colored(prefix, 'cyan'), response['data'].encode('utf-8')))
      return offset + len(response['data'].encode('utf-8'))
    else:
      return offset

  def create_grep_file(self, args, filename, content):
    grep_file = open(filename, 'wb')
    grep_file.write(content.encode('utf-8'))
    grep_file.close()

  def remove_grep_file(self, grep_file):
    if os.path.isfile(grep_file):
      os.remove(grep_file)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# file: sphere.py
# author: Anush Krishnan (anush@bu.edu), Olivier Mesnard (mesnardo@gwu.edu)
# description: Generates coordinates of a sphere.

import argparse
import os

import numpy

def read_inputs():
  """Creates the parser and parses the command-line."""
  # create parser
  parser = argparse.ArgumentParser(description='Generates a spherical body',
                         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
  # add arguments to parser
  parser.add_argument('--radius', '-r', dest='radius', type=float, default=0.5,
            help='radius of the sphere')
  parser.add_argument('--center', '-c', dest=center, type=float, nargs='+',
                      default=[0.0, 0.0, 0.0],
                      help='coordinates of the center of the sphere')
  parser.add_argument('-ds', dest='ds', type=float, default='0.015',
                      help='mesh-spacing')
  parser.add_argument('--name', dest='file_name', type=str, default=None,
                      help='name of the output file (without extension)')
  parser.add_argument('--save-dir', dest='save_dir', type=str,
                      default=os.getcwd(),
                      help='directory where file will be saved')
  return parser.parse_args()

def main():
  """Generates the coordinates of a sphere and writes into file."""
  # parse the command-line
  args = read_inputs()

  # parameters
  R = args.radius
  xc, yc, zc = args.center
  h = args.ds

  # `total` keeps track of the number of points on the body
  # it is initialized with 2 to count the points on the poles
  total = 2

  # calculate angle increments in altitude and azimuth
  nCircHalf = int(np.pi*R/h)+1
  dPhi = np.pi/nCircHalf
  for i in range(1,nCircHalf):
    phi = i*dPhi
    r = R*np.sin(phi)
    nCircAzim = int(2*np.pi*r/h)+1
    total += nCircAzim

  # write body data to file
  f = open(filename, 'w')
  f.write("%d\n" % total)
  f.write("%f\t%f\t%f\n" % (x0, y0, R+z0))
  for i in range(1,nCircHalf):
    phi = i*dPhi
    z = R*np.cos(phi)
    r = R*np.sin(phi)
    nCircAzim = int(2*np.pi*r/h)+1
    dTheta = 2*np.pi/nCircAzim
    for j in range(nCircAzim):
      theta = j*dTheta
      x = r*np.cos(theta)
      y = r*np.sin(theta)
      f.write("%f\t%f\t%f\n" % (x+x0, y+y0, z+z0))
  f.write("%f\t%f\t%f\n" % (x0, y0, -R+z0))
  f.close()

if __name__=="__main__":
  main()<commit_msg>WIP: rewrite generation of sphere<commit_after>#!/usr/bin/env python

# file: sphere.py
# author: Anush Krishnan (anush@bu.edu), Olivier Mesnard (mesnardo@gwu.edu)
# description: Generates coordinates of a sphere.

import argparse
import os
import math

import numpy

def read_inputs():
  """Creates the parser and parses the command-line."""
  # create parser
  parser = argparse.ArgumentParser(description='Generates a spherical body',
                         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
  # add arguments to parser
  parser.add_argument('--radius', '-r', dest='radius', type=float, default=0.5,
            help='radius of the sphere')
  parser.add_argument('--center', '-c', dest='center', type=float, nargs='+',
                      default=[0.0, 0.0, 0.0],
                      help='coordinates of the center of the sphere')
  parser.add_argument('-ds', dest='ds', type=float, default='0.015',
                      help='mesh-spacing')
  parser.add_argument('--name', dest='file_name', type=str, default=None,
                      help='name of the output file (without extension)')
  parser.add_argument('--save-dir', dest='save_dir', type=str,
                      default=os.getcwd(),
                      help='directory where file will be saved')
  return parser.parse_args()

def main():
  """Generates the coordinates of a sphere and writes into file."""
  # parse the command-line
  args = read_inputs()

  # parameters
  R = args.radius
  xc, yc, zc = args.center
  h = args.ds

  if not args.file_name:
    args.file_name = 'sphere_%g' % args.ds

  n_phi = int(math.ceil(math.pi*R/h))
  phi = numpy.linspace(0.0, math.pi, n_phi)[1:-1]
  n_theta = int(math.ceil(2.0*math.pi*R/h))
  theta = numpy.linspace(0.0, 2.0*math.pi, n_theta)[1:-1]

  x = xc + R*numpy.outer(numpy.sin(phi), numpy.cos(theta)).flatten()
  x = numpy.insert(x, 0, xc)
  x = numpy.insert(x, -1, xc)
  y = yc + R*numpy.outer(numpy.sin(phi), numpy.sin(theta)).flatten()
  #y = numpy.ravel(yc + R*numpy.outer(numpy.sin(phi), numpy.sin(theta)))
  y = numpy.insert(y, 0, yc)
  y = numpy.insert(y, -1, yc)
  z = zc + R*numpy.outer(numpy.cos(phi), numpy.ones(theta.size)).flatten()
  #z = numpy.ravel(zc + R*numpy.outer(numpy.cos(phi), numpy.ones(theta.size)))
  z = numpy.insert(z, 0, R+zc)
  z = numpy.insert(z, -1, -R+zc)

  with open('%s/%s.body' % (args.save_dir, args.file_name), 'w') as outfile:
    outfile.write('%d\n' % x.size)
    numpy.savetxt(outfile, numpy.c_[x, y, z], fmt='%.6f', delimiter='\t')

if __name__=="__main__":
  main()<|endoftext|>


<commit_before>#!/usr/bin/python
import RPi.GPIO as GPIO
import argparse

GPIO.setwarnings(False)
GPIO.setmode(GPIO.BOARD)

# GPIO/BOARD | Relay IN | Rotors | Zone
# 22/15	     | R2 IN2   | 1      | B
# 18/12	     | R1 IN2   | 2      | A
# 24/18	     | R1 IN3   | 3      | D
# 17/11	     | R1 IN4   | 4      | C
# 27/13	     | R2 IN1   | 5      | E

relayIO = { "1": 15, "2": 12, "3": 18, "4": 11, "5": 13}

def setState(relay, state):
	print("Trying to set relay: " + str(relayIO[relay]) + " to state: " + state)
	GPIO.output(relayIO[relay], int(state))

	if getState(relay) != state:
		print("relay: " + relay + " is not set to " + state)

	print("relay: " + relay + " is set to " + str(getState(relay)))

def toggle(relay):
	GPIO.output(relayIO[relay], not GPIO.input(relayIO[relay]))

def getState(relay):
	return GPIO.input(int(relayIO[relay]))

def setAll(state):
	chan_list = []
	for relay in relayIO:
    		chan_list.append(relayIO[relay])
	GPIO.output(chan_list, int(state))

def getAll():
	chan_list = []
	for relay in relayIO:
    		chan_list.append(relayIO[relay])
	GPIO.setup(chan_list, GPIO.OUT)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--relay', help='Set relay 1/2/3/4/5', required=True)
    parser.add_argument('--state',help='Set state high=1 or low=0', required=false)
    parser.add_argument('--toggle',help='Toggle state', required=false)
    parser.add_argument('--info',help='Get state high=1 or low=0', required=false)
    parser.add_argument('--all',help='Set all to state high=1 or low=0', required=false)

    args = parser.parse_args()
    GPIO.setup(relayIO[relay], GPIO.OUT)

    if args.toggle:
	toggle(args.relay)
    elif args.state:
	setState(args.relay, args.state)
    elif args.info:
	getState(args.relay)
    elif args.all and args.state:
	setAll(args.state)

    GPIO.cleanup()

if __name__ == '__main__':
    main()
<commit_msg>remove required args<commit_after>#!/usr/bin/python
import RPi.GPIO as GPIO
import argparse

GPIO.setwarnings(False)
GPIO.setmode(GPIO.BOARD)

# GPIO/BOARD | Relay IN | Rotors | Zone
# 22/15	     | R2 IN2   | 1      | B
# 18/12	     | R1 IN2   | 2      | A
# 24/18	     | R1 IN3   | 3      | D
# 17/11	     | R1 IN4   | 4      | C
# 27/13	     | R2 IN1   | 5      | E

relayIO = { "1": 15, "2": 12, "3": 18, "4": 11, "5": 13}

def setState(relay, state):
	print("Trying to set relay: " + str(relayIO[relay]) + " to state: " + state)
	GPIO.output(relayIO[relay], int(state))

	if getState(relay) != state:
		print("relay: " + relay + " is not set to " + state)

	print("relay: " + relay + " is set to " + str(getState(relay)))

def toggle(relay):
	GPIO.output(relayIO[relay], not GPIO.input(relayIO[relay]))

def getState(relay):
	return GPIO.input(int(relayIO[relay]))

def setAll(state):
	chan_list = []
	for relay in relayIO:
    		chan_list.append(relayIO[relay])
	GPIO.output(chan_list, int(state))

def getAll():
	chan_list = []
	for relay in relayIO:
    		chan_list.append(relayIO[relay])
	GPIO.setup(chan_list, GPIO.OUT)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--relay', help='Set relay 1/2/3/4/5', required=false)
    parser.add_argument('--state',help='Set state high=1 or low=0', required=false)
    parser.add_argument('--toggle',help='Toggle state', required=false)
    parser.add_argument('--info',help='Get state high=1 or low=0', required=false)
    parser.add_argument('--all',help='Set all to state high=1 or low=0', required=false)

    args = parser.parse_args()
    GPIO.setup(relayIO[relay], GPIO.OUT)

    if args.toggle:
	toggle(args.relay)
    elif args.state:
	setState(args.relay, args.state)
    elif args.info:
	getState(args.relay)
    elif args.all and args.state:
	setAll(args.state)

    GPIO.cleanup()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import re
import os
lastNum = re.compile(r'(?:[^\d]*(\d+)[^\d]*)+')

def title(t):
	print("\n---------------------------------")
	print(t)
	print("---------------------------------")

def cmd(c):
	print("\n" + c)
	os.system(c)

def increment(s):
    """ look for the last sequence of number(s) in a string and increment """
    m = lastNum.search(s)
    if m:
        next = str(int(m.group(1))+1)
        start, end = m.span(1)
        s = s[:max(end-len(next), start)] + next + s[end:]
    return s, str(next)

def lvgl_clone():
	title("lvgl: Clone")
	cmd("git clone https://github.com/littlevgl/lvgl.git")
	os.chdir("./lvgl")
	cmd("git co master")

def lvgl_update_version():
	title("lvgl: Update version number")

	f = open("./src/lv_version.h", "r")

	outbuf = ""
	major_ver = -1
	minor_ver = -1
	patch_ver = -1

	for i in f.read().splitlines():
		r = re.search(r'^#define LVGL_VERSION_MAJOR ', i)
		if r:
		  m = lastNum.search(i)
		  if m: major_ver = m.group(1)

		r = re.search(r'^#define LVGL_VERSION_MINOR ', i)
		if r:
		  m = lastNum.search(i)
		  if m: minor_ver = m.group(1)

		r = re.search(r'^#define LVGL_VERSION_PATCH ', i)
		if r:
		  i, patch_ver = increment(i)

		r = re.search(r'^#define LVGL_VERSION_INFO ', i)
		if r:
		  i = "#define LVGL_VERSION_INFO \"\""

		outbuf += i + '\n'

	f.close()

	f = open("./src/lv_version.h", "w")

	f.write(outbuf)
	f.close()

	s = "v" + str(major_ver) + "." + str(minor_ver) + "." + str(patch_ver)
	print("New version:" + s)
	return s

def lvgl_update_library_json(v):
	title("lvgl: Update version number in library.json")

	f = open("./library.json", "r")

	outbuf = ""

	for i in f.read().splitlines():
		r = re.search(r'"version": ', i)
		if r:
		  i = '    "version": "' + v + '",'

		outbuf += i + '\n'

	f.close()

	f = open("./library.json", "w")

	f.write(outbuf)
	f.close()

def lvgl_commit_push(v):
	title("lvgl: commit and push release")

	cmd('git ci -am "Release ' + v + '"')
	cmd('git tag -a ' + v + ' -m "Release ' + v +'"')
	cmd('git push origin master')
	cmd('git push origin ' + v)

def lvgl_update_api_docs():
	title("lvgl: Update API with Doxygen")

	cmd("cd scripts; doxygen");

def docs_clone():
	title("docs: Clone")
	os.chdir("../")
	cmd("git clone --recursive https://github.com/littlevgl/docs.git")
	os.chdir("./docs")
	cmd("git co master")

def docs_get_api():
	title("docs: Get API files")

	cmd("rm -rf xml");
	cmd("cp -r ../lvgl/docs/api_doc/xml .");

def docs_update_version(v):
	title("docs: Update version number")

	f = open("./conf.py", "r")

	outbuf = ""

	for i in f.read().splitlines():
		r = re.search(r'^version = ', i)
		if r:
		  i = "version = '" + v + "'"

		r = re.search(r'^release = ', i)
		if r:
		  i = "version = '" + v + "'"

		outbuf += i + '\n'

	f.close()

	f = open("./conf.py", "w")

	f.write(outbuf)
	f.close()

def docs_update_trans():
	title("docs: Update translations")
	cmd("cd en && ./trans_push.py && ./trans_pull.py")

def docs_build():
	title("docs: Build")
	cmd("./build.py clean")

def docs_commit_push(v):
	title("docs: Commit release")

	cmd('git add .')
	cmd('git ci -am "Release ' + v + '"')
	cmd('git tag -a ' + v + ' -m "Release ' + v +'"')
	cmd('git push origin master')
	cmd('git push origin ' + v)

def clean_up():
	title("Clean up repos")
	os.chdir("..")
	cmd("rm -rf lvgl docs")

lvgl_clone()
lvgl_update_api_docs()
ver_str = lvgl_update_version()
lvgl_update_library_json(ver_str)
lvgl_commit_push(ver_str)

docs_clone()
docs_get_api()
docs_update_version(ver_str)
docs_update_trans()
docs_build()
docs_commit_push(ver_str)

clean_up()
<commit_msg>update release script<commit_after>import re
import os
lastNum = re.compile(r'(?:[^\d]*(\d+)[^\d]*)+')

def title(t):
	print("\n---------------------------------")
	print(t)
	print("---------------------------------")

def cmd(c):
	print("\n" + c)
	os.system(c)

def increment(s):
    """ look for the last sequence of number(s) in a string and increment """
    m = lastNum.search(s)
    if m:
        next = str(int(m.group(1))+1)
        start, end = m.span(1)
        s = s[:max(end-len(next), start)] + next + s[end:]
    return s, str(next)

def lvgl_clone():
	title("lvgl: Clone")
	cmd("git clone https://github.com/lvgl/lvgl.git")
	os.chdir("./lvgl")
	cmd("git co master")

def lvgl_update_version():
	title("lvgl: Update version number")

	f = open("./src/lv_version.h", "r")

	outbuf = ""
	major_ver = -1
	minor_ver = -1
	patch_ver = -1

	for i in f.read().splitlines():
		r = re.search(r'^#define LVGL_VERSION_MAJOR ', i)
		if r:
		  m = lastNum.search(i)
		  if m: major_ver = m.group(1)

		r = re.search(r'^#define LVGL_VERSION_MINOR ', i)
		if r:
		  m = lastNum.search(i)
		  if m: minor_ver = m.group(1)

		r = re.search(r'^#define LVGL_VERSION_PATCH ', i)
		if r:
		  m = lastNum.search(i)
		  if m: patch_ver = m.group(1)

		r = re.search(r'^#define LVGL_VERSION_INFO ', i)
		if r:
		  i = "#define LVGL_VERSION_INFO \"\""

		outbuf += i + '\n'

	f.close()

	f = open("./src/lv_version.h", "w")

	f.write(outbuf)
	f.close()

	s = "v" + str(major_ver) + "." + str(minor_ver) + "." + str(patch_ver)
	print("New version:" + s)
	return s

def lvgl_update_library_json(v):
	title("lvgl: Update version number in library.json")

	f = open("./library.json", "r")

	outbuf = ""

	for i in f.read().splitlines():
		r = re.search(r'"version": ', i)
		if r:
		  i = '    "version": "' + v + '",'

		outbuf += i + '\n'

	f.close()

	f = open("./library.json", "w")

	f.write(outbuf)
	f.close()

def lvgl_commit_push(v):
	title("lvgl: commit and push release")

	cmd('git ci -am "Release ' + v + '"')
	cmd('git tag -a ' + v + ' -m "Release ' + v +'"')
	cmd('git push origin master')
	cmd('git push origin ' + v)

def lvgl_update_api_docs():
	title("lvgl: Update API with Doxygen")

	cmd("cd scripts; doxygen");

def docs_clone():
	title("docs: Clone")
	os.chdir("../")
	cmd("git clone --recursive https://github.com/lvgl/docs.git")
	os.chdir("./docs/v7")
	cmd("git co master")

def docs_get_api():
	title("docs: Get API files")

	cmd("rm -rf xml");
	cmd("cp -r ../lvgl/docs/api_doc/xml .");

def docs_update_version(v):
	title("docs: Update version number")

	f = open("./conf.py", "r")

	outbuf = ""

	for i in f.read().splitlines():
		r = re.search(r'^version = ', i)
		if r:
		  i = "version = '" + v + "'"

		r = re.search(r'^release = ', i)
		if r:
		  i = "version = '" + v + "'"

		outbuf += i + '\n'

	f.close()

	f = open("./conf.py", "w")

	f.write(outbuf)
	f.close()

def docs_update_trans():
	title("docs: Update translations")
	cmd("cd en && ./trans_push.py && ./trans_pull.py")

def docs_build():
	title("docs: Build")
	cmd("./build.py clean")

def docs_commit_push(v):
	title("docs: Commit release")

	cmd('git add .')
	cmd('git ci -am "Release ' + v + '"')
	cmd('git tag -a ' + v + ' -m "Release ' + v +'"')
	cmd('git push origin master')
	cmd('git push origin ' + v)

def clean_up():
	title("Clean up repos")
	os.chdir("..")
	cmd("rm -rf lvgl docs")

lvgl_clone()
lvgl_update_api_docs()
ver_str = lvgl_update_version()
lvgl_update_library_json(ver_str)
lvgl_commit_push(ver_str)

docs_clone()
docs_get_api()
docs_update_version(ver_str)
#docs_update_trans() # Zanata is not working now
docs_build()
docs_commit_push(ver_str)

clean_up()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding=utf-8
from decimal import Decimal

from six import iteritems
from funclib.utils import resolve_args

class Interval(object):
    """
    Interval(stop)
    Interval(start, stop)

    Represents an interval between two numbers.

    Parameters
    ----------
    start : decimal.Decimal or float or int
        Start of the interval.
    stop : decimal.Decimal or float or int
        Inclusive stop of the interval.

    Examples
    --------
    >>> interval = Interval(5)
    >>> interval
    ... Interval(start=0.0, stop=5.0)
    >>> 3 in interval
    ... True
    >>> 10 in interval
    ... False
    """
    _DEFAULTS = {'start': 0}

    def __init__(self, *args, **kwargs):
        try:
            arguments = resolve_args(['stop'], args, kwargs)
        except TypeError:
            arguments = resolve_args(['start', 'stop'], args, kwargs,
                                     self._DEFAULTS)

        final_arguments = self._DEFAULTS.copy()
        final_arguments.update(arguments._asdict())
        final_arguments = {k: Decimal(str(v))
                           for k, v in iteritems(final_arguments)}

        self.__dict__.update(final_arguments)

    def __contains__(self, item):
        """Containment test.

        Parameters
        ----------
        item : int or float or decimal.Decimal
            Number to test.

        Returns
        -------
        bool
        """
        return self.start <= item <= self.stop

    def __str__(self):
        return ('Interval(start={start}, stop={stop})'
                .format(start=repr(float(self.start)),
                        stop=repr(float(self.stop))))

    def __repr__(self):
        return self.__str__()

    def __eq__(self, other):
        return (self.start == other.start and
                self.stop == other.stop)

    def __hash__(self):
        return hash((self.start, self.stop))
<commit_msg>Remove forceful conversion to Decimal in Interval<commit_after>#!/usr/bin/env python
# coding=utf-8
from funclib.utils import resolve_args

class Interval(object):
    """
    Interval(stop)
    Interval(start, stop)

    Represents an interval between two numbers.

    Parameters
    ----------
    start : decimal.Decimal or float or int
        Start of the interval.
    stop : decimal.Decimal or float or int
        Inclusive stop of the interval.

    Examples
    --------
    >>> interval = Interval(5)
    >>> interval
    ... Interval(start=0.0, stop=5.0)
    >>> 3 in interval
    ... True
    >>> 10 in interval
    ... False
    """
    _DEFAULTS = {'start': 0}

    def __init__(self, *args, **kwargs):
        try:
            arguments = resolve_args(['stop'], args, kwargs)
        except TypeError:
            arguments = resolve_args(['start', 'stop'], args, kwargs,
                                     self._DEFAULTS)

        final_arguments = self._DEFAULTS.copy()
        final_arguments.update(arguments._asdict())

        self.__dict__.update(final_arguments)

    def __contains__(self, item):
        """Containment test.

        Parameters
        ----------
        item : int or float or decimal.Decimal
            Number to test.

        Returns
        -------
        bool
        """
        return self.start <= item <= self.stop

    def __str__(self):
        return ('Interval(start={start}, stop={stop})'
                .format(start=repr(float(self.start)),
                        stop=repr(float(self.stop))))

    def __repr__(self):
        return self.__str__()

    def __eq__(self, other):
        return (self.start == other.start and
                self.stop == other.stop)

    def __hash__(self):
        return hash((self.start, self.stop))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# pylint: disable=bad-continuation
""" Tools to discover and modify content.
"""
# Copyright ©  2015 1&1 Group <git@1and1.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, unicode_literals, print_function

import re
import difflib
try:
    import html.entities as htmlentitydefs
except ImportError:  # Python 2
    import htmlentitydefs  # pylint: disable=import-error,wrong-import-order
from xml.sax.saxutils import quoteattr  # pylint: disable=wrong-import-order

import arrow
from bunch import bunchify
from lxml.etree import fromstring, HTMLParser, XMLParser, XMLSyntaxError  # pylint: disable=no-name-in-module
from rudiments.reamed import click

from .._compat import StringIO

# Mapping of CLI content format names to Confluence API names
CLI_CONTENT_FORMATS = dict(view='view', editor='editor', storage='storage', export='export_view', anon='anonymous_export_view')

# Simple replacement rules, order is important!
TIDY_REGEX_RULES = ((_name, re.compile(_rule), _subst) for _name, _rule, _subst in [
    ("FosWiki: Remove CSS class from section title",
     r'<(h[1-5]) class="[^"]*">', r'<\1>'),
    ("FosWiki: Remove static section numbering",
     r'(?<=<h.>)(|)[0-9.]+?\s*(?= )', r'\1'),
    ("FosWiki: Empty anchor in headers",
     r'(?<=<h.>)\s* +', ''),
    ("FosWiki: 'tok' spans in front of headers",
     r'(?<=<h.>)(|)\s* ', r'\1'),
    ("FosWiki: Section edit icons at the end of headers",
     r'\s*<ac:image [^>]+><ri:url ri:value="[^"]+/EditChapterPlugin/pencil.png" ?/>'
     r'</ac:image>(?=</h)', ''),
    ("FosWiki: 'Edit Chapter Plugin' spans (old)",
     r'(?<=<h.>)(|)\s*\s*([^<]+)(?:
)\s*(?=</h.>)', r'\1\2'),
    ("FosWiki: 'Edit Chapter Plugin' spans (new)",
     r'(?<=<h.>)(|)\s*\s*([^<]+)(?:
)<a class="ecpEdit".+?\s*(?=</h.>)', r'\1\2'),
    ("FosWiki: Residual leading whitespace in headers",
     r'(?<=<h.>)(|)\s* +', r'\1'),
    ("FosWiki: Replace TOC div with macro",
     r'()?
.*?
', '''
          <ac:structured-macro ac:name="panel" ac:schema-version="1">
            <ac:parameter ac:name="title">Contents</ac:parameter>
            <ac:rich-text-body>

                <ac:structured-macro ac:name="toc" ac:schema-version="1"/>

            </ac:rich-text-body>
          </ac:structured-macro>'''),
    ("FosWiki: Replace TOC in a Twisty with Expand+TOC macro",
     r'
.+?<big>Table of Contents</big>
', '''
          <ac:structured-macro ac:name="expand" ac:schema-version="1">
            <ac:parameter ac:name="title">Table of Contents</ac:parameter>
            <ac:rich-text-body>

                <ac:structured-macro ac:name="toc" ac:schema-version="1"/>

            </ac:rich-text-body>
          </ac:structured-macro>'''),
    ("FosWiki: Named anchors (#WikiWords)",
     r'(]+><big>[^<]+</big>)',
     r'\1\2; float: right;\3'),
    ("FosWiki: Wrap HTML '
' into 'panel' macro",
     r'(?<!<ac:rich-text-body>)(
)',
     r'<ac:structured-macro ac:name="panel" ac:schema-version="1">'
     r'<ac:parameter ac:name="bgColor">#eeeeee</ac:parameter>'
     r'<ac:rich-text-body>'
     r'\1'),
    ("FosWiki: Wrap HTML '
' into 'panel' macro",
     r'
(?!</ac:rich-text-body>)', '
</ac:rich-text-body></ac:structured-macro>'),
    ("FosWiki: Embedded CSS - custom list indent",
     r'
', '
'),
    ("FosWiki: Empty paragraphs",
     r'

', r''),
])

def _apply_tidy_regex_rules(body, log=None):
    """Return tidied body after applying regex rules."""
    body = body.replace(u'\u00A0', ' ')
    for name, rule, subst in TIDY_REGEX_RULES:
        length = len(body)
        try:
            body, count = rule.subn(subst, body)
        except re.error as cause:
            raise click.LoggedFailure('Error "{}" in "{}" replacement: {} => {}'.format(
                cause, name, rule.pattern, subst,
            ))
        if count and log:
            length -= len(body)
            log.info('Replaced %d matche(s) of "%s" (%d chars %s)',
                     count, name, abs(length), "added" if length < 0 else "removed")
    return body

def _make_etree(body, content_format='storage', attrs=None):
    """Create an ElementTree from a page's body."""
    attrs = (attrs or {}).copy()
    attrs.update({
        'xmlns:ac': 'http://www.atlassian.com/schema/confluence/4/ac/',
        'xmlns:ri': 'http://www.atlassian.com/schema/confluence/4/ri/',
    })
    xml_body = re.sub(r'&(?!(amp|lt|gt|quot|apos))([a-zA-Z0-9]+);',
                  lambda cref: '&#{};'.format(htmlentitydefs.name2codepoint[cref.group(2)]), body)
    #print(body.encode('utf8'))
    xmldoc = u'<{root} {attrs}>{body}</{root}>'.format(
        root=content_format,
        attrs=' '.join('{}={}'.format(k, quoteattr(v)) for k, v in sorted(attrs.items())),
        body=xml_body)

    parser = (XMLParser if content_format == 'storage' else HTMLParser)(remove_blank_text=True)
    try:
        return fromstring(xmldoc, parser)
    except XMLSyntaxError as cause:
        raise click.LoggedFailure('{}\n{}'.format(
            cause, '\n'.join(['{:7d} {}'.format(i+1, k) for i, k in enumerate(xmldoc.splitlines())])
        ))

def _pretty_xml(body, content_format='storage', attrs=None):
    """Pretty-print the given page body and return a list of lines."""
    root = _make_etree(body, content_format=content_format, attrs=attrs)
    prettyfied = StringIO()
    root.getroottree().write(prettyfied, encoding='utf8', pretty_print=True, xml_declaration=False)
    return prettyfied.getvalue().splitlines()

class ConfluencePage(object):
    """A page that holds enough state so it can be modified."""

    DIFF_COLS = {
        '+': 'green',
        '-': 'red',
        '@': 'yellow',
    }

    def __init__(self, cf, url, markup='storage'):
        """Load the given page."""
        self.cf = cf
        self.url = url
        self.markup = markup
        self._data = cf.get(self.url, expand='space,version,body.' + self.markup)
        self.body = self._data.body[self.markup].value

    @property
    def page_id(self):
        """The numeric page ID."""
        return self._data.id

    @property
    def space_key(self):
        """The space this page belongs to."""
        return self._data.space.key

    @property
    def title(self):
        """The page's title."""
        return self._data.title

    @property
    def version(self):
        """The page's version number in history."""
        return self._data.version.number

    def etree(self):
        """Parse the page's body into an ElementTree."""
        attrs = {
            'id': 'page-' + self._data.id,
            'href': self._data._links.base + self._data._links.tinyui,
            'status': self._data.status,
            'title': self._data.title,
        }
        return _make_etree(self.body, content_format=self.markup, attrs=attrs)

    def tidy(self, log=None):
        """Return a tidy copy of this page's body."""
        assert self.markup == 'storage', "Can only clean up pages in storage format!"
        return _apply_tidy_regex_rules(self.body, log=log)

    def update(self, body=None, minor=True):
        """Update a page's content."""
        assert self.markup == 'storage', "Cannot update non-storage page markup!"
        if body is None:
            body = self.body
        if body == self._data.body[self.markup].value:
            return  # No changes

        data = {
            #'id': self._data.id,
            'type': 'page',
            'space': {'key': self.space_key},
            'title': self.title,
            'version': dict(number=self.version + 1, minorEdit=minor),
            'body': {
                'storage': {
                    'value': body,
                    'representation': self.markup,
                }
            },
            'expand': 'version',
        }
        response = self.cf.session.put(self._data._links.self, json=data)
        response.raise_for_status()
        ##page = response.json(); print(page)
        result = bunchify(response.json())
        self._data.body[self.markup].value = body
        self._data.version = result.version
        return result

    def dump_diff(self, changed):
        """Dump a diff to terminal between changed and stored body."""
        if self.body == changed:
            click.secho('=== No changes to "{0}"'.format(self.title), fg='green')
            return

        diff = difflib.unified_diff(
            _pretty_xml(self.body, self.markup),
            _pretty_xml(changed, self.markup),
            u'v. {0} of "{1}"'.format(self.version, self.title).encode('utf8'),
            u'v. {0} of "{1}"'.format(self.version + 1, self.title).encode('utf8'),
            arrow.get(self._data.version.when).replace(microsecond=0).isoformat(sep=b' '),
            arrow.now().replace(microsecond=0).isoformat(sep=b' '),
            lineterm='', n=2)
        for line in diff:
            click.secho(line, fg=self.DIFF_COLS.get(line and line[0], None))
<commit_msg>tidy: new rule 'FosWiki: Obsolete CSS classes'<commit_after># -*- coding: utf-8 -*-
# pylint: disable=bad-continuation
""" Tools to discover and modify content.
"""
# Copyright ©  2015 1&1 Group <git@1and1.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, unicode_literals, print_function

import re
import difflib
try:
    import html.entities as htmlentitydefs
except ImportError:  # Python 2
    import htmlentitydefs  # pylint: disable=import-error,wrong-import-order
from xml.sax.saxutils import quoteattr  # pylint: disable=wrong-import-order

import arrow
from bunch import bunchify
from lxml.etree import fromstring, HTMLParser, XMLParser, XMLSyntaxError  # pylint: disable=no-name-in-module
from rudiments.reamed import click

from .._compat import StringIO

# Mapping of CLI content format names to Confluence API names
CLI_CONTENT_FORMATS = dict(view='view', editor='editor', storage='storage', export='export_view', anon='anonymous_export_view')

# Simple replacement rules, order is important!
TIDY_REGEX_RULES = ((_name, re.compile(_rule), _subst) for _name, _rule, _subst in [
    ("FosWiki: Remove CSS class from section title",
     r'<(h[1-5]) class="[^"]*">', r'<\1>'),
    ("FosWiki: Remove static section numbering",
     r'(?<=<h.>)(|)[0-9.]+?\s*(?= )', r'\1'),
    ("FosWiki: Empty anchor in headers",
     r'(?<=<h.>)\s* +', ''),
    ("FosWiki: 'tok' spans in front of headers",
     r'(?<=<h.>)(|)\s* ', r'\1'),
    ("FosWiki: Section edit icons at the end of headers",
     r'\s*<ac:image [^>]+><ri:url ri:value="[^"]+/EditChapterPlugin/pencil.png" ?/>'
     r'</ac:image>(?=</h)', ''),
    ("FosWiki: 'Edit Chapter Plugin' spans (old)",
     r'(?<=<h.>)(|)\s*\s*([^<]+)(?:
)\s*(?=</h.>)', r'\1\2'),
    ("FosWiki: 'Edit Chapter Plugin' spans (new)",
     r'(?<=<h.>)(|)\s*\s*([^<]+)(?:
)<a class="ecpEdit".+?\s*(?=</h.>)', r'\1\2'),
    ("FosWiki: Residual leading whitespace in headers",
     r'(?<=<h.>)(|)\s* +', r'\1'),
    ("FosWiki: Replace TOC div with macro",
     r'()?
.*?
', '''
          <ac:structured-macro ac:name="panel" ac:schema-version="1">
            <ac:parameter ac:name="title">Contents</ac:parameter>
            <ac:rich-text-body>

                <ac:structured-macro ac:name="toc" ac:schema-version="1"/>

            </ac:rich-text-body>
          </ac:structured-macro>'''),
    ("FosWiki: Replace TOC in a Twisty with Expand+TOC macro",
     r'
.+?<big>Table of Contents</big>
', '''
          <ac:structured-macro ac:name="expand" ac:schema-version="1">
            <ac:parameter ac:name="title">Table of Contents</ac:parameter>
            <ac:rich-text-body>

                <ac:structured-macro ac:name="toc" ac:schema-version="1"/>

            </ac:rich-text-body>
          </ac:structured-macro>'''),
    ("FosWiki: Named anchors (#WikiWords)",
     r'(]+><big>[^<]+</big>)',
     r'\1\2; float: right;\3'),
    ("FosWiki: Wrap HTML '
' into 'panel' macro",
     r'(?<!<ac:rich-text-body>)(
)',
     r'<ac:structured-macro ac:name="panel" ac:schema-version="1">'
     r'<ac:parameter ac:name="bgColor">#eeeeee</ac:parameter>'
     r'<ac:rich-text-body>'
     r'\1'),
    ("FosWiki: Wrap HTML '
' into 'panel' macro",
     r'
(?!</ac:rich-text-body>)', '
</ac:rich-text-body></ac:structured-macro>'),
    ("FosWiki: Embedded CSS - custom list indent",
     r'
', '
'),
    ("FosWiki: Empty paragraphs",
     r'

', r''),
    ("FosWiki: Obsolete CSS classes",
     r'(<(?:div|p|span|h[1-5])) class="(foswikiTopic)"', r'\1'),
])

def _apply_tidy_regex_rules(body, log=None):
    """Return tidied body after applying regex rules."""
    body = body.replace(u'\u00A0', ' ')
    for name, rule, subst in TIDY_REGEX_RULES:
        length = len(body)
        try:
            body, count = rule.subn(subst, body)
        except re.error as cause:
            raise click.LoggedFailure('Error "{}" in "{}" replacement: {} => {}'.format(
                cause, name, rule.pattern, subst,
            ))
        if count and log:
            length -= len(body)
            log.info('Replaced %d matche(s) of "%s" (%d chars %s)',
                     count, name, abs(length), "added" if length < 0 else "removed")
    return body

def _make_etree(body, content_format='storage', attrs=None):
    """Create an ElementTree from a page's body."""
    attrs = (attrs or {}).copy()
    attrs.update({
        'xmlns:ac': 'http://www.atlassian.com/schema/confluence/4/ac/',
        'xmlns:ri': 'http://www.atlassian.com/schema/confluence/4/ri/',
    })
    xml_body = re.sub(r'&(?!(amp|lt|gt|quot|apos))([a-zA-Z0-9]+);',
                  lambda cref: '&#{};'.format(htmlentitydefs.name2codepoint[cref.group(2)]), body)
    #print(body.encode('utf8'))
    xmldoc = u'<{root} {attrs}>{body}</{root}>'.format(
        root=content_format,
        attrs=' '.join('{}={}'.format(k, quoteattr(v)) for k, v in sorted(attrs.items())),
        body=xml_body)

    parser = (XMLParser if content_format == 'storage' else HTMLParser)(remove_blank_text=True)
    try:
        return fromstring(xmldoc, parser)
    except XMLSyntaxError as cause:
        raise click.LoggedFailure('{}\n{}'.format(
            cause, '\n'.join(['{:7d} {}'.format(i+1, k) for i, k in enumerate(xmldoc.splitlines())])
        ))

def _pretty_xml(body, content_format='storage', attrs=None):
    """Pretty-print the given page body and return a list of lines."""
    root = _make_etree(body, content_format=content_format, attrs=attrs)
    prettyfied = StringIO()
    root.getroottree().write(prettyfied, encoding='utf8', pretty_print=True, xml_declaration=False)
    return prettyfied.getvalue().splitlines()

class ConfluencePage(object):
    """A page that holds enough state so it can be modified."""

    DIFF_COLS = {
        '+': 'green',
        '-': 'red',
        '@': 'yellow',
    }

    def __init__(self, cf, url, markup='storage'):
        """Load the given page."""
        self.cf = cf
        self.url = url
        self.markup = markup
        self._data = cf.get(self.url, expand='space,version,body.' + self.markup)
        self.body = self._data.body[self.markup].value

    @property
    def page_id(self):
        """The numeric page ID."""
        return self._data.id

    @property
    def space_key(self):
        """The space this page belongs to."""
        return self._data.space.key

    @property
    def title(self):
        """The page's title."""
        return self._data.title

    @property
    def version(self):
        """The page's version number in history."""
        return self._data.version.number

    def etree(self):
        """Parse the page's body into an ElementTree."""
        attrs = {
            'id': 'page-' + self._data.id,
            'href': self._data._links.base + self._data._links.tinyui,
            'status': self._data.status,
            'title': self._data.title,
        }
        return _make_etree(self.body, content_format=self.markup, attrs=attrs)

    def tidy(self, log=None):
        """Return a tidy copy of this page's body."""
        assert self.markup == 'storage', "Can only clean up pages in storage format!"
        return _apply_tidy_regex_rules(self.body, log=log)

    def update(self, body=None, minor=True):
        """Update a page's content."""
        assert self.markup == 'storage', "Cannot update non-storage page markup!"
        if body is None:
            body = self.body
        if body == self._data.body[self.markup].value:
            return  # No changes

        data = {
            #'id': self._data.id,
            'type': 'page',
            'space': {'key': self.space_key},
            'title': self.title,
            'version': dict(number=self.version + 1, minorEdit=minor),
            'body': {
                'storage': {
                    'value': body,
                    'representation': self.markup,
                }
            },
            'expand': 'version',
        }
        response = self.cf.session.put(self._data._links.self, json=data)
        response.raise_for_status()
        ##page = response.json(); print(page)
        result = bunchify(response.json())
        self._data.body[self.markup].value = body
        self._data.version = result.version
        return result

    def dump_diff(self, changed):
        """Dump a diff to terminal between changed and stored body."""
        if self.body == changed:
            click.secho('=== No changes to "{0}"'.format(self.title), fg='green')
            return

        diff = difflib.unified_diff(
            _pretty_xml(self.body, self.markup),
            _pretty_xml(changed, self.markup),
            u'v. {0} of "{1}"'.format(self.version, self.title).encode('utf8'),
            u'v. {0} of "{1}"'.format(self.version + 1, self.title).encode('utf8'),
            arrow.get(self._data.version.when).replace(microsecond=0).isoformat(sep=b' '),
            arrow.now().replace(microsecond=0).isoformat(sep=b' '),
            lineterm='', n=2)
        for line in diff:
            click.secho(line, fg=self.DIFF_COLS.get(line and line[0], None))
<|endoftext|>


<commit_before># coding=utf-8
"""Latent Dirichlet allocation using collapsed Gibbs sampling"""

from __future__ import absolute_import, division, unicode_literals  # noqa
import logging
import sys

import numpy as np

import lda._lda
import lda.utils

logger = logging.getLogger('lda')

PY2 = sys.version_info[0] == 2
if PY2:
    range = xrange

class LDA:
    """Latent Dirichlet allocation using collapsed Gibbs sampling

    Parameters
    ----------
    n_topics : int
        Number of topics

    n_iter : int, default 2000
        Number of sampling iterations

    alpha : float, default 0.1
        Dirichlet parameter for distribution over topics

    eta : float, default 0.01
        Dirichlet parameter for distribution over words

    random_state : int or RandomState, optional
        The generator used for the initial topics.

    Attributes
    ----------
    `components_` : array, shape = [n_topics, n_features]
        Point estimate of the topic-word distributions (Phi in literature)
    `topic_word_` :
        Alias for `components_`
    `nzw_` : array, shape = [n_topics, n_features]
        Matrix of counts recording topic-word assignments in final iteration.
    `ndz_` : array, shape = [n_samples, n_features]
        Matrix of counts recording document-topic assignments in final iteration.
    `doc_topic_` : array, shape = [n_samples, n_features]
        Point estimate of the document-topic distributions (Theta in literature)
    `nz_` : array, shape = [n_topics]
        Array of topic assignment counts in final iteration.

    Examples
    --------
    >>> import numpy
    >>> X = numpy.array([[1,1], [2, 1], [3, 1], [4, 1], [5, 8], [6, 1]])
    >>> import lda
    >>> model = lda.LDA(n_topics=2, random_state=0, n_iter=100)
    >>> model.fit(X) #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
    LDA(alpha=...
    >>> model.components_
    array([[ 0.85714286,  0.14285714],
           [ 0.45      ,  0.55      ]])
    >>> model.loglikelihood() #doctest: +ELLIPSIS
    -40.395...

    References
    ----------
    Blei, David M., Andrew Y. Ng, and Michael I. Jordan. "Latent Dirichlet
    Allocation." Journal of Machine Learning Research 3 (2003): 993–1022.

    Griffiths, Thomas L., and Mark Steyvers. "Finding Scientific Topics."
    Proceedings of the National Academy of Sciences 101 (2004): 5228–5235.
    doi:10.1073/pnas.0307752101.

    Wallach, Hanna, David Mimno, and Andrew McCallum. "Rethinking LDA: Why
    Priors Matter." In Advances in Neural Information Processing Systems 22,
    edited by Y.  Bengio, D. Schuurmans, J. Lafferty, C. K. I. Williams, and A.
    Culotta, 1973–1981, 2009.
    """

    def __init__(self, n_topics, n_iter=2000, alpha=0.1, eta=0.01, random_state=None,
                 refresh=10):
        self.n_topics = n_topics
        self.n_iter = n_iter
        self.alpha = alpha
        self.eta = eta
        # if random_state is None, check_random_state(None) does nothing
        # other than return the current numpy RandomState
        self.random_state = random_state
        self.refresh = refresh

        if alpha <= 0 or eta <= 0:
            raise ValueError("alpha and eta must be greater than zero")

        # random numbers that are reused
        rng = lda.utils.check_random_state(random_state)
        self._rands = rng.rand(1024**2 // 8)  # 1MiB of random variates

        # configure console logging if not already configured
        if len(logger.handlers) == 1 and isinstance(logger.handlers[0], logging.NullHandler):
            logging.basicConfig(level=logging.INFO)

    def fit(self, X, y=None):
        """Fit the model with X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training data, where n_samples in the number of samples
            and n_features is the number of features. Sparse matrix allowed.

        Returns
        -------
        self : object
            Returns the instance itself.
        """
        self._fit(X)
        return self

    def fit_transform(self, X, y=None):
        """Apply dimensionality reduction on X

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            New data, where n_samples in the number of samples
            and n_features is the number of features. Sparse matrix allowed.

        Returns
        -------
        doc_topic : array-like, shape (n_samples, n_topics)
            Point estimate of the document-topic distributions

        """
        if isinstance(X, np.ndarray):
            # in case user passes a (non-sparse) array of shape (n_features,)
            # turn it into an array of shape (1, n_features)
            X = np.atleast_2d(X)
        self._fit(X)
        return self.doc_topic_

    def transform(self, X, max_iter=20, tol=1e-16):
        """Transform the data X according to previously fitted model

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            New data, where n_samples in the number of samples
            and n_features is the number of features.
        max_iter : int, optional
            Maximum number of iterations in iterated-pseudocount estimation.
        tol: double, optional
            Tolerance value used in stopping condition.

        Returns
        -------
        doc_topic : array-like, shape (n_samples, n_topics)
            Point estimate of the document-topic distributions

        Note
        ----
        This uses the "iterated pseudo-counts" approach described
        in Wallach et al. (2009) and discussed in Buntine (2009).

        """
        X = np.atleast_2d(X)
        phi = self.components_
        alpha = self.alpha
        # for debugging, let's not worry about the documents
        n_topics = len(self.components_)
        doc_topic = np.empty((len(X), n_topics))
        WS, DS = lda.utils.matrix_to_lists(X)
        # TODO: this loop is parallelizable
        for d in range(len(X)):
            # initialization step
            ws_doc = WS[DS == d]
            PZS = phi[:, ws_doc].T * alpha
            PZS /= PZS.sum(axis=1)[:, np.newaxis]
            assert PZS.shape == (len(ws_doc), n_topics)
            PZS_new = np.empty_like(PZS)
            for s in range(max_iter):
                PZS_sum = PZS.sum(axis=0)
                for i, w in enumerate(ws_doc):
                    PZS_sum -= PZS[i]
                    PZS_new[i] = phi[:, w] * (PZS_sum + alpha)
                    PZS_sum += PZS[i]
                PZS_new /= PZS_new.sum(axis=1)[:, np.newaxis]
                delta_naive = np.abs(PZS_new - PZS).sum()
                logger.debug('transform iter {}, delta {}'.format(s, delta_naive))
                PZS = PZS_new.copy()
                if delta_naive < tol:
                    break
            theta_doc = PZS.sum(axis=0)
            theta_doc /= sum(theta_doc)
            assert len(theta_doc) == n_topics
            assert theta_doc.shape == (n_topics,)
            doc_topic[d] = theta_doc
        return doc_topic

    def _fit(self, X):
        """Fit the model to the data X

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training vector, where n_samples in the number of samples and
            n_features is the number of features. Sparse matrix allowed.
        """
        random_state = lda.utils.check_random_state(self.random_state)
        rands = self._rands.copy()
        self._initialize(X)
        for it in range(self.n_iter):
            # FIXME: using numpy.roll with a random shift might be faster
            random_state.shuffle(rands)
            if it % self.refresh == 0:
                ll = self.loglikelihood()
                logger.info("<{}> log likelihood: {:.0f}".format(it, ll))
                # keep track of loglikelihoods for monitoring convergence
                self.loglikelihoods_.append(ll)
            self._sample_topics(rands)
        ll = self.loglikelihood()
        logger.info("<{}> log likelihood: {:.0f}".format(self.n_iter - 1, ll))
        self.components_ = self.nzw_ + self.eta
        self.components_ /= np.sum(self.components_, axis=1)[:, np.newaxis]
        self.topic_word_ = self.components_
        self.doc_topic_ = self.ndz_ + self.alpha
        self.doc_topic_ /= np.sum(self.doc_topic_, axis=1)[:, np.newaxis]

        # delete attributes no longer needed after fitting to save memory and reduce clutter
        del self.WS
        del self.DS
        del self.ZS
        return self

    def _initialize(self, X):
        D, W = X.shape
        N = int(X.sum())
        n_topics = self.n_topics
        n_iter = self.n_iter
        logger.info("n_documents: {}".format(D))
        logger.info("vocab_size: {}".format(W))
        logger.info("n_words: {}".format(N))
        logger.info("n_topics: {}".format(n_topics))
        logger.info("n_iter: {}".format(n_iter))

        self.nzw_ = nzw_ = np.zeros((n_topics, W), dtype=np.intc)
        self.ndz_ = ndz_ = np.zeros((D, n_topics), dtype=np.intc)
        self.nz_ = nz_ = np.zeros(n_topics, dtype=np.intc)

        self.WS, self.DS = WS, DS = lda.utils.matrix_to_lists(X)
        self.ZS = ZS = np.empty_like(self.WS, dtype=np.intc)
        np.testing.assert_equal(N, len(WS))
        for i in range(N):
            w, d = WS[i], DS[i]
            z_new = i % n_topics
            ZS[i] = z_new
            ndz_[d, z_new] += 1
            nzw_[z_new, w] += 1
            nz_[z_new] += 1
        self.loglikelihoods_ = []

    def loglikelihood(self):
        """Calculate complete log likelihood, log p(w,z)

        Formula used is log p(w,z) = log p(w|z) + log p(z)
        """
        nzw, ndz, nz = self.nzw_, self.ndz_, self.nz_
        alpha = self.alpha
        eta = self.eta
        nd = np.sum(ndz, axis=1).astype(np.intc)
        return lda._lda._loglikelihood(nzw, ndz, nz, nd, alpha, eta)

    def _sample_topics(self, rands):
        """Samples all topic assignments. Called once per iteration."""
        n_topics, vocab_size = self.nzw_.shape
        alpha = np.repeat(self.alpha, n_topics).astype(np.float64)
        eta = np.repeat(self.eta, vocab_size).astype(np.float64)
        lda._lda._sample_topics(self.WS, self.DS, self.ZS, self.nzw_, self.ndz_, self.nz_,
                                alpha, eta, rands)
<commit_msg>NumPy /= is integer division even in Python 3<commit_after># coding=utf-8
"""Latent Dirichlet allocation using collapsed Gibbs sampling"""

from __future__ import absolute_import, division, unicode_literals  # noqa
import logging
import sys

import numpy as np

import lda._lda
import lda.utils

logger = logging.getLogger('lda')

PY2 = sys.version_info[0] == 2
if PY2:
    range = xrange

class LDA:
    """Latent Dirichlet allocation using collapsed Gibbs sampling

    Parameters
    ----------
    n_topics : int
        Number of topics

    n_iter : int, default 2000
        Number of sampling iterations

    alpha : float, default 0.1
        Dirichlet parameter for distribution over topics

    eta : float, default 0.01
        Dirichlet parameter for distribution over words

    random_state : int or RandomState, optional
        The generator used for the initial topics.

    Attributes
    ----------
    `components_` : array, shape = [n_topics, n_features]
        Point estimate of the topic-word distributions (Phi in literature)
    `topic_word_` :
        Alias for `components_`
    `nzw_` : array, shape = [n_topics, n_features]
        Matrix of counts recording topic-word assignments in final iteration.
    `ndz_` : array, shape = [n_samples, n_features]
        Matrix of counts recording document-topic assignments in final iteration.
    `doc_topic_` : array, shape = [n_samples, n_features]
        Point estimate of the document-topic distributions (Theta in literature)
    `nz_` : array, shape = [n_topics]
        Array of topic assignment counts in final iteration.

    Examples
    --------
    >>> import numpy
    >>> X = numpy.array([[1,1], [2, 1], [3, 1], [4, 1], [5, 8], [6, 1]])
    >>> import lda
    >>> model = lda.LDA(n_topics=2, random_state=0, n_iter=100)
    >>> model.fit(X) #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
    LDA(alpha=...
    >>> model.components_
    array([[ 0.85714286,  0.14285714],
           [ 0.45      ,  0.55      ]])
    >>> model.loglikelihood() #doctest: +ELLIPSIS
    -40.395...

    References
    ----------
    Blei, David M., Andrew Y. Ng, and Michael I. Jordan. "Latent Dirichlet
    Allocation." Journal of Machine Learning Research 3 (2003): 993–1022.

    Griffiths, Thomas L., and Mark Steyvers. "Finding Scientific Topics."
    Proceedings of the National Academy of Sciences 101 (2004): 5228–5235.
    doi:10.1073/pnas.0307752101.

    Wallach, Hanna, David Mimno, and Andrew McCallum. "Rethinking LDA: Why
    Priors Matter." In Advances in Neural Information Processing Systems 22,
    edited by Y.  Bengio, D. Schuurmans, J. Lafferty, C. K. I. Williams, and A.
    Culotta, 1973–1981, 2009.
    """

    def __init__(self, n_topics, n_iter=2000, alpha=0.1, eta=0.01, random_state=None,
                 refresh=10):
        self.n_topics = n_topics
        self.n_iter = n_iter
        self.alpha = alpha
        self.eta = eta
        # if random_state is None, check_random_state(None) does nothing
        # other than return the current numpy RandomState
        self.random_state = random_state
        self.refresh = refresh

        if alpha <= 0 or eta <= 0:
            raise ValueError("alpha and eta must be greater than zero")

        # random numbers that are reused
        rng = lda.utils.check_random_state(random_state)
        self._rands = rng.rand(1024**2 // 8)  # 1MiB of random variates

        # configure console logging if not already configured
        if len(logger.handlers) == 1 and isinstance(logger.handlers[0], logging.NullHandler):
            logging.basicConfig(level=logging.INFO)

    def fit(self, X, y=None):
        """Fit the model with X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training data, where n_samples in the number of samples
            and n_features is the number of features. Sparse matrix allowed.

        Returns
        -------
        self : object
            Returns the instance itself.
        """
        self._fit(X)
        return self

    def fit_transform(self, X, y=None):
        """Apply dimensionality reduction on X

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            New data, where n_samples in the number of samples
            and n_features is the number of features. Sparse matrix allowed.

        Returns
        -------
        doc_topic : array-like, shape (n_samples, n_topics)
            Point estimate of the document-topic distributions

        """
        if isinstance(X, np.ndarray):
            # in case user passes a (non-sparse) array of shape (n_features,)
            # turn it into an array of shape (1, n_features)
            X = np.atleast_2d(X)
        self._fit(X)
        return self.doc_topic_

    def transform(self, X, max_iter=20, tol=1e-16):
        """Transform the data X according to previously fitted model

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            New data, where n_samples in the number of samples
            and n_features is the number of features.
        max_iter : int, optional
            Maximum number of iterations in iterated-pseudocount estimation.
        tol: double, optional
            Tolerance value used in stopping condition.

        Returns
        -------
        doc_topic : array-like, shape (n_samples, n_topics)
            Point estimate of the document-topic distributions

        Note
        ----
        This uses the "iterated pseudo-counts" approach described
        in Wallach et al. (2009) and discussed in Buntine (2009).

        """
        X = np.atleast_2d(X)
        phi = self.components_
        alpha = self.alpha
        # for debugging, let's not worry about the documents
        n_topics = len(self.components_)
        doc_topic = np.empty((len(X), n_topics))
        WS, DS = lda.utils.matrix_to_lists(X)
        # TODO: this loop is parallelizable
        for d in range(len(X)):
            # initialization step
            ws_doc = WS[DS == d]
            PZS = (phi[:, ws_doc].T * alpha).astype(float)
            # NOTE: numpy /= is integer division
            PZS /= PZS.sum(axis=1)[:, np.newaxis]
            assert PZS.shape == (len(ws_doc), n_topics)
            PZS_new = np.empty_like(PZS)
            for s in range(max_iter):
                PZS_sum = PZS.sum(axis=0)
                for i, w in enumerate(ws_doc):
                    PZS_sum -= PZS[i]
                    PZS_new[i] = phi[:, w] * (PZS_sum + alpha)
                    PZS_sum += PZS[i]
                PZS_new /= PZS_new.sum(axis=1)[:, np.newaxis]
                delta_naive = np.abs(PZS_new - PZS).sum()
                logger.debug('transform iter {}, delta {}'.format(s, delta_naive))
                PZS = PZS_new.copy()
                if delta_naive < tol:
                    break
            theta_doc = PZS.sum(axis=0)
            theta_doc /= sum(theta_doc)
            assert len(theta_doc) == n_topics
            assert theta_doc.shape == (n_topics,)
            doc_topic[d] = theta_doc
        return doc_topic

    def _fit(self, X):
        """Fit the model to the data X

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training vector, where n_samples in the number of samples and
            n_features is the number of features. Sparse matrix allowed.
        """
        random_state = lda.utils.check_random_state(self.random_state)
        rands = self._rands.copy()
        self._initialize(X)
        for it in range(self.n_iter):
            # FIXME: using numpy.roll with a random shift might be faster
            random_state.shuffle(rands)
            if it % self.refresh == 0:
                ll = self.loglikelihood()
                logger.info("<{}> log likelihood: {:.0f}".format(it, ll))
                # keep track of loglikelihoods for monitoring convergence
                self.loglikelihoods_.append(ll)
            self._sample_topics(rands)
        ll = self.loglikelihood()
        logger.info("<{}> log likelihood: {:.0f}".format(self.n_iter - 1, ll))
        # note: numpy /= is integer division
        self.components_ = (self.nzw_ + self.eta).astype(float)
        self.components_ /= np.sum(self.components_, axis=1)[:, np.newaxis]
        self.topic_word_ = self.components_
        self.doc_topic_ = (self.ndz_ + self.alpha).astype(float)
        self.doc_topic_ /= np.sum(self.doc_topic_, axis=1)[:, np.newaxis]

        # delete attributes no longer needed after fitting to save memory and reduce clutter
        del self.WS
        del self.DS
        del self.ZS
        return self

    def _initialize(self, X):
        D, W = X.shape
        N = int(X.sum())
        n_topics = self.n_topics
        n_iter = self.n_iter
        logger.info("n_documents: {}".format(D))
        logger.info("vocab_size: {}".format(W))
        logger.info("n_words: {}".format(N))
        logger.info("n_topics: {}".format(n_topics))
        logger.info("n_iter: {}".format(n_iter))

        self.nzw_ = nzw_ = np.zeros((n_topics, W), dtype=np.intc)
        self.ndz_ = ndz_ = np.zeros((D, n_topics), dtype=np.intc)
        self.nz_ = nz_ = np.zeros(n_topics, dtype=np.intc)

        self.WS, self.DS = WS, DS = lda.utils.matrix_to_lists(X)
        self.ZS = ZS = np.empty_like(self.WS, dtype=np.intc)
        np.testing.assert_equal(N, len(WS))
        for i in range(N):
            w, d = WS[i], DS[i]
            z_new = i % n_topics
            ZS[i] = z_new
            ndz_[d, z_new] += 1
            nzw_[z_new, w] += 1
            nz_[z_new] += 1
        self.loglikelihoods_ = []

    def loglikelihood(self):
        """Calculate complete log likelihood, log p(w,z)

        Formula used is log p(w,z) = log p(w|z) + log p(z)
        """
        nzw, ndz, nz = self.nzw_, self.ndz_, self.nz_
        alpha = self.alpha
        eta = self.eta
        nd = np.sum(ndz, axis=1).astype(np.intc)
        return lda._lda._loglikelihood(nzw, ndz, nz, nd, alpha, eta)

    def _sample_topics(self, rands):
        """Samples all topic assignments. Called once per iteration."""
        n_topics, vocab_size = self.nzw_.shape
        alpha = np.repeat(self.alpha, n_topics).astype(np.float64)
        eta = np.repeat(self.eta, vocab_size).astype(np.float64)
        lda._lda._sample_topics(self.WS, self.DS, self.ZS, self.nzw_, self.ndz_, self.nz_,
                                alpha, eta, rands)
<|endoftext|>