<commit_before>import random as _random

from .perlin import noise, noiseSeed

# Random

def random(*args):
    """Generates random numbers.

    Each time the random() function is called, it returns an unexpected value
    within the specified range. If only one parameter is passed to the function,
    it will return a float between zero and the value of the high parameter.
    For example, random(5) returns values between 0 and 5 (starting at zero,
    and up to, but not including, 5).

    If two parameters are specified, the function will return a float with a
    value between the two values. For example, random(-5, 10.2) returns values
    starting at -5 and up to (but not including) 10.2. To convert a
    floating-point random number to an integer, use the int() function.

    Arguments:
    low -- lower limit
    high -- upper limit

    Returns:
    float

    """
    low = args[0] if len(args) > 1 else 0
    high = args[1] if len(args) > 1 else args[0]
    offset = _random.random() * (high - low)
    return low + offset

def random_gaussian():
    """Returns a float from a random series of numbers.

    Each time the randomGaussian() function is called, it returns a number
    fitting a Gaussian, or normal, distribution having a mean of 0 and a
    standard deviation of 1. There is theoretically no minimum or maximum value
    that randomGaussian() might return. Rather, there is just a very low
    probability that values far from the mean will be returned; and a higher
    probability that numbers near the mean will be returned.

    """
    return _random.gauss(0, 1)

def random_seed(seed):
    """Sets the seed value for random().

    By default, random() produces different results each time the program is
    run. Set the seed parameter to a constant to return the same pseudo-random
    numbers each time the software is run.

    Arguments:
    seed -- seed value

    """
    _random.seed(seed)

# TODO: Make sure this matches the one in Processing
def constrain(amt, low, high):
    """Constrains a value to not exceed a maximum and minimum value.
    """
    if amt < low:
        return low
    elif amt > high:
        return high
    else:
        return amt

def map(value, start1, stop1, start2, stop2):
    """Re-maps a number from one range to another.
    """
    return (value - start1) / float(stop1 - start1) * (stop2 - start2) + start2

# def noise(*args):
#     """Returns the Perlin noise value at specified coordinates.

#     Perlin noise is a random sequence generator producing a more natural,
#     harmonic succession of numbers than that of the standard random() function.
#     It was invented by Ken Perlin in the 1980s and has been used in graphical
#     applications to generate procedural textures, shapes, terrains, and other
#     seemingly organic forms.

#     In contrast to the random() function, Perlin noise is defined in an infinite
#     n-dimensional space, in which each pair of coordinates corresponds to a
#     fixed semi-random value (fixed only for the lifespan of the program). The
#     resulting value will always be between 0.0 and 1.0. Processing can compute
#     1D, 2D and 3D noise, depending on the number of coordinates given. The noise
#     value can be animated by moving through the noise space, as demonstrated in
#     the first example above. The 2nd and 3rd dimensions can also be interpreted
#     as time.

#     The actual noise structure is similar to that of an audio signal, in respect
#     to the function's use of frequencies. Similar to the concept of harmonics in
#     physics, Perlin noise is computed over several octaves which are added
#     together for the final result.

#     Another way to adjust the character of the resulting sequence is the scale
#     of the input coordinates. As the function works within an infinite space,
#     the value of the coordinates doesn't matter as such; only the distance
#     between successive coordinates is important (such as when using noise()
#     within a loop). As a general rule, the smaller the difference between
#     coordinates, the smoother the resulting noise sequence. Steps of 0.005-0.03
#     work best for most applications, but this will differ depending on use.

#     """
#     if len(args) == 1:
#         return pnoise1(*args)
#     if len(args) == 2:
#         return pnoise2(*args)
#     if len(args) == 3:
#         return pnoise3(*args)

<commit_msg>Added a few more processing functions<commit_after>import random as _random
from math import sqrt

from .perlin import noise, noiseSeed

# Random

def random(*args):
    """Generates random numbers.

    Each time the random() function is called, it returns an unexpected value
    within the specified range. If only one parameter is passed to the function,
    it will return a float between zero and the value of the high parameter.
    For example, random(5) returns values between 0 and 5 (starting at zero,
    and up to, but not including, 5).

    If two parameters are specified, the function will return a float with a
    value between the two values. For example, random(-5, 10.2) returns values
    starting at -5 and up to (but not including) 10.2. To convert a
    floating-point random number to an integer, use the int() function.

    Arguments:
    low -- lower limit
    high -- upper limit

    Returns:
    float

    """
    low = args[0] if len(args) > 1 else 0
    high = args[1] if len(args) > 1 else args[0]
    offset = _random.random() * (high - low)
    return low + offset

def random_gaussian():
    """Returns a float from a random series of numbers.

    Each time the randomGaussian() function is called, it returns a number
    fitting a Gaussian, or normal, distribution having a mean of 0 and a
    standard deviation of 1. There is theoretically no minimum or maximum value
    that randomGaussian() might return. Rather, there is just a very low
    probability that values far from the mean will be returned; and a higher
    probability that numbers near the mean will be returned.

    """
    return _random.gauss(0, 1)

def random_seed(seed):
    """Sets the seed value for random().

    By default, random() produces different results each time the program is
    run. Set the seed parameter to a constant to return the same pseudo-random
    numbers each time the software is run.

    Arguments:
    seed -- seed value

    """
    _random.seed(seed)

# TODO: Make sure this matches the one in Processing
def constrain(amt, low, high):
    """Constrains a value to not exceed a maximum and minimum value.
    """
    if amt < low:
        return low
    elif amt > high:
        return high
    else:
        return amt

def map(value, start1, stop1, start2, stop2):
    """Re-maps a number from one range to another.
    """
    return (value - start1) / float(stop1 - start1) * (stop2 - start2) + start2

def dist(*args):
    """Returns the distance between two points (2D or 3D).
    """
    step = len(args)/2
    return sqrt(sum((args[i] - args[i+step])**2 for i in range(step)))

def sq(num):
    """Returns the square of a number
    """
    return num**2

# def noise(*args):
#     """Returns the Perlin noise value at specified coordinates.

#     Perlin noise is a random sequence generator producing a more natural,
#     harmonic succession of numbers than that of the standard random() function.
#     It was invented by Ken Perlin in the 1980s and has been used in graphical
#     applications to generate procedural textures, shapes, terrains, and other
#     seemingly organic forms.

#     In contrast to the random() function, Perlin noise is defined in an infinite
#     n-dimensional space, in which each pair of coordinates corresponds to a
#     fixed semi-random value (fixed only for the lifespan of the program). The
#     resulting value will always be between 0.0 and 1.0. Processing can compute
#     1D, 2D and 3D noise, depending on the number of coordinates given. The noise
#     value can be animated by moving through the noise space, as demonstrated in
#     the first example above. The 2nd and 3rd dimensions can also be interpreted
#     as time.

#     The actual noise structure is similar to that of an audio signal, in respect
#     to the function's use of frequencies. Similar to the concept of harmonics in
#     physics, Perlin noise is computed over several octaves which are added
#     together for the final result.

#     Another way to adjust the character of the resulting sequence is the scale
#     of the input coordinates. As the function works within an infinite space,
#     the value of the coordinates doesn't matter as such; only the distance
#     between successive coordinates is important (such as when using noise()
#     within a loop). As a general rule, the smaller the difference between
#     coordinates, the smoother the resulting noise sequence. Steps of 0.005-0.03
#     work best for most applications, but this will differ depending on use.

#     """
#     if len(args) == 1:
#         return pnoise1(*args)
#     if len(args) == 2:
#         return pnoise2(*args)
#     if len(args) == 3:
#         return pnoise3(*args)
<|endoftext|>


<commit_before>import argparse
import threading
from copy import deepcopy
from os import system

import numpy as np
import cv2
from cv2 import cv
from cv_bridge import CvBridge

import rospy
import baxter_interface

from geometry_msgs.msg import (
  PolygonStamped,
)

from sensor_msgs.msg import (
  Image,
)

from std_msgs.msg import (
  String,
)

# TODO: Document all the things.

class LPVision(object):
    def __init__(self, limb):
        # Start camera
        self._side = limb
        self._camera_name = self._side + "_hand_camera"
        print ("Opening " + self._camera_name + "...")
        self._camera = baxter_interface.CameraController(self._camera_name)
        self._camera.open()
        self._camera.resolution = [1280, 800]
        self._camera.gain = 20

        self._side_roi = 400
        self._no_squares = 8
        # 400 / 8 = 50
        self._square_side_roi = self._side_roi / self._no_squares

        self.grid = [[0 for _i in range(self._no_squares)] for _j in range(self._no_squares)]
        print self.grid
        self.cv_image = None
        self._bridge = CvBridge()

        # self.yellow_sample = ()
        self.red_sample = ()
        self.blue_sample = ()

        # self.yellows = None
        self.reds = None
        self.blues = None

        # more central
        # roi = region of interest
        self._roi_points = [[100, 200], [100, 100], [200, 100], [200, 200]] # magic
        self._roi_move = False
        self._point_selected = -1
        self._gain_slider = 30
        self._red_thresh = 30
        # self._yellow_thresh = 100
        self._slider_time = rospy.Time.now()
        self._gain_set = False
        self._text = ['X', 'Y', 'R', 'G', 'B']
        self._pixel = {}
        # initialize data structure
        for label in self._text:
            self._pixel[label] = 0.0
        self._vector = {}

        self._grid = [[0 for _i in range(self._no_squares)] for _j in range(self._no_squares)]
        self._pnts = [[0 for i in range(self._square_side_roi + 1)] for j in range(self._square_side_roi + 1)]

        self._pieces_positions = []

        # initialize images
        self._np_image = np.zeros((self._side_roi, self._side_roi, 3), np.uint8)
        self._image_grid = np.zeros((self._side_roi, self._side_roi, 3), np.uint8)
        # self._yellow = np.zeros((self._side_roi, self._side_roi), np.uint8)
        self._red = np.zeros((self._side_roi, self._side_roi), np.uint8)
        self._projected = np.zeros((self._side_roi, self._side_roi, 3), np.uint8)

        self.subLock = threading.Lock()

        camera_topic = '/cameras/' + self._camera_name + '/image'
        _camera_sub = rospy.Subscriber(
            camera_topic,
            Image,
            self._on_camera)

        roi_topic = '/learn_play/localize/grid_pixels'
        _roi_sub = rospy.Subscriber(
            roi_topic,
            PolygonStamped,
            self._on_roi)

        board_state_topic = '/vision/learn_play_state'
        self._board_state_pub = rospy.Publisher(
            board_state_topic,
            String)

        print ' - All set - '
        print " - - - - - - - "
        self._process_images()

    def _process_images(self):

        print " - Starting to process images! - "
        while not rospy.is_shutdown():

            # gain changed from slider settled
            if (rospy.Time.now() - self._slider_time > rospy.Duration(2.0)
                and self._gain_set == True):
                self._gain_set = False
                print 'Setting GAIN!'
                self._camera.gain = self._gain_slider

            # process red/yellow image
            self._show_image()
            self._project_roi()
            # self._filter_yellow()
            self._filter_red()
            self._process_colors(deepcopy(self._red))
            self._update_image_grid()

            # publish state
            self._pub_state()
            rospy.sleep(0.1)

    def _show_image(self):

        self.subLock.acquire(True)
        local_image = deepcopy(self._np_image)
        self.subLock.release()

        # draw circles
        for idx, points in enumerate(self._roi_points):
            cv2.circle(local_image, (points[0], points[1]), 5, (255, 0, 0), 2)

        # draw green lines
        cv2.polylines(local_image, np.int32([np.array(self._roi_points)]),
                      1, (0, 255, 0), 2)

        cv.ShowImage("Learn Play game RGB", cv.fromarray(local_image))

        cv.SetMouseCallback("Learn Play game RGB", self._on_mouse_click, 0)
        cv.CreateTrackbar("Gain", "Learn Play game RGB", self._gain_slider,
                          100, self._on_gain_slider)
        cv.CreateTrackbar("Red Threshold", "Learn Play game RGB",
                          self._red_thresh, 500, self._on_red_slider)
        # TODO: create HSV bar for colour
        cv.WaitKey(3)

        # projects roi chosen by user

    def _project_roi(self):
        warped_in = np.float32([np.array(self._roi_points)])
        project_out = np.float32([[0, 0], [self._side_roi, 0], [self._side_roi, self._side_roi], [0, self._side_roi]])
        M = cv2.getPerspectiveTransform(warped_in, project_out)
        self.subLock.acquire(True)
        local_image = deepcopy(self._np_image)
        self.subLock.release()
        self._projected = cv2.warpPerspective(local_image, M, (self._side_roi, self._side_roi))
        # ENHANCE! *puts glasses on*
        self._blurred = cv2.GaussianBlur(self._projected, (0,0), 3)
        self._projected = cv2.addWeighted(self._projected, 1.5, self._blurred, -0.5, 0, self._projected)

    def _filter_red(self):
        # Finds red colors in HSV space
        hsv = cv2.cvtColor(self._projected, cv2.COLOR_BGR2HSV)
        # lower_red = np.array([165, 60, 60])
        # upper_red = np.array([180, 255, 255])
        lower_orange = np.array([3, 50, 50]) # TODO: those suck.
        upper_orange = np.array([16, 255, 255])
        self._red = cv2.inRange(hsv, lower_orange, upper_orange)
        cv.ShowImage('Orange', cv.fromarray(self._red))
        # print rospy.Time.now()

    def _process_colors(self, red):
        # look down each column building up from bottom
        self._grid = [[0 for _i in range(self._no_squares)] for _j in range(self._no_squares)]
        # print self._grid
        # print '----------'
        self._image_grid = deepcopy(self._projected)
        # print self._image_grid
        self._pieces_positions = []
        for col in xrange(self._no_squares):
            cur_row = True
            x_offset = self._square_side_roi * col
            # print "x = ", x_offset
            # Look from the bottom up checking if piece is there
            for row in xrange(self._no_squares):
                if cur_row == True: # runs first time
                    y_offset = self._square_side_roi * row
                    # print "y = ", y_offset
                    red_cnt = 0
                    # yellow_cnt = 0
                    # look though each pixel in current grid location
                    if len(red) != self._side_roi:
                        print 'BAILING - IMAGE SIZE IS UNEXPECTED'
                        return

                    for y in xrange(0, self._square_side_roi, 2):
                        for x in xrange(0, self._square_side_roi, 2):
                            if red[y + y_offset, x + x_offset] == 255:
                                red_cnt += 1
                                if red_cnt > self._red_thresh: # Speed tweak
                                    cv2.putText(self._image_grid,
                                    'o',
                                    (x_offset + 20, y_offset + 40),
                                    cv2.FONT_HERSHEY_COMPLEX_SMALL,
                                    2,
                                    (0, 0, 255))
                                    self._grid[row][col] = 1
                                    self._pieces_positions.append((row,col))
                                    break
                        if red_cnt > self._red_thresh:
                            break # (sigh)
                                # else:
                                #     # print "what"
                                #     cur_row = False
                                #     break

    def _update_image_grid(self):
        for idx in xrange(1, self._no_squares):
            cv2.line(self._image_grid, (self._square_side_roi * idx, 0), (self._square_side_roi * idx, self._side_roi),
                     (0, 255, 0), 1)
            cv2.line(self._image_grid, (0, self._square_side_roi * idx), (self._side_roi, self._square_side_roi * idx),
                     (0, 255, 0), 1)
            cv2.line(self._image_grid, (self._square_side_roi * self._no_squares, 0), (self._square_side_roi * self._no_squares, self._side_roi),
                     (0, 255, 0), 1)
            cv.ShowImage('Board State', cv.fromarray(self._image_grid))

    def _transform_positions(self, positions):
        return [(y, 7 - x) for (x,y) in positions]

    def _pub_state(self):
        state = dict()
        self._pieces_positions = self._transform_positions(self._pieces_positions)
        print rospy.Time.now(), " - ", self._pieces_positions
        state['baxter_count'] = self._pieces_positions
        # state['user_count'] = self._user_cnt
        state['board'] = self._grid
        self._board_state_pub.publish(str(state))

    def _on_roi(self, data):
        if data.polygon.points:
            for idx, point in enumerate(data.polygon.points):
                self._roi_points[3 - idx] = [int(point.x), int(point.y)]

    def _on_camera(self, data):
        try:
            self.cv_image = self._bridge.imgmsg_to_cv(data, "bgr8")
            local_image = np.asarray(self.cv_image)
        except Exception:
            print 'Cannot get image from Baxter'

        self.subLock.acquire(True)
        self._np_image = deepcopy(local_image)
        self.subLock.release()

    def _on_gain_slider(self, pos):
        self._gain_slider = pos
        self._gain_set = True
        self._slider_time = rospy.Time.now()

    def _on_red_slider(self, pos):
        self._red_thresh = pos

    def _on_mouse_click(self, event, x, y, flags, param):
        if event == cv.CV_EVENT_LBUTTONDOWN:
            width, height = cv.GetSize(self.cv_image)
            for idx, points in enumerate(self._roi_points):
                if (x <= points[0] + 5 and x >= points[0] - 5
                    and y <= points[1] + 5 and y >= points[1] - 5):
                    self._roi_move = True
                    self._point_selected = idx

        elif event == cv.CV_EVENT_MOUSEMOVE and self._roi_move:
            self._roi_points[self._point_selected] = [x, y]

        elif event == cv.CV_EVENT_LBUTTONUP and self._roi_move:
            self._roi_move = False
<commit_msg>clean vision<commit_after>import argparse
import threading
from copy import deepcopy
from os import system

import numpy as np
import cv2
from cv2 import cv
from cv_bridge import CvBridge

import rospy
import baxter_interface

from geometry_msgs.msg import (
  PolygonStamped,
)

from sensor_msgs.msg import (
  Image,
)

from std_msgs.msg import (
  String,
)

# TODO: Document all the things.

class LPVision(object):
    def __init__(self, limb):
        # Start camera
        self._side = limb
        self._camera_name = self._side + "_hand_camera"
        print ("Opening " + self._camera_name + "...")
        self._camera = baxter_interface.CameraController(self._camera_name)
        self._camera.open()
        self._camera.resolution = [1280, 800]
        self._camera.gain = 20

        self._side_roi = 400
        self._no_squares = 8
        # 400 / 8 = 50
        self._square_side_roi = self._side_roi / self._no_squares

        self.grid = [[0 for _i in range(self._no_squares)] for _j in range(self._no_squares)]
        print self.grid
        self.cv_image = None
        self._bridge = CvBridge()

        # self.yellow_sample = ()
        self.red_sample = ()
        self.blue_sample = ()

        # self.yellows = None
        self.reds = None
        self.blues = None

        # more central
        # roi = region of interest
        self._roi_points = [[100, 200],
                            [100, 100],
                            [200, 100],
                            [200, 200]] # magic
        self._roi_move = False
        self._point_selected = -1
        self._gain_slider = 30
        self._red_thresh = 30
        # self._yellow_thresh = 100
        self._slider_time = rospy.Time.now()
        self._gain_set = False
        self._text = ['X', 'Y', 'R', 'G', 'B']
        self._pixel = {}
        # initialize data structure
        for label in self._text:
            self._pixel[label] = 0.0
        self._vector = {}

        self._grid = [[0 for _i in range(self._no_squares)] for _j in range(self._no_squares)]
        self._pnts = [[0 for i in range(self._square_side_roi + 1)] for j in range(self._square_side_roi + 1)]

        self._pieces_positions = []

        # initialize images
        self._np_image = np.zeros((self._side_roi, self._side_roi, 3), np.uint8)
        self._image_grid = np.zeros((self._side_roi, self._side_roi, 3), np.uint8)
        self._red = np.zeros((self._side_roi, self._side_roi), np.uint8)
        self._projected = np.zeros((self._side_roi, self._side_roi, 3), np.uint8)

        self.subLock = threading.Lock()

        camera_topic = '/cameras/' + self._camera_name + '/image'
        _camera_sub = rospy.Subscriber(
            camera_topic,
            Image,
            self._on_camera)

        roi_topic = '/learn_play/localize/grid_pixels'
        _roi_sub = rospy.Subscriber(
            roi_topic,
            PolygonStamped,
            self._on_roi)

        board_state_topic = '/vision/learn_play_state'
        self._board_state_pub = rospy.Publisher(
            board_state_topic,
            String)

        print ' - All set - '
        print " - - - - - - - "
        self._process_images()

    def _process_images(self):

        print " - Starting to process images! - "
        while not rospy.is_shutdown():

            # gain changed from slider settled
            if (rospy.Time.now() - self._slider_time > rospy.Duration(2.0)
                and self._gain_set == True):
                self._gain_set = False
                print 'Setting GAIN!'
                self._camera.gain = self._gain_slider

            # process red/yellow image
            self._show_image()
            self._project_roi()
            # self._filter_yellow()
            self._filter_red()
            self._process_colors(deepcopy(self._red))
            self._update_image_grid()

            # publish state
            self._pub_state()
            rospy.sleep(0.1)

    def _show_image(self):

        self.subLock.acquire(True)
        local_image = deepcopy(self._np_image)
        self.subLock.release()

        # draw circles
        for idx, points in enumerate(self._roi_points):
            cv2.circle(local_image, (points[0], points[1]), 5, (255, 0, 0), 2)

        # draw green lines
        cv2.polylines(local_image, np.int32([np.array(self._roi_points)]),
                      1, (0, 255, 0), 2)

        cv.ShowImage("Learn Play game RGB", cv.fromarray(local_image))

        cv.SetMouseCallback("Learn Play game RGB", sel_on_mouse_click, 0)
        cv.CreateTrackbar("Gain", "Learn Play game RGB", self._gain_slider,
                          100, self._on_gain_slider)
        cv.CreateTrackbar("Red Threshold", "Learn Play game RGB",
                          self._red_thresh, 500, self._on_red_slider)
        # TODO: create HSV bar for colour
        cv.WaitKey(3)

        # projects roi chosen by user

    def _project_roi(self):
        warped_in = np.float32([np.array(self._roi_points)])
        project_out = np.float32([[0, 0],
                                  [self._side_roi, 0],
                                  [self._side_roi, self._side_roi],
                                  [0, self._side_roi]])
        M = cv2.getPerspectiveTransform(warped_in, project_out)
        self.subLock.acquire(True)
        local_image = deepcopy(self._np_image)
        self.subLock.release()
        self._projected = cv2.warpPerspective(local_image,
                                              M,
                                              (self._side_roi,
                                               self._side_roi))
        # ENHANCE! *puts glasses on*
        self._blurred = cv2.GaussianBlur(self._projected, (0,0), 3)
        self._projected = cv2.addWeighted(self._projected,
                                          1.5,
                                          self._blurred,
                                          -0.5,
                                          0,
                                          self._projected)

    def _filter_red(self):
        # Finds red colors in HSV space
        hsv = cv2.cvtColor(self._projected, cv2.COLOR_BGR2HSV)
        # lower_red = np.array([165, 60, 60])
        # upper_red = np.array([180, 255, 255])
        lower_orange = np.array([3, 50, 50]) # TODO: those suck.
        upper_orange = np.array([16, 255, 255])
        self._red = cv2.inRange(hsv, lower_orange, upper_orange)
        cv.ShowImage('Orange', cv.fromarray(self._red))
        # print rospy.Time.now()

    def _process_colors(self, red):
        # look down each column building up from bottom
        self._grid = [[0 for _i in range(self._no_squares)]
                      for _j in range(self._no_squares)]
        # print self._grid
        # print '----------'
        self._image_grid = deepcopy(self._projected)
        # print self._image_grid
        self._pieces_positions = []
        for col in xrange(self._no_squares):
            cur_row = True
            x_offset = self._square_side_roi * col
            # print "x = ", x_offset
            # Look from the bottom up checking if piece is there
            for row in xrange(self._no_squares):
                if cur_row == True: # runs first time
                    y_offset = self._square_side_roi * row
                    # print "y = ", y_offset
                    red_cnt = 0
                    # look though each pixel in current grid location
                    if len(red) != self._side_roi:
                        print 'BAILING - IMAGE SIZE IS UNEXPECTED'
                        return

                    for y in xrange(0, self._square_side_roi, 2):
                        for x in xrange(0, self._square_side_roi, 2):
                            if red[y + y_offset, x + x_offset] == 255:
                                red_cnt += 1
                                if red_cnt > self._red_thresh: # Speed tweak
                                    cv2.putText(self._image_grid,
                                    'o',
                                    (x_offset + 20, y_offset + 40),
                                    cv2.FONT_HERSHEY_COMPLEX_SMALL,
                                    2,
                                    (0, 0, 255))
                                    self._grid[row][col] = 1
                                    self._pieces_positions.append((row,col))
                                    break
                        if red_cnt > self._red_thresh:
                            break # (sigh)
                                # else:
                                #     # print "what"
                                #     cur_row = False
                                #     break

    def _update_image_grid(self):
        for idx in xrange(1, self._no_squares):
            cv2.line(self._image_grid,
                     (self._square_side_roi * idx, 0),
                     (self._square_side_roi * idx, self._side_roi),
                     (0, 255, 0),
                     1)
            cv2.line(self._image_grid, (0, self._square_side_roi * idx), (self._side_roi, self._square_side_roi * idx),
                     (0, 255, 0), 1)
            cv2.line(self._image_grid,
                     (self._square_side_roi * self._no_squares, 0),
                     (self._square_side_roi * self._no_squares, self._side_roi),
                     (0, 255, 0),
                     1)
            cv.ShowImage('Board State', cv.fromarray(self._image_grid))

    def _transform_positions(self, positions):
        return [(y, 7 - x) for (x,y) in positions]

    def _pub_state(self):
        state = dict()
        self._pieces_positions = self._transform_positions(self._pieces_positions)
        print rospy.Time.now(), " - ", self._pieces_positions
        state['baxter_count'] = self._pieces_positions
        # state['user_count'] = self._user_cnt
        state['board'] = self._grid
        self._board_state_pub.publish(str(state))

    def _on_roi(self, data):
        if data.polygon.points:
            for idx, point in enumerate(data.polygon.points):
                self._roi_points[3 - idx] = [int(point.x), int(point.y)]

    def _on_camera(self, data):
        try:
            self.cv_image = self._bridge.imgmsg_to_cv(data, "bgr8")
            local_image = np.asarray(self.cv_image)
        except Exception:
            print 'Cannot get image from Baxter'

        self.subLock.acquire(True)
        self._np_image = deepcopy(local_image)
        self.subLock.release()

    def _on_gain_slider(self, pos):
        self._gain_slider = pos
        self._gain_set = True
        self._slider_time = rospy.Time.now()

    def _on_red_slider(self, pos):
        self._red_thresh = pos

    def _on_mouse_click(self, event, x, y, flags, param):
        if event == cv.CV_EVENT_LBUTTONDOWN:
            width, height = cv.GetSize(self.cv_image)
            for idx, points in enumerate(self._roi_points):
                if (x <= points[0] + 5 and x >= points[0] - 5
                    and y <= points[1] + 5 and y >= points[1] - 5):
                    self._roi_move = True
                    self._point_selected = idx

        elif event == cv.CV_EVENT_MOUSEMOVE and self._roi_move:
            self._roi_points[self._point_selected] = [x, y]

        elif event == cv.CV_EVENT_LBUTTONUP and self._roi_move:
            self._roi_move = False
<|endoftext|>


<commit_before>LOGIN = "http://localhost:8080/"
<commit_msg>Add uri constants<commit_after>BASE = "http://localhost:8080/"
DASHBOARD = "dashboard"
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2015-2016 Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

"""
topology_docker base node module.
"""

from __future__ import unicode_literals, absolute_import
from __future__ import print_function, division

from os import getpid
from json import loads
from logging import getLogger
from datetime import datetime
from shlex import split as shsplit
from subprocess import check_output
from abc import ABCMeta, abstractmethod

from docker import Client
from six import add_metaclass

from topology.platforms.base import CommonNode
from topology_docker.utils import ensure_dir

log = getLogger(__name__)

@add_metaclass(ABCMeta)
class DockerNode(CommonNode):
    """
    An instance of this class will create a detached Docker container.

    This node binds the ``shared_dir_mount`` directory of the container to a
    local path in the host system defined in ``self.shared_dir``.

    :param str identifier: Node unique identifier in the topology being built.
    :param str image: The image to run on this node, in the
     form ``repository:tag``.
    :param str registry: Docker registry to pull image from.
    :param str command: The command to run when the container is brought up.
    :param str binds: Directories to bind for this container separated by a
     ``;`` in the form:

     ::

        '/tmp:/tmp;/dev/log:/dev/log;/sys/fs/cgroup:/sys/fs/cgroup'

    :param str network_mode: Network mode for this container.
    :param str container_shared_dir: Mount point of the shared directory in the
     container.

    Read only public attributes:

    :var image: Name of the Docker image being used by this node.
     Same as the ``image`` keyword argument.
    :var container_id: Unique container identifier assigned by the Docker
     daemon in the form of a hash.
    :var container_name: Unique container name assigned by the framework in the
     form ``{identifier}_{pid}_{timestamp}``.
    :var shared_dir: Share directory in the host for this container. Always
     ``/tmp/topology/{container_name}``.
    :var shared_dir_mount: Directory inside the container where the
     ``shared_dir`` is mounted. Same as the ``shared_dir_mount`` keyword
     argument.
    """

    @abstractmethod
    def __init__(
            self, identifier,
            image='ubuntu:latest', registry=None, command='bash',
            binds=None, network_mode='none', hostname=None,
            shared_dir_mount='/var/topology',
            **kwargs):

        super(DockerNode, self).__init__(identifier, **kwargs)

        self._pid = None
        self._image = image
        self._registry = registry
        self._command = command
        self._hostname = hostname
        self._client = Client(version='auto')

        self._container_name = '{identifier}_{pid}_{timestamp}'.format(
            identifier=identifier, pid=getpid(),
            timestamp=datetime.now().isoformat().replace(':', '-')
        )
        self._shared_dir_mount = shared_dir_mount
        self._shared_dir = '/tmp/topology/{}'.format(self._container_name)

        # Autopull docker image if necessary
        self._autopull()

        # Create shared directory
        ensure_dir(self._shared_dir)

        # Add binded directories
        container_binds = [
            '{}:{}'.format(
                self._shared_dir,
                self._shared_dir_mount
            )
        ]
        if binds is not None:
            container_binds.extend(binds.split(';'))

        # Create host config
        self._host_config = self._client.create_host_config(
            # Container is given access to all devices
            privileged=True,
            # Avoid connecting to host bridge, usually docker0
            network_mode=network_mode,
            binds=container_binds
        )

        # Create container
        self._container_id = self._client.create_container(
            image=self._image,
            command=self._command,
            name=self._container_name,
            detach=True,
            tty=True,
            hostname=self._hostname,
            host_config=self._host_config
        )['Id']

    @property
    def image(self):
        return self._image

    @property
    def container_id(self):
        return self._container_id

    @property
    def container_name(self):
        return self._container_name

    @property
    def shared_dir(self):
        return self._shared_dir

    @property
    def shared_dir_mount(self):
        return self._shared_dir_mount

    def _autopull(self):
        """
        Autopulls the docker image of the node, if necessary.
        """
        # Search for image in available images
        for tags in [img['RepoTags'] for img in self._client.images()]:
            if self._image in tags:
                return

        # Determine image parts
        registry = self._registry
        image = self._image
        tag = 'latest'

        if ':' in image:
            image, tag = image.split(':')

        # Pull image
        pull_uri = image
        if registry:
            pull_uri = '{}/{}'.format(registry, image)
        pull_name = '{}:{}'.format(pull_uri, tag)

        log.info('Trying to pull image {} ...'.format(pull_name))

        last = ''
        for line in self._client.pull(pull_uri, tag=tag, stream=True):
            last = line
        status = loads(last.decode('utf8'))

        log.debug('Pulling result :: {}'.format(status))

        if 'error' in status:
            raise Exception(status['error'])

        # Retag if required
        if pull_name != self._image:
            if not self._client.tag(pull_name, image, tag):
                raise Exception(
                    'Error when tagging image {} with tag {}:{}'.format(
                        pull_name, image, tag
                    )
                )

            log.info(
                'Tagged image {} with tag {}:{}'.format(
                    pull_name, image, tag
                )
            )

    def notify_add_biport(self, node, biport):
        """
        Get notified that a new biport was added to this engine node.

        :param node: The specification node that spawn this engine node.
        :type node: pynml.nml.Node
        :param biport: The specification bidirectional port added.
        :type biport: pynml.nml.BidirectionalPort
        :rtype: str
        :return: The assigned interface name of the port.
        """
        return biport.metadata.get('label', biport.identifier)

    def notify_add_bilink(self, nodeport, bilink):
        """
        Get notified that a new bilink was added to a port of this engine node.

        :param nodeport: A tuple with the specification node and port being
         linked.
        :type nodeport: (pynml.nml.Node, pynml.nml.BidirectionalPort)
        :param bilink: The specification bidirectional link added.
        :type bilink: pynml.nml.BidirectionalLink
        """

    def notify_post_build(self):
        """
        Get notified that the post build stage of the topology build was
        reached.
        """
        # Log container data
        image_data = self._client.inspect_image(
            image=self._image
        )
        log.info(
            'Started container {}:\n'
            '    Image name: {}\n'
            '    Image id: {}\n'
            '    Image creation date: {}'
            '    Image tags: {}'.format(
                self._container_name,
                self._image,
                image_data.get('Id', '????'),
                image_data.get('Created', '????'),
                ', '.join(image_data.get('RepoTags', []))
            )
        )
        container_data = self._client.inspect_container(
            container=self._container_id
        )
        log.debug(container_data)

    def start(self):
        """
        Start the docker node and configures a netns for it.
        """
        self._client.start(self._container_id)
        self._pid = self._client.inspect_container(
            self._container_id)['State']['Pid']

    def stop(self):
        """
        Request container to stop.
        """
        self._client.stop(self._container_id)
        self._client.wait(self._container_id)
        self._client.remove_container(self._container_id)

    def disable(self):
        """
        Disable the node.

        In Docker implementation this pauses the container.
        """
        for portlbl in self.ports:
            self.set_port_state(portlbl, False)
        self._client.pause(self._container_id)

    def enable(self):
        """
        Enable the node.

        In Docker implementation this unpauses the container.
        """
        self._client.unpause(self._container_id)
        for portlbl in self.ports:
            self.set_port_state(portlbl, True)

    def set_port_state(self, portlbl, state):
        """
        Set the given port label to the given state.

        :param str portlbl: The label of the port.
        :param bool state: True for up, False for down.
        """
        iface = self.ports[portlbl]
        state = 'up' if state else 'down'

        command = 'ip link set dev {iface} {state}'.format(**locals())
        self._docker_exec(command)

    def _docker_exec(self, command):
        """
        Execute a command inside the docker.

        :param str command: The command to execute.
        """
        log.debug(
            '[{}]._docker_exec(\'{}\') ::'.format(self._container_id, command)
        )

        response = check_output(shsplit(
            'docker exec {container_id} {command}'.format(
                container_id=self._container_id, command=command.strip()
            )
        )).decode('utf8')

        log.debug(response)
        return response

__all__ = ['DockerNode']
<commit_msg>chg: usr: Made configurable the base path to store the shared directory using the shared_dir_base attribute in the base node.<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2015-2016 Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

"""
topology_docker base node module.
"""

from __future__ import unicode_literals, absolute_import
from __future__ import print_function, division

from os import getpid
from json import loads
from os.path import join
from logging import getLogger
from datetime import datetime
from shlex import split as shsplit
from subprocess import check_output
from abc import ABCMeta, abstractmethod

from docker import Client
from six import add_metaclass

from topology.platforms.base import CommonNode
from topology_docker.utils import ensure_dir

log = getLogger(__name__)

@add_metaclass(ABCMeta)
class DockerNode(CommonNode):
    """
    An instance of this class will create a detached Docker container.

    This node binds the ``shared_dir_mount`` directory of the container to a
    local path in the host system defined in ``self.shared_dir``.

    :param str identifier: Node unique identifier in the topology being built.
    :param str image: The image to run on this node, in the
     form ``repository:tag``.
    :param str registry: Docker registry to pull image from.
    :param str command: The command to run when the container is brought up.
    :param str binds: Directories to bind for this container separated by a
     ``;`` in the form:

     ::

        '/tmp:/tmp;/dev/log:/dev/log;/sys/fs/cgroup:/sys/fs/cgroup'

    :param str network_mode: Network mode for this container.
    :param str shared_dir_base: Base path in the host where the shared
     directory will be created. The shared directory will always have the name
     of the container inside this directory.
    :param str shared_dir_mount: Mount point of the shared directory in the
     container.

    Read only public attributes:

    :var image: Name of the Docker image being used by this node.
     Same as the ``image`` keyword argument.
    :var container_id: Unique container identifier assigned by the Docker
     daemon in the form of a hash.
    :var container_name: Unique container name assigned by the framework in the
     form ``{identifier}_{pid}_{timestamp}``.
    :var shared_dir: Share directory in the host for this container. Always
     ``/tmp/topology/{container_name}``.
    :var shared_dir_mount: Directory inside the container where the
     ``shared_dir`` is mounted. Same as the ``shared_dir_mount`` keyword
     argument.
    """

    @abstractmethod
    def __init__(
            self, identifier,
            image='ubuntu:latest', registry=None, command='bash',
            binds=None, network_mode='none', hostname=None,
            shared_dir_base='/tmp/topology/docker/',
            shared_dir_mount='/var/topology',
            **kwargs):

        super(DockerNode, self).__init__(identifier, **kwargs)

        self._pid = None
        self._image = image
        self._registry = registry
        self._command = command
        self._hostname = hostname
        self._client = Client(version='auto')

        self._container_name = '{identifier}_{pid}_{timestamp}'.format(
            identifier=identifier, pid=getpid(),
            timestamp=datetime.now().isoformat().replace(':', '-')
        )
        self._shared_dir_base = shared_dir_base
        self._shared_dir_mount = shared_dir_mount
        self._shared_dir = join(
            shared_dir_base,
            self._container_name
        )

        # Autopull docker image if necessary
        self._autopull()

        # Create shared directory
        ensure_dir(self._shared_dir)

        # Add binded directories
        container_binds = [
            '{}:{}'.format(
                self._shared_dir,
                self._shared_dir_mount
            )
        ]
        if binds is not None:
            container_binds.extend(binds.split(';'))

        # Create host config
        self._host_config = self._client.create_host_config(
            # Container is given access to all devices
            privileged=True,
            # Avoid connecting to host bridge, usually docker0
            network_mode=network_mode,
            binds=container_binds
        )

        # Create container
        self._container_id = self._client.create_container(
            image=self._image,
            command=self._command,
            name=self._container_name,
            detach=True,
            tty=True,
            hostname=self._hostname,
            host_config=self._host_config
        )['Id']

    @property
    def image(self):
        return self._image

    @property
    def container_id(self):
        return self._container_id

    @property
    def container_name(self):
        return self._container_name

    @property
    def shared_dir(self):
        return self._shared_dir

    @property
    def shared_dir_mount(self):
        return self._shared_dir_mount

    def _autopull(self):
        """
        Autopulls the docker image of the node, if necessary.
        """
        # Search for image in available images
        for tags in [img['RepoTags'] for img in self._client.images()]:
            if self._image in tags:
                return

        # Determine image parts
        registry = self._registry
        image = self._image
        tag = 'latest'

        if ':' in image:
            image, tag = image.split(':')

        # Pull image
        pull_uri = image
        if registry:
            pull_uri = '{}/{}'.format(registry, image)
        pull_name = '{}:{}'.format(pull_uri, tag)

        log.info('Trying to pull image {} ...'.format(pull_name))

        last = ''
        for line in self._client.pull(pull_uri, tag=tag, stream=True):
            last = line
        status = loads(last.decode('utf8'))

        log.debug('Pulling result :: {}'.format(status))

        if 'error' in status:
            raise Exception(status['error'])

        # Retag if required
        if pull_name != self._image:
            if not self._client.tag(pull_name, image, tag):
                raise Exception(
                    'Error when tagging image {} with tag {}:{}'.format(
                        pull_name, image, tag
                    )
                )

            log.info(
                'Tagged image {} with tag {}:{}'.format(
                    pull_name, image, tag
                )
            )

    def notify_add_biport(self, node, biport):
        """
        Get notified that a new biport was added to this engine node.

        :param node: The specification node that spawn this engine node.
        :type node: pynml.nml.Node
        :param biport: The specification bidirectional port added.
        :type biport: pynml.nml.BidirectionalPort
        :rtype: str
        :return: The assigned interface name of the port.
        """
        return biport.metadata.get('label', biport.identifier)

    def notify_add_bilink(self, nodeport, bilink):
        """
        Get notified that a new bilink was added to a port of this engine node.

        :param nodeport: A tuple with the specification node and port being
         linked.
        :type nodeport: (pynml.nml.Node, pynml.nml.BidirectionalPort)
        :param bilink: The specification bidirectional link added.
        :type bilink: pynml.nml.BidirectionalLink
        """

    def notify_post_build(self):
        """
        Get notified that the post build stage of the topology build was
        reached.
        """
        # Log container data
        image_data = self._client.inspect_image(
            image=self._image
        )
        log.info(
            'Started container {}:\n'
            '    Image name: {}\n'
            '    Image id: {}\n'
            '    Image creation date: {}'
            '    Image tags: {}'.format(
                self._container_name,
                self._image,
                image_data.get('Id', '????'),
                image_data.get('Created', '????'),
                ', '.join(image_data.get('RepoTags', []))
            )
        )
        container_data = self._client.inspect_container(
            container=self._container_id
        )
        log.debug(container_data)

    def start(self):
        """
        Start the docker node and configures a netns for it.
        """
        self._client.start(self._container_id)
        self._pid = self._client.inspect_container(
            self._container_id)['State']['Pid']

    def stop(self):
        """
        Request container to stop.
        """
        self._client.stop(self._container_id)
        self._client.wait(self._container_id)
        self._client.remove_container(self._container_id)

    def disable(self):
        """
        Disable the node.

        In Docker implementation this pauses the container.
        """
        for portlbl in self.ports:
            self.set_port_state(portlbl, False)
        self._client.pause(self._container_id)

    def enable(self):
        """
        Enable the node.

        In Docker implementation this unpauses the container.
        """
        self._client.unpause(self._container_id)
        for portlbl in self.ports:
            self.set_port_state(portlbl, True)

    def set_port_state(self, portlbl, state):
        """
        Set the given port label to the given state.

        :param str portlbl: The label of the port.
        :param bool state: True for up, False for down.
        """
        iface = self.ports[portlbl]
        state = 'up' if state else 'down'

        command = 'ip link set dev {iface} {state}'.format(**locals())
        self._docker_exec(command)

    def _docker_exec(self, command):
        """
        Execute a command inside the docker.

        :param str command: The command to execute.
        """
        log.debug(
            '[{}]._docker_exec(\'{}\') ::'.format(self._container_id, command)
        )

        response = check_output(shsplit(
            'docker exec {container_id} {command}'.format(
                container_id=self._container_id, command=command.strip()
            )
        )).decode('utf8')

        log.debug(response)
        return response

__all__ = ['DockerNode']
<|endoftext|>


<commit_before>#!/usr/bin/python
#
#     Copyright 2011, Kay Hayen, mailto:kayhayen@gmx.de
#
#     Part of "Nuitka", an optimizing Python compiler that is compatible and
#     integrates with CPython, but also works on its own.
#
#     If you submit Kay Hayen patches to this software in either form, you
#     automatically grant him a copyright assignment to the code, or in the
#     alternative a BSD license to the code, should your jurisdiction prevent
#     this. Obviously it won't affect code that comes to him indirectly or
#     code you don't submit to him.
#
#     This is to reserve my ability to re-license the code at any time, e.g.
#     the PSF. With this version of Nuitka, using it for Closed Source will
#     not be allowed.
#
#     This program is free software: you can redistribute it and/or modify
#     it under the terms of the GNU General Public License as published by
#     the Free Software Foundation, version 3 of the License.
#
#     This program is distributed in the hope that it will be useful,
#     but WITHOUT ANY WARRANTY; without even the implied warranty of
#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#     GNU General Public License for more details.
#
#     You should have received a copy of the GNU General Public License
#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#     Please leave the whole of this copyright notice intact.
#

import os, sys, shutil

assert os.path.isfile( "setup.py" ) and open( ".git/description" ).read().strip() == "Nuitka Staging"

shutil.rmtree( "dist", ignore_errors = True )
shutil.rmtree( "build", ignore_errors = True )

assert 0 == os.system( "python setup.py sdist --formats=bztar,gztar,zip" )

os.chdir( "dist" )

if os.path.exists( "deb_dist" ):
    shutil.rmtree( "deb_dist" )

assert 0 == os.system( "py2dsc *.tar.gz" )

os.chdir( "deb_dist" )

for entry in os.listdir( "." ):
    if os.path.isdir( entry ) and entry.startswith( "nuitka" ) and not entry.endswith( ".orig" ):
        break
else:
    assert False

print "Found", entry

assert 0 == os.system( "rsync -a ../../debian/ %s/debian/" % entry )

assert 0 == os.system( "rm *.dsc *.debian.tar.gz" )

os.chdir( entry )

# 1. Remove the inline copy of Scons. On Debian there is a dependency.
shutil.rmtree( "nuitka/build/inline_copy", False )

assert 0 == os.system( "EDITOR='true' dpkg-source --commit --include-removal . remove-inline-scons" )

assert 0 == os.system( "debuild" )

os.chdir( "../../.." )

assert os.path.isfile( "setup.py" ) and open( ".git/description" ).read().strip() == "Nuitka Staging"

assert 0 == os.system( "lintian --fail-on-warnings dist/deb_dist/*.changes" )

os.system( "cp dist/deb_dist/*.deb dist/" )

assert os.path.exists( "dist/deb_dist" )

for filename in os.listdir( "dist/deb_dist" ):
    if os.path.isdir( "dist/deb_dist/" + filename ):
        shutil.rmtree( "dist/deb_dist/" + filename )

assert 0 == os.system( r"wine c:\\python27\\python.exe setup.py bdist_wininst --bitmap misc/Nuitka-Installer.bmp" )

for filename in os.listdir( "dist" ):
    if os.path.isfile( "dist/" + filename ):
        assert 0 == os.system( "chmod 644 dist/" + filename )
        assert 0 == os.system( "gpg --local-user 0BCF7396 --detach-sign dist/" + filename )
<commit_msg>Remove distutils "build" directory afterwards, enable --pedantic checks to debian package<commit_after>#!/usr/bin/python
#
#     Copyright 2011, Kay Hayen, mailto:kayhayen@gmx.de
#
#     Part of "Nuitka", an optimizing Python compiler that is compatible and
#     integrates with CPython, but also works on its own.
#
#     If you submit Kay Hayen patches to this software in either form, you
#     automatically grant him a copyright assignment to the code, or in the
#     alternative a BSD license to the code, should your jurisdiction prevent
#     this. Obviously it won't affect code that comes to him indirectly or
#     code you don't submit to him.
#
#     This is to reserve my ability to re-license the code at any time, e.g.
#     the PSF. With this version of Nuitka, using it for Closed Source will
#     not be allowed.
#
#     This program is free software: you can redistribute it and/or modify
#     it under the terms of the GNU General Public License as published by
#     the Free Software Foundation, version 3 of the License.
#
#     This program is distributed in the hope that it will be useful,
#     but WITHOUT ANY WARRANTY; without even the implied warranty of
#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#     GNU General Public License for more details.
#
#     You should have received a copy of the GNU General Public License
#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#     Please leave the whole of this copyright notice intact.
#

import os, sys, shutil

assert os.path.isfile( "setup.py" ) and open( ".git/description" ).read().strip() == "Nuitka Staging"

shutil.rmtree( "dist", ignore_errors = True )
shutil.rmtree( "build", ignore_errors = True )

assert 0 == os.system( "python setup.py sdist --formats=bztar,gztar,zip" )

os.chdir( "dist" )

if os.path.exists( "deb_dist" ):
    shutil.rmtree( "deb_dist" )

assert 0 == os.system( "py2dsc *.tar.gz" )

os.chdir( "deb_dist" )

for entry in os.listdir( "." ):
    if os.path.isdir( entry ) and entry.startswith( "nuitka" ) and not entry.endswith( ".orig" ):
        break
else:
    assert False

print "Found", entry

assert 0 == os.system( "rsync -a ../../debian/ %s/debian/" % entry )

assert 0 == os.system( "rm *.dsc *.debian.tar.gz" )

os.chdir( entry )

# 1. Remove the inline copy of Scons. On Debian there is a dependency.
shutil.rmtree( "nuitka/build/inline_copy", False )

assert 0 == os.system( "EDITOR='true' dpkg-source --commit --include-removal . remove-inline-scons" )

assert 0 == os.system( "debuild" )

os.chdir( "../../.." )

assert os.path.isfile( "setup.py" ) and open( ".git/description" ).read().strip() == "Nuitka Staging"

assert 0 == os.system( "lintian --pedantic --fail-on-warnings dist/deb_dist/*.changes" )

os.system( "cp dist/deb_dist/*.deb dist/" )

assert os.path.exists( "dist/deb_dist" )

for filename in os.listdir( "dist/deb_dist" ):
    if os.path.isdir( "dist/deb_dist/" + filename ):
        shutil.rmtree( "dist/deb_dist/" + filename )

assert 0 == os.system( r"wine c:\\python27\\python.exe setup.py bdist_wininst --bitmap misc/Nuitka-Installer.bmp" )

for filename in os.listdir( "dist" ):
    if os.path.isfile( "dist/" + filename ):
        assert 0 == os.system( "chmod 644 dist/" + filename )
        assert 0 == os.system( "gpg --local-user 0BCF7396 --detach-sign dist/" + filename )

shutil.rmtree( "build", ignore_errors = True )
<|endoftext|>


<commit_before># Copyright (c) 2017 Zordsdavini
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from datetime import datetime, timedelta
from time import time

from libqtile.utils import send_notification
from libqtile.widget import base

class Pomodoro(base.ThreadPoolText):
    """Pomodoro technique widget"""
    orientations = base.ORIENTATION_HORIZONTAL
    defaults = [
        ("fmt", "{}", "fmt"),
        ("num_pomodori", 4, "Number of pomodori to do in a cycle"),
        ("length_pomodori", 25, "Length of one pomodori in minutes"),
        ("length_short_break", 5, "Length of a short break in minutes"),
        ("length_long_break", 15, "Length of a long break in minutes"),
        ("color_inactive", "ff0000", "Colour then pomodoro is inactive"),
        ("color_active", "00ff00", "Colour then pomodoro is running"),
        ("color_break", "ffff00", "Colour then it is break time"),
        ("notification_on", True, "Turn notifications on"),
        ("prefix_inactive", "POMODORO", "Prefix when app is inactive"),
        ("prefix_active", "", "Prefix then app is active"),
        ("prefix_break", "B ", "Prefix during short break"),
        ("prefix_long_break", "LB ", "Prefix during long break"),
        ("prefix_paused", "PAUSE", "Prefix during pause"),
        ("update_interval", 1, "Update interval in seconds, if none, the "
            "widget updates whenever the event loop is idle."),
    ]

    STATUS_START = "start"
    STATUS_INACTIVE = "inactive"
    STATUS_ACTIVE = "active"
    STATUS_BREAK = "break"
    STATUS_LONG_BREAK = "long_break"
    STATUS_PAUSED = "paused"

    status = "inactive"
    paused_status = None
    end_time = datetime.now()
    time_left = None
    pomodoros = 1

    def __init__(self, **config):
        base.ThreadPoolText.__init__(self, "", **config)
        self.add_defaults(Pomodoro.defaults)
        self.prefix = {
            'inactive': self.prefix_inactive,
            'active': self.prefix_active,
            'break': self.prefix_break,
            'long_break': self.prefix_long_break,
            'paused': self.prefix_paused
        }

        self.add_callbacks({
            'Button1': self._toggle_break,
            'Button3': self._toggle_active,
        })

    def tick(self):
        self.update(self.poll())
        return self.update_interval - time() % self.update_interval

    def _update(self):
        if self.status in [self.STATUS_INACTIVE, self.STATUS_PAUSED]:
            return

        if self.end_time > datetime.now() and self.status != self.STATUS_START:
            return

        if self.status == self.STATUS_ACTIVE and self.pomodoros == self.num_pomodori:
            self.status = self.STATUS_LONG_BREAK
            self.end_time = datetime.now() + timedelta(minutes=self.length_long_break)
            self.pomodoros = 1
            if self.notification_on:
                self._send_notification(
                    "normal", "Please take a long break! End Time: " + self.end_time.strftime("%H:%M")
                )
            return

        if self.status == self.STATUS_ACTIVE:
            self.status = self.STATUS_BREAK
            self.end_time = datetime.now() + timedelta(minutes=self.length_short_break)
            self.pomodoros += 1
            if self.notification_on:
                self._send_notification(
                    "normal", "Please take a short break! End Time: " + self.end_time.strftime("%H:%M")
                )
            return

        self.status = self.STATUS_ACTIVE
        self.end_time = datetime.now() + timedelta(minutes=self.length_pomodori)
        if self.notification_on:
            self._send_notification(
                "critical", "Please start with the next Pomodori! End Time: " + self.end_time.strftime("%H:%M")
            )

        return

    def _get_text(self):
        self._update()

        if self.status in [self.STATUS_INACTIVE, self.STATUS_PAUSED]:
            self.layout.colour = self.color_inactive
            return self.prefix[self.status]

        time_left = self.end_time - datetime.now()

        if self.status == self.STATUS_ACTIVE:
            self.layout.colour = self.color_active
        else:
            self.layout.colour = self.color_break

        time_string = "%i:%i:%s" % (time_left.seconds // 3600, time_left.seconds % 3600 // 60, time_left.seconds % 60)
        return self.prefix[self.status] + time_string

    def _toggle_break(self):
        if self.status == self.STATUS_INACTIVE:
            self.status = self.STATUS_START
            return

        if self.paused_status is None:
            self.paused_status = self.status
            self.time_left = self.end_time - datetime.now()
            self.status = self.STATUS_PAUSED
            if self.notification_on:
                self._send_notification('low', "Pomodoro has been paused")
        else:
            self.status = self.paused_status
            self.paused_status = None
            self.end_time = self.time_left + datetime.now()
            if self.notification_on:
                if self.status == self.STATUS_ACTIVE:
                    status = 'Pomodoro'
                else:
                    status = 'break'

                self._send_notification(
                    "normal",
                    "Please continue on %s! End Time: " % status + self.end_time.strftime("%H:%M")
                )

    def _toggle_active(self):
        if self.status != self.STATUS_INACTIVE:
            self.status = self.STATUS_INACTIVE
            if self.notification_on:
                self._send_notification('critical', "Pomodoro has been suspended")
        else:
            self.status = self.STATUS_START

    def _send_notification(self, urgent, message):
        send_notification("Pomodoro", message, urgent=urgent)

    def poll(self):
        return self.fmt.format(self._get_text())
<commit_msg>Fix double parsing of 'fmt' in Pomodoro widget<commit_after># Copyright (c) 2017 Zordsdavini
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from datetime import datetime, timedelta
from time import time

from libqtile.utils import send_notification
from libqtile.widget import base

class Pomodoro(base.ThreadPoolText):
    """Pomodoro technique widget"""
    orientations = base.ORIENTATION_HORIZONTAL
    defaults = [
        ("num_pomodori", 4, "Number of pomodori to do in a cycle"),
        ("length_pomodori", 25, "Length of one pomodori in minutes"),
        ("length_short_break", 5, "Length of a short break in minutes"),
        ("length_long_break", 15, "Length of a long break in minutes"),
        ("color_inactive", "ff0000", "Colour then pomodoro is inactive"),
        ("color_active", "00ff00", "Colour then pomodoro is running"),
        ("color_break", "ffff00", "Colour then it is break time"),
        ("notification_on", True, "Turn notifications on"),
        ("prefix_inactive", "POMODORO", "Prefix when app is inactive"),
        ("prefix_active", "", "Prefix then app is active"),
        ("prefix_break", "B ", "Prefix during short break"),
        ("prefix_long_break", "LB ", "Prefix during long break"),
        ("prefix_paused", "PAUSE", "Prefix during pause"),
        ("update_interval", 1, "Update interval in seconds, if none, the "
            "widget updates whenever the event loop is idle."),
    ]

    STATUS_START = "start"
    STATUS_INACTIVE = "inactive"
    STATUS_ACTIVE = "active"
    STATUS_BREAK = "break"
    STATUS_LONG_BREAK = "long_break"
    STATUS_PAUSED = "paused"

    status = "inactive"
    paused_status = None
    end_time = datetime.now()
    time_left = None
    pomodoros = 1

    def __init__(self, **config):
        base.ThreadPoolText.__init__(self, "", **config)
        self.add_defaults(Pomodoro.defaults)
        self.prefix = {
            'inactive': self.prefix_inactive,
            'active': self.prefix_active,
            'break': self.prefix_break,
            'long_break': self.prefix_long_break,
            'paused': self.prefix_paused
        }

        self.add_callbacks({
            'Button1': self._toggle_break,
            'Button3': self._toggle_active,
        })

    def tick(self):
        self.update(self.poll())
        return self.update_interval - time() % self.update_interval

    def _update(self):
        if self.status in [self.STATUS_INACTIVE, self.STATUS_PAUSED]:
            return

        if self.end_time > datetime.now() and self.status != self.STATUS_START:
            return

        if self.status == self.STATUS_ACTIVE and self.pomodoros == self.num_pomodori:
            self.status = self.STATUS_LONG_BREAK
            self.end_time = datetime.now() + timedelta(minutes=self.length_long_break)
            self.pomodoros = 1
            if self.notification_on:
                self._send_notification(
                    "normal", "Please take a long break! End Time: " + self.end_time.strftime("%H:%M")
                )
            return

        if self.status == self.STATUS_ACTIVE:
            self.status = self.STATUS_BREAK
            self.end_time = datetime.now() + timedelta(minutes=self.length_short_break)
            self.pomodoros += 1
            if self.notification_on:
                self._send_notification(
                    "normal", "Please take a short break! End Time: " + self.end_time.strftime("%H:%M")
                )
            return

        self.status = self.STATUS_ACTIVE
        self.end_time = datetime.now() + timedelta(minutes=self.length_pomodori)
        if self.notification_on:
            self._send_notification(
                "critical", "Please start with the next Pomodori! End Time: " + self.end_time.strftime("%H:%M")
            )

        return

    def _get_text(self):
        self._update()

        if self.status in [self.STATUS_INACTIVE, self.STATUS_PAUSED]:
            self.layout.colour = self.color_inactive
            return self.prefix[self.status]

        time_left = self.end_time - datetime.now()

        if self.status == self.STATUS_ACTIVE:
            self.layout.colour = self.color_active
        else:
            self.layout.colour = self.color_break

        time_string = "%i:%i:%s" % (time_left.seconds // 3600, time_left.seconds % 3600 // 60, time_left.seconds % 60)
        return self.prefix[self.status] + time_string

    def _toggle_break(self):
        if self.status == self.STATUS_INACTIVE:
            self.status = self.STATUS_START
            return

        if self.paused_status is None:
            self.paused_status = self.status
            self.time_left = self.end_time - datetime.now()
            self.status = self.STATUS_PAUSED
            if self.notification_on:
                self._send_notification('low', "Pomodoro has been paused")
        else:
            self.status = self.paused_status
            self.paused_status = None
            self.end_time = self.time_left + datetime.now()
            if self.notification_on:
                if self.status == self.STATUS_ACTIVE:
                    status = 'Pomodoro'
                else:
                    status = 'break'

                self._send_notification(
                    "normal",
                    "Please continue on %s! End Time: " % status + self.end_time.strftime("%H:%M")
                )

    def _toggle_active(self):
        if self.status != self.STATUS_INACTIVE:
            self.status = self.STATUS_INACTIVE
            if self.notification_on:
                self._send_notification('critical', "Pomodoro has been suspended")
        else:
            self.status = self.STATUS_START

    def _send_notification(self, urgent, message):
        send_notification("Pomodoro", message, urgent=urgent)

    def poll(self):
        return self._get_text()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import codecs, os, re, subprocess, sys, yaml

# We don't read the index
IGNORED_FILES = "index.md"

# The class for all reql objects
query = 'rethinkdb.ast.RqlQuery.'

# The python class associated with each type
parents = {
    None: '',
    'r': 'rethinkdb.',
    'sequence': query,
    'query': query,
    'stream': query,
    'grouped_stream': query,
    'singleSelection': query,
    'array': query,
    'number': query,
    'bool': query,
    'value': query,
    'string': query,
    'time': query,
    'any': query,
    'geometry': query,
    'point': query,
    'line': query,
    'polygon': query,
    'connection': 'rethinkdb.net.Connection.',
    'cursor': 'rethinkdb.net.Cursor.',
    'db': 'rethinkdb.ast.DB.',
    'table': 'rethinkdb.ast.Table.',
}

# The real python names for names used in the docs
tags = {
    '[] (get_field)': [(query, '__getitem__')],
    'nth, []': [(query, 'nth')],
    'slice, []': [(query, 'slice')],
    '+': [(query, '__add__'), ('rethinkdb.', 'add')],
    '-': [(query, '__sub__'), ('rethinkdb.', 'sub')],
    '*': [(query, '__mul__'), ('rethinkdb.', 'mul')],
    '/': [(query, '__div__'), ('rethinkdb.', 'div')],
    '%': [(query, '__mod__'), ('rethinkdb.', 'mod')],
    '&, and_': [(query, '__and__'), ('rethinkdb.', 'and_')],
    '|, or_': [(query, '__or__'), ('rethinkdb.', 'or_')],
    '==, eq': [(query, '__eq__'), (query, 'eq')],
    '!=, ne': [(query, '__ne__'), (query, 'ne')],
    '<, lt': [(query, '__lt__'), (query, 'lt')],
    '>, gt': [(query, '__gt__'), (query, 'gt')],
    '<=, le': [(query, '__le__'), (query, 'le')],
    '>=, ge': [(query, '__ge__'), (query, 'ge')],
    '~, not_': [(query, '__invert__'), (query, 'not_'), ('rethinkdb.', 'not_')],
    'r': [('', 'rethinkdb')],
    'repl': [('rethinkdb.net.Connection.', 'repl')],
    'count': lambda parent: not parent == 'rethinkdb.' and [(query, 'count')] or [],
    'rethinkdb': [('', 'rethinkdb')]
}

# Write the header of the docs.py file
def write_header(file):
    commit = subprocess.Popen(['git', 'log', '-n', '1', '--pretty=format:"%H"'], stdout=subprocess.PIPE).communicate()[0].decode('utf-8')
    file.write('''# This file was generated by _scripts/gen_python.py from the rethinkdb documentation in http://github.com/rethinkdb/docs
# hash: %s

import rethinkdb

docsSource = [
''' % commit)

def write_footer(file):
    '''write the ending of the file'''
    file.write('''
]

for function, text in docsSource:
	try:
		text = str(text.decode('utf-8'))
	except UnicodeEncodeError:
		pass
	if hasattr(function, "__func__"):
		function.__func__.__doc__ = text
	else:
		function.__doc__ = text
''')

# Browse all the docs
def browse_files(base, result_file):
    subdirlist = []
    # Because we don't read from the json file, that is enough to guarantee an order
    for item in sorted(os.listdir(base)):
        if item[0] != '.' and item not in IGNORED_FILES:
            full_path = os.path.join(base, item)
            if os.path.isfile(full_path):
                add_doc(full_path, result_file)
            else:
                subdirlist.append(full_path)

    for subdir in subdirlist:
        browse_files(subdir, result_file)

# Add docs in result for one file
def add_doc(file_name, result_file):
    limiter_yaml = re.compile('---\s*')
    is_yaml = False
    yaml_header = ""

    parent = ""
    func = ""

    # Reading the JS file to extract the io data
    file_name_js = file_name.replace('python', 'javascript')
    try:
        details_file_js = codecs.open(file_name_js, "r", "utf-8")

        yaml_header_js = ""
        for line in details_file_js:
            if limiter_yaml.match(line) != None:
                # We ignore the yaml header
                if is_yaml == False:
                    is_yaml = True
                else:
                    break
            elif is_yaml == True:
                yaml_header_js += line

        yaml_data_js = yaml.load(yaml_header_js)

        parent = parents[yaml_data_js['io'][0][0]]

    except:
        # The file may not exist (for repl for example)
        pass

    # Open the python file
    details_file = codecs.open(file_name, "r", "utf-8")

    # Define some regex that we will use
    ignore_pattern = re.compile("#.*#.*|<img.*/>") # Used to skip the titles like Description, Related commands etc.

    # Used to skip the body (command syntax)
    start_body_pattern = re.compile("{%\s*apibody\s*%}\s*")
    end_body_pattern = re.compile("{%\s*endapibody\s*%}\s*")
    parsing_body = False

    # Tracking the yaml header, we need it for the command name
    is_yaml = False
    yaml_header_py = ""

    # Track if we are parsing some code
    example_code_start_pattern = re.compile("```py")
    example_code_end_pattern = re.compile("```")
    parsing_example_code = False

    text = ""

    for line in details_file:
        # Ignore titles (h1 tags)
        if ignore_pattern.match(line) != None:
            continue

        if limiter_yaml.match(line) != None:
            # We ignore the yaml header
            if is_yaml == False:
                is_yaml = True
            else:
                yaml_data_py = yaml.load(yaml_header_py)
                name = yaml_data_py["command"]
                is_yaml = False
        elif is_yaml == True:
            yaml_header_py += line
        elif is_yaml == False:
            if start_body_pattern.match(line) != None:
                parsing_body = True
            elif end_body_pattern.match(line) != None:
                parsing_body = False
            elif parsing_body == False:
                if example_code_start_pattern.match(line) != None:
                    parsing_example_code = True
                elif example_code_end_pattern.match(line) != None:
                    parsing_example_code = False
                else:
                    if parsing_example_code == True:
                        text += "    " + line
                    else:
                        text += line
            else:
                text += line.replace('→', '->')

    encoded = repr(re.sub("(__Example:__)|(__Example__:)", "*Example*", re.sub("^\n+", "", re.sub("\n{2,}", "\n\n", text))).encode('utf-8'))
    if not encoded.startswith('b'): # append the binary marker for Python3 when generateing from Python2.6
        encoded = 'b' + encoded

    # If the command has multiple name, parents
    if name in tags:
        names = tags[name]
        if type(names) == type(lambda x: x):
            names = names(parent)

        for parent, name in names:
            result_file.write("\n\t(" + parent + name + ", " + encoded + '),')
    else: # If the command has just one name and one parent
        assert parent not in (None, ''), 'Missing the parent entry for: %s in %s' % (name, file_name)
        result_file.write("\n\t(" + parent + name + ", " + encoded + '),')

if __name__ == "__main__":
    script_path = os.path.dirname(os.path.realpath(__file__))

    result_file = codecs.open(script_path+"/docs.py", "w", "utf-8")

    write_header(result_file)

    browse_files(script_path+"/../api/python/", result_file)

    write_footer(result_file)

    result_file.close()
<commit_msg>Update gen_python for Markdown URL munging<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import codecs, os, re, subprocess, sys, yaml

# We don't read the index
IGNORED_FILES = "index.md"

# The class for all reql objects
query = 'rethinkdb.ast.RqlQuery.'

# The python class associated with each type
parents = {
    None: '',
    'r': 'rethinkdb.',
    'sequence': query,
    'query': query,
    'stream': query,
    'grouped_stream': query,
    'singleSelection': query,
    'array': query,
    'number': query,
    'bool': query,
    'value': query,
    'string': query,
    'time': query,
    'any': query,
    'geometry': query,
    'point': query,
    'line': query,
    'polygon': query,
    'connection': 'rethinkdb.net.Connection.',
    'cursor': 'rethinkdb.net.Cursor.',
    'db': 'rethinkdb.ast.DB.',
    'table': 'rethinkdb.ast.Table.',
}

# The real python names for names used in the docs
tags = {
    '[] (get_field)': [(query, '__getitem__')],
    'nth, []': [(query, 'nth')],
    'slice, []': [(query, 'slice')],
    '+': [(query, '__add__'), ('rethinkdb.', 'add')],
    '-': [(query, '__sub__'), ('rethinkdb.', 'sub')],
    '*': [(query, '__mul__'), ('rethinkdb.', 'mul')],
    '/': [(query, '__div__'), ('rethinkdb.', 'div')],
    '%': [(query, '__mod__'), ('rethinkdb.', 'mod')],
    '&, and_': [(query, '__and__'), ('rethinkdb.', 'and_')],
    '|, or_': [(query, '__or__'), ('rethinkdb.', 'or_')],
    '==, eq': [(query, '__eq__'), (query, 'eq')],
    '!=, ne': [(query, '__ne__'), (query, 'ne')],
    '<, lt': [(query, '__lt__'), (query, 'lt')],
    '>, gt': [(query, '__gt__'), (query, 'gt')],
    '<=, le': [(query, '__le__'), (query, 'le')],
    '>=, ge': [(query, '__ge__'), (query, 'ge')],
    '~, not_': [(query, '__invert__'), (query, 'not_'), ('rethinkdb.', 'not_')],
    'r': [('', 'rethinkdb')],
    'repl': [('rethinkdb.net.Connection.', 'repl')],
    'count': lambda parent: not parent == 'rethinkdb.' and [(query, 'count')] or [],
    'rethinkdb': [('', 'rethinkdb')]
}

# Write the header of the docs.py file
def write_header(file):
    commit = subprocess.Popen(['git', 'log', '-n', '1', '--pretty=format:"%H"'], stdout=subprocess.PIPE).communicate()[0].decode('utf-8')
    file.write('''# This file was generated by _scripts/gen_python.py from the rethinkdb documentation in http://github.com/rethinkdb/docs
# hash: %s

import rethinkdb

docsSource = [
''' % commit)

def write_footer(file):
    '''write the ending of the file'''
    file.write('''
]

for function, text in docsSource:
	try:
		text = str(text.decode('utf-8'))
	except UnicodeEncodeError:
		pass
	if hasattr(function, "__func__"):
		function.__func__.__doc__ = text
	else:
		function.__doc__ = text
''')

# Browse all the docs
def browse_files(base, result_file):
    subdirlist = []
    # Because we don't read from the json file, that is enough to guarantee an order
    for item in sorted(os.listdir(base)):
        if item[0] != '.' and item not in IGNORED_FILES:
            full_path = os.path.join(base, item)
            if os.path.isfile(full_path):
                add_doc(full_path, result_file)
            else:
                subdirlist.append(full_path)

    for subdir in subdirlist:
        browse_files(subdir, result_file)

# Add docs in result for one file
def add_doc(file_name, result_file):
    limiter_yaml = re.compile('---\s*')
    is_yaml = False
    yaml_header = ""

    parent = ""
    func = ""

    # Reading the JS file to extract the io data
    file_name_js = file_name.replace('python', 'javascript')
    try:
        details_file_js = codecs.open(file_name_js, "r", "utf-8")

        yaml_header_js = ""
        for line in details_file_js:
            if limiter_yaml.match(line) != None:
                # We ignore the yaml header
                if is_yaml == False:
                    is_yaml = True
                else:
                    break
            elif is_yaml == True:
                yaml_header_js += line

        yaml_data_js = yaml.load(yaml_header_js)

        parent = parents[yaml_data_js['io'][0][0]]

    except:
        # The file may not exist (for repl for example)
        pass

    # Open the python file
    details_file = codecs.open(file_name, "r", "utf-8")

    # Define some regex that we will use
    ignore_pattern = re.compile("#.*#.*|<img.*/>") # Used to skip the titles like Description, Related commands etc.

    # Used to skip the body (command syntax)
    start_body_pattern = re.compile("{%\s*apibody\s*%}\s*")
    end_body_pattern = re.compile("{%\s*endapibody\s*%}\s*")
    parsing_body = False

    # Used to convert relative Markdown links to absolute
    link_match_pattern = re.compile(r'\[(.*?)\]\(/')
    link_replace_pattern = r'[\1](http://rethinkdb.com/'

    # Tracking the yaml header, we need it for the command name
    is_yaml = False
    yaml_header_py = ""

    # Track if we are parsing some code
    example_code_start_pattern = re.compile("```py")
    example_code_end_pattern = re.compile("```")
    parsing_example_code = False

    text = ""

    for line in details_file:
        # Ignore titles (h1 tags)
        if ignore_pattern.match(line) != None:
            continue

        if limiter_yaml.match(line) != None:
            # We ignore the yaml header
            if is_yaml == False:
                is_yaml = True
            else:
                yaml_data_py = yaml.load(yaml_header_py)
                name = yaml_data_py["command"]
                is_yaml = False
        elif is_yaml == True:
            yaml_header_py += line
        elif is_yaml == False:
            if start_body_pattern.match(line) != None:
                parsing_body = True
            elif end_body_pattern.match(line) != None:
                parsing_body = False
            elif parsing_body == False:
                if example_code_start_pattern.match(line) != None:
                    parsing_example_code = True
                elif example_code_end_pattern.match(line) != None:
                    parsing_example_code = False
                else:
                    if parsing_example_code == True:
                        text += "    " + line
                    else:
                        line = re.sub(link_match_pattern,
                            link_replace_pattern, line)
                        text += line
            else:
                text += line.replace('→', '->')

    encoded = repr(re.sub("(__Example:__)|(__Example__:)", "*Example*", re.sub("^\n+", "", re.sub("\n{2,}", "\n\n", text))).encode('utf-8'))
    if not encoded.startswith('b'): # append the binary marker for Python3 when generateing from Python2.6
        encoded = 'b' + encoded

    # If the command has multiple name, parents
    if name in tags:
        names = tags[name]
        if type(names) == type(lambda x: x):
            names = names(parent)

        for parent, name in names:
            result_file.write("\n\t(" + parent + name + ", " + encoded + '),')
    else: # If the command has just one name and one parent
        assert parent not in (None, ''), 'Missing the parent entry for: %s in %s' % (name, file_name)
        result_file.write("\n\t(" + parent + name + ", " + encoded + '),')

if __name__ == "__main__":
    script_path = os.path.dirname(os.path.realpath(__file__))

    result_file = codecs.open(script_path+"/docs.py", "w", "utf-8")

    write_header(result_file)

    browse_files(script_path+"/../api/python/", result_file)

    write_footer(result_file)

    result_file.close()
<|endoftext|>