<commit_before>"""
Project management classes and functions
"""

from PyQt5 import QtWidgets, QtCore
from views.new_project import Ui_create_new_project
import os
import re
from ConfigParser import SafeConfigParser, NoSectionError, NoOptionError
import time
import datetime
from shutil import copy, rmtree
import cv2
try:
    from PIL import Image
except:
    import Image
import numpy as np

from app_config import AppConfig as ac
from app_config import get_default_project_dir, get_project_path, get_config_path, config_section_exists, get_config_with_sections, update_config_with_sections
from cloud_api import api
import message_helper

class ProjectWizard(QtWidgets.QWizard):

    def __init__(self, parent):
        super(ProjectWizard, self).__init__(parent)
        self.ui = Ui_create_new_project()
        self.ui.setupUi(self)
        self.ui.newp_aerial_image_browse.clicked.connect(self.open_aerial_image)
        self.ui.newp_video_browse.clicked.connect(self.open_video)
        self.aerial_image_selected = False
        self.video_selected = False

        # Remove '?' icon
        flags = self.windowFlags() & (~QtCore.Qt.WindowContextHelpButtonHint)
        self.setWindowFlags(flags)

        # Make 'Back' button non-destructive
        self.setOption(QtWidgets.QWizard.IndependentPages)

        self.ui.newp_start_creation.clicked.connect(self.start_create_project)
        self.config_parser = SafeConfigParser()

        self.creating_project = False

        self.ui.newp_p1.registerField("project_name*", self.ui.newp_projectname_input)

        self.ui.newp_p2.registerField("video_path*", self.ui.newp_video_input)
        self.ui.newp_p2.registerField("video_start_datetime", self.ui.newp_video_start_time_input)
        self.ui.newp_p2.registerField("video_server_input*", self.ui.newp_video_server_input)
        self.ui.newp_p2.registerField("video_email", self.ui.newp_video_email_input)

        self.ui.newp_p2_5.registerField("aerial_image*", self.ui.newp_aerial_image_input)
        self.ui.newp_p3.registerField("create_project", self.ui.newp_start_creation)

        # Keep track of where they last opened a video or image from, hopefully it will save them
        # time next time if the next one is in the same spot
        self.last_known_location = None

    def showEvent(self, event):
        # We want to autofill the default server every time the creation page is shown
        self.ui.newp_video_server_input.setText("http://localhost:8088")

    def validateCurrentPage(self):
        # If on last page, treat 'Finish' click as project creation click. Otherwise, return True to advance.
        page = self.currentPage()
        if page.isFinalPage():
            self.start_create_project()
            return False
        else:
            return True

    def show_message(self, message):
        helper = message_helper.MessageHelper(self)
        helper.show_message(message)

    def open_aerial_image(self):
        filt = "Images (*.png *.jpg *.jpeg *.bmp *.tif *.gif)"  # Select only images
        if self.last_known_location is not None:
            folder = self.last_known_location
        else:
            folder = os.path.realpath(os.path.join(os.path.expanduser('~'), "Documents"))
        fname = self.open_fd(dialog_text="Select aerial image", file_filter=filt, default_dir=folder)
        if fname:
            filepath = self.get_filepath(fname)
            self.last_known_location = os.path.dirname(filepath)
            self.ui.newp_aerial_image_input.setText(filepath)
            self.aerialpath = filepath
            self.aerial_image_selected = True
        else:
            self.ui.newp_aerial_image_input.setText("NO FILE SELECTED")

    def open_video(self):
        filt = "Videos (*.mp4 *.avi *.mpg *.mpeg *.mov *.ogg *.wmv *.m4v *.m4p *.mp2 *.mpe *.mpv *.m2v)"  # Select only videos
        if self.last_known_location is not None:
            folder = self.last_known_location
        else:
            folder = os.path.realpath(os.path.join(os.path.expanduser('~'), "Documents"))
        fname = self.open_fd(dialog_text="Select video for analysis", file_filter=filt, default_dir=folder)
        if fname:
            filepath = self.get_filepath(fname)
            self.last_known_location = os.path.dirname(filepath)
            self.ui.newp_video_input.setText(filepath)
            self.video_selected = True
            self.videopath = filepath
        else:
            self.ui.newp_video_input.setText("NO VIDEO SELECTED")

    def get_filepath(self, filepath):
        """Cleans the filepath to only include the path, not file options.

        Returns the selected filepath only, found between the ''.

        Args:
            filepath: uncleaned filename and file types.

        Returns:
            str: Cleaned string of only the location of video/image file.
        """
        return re.findall(r"'(.*?)'", filepath, re.DOTALL)[0]

    def open_fd(self, dialog_text="Open", file_filter="", default_dir=""):
        """Opens a file dialog, allowing user to select a file.

        Returns the selected filename. If the user presses cancel, this returns
        a null string ("").

        Args:
            dialog_text [Optional(str.)]: Text to prompt user with in open file
                dialog. Defaults to "Open".
            file_filter [Optional(str.)]: String used to filter selectable file
                types. Defaults to "".
            default_dir [Optional(str.)]: Path of the default directory to open
                the file dialog box to. Defaults to "".

        Returns:
            str: Filename selected. Null string ("") if no file selected.
        """
        fname = QtWidgets.QFileDialog.getOpenFileName(self, dialog_text, default_dir, file_filter)  # TODO: Filter to show only image files
        return str(fname)

    def start_create_project(self):
        if not self.creating_project:
            self.creating_project = True
            self.config_parser = SafeConfigParser()
            self.create_project_dir()

    def create_project_dir(self):
        project_name = str(self.ui.newp_projectname_input.text())
        ac.CURRENT_PROJECT_PATH = os.path.join(get_default_project_dir(), project_name)
        directory_names = ["homography", "results"]
        pr_path = get_project_path()

        progress_msg = self.ui.newp_creation_status
        progress_bar = self.ui.newp_creation_progress

        # Update UI while creating
        self._update_ui_for_project_creation()

        if not os.path.exists(pr_path):
            # Set URL to use before doing anything
            server = str(self.ui.newp_video_server_input.text())
            update_api(server)

            progress_msg.setText("Creating project directories...")
            progress_bar.show()
            for new_dir in directory_names:
                progress_bar.setValue(progress_bar.value() + 5)
                os.makedirs(os.path.join(pr_path, new_dir))

            progress_bar.setValue(progress_bar.value() + 5)
            progress_msg.setText("Writing configuration files...")
            self._write_to_project_config()

            progress_msg.setText("Copying video file...")
            video_extension = self.videopath.split('.')[-1]
            video_dest = os.path.join(pr_path, 'video.' + video_extension)
            copy(self.videopath, video_dest)

            progress_msg.setText("Uploading video file...")
            success, err, identifier = api.uploadVideo(self.videopath)
            if not success:
                self._project_creation_error(err, project_name_to_delete=project_name)
                return
            update_config_with_sections(get_config_path(), 'info', 'identifier', identifier)
            progress_bar.setValue(80)

            progress_msg.setText("Extracting camera image...")
            vidcap = cv2.VideoCapture(video_dest)
            vidcap.set(cv2.cv.CV_CAP_PROP_FRAME_COUNT, 1000)
            success, image = vidcap.read()
            progress_bar.setValue(85)
            if success:
                cv2.imwrite(os.path.join(pr_path, "homography", "camera.png"), image)
            else:
                self.show_message("Error getting image. Use the 'Open Camera Image' button and input an image from a frame of the video.")
            progress_bar.setValue(90)

            progress_msg.setText("Copying aerial image...")
            im = Image.open(self.aerialpath)
            aerial_dest = os.path.join(pr_path, "homography", "aerial.png")
            im.save(aerial_dest)
            progress_bar.setValue(95)
            progress_msg.setText("Complete.")

            progress_msg.setText("Opening {} project...".format(project_name))
            self.load_new_project()
            progress_bar.setValue(100)
            progress_msg.setText("Complete.")

            self.creating_project = False
            self._update_ui_for_project_creation()

            self.close()

        else:
            self._project_creation_error("Project exists. No new project created.")
            return

    def _update_ui_for_project_creation(self):
        progress_bar = self.ui.newp_creation_progress
        progress_bar.setHidden(not self.creating_project)
        progress_bar.setValue(0)
        progress_msg = self.ui.newp_creation_status
        progress_msg.setHidden(not self.creating_project)
        creation_button = self.ui.newp_start_creation
        creation_button.setHidden(self.creating_project)

    def _project_creation_error(self, error, project_name_to_delete=None):
        ac.CURRENT_PROJECT_PATH = None
        self.creating_project = False
        self._update_ui_for_project_creation()
        self.show_message(error, title="Error")

        if project_name_to_delete is not None:
            path = os.path.join(get_default_project_dir(), project_name_to_delete)
            rmtree(path)

    def _write_to_project_config(self):
        ts = time.time()
        vid_ts = self.ui.newp_video_start_time_input.dateTime().toPyDateTime()
        email = str(self.ui.newp_video_email_input.text())
        server = str(self.ui.newp_video_server_input.text())

        timestamp = datetime.datetime.fromtimestamp(ts).strftime('%d-%m-%Y %H:%M:%S %Z')
        video_timestamp = vid_ts.strftime('%d-%m-%Y %H:%M:%S %Z')
        self.config_parser.add_section("info")
        self.config_parser.set("info", "project_name", os.path.basename(ac.CURRENT_PROJECT_PATH))
        self.config_parser.set("info", "creation_date", timestamp)
        self.config_parser.set("info", "server", server)
        self.config_parser.set("info", "email", email)
        self.config_parser.add_section("video")
        video_extension = self.videopath.split('.')[-1]
        self.config_parser.set("video", "name", 'video.'+video_extension)
        self.config_parser.set("video", "source", self.videopath)
        self.config_parser.set("video", "start", video_timestamp)

        self.config_parser.add_section("config")
        success, _, config = api.defaultConfig()

        # If we can't get defaults, don't worry. Server fills them in anyway.
        if success:
            for (key, value) in config.iteritems():
                self.config_parser.set("config", key, str(value))

        with open(os.path.join(get_config_path()), 'wb') as configfile:
            self.config_parser.write(configfile)

    def load_new_project(self):
        load_project(ac.CURRENT_PROJECT_PATH, self.parent())

def load_project(project_path, main_window):
    ac.CURRENT_PROJECT_PATH = project_path

    load_homography(main_window)

    # Reload the URL for the project
    addr = get_config_with_sections(get_config_path(), "info", "server")
    update_api(addr)

    load_config(main_window)

def loadPointCorrespondences(filename):
    '''Loads and returns the corresponding points in world (first 2 lines) and image spaces (last 2 lines)'''
    points = np.loadtxt(filename, dtype=np.float32)
    return  (points[:2,:].T, points[2:,:].T) # (world points, image points)

def load_homography(main_window):
    """
    Loads homography information into the specified main window.
    """
    path = get_project_path()
    aerial_path = os.path.join(path, "homography", "aerial.png")
    camera_path = os.path.join(path, "homography", "camera.png")
    # TODO: Handle if above two paths do not exist
    load_from = 'image_pts'  # "image_pts" or "pt_corrs"
    gui = main_window.ui
    # Load images
    gui.homography_aerialview.load_image_from_path(aerial_path)
    gui.homography_cameraview.load_image_from_path(camera_path)

    goodness_path = os.path.join(path, "homography", "homography_goodness_aerial.png")
    image_pts_path = os.path.join(path, "homography", "image-points.txt")
    pt_corrs_path = os.path.join(path, "homography", "point-correspondences.txt")
    homo_path = os.path.join(path, "homography", "homography.txt")

    if load_from is "image_pts":
        corr_path = image_pts_path
    else:
        corr_path = pt_corrs_path

    # Has a homography been previously computed?
    if config_section_exists(get_config_path(), "homography"):  # If we can load homography unit-pix ratio load it
        # load unit-pixel ratio
        upr = get_config_with_sections(get_config_path(), "homography", "unitpixelratio")
        if upr:
            gui.unit_px_input.setText(upr)
    if os.path.exists(corr_path):  # If points have been previously selected
        worldPts, videoPts = loadPointCorrespondences(corr_path)
        main_window.homography = np.loadtxt(homo_path)
        if load_from is "image_pts":
            for point in worldPts:
                main_window.ui.homography_aerialview.scene().add_point(point)
        elif load_from is "pt_corrs":
            for point in worldPts:
                main_window.ui.homography_aerialview.scene().add_point(point/float(upr))
        else:
            print("ERR: Invalid point source {} specified. Points not loaded".format(load_from))
        for point in videoPts:
            main_window.ui.homography_cameraview.scene().add_point(point)
        gui.homography_results.load_image_from_path(goodness_path)
    else:
        print ("{} does not exist. No points loaded.".format(corr_path))

def update_api(addr):
    if addr:
        api.set_url(addr)
    else:
        print("No server, resorting to localhost")
        api.set_url('localhost')

def load_config(main_window):
    main_window.configGui_features.loadConfig_features()
    main_window.configGui_object.loadConfig_objects()
<commit_msg>Don't show 'Cancel' button in project creation<commit_after>"""
Project management classes and functions
"""

from PyQt5 import QtWidgets, QtCore
from views.new_project import Ui_create_new_project
import os
import re
from ConfigParser import SafeConfigParser, NoSectionError, NoOptionError
import time
import datetime
from shutil import copy, rmtree
import cv2
try:
    from PIL import Image
except:
    import Image
import numpy as np

from app_config import AppConfig as ac
from app_config import get_default_project_dir, get_project_path, get_config_path, config_section_exists, get_config_with_sections, update_config_with_sections
from cloud_api import api
import message_helper

class ProjectWizard(QtWidgets.QWizard):

    def __init__(self, parent):
        super(ProjectWizard, self).__init__(parent)
        self.ui = Ui_create_new_project()
        self.ui.setupUi(self)
        self.ui.newp_aerial_image_browse.clicked.connect(self.open_aerial_image)
        self.ui.newp_video_browse.clicked.connect(self.open_video)
        self.aerial_image_selected = False
        self.video_selected = False

        # Remove '?' icon
        flags = self.windowFlags() & (~QtCore.Qt.WindowContextHelpButtonHint)
        self.setWindowFlags(flags)

        # Make 'Back' button non-destructive
        self.setOption(QtWidgets.QWizard.IndependentPages)

        # Don't show 'Cancel' button
        self.setOption(QtWidgets.QWizard.NoCancelButton)

        self.ui.newp_start_creation.clicked.connect(self.start_create_project)
        self.config_parser = SafeConfigParser()

        self.creating_project = False

        self.ui.newp_p1.registerField("project_name*", self.ui.newp_projectname_input)

        self.ui.newp_p2.registerField("video_path*", self.ui.newp_video_input)
        self.ui.newp_p2.registerField("video_start_datetime", self.ui.newp_video_start_time_input)
        self.ui.newp_p2.registerField("video_server_input*", self.ui.newp_video_server_input)
        self.ui.newp_p2.registerField("video_email", self.ui.newp_video_email_input)

        self.ui.newp_p2_5.registerField("aerial_image*", self.ui.newp_aerial_image_input)
        self.ui.newp_p3.registerField("create_project", self.ui.newp_start_creation)

        # Keep track of where they last opened a video or image from, hopefully it will save them
        # time next time if the next one is in the same spot
        self.last_known_location = None

    def showEvent(self, event):
        # We want to autofill the default server every time the creation page is shown
        self.ui.newp_video_server_input.setText("http://localhost:8088")

    def validateCurrentPage(self):
        # If on last page, treat 'Finish' click as project creation click. Otherwise, return True to advance.
        page = self.currentPage()
        if page.isFinalPage():
            self.start_create_project()
            return False
        else:
            return True

    def show_message(self, message):
        helper = message_helper.MessageHelper(self)
        helper.show_message(message)

    def open_aerial_image(self):
        filt = "Images (*.png *.jpg *.jpeg *.bmp *.tif *.gif)"  # Select only images
        if self.last_known_location is not None:
            folder = self.last_known_location
        else:
            folder = os.path.realpath(os.path.join(os.path.expanduser('~'), "Documents"))
        fname = self.open_fd(dialog_text="Select aerial image", file_filter=filt, default_dir=folder)
        if fname:
            filepath = self.get_filepath(fname)
            self.last_known_location = os.path.dirname(filepath)
            self.ui.newp_aerial_image_input.setText(filepath)
            self.aerialpath = filepath
            self.aerial_image_selected = True
        else:
            self.ui.newp_aerial_image_input.setText("NO FILE SELECTED")

    def open_video(self):
        filt = "Videos (*.mp4 *.avi *.mpg *.mpeg *.mov *.ogg *.wmv *.m4v *.m4p *.mp2 *.mpe *.mpv *.m2v)"  # Select only videos
        if self.last_known_location is not None:
            folder = self.last_known_location
        else:
            folder = os.path.realpath(os.path.join(os.path.expanduser('~'), "Documents"))
        fname = self.open_fd(dialog_text="Select video for analysis", file_filter=filt, default_dir=folder)
        if fname:
            filepath = self.get_filepath(fname)
            self.last_known_location = os.path.dirname(filepath)
            self.ui.newp_video_input.setText(filepath)
            self.video_selected = True
            self.videopath = filepath
        else:
            self.ui.newp_video_input.setText("NO VIDEO SELECTED")

    def get_filepath(self, filepath):
        """Cleans the filepath to only include the path, not file options.

        Returns the selected filepath only, found between the ''.

        Args:
            filepath: uncleaned filename and file types.

        Returns:
            str: Cleaned string of only the location of video/image file.
        """
        return re.findall(r"'(.*?)'", filepath, re.DOTALL)[0]

    def open_fd(self, dialog_text="Open", file_filter="", default_dir=""):
        """Opens a file dialog, allowing user to select a file.

        Returns the selected filename. If the user presses cancel, this returns
        a null string ("").

        Args:
            dialog_text [Optional(str.)]: Text to prompt user with in open file
                dialog. Defaults to "Open".
            file_filter [Optional(str.)]: String used to filter selectable file
                types. Defaults to "".
            default_dir [Optional(str.)]: Path of the default directory to open
                the file dialog box to. Defaults to "".

        Returns:
            str: Filename selected. Null string ("") if no file selected.
        """
        fname = QtWidgets.QFileDialog.getOpenFileName(self, dialog_text, default_dir, file_filter)  # TODO: Filter to show only image files
        return str(fname)

    def start_create_project(self):
        if not self.creating_project:
            self.creating_project = True
            self.config_parser = SafeConfigParser()
            self.create_project_dir()

    def create_project_dir(self):
        project_name = str(self.ui.newp_projectname_input.text())
        ac.CURRENT_PROJECT_PATH = os.path.join(get_default_project_dir(), project_name)
        directory_names = ["homography", "results"]
        pr_path = get_project_path()

        progress_msg = self.ui.newp_creation_status
        progress_bar = self.ui.newp_creation_progress

        # Update UI while creating
        self._update_ui_for_project_creation()

        if not os.path.exists(pr_path):
            # Set URL to use before doing anything
            server = str(self.ui.newp_video_server_input.text())
            update_api(server)

            progress_msg.setText("Creating project directories...")
            progress_bar.show()
            for new_dir in directory_names:
                progress_bar.setValue(progress_bar.value() + 5)
                os.makedirs(os.path.join(pr_path, new_dir))

            progress_bar.setValue(progress_bar.value() + 5)
            progress_msg.setText("Writing configuration files...")
            self._write_to_project_config()

            progress_msg.setText("Copying video file...")
            video_extension = self.videopath.split('.')[-1]
            video_dest = os.path.join(pr_path, 'video.' + video_extension)
            copy(self.videopath, video_dest)

            progress_msg.setText("Uploading video file...")
            success, err, identifier = api.uploadVideo(self.videopath)
            if not success:
                self._project_creation_error(err, project_name_to_delete=project_name)
                return
            update_config_with_sections(get_config_path(), 'info', 'identifier', identifier)
            progress_bar.setValue(80)

            progress_msg.setText("Extracting camera image...")
            vidcap = cv2.VideoCapture(video_dest)
            vidcap.set(cv2.cv.CV_CAP_PROP_FRAME_COUNT, 1000)
            success, image = vidcap.read()
            progress_bar.setValue(85)
            if success:
                cv2.imwrite(os.path.join(pr_path, "homography", "camera.png"), image)
            else:
                self.show_message("Error getting image. Use the 'Open Camera Image' button and input an image from a frame of the video.")
            progress_bar.setValue(90)

            progress_msg.setText("Copying aerial image...")
            im = Image.open(self.aerialpath)
            aerial_dest = os.path.join(pr_path, "homography", "aerial.png")
            im.save(aerial_dest)
            progress_bar.setValue(95)
            progress_msg.setText("Complete.")

            progress_msg.setText("Opening {} project...".format(project_name))
            self.load_new_project()
            progress_bar.setValue(100)
            progress_msg.setText("Complete.")

            self.creating_project = False
            self._update_ui_for_project_creation()

            self.close()

        else:
            self._project_creation_error("Project exists. No new project created.")
            return

    def _update_ui_for_project_creation(self):
        progress_bar = self.ui.newp_creation_progress
        progress_bar.setHidden(not self.creating_project)
        progress_bar.setValue(0)
        progress_msg = self.ui.newp_creation_status
        progress_msg.setHidden(not self.creating_project)
        creation_button = self.ui.newp_start_creation
        creation_button.setHidden(self.creating_project)

    def _project_creation_error(self, error, project_name_to_delete=None):
        ac.CURRENT_PROJECT_PATH = None
        self.creating_project = False
        self._update_ui_for_project_creation()
        self.show_message(error, title="Error")

        if project_name_to_delete is not None:
            path = os.path.join(get_default_project_dir(), project_name_to_delete)
            rmtree(path)

    def _write_to_project_config(self):
        ts = time.time()
        vid_ts = self.ui.newp_video_start_time_input.dateTime().toPyDateTime()
        email = str(self.ui.newp_video_email_input.text())
        server = str(self.ui.newp_video_server_input.text())

        timestamp = datetime.datetime.fromtimestamp(ts).strftime('%d-%m-%Y %H:%M:%S %Z')
        video_timestamp = vid_ts.strftime('%d-%m-%Y %H:%M:%S %Z')
        self.config_parser.add_section("info")
        self.config_parser.set("info", "project_name", os.path.basename(ac.CURRENT_PROJECT_PATH))
        self.config_parser.set("info", "creation_date", timestamp)
        self.config_parser.set("info", "server", server)
        self.config_parser.set("info", "email", email)
        self.config_parser.add_section("video")
        video_extension = self.videopath.split('.')[-1]
        self.config_parser.set("video", "name", 'video.'+video_extension)
        self.config_parser.set("video", "source", self.videopath)
        self.config_parser.set("video", "start", video_timestamp)

        self.config_parser.add_section("config")
        success, _, config = api.defaultConfig()

        # If we can't get defaults, don't worry. Server fills them in anyway.
        if success:
            for (key, value) in config.iteritems():
                self.config_parser.set("config", key, str(value))

        with open(os.path.join(get_config_path()), 'wb') as configfile:
            self.config_parser.write(configfile)

    def load_new_project(self):
        load_project(ac.CURRENT_PROJECT_PATH, self.parent())

def load_project(project_path, main_window):
    ac.CURRENT_PROJECT_PATH = project_path

    load_homography(main_window)

    # Reload the URL for the project
    addr = get_config_with_sections(get_config_path(), "info", "server")
    update_api(addr)

    load_config(main_window)

def loadPointCorrespondences(filename):
    '''Loads and returns the corresponding points in world (first 2 lines) and image spaces (last 2 lines)'''
    points = np.loadtxt(filename, dtype=np.float32)
    return  (points[:2,:].T, points[2:,:].T) # (world points, image points)

def load_homography(main_window):
    """
    Loads homography information into the specified main window.
    """
    path = get_project_path()
    aerial_path = os.path.join(path, "homography", "aerial.png")
    camera_path = os.path.join(path, "homography", "camera.png")
    # TODO: Handle if above two paths do not exist
    load_from = 'image_pts'  # "image_pts" or "pt_corrs"
    gui = main_window.ui
    # Load images
    gui.homography_aerialview.load_image_from_path(aerial_path)
    gui.homography_cameraview.load_image_from_path(camera_path)

    goodness_path = os.path.join(path, "homography", "homography_goodness_aerial.png")
    image_pts_path = os.path.join(path, "homography", "image-points.txt")
    pt_corrs_path = os.path.join(path, "homography", "point-correspondences.txt")
    homo_path = os.path.join(path, "homography", "homography.txt")

    if load_from is "image_pts":
        corr_path = image_pts_path
    else:
        corr_path = pt_corrs_path

    # Has a homography been previously computed?
    if config_section_exists(get_config_path(), "homography"):  # If we can load homography unit-pix ratio load it
        # load unit-pixel ratio
        upr = get_config_with_sections(get_config_path(), "homography", "unitpixelratio")
        if upr:
            gui.unit_px_input.setText(upr)
    if os.path.exists(corr_path):  # If points have been previously selected
        worldPts, videoPts = loadPointCorrespondences(corr_path)
        main_window.homography = np.loadtxt(homo_path)
        if load_from is "image_pts":
            for point in worldPts:
                main_window.ui.homography_aerialview.scene().add_point(point)
        elif load_from is "pt_corrs":
            for point in worldPts:
                main_window.ui.homography_aerialview.scene().add_point(point/float(upr))
        else:
            print("ERR: Invalid point source {} specified. Points not loaded".format(load_from))
        for point in videoPts:
            main_window.ui.homography_cameraview.scene().add_point(point)
        gui.homography_results.load_image_from_path(goodness_path)
    else:
        print ("{} does not exist. No points loaded.".format(corr_path))

def update_api(addr):
    if addr:
        api.set_url(addr)
    else:
        print("No server, resorting to localhost")
        api.set_url('localhost')

def load_config(main_window):
    main_window.configGui_features.loadConfig_features()
    main_window.configGui_object.loadConfig_objects()
<|endoftext|>


<commit_before>from django.conf.urls.defaults import patterns, url
from django.views.generic import list_detail, ListView
from memopol2 import utils

from meps.models import MEP, Country, Group, Committee, Delegation, Organization, Building
from reps.models import Party

from views import BuildingDetailView

country_dict = {
  'queryset': Country.objects.all(),
  'slug_field': 'code',
  'template_name': 'meps/container_detail.html',
  'extra_context': {
    'hidden_fields': ['country'],
    'header_template': 'meps/country_header.html',
  },
}
party_dict = {
  'queryset': Party.objects.all(),
  'template_name': 'meps/container_detail.html',
  'extra_context': {
    'hidden_fields': ['party'],
    'header_template': 'meps/named_header.html',
  },
}
group_dict = {
  'queryset': Group.objects.all(),
  'slug_field': 'abbreviation',
  'template_name': 'meps/container_detail.html',
  'extra_context': {
    'hidden_fields': ['group'],
    'header_template': 'meps/group_header.html',
  },
}
delegation_dict = {
  'queryset': Delegation.objects.all(),
  'template_name': 'meps/container_detail.html',
  'extra_context': {
    'hidden_fields': [],
    'header_template': 'meps/named_header.html',
  },
}
committe_dict = {
  'queryset': Committee.objects.all(),
  'slug_field': 'abbreviation',
  'template_name': 'meps/container_detail.html',
  'extra_context': {
    'hidden_fields': [],
    'header_template': 'meps/named_header.html',
  },
}
organization_dict = {
  'queryset': Organization.objects.all(),
  'template_name': 'meps/container_detail.html',
  'extra_context': {
    'hidden_fields': [],
    'header_template': 'meps/named_header.html',
  },
}
mep_dict = {'queryset': MEP.objects.all(), 'slug_field': 'id', 'template_object_name': 'mep'}
mep_dict_dataporn = {'queryset': MEP.objects.all(), 'slug_field': 'id', 'template_object_name': 'mep', 'template_name': 'meps/dataporn.html'}
mep_contact_dict = {'queryset': MEP.objects.all(), 'slug_field': 'id',
                    'template_name': 'meps/mep_contact.html',
                    'template_object_name': 'mep'}

urlpatterns = patterns('meps.views',
    # those view are *very* expansive. we cache them in RAM for a week
    url(r'^names/$', utils.cached(3600*24*7)(list_detail.object_list), {'queryset': MEP.objects.filter(active=True)}, name='index_names'),
    url(r'^inactive/$', utils.cached(3600*24*7)(list_detail.object_list), {'queryset': MEP.objects.filter(active=False)}, name='index_inactive'),
    url(r'^score/$', list_detail.object_list, {'queryset': MEP.objects.filter(active=True).order_by('-total_score'), 'extra_context' : {'score_listing' : True}}, name='scores'),

    url(r'^organization/$', ListView.as_view(model=Organization), name='index_organizations'),
    url(r'^organization/(?P<object_id>[0-9]+)/$', list_detail.object_detail, organization_dict, name='index_by_organization'),
    url(r'^country/$', ListView.as_view(model=Country), name='index_countries'),
    url(r'^country/(?P<slug>[a-zA-Z][a-zA-Z])/$', list_detail.object_detail, country_dict, name='index_by_country'),
    url(r'^group/$', ListView.as_view(model=Group), name='index_groups'),
    url(r'^group/(?P<slug>[a-zA-Z/-]+)/$', list_detail.object_detail, group_dict,  name='index_by_group'),
    url(r'^committee/$', ListView.as_view(model=Committee), name='index_committees'),
    url(r'^committee/(?P<slug>[A-Z]+)/$', list_detail.object_detail, committe_dict, name='index_by_committee'),
    url(r'^delegation/$', ListView.as_view(model=Delegation), name='index_delegations'),
    url(r'^delegation/(?P<object_id>[0-9]+)/$', list_detail.object_detail, delegation_dict, name='index_by_delegation'),
    url(r'^party/$', ListView.as_view(model=Party), name='index_parties'),
    url(r'^party/(?P<object_id>[0-9]+)/$', list_detail.object_detail, party_dict,  name='index_by_party'),
    url(r'^floor/$', list_detail.object_list, {'queryset': Building.objects.all().order_by('postcode')}, name='index_floor'),
    url(r'^floor/brussels/(?P<pk>\w+)/(?P<floor>\w+)/$', BuildingDetailView.as_view(), name='bxl_floor'),

    url(r'^deputy/(?P<slug>\w+)/$', list_detail.object_detail, mep_dict, name='mep'),
    url(r'^deputy/(?P<slug>\w+)/dataporn/$', list_detail.object_detail, mep_dict_dataporn, name='mep_dataporn'),
    url(r'^deputy/(?P<slug>\w+)/contact$', list_detail.object_detail, mep_contact_dict, name='mep_contact'),
)
urlpatterns += patterns('meps.views',
    url(r'^mep/(?P<ep_id>[0-9]+)/picture.jpg$', 'get_mep_picture',
        name='mep-picture'),
)
<commit_msg>[mod] blank space<commit_after>from django.conf.urls.defaults import patterns, url
from django.views.generic import list_detail, ListView
from memopol2 import utils

from meps.models import MEP, Country, Group, Committee, Delegation, Organization, Building
from reps.models import Party

from views import BuildingDetailView

country_dict = {
  'queryset': Country.objects.all(),
  'slug_field': 'code',
  'template_name': 'meps/container_detail.html',
  'extra_context': {
    'hidden_fields': ['country'],
    'header_template': 'meps/country_header.html',
  },
}
party_dict = {
  'queryset': Party.objects.all(),
  'template_name': 'meps/container_detail.html',
  'extra_context': {
    'hidden_fields': ['party'],
    'header_template': 'meps/named_header.html',
  },
}
group_dict = {
  'queryset': Group.objects.all(),
  'slug_field': 'abbreviation',
  'template_name': 'meps/container_detail.html',
  'extra_context': {
    'hidden_fields': ['group'],
    'header_template': 'meps/group_header.html',
  },
}
delegation_dict = {
  'queryset': Delegation.objects.all(),
  'template_name': 'meps/container_detail.html',
  'extra_context': {
    'hidden_fields': [],
    'header_template': 'meps/named_header.html',
  },
}
committe_dict = {
  'queryset': Committee.objects.all(),
  'slug_field': 'abbreviation',
  'template_name': 'meps/container_detail.html',
  'extra_context': {
    'hidden_fields': [],
    'header_template': 'meps/named_header.html',
  },
}
organization_dict = {
  'queryset': Organization.objects.all(),
  'template_name': 'meps/container_detail.html',
  'extra_context': {
    'hidden_fields': [],
    'header_template': 'meps/named_header.html',
  },
}
mep_dict = {'queryset': MEP.objects.all(), 'slug_field': 'id', 'template_object_name': 'mep'}
mep_dict_dataporn = {'queryset': MEP.objects.all(), 'slug_field': 'id', 'template_object_name': 'mep', 'template_name': 'meps/dataporn.html'}
mep_contact_dict = {'queryset': MEP.objects.all(), 'slug_field': 'id',
                    'template_name': 'meps/mep_contact.html',
                    'template_object_name': 'mep'}

urlpatterns = patterns('meps.views',
    # those view are *very* expansive. we cache them in RAM for a week
    url(r'^names/$', utils.cached(3600*24*7)(list_detail.object_list), {'queryset': MEP.objects.filter(active=True)}, name='index_names'),
    url(r'^inactive/$', utils.cached(3600*24*7)(list_detail.object_list), {'queryset': MEP.objects.filter(active=False)}, name='index_inactive'),
    url(r'^score/$', list_detail.object_list, {'queryset': MEP.objects.filter(active=True).order_by('-total_score'), 'extra_context' : {'score_listing' : True}}, name='scores'),

    url(r'^organization/$', ListView.as_view(model=Organization), name='index_organizations'),
    url(r'^organization/(?P<object_id>[0-9]+)/$', list_detail.object_detail, organization_dict, name='index_by_organization'),
    url(r'^country/$', ListView.as_view(model=Country), name='index_countries'),
    url(r'^country/(?P<slug>[a-zA-Z][a-zA-Z])/$', list_detail.object_detail, country_dict, name='index_by_country'),
    url(r'^group/$', ListView.as_view(model=Group), name='index_groups'),
    url(r'^group/(?P<slug>[a-zA-Z/-]+)/$', list_detail.object_detail, group_dict,  name='index_by_group'),
    url(r'^committee/$', ListView.as_view(model=Committee), name='index_committees'),
    url(r'^committee/(?P<slug>[A-Z]+)/$', list_detail.object_detail, committe_dict, name='index_by_committee'),
    url(r'^delegation/$', ListView.as_view(model=Delegation), name='index_delegations'),
    url(r'^delegation/(?P<object_id>[0-9]+)/$', list_detail.object_detail, delegation_dict, name='index_by_delegation'),
    url(r'^party/$', ListView.as_view(model=Party), name='index_parties'),
    url(r'^party/(?P<object_id>[0-9]+)/$', list_detail.object_detail, party_dict,  name='index_by_party'),
    url(r'^floor/$', list_detail.object_list, {'queryset': Building.objects.all().order_by('postcode')}, name='index_floor'),
    url(r'^floor/brussels/(?P<pk>\w+)/(?P<floor>\w+)/$', BuildingDetailView.as_view(), name='bxl_floor'),

    url(r'^deputy/(?P<slug>\w+)/$', list_detail.object_detail, mep_dict, name='mep'),
    url(r'^deputy/(?P<slug>\w+)/dataporn/$', list_detail.object_detail, mep_dict_dataporn, name='mep_dataporn'),
    url(r'^deputy/(?P<slug>\w+)/contact$', list_detail.object_detail, mep_contact_dict, name='mep_contact'),
)

urlpatterns += patterns('meps.views',
    url(r'^mep/(?P<ep_id>[0-9]+)/picture.jpg$', 'get_mep_picture',
        name='mep-picture'),
)
<|endoftext|>


<commit_before># Django settings for pipeye project.

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': '',                      # Or path to database file if using sqlite3.
        'USER': '',                      # Not used with sqlite3.
        'PASSWORD': '',                  # Not used with sqlite3.
        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
    }
}

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = ''

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = '$9-lfu%()3*dgup6q2)0jb2-^dcwj_ac@z+(^f8h$rr0^#=r$('

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'pipeye.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'pipeye.wsgi.application'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    # Uncomment the next line to enable the admin:
    # 'django.contrib.admin',
    # Uncomment the next line to enable admin documentation:
    # 'django.contrib.admindocs',
)

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}
<commit_msg>Changes settings<commit_after># Django settings for pipeye project.
from os import path, environ
ROOT = path.dirname(path.dirname(__file__))

DEBUG = environ.get('DEBUG', False)
TEMPLATE_DEBUG = DEBUG

ADMINS = [('Tomek Paczkowski', 'tomek@hauru.eu')]

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': '',
        'USER': '',
        'PASSWORD': '',
        'HOST': '',
        'PORT': '',
    }
}

# SITE_ID = 1

LANGUAGE_CODE = 'en-us'
USE_I18N = True
USE_L10N = True

TIME_ZONE = 'UTC'
USE_TZ = True

MEDIA_ROOT = ''
MEDIA_URL = ''

STATIC_ROOT = path.join(ROOT, 'public')
STATIC_URL = '/static/'
STATICFILES_DIRS = [path.join(ROOT, 'static')]
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
)

TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)
TEMPLATE_DIRS = [path.join(ROOT, 'templates')]

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'pipeye.urls'
WSGI_APPLICATION = 'pipeye.wsgi.application'

SECRET_KEY = environ.get('SECRET', '')

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
)

# A sample logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}
<|endoftext|>


<commit_before>#!/usr/bin/python

# Copyright (c) 2015 SUSE LINUX GmbH, Nuernberg, Germany.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

import argparse
import functools
import os
import sys
import tempfile

import sh
from sh import Command

IBS_MAPPING = {
    'release/stoney/master': 'Devel:Cloud:4:Staging',
    'release/tex/master':    'Devel:Cloud:5:Staging',
    'master':                'Devel:Cloud:6:Staging'
}

CLOUDSRC = {
    'release/stoney/master': 'develcloud4',
    'release/tex/master':    'develcloud5',
    'master':                'develcloud6'
}

JOB_PARAMETERS = {
    'barclamp-ceph': ('nodenumber=3', 'networkingplugin=linuxbridge'),
    'barclamp-pacemaker': ('nodenumber=3', 'hacloud=1')
}

htdocs_dir = '/srv/www/htdocs/mkcloud'
htdocs_url = 'http://tu-sle12.j.cloud.suse.de/mkcloud/'

iosc = functools.partial(
    Command('/usr/bin/osc'), '-A', 'https://api.suse.de')

def ghs_set_status(repo, pr_id, head_sha1, status):
    ghs = Command(
        os.path.abspath(
            os.path.join(os.path.dirname(sys.argv[0]),
                         'github-status/github-status.rb')))

    ghs('-r', 'crowbar/' + repo,
        '-p', pr_id, '-c', head_sha1, '-a', 'set-status',
        '-s', status)

def jenkins_job_trigger(repo, github_opts, cloudsource, ptfdir):
    print("triggering jenkins job with " + htdocs_url + ptfdir)

    jenkins = Command(
        os.path.abspath(
            os.path.join(os.path.dirname(sys.argv[0]),
                         'jenkins/jenkins-job-trigger')))

    job_parameters = (
        'nodenumber=2', 'networkingplugin=openvswitch')

    if repo in JOB_PARAMETERS:
        job_parameters = JOB_PARAMETERS[repo]

    job_parameters += ('all_noreboot',)

    print(jenkins(
        'openstack-mkcloud',
        '-p', 'mode=standard',
        "github_pr=crowbar/%s:%s" % (repo, github_opts),
        "cloudsource=" + cloudsource,
        'label=openstack-mkcloud-SLE12',
        'UPDATEREPOS=' + htdocs_url + ptfdir,
        'mkcloudtarget=all_noreboot',
        *job_parameters))

def add_pr_to_checkout(repo, pr_id, head_sha1, pr_branch, spec):
    sh.curl(
        '-s', '-k', '-L',
        "https://github.com/crowbar/%s/compare/%s...%s.patch" % (
            repo, pr_branch, head_sha1),
        '-o', 'prtest.patch')
    sh.sed('-i', '-e', 's,Url:.*,%define _default_patch_fuzz 2,',
           '-e', 's,%patch[0-36-9].*,,', spec)
    Command('/usr/lib/build/spec_add_patch')(spec, 'prtest.patch')
    iosc('vc', '-m', " added PR test patch from %s#%s (%s)" % (
        repo, pr_id, head_sha1))

def trigger_testbuild(repo, github_opts):
    pr_id, head_sha1, pr_branch = github_opts.split(':')

    olddir = os.getcwd()
    workdir = tempfile.mkdtemp()
    build_failed = False
    try:
        ptfdir = repo + ':' + github_opts
        webroot = os.path.join(htdocs_dir, ptfdir)
        pkg = repo if repo == "crowbar" else "crowbar-" + repo
        spec = pkg + '.spec'

        sh.rm('-rf', webroot)
        sh.mkdir('-p', webroot)
        try:
            os.chdir(workdir)
            buildroot = os.path.join(os.getcwd(), 'BUILD')
            iosc('co', IBS_MAPPING[pr_branch], pkg, '-c')
            os.chdir(pkg)
            add_pr_to_checkout(repo, pr_id, head_sha1, pr_branch, spec)
            repository = 'SLE_12' if pr_branch == 'master' else 'SLE_11_SP3'
            iosc('build', '--root', buildroot, '--noverify', '--noservice',
                 repository, 'x86_64', spec, _out=sys.stdout)
        except:
            build_failed = True
            print("Build failed: " + str(sys.exc_info()[0]))
            raise
        else:
            sh.cp('-p', sh.glob(
                os.path.join(buildroot, '.build.packages/RPMS/*/*.rpm')),
                webroot)
        finally:
            os.chdir(olddir)
            sh.cp('-p', os.path.join(buildroot, '.build.log'),
                  os.path.join(webroot, 'build.log'))
    finally:
        sh.sudo.rm('-rf', workdir)

    if not build_failed:
        jenkins_job_trigger(
            repo, github_opts, CLOUDSRC[pr_branch], ptfdir)

    ghs_set_status(
        repo, pr_id, head_sha1,
        'failure' if build_failed else'pending')

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Test a crowbar/ PR')
    parser.add_argument("repo", help='github ORG/REPO')
    parser.add_argument('pr', help='github PR <PRID>:<SHA1>:<BRANCH>')

    args = parser.parse_args()

    trigger_testbuild(args.repo, args.pr)
    sys.exit(0)
<commit_msg>Fix mkcloud testbuild script for gating<commit_after>#!/usr/bin/python

# Copyright (c) 2015 SUSE LINUX GmbH, Nuernberg, Germany.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

import argparse
import functools
import os
import sys
import tempfile

import sh
from sh import Command

IBS_MAPPING = {
    'release/stoney/master': 'Devel:Cloud:4:Staging',
    'release/tex/master':    'Devel:Cloud:5:Staging',
    'master':                'Devel:Cloud:6:Staging'
}

CLOUDSRC = {
    'release/stoney/master': 'develcloud4',
    'release/tex/master':    'develcloud5',
    'master':                'develcloud6'
}

JOB_PARAMETERS = {
    'barclamp-ceph': ('nodenumber=3', 'networkingplugin=linuxbridge'),
    'barclamp-pacemaker': ('nodenumber=3', 'hacloud=1')
}

htdocs_dir = '/srv/www/htdocs/mkcloud'
htdocs_url = 'http://tu-sle12.j.cloud.suse.de/mkcloud/'

iosc = functools.partial(
    Command('/usr/bin/osc'), '-A', 'https://api.suse.de')

def ghs_set_status(repo, pr_id, head_sha1, status):
    ghs = Command(
        os.path.abspath(
            os.path.join(os.path.dirname(sys.argv[0]),
                         'github-status/github-status.rb')))

    ghs('-r', 'crowbar/' + repo,
        '-p', pr_id, '-c', head_sha1, '-a', 'set-status',
        '-s', status)

def jenkins_job_trigger(repo, github_opts, cloudsource, ptfdir):
    print("triggering jenkins job with " + htdocs_url + ptfdir)

    jenkins = Command(
        os.path.abspath(
            os.path.join(os.path.dirname(sys.argv[0]),
                         'jenkins/jenkins-job-trigger')))

    job_parameters = (
        'nodenumber=2', 'networkingplugin=openvswitch')

    if repo in JOB_PARAMETERS:
        job_parameters = JOB_PARAMETERS[repo]

    job_parameters += ('all_noreboot',)

    print(jenkins(
        'openstack-mkcloud',
        '-p', 'mode=standard',
        "github_pr=crowbar/%s:%s" % (repo, github_opts),
        "cloudsource=" + cloudsource,
        'label=openstack-mkcloud-SLE12',
        'UPDATEREPOS=' + htdocs_url + ptfdir,
        'mkcloudtarget=all_noreboot',
        *job_parameters))

def add_pr_to_checkout(repo, pr_id, head_sha1, pr_branch, spec):
    sh.curl(
        '-s', '-k', '-L',
        "https://github.com/crowbar/%s/compare/%s...%s.patch" % (
            repo, pr_branch, head_sha1),
        '-o', 'prtest.patch')
    sh.sed('-i', '-e', 's,Url:.*,%define _default_patch_fuzz 2,',
           '-e', 's,%patch[0-36-9].*,,', spec)
    Command('/usr/lib/build/spec_add_patch')(spec, 'prtest.patch')
    iosc('vc', '-m', " added PR test patch from %s#%s (%s)" % (
        repo, pr_id, head_sha1))

def trigger_testbuild(repo, github_opts):
    pr_id, head_sha1, pr_branch = github_opts.split(':')

    olddir = os.getcwd()
    workdir = tempfile.mkdtemp()
    build_failed = False
    try:
        ptfdir = repo + ':' + github_opts
        webroot = os.path.join(htdocs_dir, ptfdir)

        if "crowbar" in repo:
            pkg = repo
        else:
            pkg = "crowbar-" + repo

        spec = pkg + '.spec'

        sh.rm('-rf', webroot)
        sh.mkdir('-p', webroot)
        try:
            os.chdir(workdir)
            buildroot = os.path.join(os.getcwd(), 'BUILD')
            iosc('co', IBS_MAPPING[pr_branch], pkg, '-c')
            os.chdir(pkg)
            add_pr_to_checkout(repo, pr_id, head_sha1, pr_branch, spec)
            repository = 'SLE_12' if pr_branch == 'master' else 'SLE_11_SP3'
            iosc('build', '--root', buildroot, '--noverify', '--noservice',
                 repository, 'x86_64', spec, _out=sys.stdout)
        except:
            build_failed = True
            print("Build failed: " + str(sys.exc_info()[0]))
            raise
        else:
            sh.cp('-p', sh.glob(
                os.path.join(buildroot, '.build.packages/RPMS/*/*.rpm')),
                webroot)
        finally:
            os.chdir(olddir)
            sh.cp('-p', os.path.join(buildroot, '.build.log'),
                  os.path.join(webroot, 'build.log'))
    finally:
        sh.sudo.rm('-rf', workdir)

    if not build_failed:
        jenkins_job_trigger(
            repo, github_opts, CLOUDSRC[pr_branch], ptfdir)

    ghs_set_status(
        repo, pr_id, head_sha1,
        'failure' if build_failed else'pending')

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Test a crowbar/ PR')
    parser.add_argument("repo", help='github ORG/REPO')
    parser.add_argument('pr', help='github PR <PRID>:<SHA1>:<BRANCH>')

    args = parser.parse_args()

    trigger_testbuild(args.repo, args.pr)
    sys.exit(0)
<|endoftext|>


<commit_before>import argparse
import os
import pip
import piprepo
import re
import tempfile

class Index(object):
    html_header = '<!DOCTYPE html>\n'
    html_footer = '\n'

    def __init__(self, directory):
        self.directory = directory
        self.index_root = directory.rstrip('/') + '/simple'
        self.packages = self._build_packages()

    def _build_packages(self):
        packages = {}
        files = [f for f in os.listdir(self.directory) if os.path.isfile(self.directory + '/' + f)]
        for p in files:
            name = os.path.basename(p).split('-')[0]
            # normalize name as per https://www.python.org/dev/peps/pep-0503/
            name = re.sub(r"[-_.]+", "-", name).lower()
            if name in packages:
                packages[name].append(p)
            else:
                packages[name] = [p]
        return packages

    def save(self):
        # create index directories
        indexes = [self.index_root] + [self.index_root + '/' + p for p in self.packages.keys()]
        for i in indexes:
            try:
                os.mkdir(i)
            except OSError:
                pass

        # create root index
        packages = ['{0}
\n'.format(p) for p in sorted(self.packages.keys())]
        lines = [self.html_header] + packages + [self.html_footer]
        self._write_file(self.index_root + '/index.html', lines)

        # create package indexes
        for package, files in self.packages.iteritems():
            versions = ['{0}
\n'.format(p) for p in sorted(files)]
            lines = [self.html_header] + versions + [self.html_footer]
            self._write_file(self.index_root + '/' + package + '/index.html', lines)

    def _write_file(self, filename, lines):
        with open(filename, 'w') as f:
            f.writelines(lines)

def parse_args():
    ''' Sets up argparse and returns a tuple of (piprepo_args, pip_args) '''

    usage = '''
  piprepo [-hsw] target [pip_options] packages
  piprepo [-hsw] target [pip_options] -r requirements_file'''

    parser = argparse.ArgumentParser(
        description=piprepo.__description__,
        usage=usage
    )
    parser.add_argument('target', help='Repository target')
    parser.add_argument('-s', '--get-source',
                        default=False, action='store_true',
                        help='Download source tarballs')
    parser.add_argument('-w', '--build-wheels',
                        default=False, action='store_true',
                        help='Build wheels from source packages')

    return parser.parse_known_args()

def get_working_directory(target):
    ''' Returns temporary working directory for remote targets '''
    if target.startswith('s3://'):
        return tempfile.mkdtemp()
    elif target.startswith('rsync://'):
        return tempfile.mkdtemp()
    return target

def main():
    args, pip_args = parse_args()
    working_directory = get_working_directory(args.target)

    # let index decide what to download unless forcing wheels and tarballs
    if not (args.get_source and args.build_wheels):
        pip.main(['download', '-d', working_directory] + pip_args)
    # download source tarballs only
    if args.get_source:
        pip.main(['download', '-d', working_directory, '--no-binary', ':all:'] + pip_args)
    # download/compile wheels only
    if args.build_wheels:
        pip.main(['wheel', '--wheel-dir', working_directory] + pip_args)

    # create index and write index files
    Index(working_directory).save()

    # TODO: publish to remote targets
<commit_msg>rescope variable<commit_after>import argparse
import os
import pip
import piprepo
import re
import tempfile

class Index(object):
    html_header = '<!DOCTYPE html>\n'
    html_footer = '\n'

    def __init__(self, directory):
        self.directory = directory
        self.index_root = directory.rstrip('/') + '/simple'
        self.packages = self._build_packages()

    def _build_packages(self):
        packages = {}
        files = [f for f in os.listdir(self.directory) if os.path.isfile(self.directory + '/' + f)]
        for p in files:
            name = os.path.basename(p).split('-')[0]
            # normalize name as per https://www.python.org/dev/peps/pep-0503/
            name = re.sub(r"[-_.]+", "-", name).lower()
            if name in packages:
                packages[name].append(p)
            else:
                packages[name] = [p]
        return packages

    def save(self):
        # create index directories
        for i in [self.index_root] + [self.index_root + '/' + p for p in self.packages.keys()]:
            try:
                os.mkdir(i)
            except OSError:
                pass

        # create root index
        packages = ['{0}
\n'.format(p) for p in sorted(self.packages.keys())]
        lines = [self.html_header] + packages + [self.html_footer]
        self._write_file(self.index_root + '/index.html', lines)

        # create package indexes
        for package, files in self.packages.iteritems():
            versions = ['{0}
\n'.format(p) for p in sorted(files)]
            lines = [self.html_header] + versions + [self.html_footer]
            self._write_file(self.index_root + '/' + package + '/index.html', lines)

    def _write_file(self, filename, lines):
        with open(filename, 'w') as f:
            f.writelines(lines)

def parse_args():
    ''' Sets up argparse and returns a tuple of (piprepo_args, pip_args) '''

    usage = '''
  piprepo [-hsw] target [pip_options] packages
  piprepo [-hsw] target [pip_options] -r requirements_file'''

    parser = argparse.ArgumentParser(
        description=piprepo.__description__,
        usage=usage
    )
    parser.add_argument('target', help='Repository target')
    parser.add_argument('-s', '--get-source',
                        default=False, action='store_true',
                        help='Download source tarballs')
    parser.add_argument('-w', '--build-wheels',
                        default=False, action='store_true',
                        help='Build wheels from source packages')

    return parser.parse_known_args()

def get_working_directory(target):
    ''' Returns temporary working directory for remote targets '''
    if target.startswith('s3://'):
        return tempfile.mkdtemp()
    elif target.startswith('rsync://'):
        return tempfile.mkdtemp()
    return target

def main():
    args, pip_args = parse_args()
    working_directory = get_working_directory(args.target)

    # let index decide what to download unless forcing wheels and tarballs
    if not (args.get_source and args.build_wheels):
        pip.main(['download', '-d', working_directory] + pip_args)
    # download source tarballs only
    if args.get_source:
        pip.main(['download', '-d', working_directory, '--no-binary', ':all:'] + pip_args)
    # download/compile wheels only
    if args.build_wheels:
        pip.main(['wheel', '--wheel-dir', working_directory] + pip_args)

    # create index and write index files
    Index(working_directory).save()

    # TODO: publish to remote targets
<|endoftext|>


<commit_before>#!/usr/bin/python

# Copyright (c) 2015 SUSE LINUX GmbH, Nuernberg, Germany.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

import argparse
import functools
import os
import sys
import tempfile

import sh
from sh import Command

IBS_MAPPING = {
    'release/stoney/master': 'Devel:Cloud:4:Staging',
    'release/tex/master':    'Devel:Cloud:5:Staging',
    'master':                'Devel:Cloud:6:Staging'
}

CLOUDSRC = {
    'release/stoney/master': 'develcloud4',
    'release/tex/master':    'develcloud5',
    'master':                'develcloud6'
}

JOB_PARAMETERS = {
    'barclamp-ceph': ('nodenumber=3', 'networkingplugin=linuxbridge'),
    'barclamp-pacemaker': ('nodenumber=3', 'hacloud=1')
}

htdocs_dir = '/srv/www/htdocs/mkcloud'
htdocs_url = 'http://tu-sle12.j.cloud.suse.de/mkcloud/'

iosc = functools.partial(
    Command('/usr/bin/osc'), '-A', 'https://api.suse.de')

def ghs_set_status(repo, pr_id, head_sha1, status):
    ghs = Command(
        os.path.abspath(
            os.path.join(os.path.dirname(sys.argv[0]),
                         'github-status/github-status.rb')))

    ghs('-r', 'crowbar/' + repo,
        '-p', pr_id, '-c', head_sha1, '-a', 'set-status',
        '-s', status)

def jenkins_job_trigger(repo, github_opts, cloudsource, ptfdir):
    print("triggering jenkins job with " + htdocs_url + ptfdir)

    jenkins = Command(
        os.path.abspath(
            os.path.join(os.path.dirname(sys.argv[0]),
                         'jenkins/jenkins-job-trigger')))

    job_parameters = (
        'nodenumber=2', 'networkingplugin=openvswitch')

    if repo in JOB_PARAMETERS:
        job_parameters = JOB_PARAMETERS[repo]

    job_parameters += ('all_noreboot',)

    print(jenkins(
        'openstack-mkcloud',
        '-p', 'mode=standard',
        "github_pr=crowbar/%s:%s" % (repo, github_opts),
        "cloudsource=" + cloudsource,
        'label=openstack-mkcloud-SLE12',
        'UPDATEREPOS=' + htdocs_url + ptfdir,
        'mkcloudtarget=all_noreboot',
        *job_parameters))

def add_pr_to_checkout(repo, pr_id, head_sha1, pr_branch, spec):
    sh.curl(
        '-s', '-k', '-L',
        "https://github.com/crowbar/%s/compare/%s...%s.patch" % (
            repo, pr_branch, head_sha1),
        '-o', 'prtest.patch')
    sh.sed('-i', '-e', 's,Url:.*,%define _default_patch_fuzz 2,',
           '-e', 's,%patch[0-36-9].*,,', spec)
    Command('/usr/lib/build/spec_add_patch')(spec, 'prtest.patch')
    iosc('vc', '-m', " added PR test patch from %s#%s (%s)" % (
        repo, pr_id, head_sha1))

def trigger_testbuild(repo, github_opts):
    pr_id, head_sha1, pr_branch = github_opts.split(':')

    olddir = os.getcwd()
    workdir = tempfile.mkdtemp()
    build_failed = False
    try:
        ptfdir = repo + ':' + github_opts
        webroot = os.path.join(htdocs_dir, ptfdir)
        pkg = repo if repo == "crowbar" else "crowbar-" + repo
        spec = pkg + '.spec'

        sh.rm('-rf', webroot)
        sh.mkdir('-p', webroot)

        try:
            os.chdir(workdir)
            buildroot = os.path.join(os.getcwd(), 'BUILD')
            iosc('co', IBS_MAPPING[pr_branch], pkg)
            os.chdir(os.path.join(IBS_MAPPING[pr_branch], pkg))
            add_pr_to_checkout(repo, pr_id, head_sha1, pr_branch, spec)
            iosc('build', '--root', buildroot,
                 '--noverify', '--noservice', 'SLE_11_SP3', 'x86_64',
                 spec, _out=sys.stdout)
        except:
            build_failed = True
            print("Build failed: " + str(sys.exc_info()[0]))
            raise
        else:
            sh.cp('-p',
                  sh.glob(os.path.join(buildroot,
                                       'usr/src/packages/RPMS/*/*.rpm')),
                  webroot)

        finally:
            os.chdir(olddir)
            sh.cp(
                '-p', os.path.join(buildroot, '.build.log'),
                os.path.join(webroot, 'build.log'))
    finally:
        sh.sudo.rm('-rf', workdir)

    if not build_failed:
        jenkins_job_trigger(
            repo, github_opts, CLOUDSRC[pr_branch], ptfdir)

    ghs_set_status(
        repo, pr_id, head_sha1,
        'failure' if build_failed else'pending')

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Test a crowbar/ PR')
    parser.add_argument("repo", help='github ORG/REPO')
    parser.add_argument('pr', help='github PR <PRID>:<SHA1>:<BRANCH>')

    args = parser.parse_args()

    trigger_testbuild(args.repo, args.pr)
    sys.exit(0)
<commit_msg>Switch build of PR PTFs for master to SLE_12<commit_after>#!/usr/bin/python

# Copyright (c) 2015 SUSE LINUX GmbH, Nuernberg, Germany.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

import argparse
import functools
import os
import sys
import tempfile

import sh
from sh import Command

IBS_MAPPING = {
    'release/stoney/master': 'Devel:Cloud:4:Staging',
    'release/tex/master':    'Devel:Cloud:5:Staging',
    'master':                'Devel:Cloud:6:Staging'
}

CLOUDSRC = {
    'release/stoney/master': 'develcloud4',
    'release/tex/master':    'develcloud5',
    'master':                'develcloud6'
}

JOB_PARAMETERS = {
    'barclamp-ceph': ('nodenumber=3', 'networkingplugin=linuxbridge'),
    'barclamp-pacemaker': ('nodenumber=3', 'hacloud=1')
}

htdocs_dir = '/srv/www/htdocs/mkcloud'
htdocs_url = 'http://tu-sle12.j.cloud.suse.de/mkcloud/'

iosc = functools.partial(
    Command('/usr/bin/osc'), '-A', 'https://api.suse.de')

def ghs_set_status(repo, pr_id, head_sha1, status):
    ghs = Command(
        os.path.abspath(
            os.path.join(os.path.dirname(sys.argv[0]),
                         'github-status/github-status.rb')))

    ghs('-r', 'crowbar/' + repo,
        '-p', pr_id, '-c', head_sha1, '-a', 'set-status',
        '-s', status)

def jenkins_job_trigger(repo, github_opts, cloudsource, ptfdir):
    print("triggering jenkins job with " + htdocs_url + ptfdir)

    jenkins = Command(
        os.path.abspath(
            os.path.join(os.path.dirname(sys.argv[0]),
                         'jenkins/jenkins-job-trigger')))

    job_parameters = (
        'nodenumber=2', 'networkingplugin=openvswitch')

    if repo in JOB_PARAMETERS:
        job_parameters = JOB_PARAMETERS[repo]

    job_parameters += ('all_noreboot',)

    print(jenkins(
        'openstack-mkcloud',
        '-p', 'mode=standard',
        "github_pr=crowbar/%s:%s" % (repo, github_opts),
        "cloudsource=" + cloudsource,
        'label=openstack-mkcloud-SLE12',
        'UPDATEREPOS=' + htdocs_url + ptfdir,
        'mkcloudtarget=all_noreboot',
        *job_parameters))

def add_pr_to_checkout(repo, pr_id, head_sha1, pr_branch, spec):
    sh.curl(
        '-s', '-k', '-L',
        "https://github.com/crowbar/%s/compare/%s...%s.patch" % (
            repo, pr_branch, head_sha1),
        '-o', 'prtest.patch')
    sh.sed('-i', '-e', 's,Url:.*,%define _default_patch_fuzz 2,',
           '-e', 's,%patch[0-36-9].*,,', spec)
    Command('/usr/lib/build/spec_add_patch')(spec, 'prtest.patch')
    iosc('vc', '-m', " added PR test patch from %s#%s (%s)" % (
        repo, pr_id, head_sha1))

def trigger_testbuild(repo, github_opts):
    pr_id, head_sha1, pr_branch = github_opts.split(':')

    olddir = os.getcwd()
    workdir = tempfile.mkdtemp()
    build_failed = False
    try:
        ptfdir = repo + ':' + github_opts
        webroot = os.path.join(htdocs_dir, ptfdir)
        pkg = repo if repo == "crowbar" else "crowbar-" + repo
        spec = pkg + '.spec'

        sh.rm('-rf', webroot)
        sh.mkdir('-p', webroot)
        try:
            os.chdir(workdir)
            buildroot = os.path.join(os.getcwd(), 'BUILD')
            iosc('co', IBS_MAPPING[pr_branch], pkg)
            os.chdir(os.path.join(IBS_MAPPING[pr_branch], pkg))
            add_pr_to_checkout(repo, pr_id, head_sha1, pr_branch, spec)
            repository = 'SLE_12' if pr_branch == 'master' else 'SLE_11_SP3'
            iosc('build', '--root', buildroot, '--noverify', '--noservice',
                 repository, 'x86_64', spec, _out=sys.stdout)
        except:
            build_failed = True
            print("Build failed: " + str(sys.exc_info()[0]))
            raise
        else:
            sh.cp('-p', sh.glob(
                os.path.join(buildroot, '.build.packages/RPMS/*/*.rpm')),
                webroot)
        finally:
            os.chdir(olddir)
            sh.cp('-p', os.path.join(buildroot, '.build.log'),
                  os.path.join(webroot, 'build.log'))
    finally:
        sh.sudo.rm('-rf', workdir)

    if not build_failed:
        jenkins_job_trigger(
            repo, github_opts, CLOUDSRC[pr_branch], ptfdir)

    ghs_set_status(
        repo, pr_id, head_sha1,
        'failure' if build_failed else'pending')

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Test a crowbar/ PR')
    parser.add_argument("repo", help='github ORG/REPO')
    parser.add_argument('pr', help='github PR <PRID>:<SHA1>:<BRANCH>')

    args = parser.parse_args()

    trigger_testbuild(args.repo, args.pr)
    sys.exit(0)
<|endoftext|>


<commit_before># -*- coding: utf8 -*-

"""Parsers for extracting @mentions, #hashtags and URLs from posts.

.. note: These have been split appart from posts backend so they can be
         maintained more easily.

:license: AGPL v3, see LICENSE for more details
:copyright: 2014-2015 Joe Doherty

"""

import re

from pjuu.auth.utils import get_uid_username
from pjuu.lib import fix_url

# Regular expressions for highlighting URLs, @mentions and #hashtags
# URL matching pattern; thanks to John Gruber @ http://daringfireball.net/
# https://gist.github.com/gruber/8891611
URL_RE_PATTERN = (
    r'(?i)\b((?:https?:(?:/{1,3}|[a-z0-9%])|[a-z0-9.\-]+[.](?:com|net|org|edu|'
    r'gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|'
    r'tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|'
    r'bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|'
    r'ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|'
    r'er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|'
    r'gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|'
    r'jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|'
    r'mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|'
    r'nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|'
    r'qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|'
    r'su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|'
    r'uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)/)(?:[^\s()<>{}'
    r'\[\]]+|\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\))+(?:\([^\s()]*?\(['
    r'^\s()]+\)[^\s()]*?\)|\([^\s]+?\)|[^\s`!()\[\]{};:\'".,<>?«»“”‘’])|(?:(?<'
    r'!@)[a-z0-9]+(?:[.\-][a-z0-9]+)*[.](?:com|net|org|edu|gov|mil|aero|asia|'
    r'biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|'
    r'ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|'
    r'bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|'
    r'cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|'
    r'fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|'
    r'hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|'
    r'kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|'
    r'ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|'
    r'nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|'
    r'sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|'
    r'td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|'
    r've|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)\b/?(?!@)))'
)
URL_RE = re.compile(URL_RE_PATTERN)

MENTION_RE = re.compile(
    r'(?:^|(?<=[\(\[\{ \t\r\n]))@(\w{3,16})(?:$|(?=[.;,:?\)\]\} \t\r\n]))'
)

HASHTAG_RE = re.compile(
    r'(?:^|(?<=[\(\[\{ \t\r\n]))#(\w{2,32})(?:$|(?=[.;,:?\)\]\} \t\r\n]))'
)

def parse_links(body):
    """Parses URLs out of a post.

    .. note: This will need to be refined as edge cases are discovered.

    """
    links = URL_RE.finditer(body)

    result = []
    for link in links:
        result.append({
            'link': fix_url(link.group(0)),
            'span': link.span()
        })

    return result

def parse_mentions(body, check_user=True):
    """Parses @mentions out of a post.

    .. note: This will need to be refined as edge cases are discovered.

    """
    mentions = MENTION_RE.finditer(body)

    result = []
    for mention in mentions:
        username = mention.group(1)
        if check_user:
            user_id = get_uid_username(username)
        else:
            user_id = 'NA'

        if user_id:
            result.append({
                'user_id': user_id,
                'username': username,
                'span': mention.span()
            })

    return result

def parse_hashtags(body):
    """Parsed #hashtags out of a post.

    .. note: This will need to be refined as edge cases are discovered.

    """
    hashtags = HASHTAG_RE.finditer(body)

    result = []
    for hashtag in hashtags:
        result.append({
            'hashtag': hashtag.group(1).lower(),
            'span': hashtag.span()
        })

    return result

def parse_post(body):
    """Calls the individual parsers and returns them all in a multiple return
    statement.

    """
    links = parse_links(body)
    mentions = parse_mentions(body)
    hashtags = parse_hashtags(body)

    return links, mentions, hashtags
<commit_msg>Cleaned up delimers in regex and added '!'.<commit_after># -*- coding: utf8 -*-

"""Parsers for extracting @mentions, #hashtags and URLs from posts.

.. note: These have been split appart from posts backend so they can be
         maintained more easily.

:license: AGPL v3, see LICENSE for more details
:copyright: 2014-2015 Joe Doherty

"""

import re

from pjuu.auth.utils import get_uid_username
from pjuu.lib import fix_url

# Regular expressions for highlighting URLs, @mentions and #hashtags
# URL matching pattern; thanks to John Gruber @ http://daringfireball.net/
# https://gist.github.com/gruber/8891611
URL_RE_PATTERN = (
    r'(?i)\b((?:https?:(?:/{1,3}|[a-z0-9%])|[a-z0-9.\-]+[.](?:com|net|org|edu|'
    r'gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|'
    r'tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|'
    r'bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|'
    r'ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|'
    r'er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|'
    r'gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|'
    r'jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|'
    r'mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|'
    r'nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|'
    r'qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|'
    r'su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|'
    r'uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)/)(?:[^\s()<>{}'
    r'\[\]]+|\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\))+(?:\([^\s()]*?\(['
    r'^\s()]+\)[^\s()]*?\)|\([^\s]+?\)|[^\s`!()\[\]{};:\'".,<>?«»“”‘’])|(?:(?<'
    r'!@)[a-z0-9]+(?:[.\-][a-z0-9]+)*[.](?:com|net|org|edu|gov|mil|aero|asia|'
    r'biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|'
    r'ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|'
    r'bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|'
    r'cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|'
    r'fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|'
    r'hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|'
    r'kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|'
    r'ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|'
    r'nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|'
    r'sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|'
    r'td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|'
    r've|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)\b/?(?!@)))'
)
URL_RE = re.compile(URL_RE_PATTERN)

DELIMITERS = r'\(\)\[\]\{\}\.\;\,\:\?\!\ \t\r\n'

MENTION_RE = re.compile(
    r'(?:^|(?<=[{0}]))@(\w{{3,16}})(?:$|(?=[{0}]))'.format(DELIMITERS)
)

HASHTAG_RE = re.compile(
    r'(?:^|(?<=[{0}]))#(\w{{2,32}})(?:$|(?=[{0}]))'.format(DELIMITERS)
)

def parse_links(body):
    """Parses URLs out of a post.

    .. note: This will need to be refined as edge cases are discovered.

    """
    links = URL_RE.finditer(body)

    result = []
    for link in links:
        result.append({
            'link': fix_url(link.group(0)),
            'span': link.span()
        })

    return result

def parse_mentions(body, check_user=True):
    """Parses @mentions out of a post.

    .. note: This will need to be refined as edge cases are discovered.

    """
    mentions = MENTION_RE.finditer(body)

    result = []
    for mention in mentions:
        username = mention.group(1)
        if check_user:
            user_id = get_uid_username(username)
        else:
            user_id = 'NA'

        if user_id:
            result.append({
                'user_id': user_id,
                'username': username,
                'span': mention.span()
            })

    return result

def parse_hashtags(body):
    """Parsed #hashtags out of a post.

    .. note: This will need to be refined as edge cases are discovered.

    """
    hashtags = HASHTAG_RE.finditer(body)

    result = []
    for hashtag in hashtags:
        result.append({
            'hashtag': hashtag.group(1).lower(),
            'span': hashtag.span()
        })

    return result

def parse_post(body):
    """Calls the individual parsers and returns them all in a multiple return
    statement.

    """
    links = parse_links(body)
    mentions = parse_mentions(body)
    hashtags = parse_hashtags(body)

    return links, mentions, hashtags
<|endoftext|>


<commit_before>"""
This file demonstrates writing tests using the unittest module. These will pass
when you run "manage.py test".

Replace this with more appropriate tests for your application.
"""

from django.conf import settings
from django.test import TestCase
from django.test.client import Client
from mock import MagicMock, patch
from nose.tools import eq_, ok_
from oabutton.apps.bookmarklet.models import OAEvent
import BeautifulSoup

# TODO: tests should probably use real database data as the views
# actually load from disk and render to JSON. Remove these mocks.

# This sets up a mock for the OAEvent class
all_objs = MagicMock()
all_objs.to_json.return_value = [MagicMock(), MagicMock()]
MockOAEvent = MagicMock(wraps=OAEvent)
MockOAEvent.objects.count.return_value = 2
MockOAEvent.objects.all.return_value = all_objs

class SimpleTest(TestCase):

    def setUp(self):
        self.client = Client()

    @patch('oabutton.apps.bookmarklet.models.OAEvent', MockOAEvent)
    def test_count_denied_pursuits(self):
        response = self.client.get('/')
        eq_(response.context['count'], 2)

        soup = BeautifulSoup.BeautifulSoup(response.content)
        node = soup.find('span', attrs={'id': 'counter'})
        eq_(node.text, '2')
        eq_(node.nextSibling.text, u'Paywalls Hit')

    @patch('oabutton.apps.bookmarklet.models.OAEvent', MockOAEvent)
    def test_versioned_static_content(self):
        response = self.client.get('/')
        soup = BeautifulSoup.BeautifulSoup(response.content)
        scripts = [s for s in soup.findAll('script') if s.has_key('src')]  # NOQA
        scripts = [s for s in scripts if s['src'].startswith("/static")]
        sheets = soup.findAll('link', attrs={'rel': 'stylesheet'})

        ok_(len(scripts) > 0)
        ok_(len(sheets) > 0)

        v = settings.VERSION
        for s in scripts:
            version_check = s['src'].endswith(".js?version=%s" % v)
            ok_(version_check)

        for s in sheets:
            ok_(s['href'].endswith(".css?version=%s" % v))
<commit_msg>Minor test case fix<commit_after>"""
This file demonstrates writing tests using the unittest module. These will pass
when you run "manage.py test".

Replace this with more appropriate tests for your application.
"""

from django.conf import settings
from django.test import TestCase
from django.test.client import Client
from mock import MagicMock, patch
from nose.tools import eq_, ok_
from oabutton.apps.bookmarklet.models import OAEvent
import BeautifulSoup

# TODO: tests should probably use real database data as the views
# actually load from disk and render to JSON. Remove these mocks.

# This sets up a mock for the OAEvent class
all_objs = MagicMock()
all_objs.to_json.return_value = [MagicMock(), MagicMock()]
MockOAEvent = MagicMock(wraps=OAEvent)
MockOAEvent.objects.count.return_value = 2
MockOAEvent.objects.all.return_value = all_objs

class SimpleTest(TestCase):

    def setUp(self):
        self.client = Client()

    @patch('oabutton.apps.bookmarklet.models.OAEvent', MockOAEvent)
    def test_count_denied_pursuits(self):
        response = self.client.get('/')
        eq_(response.context['count'], 2)

        soup = BeautifulSoup.BeautifulSoup(response.content)
        node = soup.find('span', attrs={'id': 'counter'})
        eq_(node.text, '2')
        eq_(node.nextSibling.text, u'Paywalls Hit')

    @patch('oabutton.apps.bookmarklet.models.OAEvent', MockOAEvent)
    def test_versioned_static_content(self):
        response = self.client.get('/')
        soup = BeautifulSoup.BeautifulSoup(response.content)
        scripts = [s for s in soup.findAll('script')
                   if s.get('src', '').startswith("/static")]
        sheets = soup.findAll('link', attrs={'rel': 'stylesheet'})

        ok_(len(scripts) > 0)
        ok_(len(sheets) > 0)

        v = settings.VERSION
        for s in scripts:
            version_check = s['src'].endswith(".js?version=%s" % v)
            ok_(version_check)

        for s in sheets:
            ok_(s['href'].endswith(".css?version=%s" % v))
<|endoftext|>


<commit_before>"""
Dump iem database of OT data to archive
Runs at 00 and 12 UTC
"""
import datetime
import psycopg2
import sys
import iemdb
import pytz
import psycopg2.extras

OTHER = psycopg2.connect(database='other', host='iemdb')
ocursor = OTHER.cursor(cursor_factory=psycopg2.extras.DictCursor)
IEM = psycopg2.connect(database='iem', host='iemdb')
icursor = IEM.cursor(cursor_factory=psycopg2.extras.DictCursor)

ts = datetime.datetime( int(sys.argv[1]), int(sys.argv[2]), int(sys.argv[3]))
ts = ts.replace(hour=0, second=0, minute=0, microsecond=0,
                tzinfo=pytz.timezone("UTC"))
ts2 = ts + datetime.timedelta(hours=24)

# Delete any obs from yesterday
sql = "DELETE from t%s WHERE valid >= '%s' or valid < '%s'" % (ts.year,
      ts, ts2 )
ocursor.execute(sql)

# Get obs from Access
sql = """SELECT c.*, t.id from current_log c JOIN stations t on
    (t.iemid = c.iemid) WHERE valid >= '%s' or valid < '%s'
    and t.network = 'OT'""" % (ts, ts2 )
icursor.execute( sql )

for row in icursor:
    pday = 0
    if (row['pday'] is not None and float(row['pday']) > 0):
        pday = row['pday']
    alti = row['alti']
    if alti is None and row['mslp'] is not None:
        alti = row['mslp'] * .03
    sql = """INSERT into t%s (station, valid, tmpf, dwpf, drct, sknt,  alti,
         pday, gust, c1tmpf,srad) values('%s','%s',%s,%s,%s,%s,%s,%s,%s,%s,%s)
         """ % (ts.year, row['id'], row['valid'], (row['tmpf'] or "Null"),
  (row['dwpf'] or "Null"), (row['drct'] or "Null"), (row['sknt'] or "Null"),
  (alti or "Null"), pday, (row['gust'] or "Null") ,
   (row['c1tmpf'] or "Null"), (row['srad'] or "Null"))
    ocursor.execute(sql)

ocursor.close()
OTHER.commit()<commit_msg>typo, yikes :(<commit_after>"""
Dump iem database of OT data to archive
Runs at 00 and 12 UTC
"""
import datetime
import psycopg2
import sys
import iemdb
import pytz
import psycopg2.extras

OTHER = psycopg2.connect(database='other', host='iemdb')
ocursor = OTHER.cursor(cursor_factory=psycopg2.extras.DictCursor)
IEM = psycopg2.connect(database='iem', host='iemdb')
icursor = IEM.cursor(cursor_factory=psycopg2.extras.DictCursor)

ts = datetime.datetime( int(sys.argv[1]), int(sys.argv[2]), int(sys.argv[3]))
ts = ts.replace(hour=0, second=0, minute=0, microsecond=0,
                tzinfo=pytz.timezone("UTC"))
ts2 = ts + datetime.timedelta(hours=24)

# Delete any obs from yesterday
sql = "DELETE from t%s WHERE valid >= '%s' and valid < '%s'" % (ts.year,
      ts, ts2 )
ocursor.execute(sql)

# Get obs from Access
sql = """SELECT c.*, t.id from current_log c JOIN stations t on
    (t.iemid = c.iemid) WHERE valid >= '%s' and valid < '%s'
    and t.network = 'OT'""" % (ts, ts2 )
icursor.execute( sql )

for row in icursor:
    pday = 0
    if (row['pday'] is not None and float(row['pday']) > 0):
        pday = row['pday']
    alti = row['alti']
    if alti is None and row['mslp'] is not None:
        alti = row['mslp'] * .03
    sql = """INSERT into t%s (station, valid, tmpf, dwpf, drct, sknt,  alti,
         pday, gust, c1tmpf,srad) values('%s','%s',%s,%s,%s,%s,%s,%s,%s,%s,%s)
         """ % (ts.year, row['id'], row['valid'], (row['tmpf'] or "Null"),
  (row['dwpf'] or "Null"), (row['drct'] or "Null"), (row['sknt'] or "Null"),
  (alti or "Null"), pday, (row['gust'] or "Null") ,
   (row['c1tmpf'] or "Null"), (row['srad'] or "Null"))
    ocursor.execute(sql)

ocursor.close()
OTHER.commit()<|endoftext|>


<commit_before>#! /usr/bin/env python
import khmer, sys

CHECKPOINT_PERIOD=1000000
K=32
HASHTABLE_SIZE=int(4**15)+1

def make_reporting_fn(ht, filename, period):

    def _report(name, count1, count2,
                ht=ht, f=filename, checkpoint_period=period):
        print name, count1, count2
        if name == 'do_truncated_partition/read' and \
               count1 % checkpoint_period == 0:
            ht.save_checkpoint(f + '.pmap',
                               f + '.surrender')

    return _report

def main():
    infile = sys.argv[1]
    outfile = sys.argv[2]
    k = K
    hashtable_size = HASHTABLE_SIZE
    checkpoint_period = CHECKPOINT_PERIOD

    print 'making hashtable: k=%d, hashtable size=%dbn' % (k,
                                                          hashtable_size / 1e9)
    ht = khmer.new_hashtable(k, hashtable_size)

    report_fn = make_reporting_fn(ht, outfile, checkpoint_period)

    n_partitions = ht.do_truncated_partition(infile, outfile, report_fn)
    print n_partitions, 'partitions kept'

    ht.save_checkpoint(infile + '.pmap.end',
                       outfile + '.surrender.end')

if __name__ == '__main__':
    main()

<commit_msg>add options and things to do-partition-calc<commit_after>#! /usr/bin/env python
import khmer, sys
from optparse import OptionParser

### defaults

CHECKPOINT_PERIOD=1000000               # 1m
K=32
HASHTABLE_SIZE=15

###

parser = OptionParser()
parser.add_option("-k", "--k", dest="ksize", help="k-mer size", default=K,
                  type=int)
parser.add_option("-s", "--size", dest="htsize4", help="hashtable size",
                  default=HASHTABLE_SIZE, type=int)
parser.add_option("-c", "--checkpoint", dest="checkpoint",
                  help="checkpoint period",
                  default=CHECKPOINT_PERIOD, type=int)

###

def make_reporting_fn(ht, filename, period):

    def _report(name, count1, count2,
                ht=ht, f=filename, checkpoint_period=period):
        print name, count1, count2
        if name == 'do_truncated_partition/read' and \
               count1 % checkpoint_period == 0:
            ht.save_checkpoint(f + '.pmap',
                               f + '.surrender')

    return _report

def main():
    (options, args) = parser.parse_args()

    (infile, outfile) = args
    k = options.ksize
    hashtable_size = 4**options.htsize4 + 1
    checkpoint_period = options.checkpoint

    if k > 32:
        raise Exception, "invalid k; must be <= 32"

    ###

    print 'making hashtable: k=%d, hashtable size=%.2f bn' % (k,
                                                 hashtable_size / float(1e9))
    ht = khmer.new_hashtable(k, hashtable_size)

    report_fn = make_reporting_fn(ht, outfile, checkpoint_period)

    n_partitions = ht.do_truncated_partition(infile, outfile, report_fn)
    print n_partitions, 'partitions kept'

    ht.save_checkpoint(infile + '.pmap.end',
                       outfile + '.surrender.end')

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>from django.contrib.auth.forms import UserCreationForm
from profiles.models import Profile

class RegistrationForm(UserCreationForm):
    class Meta(UserCreationForm.Meta):
        fields = ("username", "email")
        model = Profile
<commit_msg>Fix clean_username method<commit_after>from django import forms
from django.contrib.auth.forms import UserCreationForm

from profiles.models import Profile

class RegistrationForm(UserCreationForm):
    class Meta(UserCreationForm.Meta):
        fields = ("username", "email")
        model = Profile

    def clean_username(self):
        # Since User.username is unique, this check is redundant,
        # but it sets a nicer error message than the ORM. See #13147.
        username = self.cleaned_data["username"]
        try:
            Profile._default_manager.get(username=username)
        except Profile.DoesNotExist:
            return username
        raise forms.ValidationError(
            self.error_messages['duplicate_username'],
            code='duplicate_username',
        )<|endoftext|>