<commit_before>import json
from pathlib import Path
import numpy as np
from tqdm import tqdm
from moviepy.editor import VideoFileClip, concatenate_videoclips  # requires ffmpeg

from utils import read_img

def process(video_dir, gen_frames=False, gen_highlight=False, gen_label=False):
    """Generating frames, highlights and labels of the video.
    Frames is written as `video_dir/%08d.jpg`
    Highlight is generated as `highlight.mp4`
    Label is written back to `info.json`

    Arguments:
    video_dir:  A pathlib.Path object pointing to the folder of the target video.
                The directory should contain `video.mp4` and `info.json`.
    gen_frames, gen_highlight,  gen_label:
                Controls whether to generate frames, highlight, label repectively.
                Default to True
    """

    video_path = video_dir / 'video.mp4'
    hl_path = video_dir / 'highlight.mp4'
    info_path = video_dir / 'info.json'
    label_path = video_dir / 'label.json'
    frame_dir = video_dir / 'frames/'
    frame_fmt = frame_dir / '%08d.jpg'
    frame_dir.mkdir(exist_ok=True)

    video = VideoFileClip(str(video_path))
    info = json.load(info_path.open())

    if gen_frames:
        print('Generating frames')
        video.write_images_sequence(str(frame_fmt))

    if gen_highlight:
        print('Generating highlight')
        clips = [
            video.subclip(s, e) for s, e in zip(info['starts'], info['ends'])
        ]
        hl = concatenate_videoclips(clips)
        hl.write_videofile(str(hl_path), threads=3)

    if gen_label:
        print('Generating label...', end='')
        n_frames = len(list(frame_dir.iterdir()))
        print('n_frames =', n_frames)
        label = {'label': [0 for _ in range(n_frames)]}
        for s, e in zip(info['starts'], info['ends']):
            fs = round(s * video.fps)
            fe = round(e * video.fps)
            for i in range(fs, fe + 1):
                if i >= n_frames:
                    print(i)
                label['label'][i] = 1
        with label_path.open('w') as f:
            json.dump(label, f, ensure_ascii=False)
        print('ok')

def process_all():
    dataset_dir = Path('~/dataset').expanduser().resolve()
    video_dirs = [x for x in dataset_dir.iterdir() if x.is_dir()]
    for i, video_dir in enumerate(video_dirs):
        print(video_dir, '({}/{})'.format(i + 1, len(video_dirs)))
        process(video_dir, gen_label=True)
        print('*' * 50)

if __name__ == '__main__':
    process_all()
    # video_dir = Path('~/dataset/video05').expanduser().resolve()
    # process(video_dir, gen_frames=True, gen_highlight=False, gen_label=True)
<commit_msg>Update at 2017-07-19 00-35-12<commit_after>import json
from pathlib import Path
import numpy as np
from tqdm import tqdm
from moviepy.editor import VideoFileClip, concatenate_videoclips  # requires ffmpeg

from utils import read_img

def process(video_dir, gen_frames=False, gen_highlight=False, gen_label=False):
    """Generating frames, highlights and labels of the video.
    Frames is written as `video_dir/%08d.jpg`
    Highlight is generated as `highlight.mp4`
    Label is written back to `info.json`

    Arguments:
    video_dir:  A pathlib.Path object pointing to the folder of the target video.
                The directory should contain `video.mp4` and `info.json`.
    gen_frames, gen_highlight,  gen_label:
                Controls whether to generate frames, highlight, label repectively.
                Default to True
    """

    video_path = video_dir / 'video.mp4'
    hl_path = video_dir / 'highlight.mp4'
    info_path = video_dir / 'info.json'
    label_path = video_dir / 'label.json'
    frame_dir = video_dir / 'frames/'
    frame_fmt = frame_dir / '%08d.jpg'
    frame_dir.mkdir(exist_ok=True)

    video = VideoFileClip(str(video_path))
    info = json.load(info_path.open())

    if gen_frames:
        print('Generating frames')
        video.write_images_sequence(str(frame_fmt))

    if gen_highlight:
        print('Generating highlight')
        clips = [
            video.subclip(s, e) for s, e in zip(info['starts'], info['ends'])
        ]
        hl = concatenate_videoclips(clips)
        hl.write_videofile(str(hl_path), threads=3)

    if gen_label:
        print('Generating label...', end='')
        n_frames = len(list(frame_dir.iterdir()))
        label = {'label': [0 for _ in range(n_frames)]}
        for s, e in zip(info['starts'], info['ends']):
            fs = round(s * video.fps)
            fe = round(e * video.fps)
            for i in range(fs, fe + 1):
                label['label'][i] = 1
        with label_path.open('w') as f:
            json.dump(label, f, ensure_ascii=False)
        print('ok')

def process_all():
    dataset_dir = Path('~/dataset').expanduser().resolve()
    video_dirs = [x for x in dataset_dir.iterdir() if x.is_dir()]
    for i, video_dir in enumerate(video_dirs):
        print(video_dir, '({}/{})'.format(i + 1, len(video_dirs)))
        process(video_dir, gen_frames=True, gen_label=True)
        print('*' * 50)

if __name__ == '__main__':
    process_all()
    # video_dir = Path('~/dataset/video05').expanduser().resolve()
    # process(video_dir, gen_frames=True, gen_highlight=False, gen_label=True)
<|endoftext|>


<commit_before>import json
import time
import random
import datetime as dt
import re as regex
from slackclient import SlackClient

with open("settings.json") as settings_file:
    settings = json.loads(settings_file.read())

# Establish settings
BOT_ID = settings["bot_id"]
API_KEY = settings["api"]
GENERAL_CHANNEL = settings["general_channel"]
USER_CHANNELS = settings["user_channels"]
USER_GROUPS = settings["user_groups"]
ADMIN_CHANNELS = settings["admin_channels"]
ADMIN_GROUPS = settings["admin_groups"]

# Constants
AT_BOT = "<@" + BOT_ID + ">"
ALL_CLEAR = ["No issues!", "Nothing going on!", "No problems!", "Nothing's broken!", "Not much happening today!"]

# Instantiate client
slack_client = SlackClient(API_KEY)

# Instantiate working variable
working = {"dirty": False, "channel": "", "timestamp": "", "text": ""}
list_of_messages = []

class Message:
    def __init__(self, chnnl, tmstmp, txt):
        self.channel, self.timestamp, self.text = chnnl, tmstmp, txt
        self.dirty = False

    def __str__(self):
        return "Channel: " + self.channel + ", TS: " + self.timestamp + ", Text: " + self.text

def post_to_general(pending=working):
    prepend = ":rotating_light::rotating_light::rotating_light:\n" \
              "The University is currently experiencing the following issue:\n```" + pending["text"] + "```"
    api_result = slack_client.api_call("chat.postMessage", channel=GENERAL_CHANNEL, text=prepend, as_user=True)
    message = Message(api_result["channel"], api_result["ts"], pending["text"])
    list_of_messages.append(message)
    slack_client.api_call("pins.add", channel=GENERAL_CHANNEL, timestamp=api_result["ts"])
    return

def handle_command(slack_command, slack_user, slack_channel, item_timestamp, pending=working):
    response = "Couldn't understand you. No problem posted. Please try again or send `!problem help` for tips."
    # Play with this regex here: https://regex101.com/r/3UZNxU/2
    # TODO: Allow specific channel names to be targeted for submitting problems
    to_be_posted = regex.search(r"(?:^post +)((?:\"|')(.*)(?:\"|'))", slack_command, regex.IGNORECASE)
    allow_command = regex.search(r"(?:^allow)", slack_command, regex.IGNORECASE)
    deny_command = regex.search(r"(?:^deny) *((?:\")(.*)(?:\"))?", slack_command, regex.IGNORECASE)
    close_command = regex.search(r"(?:^close +)(\d+)", slack_command, regex.IGNORECASE)
    update_command = regex.search(r"(?:^update +)(\d) +((?:\"|')(.*)(?:\"|'))", slack_command, regex.IGNORECASE)
    list_command = regex.search(r"(?:^list)", slack_command, regex.IGNORECASE)
    help_command = regex.search(r"(?:^help)", slack_command, regex.IGNORECASE)
    # Easy and tangible reaction to acknowledge that a problem has been seen by the bot
    slack_client.api_call("reactions.add", name="rotating_light", channel=slack_channel, timestamp=item_timestamp)
    if not pending["dirty"]:
        pending["channel"], pending["timestamp"] = slack_channel, item_timestamp
        if to_be_posted:
            pending["dirty"] = True
            try:
                pending["text"] = to_be_posted.group(2)
            except AttributeError:
                print("No regex match found! Something may be wrong!")
                pending["text"], pending["dirty"] = "", False
    # Post a problem
    if to_be_posted:
        # TODO: Add admin groups, not just admin channels
        if (slack_channel in ADMIN_CHANNELS) or (slack_channel in ADMIN_GROUPS):
            # Admin requests are automatically approved
            prepend = "Posting the following:\n```" + pending["text"] + "```"
            for USER_CHANNEL in USER_CHANNELS:
                slack_client.api_call("channels.setTopic", channel=USER_CHANNEL,
                                      topic=(":rotating_light: " + pending["text"] + " :rotating_light:"))
            for USER_GROUP in USER_GROUPS:
                slack_client.api_call("groups.setTopic", channel=USER_GROUP,
                                      topic=(":rotating_light: " + pending["text"] + " :rotating_light:"))
            # Notify admin channel that problem was posted
            for ADMIN_CHANNEL in ADMIN_CHANNELS:
                slack_client.api_call("chat.postMessage", channel=ADMIN_CHANNEL, text=prepend, as_user=True)
            for ADMIN_GROUP in ADMIN_GROUPS:
                slack_client.api_call("chat.postMessage", channel=ADMIN_GROUP, text=prepend, as_user=True)
            slack_client.api_call("reactions.add", name="ok",
                                  channel=pending["channel"], timestamp=pending["timestamp"])
            post_to_general()
            pending["dirty"] = False
        else:
            slack_client.api_call("reactions.add", name="question", channel=pending["channel"],
                                  timestamp=pending["timestamp"])
            prepend = "<@" + slack_user + "> has requested that the following problem be posted:\n```" + \
                      pending["text"] + "```\n\n`Allow` or `Deny`?"
            for ADMIN_CHANNEL in ADMIN_CHANNELS:
                slack_client.api_call("chat.postMessage", channel=ADMIN_CHANNEL, text=prepend, as_user=True)
            for ADMIN_GROUP in ADMIN_GROUPS:
                slack_client.api_call("chat.postMessage", channel=ADMIN_GROUP, text=prepend, as_user=True)
    # Approve a posting
    if allow_command and (slack_channel in ADMIN_CHANNELS or slack_channel in ADMIN_GROUPS):
        # Problem will be posted
        approval = "Problem posted."
        for ADMIN_CHANNEL in ADMIN_CHANNELS:
            slack_client.api_call("chat.postMessage", channel=ADMIN_CHANNEL, text=approval, as_user=True)
        for ADMIN_GROUP in ADMIN_GROUPS:
            slack_client.api_call("chat.postMessage", channel=ADMIN_GROUP, text=approval, as_user=True)
        # Set the topic in the user channels
        for USER_CHANNEL in USER_CHANNELS:
            slack_client.api_call("channels.setTopic", channel=USER_CHANNEL,
                                  topic=":rotating_light: " + pending["text"] + " :rotating_light:")
        for USER_GROUP in USER_GROUPS:
            slack_client.api_call("groups.setTopic", channel=USER_GROUP,
                                  topic=":rotating_light: " + pending["text"] + " :rotating_light:")
        # Go back to the problem and remove its :question:
        slack_client.api_call("reactions.remove", name="question",
                              channel=pending["channel"], timestamp=pending["timestamp"])
        slack_client.api_call("reactions.add", name="ok", channel=pending["channel"], timestamp=pending["timestamp"])
        post_to_general()
        pending["dirty"] = False
    # Deny a posting
    if deny_command and (slack_channel in ADMIN_CHANNELS or slack_channel in ADMIN_GROUPS):
        # Problem won't be posted
        denial = "Problem will not be posted."
        try:
            reason = deny_command.group(2)
        except AttributeError:
            reason = None
        prepend = "This posting:\n```" + pending["text"] + "```\nhas been rejected."
        if reason:
            prepend += "\nThis reason was given:\n```" + reason + "```\n"
        for ADMIN_CHANNEL in ADMIN_CHANNELS:
            slack_client.api_call("chat.postMessage", channel=ADMIN_CHANNEL, text=denial, as_user=True)
        for ADMIN_GROUP in ADMIN_GROUPS:
            slack_client.api_call("chat.postMessage", channel=ADMIN_GROUP, text=denial, as_user=True)
        # Go back to the problem and remove its :question:
        slack_client.api_call("reactions.remove", name="question",
                              channel=pending["channel"], timestamp=pending["timestamp"])
        slack_client.api_call("reactions.add", name="no_entry_sign",
                              channel=pending["channel"], timestamp=pending["timestamp"])
        slack_client.api_call("chat.postMessage", channel=pending["channel"], text=prepend, as_user=True)
        pending["dirty"] = False
    # List open postings
    if list_command and (slack_channel in ADMIN_CHANNELS or slack_channel in ADMIN_GROUPS):
        # List currently posted problems
        prepend = "Currently, these issues are posted:\n```"
        counter = 1
        if len(list_of_messages) > 0:
            for message in list_of_messages:
                prepend += str(counter) + ")\t" + message.text + "\t" + \
                           dt.datetime.fromtimestamp(float(message.timestamp)).strftime('%H:%M %p, %m-%d-%Y') + "\n"
                counter += 1
        else:
            prepend += "No pending issues."
        prepend += "```"
        # Send this list only to the channel requesting it
        slack_client.api_call("chat.postMessage", channel=slack_channel, text=prepend, as_user=True)
    # Close a posting
    if close_command and (slack_channel in ADMIN_CHANNELS or slack_channel in ADMIN_GROUPS):
        index_to_close = None
        try:
            index_to_close = close_command.group(1)
        except AttributeError:
            print("Nothing to close.")
        if index_to_close:
            response = "Problem #" + index_to_close + " closed."
            to_close = list_of_messages[int(index_to_close) - 1]  # -1 adjust for human indexing
            # Un-pin from channel
            slack_client.api_call("pins.remove", channel=to_close.channel, timestamp=to_close.timestamp)
            # Thread closing message
            slack_client.api_call("chat.postMessage", channel=to_close.channel,
                                  thread_ts=to_close.timestamp, reply_broadcast=True,
                                  text="This problem has been closed.", as_user=True)
            del list_of_messages[int(index_to_close) - 1]
            # If there are still problems, update the topic of the user channel
            if len(list_of_messages) > 0:
                for USER_CHANNEL in USER_CHANNELS:
                    slack_client.api_call("channels.setTopic", channel=USER_CHANNEL,
                                          topic=":rotating_light: " + list_of_messages[-1].text + " :rotating_light:")
                for USER_GROUP in USER_GROUPS:
                    slack_client.api_call("groups.setTopic", channel=USER_GROUP,
                                          topic=":rotating_light: " + list_of_messages[-1].text + " :rotating_light:")
            elif len(list_of_messages) == 0:
                for USER_CHANNEL in USER_CHANNELS:
                    slack_client.api_call("channels.setTopic", channel=USER_CHANNEL,
                                          topic=random.choice(ALL_CLEAR))
                for USER_GROUP in USER_GROUPS:
                    slack_client.api_call("groups.setTopic", channel=USER_GROUP,
                                          topic=random.choice(ALL_CLEAR))
            for ADMIN_CHANNEL in ADMIN_CHANNELS:
                slack_client.api_call("chat.postMessage", channel=ADMIN_CHANNEL, text=response, as_user=True)
    # Update a posting
    if update_command and (slack_channel in ADMIN_CHANNELS or slack_channel in ADMIN_GROUPS):
        index_to_update = None
        update_text = None
        try:
            index_to_update = update_command.group(1)
        except AttributeError:
            print("Nothing to update.")
        try:
            update_text = update_command.group(3)
        except AttributeError:
            print("No update text given.")
        if index_to_update and update_text:
            response = "Problem #" + index_to_update + " updated with:\n```" + update_text + "```"
            to_update = list_of_messages[int(index_to_update) - 1]  # -1 to adjust for human indexing
            prepend = "The following update has been posted:\n```"
            prepend += update_command.group(3) + "```"
            # Thread the update message on
            slack_client.api_call("chat.postMessage", channel=to_update.channel,
                                  thread_ts=to_update.timestamp, reply_broadcast=True, text=prepend, as_user=True)
            # Notify sending channel of posting
            slack_client.api_call("chat.postMessage", channel=slack_channel, text=response, as_user=True)
    if help_command:
        response = "Invoke any of these commands using `!problem` or `@problem-bot`:\n" \
                   "`help`: Posts this help.\n" \
                   "`post \"...\"`: Submits a problem posting for approval.\n"
        if slack_channel in ADMIN_CHANNELS:
            response += "\n*Admin-only commands:*\n" \
                        "`allow`: Approves problem to be posted to the users' channel.\n" \
                        "`deny`: Denies a problem being posted to the users' channel.\n" \
                        "`list`: Lists all pinned problems.\n" \
                        "`close #`: Closes problem according to its list number."
        slack_client.api_call("chat.postMessage", channel=slack_channel, text=response, as_user=True)
    if not to_be_posted and not allow_command and not deny_command and not list_command and not close_command \
       and not update_command and not help_command:
        slack_client.api_call("chat.postMessage", channel=slack_channel, text=response, as_user=True)

def parse_slack_output(slack_rtm_output):
    output_list = slack_rtm_output
    if output_list and len(output_list) > 0:
        for output in output_list:
            if output and 'text' in output and output['text'].startswith('!problem') and output['user'] != BOT_ID:
                return output['text'].split("!problem")[1].strip(), output['user'], output['channel'], output['ts']
            elif output and 'text' in output and output['text'].startswith(AT_BOT) and output['user'] != BOT_ID:
                return output['text'].split(AT_BOT)[1].strip(), output['user'], output['channel'], output['ts']
    return None, None, None, None

if __name__ == "__main__":
    READ_WEBSOCKET_DELAY = 1  # 1 second firehose delay
    if slack_client.rtm_connect():
        print("Problem-Bot connected and running!")
        print("Grabbing currently posted problems!")
        api_response = slack_client.api_call("pins.list", channel=GENERAL_CHANNEL)
        for item in reversed(api_response["items"]):
            if item["type"] == "message" and item["message"]["user"] == BOT_ID:
                text = regex.search(r"(?:```)(.*)(?:```)", item["message"]["text"], regex.IGNORECASE).group(1)
                list_of_messages.append(Message(GENERAL_CHANNEL, item["message"]["ts"], text))
        while True:
            command, user, channel, timestamp = parse_slack_output(slack_client.rtm_read())
            if command and user and channel:
                handle_command(command, user, channel, timestamp)
            time.sleep(READ_WEBSOCKET_DELAY)
    else:
        print("Connection failed! Check API key and Bot ID!")
<commit_msg>Implemented specific channel targeting<commit_after>import json
import time
import random
import datetime as dt
import re as regex
from slackclient import SlackClient

with open("settings.json") as settings_file:
    settings = json.loads(settings_file.read())

# Establish settings
BOT_ID = settings["bot_id"]
API_KEY = settings["api"]
GENERAL_CHANNEL = settings["general_channel"]
USER_CHANNELS = settings["user_channels"]
USER_GROUPS = settings["user_groups"]
ADMIN_CHANNELS = settings["admin_channels"]
ADMIN_GROUPS = settings["admin_groups"]

# Constants
AT_BOT = "<@" + BOT_ID + ">"
ALL_CLEAR = ["No issues!", "Nothing going on!", "No problems!", "Nothing's broken!", "Not much happening today!"]

# Instantiate client
slack_client = SlackClient(API_KEY)

# Instantiate working variable
working = {"dirty": False, "channel": "", "timestamp": "", "text": ""}
list_of_messages = []

class Message:
    def __init__(self, chnnl, tmstmp, txt):
        self.channel, self.timestamp, self.text = chnnl, tmstmp, txt
        self.dirty = False

    def __str__(self):
        return "Channel: " + self.channel + ", TS: " + self.timestamp + ", Text: " + self.text

def post_to_general(pending=working):
    prepend = ":rotating_light::rotating_light::rotating_light:\n" \
              "The University is currently experiencing the following issue:\n```" + pending["text"] + "```"
    api_result = slack_client.api_call("chat.postMessage", channel=GENERAL_CHANNEL, text=prepend, as_user=True)
    message = Message(api_result["channel"], api_result["ts"], pending["text"])
    list_of_messages.append(message)
    slack_client.api_call("pins.add", channel=GENERAL_CHANNEL, timestamp=api_result["ts"])
    return

def handle_command(slack_command, slack_user, slack_channel, item_timestamp, pending=working):
    response = "Couldn't understand you. No problem posted. Please try again or send `!problem help` for tips."
    # Play with this regex here: https://regex101.com/r/3UZNxU/3
    to_be_posted = regex.search(r"(?:^post +)((?:\"|')(.*)(?:\"|')) *"
                                r"((?:<#)(\w*)(?:\|(\w*)>))?", slack_command, regex.IGNORECASE)
    allow_command = regex.search(r"(?:^allow)", slack_command, regex.IGNORECASE)
    deny_command = regex.search(r"(?:^deny) *((?:\")(.*)(?:\"))?", slack_command, regex.IGNORECASE)
    close_command = regex.search(r"(?:^close +)(\d+)", slack_command, regex.IGNORECASE)
    update_command = regex.search(r"(?:^update +)(\d) +((?:\"|')(.*)(?:\"|'))", slack_command, regex.IGNORECASE)
    list_command = regex.search(r"(?:^list)", slack_command, regex.IGNORECASE)
    help_command = regex.search(r"(?:^help)", slack_command, regex.IGNORECASE)
    # Easy and tangible reaction to acknowledge that a problem has been seen by the bot
    slack_client.api_call("reactions.add", name="rotating_light", channel=slack_channel, timestamp=item_timestamp)
    if not pending["dirty"]:
        pending["channel"], pending["timestamp"] = slack_channel, item_timestamp
        if to_be_posted:
            pending["dirty"] = True
            try:
                pending["text"] = to_be_posted.group(2)
            except AttributeError:
                print("No regex match found! Something may be wrong!")
                pending["text"], pending["dirty"] = "", False
    # Post a problem
    if to_be_posted:
        if (slack_channel in ADMIN_CHANNELS) or (slack_channel in ADMIN_GROUPS):
            # Admin requests are automatically approved
            prepend = "Posting the following:\n```" + pending["text"] + "```"
            for USER_CHANNEL in USER_CHANNELS:
                slack_client.api_call("channels.setTopic", channel=USER_CHANNEL,
                                      topic=(":rotating_light: " + pending["text"] + " :rotating_light:"))
            for USER_GROUP in USER_GROUPS:
                slack_client.api_call("groups.setTopic", channel=USER_GROUP,
                                      topic=(":rotating_light: " + pending["text"] + " :rotating_light:"))
            # Notify admin channel that problem was posted
            for ADMIN_CHANNEL in ADMIN_CHANNELS:
                slack_client.api_call("chat.postMessage", channel=ADMIN_CHANNEL, text=prepend, as_user=True)
            for ADMIN_GROUP in ADMIN_GROUPS:
                slack_client.api_call("chat.postMessage", channel=ADMIN_GROUP, text=prepend, as_user=True)
            slack_client.api_call("reactions.add", name="ok",
                                  channel=pending["channel"], timestamp=pending["timestamp"])
            post_to_general()
            pending["dirty"] = False
        else:
            slack_client.api_call("reactions.add", name="question", channel=pending["channel"],
                                  timestamp=pending["timestamp"])
            prepend = "<@" + slack_user + "> has requested that the following problem be posted:\n```" + \
                      pending["text"] + "```\n\n`Allow` or `Deny`?"
            try:
                target_group = to_be_posted.group(4)
            except AttributeError:
                # No targeted group
                target_group = None
            if target_group:
                # Send message to target group
                slack_client.api_call("chat.postMessage", channel=target_group, text=prepend, as_user=True)
            else:
                for ADMIN_CHANNEL in ADMIN_CHANNELS:
                    slack_client.api_call("chat.postMessage", channel=ADMIN_CHANNEL, text=prepend, as_user=True)
                for ADMIN_GROUP in ADMIN_GROUPS:
                    slack_client.api_call("chat.postMessage", channel=ADMIN_GROUP, text=prepend, as_user=True)
    # Approve a posting
    if allow_command and (slack_channel in ADMIN_CHANNELS or slack_channel in ADMIN_GROUPS):
        # Problem will be posted
        confirmation = "Problem has been posted."
        slack_client.api_call("chat.postMessage", channel=slack_channel, text=confirmation, as_user=True)
        approval = "The following problem has been posted in <#" + GENERAL_CHANNEL + ">:\n```" + pending["text"] + "```"
        for ADMIN_CHANNEL in ADMIN_CHANNELS:
            # Don't double up on sending confirmation/approval
            if ADMIN_CHANNEL != slack_channel:
                slack_client.api_call("chat.postMessage", channel=ADMIN_CHANNEL, text=approval, as_user=True)
        for ADMIN_GROUP in ADMIN_GROUPS:
            # Don't double up on sending confirmation/approval
            if ADMIN_GROUP != slack_channel:
                slack_client.api_call("chat.postMessage", channel=ADMIN_GROUP, text=approval, as_user=True)
        # Set the topic in the user channels
        for USER_CHANNEL in USER_CHANNELS:
            slack_client.api_call("channels.setTopic", channel=USER_CHANNEL,
                                  topic=":rotating_light: " + pending["text"] + " :rotating_light:")
        for USER_GROUP in USER_GROUPS:
            slack_client.api_call("groups.setTopic", channel=USER_GROUP,
                                  topic=":rotating_light: " + pending["text"] + " :rotating_light:")
        # Go back to the problem and remove its :question:
        slack_client.api_call("reactions.remove", name="question",
                              channel=pending["channel"], timestamp=pending["timestamp"])
        slack_client.api_call("reactions.add", name="ok", channel=pending["channel"], timestamp=pending["timestamp"])
        post_to_general()
        pending["dirty"] = False
    # Deny a posting
    if deny_command and (slack_channel in ADMIN_CHANNELS or slack_channel in ADMIN_GROUPS):
        # Problem won't be posted
        denial = "Problem will not be posted."
        try:
            reason = deny_command.group(2)
        except AttributeError:
            reason = None
        prepend = "This posting:\n```" + pending["text"] + "```\nhas been rejected."
        if reason:
            prepend += "\nThis reason was given:\n```" + reason + "```\n"
        for ADMIN_CHANNEL in ADMIN_CHANNELS:
            slack_client.api_call("chat.postMessage", channel=ADMIN_CHANNEL, text=denial, as_user=True)
        for ADMIN_GROUP in ADMIN_GROUPS:
            slack_client.api_call("chat.postMessage", channel=ADMIN_GROUP, text=denial, as_user=True)
        # Go back to the problem and remove its :question:
        slack_client.api_call("reactions.remove", name="question",
                              channel=pending["channel"], timestamp=pending["timestamp"])
        slack_client.api_call("reactions.add", name="no_entry_sign",
                              channel=pending["channel"], timestamp=pending["timestamp"])
        slack_client.api_call("chat.postMessage", channel=pending["channel"], text=prepend, as_user=True)
        pending["dirty"] = False
    # List open postings
    if list_command and (slack_channel in ADMIN_CHANNELS or slack_channel in ADMIN_GROUPS):
        # List currently posted problems
        prepend = "Currently, these issues are posted:\n```"
        counter = 1
        if len(list_of_messages) > 0:
            for message in list_of_messages:
                prepend += str(counter) + ")\t" + message.text + "\t" + \
                           dt.datetime.fromtimestamp(float(message.timestamp)).strftime('%H:%M %p, %m-%d-%Y') + "\n"
                counter += 1
        else:
            prepend += "No pending issues."
        prepend += "```"
        # Send this list only to the channel requesting it
        slack_client.api_call("chat.postMessage", channel=slack_channel, text=prepend, as_user=True)
    # Close a posting
    if close_command and (slack_channel in ADMIN_CHANNELS or slack_channel in ADMIN_GROUPS):
        index_to_close = None
        try:
            index_to_close = close_command.group(1)
        except AttributeError:
            print("Nothing to close.")
        if index_to_close:
            response = "Problem #" + index_to_close + " closed."
            to_close = list_of_messages[int(index_to_close) - 1]  # -1 adjust for human indexing
            # TODO: Fix the removal timing issue
            # If a user doesn't wait for this to finish, it won't unpin properly, but will remove it from the list
            # Un-pin from channel
            slack_client.api_call("pins.remove", channel=to_close.channel, timestamp=to_close.timestamp)
            # Thread closing message
            slack_client.api_call("chat.postMessage", channel=to_close.channel,
                                  thread_ts=to_close.timestamp, reply_broadcast=True,
                                  text="This problem has been closed.", as_user=True)
            del list_of_messages[int(index_to_close) - 1]
            # If there are still problems, update the topic of the user channel
            if len(list_of_messages) > 0:
                for USER_CHANNEL in USER_CHANNELS:
                    slack_client.api_call("channels.setTopic", channel=USER_CHANNEL,
                                          topic=":rotating_light: " + list_of_messages[-1].text + " :rotating_light:")
                for USER_GROUP in USER_GROUPS:
                    slack_client.api_call("groups.setTopic", channel=USER_GROUP,
                                          topic=":rotating_light: " + list_of_messages[-1].text + " :rotating_light:")
            elif len(list_of_messages) == 0:
                for USER_CHANNEL in USER_CHANNELS:
                    slack_client.api_call("channels.setTopic", channel=USER_CHANNEL,
                                          topic=random.choice(ALL_CLEAR))
                for USER_GROUP in USER_GROUPS:
                    slack_client.api_call("groups.setTopic", channel=USER_GROUP,
                                          topic=random.choice(ALL_CLEAR))
            for ADMIN_CHANNEL in ADMIN_CHANNELS:
                slack_client.api_call("chat.postMessage", channel=ADMIN_CHANNEL, text=response, as_user=True)
    # Update a posting
    if update_command and (slack_channel in ADMIN_CHANNELS or slack_channel in ADMIN_GROUPS):
        index_to_update = None
        update_text = None
        try:
            index_to_update = update_command.group(1)
        except AttributeError:
            print("Nothing to update.")
        try:
            update_text = update_command.group(3)
        except AttributeError:
            print("No update text given.")
        if index_to_update and update_text:
            response = "Problem #" + index_to_update + " updated with:\n```" + update_text + "```"
            to_update = list_of_messages[int(index_to_update) - 1]  # -1 to adjust for human indexing
            prepend = "The following update has been posted:\n```"
            prepend += update_command.group(3) + "```"
            # Thread the update message on
            slack_client.api_call("chat.postMessage", channel=to_update.channel,
                                  thread_ts=to_update.timestamp, reply_broadcast=True, text=prepend, as_user=True)
            # Notify sending channel of posting
            slack_client.api_call("chat.postMessage", channel=slack_channel, text=response, as_user=True)
    if help_command:
        response = "Invoke any of these commands using `!problem` or `@problem-bot`:\n" \
                   "`help`: Posts this help.\n" \
                   "`post \"...\"`: Submits a problem posting for approval.\n"
        if slack_channel in ADMIN_CHANNELS:
            response += "\n*Admin-only commands:*\n" \
                        "`allow`: Approves problem to be posted to the users' channel.\n" \
                        "`deny`: Denies a problem being posted to the users' channel.\n" \
                        "`list`: Lists all pinned problems.\n" \
                        "`close #`: Closes problem according to its list number."
        slack_client.api_call("chat.postMessage", channel=slack_channel, text=response, as_user=True)
    if not to_be_posted and not allow_command and not deny_command and not list_command and not close_command \
       and not update_command and not help_command:
        slack_client.api_call("chat.postMessage", channel=slack_channel, text=response, as_user=True)

def parse_slack_output(slack_rtm_output):
    output_list = slack_rtm_output
    if output_list and len(output_list) > 0:
        for output in output_list:
            if output and 'text' in output and output['text'].startswith('!problem') and output['user'] != BOT_ID:
                return output['text'].split("!problem")[1].strip(), output['user'], output['channel'], output['ts']
            elif output and 'text' in output and output['text'].startswith(AT_BOT) and output['user'] != BOT_ID:
                return output['text'].split(AT_BOT)[1].strip(), output['user'], output['channel'], output['ts']
    return None, None, None, None

if __name__ == "__main__":
    READ_WEBSOCKET_DELAY = 1  # 1 second firehose delay
    if slack_client.rtm_connect():
        print("Problem-Bot connected and running!")
        print("Grabbing currently posted problems!")
        api_response = slack_client.api_call("pins.list", channel=GENERAL_CHANNEL)
        for item in reversed(api_response["items"]):
            if item["type"] == "message" and item["message"]["user"] == BOT_ID:
                text = regex.search(r"(?:```)(.*)(?:```)", item["message"]["text"], regex.IGNORECASE).group(1)
                list_of_messages.append(Message(GENERAL_CHANNEL, item["message"]["ts"], text))
        while True:
            command, user, channel, timestamp = parse_slack_output(slack_client.rtm_read())
            if command and user and channel:
                handle_command(command, user, channel, timestamp)
            time.sleep(READ_WEBSOCKET_DELAY)
    else:
        print("Connection failed! Check API key and Bot ID!")
<|endoftext|>


<commit_before># ex: set expandtab softtabstop=4 shiftwidth=4: -*- cpy-indent-level: 4; indent-tabs-mode: nil -*-
#
# Copyright (C) 2009,2010  Contributor
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the EU DataGrid Software License.  You should
# have received a copy of the license with this program, and the
# license is published at
# http://eu-datagrid.web.cern.ch/eu-datagrid/license.html.
#
# THE FOLLOWING DISCLAIMER APPLIES TO ALL SOFTWARE CODE AND OTHER
# MATERIALS CONTRIBUTED IN CONNECTION WITH THIS PROGRAM.
#
# THIS SOFTWARE IS LICENSED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE AND ANY WARRANTY OF NON-INFRINGEMENT, ARE
# DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. THIS
# SOFTWARE MAY BE REDISTRIBUTED TO OTHERS ONLY BY EFFECTIVELY USING
# THIS OR ANOTHER EQUIVALENT DISCLAIMER AS WELL AS ANY OTHER LICENSE
# TERMS THAT MAY APPLY.
import sys

from inspect import isclass

from sqlalchemy.schema import CreateTable
from sqlalchemy.orm.session import Session
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
from sqlalchemy.orm.properties import RelationProperty, ColumnProperty
from sqlalchemy.ext.associationproxy import AssociationProxy, _lazy_collection

from aquilon.utils import monkeypatch
from aquilon.exceptions_ import InternalError, NotFoundException, ArgumentError

def _raise_custom(cls, defcls, msg):
    if isclass(cls) and issubclass(cls, Exception):
        raise cls(msg)
    else:
        raise defcls(msg)

class Base(object):
    """ The abstract base class for all aqdb objects """

    def __init__(self, **kw):
        for k in kw:
            if not hasattr(type(self), k):  # pragma: no cover
                msg = "%r is an invalid argument for %s" % (
                    k, type(self).__name__)
                raise TypeError(msg)
            setattr(self, k, kw[k])

    def __repr__(self):
        """
        Return an internal representation of the object.

        This representation is intended mainly for debugging, and it should
        uniquely identify the instance. The output of repr() should never be
        sent to the client, and should only appear in server-side logs at the
        debug level.
        """
        label = self.__class__.__name__
        attrs = []
        mapper = self.__class__.__mapper__
        for prop in mapper.iterate_properties:
            if not isinstance(prop, ColumnProperty):
                continue
            field = prop.columns[0].name

            # These fields are not really interesting
            if field == 'id' or field == 'creation_date' or field == 'comments':
                continue

            # Convert foreign IDs to names
            if field.endswith("_id") and mapper.has_property(field[:-3]) and \
                    isinstance(mapper.get_property(field[:-3]),
                               RelationProperty):
                field = field[:-3]
            value = getattr(self, field, None)

            attrs.append("%s: %s" % (field, value))
        return "<%s %s>" % (label, ", ".join(attrs))

    def __str__(self):
        """
        Return the most significant attribute of the object.

        If the object has a single field that makes it unique, then that field
        should be returned.
        """
        return str(self._get_instance_label())

    def format_helper(self, format_spec, instance):
        """ Common helper for formatting functions """
        lowercase = False
        class_only = False
        passthrough = ""
        for letter in format_spec:
            if letter == "l":
                lowercase = True
            elif letter == "c":
                class_only = True
            else:
                passthrough += letter

        clsname = self.__class__._get_class_label(tolower=lowercase)
        if class_only:
            return clsname.__format__(passthrough)
        val = "%s %s" % (clsname, instance)
        return val.__format__(passthrough)

    def __format__(self, format_spec):
        """
        Return a pretty-printed representation of the object.

        The output of format() should be the preferred form when referring to
        this object in messages sent to the client. It should be readable as
        plain text, and should uniquely identify the object.

        The format_spec argument can be a standard format specifier suitable for
        strings, with some extensions:

        - specifying the 'l' flag will format the class label in lower case,
          except abbreviations.

        - specifying the 'c' flag will return the pretty printed class name
        """

        return self.format_helper(format_spec, self._get_instance_label())

    @classmethod
    def _get_class_label(cls, tolower=False):
        label = getattr(cls, "_class_label", cls.__name__)
        if tolower:
            parts = label.split()
            # 'Operating System' -> 'operating system', but:
            # 'ESX Cluster' -> 'ESX cluster'
            # 'ToR Switch' -> 'ToR switch'
            #
            # Heuristic: a word is an acronym if the last letter is in upper
            # case
            label = ' '.join(map(lambda x: x if x[:-1].isupper() else x.lower(), parts))
        return label

    def _get_instance_label(self):
        """ Subclasses can override this method or just set a property to check

            If an instance has an attribute named _instance_label, the property
            named by that attribute will be checked for an identifier.

            Without _instance_label set, the properties 'name' and 'type' are
            checked, followed by service.name and system.name.

            For situations more complex than just checking a property this
            method should be overridden with the necessary logic.
        """
        if hasattr(self, "_instance_label"):
            return getattr(self, getattr(self, "_instance_label"))
        for attr in ['name', 'type']:
            if hasattr(self, attr):
                return getattr(self, attr)
        # These might not be necessary any more...
        for attr in ['service', 'system']:
            if hasattr(self, attr):
                return getattr(self, attr).name
        return 'instance'

    @classmethod
    def _selection_helper(cls, session, query, *args, **kwargs):
        """ Helper method for get_unique and get_matching_ids

            Every class that wishes to support get_unique() must have
            'unique_fields' defined in the table's info dictionary.
            'unique_fields' is a list that contains the names of fields that
            make the object unique. Every field can be either a column or a
            relation. In the latter case, 'unique_fields' of the referenced
            class must contain a single field only
        """

        compel = kwargs.pop('compel', False)
        table = cls.__table__
        mapper = cls.__mapper__
        caller = sys._getframe(1).f_code.co_name
        clslabel = cls._get_class_label()

        if not isinstance(session, Session):  # pragma: no cover
            raise TypeError("The first argument of %s() must be an "
                            "SQLAlchemy session." % caller)

        if 'unique_fields' not in table.info:
            raise InternalError("Class %s is not annotated to be used with "
                                "%s()." % (cls.__name__, caller))

        # Handle positional arguments
        if args:
            if len(args) > 1:
                raise InternalError("%s() does not support multiple positional"
                                    "arguments, use named arguments." % caller)
            if len(table.info['unique_fields']) > 1:
                raise InternalError("The uniqueness criteria for class %s "
                                    "includes multiple fields, positional "
                                    "arguments cannot be used." % cls.__name__)
            if kwargs:
                raise InternalError("Cannot mix positional and named "
                                    "arguments with %s()." % caller)
            kwargs = {table.info['unique_fields'][0]: args[0]}

        desc = []
        # We don't want to modify the table description below, so make a copy
        fields = table.info['unique_fields'][:]
        if 'extra_search_fields' in table.info:
            fields.extend(table.info['extra_search_fields'])
        for field in fields:
            value = kwargs.pop(field, None)
            if value is None:
                continue

            # Do a lookup if the field refers to a relation but the argument
            # given is not a DB object
            if mapper.has_property(field):
                rel = mapper.get_property(field)
                if isinstance(rel, RelationProperty) and \
                        not isinstance(value, rel.argument):
                    value = rel.argument.get_unique(session, value,
                                                    compel=compel)

            # filter_by() would be simpler but it would not allow querying just
            # one column
            query = query.filter(getattr(cls, field) == value)

            # Now some beautification...
            poly_column = getattr(mapper, "polymorphic_on", None)
            if poly_column is not None and poly_column.name == field:
                # Return "Building foo" instead of "Location foo, location_type
                # building"
                clslabel = mapper.polymorphic_map[value].class_._get_class_label()
            else:
                name = value.name if hasattr(value, "name") else str(value)
                if field == "name":
                    desc.append(name)
                else:
                    desc.append(field + " " + name)

        # Check for arguments we don't know about
        if kwargs:
            raise InternalError("Extra arguments to %s(): %s." %
                                (caller, kwargs))
        return (query, clslabel, desc)

    @classmethod
    def get_unique(cls, session, *args, **kwargs):
        compel = kwargs.get('compel', False)
        preclude = kwargs.pop('preclude', False)

        query = session.query(cls)
        (query, clslabel, desc) = cls._selection_helper(session, query, *args,
                                                        **kwargs)
        try:
            obj = query.one()
            if preclude:
                msg = "%s %s already exists." % (clslabel, ", ".join(desc))
                _raise_custom(preclude, ArgumentError, msg)
            return obj
        except NoResultFound:
            if not compel:
                return None
            msg = "%s %s not found." % (clslabel, ", ".join(desc))
            _raise_custom(compel, NotFoundException, msg)
        except MultipleResultsFound:
            msg = "%s %s is not unique." % (clslabel, ", ".join(desc))
            raise ArgumentError(msg)

    @classmethod
    def get_matching_query(cls, session, *args, **kwargs):
        compel = kwargs.get('compel', False)
        query = session.query(cls.__table__.c.id)
        (query, clslabel, desc) = cls._selection_helper(session, query, *args,
                                                        **kwargs)
        if compel:
            obj = query.first()
            if obj is None:
                msg = "%s %s not found." % (clslabel, ", ".join(desc))
                _raise_custom(compel, NotFoundException, msg)
        return query.subquery()

    @classmethod
    def ddl(self):
        return str(CreateTable(self.__table__))

#Base = declarative_base(metaclass=VersionedMeta, cls=Base)
Base = declarative_base(cls=Base)

# WAY too much magic in AssociationProxy.  This bug and proposed patch is
# listed in the second half of this message:
# http://groups.google.com/group/sqlalchemy-devel/browse_thread/thread/973f4104007f6964/9a001201b3179c58
# Basically, scalar assocation proxies are much more annoying without this
# patch.  Accessing a null attribute normally returns None.  However, the AP
# tries to proxy through the None.  This raises an exception when the AP
# (in a scalar context) should just itself return None.
@monkeypatch(AssociationProxy)
def __get__(self, obj, class_):
    if self.owning_class is None:
        self.owning_class = class_ and class_ or type(obj)
    if obj is None:
        return self
    elif self.scalar is None:
        self.scalar = self._target_is_scalar()
        if self.scalar:
            self._initialize_scalar_accessors()

    if self.scalar:
        # Original line from 0.5.8
        #proxy = self._new(self._lazy_collection(weakref.ref(obj)))
        #setattr(obj, self.key, (id(obj), proxy))
        #return proxy

        target = getattr(obj, self.target_collection)
        if target is None:
            return None
        else:
            return self._scalar_get(target)
    else:
        try:
            # If the owning instance is reborn (orm session resurrect,
            # etc.), refresh the proxy cache.
            creator_id, proxy = getattr(obj, self.key)
            if id(obj) == creator_id:
                return proxy
        except AttributeError:
            pass
        proxy = self._new(_lazy_collection(obj, self.target_collection))
        setattr(obj, self.key, (id(obj), proxy))
        return proxy
<commit_msg>add comments, pragma: no cover for the ddl function<commit_after># ex: set expandtab softtabstop=4 shiftwidth=4: -*- cpy-indent-level: 4; indent-tabs-mode: nil -*-
#
# Copyright (C) 2009,2010  Contributor
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the EU DataGrid Software License.  You should
# have received a copy of the license with this program, and the
# license is published at
# http://eu-datagrid.web.cern.ch/eu-datagrid/license.html.
#
# THE FOLLOWING DISCLAIMER APPLIES TO ALL SOFTWARE CODE AND OTHER
# MATERIALS CONTRIBUTED IN CONNECTION WITH THIS PROGRAM.
#
# THIS SOFTWARE IS LICENSED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE AND ANY WARRANTY OF NON-INFRINGEMENT, ARE
# DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. THIS
# SOFTWARE MAY BE REDISTRIBUTED TO OTHERS ONLY BY EFFECTIVELY USING
# THIS OR ANOTHER EQUIVALENT DISCLAIMER AS WELL AS ANY OTHER LICENSE
# TERMS THAT MAY APPLY.
import sys

from inspect import isclass

from sqlalchemy.schema import CreateTable
from sqlalchemy.orm.session import Session
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
from sqlalchemy.orm.properties import RelationProperty, ColumnProperty
from sqlalchemy.ext.associationproxy import AssociationProxy, _lazy_collection

from aquilon.utils import monkeypatch
from aquilon.exceptions_ import InternalError, NotFoundException, ArgumentError

def _raise_custom(cls, defcls, msg):
    if isclass(cls) and issubclass(cls, Exception):
        raise cls(msg)
    else:
        raise defcls(msg)

class Base(object):
    """ The abstract base class for all aqdb objects """

    def __init__(self, **kw):
        for k in kw:
            if not hasattr(type(self), k):  # pragma: no cover
                msg = "%r is an invalid argument for %s" % (
                    k, type(self).__name__)
                raise TypeError(msg)
            setattr(self, k, kw[k])

    def __repr__(self):
        """
        Return an internal representation of the object.

        This representation is intended mainly for debugging, and it should
        uniquely identify the instance. The output of repr() should never be
        sent to the client, and should only appear in server-side logs at the
        debug level.
        """
        label = self.__class__.__name__
        attrs = []
        mapper = self.__class__.__mapper__
        for prop in mapper.iterate_properties:
            if not isinstance(prop, ColumnProperty):
                continue
            field = prop.columns[0].name

            # These fields are not really interesting
            if field == 'id' or field == 'creation_date' or field == 'comments':
                continue

            # Convert foreign IDs to names
            if field.endswith("_id") and mapper.has_property(field[:-3]) and \
                    isinstance(mapper.get_property(field[:-3]),
                               RelationProperty):
                field = field[:-3]
            value = getattr(self, field, None)

            attrs.append("%s: %s" % (field, value))
        return "<%s %s>" % (label, ", ".join(attrs))

    def __str__(self):
        """
        Return the most significant attribute of the object.

        If the object has a single field that makes it unique, then that field
        should be returned.
        """
        return str(self._get_instance_label())

    def format_helper(self, format_spec, instance):
        """ Common helper for formatting functions """
        lowercase = False
        class_only = False
        passthrough = ""
        for letter in format_spec:
            if letter == "l":
                lowercase = True
            elif letter == "c":
                class_only = True
            else:
                passthrough += letter

        clsname = self.__class__._get_class_label(tolower=lowercase)
        if class_only:
            return clsname.__format__(passthrough)
        val = "%s %s" % (clsname, instance)
        return val.__format__(passthrough)

    def __format__(self, format_spec):
        """
        Return a pretty-printed representation of the object.

        The output of format() should be the preferred form when referring to
        this object in messages sent to the client. It should be readable as
        plain text, and should uniquely identify the object.

        The format_spec argument can be a standard format specifier suitable for
        strings, with some extensions:

        - specifying the 'l' flag will format the class label in lower case,
          except abbreviations.

        - specifying the 'c' flag will return the pretty printed class name
        """

        return self.format_helper(format_spec, self._get_instance_label())

    @classmethod
    def _get_class_label(cls, tolower=False):
        label = getattr(cls, "_class_label", cls.__name__)
        if tolower:
            parts = label.split()
            # 'Operating System' -> 'operating system', but:
            # 'ESX Cluster' -> 'ESX cluster'
            # 'ToR Switch' -> 'ToR switch'
            #
            # Heuristic: a word is an acronym if the last letter is in upper
            # case
            label = ' '.join(map(lambda x: x if x[:-1].isupper() else x.lower(), parts))
        return label

    def _get_instance_label(self):
        """ Subclasses can override this method or just set a property to check

            If an instance has an attribute named _instance_label, the property
            named by that attribute will be checked for an identifier.

            Without _instance_label set, the properties 'name' and 'type' are
            checked, followed by service.name and system.name.

            For situations more complex than just checking a property this
            method should be overridden with the necessary logic.
        """
        if hasattr(self, "_instance_label"):
            return getattr(self, getattr(self, "_instance_label"))
        for attr in ['name', 'type']:
            if hasattr(self, attr):
                return getattr(self, attr)
        # These might not be necessary any more...
        for attr in ['service', 'system']:
            if hasattr(self, attr):
                return getattr(self, attr).name
        return 'instance'

    @classmethod
    def _selection_helper(cls, session, query, *args, **kwargs):
        """ Helper method for get_unique and get_matching_ids

            Every class that wishes to support get_unique() must have
            'unique_fields' defined in the table's info dictionary.
            'unique_fields' is a list that contains the names of fields that
            make the object unique. Every field can be either a column or a
            relation. In the latter case, 'unique_fields' of the referenced
            class must contain a single field only
        """

        compel = kwargs.pop('compel', False)
        table = cls.__table__
        mapper = cls.__mapper__
        caller = sys._getframe(1).f_code.co_name
        clslabel = cls._get_class_label()

        if not isinstance(session, Session):  # pragma: no cover
            raise TypeError("The first argument of %s() must be an "
                            "SQLAlchemy session." % caller)

        if 'unique_fields' not in table.info:
            raise InternalError("Class %s is not annotated to be used with "
                                "%s()." % (cls.__name__, caller))

        # Handle positional arguments
        if args:
            if len(args) > 1:
                raise InternalError("%s() does not support multiple positional"
                                    "arguments, use named arguments." % caller)
            if len(table.info['unique_fields']) > 1:
                raise InternalError("The uniqueness criteria for class %s "
                                    "includes multiple fields, positional "
                                    "arguments cannot be used." % cls.__name__)
            if kwargs:
                raise InternalError("Cannot mix positional and named "
                                    "arguments with %s()." % caller)
            kwargs = {table.info['unique_fields'][0]: args[0]}

        desc = []
        # We don't want to modify the table description below, so make a copy
        fields = table.info['unique_fields'][:]
        if 'extra_search_fields' in table.info:
            fields.extend(table.info['extra_search_fields'])
        for field in fields:
            value = kwargs.pop(field, None)
            if value is None:
                continue

            # Do a lookup if the field refers to a relation but the argument
            # given is not a DB object
            if mapper.has_property(field):
                rel = mapper.get_property(field)
                if isinstance(rel, RelationProperty) and \
                        not isinstance(value, rel.argument):
                    value = rel.argument.get_unique(session, value,
                                                    compel=compel)

            # filter_by() would be simpler but it would not allow querying just
            # one column
            query = query.filter(getattr(cls, field) == value)

            # Now some beautification...
            poly_column = getattr(mapper, "polymorphic_on", None)
            if poly_column is not None and poly_column.name == field:
                # Return "Building foo" instead of "Location foo, location_type
                # building"
                clslabel = mapper.polymorphic_map[value].class_._get_class_label()
            else:
                name = value.name if hasattr(value, "name") else str(value)
                if field == "name":
                    desc.append(name)
                else:
                    desc.append(field + " " + name)

        # Check for arguments we don't know about
        if kwargs:
            raise InternalError("Extra arguments to %s(): %s." %
                                (caller, kwargs))
        return (query, clslabel, desc)

    @classmethod
    def get_unique(cls, session, *args, **kwargs):
        compel = kwargs.get('compel', False)
        preclude = kwargs.pop('preclude', False)

        query = session.query(cls)
        (query, clslabel, desc) = cls._selection_helper(session, query, *args,
                                                        **kwargs)
        try:
            obj = query.one()
            if preclude:
                msg = "%s %s already exists." % (clslabel, ", ".join(desc))
                _raise_custom(preclude, ArgumentError, msg)
            return obj
        except NoResultFound:
            if not compel:
                return None
            msg = "%s %s not found." % (clslabel, ", ".join(desc))
            _raise_custom(compel, NotFoundException, msg)
        except MultipleResultsFound:
            msg = "%s %s is not unique." % (clslabel, ", ".join(desc))
            raise ArgumentError(msg)

    @classmethod
    def get_matching_query(cls, session, *args, **kwargs):
        compel = kwargs.get('compel', False)
        query = session.query(cls.__table__.c.id)
        (query, clslabel, desc) = cls._selection_helper(session, query, *args,
                                                        **kwargs)
        if compel:
            obj = query.first()
            if obj is None:
                msg = "%s %s not found." % (clslabel, ", ".join(desc))
                _raise_custom(compel, NotFoundException, msg)
        return query.subquery()

    @classmethod
    def ddl(self):  # pragma: no cover
        """ Returns the DDL SqlAlchemy will use to generate the table.

            This method is used to aid rapid upgrade sql scripts for new
            tables, but make careful note that it won't properly rename any
            constraints made on a per column basis. The utils.contstraints
            module contains utilities for this and should also be part of the
            creation of any new tables during upgrade procedures """

        return str(CreateTable(self.__table__))

#Base = declarative_base(metaclass=VersionedMeta, cls=Base)
Base = declarative_base(cls=Base)

# WAY too much magic in AssociationProxy.  This bug and proposed patch is
# listed in the second half of this message:
# http://groups.google.com/group/sqlalchemy-devel/browse_thread/thread/973f4104007f6964/9a001201b3179c58
# Basically, scalar assocation proxies are much more annoying without this
# patch.  Accessing a null attribute normally returns None.  However, the AP
# tries to proxy through the None.  This raises an exception when the AP
# (in a scalar context) should just itself return None.
@monkeypatch(AssociationProxy)
def __get__(self, obj, class_):
    if self.owning_class is None:
        self.owning_class = class_ and class_ or type(obj)
    if obj is None:
        return self
    elif self.scalar is None:
        self.scalar = self._target_is_scalar()
        if self.scalar:
            self._initialize_scalar_accessors()

    if self.scalar:
        # Original line from 0.5.8
        #proxy = self._new(self._lazy_collection(weakref.ref(obj)))
        #setattr(obj, self.key, (id(obj), proxy))
        #return proxy

        target = getattr(obj, self.target_collection)
        if target is None:
            return None
        else:
            return self._scalar_get(target)
    else:
        try:
            # If the owning instance is reborn (orm session resurrect,
            # etc.), refresh the proxy cache.
            creator_id, proxy = getattr(obj, self.key)
            if id(obj) == creator_id:
                return proxy
        except AttributeError:
            pass
        proxy = self._new(_lazy_collection(obj, self.target_collection))
        setattr(obj, self.key, (id(obj), proxy))
        return proxy
<|endoftext|>


<commit_before>__version_info__ = ('1', '14', '1dev1')
__version__ = '.'.join(__version_info__)

from .wrappers import (ObjectProxy, CallableObjectProxy, FunctionWrapper,
        BoundFunctionWrapper, WeakFunctionProxy, PartialCallableObjectProxy,
        resolve_path, apply_patch, wrap_object, wrap_object_attribute,
        function_wrapper, wrap_function_wrapper, patch_function_wrapper,
        transient_function_wrapper)

from .decorators import (adapter_factory, AdapterFactory, decorator,
        synchronized)

from .importer import (register_post_import_hook, when_imported,
        notify_module_loaded, discover_post_import_hooks)

# Import of inspect.getcallargs() included for backward compatibility. An
# implementation of this was previously bundled and made available here for
# Python <2.7. Avoid using this in future.

from inspect import getcallargs

# Variant of inspect.formatargspec() included here for forward compatibility.
# This is being done because Python 3.11 dropped inspect.formatargspec() but
# code for handling signature changing decorators relied on it. Exposing the
# bundled implementation here in case any user of wrapt was also needing it.

from .arguments import formatargspec
<commit_msg>Update version in preparation for 1.14.1 release.<commit_after>__version_info__ = ('1', '14', '1')
__version__ = '.'.join(__version_info__)

from .wrappers import (ObjectProxy, CallableObjectProxy, FunctionWrapper,
        BoundFunctionWrapper, WeakFunctionProxy, PartialCallableObjectProxy,
        resolve_path, apply_patch, wrap_object, wrap_object_attribute,
        function_wrapper, wrap_function_wrapper, patch_function_wrapper,
        transient_function_wrapper)

from .decorators import (adapter_factory, AdapterFactory, decorator,
        synchronized)

from .importer import (register_post_import_hook, when_imported,
        notify_module_loaded, discover_post_import_hooks)

# Import of inspect.getcallargs() included for backward compatibility. An
# implementation of this was previously bundled and made available here for
# Python <2.7. Avoid using this in future.

from inspect import getcallargs

# Variant of inspect.formatargspec() included here for forward compatibility.
# This is being done because Python 3.11 dropped inspect.formatargspec() but
# code for handling signature changing decorators relied on it. Exposing the
# bundled implementation here in case any user of wrapt was also needing it.

from .arguments import formatargspec
<|endoftext|>


<commit_before>__version_info__ = ('1', '13', '3rc2')
__version__ = '.'.join(__version_info__)

from .wrappers import (ObjectProxy, CallableObjectProxy, FunctionWrapper,
        BoundFunctionWrapper, WeakFunctionProxy, PartialCallableObjectProxy,
        resolve_path, apply_patch, wrap_object, wrap_object_attribute,
        function_wrapper, wrap_function_wrapper, patch_function_wrapper,
        transient_function_wrapper)

from .decorators import (adapter_factory, AdapterFactory, decorator,
        synchronized)

from .importer import (register_post_import_hook, when_imported,
        notify_module_loaded, discover_post_import_hooks)

from inspect import getcallargs
<commit_msg>Increment version to 1.13.3.<commit_after>__version_info__ = ('1', '13', '3')
__version__ = '.'.join(__version_info__)

from .wrappers import (ObjectProxy, CallableObjectProxy, FunctionWrapper,
        BoundFunctionWrapper, WeakFunctionProxy, PartialCallableObjectProxy,
        resolve_path, apply_patch, wrap_object, wrap_object_attribute,
        function_wrapper, wrap_function_wrapper, patch_function_wrapper,
        transient_function_wrapper)

from .decorators import (adapter_factory, AdapterFactory, decorator,
        synchronized)

from .importer import (register_post_import_hook, when_imported,
        notify_module_loaded, discover_post_import_hooks)

from inspect import getcallargs
<|endoftext|>


<commit_before>"""
Fixer for import statements, with a __future__ import line.

Based on lib2to3/fixes/fix_import.py, but extended slightly so it also
supports Cython modules.

If spam is being imported from the local directory, this import:
    from spam import eggs
becomes:
    from .spam import eggs

and this import:
    import spam
becomes:
    from . import spam
"""

from lib2to3.fixes.fix_import import FixImport
from lib2to3.fixer_util import FromImport, syms
from lib2to3.fixes.fix_import import traverse_imports
from libfuturize.fixer_util import future_import

class FixAbsoluteImport(FixImport):
    run_order = 9

    def transform(self, node, results):
        """
        Copied from FixImport.transform(), but with this line added in
        any modules that had implicit relative imports changed:

            from __future__ import absolute_import"
        """
        if self.skip:
            return
        imp = results['imp']

        if node.type == syms.import_from:
            # Some imps are top-level (eg: 'import ham')
            # some are first level (eg: 'import ham.eggs')
            # some are third level (eg: 'import ham.eggs as spam')
            # Hence, the loop
            while not hasattr(imp, 'value'):
                imp = imp.children[0]
            if self.probably_a_local_import(imp.value):
                imp.value = u"." + imp.value
                imp.changed()
                future_import(u"absolute_import", node)
        else:
            have_local = False
            have_absolute = False
            for mod_name in traverse_imports(imp):
                if self.probably_a_local_import(mod_name):
                    have_local = True
                else:
                    have_absolute = True
            if have_absolute:
                if have_local:
                    # We won't handle both sibling and absolute imports in the
                    # same statement at the moment.
                    self.warning(node, "absolute and local imports together")
                return

            new = FromImport(u".", [imp])
            new.prefix = node.prefix
            future_import(u"absolute_import", node)
            return new

    def probably_a_local_import(self, imp_name):
        """
        Like the corresponding method in the base class, but this also
        supports Cython modules.
        """
        if imp_name.startswith(u"."):
            # Relative imports are certainly not local imports.
            return False
        imp_name = imp_name.split(u".", 1)[0]
        base_path = dirname(self.filename)
        base_path = join(base_path, imp_name)
        # If there is no __init__.py next to the file its not in a package
        # so can't be a relative import.
        if not exists(join(dirname(base_path), "__init__.py")):
            return False
        for ext in [".py", sep, ".pyc", ".so", ".sl", ".pyd", ".pyx"]:
            if exists(base_path + ext):
                return True
        return False

<commit_msg>Add missing imports to fix_absolute_import.py<commit_after>"""
Fixer for import statements, with a __future__ import line.

Based on lib2to3/fixes/fix_import.py, but extended slightly so it also
supports Cython modules.

If spam is being imported from the local directory, this import:
    from spam import eggs
becomes:
    from .spam import eggs

and this import:
    import spam
becomes:
    from . import spam
"""

from os.path import dirname, join, exists, sep
from lib2to3.fixes.fix_import import FixImport
from lib2to3.fixer_util import FromImport, syms
from lib2to3.fixes.fix_import import traverse_imports

from libfuturize.fixer_util import future_import

class FixAbsoluteImport(FixImport):
    run_order = 9

    def transform(self, node, results):
        """
        Copied from FixImport.transform(), but with this line added in
        any modules that had implicit relative imports changed:

            from __future__ import absolute_import"
        """
        if self.skip:
            return
        imp = results['imp']

        if node.type == syms.import_from:
            # Some imps are top-level (eg: 'import ham')
            # some are first level (eg: 'import ham.eggs')
            # some are third level (eg: 'import ham.eggs as spam')
            # Hence, the loop
            while not hasattr(imp, 'value'):
                imp = imp.children[0]
            if self.probably_a_local_import(imp.value):
                imp.value = u"." + imp.value
                imp.changed()
                future_import(u"absolute_import", node)
        else:
            have_local = False
            have_absolute = False
            for mod_name in traverse_imports(imp):
                if self.probably_a_local_import(mod_name):
                    have_local = True
                else:
                    have_absolute = True
            if have_absolute:
                if have_local:
                    # We won't handle both sibling and absolute imports in the
                    # same statement at the moment.
                    self.warning(node, "absolute and local imports together")
                return

            new = FromImport(u".", [imp])
            new.prefix = node.prefix
            future_import(u"absolute_import", node)
            return new

    def probably_a_local_import(self, imp_name):
        """
        Like the corresponding method in the base class, but this also
        supports Cython modules.
        """
        if imp_name.startswith(u"."):
            # Relative imports are certainly not local imports.
            return False
        imp_name = imp_name.split(u".", 1)[0]
        base_path = dirname(self.filename)
        base_path = join(base_path, imp_name)
        # If there is no __init__.py next to the file its not in a package
        # so can't be a relative import.
        if not exists(join(dirname(base_path), "__init__.py")):
            return False
        for ext in [".py", sep, ".pyc", ".so", ".sl", ".pyd", ".pyx"]:
            if exists(base_path + ext):
                return True
        return False

<|endoftext|>


<commit_before>from lino.projects.std.settings import *

class Site(Site):

    title = "Cool Polls"

    def get_installed_apps(self):
        for a in super(Site,self).get_installed_apps():
            yield a
        yield 'lino.projects.polls_tutorial.polls' # 'mysite.polls'

SITE = Site(__file__,globals())

DEBUG = True

# The DATABASES setting is the only thing you should take
# over from your original file:
#~ DATABASES = {
    #~ 'default': {
        #~ 'ENGINE': 'django.db.backends.sqlite3',
        #~ 'NAME': abspath(join(dirname(__file__),'test.db'))
    #~ }
#~ }
<commit_msg>http://code.google.com/p/lino/source/browse/docs/blog/2013/0309.rst<commit_after>from lino.projects.std.settings import *

class Site(Site):

    title = "Cool Polls"
    languages = ['en'] # TODO: remove this here

    def get_installed_apps(self):
        for a in super(Site,self).get_installed_apps():
            yield a
        yield 'lino.projects.polls_tutorial.polls' # 'mysite.polls'

SITE = Site(__file__,globals())

DEBUG = True

# The DATABASES setting is the only thing you should take
# over from your original file:
#~ DATABASES = {
    #~ 'default': {
        #~ 'ENGINE': 'django.db.backends.sqlite3',
        #~ 'NAME': abspath(join(dirname(__file__),'test.db'))
    #~ }
#~ }
<|endoftext|>


<commit_before>""" Users API URI specification """
from django.conf.urls import patterns, url

from rest_framework.urlpatterns import format_suffix_patterns

from api_manager.users import views as users_views

urlpatterns = patterns(
    '',
    url(r'^metrics/cities/$', users_views.UsersMetricsCitiesList.as_view(), name='apimgr-users-metrics-cities-list'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/courses/grades$', users_views.UsersCoursesGradesList.as_view(), name='users-courses-grades-list'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/grades$', users_views.UsersCoursesGradesDetail.as_view(), name='users-courses-grades-detail'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/metrics/social/$', users_views.UsersSocialMetrics.as_view(), name='users-social-metrics'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/completions/$', users_views.UsersCoursesCompletionsList.as_view(), name='users-courses-completions-list'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/courses/(?P<course_id>[^/]+/[^/]+/[^/]+)$', users_views.UsersCoursesDetail.as_view(), name='users-courses-detail'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/courses/(?P<course_id>[a-zA-Z0-9_+\/:]+)/grades$', users_views.UsersCoursesGradesDetail.as_view(), name='users-courses-grades-detail'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/courses/(?P<course_id>[a-zA-Z0-9_+\/:]+)/metrics/social/$', users_views.UsersSocialMetrics.as_view(), name='users-social-metrics'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/courses/(?P<course_id>[a-zA-Z0-9_+\/:]+)/completions/$', users_views.UsersCoursesCompletionsList.as_view(), name='users-courses-completions-list'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/courses/(?P<course_id>[a-zA-Z0-9_+\/:]+)$', users_views.UsersCoursesDetail.as_view(), name='users-courses-detail'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/courses/*$', users_views.UsersCoursesList.as_view(), name='users-courses-list'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/groups/*$', users_views.UsersGroupsList.as_view(), name='users-groups-list'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/groups/(?P<group_id>[0-9]+)$', users_views.UsersGroupsDetail.as_view(), name='users-groups-detail'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/preferences$', users_views.UsersPreferences.as_view(), name='users-preferences-list'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/preferences/(?P<preference_id>[a-zA-Z0-9_]+)$', users_views.UsersPreferencesDetail.as_view(), name='users-preferences-detail'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/organizations/$', users_views.UsersOrganizationsList.as_view(), name='users-organizations-list'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/roles/(?P<role>[a-z_]+)/courses/(?P<course_id>[a-zA-Z0-9_+\/:]+)$', users_views.UsersRolesCoursesDetail.as_view(), name='users-roles-courses-detail'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/roles/*$', users_views.UsersRolesList.as_view(), name='users-roles-list'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/workgroups/$', users_views.UsersWorkgroupsList.as_view(), name='users-workgroups-list'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)$', users_views.UsersDetail.as_view(), name='apimgr-users-detail'),
    url(r'/*$^', users_views.UsersList.as_view(), name='apimgr-users-list'),
)

urlpatterns = format_suffix_patterns(urlpatterns)
<commit_msg>Make user API use the same regex as everyone else.<commit_after>""" Users API URI specification """
from django.conf import settings
from django.conf.urls import patterns, url

from rest_framework.urlpatterns import format_suffix_patterns

from api_manager.users import views as users_views

COURSE_ID_PATTERN = settings.COURSE_ID_PATTERN

urlpatterns = patterns(
    '',
    url(r'^metrics/cities/$', users_views.UsersMetricsCitiesList.as_view(), name='apimgr-users-metrics-cities-list'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/courses/grades$', users_views.UsersCoursesGradesList.as_view(), name='users-courses-grades-list'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/courses/{0}/grades$'.format(COURSE_ID_PATTERN), users_views.UsersCoursesGradesDetail.as_view(), name='users-courses-grades-detail'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/courses/{0}/metrics/social/$'.format(COURSE_ID_PATTERN), users_views.UsersSocialMetrics.as_view(), name='users-social-metrics'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/courses/{0}/completions/$'.format(COURSE_ID_PATTERN), users_views.UsersCoursesCompletionsList.as_view(), name='users-courses-completions-list'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/courses/{0}$'.format(COURSE_ID_PATTERN), users_views.UsersCoursesDetail.as_view(), name='users-courses-detail'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/courses/{0}/grades$'.format(COURSE_ID_PATTERN), users_views.UsersCoursesGradesDetail.as_view(), name='users-courses-grades-detail'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/courses/{0}/metrics/social/$'.format(COURSE_ID_PATTERN), users_views.UsersSocialMetrics.as_view(), name='users-social-metrics'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/courses/{0}/completions/$'.format(COURSE_ID_PATTERN), users_views.UsersCoursesCompletionsList.as_view(), name='users-courses-completions-list'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/courses/{0}$'.format(COURSE_ID_PATTERN), users_views.UsersCoursesDetail.as_view(), name='users-courses-detail'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/courses/*$', users_views.UsersCoursesList.as_view(), name='users-courses-list'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/groups/*$', users_views.UsersGroupsList.as_view(), name='users-groups-list'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/groups/(?P<group_id>[0-9]+)$', users_views.UsersGroupsDetail.as_view(), name='users-groups-detail'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/preferences$', users_views.UsersPreferences.as_view(), name='users-preferences-list'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/preferences/(?P<preference_id>[a-zA-Z0-9_]+)$', users_views.UsersPreferencesDetail.as_view(), name='users-preferences-detail'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/organizations/$', users_views.UsersOrganizationsList.as_view(), name='users-organizations-list'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/roles/(?P<role>[a-z_]+)/courses/{0}$'.format(COURSE_ID_PATTERN), users_views.UsersRolesCoursesDetail.as_view(), name='users-roles-courses-detail'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/roles/*$', users_views.UsersRolesList.as_view(), name='users-roles-list'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)/workgroups/$', users_views.UsersWorkgroupsList.as_view(), name='users-workgroups-list'),
    url(r'^(?P<user_id>[a-zA-Z0-9]+)$', users_views.UsersDetail.as_view(), name='apimgr-users-detail'),
    url(r'/*$^', users_views.UsersList.as_view(), name='apimgr-users-list'),
)

urlpatterns = format_suffix_patterns(urlpatterns)
<|endoftext|>


<commit_before>#!python
import os
import tempfile
#
from starcluster.clustersetup import ClusterSetup
from starcluster.logger import log
#
import tabular_predDB.settings as S

run_as_user = lambda node, user, command_str: \
    node.ssh.execute('sudo -u %s %s' % (user, command_str))

def auto_accept_key(node, user):
     # make sure you can programmatically ssh into node
     cmd_str = 'ssh -i %s -o StrictHostKeyChecking=no %s@%s echo $(hostname)'
     cmd_str %= (node.key_location, user, node.ip_address)
     os.system(cmd_str)

def create_pushable_remote_repo(remote_code_dir, node):
     # make sure you remove the old code
     cmd_str = 'rm -rf %s'
     cmd_str %= remote_code_dir
     node.ssh.execute(cmd_str)
     # prepare remote for push
     cmd_str = ' && '.join([
               'mkdir -p %s',
               'cd %s',
               'git init',
               'git config --add receive.denyCurrentBranch ignore'
               ])
     cmd_str %= (remote_code_dir, remote_code_dir)
     node.ssh.execute(cmd_str)

def push_repo_using_node_key(remote_code_dir, node,
                             git_ssh_file='/tmp/git_ssh.sh'):
     # make GIT_SSH script
     cmd_str = 'echo \'ssh -i %s $1 $2\' > %s && chmod a+x %s'
     cmd_str %= (node.key_location, git_ssh_file, git_ssh_file)
     os.system(cmd_str)
     # push up from LOCAL to remote
     cmd_str = 'export GIT_SSH=%s && git push ssh://root@%s%s/ master'
     cmd_str %= (git_ssh_file, node.ip_address, remote_code_dir)
     os.system(cmd_str)

def update_remote_repo_working_tree(remote_code_dir, node):
     # update remote working tree
     cmd_str = 'cd %s && git reset --hard'
     cmd_str %= remote_code_dir
     node.ssh.execute(cmd_str)
     # chown
     cmd_str = 'chown -R %s %s'
     cmd_str %= (user, remote_code_dir)
     node.ssh.execute(cmd_str)

# this complains about
# fatal: Not a git repository (or any of the parent directories): .git
def push_local_repo_to_new_remote(remote_code_dir, node, user):
     auto_accept_key(node, user)
     create_pushable_remote_repo(remote_code_dir, node)
     push_repo_using_node_key(remote_code_dir, node)
     update_remote_repot_working_tree(remote_code_dir, node)

def copy_this_repo(remote_code_dir, node, user, branch=None,
                   temp_base_dir='/tmp/'):
     local_dir = os.path.split(__file__)[0]
     # generate temporary resources
     temp_dir_name = tempfile.mkdtemp(dir=temp_base_dir)
     temp_tgz_name = tempfile.mktemp(dir=temp_base_dir)
     # clone into temp dir
     cmd_str = 'git clone %s %s'
     cmd_str %= (local_dir, temp_dir_name)
     os.system(cmd_str)
     # tgz for speed
     cmd_str = 'cd %s && tar cvfz %s .'
     cmd_str %= (temp_dir_name, temp_tgz_name)
     os.system(cmd_str)
     # put up to node and untar
     node.ssh.put(temp_tgz_name, temp_tgz_name)
     cmd_str = 'mkdir -p %s && cd %s && tar xvfz %s && chown -R %s .'
     cmd_str %= (remote_code_dir, remote_code_dir, temp_tgz_name, user)
     node.ssh.execute(cmd_str)
     # clean up
     cmd_str = 'rm -rf %s %s'
     cmd_str %= (temp_tgz_name, temp_dir_name)
     os.system(cmd_str)
     # switch to branch
     if branch is not None:
          cmd_str = 'cd %s && git checkout %s'
          cmd_str %= (remote_code_dir, branch)
          node.ssh.execute(cmd_str)

class tabular_predDBSetup(ClusterSetup):
     def __init__(self):
         # TODO: Could be generalized to "install a python package plugin"
         pass

     def run(self, nodes, master, user, user_shell, volumes):
          # NOTE: nodes includes master
          remote_home_dir = os.path.join('/home/', user)
          for node in nodes:
               log.info("Installing tabular_predDB (part 1) on %s"
                        % node.alias)
               # push code up
               copy_this_repo(S.path.remote_code_dir, node, user,
                              S.git.branch)
          for node in nodes:
               log.info("Installing tabular_predDB (part 2) on %s"
                        % node.alias)
               # install ubuntu packages
               cmd_str = 'bash %s >ubuntu.out 2>ubuntu.err'
               cmd_str %= S.path.install_ubuntu_script
               node.ssh.execute(cmd_str)
               # install boost
               cmd_str = 'bash %s >boost.out 2>boost.err'
               cmd_str %= S.path.install_boost_script
               node.ssh.execute(cmd_str)
               # install python packages in a virtual env
               cmd_str = 'bash -i %s %s >virtualenv.out 2>virtualenv.err'
               cmd_str %= (S.path.virtualenv_setup_script, user)
               node.ssh.execute(cmd_str)
               # make sure agg backend is used
               cmd_str = ' && '.join([
                         'mkdir -p %s/.matplotlib',
                         'echo backend: Agg > %s/.matplotlib/matplotlibrc',
                         ])
               cmd_str %= (remote_home_dir, remote_home_dir)
               node.ssh.execute(cmd_str)
<commit_msg>clean up install process<commit_after>#!python
import os
import tempfile
#
from starcluster.clustersetup import ClusterSetup
from starcluster.logger import log
#
import tabular_predDB.settings as S

# maybe should prefix the command with "source /etc/profile"
# as starclusters' sshutils.ssh.execute(..., source_profile=True) does
run_as_user = lambda node, user, command_str: \
    node.ssh.execute('sudo -H -u %s %s' % (user, command_str))

def auto_accept_key(node, user):
     # make sure you can programmatically ssh into node
     cmd_str = 'ssh -i %s -o StrictHostKeyChecking=no %s@%s echo $(hostname)'
     cmd_str %= (node.key_location, user, node.ip_address)
     os.system(cmd_str)

def create_pushable_remote_repo(remote_code_dir, node):
     # make sure you remove the old code
     cmd_str = 'rm -rf %s'
     cmd_str %= remote_code_dir
     node.ssh.execute(cmd_str)
     # prepare remote for push
     cmd_str = ' && '.join([
               'mkdir -p %s',
               'cd %s',
               'git init',
               'git config --add receive.denyCurrentBranch ignore'
               ])
     cmd_str %= (remote_code_dir, remote_code_dir)
     node.ssh.execute(cmd_str)

def push_repo_using_node_key(remote_code_dir, node,
                             git_ssh_file='/tmp/git_ssh.sh'):
     # make GIT_SSH script
     cmd_str = 'echo \'ssh -i %s $1 $2\' > %s && chmod a+x %s'
     cmd_str %= (node.key_location, git_ssh_file, git_ssh_file)
     os.system(cmd_str)
     # push up from LOCAL to remote
     cmd_str = 'export GIT_SSH=%s && git push ssh://root@%s%s/ master'
     cmd_str %= (git_ssh_file, node.ip_address, remote_code_dir)
     os.system(cmd_str)

def update_remote_repo_working_tree(remote_code_dir, node):
     # update remote working tree
     cmd_str = 'cd %s && git reset --hard'
     cmd_str %= remote_code_dir
     node.ssh.execute(cmd_str)
     # chown
     cmd_str = 'chown -R %s %s'
     cmd_str %= (user, remote_code_dir)
     node.ssh.execute(cmd_str)

# this complains about
# fatal: Not a git repository (or any of the parent directories): .git
def push_local_repo_to_new_remote(remote_code_dir, node, user):
     auto_accept_key(node, user)
     create_pushable_remote_repo(remote_code_dir, node)
     push_repo_using_node_key(remote_code_dir, node)
     update_remote_repot_working_tree(remote_code_dir, node)

def copy_this_repo(remote_code_dir, node, user, branch='master',
                   temp_base_dir='/tmp/'):
     local_dir = os.path.split(__file__)[0]
     # generate temporary resources
     temp_dir_name = tempfile.mkdtemp(dir=temp_base_dir)
     temp_tgz_name = tempfile.mktemp(dir=temp_base_dir)
     # clone into temp dir
     cmd_str = 'git clone %s %s'
     cmd_str %= (local_dir, temp_dir_name)
     os.system(cmd_str)
     # tgz for speed
     cmd_str = 'cd %s && tar cvfz %s . >tar.out 2>tar.err'
     cmd_str %= (temp_dir_name, temp_tgz_name)
     os.system(cmd_str)
     # put up to node and untar
     node.ssh.put(temp_tgz_name, temp_tgz_name)
     cmd_str = ' && '.join([
               'mkdir -p %s',
               'cd %s',
               'tar xvfz %s',
               'git checkout %s',
               'chown -R %s .'
               ])
     cmd_str %= (remote_code_dir, remote_code_dir, temp_tgz_name,
                 branch, user)
     node.ssh.execute(cmd_str)
     # clean up
     cmd_str = 'rm -rf %s %s'
     cmd_str %= (temp_tgz_name, temp_dir_name)
     os.system(cmd_str)

class tabular_predDBSetup(ClusterSetup):
     def __init__(self):
         # TODO: Could be generalized to "install a python package plugin"
         pass

     def run(self, nodes, master, user, user_shell, volumes):
          # NOTE: nodes includes master
          remote_home_dir = os.path.join('/home/', user)
          for node in nodes:
               log.info("Installing tabular_predDB (part 1) on %s"
                        % node.alias)
               # push code up
               copy_this_repo(S.path.remote_code_dir, node, user,
                              S.git.branch)
          for node in nodes:
               log.info("Installing tabular_predDB (part 2) on %s"
                        % node.alias)
               # install ubuntu packages
               cmd_str = 'bash %s >ubuntu.out 2>ubuntu.err'
               cmd_str %= S.path.install_ubuntu_script
               node.ssh.execute(cmd_str)
               # install boost
               cmd_str = 'bash %s >boost.out 2>boost.err'
               cmd_str %= S.path.install_boost_script
               node.ssh.execute(cmd_str)
               # install python packages in a virtual env
               cmd_str = 'bash -i %s %s >virtualenv.out 2>virtualenv.err'
               cmd_str %= (S.path.virtualenv_setup_script, user)
               node.ssh.execute(cmd_str)
               # make sure agg backend is used
               cmd_str = ' && '.join([
                         'mkdir -p %s/.matplotlib',
                         'echo backend: Agg > %s/.matplotlib/matplotlibrc',
                         ])
               cmd_str %= (remote_home_dir, remote_home_dir)
               node.ssh.execute(cmd_str)
               # compile cython
               cmd_str = 'bash -c -i "workon tabular_predDB && make cython"'
               run_as_user(node, user, cmd_str)
               # set up a server
               server_script = os.path.join(S.path.remote_code_dir,
                                            S.path.server_script)
               cmd_str = 'bash -c -i "workon tabular_predDB && python %s"'
               cmd_str %= server_script
               node.ssh.execute_async(cmd_str)
<|endoftext|>


<commit_before>from starlette.decorators import asgi_application
from starlette.response import (
    FileResponse,
    HTMLResponse,
    JSONResponse,
    Response,
    PlainTextResponse,
    StreamingResponse,
)
from starlette.request import Request
from starlette.testclient import TestClient

__all__ = (
    "asgi_application",
    "FileResponse",
    "HTMLResponse",
    "JSONResponse",
    "PlainTextResponse",
    "Response",
    "StreamingResponse",
    "Request",
    "TestClient",
)
__version__ = "0.1.10"
<commit_msg>Version 0.1.11<commit_after>from starlette.decorators import asgi_application
from starlette.response import (
    FileResponse,
    HTMLResponse,
    JSONResponse,
    Response,
    PlainTextResponse,
    StreamingResponse,
)
from starlette.request import Request
from starlette.testclient import TestClient

__all__ = (
    "asgi_application",
    "FileResponse",
    "HTMLResponse",
    "JSONResponse",
    "PlainTextResponse",
    "Response",
    "StreamingResponse",
    "Request",
    "TestClient",
)
__version__ = "0.1.11"
<|endoftext|>


<commit_before>__version__ = "0.20.3"
<commit_msg>Fix __version__ (#1713)<commit_after>__version__ = "0.20.4"
<|endoftext|>


<commit_before>from sklearn.datasets import fetch_20newsgroups
import matplotlib.pyplot as plt
from collections import Counter

comp_tech = [
    "comp.graphics",
    "comp.os.ms-windows.misc",
    "comp.sys.ibm.pc.hardware",
    "comp.sys.mac.hardware"
]

rec_act = [
    "rec.autos",
    "rec.motorcycles",
    "rec.sport.baseball",
    "rec.sport.hockey"
]

train = fetch_20newsgroups(
    subset = 'train',
    categories = comp_tech + rec_act,
    shuffle=True,
    random_state=23
)
topic_names = train.target_names

cc = Counter(train.target)
freqs = [cc[i] for i in cc]

plt.bar(range(1, 9), freqs, 1/1.5, color='blue')
plt.xticks(range(1, 9), train.target_names, rotation=90)

plt.subplots_adjust(bottom=0.45)
plt.xlabel("Categories")
plt.ylabel("Frequencies")

plt.show()

plt.savefig('plots/histogram.png', format='png')
<commit_msg>added counts for top level classes (a)<commit_after>from sklearn.datasets import fetch_20newsgroups
import matplotlib.pyplot as plt
from collections import Counter

comp_tech = [
    "comp.graphics",
    "comp.os.ms-windows.misc",
    "comp.sys.ibm.pc.hardware",
    "comp.sys.mac.hardware"
]

rec_act = [
    "rec.autos",
    "rec.motorcycles",
    "rec.sport.baseball",
    "rec.sport.hockey"
]

train = fetch_20newsgroups(
    subset = 'train',
    categories = comp_tech + rec_act,
    shuffle=True,
    random_state=23
)
topic_names = train.target_names

cc = Counter(train.target)
freqs = [cc[i] for i in cc]

plt.bar(range(1, 9), freqs, 1/1.5, color='blue')
plt.xticks(range(1, 9), train.target_names, rotation=90)

plt.subplots_adjust(bottom=0.45)
plt.xlabel("Categories")
plt.ylabel("Frequencies")

plt.show()

plt.savefig('plots/histogram.png', format='png')

num_ct = sum(freqs[0:3])
num_rec = sum(freqs[4:7])

print "Number of documents in the Computer Technology class is ", num_ct
print "Number of documents in the Recreational activity class is", num_rec
<|endoftext|>


<commit_before># Load Graphene Dependencies from python-graphenelibs
try:
    from graphenebase.account import PrivateKey, PublicKey, Address, BrainKey
    import graphenebase.base58
    import graphenebase.transactions
    import graphenebase.dictionary
except:
    raise ImportError("Please install 'graphenelibs'")

__all__ = ['transactions',
           'chains',
           'objecttypes',
           'operations',
           'memo']
<commit_msg>[init] Also load PasswordKey by default<commit_after># Load Graphene Dependencies from python-graphenelibs
try:
    from graphenebase.account import PrivateKey, PublicKey, Address, BrainKey, PasswordKey
    import graphenebase.base58
    import graphenebase.transactions
    import graphenebase.dictionary
except:
    raise ImportError("Please install 'graphenelibs'")

__all__ = ['transactions',
           'chains',
           'objecttypes',
           'operations',
           'memo']
<|endoftext|>