<commit_before># coding=utf-8
import json
import requests
from globalvars import GlobalVars
import threading
# noinspection PyPackageRequirements
import websocket
from collections import Iterable
from datetime import datetime, timedelta
from glob import glob
import sys
import traceback
import time
import os
import datahandling
import parsing
import apigetpost
import spamhandling
import classes
import chatcommunicate
from helpers import log, only_blacklists_changed
from gitmanager import GitManager
from blacklists import load_blacklists

# noinspection PyClassHasNoInit,PyBroadException,PyUnresolvedReferences,PyProtectedMember
class Metasmoke:
    @staticmethod
    def init_websocket():
        has_succeeded = False
        while True:
            try:
                GlobalVars.metasmoke_ws = websocket.create_connection(GlobalVars.metasmoke_ws_host,
                                                                      origin=GlobalVars.metasmoke_host)
                payload = json.dumps({"command": "subscribe",
                                      "identifier": "{\"channel\":\"SmokeDetectorChannel\","
                                      "\"key\":\"" + GlobalVars.metasmoke_key + "\"}"})
                GlobalVars.metasmoke_ws.send(payload)

                GlobalVars.metasmoke_ws.settimeout(10)

                has_succeeded = True
                while True:
                    a = GlobalVars.metasmoke_ws.recv()
                    try:
                        data = json.loads(a)
                        GlobalVars.metasmoke_last_ping_time = datetime.now()
                        Metasmoke.handle_websocket_data(data)
                    except Exception as e:
                        GlobalVars.metasmoke_ws = websocket.create_connection(GlobalVars.metasmoke_ws_host,
                                                                              origin=GlobalVars.metasmoke_host)
                        payload = json.dumps({"command": "subscribe",
                                              "identifier": "{\"channel\":\"SmokeDetectorChannel\"}"})
                        GlobalVars.metasmoke_ws.send(payload)
                        log('error', e)
                        try:
                            exc_info = sys.exc_info()
                            traceback.print_exception(*exc_info)
                        except:
                            log('error', "Can't fetch full exception details")
            except:
                log('error', "Couldn't bind to MS websocket")
                if not has_succeeded:
                    break
                else:
                    time.sleep(10)

    @staticmethod
    def check_last_pingtime():
        now = datetime.utcnow()
        errlog = open('errorLogs.txt', 'a', encoding="utf-8")
        if GlobalVars.metasmoke_last_ping_time is None:
            errlog.write("\nINFO/WARNING: SmokeDetector has not received a ping yet, forcing SmokeDetector restart "
                         "to try and reset the connection states.\n%s UTC\n" % now)
            os._exit(10)
        elif GlobalVars.metasmoke_last_ping_time < (datetime.now() - timedelta(seconds=120)):
            errlog.write("\nWARNING: Last metasmoke ping with a response was over 120 seconds ago, "
                         "forcing SmokeDetector restart to reset all sockets.\n%s UTC\n" % now)
            # os._exit(10)
        else:
            pass  # Do nothing

    @staticmethod
    def handle_websocket_data(data):
        if "message" not in data:
            return

        message = data['message']
        if isinstance(message, Iterable):
            if "message" in message:
                chatcommunicate.tell_rooms_with("metasmoke", message['message'])
            elif "exit" in message:
                os._exit(message["exit"])
            elif "blacklist" in message:
                datahandling.add_blacklisted_user((message['blacklist']['uid'], message['blacklist']['site']),
                                                  "metasmoke", message['blacklist']['post'])
            elif "naa" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
                datahandling.add_ignored_post(post_site_id[0:2])
            elif "fp" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
                datahandling.add_false_positive(post_site_id[0:2])
            elif "report" in message:
                post_data = apigetpost.api_get_post(message["report"]["post_link"])
                if post_data is None or post_data is False:
                    return
                if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
                        and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
                    return
                user = parsing.get_user_from_url(post_data.owner_url)
                if user is not None:
                    datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url)
                why = u"Post manually reported by user *{}* from metasmoke.\n".format(message["report"]["user"])
                postobj = classes.Post(api_response={'title': post_data.title, 'body': post_data.body,
                                                     'owner': {'display_name': post_data.owner_name,
                                                               'reputation': post_data.owner_rep,
                                                               'link': post_data.owner_url},
                                                     'site': post_data.site,
                                                     'is_answer': (post_data.post_type == "answer"),
                                                     'score': post_data.score, 'link': post_data.post_url,
                                                     'question_id': post_data.post_id,
                                                     'up_vote_count': post_data.up_vote_count,
                                                     'down_vote_count': post_data.down_vote_count})
                spamhandling.handle_spam(post=postobj,
                                         reasons=["Manually reported " + post_data.post_type],
                                         why=why)
            elif "deploy_updated" in message:
                sha = message["deploy_updated"]["head_commit"]["id"]
                if sha != os.popen('git log --pretty=format:"%H" -n 1').read():
                    if "autopull" in message["deploy_updated"]["head_commit"]["message"]:
                        if only_blacklists_changed(GitManager.get_remote_diff()):
                            commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \
                                        .format(sha[:7])
                            i = []  # Currently no issues with backlists
                            for bl_file in glob('bad_*.txt') + glob('blacklisted_*.txt'):  # Check blacklists for issues
                                with open(bl_file, 'r') as lines:
                                    seen = dict()
                                    for lineno, line in enumerate(lines, 1):
                                        if line.endswith('\r\n'):
                                            i.append("DOS line ending at `{0}:{1}` in {2}".format(bl_file, lineno,
                                                                                commit_md))
                                        if not line.endswith('\n'):
                                            i.append("No newline at end of `{0}` in {1}".format(bl_file, commit_md))
                                        if line == '\n':
                                            i.append("Blank line at `{0}:{1}` in {2}".format(bl_file, lineno,
                                                                                commit_md))
                                        if line in seen:
                                            i.append("Duplicate entry of {0} at lines {1} and {2} of {3} in {4}"
                                                     .format(line.rstrip('\n'), seen[line], lineno, bl_file, commit_md))
                                        seen[line] = lineno
                            if i == []:  # No issues
                                GitManager.pull_remote()
                                load_blacklists()
                                chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists"
                                                                " reloaded.".format(commit_md))
                            else:
                                i.append("please fix before pulling.")
                                chatcommunicate.tell_rooms_with("debug", ", ".join(i))
            elif "commit_status" in message:
                c = message["commit_status"]
                sha = c["commit_sha"][:7]
                if c["commit_sha"] != os.popen('git log --pretty=format:"%H" -n 1').read():
                    if c["status"] == "success":
                        if "autopull" in c["commit_message"]:
                            s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                                "commit/{commit_sha})"\
                                " succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"],
                                                                                commit_sha=sha)
                            chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
                            time.sleep(2)
                            os._exit(3)
                        else:
                            s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                                "commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha)

                            chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
                    elif c["status"] == "failure":
                        s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                            "commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha)

                        chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
            elif "everything_is_broken" in message:
                if message["everything_is_broken"] is True:
                    os._exit(6)

    @staticmethod
    def send_stats_on_post(title, link, reasons, body, username, user_link, why, owner_rep,
                           post_score, up_vote_count, down_vote_count):
        if GlobalVars.metasmoke_host is None:
            log('info', "Metasmoke location not defined, not reporting")
            return

        metasmoke_key = GlobalVars.metasmoke_key

        try:
            if len(why) > 1024:
                why = why[:512] + '...' + why[-512:]

            post = {'title': title, 'link': link, 'reasons': reasons,
                    'body': body, 'username': username, 'user_link': user_link,
                    'why': why, 'user_reputation': owner_rep, 'score': post_score,
                    'upvote_count': up_vote_count, 'downvote_count': down_vote_count}

            # Remove None values (if they somehow manage to get through)
            post = dict((k, v) for k, v in post.items() if v)

            payload = {'post': post, 'key': metasmoke_key}
            headers = {'Content-type': 'application/json'}
            requests.post(GlobalVars.metasmoke_host + "/posts.json", data=json.dumps(payload), headers=headers)
        except Exception as e:
            log('error', e)

    @staticmethod
    def send_feedback_for_post(post_link, feedback_type, user_name, user_id, chat_host):
        if GlobalVars.metasmoke_host is None:
            log('info', "Metasmoke location not defined; not reporting")
            return

        metasmoke_key = GlobalVars.metasmoke_key

        try:
            payload = {
                'feedback': {
                    'user_name': user_name,
                    'chat_user_id': user_id,
                    'chat_host': chat_host,
                    'feedback_type': feedback_type,
                    'post_link': post_link
                },
                'key': metasmoke_key
            }

            headers = {'Content-type': 'application/json'}
            requests.post(GlobalVars.metasmoke_host + "/feedbacks.json", data=json.dumps(payload), headers=headers)

        except Exception as e:
            log('error', e)

    @staticmethod
    def send_deletion_stats_for_post(post_link, is_deleted):
        if GlobalVars.metasmoke_host is None:
            log('info', "Metasmoke location not defined; not reporting deletion stats")
            return

        metasmoke_key = GlobalVars.metasmoke_key

        try:
            payload = {
                'deletion_log': {
                    'is_deleted': is_deleted,
                    'post_link': post_link
                },
                'key': metasmoke_key
            }

            headers = {'Content-type': 'application/json'}
            requests.post(GlobalVars.metasmoke_host + "/deletion_logs.json", data=json.dumps(payload), headers=headers)
        except Exception as e:
            log('error', e)

    @staticmethod
    def send_status_ping():
        if GlobalVars.metasmoke_host is None:
            log('info', "Metasmoke location not defined; not sending status ping")
            return

        metasmoke_key = GlobalVars.metasmoke_key

        try:
            payload = {
                'location': GlobalVars.location,
                'key': metasmoke_key,
                'standby': GlobalVars.standby_mode
            }

            headers = {'content-type': 'application/json'}
            response = requests.post(GlobalVars.metasmoke_host + "/status-update.json",
                                     data=json.dumps(payload), headers=headers)

            try:
                response = response.json()

                if 'failover' in response and GlobalVars.standby_mode:
                    if response['failover']:
                        GlobalVars.standby_mode = False
                        GlobalVars.metasmoke_last_ping_time = datetime.now()  # Otherwise the ping watcher will exit(10)

                        chatcommunicate.tell_rooms_with("debug", GlobalVars.location + " received failover signal.",
                                                        notify_site="/failover")

                    if response['standby']:
                        chatcommunicate.tell_rooms_with("debug",
                                                        GlobalVars.location + " entering metasmoke-forced standby.")
                        time.sleep(2)
                        os._exit(7)

                if 'shutdown' in response:
                    if response['shutdown']:
                        os._exit(6)

            except Exception:
                pass

        except Exception as e:
            log('error', e)

    @staticmethod
    def update_code_privileged_users_list():
        payload = {'key': GlobalVars.metasmoke_key}
        headers = {'Content-type': 'application/json'}
        response = requests.get(GlobalVars.metasmoke_host + "/api/users/code_privileged",
                                data=json.dumps(payload), headers=headers).json()['items']

        GlobalVars.code_privileged_users = set()

        for id in response["stackexchange_chat_ids"]:
            GlobalVars.code_privileged_users.add(("stackexchange.com", id))

        for id in response["meta_stackexchange_chat_ids"]:
            GlobalVars.code_privileged_users.add(("meta.stackexchange.com", id))

        for id in response["stackoverflow_chat_ids"]:
            GlobalVars.code_privileged_users.add(("stackoverflow.com", id))

    @staticmethod
    def determine_if_autoflagged(post_url):
        """
        Given the URL for a post, determine whether or not it has been autoflagged.
        """
        payload = {
            'key': GlobalVars.metasmoke_key,
            'filter': 'GKNJKLILHNFMJLFKINGJJHJOLGFHJF',  # id and autoflagged
            'urls': post_url
        }
        response = requests.get(GlobalVars.metasmoke_host + "/api/v2.0/posts/urls", params=payload).json()

        if len(response["items"]) > 0 and response["items"][0]["autoflagged"]:
            # get flagger names
            id = str(response["items"][0]["id"])
            payload = {'key': GlobalVars.metasmoke_key}

            flags = requests.get(GlobalVars.metasmoke_host + "/api/v2.0/posts/" + id + "/flags", params=payload).json()

            if len(flags["items"]) > 0:
                return True, [user["username"] for user in flags["items"][0]["autoflagged"]["users"]]

        return False, []

    @staticmethod
    def stop_autoflagging():
        payload = {'key': GlobalVars.metasmoke_key}
        headers = {'Content-type': 'application/json'}

        requests.post(GlobalVars.metasmoke_host + "/flagging/smokey_disable",
                      data=json.dumps(payload), headers=headers)

    @staticmethod
    def send_statistics():
        GlobalVars.posts_scan_stats_lock.acquire()
        if GlobalVars.post_scan_time != 0:
            posts_per_second = GlobalVars.num_posts_scanned / GlobalVars.post_scan_time
            payload = {'key': GlobalVars.metasmoke_key,
                       'statistic': {'posts_scanned': GlobalVars.num_posts_scanned, 'api_quota': GlobalVars.apiquota,
                                     'post_scan_rate': posts_per_second}}
        else:
            payload = {'key': GlobalVars.metasmoke_key,
                       'statistic': {'posts_scanned': GlobalVars.num_posts_scanned, 'api_quota': GlobalVars.apiquota}}

        GlobalVars.post_scan_time = 0
        GlobalVars.num_posts_scanned = 0
        GlobalVars.posts_scan_stats_lock.release()

        headers = {'Content-type': 'application/json'}

        if GlobalVars.metasmoke_host is not None:
            requests.post(GlobalVars.metasmoke_host + "/statistics.json",
                          data=json.dumps(payload), headers=headers)
<commit_msg>rmblu if fp from MS<commit_after># coding=utf-8
import json
import requests
from globalvars import GlobalVars
import threading
# noinspection PyPackageRequirements
import websocket
from collections import Iterable
from datetime import datetime, timedelta
from glob import glob
import sys
import traceback
import time
import os
import datahandling
import parsing
import apigetpost
import spamhandling
import classes
import chatcommunicate
from helpers import log, only_blacklists_changed
from gitmanager import GitManager
from blacklists import load_blacklists

# noinspection PyClassHasNoInit,PyBroadException,PyUnresolvedReferences,PyProtectedMember
class Metasmoke:
    @staticmethod
    def init_websocket():
        has_succeeded = False
        while True:
            try:
                GlobalVars.metasmoke_ws = websocket.create_connection(GlobalVars.metasmoke_ws_host,
                                                                      origin=GlobalVars.metasmoke_host)
                payload = json.dumps({"command": "subscribe",
                                      "identifier": "{\"channel\":\"SmokeDetectorChannel\","
                                      "\"key\":\"" + GlobalVars.metasmoke_key + "\"}"})
                GlobalVars.metasmoke_ws.send(payload)

                GlobalVars.metasmoke_ws.settimeout(10)

                has_succeeded = True
                while True:
                    a = GlobalVars.metasmoke_ws.recv()
                    try:
                        data = json.loads(a)
                        GlobalVars.metasmoke_last_ping_time = datetime.now()
                        Metasmoke.handle_websocket_data(data)
                    except Exception as e:
                        GlobalVars.metasmoke_ws = websocket.create_connection(GlobalVars.metasmoke_ws_host,
                                                                              origin=GlobalVars.metasmoke_host)
                        payload = json.dumps({"command": "subscribe",
                                              "identifier": "{\"channel\":\"SmokeDetectorChannel\"}"})
                        GlobalVars.metasmoke_ws.send(payload)
                        log('error', e)
                        try:
                            exc_info = sys.exc_info()
                            traceback.print_exception(*exc_info)
                        except:
                            log('error', "Can't fetch full exception details")
            except:
                log('error', "Couldn't bind to MS websocket")
                if not has_succeeded:
                    break
                else:
                    time.sleep(10)

    @staticmethod
    def check_last_pingtime():
        now = datetime.utcnow()
        errlog = open('errorLogs.txt', 'a', encoding="utf-8")
        if GlobalVars.metasmoke_last_ping_time is None:
            errlog.write("\nINFO/WARNING: SmokeDetector has not received a ping yet, forcing SmokeDetector restart "
                         "to try and reset the connection states.\n%s UTC\n" % now)
            os._exit(10)
        elif GlobalVars.metasmoke_last_ping_time < (datetime.now() - timedelta(seconds=120)):
            errlog.write("\nWARNING: Last metasmoke ping with a response was over 120 seconds ago, "
                         "forcing SmokeDetector restart to reset all sockets.\n%s UTC\n" % now)
            # os._exit(10)
        else:
            pass  # Do nothing

    @staticmethod
    def handle_websocket_data(data):
        if "message" not in data:
            return

        message = data['message']
        if isinstance(message, Iterable):
            if "message" in message:
                chatcommunicate.tell_rooms_with("metasmoke", message['message'])
            elif "exit" in message:
                os._exit(message["exit"])
            elif "blacklist" in message:
                datahandling.add_blacklisted_user((message['blacklist']['uid'], message['blacklist']['site']),
                                                  "metasmoke", message['blacklist']['post'])
            elif "naa" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
                datahandling.add_ignored_post(post_site_id[0:2])
            elif "fp" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
                datahandling.add_false_positive(post_site_id[0:2])
                if datahandling.is_blacklisted_user(user):
                    datahandling.remove_blacklisted_user(user)
            elif "report" in message:
                post_data = apigetpost.api_get_post(message["report"]["post_link"])
                if post_data is None or post_data is False:
                    return
                if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
                        and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
                    return
                user = parsing.get_user_from_url(post_data.owner_url)
                if user is not None:
                    datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url)
                why = u"Post manually reported by user *{}* from metasmoke.\n".format(message["report"]["user"])
                postobj = classes.Post(api_response={'title': post_data.title, 'body': post_data.body,
                                                     'owner': {'display_name': post_data.owner_name,
                                                               'reputation': post_data.owner_rep,
                                                               'link': post_data.owner_url},
                                                     'site': post_data.site,
                                                     'is_answer': (post_data.post_type == "answer"),
                                                     'score': post_data.score, 'link': post_data.post_url,
                                                     'question_id': post_data.post_id,
                                                     'up_vote_count': post_data.up_vote_count,
                                                     'down_vote_count': post_data.down_vote_count})
                spamhandling.handle_spam(post=postobj,
                                         reasons=["Manually reported " + post_data.post_type],
                                         why=why)
            elif "deploy_updated" in message:
                sha = message["deploy_updated"]["head_commit"]["id"]
                if sha != os.popen('git log --pretty=format:"%H" -n 1').read():
                    if "autopull" in message["deploy_updated"]["head_commit"]["message"]:
                        if only_blacklists_changed(GitManager.get_remote_diff()):
                            commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \
                                        .format(sha[:7])
                            i = []  # Currently no issues with backlists
                            for bl_file in glob('bad_*.txt') + glob('blacklisted_*.txt'):  # Check blacklists for issues
                                with open(bl_file, 'r') as lines:
                                    seen = dict()
                                    for lineno, line in enumerate(lines, 1):
                                        if line.endswith('\r\n'):
                                            i.append("DOS line ending at `{0}:{1}` in {2}".format(bl_file, lineno,
                                                                                commit_md))
                                        if not line.endswith('\n'):
                                            i.append("No newline at end of `{0}` in {1}".format(bl_file, commit_md))
                                        if line == '\n':
                                            i.append("Blank line at `{0}:{1}` in {2}".format(bl_file, lineno,
                                                                                commit_md))
                                        if line in seen:
                                            i.append("Duplicate entry of {0} at lines {1} and {2} of {3} in {4}"
                                                     .format(line.rstrip('\n'), seen[line], lineno, bl_file, commit_md))
                                        seen[line] = lineno
                            if i == []:  # No issues
                                GitManager.pull_remote()
                                load_blacklists()
                                chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists"
                                                                " reloaded.".format(commit_md))
                            else:
                                i.append("please fix before pulling.")
                                chatcommunicate.tell_rooms_with("debug", ", ".join(i))
            elif "commit_status" in message:
                c = message["commit_status"]
                sha = c["commit_sha"][:7]
                if c["commit_sha"] != os.popen('git log --pretty=format:"%H" -n 1').read():
                    if c["status"] == "success":
                        if "autopull" in c["commit_message"]:
                            s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                                "commit/{commit_sha})"\
                                " succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"],
                                                                                commit_sha=sha)
                            chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
                            time.sleep(2)
                            os._exit(3)
                        else:
                            s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                                "commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha)

                            chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
                    elif c["status"] == "failure":
                        s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                            "commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha)

                        chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
            elif "everything_is_broken" in message:
                if message["everything_is_broken"] is True:
                    os._exit(6)

    @staticmethod
    def send_stats_on_post(title, link, reasons, body, username, user_link, why, owner_rep,
                           post_score, up_vote_count, down_vote_count):
        if GlobalVars.metasmoke_host is None:
            log('info', "Metasmoke location not defined, not reporting")
            return

        metasmoke_key = GlobalVars.metasmoke_key

        try:
            if len(why) > 1024:
                why = why[:512] + '...' + why[-512:]

            post = {'title': title, 'link': link, 'reasons': reasons,
                    'body': body, 'username': username, 'user_link': user_link,
                    'why': why, 'user_reputation': owner_rep, 'score': post_score,
                    'upvote_count': up_vote_count, 'downvote_count': down_vote_count}

            # Remove None values (if they somehow manage to get through)
            post = dict((k, v) for k, v in post.items() if v)

            payload = {'post': post, 'key': metasmoke_key}
            headers = {'Content-type': 'application/json'}
            requests.post(GlobalVars.metasmoke_host + "/posts.json", data=json.dumps(payload), headers=headers)
        except Exception as e:
            log('error', e)

    @staticmethod
    def send_feedback_for_post(post_link, feedback_type, user_name, user_id, chat_host):
        if GlobalVars.metasmoke_host is None:
            log('info', "Metasmoke location not defined; not reporting")
            return

        metasmoke_key = GlobalVars.metasmoke_key

        try:
            payload = {
                'feedback': {
                    'user_name': user_name,
                    'chat_user_id': user_id,
                    'chat_host': chat_host,
                    'feedback_type': feedback_type,
                    'post_link': post_link
                },
                'key': metasmoke_key
            }

            headers = {'Content-type': 'application/json'}
            requests.post(GlobalVars.metasmoke_host + "/feedbacks.json", data=json.dumps(payload), headers=headers)

        except Exception as e:
            log('error', e)

    @staticmethod
    def send_deletion_stats_for_post(post_link, is_deleted):
        if GlobalVars.metasmoke_host is None:
            log('info', "Metasmoke location not defined; not reporting deletion stats")
            return

        metasmoke_key = GlobalVars.metasmoke_key

        try:
            payload = {
                'deletion_log': {
                    'is_deleted': is_deleted,
                    'post_link': post_link
                },
                'key': metasmoke_key
            }

            headers = {'Content-type': 'application/json'}
            requests.post(GlobalVars.metasmoke_host + "/deletion_logs.json", data=json.dumps(payload), headers=headers)
        except Exception as e:
            log('error', e)

    @staticmethod
    def send_status_ping():
        if GlobalVars.metasmoke_host is None:
            log('info', "Metasmoke location not defined; not sending status ping")
            return

        metasmoke_key = GlobalVars.metasmoke_key

        try:
            payload = {
                'location': GlobalVars.location,
                'key': metasmoke_key,
                'standby': GlobalVars.standby_mode
            }

            headers = {'content-type': 'application/json'}
            response = requests.post(GlobalVars.metasmoke_host + "/status-update.json",
                                     data=json.dumps(payload), headers=headers)

            try:
                response = response.json()

                if 'failover' in response and GlobalVars.standby_mode:
                    if response['failover']:
                        GlobalVars.standby_mode = False
                        GlobalVars.metasmoke_last_ping_time = datetime.now()  # Otherwise the ping watcher will exit(10)

                        chatcommunicate.tell_rooms_with("debug", GlobalVars.location + " received failover signal.",
                                                        notify_site="/failover")

                    if response['standby']:
                        chatcommunicate.tell_rooms_with("debug",
                                                        GlobalVars.location + " entering metasmoke-forced standby.")
                        time.sleep(2)
                        os._exit(7)

                if 'shutdown' in response:
                    if response['shutdown']:
                        os._exit(6)

            except Exception:
                pass

        except Exception as e:
            log('error', e)

    @staticmethod
    def update_code_privileged_users_list():
        payload = {'key': GlobalVars.metasmoke_key}
        headers = {'Content-type': 'application/json'}
        response = requests.get(GlobalVars.metasmoke_host + "/api/users/code_privileged",
                                data=json.dumps(payload), headers=headers).json()['items']

        GlobalVars.code_privileged_users = set()

        for id in response["stackexchange_chat_ids"]:
            GlobalVars.code_privileged_users.add(("stackexchange.com", id))

        for id in response["meta_stackexchange_chat_ids"]:
            GlobalVars.code_privileged_users.add(("meta.stackexchange.com", id))

        for id in response["stackoverflow_chat_ids"]:
            GlobalVars.code_privileged_users.add(("stackoverflow.com", id))

    @staticmethod
    def determine_if_autoflagged(post_url):
        """
        Given the URL for a post, determine whether or not it has been autoflagged.
        """
        payload = {
            'key': GlobalVars.metasmoke_key,
            'filter': 'GKNJKLILHNFMJLFKINGJJHJOLGFHJF',  # id and autoflagged
            'urls': post_url
        }
        response = requests.get(GlobalVars.metasmoke_host + "/api/v2.0/posts/urls", params=payload).json()

        if len(response["items"]) > 0 and response["items"][0]["autoflagged"]:
            # get flagger names
            id = str(response["items"][0]["id"])
            payload = {'key': GlobalVars.metasmoke_key}

            flags = requests.get(GlobalVars.metasmoke_host + "/api/v2.0/posts/" + id + "/flags", params=payload).json()

            if len(flags["items"]) > 0:
                return True, [user["username"] for user in flags["items"][0]["autoflagged"]["users"]]

        return False, []

    @staticmethod
    def stop_autoflagging():
        payload = {'key': GlobalVars.metasmoke_key}
        headers = {'Content-type': 'application/json'}

        requests.post(GlobalVars.metasmoke_host + "/flagging/smokey_disable",
                      data=json.dumps(payload), headers=headers)

    @staticmethod
    def send_statistics():
        GlobalVars.posts_scan_stats_lock.acquire()
        if GlobalVars.post_scan_time != 0:
            posts_per_second = GlobalVars.num_posts_scanned / GlobalVars.post_scan_time
            payload = {'key': GlobalVars.metasmoke_key,
                       'statistic': {'posts_scanned': GlobalVars.num_posts_scanned, 'api_quota': GlobalVars.apiquota,
                                     'post_scan_rate': posts_per_second}}
        else:
            payload = {'key': GlobalVars.metasmoke_key,
                       'statistic': {'posts_scanned': GlobalVars.num_posts_scanned, 'api_quota': GlobalVars.apiquota}}

        GlobalVars.post_scan_time = 0
        GlobalVars.num_posts_scanned = 0
        GlobalVars.posts_scan_stats_lock.release()

        headers = {'Content-type': 'application/json'}

        if GlobalVars.metasmoke_host is not None:
            requests.post(GlobalVars.metasmoke_host + "/statistics.json",
                          data=json.dumps(payload), headers=headers)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (C) 2010 Benjamin Pollack.  All rights reserved.

from __future__ import with_statement

import datetime
import getopt
import os
import re
import pickle
import select
import socket
import sys
import threading

class RedisError(object):
    def __init__(self, message):
        self.message = message

    def __repr__(self):
        return '<RedisError(%s)>' % self.message

class RedisConstant(object):
    def __init__(self, type):
        self.type = type

    def __repr__(self):
        return '<RedisConstant(%s)>' % self.type

EMPTY_SCALAR = RedisConstant('EmptyScalar')
EMPTY_LIST = RedisConstant('EmptyList')
BAD_VALUE = RedisError('Operation against a key holding the wrong kind of value')

class RedisClient(object):
    def __init__(self, socket):
        self.socket = socket
        self.wfile = socket.makefile('wb')
        self.rfile = socket.makefile('rb')
        self.db = None
        self.table = None

class MiniRedis(threading.Thread):
    def __init__(self, host='127.0.0.1', port=6379, log_file=None, db_file=None):
        super(MiniRedis, self).__init__()
        self.host = host
        self.port = port
        self.log_file = log_file
        self.halt = True

        self.clients = {}
        self.tables = {}
        self.db_file = db_file

        self.load()

    def dump(self, client, o):
        nl = '\r\n'
        if isinstance(o, bool):
            if o:
                client.wfile.write('+OK\r\n')
            # Show nothing for a false return; that means be quiet
        elif o == EMPTY_SCALAR:
            client.wfile.write('$-1\r\n')
        elif o == EMPTY_LIST:
            client.wfile.write('*-1\r\n')
        elif isinstance(o, int):
            client.wfile.write(':' + str(o) + nl)
        elif isinstance(o, str):
            client.wfile.write('$' + str(len(o)) + nl)
            client.wfile.write(o + nl)
        elif isinstance(o, list):
            client.wfile.write('*' + str(len(o)) + nl)
            for val in o:
                self.dump(client, str(val))
        elif isinstance(o, RedisError):
            client.wfile.write('-ERR %s\r\n' % o.message)
        else:
            client.wfile.write('return type not yet implemented\r\n')
        client.wfile.flush()

    def load(self):
        if self.db_file and os.path.lexists(self.db_file):
            with open(self.db_file, 'rb') as f:
                self.tables = pickle.load(f)
                self.log(None, 'loaded database from file "%s"' % self.db_file)

    def log(self, client, s):
        if self.log_file:
            try:
                who = '%s:%s' % client.socket.getpeername() if client else 'SERVER'
            except:
                who = '<CLOSED>'
            self.log_file.write('%s - %s: %s\n' % (datetime.datetime.now(), who, s))
            self.log_file.flush()

    def handle(self, client):
        line = client.rfile.readline()
        if not line:
            self.log(client, 'client disconnected')
            del self.clients[client.socket]
            client.socket.close()
            return
        items = int(line[1:].strip())
        args = []
        for x in xrange(0, items):
            length = int(client.rfile.readline().strip()[1:])
            args.append(client.rfile.read(length))
            client.rfile.read(2) # throw out newline
        command = args[0].lower()
        self.dump(client, getattr(self, 'handle_' + command)(client, *args[1:]))

    def run(self):
        self.halt = False
        server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server.bind((self.host, self.port))
        server.listen(5)
        while not self.halt:
            readable, _, _ = select.select([server] + self.clients.keys(), [], [], 1.0)
            for sock in readable:
                if sock == server:
                    (client_socket, address) = server.accept()
                    client = RedisClient(client_socket)
                    self.clients[client_socket] = client
                    self.log(client, 'client connected')
                    self.select(client, 0)
                else:
                    try:
                        self.handle(self.clients[sock])
                    except Exception, e:
                        self.log(client, 'exception: %s' % e)
                        self.handle_quit(client)
        for client_socket in self.clients.iterkeys():
            client_socket.close()
        self.clients.clear()
        server.close()
        server = None

    def save(self):
        if self.db_file:
            with open(self.db_file, 'wb') as f:
                pickle.dump(self.tables, f, pickle.HIGHEST_PROTOCOL)

    def select(self, client, db):
        if db not in self.tables:
            self.tables[db] = {}
        client.db = db
        client.table = self.tables[db]

    def stop(self):
        self.halt = True
        self.join()

    # HANDLERS

    def handle_del(self, client, key):
        self.log(client, 'DEL %s' % key)
        if key not in client.table:
            return 0
        del client.table[key]
        return 1

    def handle_flushdb(self, client):
        self.log(client, 'FLUSHDB')
        client.table.clear()
        return True

    def handle_flushall(self, client):
        self.log(client, 'FLUSHALL')
        for table in self.tables.itervalues():
            table.clear()
        return True

    def handle_get(self, client, key):
        data = client.table.get(key, None)
        if isinstance(data, list):
            return BAD_VALUE
        if data != None:
            data = str(data)
        else:
            data = EMPTY_SCALAR
        self.log(client, 'GET %s -> %s' % (key, data))
        return data

    def handle_incr(self, client, key):
        return self.handle_incrby(client, key, 1)

    def handle_incrby(self, client, key, by):
        try:
            client.table[key] = int(client.table[key])
            client.table[key] += int(by)
        except (KeyError, TypeError, ValueError):
            client.table[key] = 1
        self.log(client, 'INCRBY %s %s -> %s' % (key, by, client.table[key]))
        return client.table[key]

    def handle_keys(self, client, pattern):
        r = re.compile(pattern.replace('*', '.*'))
        self.log(client, 'KEYS %s' % pattern)
        return ' '.join(k for k in client.table.keys() if r.search(k))

    def handle_lpush(self, client, key, data):
        if key not in client.table:
            client.table[key] = []
        elif not isinstance(client.table[key], list):
            return BAD_VALUE
        client.table[key].insert(0, data)
        self.log(client, 'LPUSH %s %s' % (key, data))
        return True

    def handle_lrange(self, client, key, low, high):
        low, high = int(low), int(high)
        if low == 0 and high == -1:
            high = None
        if key not in client.table:
            return EMPTY_LIST
        if not isinstance(client.table[key], list):
            return BAD_VALUE
        self.log(client, 'LRANGE %s %s %s -> %s' % (key, low, high, client.table[key][low:high]))
        return client.table[key][low:high]

    def handle_rpop(self, client, key):
        if key not in client.table:
            return EMPTY_SCALAR
        if not isinstance(client.table[key], list):
            return BAD_VALUE
        if len(client.table[key]) > 0:
            data = client.table[key].pop()
        else:
            data = EMPTY_SCALAR
        self.log(client, 'LPOP %s -> %s' % (key, data))
        return data

    def handle_quit(self, client):
        client.socket.shutdown(socket.SHUT_RDWR)
        client.socket.close()
        self.log(client, 'QUIT')
        del self.clients[client.socket]
        return False

    def handle_save(self, client):
        try:
            if not os.fork():
                self.save()
                sys.exit(0)
        except OSError:
            self.save()
        self.log(client, 'SAVE')
        return True

    def handle_select(self, client, db):
        db = int(db)
        self.select(client, db)
        self.log(client, 'SELECT %s' % db)
        return True

    def handle_set(self, client, key, data):
        client.table[key] = data
        self.log(client, 'SET %s -> %s' % (key, data))
        return True

    def handle_setnx(self, client, key, data):
        if key in client.table:
            self.log(client, 'SETNX %s -> %s FAILED' % (key, data))
            return 0
        client.table[key] = data
        self.log(client, 'SETNX %s -> %s' % (key, data))
        return 1

    def handle_shutdown(self, client):
        self.log(client, 'SHUTDOWN')
        self.halt = True
        return self.handle_quit(client)

def main(args):
    host, port, log_file, db_file = '127.0.0.1', 6379, None, None
    opts, args = getopt.getopt(args, 'h:p:d:l')
    for o, a in opts:
        if o == '-h':
            host = a
        elif o == '-p':
            port = int(a)
        elif o == '-l':
            log_file = sys.stdout
        elif o == '-d':
            db_file = os.path.abspath(a)
    m = MiniRedis(host=host, port=port, log_file=log_file, db_file=db_file)
    m.start()
    m.join()

if __name__ == '__main__':
    main(sys.argv[1:])
<commit_msg>enable SO_REUSEADDR to avoid address conflicts<commit_after>#!/usr/bin/env python
# Copyright (C) 2010 Benjamin Pollack.  All rights reserved.

from __future__ import with_statement

import datetime
import getopt
import os
import re
import pickle
import select
import socket
import sys
import threading

class RedisError(object):
    def __init__(self, message):
        self.message = message

    def __repr__(self):
        return '<RedisError(%s)>' % self.message

class RedisConstant(object):
    def __init__(self, type):
        self.type = type

    def __repr__(self):
        return '<RedisConstant(%s)>' % self.type

EMPTY_SCALAR = RedisConstant('EmptyScalar')
EMPTY_LIST = RedisConstant('EmptyList')
BAD_VALUE = RedisError('Operation against a key holding the wrong kind of value')

class RedisClient(object):
    def __init__(self, socket):
        self.socket = socket
        self.wfile = socket.makefile('wb')
        self.rfile = socket.makefile('rb')
        self.db = None
        self.table = None

class MiniRedis(threading.Thread):
    def __init__(self, host='127.0.0.1', port=6379, log_file=None, db_file=None):
        super(MiniRedis, self).__init__()
        self.host = host
        self.port = port
        self.log_file = log_file
        self.halt = True

        self.clients = {}
        self.tables = {}
        self.db_file = db_file

        self.load()

    def dump(self, client, o):
        nl = '\r\n'
        if isinstance(o, bool):
            if o:
                client.wfile.write('+OK\r\n')
            # Show nothing for a false return; that means be quiet
        elif o == EMPTY_SCALAR:
            client.wfile.write('$-1\r\n')
        elif o == EMPTY_LIST:
            client.wfile.write('*-1\r\n')
        elif isinstance(o, int):
            client.wfile.write(':' + str(o) + nl)
        elif isinstance(o, str):
            client.wfile.write('$' + str(len(o)) + nl)
            client.wfile.write(o + nl)
        elif isinstance(o, list):
            client.wfile.write('*' + str(len(o)) + nl)
            for val in o:
                self.dump(client, str(val))
        elif isinstance(o, RedisError):
            client.wfile.write('-ERR %s\r\n' % o.message)
        else:
            client.wfile.write('return type not yet implemented\r\n')
        client.wfile.flush()

    def load(self):
        if self.db_file and os.path.lexists(self.db_file):
            with open(self.db_file, 'rb') as f:
                self.tables = pickle.load(f)
                self.log(None, 'loaded database from file "%s"' % self.db_file)

    def log(self, client, s):
        if self.log_file:
            try:
                who = '%s:%s' % client.socket.getpeername() if client else 'SERVER'
            except:
                who = '<CLOSED>'
            self.log_file.write('%s - %s: %s\n' % (datetime.datetime.now(), who, s))
            self.log_file.flush()

    def handle(self, client):
        line = client.rfile.readline()
        if not line:
            self.log(client, 'client disconnected')
            del self.clients[client.socket]
            client.socket.close()
            return
        items = int(line[1:].strip())
        args = []
        for x in xrange(0, items):
            length = int(client.rfile.readline().strip()[1:])
            args.append(client.rfile.read(length))
            client.rfile.read(2) # throw out newline
        command = args[0].lower()
        self.dump(client, getattr(self, 'handle_' + command)(client, *args[1:]))

    def run(self):
        self.halt = False
        server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        server.bind((self.host, self.port))
        server.listen(5)
        while not self.halt:
            readable, _, _ = select.select([server] + self.clients.keys(), [], [], 1.0)
            for sock in readable:
                if sock == server:
                    (client_socket, address) = server.accept()
                    client = RedisClient(client_socket)
                    self.clients[client_socket] = client
                    self.log(client, 'client connected')
                    self.select(client, 0)
                else:
                    try:
                        self.handle(self.clients[sock])
                    except Exception, e:
                        self.log(client, 'exception: %s' % e)
                        self.handle_quit(client)
        for client_socket in self.clients.iterkeys():
            client_socket.close()
        self.clients.clear()
        server.close()
        server = None

    def save(self):
        if self.db_file:
            with open(self.db_file, 'wb') as f:
                pickle.dump(self.tables, f, pickle.HIGHEST_PROTOCOL)

    def select(self, client, db):
        if db not in self.tables:
            self.tables[db] = {}
        client.db = db
        client.table = self.tables[db]

    def stop(self):
        self.halt = True
        self.join()

    # HANDLERS

    def handle_del(self, client, key):
        self.log(client, 'DEL %s' % key)
        if key not in client.table:
            return 0
        del client.table[key]
        return 1

    def handle_flushdb(self, client):
        self.log(client, 'FLUSHDB')
        client.table.clear()
        return True

    def handle_flushall(self, client):
        self.log(client, 'FLUSHALL')
        for table in self.tables.itervalues():
            table.clear()
        return True

    def handle_get(self, client, key):
        data = client.table.get(key, None)
        if isinstance(data, list):
            return BAD_VALUE
        if data != None:
            data = str(data)
        else:
            data = EMPTY_SCALAR
        self.log(client, 'GET %s -> %s' % (key, data))
        return data

    def handle_incr(self, client, key):
        return self.handle_incrby(client, key, 1)

    def handle_incrby(self, client, key, by):
        try:
            client.table[key] = int(client.table[key])
            client.table[key] += int(by)
        except (KeyError, TypeError, ValueError):
            client.table[key] = 1
        self.log(client, 'INCRBY %s %s -> %s' % (key, by, client.table[key]))
        return client.table[key]

    def handle_keys(self, client, pattern):
        r = re.compile(pattern.replace('*', '.*'))
        self.log(client, 'KEYS %s' % pattern)
        return ' '.join(k for k in client.table.keys() if r.search(k))

    def handle_lpush(self, client, key, data):
        if key not in client.table:
            client.table[key] = []
        elif not isinstance(client.table[key], list):
            return BAD_VALUE
        client.table[key].insert(0, data)
        self.log(client, 'LPUSH %s %s' % (key, data))
        return True

    def handle_lrange(self, client, key, low, high):
        low, high = int(low), int(high)
        if low == 0 and high == -1:
            high = None
        if key not in client.table:
            return EMPTY_LIST
        if not isinstance(client.table[key], list):
            return BAD_VALUE
        self.log(client, 'LRANGE %s %s %s -> %s' % (key, low, high, client.table[key][low:high]))
        return client.table[key][low:high]

    def handle_rpop(self, client, key):
        if key not in client.table:
            return EMPTY_SCALAR
        if not isinstance(client.table[key], list):
            return BAD_VALUE
        if len(client.table[key]) > 0:
            data = client.table[key].pop()
        else:
            data = EMPTY_SCALAR
        self.log(client, 'LPOP %s -> %s' % (key, data))
        return data

    def handle_quit(self, client):
        client.socket.shutdown(socket.SHUT_RDWR)
        client.socket.close()
        self.log(client, 'QUIT')
        del self.clients[client.socket]
        return False

    def handle_save(self, client):
        try:
            if not os.fork():
                self.save()
                sys.exit(0)
        except OSError:
            self.save()
        self.log(client, 'SAVE')
        return True

    def handle_select(self, client, db):
        db = int(db)
        self.select(client, db)
        self.log(client, 'SELECT %s' % db)
        return True

    def handle_set(self, client, key, data):
        client.table[key] = data
        self.log(client, 'SET %s -> %s' % (key, data))
        return True

    def handle_setnx(self, client, key, data):
        if key in client.table:
            self.log(client, 'SETNX %s -> %s FAILED' % (key, data))
            return 0
        client.table[key] = data
        self.log(client, 'SETNX %s -> %s' % (key, data))
        return 1

    def handle_shutdown(self, client):
        self.log(client, 'SHUTDOWN')
        self.halt = True
        return self.handle_quit(client)

def main(args):
    host, port, log_file, db_file = '127.0.0.1', 6379, None, None
    opts, args = getopt.getopt(args, 'h:p:d:l')
    for o, a in opts:
        if o == '-h':
            host = a
        elif o == '-p':
            port = int(a)
        elif o == '-l':
            log_file = sys.stdout
        elif o == '-d':
            db_file = os.path.abspath(a)
    m = MiniRedis(host=host, port=port, log_file=log_file, db_file=db_file)
    m.start()
    m.join()

if __name__ == '__main__':
    main(sys.argv[1:])
<|endoftext|>


<commit_before>db = DAL('sqlite://acacrits.db')

db.define_table(
    'athlete',
    Field('last_name', required=True),
    Field('first_name', required=True),
    Field('additional_names'),
    Field('suffix'),
    Field('gender', length=1, required=True, requires=IS_IN_SET(('m', 'f'))),
    Field('usac_license_numer', 'integer'))
db.athlete.id.readable = False

db.define_table(
    'official',
    Field('last_name', required=True),
    Field('first_name', required=True),
    Field('additional_names'),
    Field('suffix'))
db.official.id.readable = False

db.define_table(
    'marshal',
    Field('last_name', required=True),
    Field('first_name', required=True),
    Field('additional_names'),
    Field('suffix'))
db.marshal.id.readable = False

db.define_table(
    'season',
    Field('year', 'integer', required=True, unique=True))

db.define_table(
    'race_class',
    Field('description', required=True, unique=True))

db.define_table(
    'race_subclass',
    Field('description', required=True, unique=True))

db.define_table(
    'raceday',
    Field('date', 'date', required=True),
    Field('conditions'),
    Field('temperature_f', 'double'))

db.define_table(
    'racedayofficial',
    Field('raceday_id', 'reference raceday'),
    Field('official_id', 'reference official'))

db.define_table(
    'race',
    Field('date', 'date', required=True),
    Field('class_id', 'reference race_class', required=True,
          requires=IS_IN_DB(db, db.race_class.id, '%(description)s')),
    Field('laps', 'integer'),
    Field('slow_lap_secs', 'double'),
    Field('fast_lap_secs', 'double'),
    Field('average_lap_secs', 'double'))

db.define_table(
    'team',
    Field('name', required=True, unique=True))
db.team.id.readable = False

db.define_table(
    'athleteteam',
    Field('athlete_id', 'reference athlete'),
    Field('team_id', 'reference team'),
    Field('season_id', 'reference season',
          requires=IS_IN_DB(db, db.season.id, '%(year)d')))

db.define_table(
    'participant',
    Field('athlete_id', 'reference athlete', required=True),
    Field('team_id', 'reference team'),
    Field('race_id', 'reference race', required=True))

db.define_table(
    'racemarshal',
    Field('race_id', 'reference race', required=True),
    Field('marshal_id', 'reference marshal', required=True))

db.define_table(
    'result',
    Field('participant_id', 'reference participant', required=True),
    Field('place', 'integer'),
    Field('points', 'double'),
    Field('mar_place', 'integer'),
    Field('point_prime', 'boolean'))

db.define_table(
    'subclassresult',
    Field('participant_id', 'reference participant', required=True),
    Field('subclass_id', 'reference race_subclass', required=True),
    Field('place', 'integer'),
    Field('points', 'double'))

db.define_table(
    'prime',
    Field('participant_id', 'reference participant'),
    Field('description'))

# Hard-coded tables.
def _():
    def update_table(table, table_data):
        cur_table_data = db().select(table.ALL)
        cur_keys = set()
        for row in cur_table_data:
            cur_keys.add(row.id)
            non_id_vals = {k: v for k, v in row.items() if k != 'id'}
            if row.id in table_data:
                table[row.id] = table_data[row.id]
            else:
                del table[row.id]
        for key, val in table_data.items():
            if not key in cur_keys:
                table.insert(**val)

    CURRENT_SEASON = 2015
    SEASONS_TABLE = {
        i+1: dict(year=year) for i, year in
        enumerate(xrange(1996, CURRENT_SEASON+1))}
    update_table(db.season, SEASONS_TABLE)

    CLASSES = ['A', 'B', 'C', 'Masters 40+/Women', 'Juniors']
    CLASSES_TABLE = {
        i+1: dict(description=description)
        for i, description in enumerate(CLASSES)}
    update_table(db.race_class, CLASSES_TABLE)

    SUBCLASSES = ['Women', 'Juniors']
    SUBCLASSES_TABLE = {
        i+1:dict(description=description)
        for i, description in enumerate(SUBCLASSES)}
    update_table(db.race_subclass, SUBCLASSES_TABLE)

    db.commit()

_()
<commit_msg>Use underscore naming uniformly for tables/columns.<commit_after>db = DAL('sqlite://acacrits.db')

db.define_table(
    'athlete',
    Field('last_name', required=True),
    Field('first_name', required=True),
    Field('additional_names'),
    Field('suffix'),
    Field('gender', length=1, required=True, requires=IS_IN_SET(('m', 'f'))),
    Field('usac_license_numer', 'integer'))
db.athlete.id.readable = False

db.define_table(
    'official',
    Field('last_name', required=True),
    Field('first_name', required=True),
    Field('additional_names'),
    Field('suffix'))
db.official.id.readable = False

db.define_table(
    'marshal',
    Field('last_name', required=True),
    Field('first_name', required=True),
    Field('additional_names'),
    Field('suffix'))
db.marshal.id.readable = False

db.define_table(
    'season',
    Field('year', 'integer', required=True, unique=True))

db.define_table(
    'race_class',
    Field('description', required=True, unique=True))

db.define_table(
    'race_subclass',
    Field('description', required=True, unique=True))

db.define_table(
    'race_day',
    Field('date', 'date', required=True),
    Field('conditions'),
    Field('temperature_f', 'double'))

db.define_table(
    'race_day_official',
    Field('race_day_id', 'reference race_day'),
    Field('official_id', 'reference official'))

db.define_table(
    'race',
    Field('date', 'date', required=True),
    Field('class_id', 'reference race_class', required=True,
          requires=IS_IN_DB(db, db.race_class.id, '%(description)s')),
    Field('laps', 'integer'),
    Field('slow_lap_secs', 'double'),
    Field('fast_lap_secs', 'double'),
    Field('average_lap_secs', 'double'))

db.define_table(
    'team',
    Field('name', required=True, unique=True))
db.team.id.readable = False

db.define_table(
    'athlete_team',
    Field('athlete_id', 'reference athlete'),
    Field('team_id', 'reference team'),
    Field('season_id', 'reference season',
          requires=IS_IN_DB(db, db.season.id, '%(year)d')))

db.define_table(
    'participant',
    Field('athlete_id', 'reference athlete', required=True),
    Field('team_id', 'reference team'),
    Field('race_id', 'reference race', required=True))

db.define_table(
    'race_marshal',
    Field('race_id', 'reference race', required=True),
    Field('marshal_id', 'reference marshal', required=True))

db.define_table(
    'result',
    Field('participant_id', 'reference participant', required=True),
    Field('place', 'integer'),
    Field('points', 'double'),
    Field('mar_place', 'integer'),
    Field('point_prime', 'boolean'))

db.define_table(
    'subclass_result',
    Field('participant_id', 'reference participant', required=True),
    Field('subclass_id', 'reference race_subclass', required=True),
    Field('place', 'integer'),
    Field('points', 'double'))

db.define_table(
    'prime',
    Field('participant_id', 'reference participant'),
    Field('description'))

# Hard-coded tables.
def _():
    def update_table(table, table_data):
        cur_table_data = db().select(table.ALL)
        cur_keys = set()
        for row in cur_table_data:
            cur_keys.add(row.id)
            non_id_vals = {k: v for k, v in row.items() if k != 'id'}
            if row.id in table_data:
                table[row.id] = table_data[row.id]
            else:
                del table[row.id]
        for key, val in table_data.items():
            if not key in cur_keys:
                table.insert(**val)

    CURRENT_SEASON = 2015
    SEASONS_TABLE = {
        i+1: dict(year=year) for i, year in
        enumerate(xrange(1996, CURRENT_SEASON+1))}
    update_table(db.season, SEASONS_TABLE)

    CLASSES = ['A', 'B', 'C', 'Masters 40+/Women', 'Juniors']
    CLASSES_TABLE = {
        i+1: dict(description=description)
        for i, description in enumerate(CLASSES)}
    update_table(db.race_class, CLASSES_TABLE)

    SUBCLASSES = ['Women', 'Juniors']
    SUBCLASSES_TABLE = {
        i+1:dict(description=description)
        for i, description in enumerate(SUBCLASSES)}
    update_table(db.race_subclass, SUBCLASSES_TABLE)

    db.commit()

_()
<|endoftext|>


<commit_before># Copyright 2011-2012 10gen, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Utilities for using Motor with Tornado web applications."""

import datetime
import email.utils
import mimetypes
import time

import tornado.web
from tornado import gen

import gridfs
import motor

# TODO: this class is not a drop-in replacement for StaticFileHandler.
#   StaticFileHandler provides class method make_static_url, which appends
#   an MD5 of the static file's contents. Templates thus can do
#   {{ static_url('image.png') }} and get "/static/image.png?v=1234abcdef",
#   which is cached forever. Problem is, it calculates the MD5 synchronously.
#   Two options: keep a synchronous GridFS available to get each grid file's
#   MD5 synchronously for every static_url call, or find some other idiom.

class GridFSHandler(tornado.web.RequestHandler):
    """A handler that can serve content from `GridFS`_, very similar to
    `tornado.web.StaticFileHandler`.

    .. code-block:: python

        db = motor.MotorClient().open_sync().my_database
        application = web.Application([
            (r"/static/(.*)", web.GridFSHandler, {"database": db}),
        ])

    By default, requests' If-Modified-Since headers are honored, but no
    specific cache-control timeout is sent to clients. Thus each request for
    a GridFS file requires a quick check of the file's ``uploadDate`` in
    MongoDB. Override :meth:`get_cache_time` in a subclass to customize this.

    .. seealso:: MongoDB and `GridFS
       <http://docs.mongodb.org/manual/applications/gridfs/>`_

    .. seealso:: `StaticFileHandler
       <http://www.tornadoweb.org/documentation/web.html#tornado.web.StaticFileHandler>`_
    """
    def initialize(self, database, root_collection='fs'):
        self.database = database
        self.root_collection = root_collection

    def get_gridfs_file(self, fs, path):
        """Overridable method to choose a GridFS file to serve at a URL.

        By default, if a URL pattern like ``"/static/(.*)"`` is mapped to this
        ``GridFSHandler``, then the trailing portion of the URL is used as the
        filename, so a request for "/static/image.png" results in a call
        to ``get_gridfs_file`` with "image.png" as the ``path`` argument. To
        customize the mapping of path to GridFS file, override
        ``get_gridfs_file`` and return a Future :class:`~motor.MotorGridOut`
        from it.

        For example, to retrieve the file by ``_id`` instead of filename::

            class CustomGridFSHandler(motor.web.GridFSHandler):
                def get_gridfs_file(self, fs, path):
                    # Path is interpreted as _id instead of name.
                    # Return a Future MotorGridOut.
                    return fs.get(file_id=path)

        :Parameters:
          - `fs`: An open :class:`~motor.MotorGridFS` object
          - `path`: A string, the trailing portion of the URL pattern being
            served

        .. versionchanged:: 0.2
           ``get_gridfs_file`` no longer accepts a callback, instead returns
           a Future.
        """
        return fs.get_last_version(path)  # A Future MotorGridOut

    @tornado.web.asynchronous
    @gen.coroutine
    def get(self, path, include_body=True):
        fs = yield motor.MotorGridFS(self.database, self.root_collection).open()

        try:
            gridout = yield self.get_gridfs_file(fs, path)
        except gridfs.NoFile:
            raise tornado.web.HTTPError(404)

        # If-Modified-Since header is only good to the second.
        modified = gridout.upload_date.replace(microsecond=0)
        self.set_header("Last-Modified", modified)

        # MD5 is calculated on the MongoDB server when GridFS file is created
        self.set_header("Etag", '"%s"' % gridout.md5)

        mime_type = gridout.content_type

        # If content type is not defined, try to check it with mimetypes
        if mime_type is None:
            mime_type, encoding = mimetypes.guess_type(path)

        # Starting from here, largely a copy of StaticFileHandler
        if mime_type:
            self.set_header("Content-Type", mime_type)

        cache_time = self.get_cache_time(path, modified, mime_type)

        if cache_time > 0:
            self.set_header("Expires", datetime.datetime.utcnow() +
                                       datetime.timedelta(seconds=cache_time))
            self.set_header("Cache-Control", "max-age=" + str(cache_time))
        else:
            self.set_header("Cache-Control", "public")

        self.set_extra_headers(path, gridout)

        # Check the If-Modified-Since, and don't send the result if the
        # content has not been modified
        ims_value = self.request.headers.get("If-Modified-Since")
        if ims_value is not None:
            date_tuple = email.utils.parsedate(ims_value)
            if_since = datetime.datetime.fromtimestamp(time.mktime(date_tuple))
            if if_since >= modified:
                self.set_status(304)
                return

        # Same for Etag
        etag = self.request.headers.get("If-None-Match")
        if etag is not None and etag.strip('"') == gridout.md5:
            self.set_status(304)
            return

        self.set_header("Content-Length", gridout.length)
        if include_body:
            yield gridout.stream_to_handler(self)

        # Needed until fix for Tornado bug 751 is released, see
        # https://github.com/facebook/tornado/issues/751 and
        # https://github.com/facebook/tornado/commit/5491685
        self.finish()

    def head(self, path):
        self.get(path, include_body=False)

    def get_cache_time(self, path, modified, mime_type):
        """Override to customize cache control behavior.

        Return a positive number of seconds to trigger aggressive caching or 0
        to mark resource as cacheable, only. 0 is the default.
        """
        return 0

    def set_extra_headers(self, path, gridout):
        """For subclass to add extra headers to the response"""
        pass
<commit_msg>Improve GridFSHandler docs a little more<commit_after># Copyright 2011-2012 10gen, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Utilities for using Motor with Tornado web applications."""

import datetime
import email.utils
import mimetypes
import time

import tornado.web
from tornado import gen

import gridfs
import motor

# TODO: this class is not a drop-in replacement for StaticFileHandler.
#   StaticFileHandler provides class method make_static_url, which appends
#   an MD5 of the static file's contents. Templates thus can do
#   {{ static_url('image.png') }} and get "/static/image.png?v=1234abcdef",
#   which is cached forever. Problem is, it calculates the MD5 synchronously.
#   Two options: keep a synchronous GridFS available to get each grid file's
#   MD5 synchronously for every static_url call, or find some other idiom.

class GridFSHandler(tornado.web.RequestHandler):
    """A handler that can serve content from `GridFS`_, very similar to
    `tornado.web.StaticFileHandler`.

    .. code-block:: python

        db = motor.MotorClient().open_sync().my_database
        application = web.Application([
            (r"/static/(.*)", web.GridFSHandler, {"database": db}),
        ])

    By default, requests' If-Modified-Since headers are honored, but no
    specific cache-control timeout is sent to clients. Thus each request for
    a GridFS file requires a quick check of the file's ``uploadDate`` in
    MongoDB. Override :meth:`get_cache_time` in a subclass to customize this.

    .. seealso:: MongoDB and `GridFS
       <http://docs.mongodb.org/manual/applications/gridfs/>`_

    .. seealso:: `StaticFileHandler
       <http://www.tornadoweb.org/documentation/web.html#tornado.web.StaticFileHandler>`_
    """
    def initialize(self, database, root_collection='fs'):
        self.database = database
        self.root_collection = root_collection

    def get_gridfs_file(self, fs, path):
        """Overridable method to choose a GridFS file to serve at a URL.

        By default, if a URL pattern like ``"/static/(.*)"`` is mapped to this
        ``GridFSHandler``, then the trailing portion of the URL is used as the
        filename, so a request for "/static/image.png" results in a call
        to ``get_gridfs_file`` with "image.png" as the ``path`` argument. To
        customize the mapping of path to GridFS file, override
        ``get_gridfs_file`` and return a Future :class:`~motor.MotorGridOut`
        from it.

        For example, to retrieve the file by ``_id`` instead of filename::

            class CustomGridFSHandler(motor.web.GridFSHandler):
                def get_gridfs_file(self, fs, path):
                    # Path is interpreted as _id instead of name.
                    # Return a Future MotorGridOut.
                    return fs.get(file_id=ObjectId(path))

        :Parameters:
          - `fs`: An open :class:`~motor.MotorGridFS` object
          - `path`: A string, the trailing portion of the URL pattern being
            served

        .. versionchanged:: 0.2
           ``get_gridfs_file`` no longer accepts a callback, instead returns
           a Future.
        """
        return fs.get_last_version(path)  # A Future MotorGridOut

    @tornado.web.asynchronous
    @gen.coroutine
    def get(self, path, include_body=True):
        fs = yield motor.MotorGridFS(self.database, self.root_collection).open()

        try:
            gridout = yield self.get_gridfs_file(fs, path)
        except gridfs.NoFile:
            raise tornado.web.HTTPError(404)

        # If-Modified-Since header is only good to the second.
        modified = gridout.upload_date.replace(microsecond=0)
        self.set_header("Last-Modified", modified)

        # MD5 is calculated on the MongoDB server when GridFS file is created
        self.set_header("Etag", '"%s"' % gridout.md5)

        mime_type = gridout.content_type

        # If content type is not defined, try to check it with mimetypes
        if mime_type is None:
            mime_type, encoding = mimetypes.guess_type(path)

        # Starting from here, largely a copy of StaticFileHandler
        if mime_type:
            self.set_header("Content-Type", mime_type)

        cache_time = self.get_cache_time(path, modified, mime_type)

        if cache_time > 0:
            self.set_header("Expires", datetime.datetime.utcnow() +
                                       datetime.timedelta(seconds=cache_time))
            self.set_header("Cache-Control", "max-age=" + str(cache_time))
        else:
            self.set_header("Cache-Control", "public")

        self.set_extra_headers(path, gridout)

        # Check the If-Modified-Since, and don't send the result if the
        # content has not been modified
        ims_value = self.request.headers.get("If-Modified-Since")
        if ims_value is not None:
            date_tuple = email.utils.parsedate(ims_value)
            if_since = datetime.datetime.fromtimestamp(time.mktime(date_tuple))
            if if_since >= modified:
                self.set_status(304)
                return

        # Same for Etag
        etag = self.request.headers.get("If-None-Match")
        if etag is not None and etag.strip('"') == gridout.md5:
            self.set_status(304)
            return

        self.set_header("Content-Length", gridout.length)
        if include_body:
            yield gridout.stream_to_handler(self)

        # Needed until fix for Tornado bug 751 is released, see
        # https://github.com/facebook/tornado/issues/751 and
        # https://github.com/facebook/tornado/commit/5491685
        self.finish()

    def head(self, path):
        self.get(path, include_body=False)

    def get_cache_time(self, path, modified, mime_type):
        """Override to customize cache control behavior.

        Return a positive number of seconds to trigger aggressive caching or 0
        to mark resource as cacheable, only. 0 is the default.
        """
        return 0

    def set_extra_headers(self, path, gridout):
        """For subclass to add extra headers to the response"""
        pass
<|endoftext|>


<commit_before>import shutil

thirdPartys = ['PlayableAds','AdColony','AdMob','Applovin','Baidu','Centrixlink','ChartBoost','Domob','Facebook','GDT','InMobi','IronSource','Mobvista','StartApp','Unity','Vungle']

for thirdParty in thirdPartys:
	print('is moving %s' % thirdParty)
	if thirdParty == 'PlayableAds':
		shutil.move("./Pods/%s/" % thirdParty, "./Pods/YumiMediationAdapters/YumiMediationPlayableAds/")
		continue
	thirdAdapter = 'YumiMediation' + thirdParty
	thirdParty = 'Yumi' + thirdParty
	shutil.move("./Pods/%s/" % thirdParty, "./Pods/YumiMediationAdapters/%s/" % thirdAdapter)
<commit_msg>update moveFiles.py<commit_after>import shutil
import os

sdkVersion = 'YumiMediationSDK_v'+'3.2.2'
thirdPartys = ['PlayableAds','AdColony','AdMob','Applovin','Baidu','Centrixlink','ChartBoost','Domob','Facebook','GDT','InMobi','IronSource','Mobvista','StartApp','Unity','Vungle']

for thirdParty in thirdPartys:
	print('is moving %s' % thirdParty)
	if thirdParty == 'PlayableAds':
		shutil.move("./Pods/%s/" % thirdParty, "./Pods/YumiMediationAdapters/YumiMediationPlayableAds/")
		shutil.make_archive('./Pods/YumiMediationAdapters/YumiMediationPlayableAds','zip','./Pods/YumiMediationAdapters/YumiMediationPlayableAds')
		shutil.rmtree('./Pods/YumiMediationAdapters/YumiMediationPlayableAds')
		continue
	thirdAdapter = 'YumiMediation' + thirdParty
	thirdParty = 'Yumi' + thirdParty
	shutil.move("./Pods/%s/" % thirdParty, "./Pods/YumiMediationAdapters/%s/" % thirdAdapter)
	shutil.make_archive("./Pods/YumiMediationAdapters/%s/" % thirdAdapter,'zip',"./Pods/YumiMediationAdapters/%s/" % thirdAdapter)
	shutil.rmtree('./Pods/YumiMediationAdapters/%s/' % thirdAdapter)

os.mkdir('./Pods/%s' % sdkVersion)
os.rename('./Pods/YumiMediationAdapters', './Pods/YumiMediationThirdPartys')

shutil.make_archive('./Pods/YumiMediationThirdPartys', 'zip','./Pods/YumiMediationThirdPartys')
shutil.rmtree('./Pods/YumiMediationThirdPartys')
shutil.move('./Pods/YumiMediationThirdPartys.zip', './Pods/%s' % sdkVersion)

shutil.move('./Pods/YumiMediationDebugCenter-iOS/YumiMediationDebugCenter-iOS.framework', './Pods/YumiMediationSDK/YumiMediationDebugCenter-iOS.framework')
shutil.make_archive('./Pods/YumiMediationSDK', 'zip','./Pods/YumiMediationSDK')
shutil.rmtree('./Pods/YumiMediationSDK')
shutil.rmtree('./Pods/YumiMediationDebugCenter-iOS')
shutil.move('./Pods/YumiMediationSDK.zip', './Pods/%s' % sdkVersion)

shutil.copy('./YumiMediationSDKConfig.xcconfig', './Pods/%s' % sdkVersion)

shutil.make_archive('./Pods/%s' % sdkVersion, 'zip','./Pods/%s' % sdkVersion)
shutil.rmtree('./Pods/%s' % sdkVersion)

<|endoftext|>


<commit_before>## Jordan Thayer  2016-03-21T14:21:37-04:00
## A stump simple IRC client for managing servers on a digital ocean instance

import irc.client
import irc.modes
from irc.dict import IRCDict
from irc.bot import SingleServerIRCBot, ServerSpec
from configuration import Configuration

# Basic commands
from command import help, running_droplet
# User management commands
from command import op_user, unop_user, add_user, rem_user
# Droplet management commands
from command import add_api_key, stop_droplet, list_images,
                    load_most_recent_image, load_named_image

def serverOfConfig(config):
    return ServerSpec(config.settings[Configuration.IRC_SERVER],
                      config.settings[Configuration.IRC_PASSWD])

class MrDo(SingleServerIRCBot):

    def __init__(self, config):
        SingleServerIRCBot.__init__(self,
                                    [serverOfConfig(config)],
                                    config[Configuration.IRC_UNAME],
                                    config[Configuration.IRC_UNAME])
        self.config = config
        self.channel = config.settings[Configuration.IRC_CHAN]

    def on_welcome(self,c,e):
        c.join(self.channel)

    def on_dccchat(self,c,e):
        print c, e

    def on_privmsg(self,c,e):
        print "got private message", e.arguments[0]

    def on_pubmsg(self,c,e):
        a = e.arguments[0].split(':', 1)
        if len(a) > 1 and irc.strings.lower(a[0]) == irc.strings.lower(self.config.settings[Configuration.IRC_UNAME]):
            print "got command", a[1].strip()
<commit_msg>Stubbing out command handling<commit_after>## Jordan Thayer  2016-03-21T14:21:37-04:00
## A stump simple IRC client for managing servers on a digital ocean instance

import irc.client
import irc.modes
from irc.dict import IRCDict
from irc.bot import SingleServerIRCBot, ServerSpec
from configuration import Configuration

# Basic commands
from command import help, running_droplet
# User management commands
from command import op_user, unop_user, add_user, rem_user
# Droplet management commands
from command import add_api_key, stop_droplet, list_images,
                    load_most_recent_image, load_named_image

def serverOfConfig(config):
    return ServerSpec(config.settings[Configuration.IRC_SERVER],
                      config.settings[Configuration.IRC_PASSWD])

class MrDo(SingleServerIRCBot):

    def __init__(self, config):
        SingleServerIRCBot.__init__(self,
                                    [serverOfConfig(config)],
                                    config[Configuration.IRC_UNAME],
                                    config[Configuration.IRC_UNAME])
        self.config = config
        self.channel = config.settings[Configuration.IRC_CHAN]

    def on_welcome(self,c,e):
        c.join(self.channel)

    def on_dccchat(self,c,e):
        print c, e

    def on_privmsg(self,c,e):
        print "got private message", e.arguments[0]

    def _respond(self, chan, message):
        pass

    def _handle_msg(self, response_chan, user, cmd):
        """
        Dispatches the appropriate command
        """
        if cmd == []:
            return
        cmd_name = cmd[0]
        if cmd_name == help.keyword:
            pass
        elif cmd_name == running_droplet.keyword:
            pass
        elif cmd_name == op_user.keyword:
            pass
        elif cmd_name == unop_user.keyword:
            pass
        elif cmd_name == add_user.keyword:
            pass
        elif cmd_name == rem_user.keyword:
            pass
        elif cmd_name == add_api_key.keyword:
            pass
        elif cmd_name == stop_droplet.keyword:
            pass
        elif cmd_name == list_images.keyword:
            pass
        elif cmd_name == load_most_recent_image.keyword:
            pass
        elif cmd_name == load_named_image.keyword:
            pass
        else: ## No recognized keyword
            self._help(response_chan, user, [])

    def _help(self, response_chan, user, args):
        if help.canIssue(self.config, user):
            if args:
                for arg in args:
                    ## look for arg as keyword in list of commands
                    ## print its long desc on the response chan if you find it
                    ## otherwise, respond with "didn't find command arg
                    pass
            else:
                self._respond(response_chan, help.shortDesc)

    def on_pubmsg(self,c,e):
        a = e.arguments[0].split(':', 1)
        if len(a) > 1 and irc.strings.lower(a[0]) == irc.strings.lower(self.config.settings[Configuration.IRC_UNAME]):
            print "got command", a[1].strip()
<|endoftext|>