<commit_before>"""
API described here: http://apfmon.lancs.ac.uk/help
"""

import requests
import json
import time
import traceback

from pandaharvester.harvesterconfig import harvester_config
from pandaharvester.harvestercore import core_utils
from pandaharvester import panda_pkg_info
from pandaharvester.harvestermisc import generic_utils
from pandaharvester.harvestercore.work_spec import WorkSpec
from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper

class Apfmon:

    def __init__(self):

        self._base_logger = core_utils.setup_logger('apfmon')

        try:
            self.__active = harvester_config.apfmon.active
        except:
            self.__active = True # TODO: decide if default is active or not

        # TODO: make proper exception handling and defaults
        try:
            self.harvester_id = harvester_config.master.harvester_id
        except:
            self.harvester_id = 'DUMMY'

        try:
            self.base_url = harvester_config.apfmon.base_url
        except:
            self.base_url = 'http://apfmon.lancs.ac.uk/api'

        self.queue_config_mapper = QueueConfigMapper()

    def __apfmon_active(cls, method, *args, **kwargs):
        if cls.__active:
            method(*args, **kwargs)
        else:
            return

    def create_factory(self):
        """
        Creates or updates a harvester instance to APF Mon. Should be done at startup of the instance.
        """
        start_time = time.time()
        tmp_log = core_utils.make_logger(self._base_logger, 'harvester_id={0}'.format(self.harvester_id),
                                         method_name='create_factory')
        try:
            tmp_log.debug('start')

            url = '{0}/factories/{1}'.format(self.base_url, self.harvester_id)

            f = {'url': 'url_to_logs', # TODO: get the URL properly. This can be problematic when harvester runs on two nodes and one DB
                 'email': 'atlas-adc-harvester-central-support@cern.ch',
                 'version': panda_pkg_info.release_version}
            payload = json.dumps(f)

            r = requests.put(url, data=payload)
            tmp_log.debug('registration ended with {0}'.format(r.status_code))
            end_time = time.time()
            tmp_log.debug('done (took {0})'.format(end_time - start_time))
        except:
            tmp_log.error('Excepted with: {0}'.format(traceback.format_exc()))

    def create_labels(self):
        """
        Creates or updates a collection of labels (=panda queues)
        """
        start_time = time.time()
        tmp_log = core_utils.make_logger(self._base_logger, 'harvester_id={0}'.format(self.harvester_id),
                                         method_name='create_labels')
        try:
            tmp_log.debug('start')

            url = '{0}/labels'.format(self.base_url)

            # get the active queues from the config mapper
            all_sites = self.queue_config_mapper.get_active_queues().keys()

            # publish the active queues to APF mon in shards
            for sites in generic_utils.create_shards(all_sites, 20):
                labels = []
                for site in sites:
                    labels.append({'name': site, 'factory': self.harvester_id})

                payload = json.dumps(labels)

                r = requests.put(url, data=payload)
                tmp_log.debug('label creation for {0} ended with {1}'.format(sites, r.status_code))

            end_time = time.time()
            tmp_log.debug('done (took {0})'.format(end_time - start_time))
        except:
            tmp_log.error('Excepted with: {0}'.format(traceback.format_exc()))

    def create_workers(self, worker_spec_list):
        """
        Creates a worker. The updates (state transitions) are expected to come from the pilot wrappers
        """
        start_time = time.time()
        tmp_log = core_utils.make_logger(self._base_logger, 'harvester_id={0}'.format(self.harvester_id),
                                         method_name='create_workers')
        try:
            tmp_log.debug('start')

            url = '{0}/jobs'.format(self.base_url)

            for worker_spec_shard in generic_utils.create_shards(worker_spec_list, 20):
                workers = []
                for worker_spec in worker_spec_shard:
                    # TODO: be sure it's expecting the condor ID and it's consistent with the wrapper info
                    batch_id = worker_spec.batchID
                    factory = self.harvester_id
                    computingsite = worker_spec.computingSite

                    # extract the log URLs
                    work_attribs = json.loads(worker_spec.workAttributes)
                    stdout_url = work_attribs['stdOut']
                    stderr_url = work_attribs['stdErr']
                    log_url = work_attribs['batchLog']

                    jdl_url = '' # TODO: publish the jdl files

                    apfmon_worker = {'cid': batch_id,
                                     'factory': factory,
                                     'label': computingsite,
                                     'jdlurl': jdl_url,
                                     'stdouturl': stdout_url,
                                     'stderrurl': stderr_url,
                                     'logurl': log_url
                                     }
                    tmp_log.debug('packed worker: {0}'.format(apfmon_worker))
                    workers.append()

                payload = json.dumps(workers)
                r = requests.put(url, data=payload)
                tmp_log.debug('worker creation for {0} ended with {1}'.format(workers, r.status_code))

            end_time = time.time()
            tmp_log.debug('done (took {0})'.format(end_time - start_time))
        except:
            tmp_log.error('Excepted with: {0}'.format(traceback.format_exc()))

if __name__== "__main__":

    """
    Quick tests
    """

    apfmon = Apfmon()
    apfmon.create_factory()
    apfmon.create_labels()

    worker_a = WorkSpec()
    worker_a.batchID = 1
    worker_a.computingsite = 'CERN-PROD-DEV_UCORE'
    worker_a.workAttributes = '{"batchLog": "https://aipanda024.cern.ch/condor_logs/18-07-19_09/grid.9659.0.log", "stdErr": "https://aipanda024.cern.ch/condor_logs/18-07-19_09/grid.9659.0.err", "stdOut": "https://aipanda024.cern.ch/condor_logs/18-07-19_09/grid.9659.0.out"}'
    worker_a.workAttributes = {}

    worker_b = WorkSpec()
    worker_a.batchID = 2
    worker_a.computingsite = 'CERN-PROD-DEV_UCORE'
    worker_a.workAttributes = '{"batchLog": "https://aipanda024.cern.ch/condor_logs/18-07-19_09/grid.9659.0.log", "stdErr": "https://aipanda024.cern.ch/condor_logs/18-07-19_09/grid.9659.0.err", "stdOut": "https://aipanda024.cern.ch/condor_logs/18-07-19_09/grid.9659.0.out"}'

    workers = [worker_a, worker_b]

    apfmon.create_workers(workers)

<commit_msg>APFMon: typo<commit_after>"""
API described here: http://apfmon.lancs.ac.uk/help
"""

import requests
import json
import time
import traceback

from pandaharvester.harvesterconfig import harvester_config
from pandaharvester.harvestercore import core_utils
from pandaharvester import panda_pkg_info
from pandaharvester.harvestermisc import generic_utils
from pandaharvester.harvestercore.work_spec import WorkSpec
from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper

class Apfmon:

    def __init__(self):

        self._base_logger = core_utils.setup_logger('apfmon')

        try:
            self.__active = harvester_config.apfmon.active
        except:
            self.__active = True # TODO: decide if default is active or not

        # TODO: make proper exception handling and defaults
        try:
            self.harvester_id = harvester_config.master.harvester_id
        except:
            self.harvester_id = 'DUMMY'

        try:
            self.base_url = harvester_config.apfmon.base_url
        except:
            self.base_url = 'http://apfmon.lancs.ac.uk/api'

        self.queue_config_mapper = QueueConfigMapper()

    def __apfmon_active(cls, method, *args, **kwargs):
        if cls.__active:
            method(*args, **kwargs)
        else:
            return

    def create_factory(self):
        """
        Creates or updates a harvester instance to APF Mon. Should be done at startup of the instance.
        """
        start_time = time.time()
        tmp_log = core_utils.make_logger(self._base_logger, 'harvester_id={0}'.format(self.harvester_id),
                                         method_name='create_factory')
        try:
            tmp_log.debug('start')

            url = '{0}/factories/{1}'.format(self.base_url, self.harvester_id)

            f = {'url': 'url_to_logs', # TODO: get the URL properly. This can be problematic when harvester runs on two nodes and one DB
                 'email': 'atlas-adc-harvester-central-support@cern.ch',
                 'version': panda_pkg_info.release_version}
            payload = json.dumps(f)

            r = requests.put(url, data=payload)
            tmp_log.debug('registration ended with {0}'.format(r.status_code))
            end_time = time.time()
            tmp_log.debug('done (took {0})'.format(end_time - start_time))
        except:
            tmp_log.error('Excepted with: {0}'.format(traceback.format_exc()))

    def create_labels(self):
        """
        Creates or updates a collection of labels (=panda queues)
        """
        start_time = time.time()
        tmp_log = core_utils.make_logger(self._base_logger, 'harvester_id={0}'.format(self.harvester_id),
                                         method_name='create_labels')
        try:
            tmp_log.debug('start')

            url = '{0}/labels'.format(self.base_url)

            # get the active queues from the config mapper
            all_sites = self.queue_config_mapper.get_active_queues().keys()

            # publish the active queues to APF mon in shards
            for sites in generic_utils.create_shards(all_sites, 20):
                labels = []
                for site in sites:
                    labels.append({'name': site, 'factory': self.harvester_id})

                payload = json.dumps(labels)

                r = requests.put(url, data=payload)
                tmp_log.debug('label creation for {0} ended with {1}'.format(sites, r.status_code))

            end_time = time.time()
            tmp_log.debug('done (took {0})'.format(end_time - start_time))
        except:
            tmp_log.error('Excepted with: {0}'.format(traceback.format_exc()))

    def create_workers(self, worker_spec_list):
        """
        Creates a worker. The updates (state transitions) are expected to come from the pilot wrappers
        """
        start_time = time.time()
        tmp_log = core_utils.make_logger(self._base_logger, 'harvester_id={0}'.format(self.harvester_id),
                                         method_name='create_workers')
        try:
            tmp_log.debug('start')

            url = '{0}/jobs'.format(self.base_url)

            for worker_spec_shard in generic_utils.create_shards(worker_spec_list, 20):
                workers = []
                for worker_spec in worker_spec_shard:
                    # TODO: be sure it's expecting the condor ID and it's consistent with the wrapper info
                    batch_id = worker_spec.batchID
                    factory = self.harvester_id
                    computingsite = worker_spec.computingSite

                    # extract the log URLs
                    work_attribs = json.loads(worker_spec.workAttributes)
                    stdout_url = work_attribs['stdOut']
                    stderr_url = work_attribs['stdErr']
                    log_url = work_attribs['batchLog']

                    jdl_url = '' # TODO: publish the jdl files

                    apfmon_worker = {'cid': batch_id,
                                     'factory': factory,
                                     'label': computingsite,
                                     'jdlurl': jdl_url,
                                     'stdouturl': stdout_url,
                                     'stderrurl': stderr_url,
                                     'logurl': log_url
                                     }
                    tmp_log.debug('packed worker: {0}'.format(apfmon_worker))
                    workers.append()

                payload = json.dumps(workers)
                r = requests.put(url, data=payload)
                tmp_log.debug('worker creation for {0} ended with {1}'.format(workers, r.status_code))

            end_time = time.time()
            tmp_log.debug('done (took {0})'.format(end_time - start_time))
        except:
            tmp_log.error('Excepted with: {0}'.format(traceback.format_exc()))

if __name__== "__main__":

    """
    Quick tests
    """

    apfmon = Apfmon()
    apfmon.create_factory()
    apfmon.create_labels()

    worker_a = WorkSpec()
    worker_a.batchID = 1
    worker_a.computingSite = 'CERN-PROD-DEV_UCORE'
    worker_a.workAttributes = '{"batchLog": "https://aipanda024.cern.ch/condor_logs/18-07-19_09/grid.9659.0.log", "stdErr": "https://aipanda024.cern.ch/condor_logs/18-07-19_09/grid.9659.0.err", "stdOut": "https://aipanda024.cern.ch/condor_logs/18-07-19_09/grid.9659.0.out"}'
    worker_a.workAttributes = {}

    worker_b = WorkSpec()
    worker_a.batchID = 2
    worker_a.computingSite = 'CERN-PROD-DEV_UCORE'
    worker_a.workAttributes = '{"batchLog": "https://aipanda024.cern.ch/condor_logs/18-07-19_09/grid.9659.0.log", "stdErr": "https://aipanda024.cern.ch/condor_logs/18-07-19_09/grid.9659.0.err", "stdOut": "https://aipanda024.cern.ch/condor_logs/18-07-19_09/grid.9659.0.out"}'

    workers = [worker_a, worker_b]

    apfmon.create_workers(workers)

<|endoftext|>


<commit_before># coding=UTF8
__author__ = 'daedaluschan'

# importing libraries

import sys, time
import telepot
from telepot.delegate import per_chat_id, create_open

from datetime import date
from types import *
from enum import Enum
import uuid

from types import  *

# Constants & global data

allPoll = {}
botName = 'HKGathering_bot'
start_private_url = 'https://telegram.me/' + botName + '?start='
start_group_url = 'https://telegram.me/' + botName + '?startgroup='

# Defining enums

class ConverType(Enum):
    nothing = 1
    create_poll = 2
    response_poll = 3

class CreatePollFlow(Enum):
    not_start = 1
    poll_question = 2
    poll_choice = 3

# Response data structure

class Response():
    def __init__(self):
        self.userid = 0
        self.display_name = ''
        self.preference = []

    def __str__(self):
        return '[response_obj][' + self.userid.__str__() + '][' + self.display_name + '][' + \
               len(self.preference).__str__() + ']'

    @property
    def userid(self):
        return self._userid

    @userid.setter
    def userid(self, value):
        self._userid = value

    @property
    def preference(self):
        return self._preference

    @preference.setter
    def preference(self, value):
        self._preference = value

    @property
    def display_name(self):
        return self._display_name

    @display_name.setter
    def display_name(self, value):
        self._display_name = value

# Poll data structure

class Poll():
    def __init__(self):
        self.question = ''
        self.creatorId = 0
        self.choices = []
        self.groupId = 0
        self.response = {}

    def __str__(self):
        choice_str = ''
        response_str = ''
        for each_choice in self.choices:
            choice_str = choice_str + ' - ' + each_choice
        for each_response in self.response:
            response_str = response_str + '<' + self.response[each_response].__str__() + '>'
        return '[Poll_obj][question: ' + self.question + '][choice: ' + choice_str + '][group: ' + \
               self.groupId.__str__() + '][all_resp: ' + response_str + ']'

    @property
    def question(self):
        return self._question

    @question.setter
    def question(self, value):
        self._question = value

    @property
    def creatorId(self):
        return self._creatorId

    @creatorId.setter
    def creatorId(self, value):
        self._creatorId = value

    @property
    def choices(self):
        return self._choices

    @choices.setter
    def choices(self, value):
        self._choices = value

    @property
    def groupId(self):
        return self._groupId

    @groupId.setter
    def groupId(self, value):
        self._groupId = value

    def gen_survey_str(self, response_attached=[]):
        choice_str = ''
        i = 1
        print(self.choices)
        for choice in self.choices:
            if len(response_attached) == 0:
                choice_str = choice_str + '- ' + choice.encode('utf-8') + '\n'
            else:
                choice_str = choice_str + '- /' + i.__str__() + ' ' + choice.encode('utf-8') + \
                ' -- 現在回應：'
                if response_attached[i-1]:
                    choice_str = choice_str + 'Yes\n'
                else:
                    choice_str = choice_str + 'No\n'
            i = i + 1

        survey_text = '依家問你：' + self.question.encode('utf-8') + '\n\n現有選擇係：\n\n' + choice_str

        return survey_text

    @property
    def response(self):
        return self._response

    @response.setter
    def response(self, value):
        self._response = value

    def genNullResponse(self):
        preference_vector = []
        for each_choice in self.choices:
            preference_vector.append(False)
        return preference_vector

    def genResponseKeyboard(self, current_response=genNullResponse()):
        show_keyboard = {'keyboard': [[]]}
        i=0
        for choice in self.choices:
            if current_response[i]:
                show_keyboard['keyboard'][0].append('反對: /' + (i+1).__str__() + ' - ' + choice)
            else:
                show_keyboard['keyboard'][0].append('撐: /' + (i+1).__str__() + ' - ' + choice)
            i = i + 1

        show_keyboard['keyboard'][0].append('加入新選項')
        show_keyboard['keyboard'][0].append('完')
        return  show_keyboard

# main class of message handling

class HKGathering(telepot.helper.ChatHandler):
    def __init__(self, seed_tuple, timeout):
        super(HKGathering, self).__init__(seed_tuple, timeout)
        print('constructor is being called')
        self._converType = ConverType.nothing
        self._createPollFlow = CreatePollFlow.not_start
        self.__uid = ''
        self._poll = Poll()

    def complte_poll_creation(self):
        self.__uid = uuid.uuid4().hex
        allPoll[self.__uid] = self._poll
        self.sender.sendMessage(text='用呢條 link 將問題放落 chat group 裡邊：\n' +
                                     start_group_url + self.__uid)
        self._converType = ConverType.nothing

    def get_invited(self, poll_id):
        self._poll = allPoll[poll_id]
        print('Linked poll: ' + self._poll.__str__())
        choice_str = ''
        for choice in self._poll.choices:
            choice_str = choice_str + choice.encode('utf-8') + '\n'
        ans_link = '/answer' +'_' + poll_id.encode(encoding='utf-8')
        show_keyboard = {'keyboard': [['開始回應']]}

        self.sender.sendMessage(text=self._poll.gen_survey_str() + '\n' +
                                     '請用 ' + ans_link +  ' 回應問題。\n' +
                                     '或者用 pop up 鍵盤開始回應。',
                                reply_markup=show_keyboard)

    def initiate_survey(self, poll_id, target_id, display_name):
        print('answer to: ' + target_id.__str__())
        new_response = Response()
        new_response.userid = target_id
        new_response.display_name = display_name
        new_response.preference = allPoll[poll_id].genNullResponse()
        allPoll[poll_id].response[target_id.__str__()] = new_response
        # allPoll[poll_id].response.append({target_id.__str__(): new_response})

        print ('new_response: ' + new_response.__str__())
        print ('saved resposne: ' + allPoll[poll_id].response[target_id.__str__()].__str__())

        show_keyboard = {'keyboard': [['開始']]}
        self.bot.sendMessage(target_id,
                             text=self._poll.gen_survey_str() + '\n' +
                                  '請用 /begin 或者用 pop up 鍵盤開始。',
                             reply_markup=show_keyboard)

    def start_survey(self, poll_id, userid):
        print('start survey with poll id: ' + poll_id)
        self._poll = allPoll[poll_id]

        show_keyboard = self._poll.genResponseKeyboard(allPoll[poll_id].response[userid.__str__()].preference)
        self.sender.sendMessage(text=self._poll.gen_survey_str(response_attached=allPoll[poll_id].response[userid.__str__()].preference) +
                                     '\n你可以用 / + 〈數字〉更改你對相關選項的回應，用 pop up 鍵盤亦可。\n' +
                                     '或者用 /add_pref 加入新選項。\n\n' +
                                     '當完成時請用 /finish 。',
                                reply_markup=show_keyboard)

    def on_message(self, msg):
        print('on_message() is being called')
        print('==== all poll in cache ====')
        if len(allPoll) == 0:
            print('No poll yet')
        else:
            for each_poll in allPoll:
                print(allPoll[each_poll].__str__())
        print('========== END ============')
        flavor = telepot.flavor(msg)

        print('flavor: ' + flavor)

        # normal message
        if flavor == 'normal':
            content_type, chat_type, _chat_id = telepot.glance2(msg)
            print('Normal Message:', content_type, chat_type, _chat_id, '; message content: ', msg)

            if content_type == 'text' and chat_type == 'private':

                if self._converType == ConverType.nothing:

                    if msg['text'] == '/start':
                        self.sender.sendMessage(text='今日的日期係：' + str(date.today()) + '. \n' +
                                                     '用 /new 黎 create 一個新問題，' +
                                                     '或者用 /help 睇其他選項。')
                    elif msg['text'] == '/new':
                        self._converType = ConverType.create_poll
                        self._createPollFlow = CreatePollFlow.poll_question
                        self._poll.creatorId = msg['from']['id']
                        self.sender.sendMessage(text='唔該 send 你個問題俾我。')

                    elif msg['text'] == '/help':
                        self.sender.sendMessage(text='用 /new 黎 create 一個新問題，\n' +
                                                     '或者用 /result 黎查詢回應統計。\n')

                    elif msg['text'].encode(encoding='utf-8') == '開始' or msg['text'].startswith('/begin'):
                        found_poll = 0
                        for poll_key in allPoll.keys():
                            print ('in_all_poll: ' + allPoll[poll_key].__str__())
                            for each_resp in allPoll[poll_key].response:
                                print('each_resp: ' + allPoll[poll_key].response[each_resp].__str__())
                                if allPoll[poll_key].response[each_resp].userid == msg['from']['id']:
                                    found_poll = poll_key
                        if found_poll != 0:
                            self._converType = ConverType.response_poll
                            self.start_survey(found_poll, msg['from']['id'])
                        else:
                            self.sender.sendMessage(text='搵唔到你的問題。')
                    else:
                        self.sender.sendMessage(text='唔知你想點，麻煩你再試過。\n' +
                                                     '或者用 /help 睇其他選項。')
                elif self._converType == ConverType.create_poll:
                    if msg['text'] == '/done':
                        self.complte_poll_creation()
                    elif msg['text'] != '':
                        if self._createPollFlow == CreatePollFlow.poll_question:
                            self._poll.question = msg['text']
                            self._createPollFlow = CreatePollFlow.poll_choice
                            self.sender.sendMessage(text='好，咁你自己有乜意見？')
                        elif self._createPollFlow == CreatePollFlow.poll_choice:
                            self._poll.choices.append(msg['text'])
                            self.sender.sendMessage(text='好，仲有冇？有就繼續 send 下個個選擇。\n' +
                                                         '如果冇就用 /done 完成建立問題。')

            elif content_type == 'text' and chat_type == 'group':
                if msg['text'].startswith('/start@' + botName):
                    print('invited into group')
                    poll_id = msg['text'].split(' ')[1]
                    self.get_invited(poll_id)
                    allPoll[poll_id].groupId = _chat_id
                elif msg['text'].startswith('/answer_'):
                    poll_id = msg['text'].split('_')[1].split('@')[0]
                    self.initiate_survey(poll_id,
                                         target_id=msg['from']['id'],
                                         display_name=msg['from']['first_name'] + ' ' + msg['from']['last_name'])
                elif msg['text'].encode(encoding='utf-8') == '開始回應':
                    orig_text = msg['reply_to_message']['text']
                    poll_id = orig_text.split('/answer_')[1].split(' ')[0]
                    self.initiate_survey(poll_id,
                                         target_id=msg['from']['id'],
                                         display_name=msg['from']['first_name'] + ' ' + msg['from']['last_name'])

            print('Poll:' + self._poll.__str__())
        else:
            raise telepot.BadFlavor(msg)

TOKEN = sys.argv[1]  # get token from command-line

bot = telepot.DelegatorBot(TOKEN, [
    (per_chat_id(), create_open(HKGathering, timeout=20)), ])
print('Listening ...')
bot.notifyOnMessage(run_forever=True)
<commit_msg>response keyboard<commit_after># coding=UTF8
__author__ = 'daedaluschan'

# importing libraries

import sys, time
import telepot
from telepot.delegate import per_chat_id, create_open

from datetime import date
from types import *
from enum import Enum
import uuid

from types import  *

# Constants & global data

allPoll = {}
botName = 'HKGathering_bot'
start_private_url = 'https://telegram.me/' + botName + '?start='
start_group_url = 'https://telegram.me/' + botName + '?startgroup='

# Defining enums

class ConverType(Enum):
    nothing = 1
    create_poll = 2
    response_poll = 3

class CreatePollFlow(Enum):
    not_start = 1
    poll_question = 2
    poll_choice = 3

# Response data structure

class Response():
    def __init__(self):
        self.userid = 0
        self.display_name = ''
        self.preference = []

    def __str__(self):
        return '[response_obj][' + self.userid.__str__() + '][' + self.display_name + '][' + \
               len(self.preference).__str__() + ']'

    @property
    def userid(self):
        return self._userid

    @userid.setter
    def userid(self, value):
        self._userid = value

    @property
    def preference(self):
        return self._preference

    @preference.setter
    def preference(self, value):
        self._preference = value

    @property
    def display_name(self):
        return self._display_name

    @display_name.setter
    def display_name(self, value):
        self._display_name = value

# Poll data structure

class Poll():
    def __init__(self):
        self.question = ''
        self.creatorId = 0
        self.choices = []
        self.groupId = 0
        self.response = {}

    def __str__(self):
        choice_str = ''
        response_str = ''
        for each_choice in self.choices:
            choice_str = choice_str + ' - ' + each_choice
        for each_response in self.response:
            response_str = response_str + '<' + self.response[each_response].__str__() + '>'
        return '[Poll_obj][question: ' + self.question + '][choice: ' + choice_str + '][group: ' + \
               self.groupId.__str__() + '][all_resp: ' + response_str + ']'

    @property
    def question(self):
        return self._question

    @question.setter
    def question(self, value):
        self._question = value

    @property
    def creatorId(self):
        return self._creatorId

    @creatorId.setter
    def creatorId(self, value):
        self._creatorId = value

    @property
    def choices(self):
        return self._choices

    @choices.setter
    def choices(self, value):
        self._choices = value

    @property
    def groupId(self):
        return self._groupId

    @groupId.setter
    def groupId(self, value):
        self._groupId = value

    def gen_survey_str(self, response_attached=[]):
        choice_str = ''
        i = 1
        print(self.choices)
        for choice in self.choices:
            if len(response_attached) == 0:
                choice_str = choice_str + '- ' + choice.encode('utf-8') + '\n'
            else:
                choice_str = choice_str + '- /' + i.__str__() + ' ' + choice.encode('utf-8') + \
                ' -- 現在回應：'
                if response_attached[i-1]:
                    choice_str = choice_str + 'Yes\n'
                else:
                    choice_str = choice_str + 'No\n'
            i = i + 1

        survey_text = '依家問你：' + self.question.encode('utf-8') + '\n\n現有選擇係：\n\n' + choice_str

        return survey_text

    @property
    def response(self):
        return self._response

    @response.setter
    def response(self, value):
        self._response = value

    def genNullResponse(self):
        preference_vector = []
        for each_choice in self.choices:
            preference_vector.append(False)
        return preference_vector

    def genResponseKeyboard(self, current_response=[]):
        show_keyboard = {'keyboard': [[]]}
        i=0

        if len(current_response) == 0:
            current_response = self.genNullResponse()

        for choice in self.choices:
            if current_response[i]:
                show_keyboard['keyboard'][0].append('反對: /' + (i+1).__str__() + ' - ' + choice)
            else:
                show_keyboard['keyboard'][0].append('撐: /' + (i+1).__str__() + ' - ' + choice)
            i = i + 1

        show_keyboard['keyboard'][0].append('加入新選項')
        show_keyboard['keyboard'][0].append('完')
        return  show_keyboard

# main class of message handling

class HKGathering(telepot.helper.ChatHandler):
    def __init__(self, seed_tuple, timeout):
        super(HKGathering, self).__init__(seed_tuple, timeout)
        print('constructor is being called')
        self._converType = ConverType.nothing
        self._createPollFlow = CreatePollFlow.not_start
        self.__uid = ''
        self._poll = Poll()

    def complte_poll_creation(self):
        self.__uid = uuid.uuid4().hex
        allPoll[self.__uid] = self._poll
        self.sender.sendMessage(text='用呢條 link 將問題放落 chat group 裡邊：\n' +
                                     start_group_url + self.__uid)
        self._converType = ConverType.nothing

    def get_invited(self, poll_id):
        self._poll = allPoll[poll_id]
        print('Linked poll: ' + self._poll.__str__())
        choice_str = ''
        for choice in self._poll.choices:
            choice_str = choice_str + choice.encode('utf-8') + '\n'
        ans_link = '/answer' +'_' + poll_id.encode(encoding='utf-8')
        show_keyboard = {'keyboard': [['開始回應']]}

        self.sender.sendMessage(text=self._poll.gen_survey_str() + '\n' +
                                     '請用 ' + ans_link +  ' 回應問題。\n' +
                                     '或者用 pop up 鍵盤開始回應。',
                                reply_markup=show_keyboard)

    def initiate_survey(self, poll_id, target_id, display_name):
        print('answer to: ' + target_id.__str__())
        new_response = Response()
        new_response.userid = target_id
        new_response.display_name = display_name
        new_response.preference = allPoll[poll_id].genNullResponse()
        allPoll[poll_id].response[target_id.__str__()] = new_response
        # allPoll[poll_id].response.append({target_id.__str__(): new_response})

        print ('new_response: ' + new_response.__str__())
        print ('saved resposne: ' + allPoll[poll_id].response[target_id.__str__()].__str__())

        show_keyboard = {'keyboard': [['開始']]}
        self.bot.sendMessage(target_id,
                             text=self._poll.gen_survey_str() + '\n' +
                                  '請用 /begin 或者用 pop up 鍵盤開始。',
                             reply_markup=show_keyboard)

    def start_survey(self, poll_id, userid):
        print('start survey with poll id: ' + poll_id)
        self._poll = allPoll[poll_id]

        show_keyboard = self._poll.genResponseKeyboard(allPoll[poll_id].response[userid.__str__()].preference)
        self.sender.sendMessage(text=self._poll.gen_survey_str(response_attached=allPoll[poll_id].response[userid.__str__()].preference) +
                                     '\n你可以用 / + 〈數字〉更改你對相關選項的回應，用 pop up 鍵盤亦可。\n' +
                                     '或者用 /add_pref 加入新選項。\n\n' +
                                     '當完成時請用 /finish 。',
                                reply_markup=show_keyboard)

    def on_message(self, msg):
        print('on_message() is being called')
        print('==== all poll in cache ====')
        if len(allPoll) == 0:
            print('No poll yet')
        else:
            for each_poll in allPoll:
                print(allPoll[each_poll].__str__())
        print('========== END ============')
        flavor = telepot.flavor(msg)

        print('flavor: ' + flavor)

        # normal message
        if flavor == 'normal':
            content_type, chat_type, _chat_id = telepot.glance2(msg)
            print('Normal Message:', content_type, chat_type, _chat_id, '; message content: ', msg)

            if content_type == 'text' and chat_type == 'private':

                if self._converType == ConverType.nothing:

                    if msg['text'] == '/start':
                        self.sender.sendMessage(text='今日的日期係：' + str(date.today()) + '. \n' +
                                                     '用 /new 黎 create 一個新問題，' +
                                                     '或者用 /help 睇其他選項。')
                    elif msg['text'] == '/new':
                        self._converType = ConverType.create_poll
                        self._createPollFlow = CreatePollFlow.poll_question
                        self._poll.creatorId = msg['from']['id']
                        self.sender.sendMessage(text='唔該 send 你個問題俾我。')

                    elif msg['text'] == '/help':
                        self.sender.sendMessage(text='用 /new 黎 create 一個新問題，\n' +
                                                     '或者用 /result 黎查詢回應統計。\n')

                    elif msg['text'].encode(encoding='utf-8') == '開始' or msg['text'].startswith('/begin'):
                        found_poll = 0
                        for poll_key in allPoll.keys():
                            print ('in_all_poll: ' + allPoll[poll_key].__str__())
                            for each_resp in allPoll[poll_key].response:
                                print('each_resp: ' + allPoll[poll_key].response[each_resp].__str__())
                                if allPoll[poll_key].response[each_resp].userid == msg['from']['id']:
                                    found_poll = poll_key
                        if found_poll != 0:
                            self._converType = ConverType.response_poll
                            self.start_survey(found_poll, msg['from']['id'])
                        else:
                            self.sender.sendMessage(text='搵唔到你的問題。')
                    else:
                        self.sender.sendMessage(text='唔知你想點，麻煩你再試過。\n' +
                                                     '或者用 /help 睇其他選項。')
                elif self._converType == ConverType.create_poll:
                    if msg['text'] == '/done':
                        self.complte_poll_creation()
                    elif msg['text'] != '':
                        if self._createPollFlow == CreatePollFlow.poll_question:
                            self._poll.question = msg['text']
                            self._createPollFlow = CreatePollFlow.poll_choice
                            self.sender.sendMessage(text='好，咁你自己有乜意見？')
                        elif self._createPollFlow == CreatePollFlow.poll_choice:
                            self._poll.choices.append(msg['text'])
                            self.sender.sendMessage(text='好，仲有冇？有就繼續 send 下個個選擇。\n' +
                                                         '如果冇就用 /done 完成建立問題。')

            elif content_type == 'text' and chat_type == 'group':
                if msg['text'].startswith('/start@' + botName):
                    print('invited into group')
                    poll_id = msg['text'].split(' ')[1]
                    self.get_invited(poll_id)
                    allPoll[poll_id].groupId = _chat_id
                elif msg['text'].startswith('/answer_'):
                    poll_id = msg['text'].split('_')[1].split('@')[0]
                    self.initiate_survey(poll_id,
                                         target_id=msg['from']['id'],
                                         display_name=msg['from']['first_name'] + ' ' + msg['from']['last_name'])
                elif msg['text'].encode(encoding='utf-8') == '開始回應':
                    orig_text = msg['reply_to_message']['text']
                    poll_id = orig_text.split('/answer_')[1].split(' ')[0]
                    self.initiate_survey(poll_id,
                                         target_id=msg['from']['id'],
                                         display_name=msg['from']['first_name'] + ' ' + msg['from']['last_name'])

            print('Poll:' + self._poll.__str__())
        else:
            raise telepot.BadFlavor(msg)

TOKEN = sys.argv[1]  # get token from command-line

bot = telepot.DelegatorBot(TOKEN, [
    (per_chat_id(), create_open(HKGathering, timeout=20)), ])
print('Listening ...')
bot.notifyOnMessage(run_forever=True)
<|endoftext|>


<commit_before>#!/usr/bin/python

import json
import re
import requests
from Queue import Queue
from requests_oauthlib import OAuth1
from threading import Thread
from firebase import firebase

with open('twitter-secrets.json') as twitter_secrets:
    s = json.load(twitter_secrets)
    access_token        = s['access_token']
    access_token_secret = s['access_token_secret']
    consumer_key        = s['consumer_key']
    consumer_secret     = s['consumer_secret']
auth = OAuth1(consumer_key, consumer_secret, access_token, access_token_secret)

firebase = firebase.FirebaseApplication(
    'https://hacker-news.firebaseio.com', None)

work_queue = Queue()

def read_tweets():
    url = 'https://userstream.twitter.com/1.1/user.json'
    r = requests.get(url, auth=auth, stream=True)
    for line in r.iter_lines():
        if line:
            tweet = json.loads(line.decode('utf-8'))
            work_queue.put(tweet)

def run():
    while True:
        tweet = work_queue.get()
        if 'user' in tweet and tweet['user']['id'] == 14335498:
            tweet_id = tweet['id']
            title = re.sub('https://t.co.*$', '', tweet['text']).strip()
            hackernews_id = find_hackernews_id(title)
            hackernews_url = 'https://news.ycombinator.com/item?id=' + str(item_id)
            post_tweet(tweet_id, title)
        work_queue.task_done()

def find_hackernews_id(title):
    top_ids = firebase.get('/v0/topstories', None)
    for item_id in top_ids:
        item = firebase.get('/v0/item', item_id)
        if 'title' in item and item['title'].strip() == title:
            return item_id

def post_tweet(tweet_id, hackernews_url):
    url = 'https://api.twitter.com/1.1/statuses/update.json'
    tweet_text = '@newsycombinator Comments: ' + hackernews_url
    params = {'status': tweet_text, 'in_reply_to_status_id': tweet_id}
    requests.post(url, auth=auth, data=params)

t = Thread(target=run)
t.daemon = True
t.start()

read_tweets()
work_queue.join()
<commit_msg>fix param<commit_after>#!/usr/bin/python

import json
import re
import requests
from Queue import Queue
from requests_oauthlib import OAuth1
from threading import Thread
from firebase import firebase

with open('twitter-secrets.json') as twitter_secrets:
    s = json.load(twitter_secrets)
    access_token        = s['access_token']
    access_token_secret = s['access_token_secret']
    consumer_key        = s['consumer_key']
    consumer_secret     = s['consumer_secret']
auth = OAuth1(consumer_key, consumer_secret, access_token, access_token_secret)

firebase = firebase.FirebaseApplication(
    'https://hacker-news.firebaseio.com', None)

work_queue = Queue()

def read_tweets():
    url = 'https://userstream.twitter.com/1.1/user.json'
    r = requests.get(url, auth=auth, stream=True)
    for line in r.iter_lines():
        if line:
            tweet = json.loads(line.decode('utf-8'))
            work_queue.put(tweet)

def run():
    while True:
        tweet = work_queue.get()
        if 'user' in tweet and tweet['user']['id'] == 14335498:
            tweet_id = tweet['id']
            title = re.sub('https://t.co.*$', '', tweet['text']).strip()
            hackernews_id = find_hackernews_id(title)
            hackernews_url = 'https://news.ycombinator.com/item?id=' + str(item_id)
            post_tweet(tweet_id, hackernews_url)
        work_queue.task_done()

def find_hackernews_id(title):
    top_ids = firebase.get('/v0/topstories', None)
    for item_id in top_ids:
        item = firebase.get('/v0/item', item_id)
        if 'title' in item and item['title'].strip() == title:
            return item_id

def post_tweet(tweet_id, hackernews_url):
    url = 'https://api.twitter.com/1.1/statuses/update.json'
    tweet_text = '@newsycombinator Comments: ' + hackernews_url
    params = {'status': tweet_text, 'in_reply_to_status_id': tweet_id}
    requests.post(url, auth=auth, data=params)

t = Thread(target=run)
t.daemon = True
t.start()

read_tweets()
work_queue.join()
<|endoftext|>


<commit_before>from ..embedder import Embedder
from os import makedirs
from os.path import exists, join, dirname
from shutil import rmtree
from filecmp import cmp

def test_insert_payload():
    if exists(join(dirname(__file__), "results")):
        rmtree(join(dirname(__file__), "results"))
    makedirs(join(dirname(__file__), "results"))

    embedder = Embedder()
    containerFilePath = join(dirname(__file__), "mock/ninja.png")
    payloadFilePath = join(dirname(__file__), "mock/ramen.png")
    resultFilePath = join(dirname(__file__), "results/ninjaThatAteTheRamen.png")
    mockfileFilePath = join(dirname(__file__), "mock/ninjaThatAteTheRamen.png")
    embedder.insertPayload(containerFilePath, payloadFilePath, resultFilePath)
    assert cmp(mockfileFilePath, resultFilePath, shallow=False)<commit_msg>Refactor results folder reset<commit_after>from ..embedder import Embedder
from os import makedirs
from os.path import exists, join, dirname
from shutil import rmtree
from filecmp import cmp

def test_insert_payload():
    _resetResultsFolder()
    embedder = Embedder()
    containerFilePath = join(dirname(__file__), "mock/ninja.png")
    payloadFilePath = join(dirname(__file__), "mock/ramen.png")
    resultFilePath = join(dirname(__file__), "results/ninjaThatAteTheRamen.png")
    mockfileFilePath = join(dirname(__file__), "mock/ninjaThatAteTheRamen.png")
    embedder.insertPayload(containerFilePath, payloadFilePath, resultFilePath)
    assert cmp(mockfileFilePath, resultFilePath, shallow=False)
def _resetResultsFolder():
    if exists(join(dirname(__file__), "results")):
        rmtree(join(dirname(__file__), "results"))
    makedirs(join(dirname(__file__), "results"))<|endoftext|>


<commit_before>from __future__ import absolute_import, division, unicode_literals

from cStringIO import StringIO
from datetime import datetime
from flask.ext.restful import reqparse
from sqlalchemy.orm import joinedload, subqueryload_all
from sqlalchemy.sql import func
from werkzeug.datastructures import FileStorage

from changes.api.base import APIView
from changes.api.validators.author import AuthorValidator
from changes.config import db, queue
from changes.constants import Status
from changes.db.funcs import coalesce
from changes.db.utils import get_or_create
from changes.events import publish_build_update, publish_job_update
from changes.models import (
    Project, Build, Job, JobPlan, Repository, Patch, ProjectOption,
    ItemOption, Source, ProjectPlan
)
from changes.utils.http import build_uri

def create_build(project, label, target, message, author, change=None,
                 patch=None, cause=None, source=None, sha=None):
    assert sha or source

    if not project.plans:
        # Legacy support
        # TODO(dcramer): remove this after we transition to plans
        raise ValueError('No plans defined for project {0}'.format(project))

    repository = project.repository

    if source is None:
        source, _ = get_or_create(Source, where={
            'repository': repository,
            'patch': patch,
            'revision_sha': sha,
        })

    jobs = []

    # TODO(dcramer): find a way to abstract this
    cur_no_query = db.session.query(
        coalesce(func.max(Build.number), 0)
    ).filter(
        Build.project_id == project.id,
    ).scalar()

    build = Build(
        number=cur_no_query + 1,
        project=project,
        project_id=project.id,
        source=source,
        source_id=source.id if source else None,
        repository=repository,
        status=Status.queued,
        author=author,
        author_id=author.id if author else None,
        label=label,
        target=target,
        revision_sha=sha,
        patch=patch,
        message=message,
        cause=cause,
    )

    db.session.add(build)

    for plan in project.plans:
        cur_no_query = db.session.query(
            coalesce(func.max(Job.number), 0)
        ).filter(
            Job.build_id == build.id,
        ).scalar()

        job = Job(
            build=build,
            build_id=build.id,
            number=cur_no_query + 1,
            project=project,
            project_id=project.id,
            source=build.source,
            source_id=build.source_id,
            status=build.status,
            label=plan.label,
        )

        db.session.add(job)

        jobplan = JobPlan(
            project=project,
            job=job,
            build=build,
            plan=plan,
        )

        db.session.add(jobplan)

        jobs.append(job)

    if change:
        change.date_modified = datetime.utcnow()
        db.session.add(change)

    db.session.commit()

    publish_build_update(build)

    for job in jobs:
        publish_job_update(job)
        queue.delay('create_job', kwargs={
            'job_id': job.id.hex,
        }, countdown=5)

    return build

class BuildIndexAPIView(APIView):
    parser = reqparse.RequestParser()
    parser.add_argument('sha', type=str, required=True)
    parser.add_argument('project', type=lambda x: Project.query.filter_by(slug=x).first())
    parser.add_argument('repository', type=lambda x: Repository.query.filter_by(url=x).first())
    parser.add_argument('author', type=AuthorValidator())
    parser.add_argument('label', type=str, default="A homeless build")
    parser.add_argument('target', type=str)
    parser.add_argument('message', type=str)
    parser.add_argument('patch', type=FileStorage, dest='patch_file', location='files')
    parser.add_argument('patch[label]', type=str, dest='patch_label')

    def get(self):
        queryset = Build.query.options(
            joinedload(Build.project),
            joinedload(Build.author),
        ).order_by(Build.date_created.desc(), Build.date_started.desc())

        return self.paginate(queryset)

    def post(self):
        args = self.parser.parse_args()

        print args

        assert args.project or args.repository

        if args.patch_file and not args.patch_label:
            raise ValueError('patch_label')

        if args.project:
            projects = [args.project]
            repository = Repository.query.get(args.project.repository_id)
        else:
            projects = list(Project.query.options(
                subqueryload_all(Project.project_plans, ProjectPlan.plan),
            ).filter(
                Project.repository_id == args.repository.id,
            ))

        if args.patch_file:
            # eliminate projects without diff builds
            options = dict(
                db.session.query(
                    ProjectOption.project_id, ProjectOption.value
                ).filter(
                    ProjectOption.project_id.in_([p.id for p in projects]),
                    ProjectOption.name.in_([
                        'build.allow-patches',
                    ])
                )
            )

            projects = [
                p for p in projects
                if options.get(p.id, '1') == '1'
            ]

        if not projects:
            return self.respond({'builds': []})

        label = args.label[:128]

        if not args.target:
            if args.patch_label:
                target = args.patch_label[:128]
            else:
                target = args.sha[:12]
        else:
            target = args.target[:128]

        if args.patch_file:
            fp = StringIO()
            for line in args.patch_file:
                fp.write(line)
            patch_file = fp
        else:
            patch_file = None

        builds = []
        for project in projects:
            plan_list = list(project.plans)
            if plan_list and patch_file:
                options = dict(
                    db.session.query(
                        ItemOption.item_id, ItemOption.value
                    ).filter(
                        ItemOption.item_id.in_([p.id for p in plan_list]),
                        ItemOption.name.in_([
                            'build.allow-patches',
                        ])
                    )
                )
                plan_list = [
                    p for p in plan_list
                    if options.get(p.id, '1') == '1'
                ]

                # no plans remained
                if not plan_list:
                    continue

            if patch_file:
                patch = Patch(
                    repository=repository,
                    project=project,
                    parent_revision_sha=args.sha,
                    label=args.patch_label,
                    diff=patch_file.getvalue(),
                )
                db.session.add(patch)
            else:
                patch = None

            builds.append(create_build(
                project=project,
                sha=args.sha,
                target=target,
                label=label,
                message=args.message,
                author=args.author,
                patch=patch,
            ))

        context = {
            'builds': [
                {
                    'id': b.id.hex,
                    'project': b.project,
                    'link': build_uri('/builds/{0}/'.format(b.id.hex)),
                } for b in builds
            ],
        }

        return self.respond(context)

    def get_stream_channels(self):
        return ['builds:*']
<commit_msg>Remove print statement<commit_after>from __future__ import absolute_import, division, unicode_literals

from cStringIO import StringIO
from datetime import datetime
from flask.ext.restful import reqparse
from sqlalchemy.orm import joinedload, subqueryload_all
from sqlalchemy.sql import func
from werkzeug.datastructures import FileStorage

from changes.api.base import APIView
from changes.api.validators.author import AuthorValidator
from changes.config import db, queue
from changes.constants import Status
from changes.db.funcs import coalesce
from changes.db.utils import get_or_create
from changes.events import publish_build_update, publish_job_update
from changes.models import (
    Project, Build, Job, JobPlan, Repository, Patch, ProjectOption,
    ItemOption, Source, ProjectPlan
)
from changes.utils.http import build_uri

def create_build(project, label, target, message, author, change=None,
                 patch=None, cause=None, source=None, sha=None):
    assert sha or source

    if not project.plans:
        # Legacy support
        # TODO(dcramer): remove this after we transition to plans
        raise ValueError('No plans defined for project {0}'.format(project))

    repository = project.repository

    if source is None:
        source, _ = get_or_create(Source, where={
            'repository': repository,
            'patch': patch,
            'revision_sha': sha,
        })

    jobs = []

    # TODO(dcramer): find a way to abstract this
    cur_no_query = db.session.query(
        coalesce(func.max(Build.number), 0)
    ).filter(
        Build.project_id == project.id,
    ).scalar()

    build = Build(
        number=cur_no_query + 1,
        project=project,
        project_id=project.id,
        source=source,
        source_id=source.id if source else None,
        repository=repository,
        status=Status.queued,
        author=author,
        author_id=author.id if author else None,
        label=label,
        target=target,
        revision_sha=sha,
        patch=patch,
        message=message,
        cause=cause,
    )

    db.session.add(build)

    for plan in project.plans:
        cur_no_query = db.session.query(
            coalesce(func.max(Job.number), 0)
        ).filter(
            Job.build_id == build.id,
        ).scalar()

        job = Job(
            build=build,
            build_id=build.id,
            number=cur_no_query + 1,
            project=project,
            project_id=project.id,
            source=build.source,
            source_id=build.source_id,
            status=build.status,
            label=plan.label,
        )

        db.session.add(job)

        jobplan = JobPlan(
            project=project,
            job=job,
            build=build,
            plan=plan,
        )

        db.session.add(jobplan)

        jobs.append(job)

    if change:
        change.date_modified = datetime.utcnow()
        db.session.add(change)

    db.session.commit()

    publish_build_update(build)

    for job in jobs:
        publish_job_update(job)
        queue.delay('create_job', kwargs={
            'job_id': job.id.hex,
        }, countdown=5)

    return build

class BuildIndexAPIView(APIView):
    parser = reqparse.RequestParser()
    parser.add_argument('sha', type=str, required=True)
    parser.add_argument('project', type=lambda x: Project.query.filter_by(slug=x).first())
    parser.add_argument('repository', type=lambda x: Repository.query.filter_by(url=x).first())
    parser.add_argument('author', type=AuthorValidator())
    parser.add_argument('label', type=str, default="A homeless build")
    parser.add_argument('target', type=str)
    parser.add_argument('message', type=str)
    parser.add_argument('patch', type=FileStorage, dest='patch_file', location='files')
    parser.add_argument('patch[label]', type=str, dest='patch_label')

    def get(self):
        queryset = Build.query.options(
            joinedload(Build.project),
            joinedload(Build.author),
        ).order_by(Build.date_created.desc(), Build.date_started.desc())

        return self.paginate(queryset)

    def post(self):
        args = self.parser.parse_args()

        assert args.project or args.repository

        if args.patch_file and not args.patch_label:
            raise ValueError('patch_label')

        if args.project:
            projects = [args.project]
            repository = Repository.query.get(args.project.repository_id)
        else:
            projects = list(Project.query.options(
                subqueryload_all(Project.project_plans, ProjectPlan.plan),
            ).filter(
                Project.repository_id == args.repository.id,
            ))

        if args.patch_file:
            # eliminate projects without diff builds
            options = dict(
                db.session.query(
                    ProjectOption.project_id, ProjectOption.value
                ).filter(
                    ProjectOption.project_id.in_([p.id for p in projects]),
                    ProjectOption.name.in_([
                        'build.allow-patches',
                    ])
                )
            )

            projects = [
                p for p in projects
                if options.get(p.id, '1') == '1'
            ]

        if not projects:
            return self.respond({'builds': []})

        label = args.label[:128]

        if not args.target:
            if args.patch_label:
                target = args.patch_label[:128]
            else:
                target = args.sha[:12]
        else:
            target = args.target[:128]

        if args.patch_file:
            fp = StringIO()
            for line in args.patch_file:
                fp.write(line)
            patch_file = fp
        else:
            patch_file = None

        builds = []
        for project in projects:
            plan_list = list(project.plans)
            if plan_list and patch_file:
                options = dict(
                    db.session.query(
                        ItemOption.item_id, ItemOption.value
                    ).filter(
                        ItemOption.item_id.in_([p.id for p in plan_list]),
                        ItemOption.name.in_([
                            'build.allow-patches',
                        ])
                    )
                )
                plan_list = [
                    p for p in plan_list
                    if options.get(p.id, '1') == '1'
                ]

                # no plans remained
                if not plan_list:
                    continue

            if patch_file:
                patch = Patch(
                    repository=repository,
                    project=project,
                    parent_revision_sha=args.sha,
                    label=args.patch_label,
                    diff=patch_file.getvalue(),
                )
                db.session.add(patch)
            else:
                patch = None

            builds.append(create_build(
                project=project,
                sha=args.sha,
                target=target,
                label=label,
                message=args.message,
                author=args.author,
                patch=patch,
            ))

        context = {
            'builds': [
                {
                    'id': b.id.hex,
                    'project': b.project,
                    'link': build_uri('/builds/{0}/'.format(b.id.hex)),
                } for b in builds
            ],
        }

        return self.respond(context)

    def get_stream_channels(self):
        return ['builds:*']
<|endoftext|>


<commit_before>"""An implementation of the Web Site Process Bus.

This module is completely standalone, depending only on the stdlib.

Web Site Process Bus
--------------------

A Bus object is used to contain and manage site-wide behavior:
daemonization, HTTP server start/stop, process reload, signal handling,
drop privileges, PID file management, logging for all of these,
and many more.

In addition, a Bus object provides a place for each web framework
to register code that runs in response to site-wide events (like
process start and stop), or which controls or otherwise interacts with
the site-wide components mentioned above. For example, a framework which
uses file-based templates would add known template filenames to an
autoreload component.

Ideally, a Bus object will be flexible enough to be useful in a variety
of invocation scenarios:

 1. The deployer starts a site from the command line via a framework-
     neutral deployment script; applications from multiple frameworks
     are mixed in a single site. Command-line arguments and configuration
     files are used to define site-wide components such as the HTTP server,
     WSGI component graph, autoreload behavior, signal handling, etc.
 2. The deployer starts a site via some other process, such as Apache;
     applications from multiple frameworks are mixed in a single site.
     Autoreload and signal handling (from Python at least) are disabled.
 3. The deployer starts a site via a framework-specific mechanism;
     for example, when running tests, exploring tutorials, or deploying
     single applications from a single framework. The framework controls
     which site-wide components are enabled as it sees fit.

The Bus object in this package uses topic-based publish-subscribe
messaging to accomplish all this. A few topic channels are built in
('start', 'stop', 'exit', 'graceful', 'log', and 'main'). Frameworks and
site containers are free to define their own. If a message is sent to a
channel that has not been defined or has no listeners, there is no effect.

In general, there should only ever be a single Bus object per process.
Frameworks and site containers share a single Bus object by publishing
messages and subscribing listeners.

The Bus object works as a finite state machine which models the current
state of the process. Bus methods move it from one state to another;
those methods then publish to subscribed listeners on the channel for
the new state.

                        O
                        |
                        V
       STOPPING --> STOPPED --> EXITING -> X
          A   A         |
          |    \___     |
          |        \    |
          |         V   V
        STARTED <-- STARTING

"""

import atexit
import os
try:
    set
except NameError:
    from sets import Set as set
import sys
import threading
import time
import traceback as _traceback
import warnings

# Here I save the value of os.getcwd(), which, if I am imported early enough,
# will be the directory from which the startup script was run.  This is needed
# by _do_execv(), to change back to the original directory before execv()ing a
# new process.  This is a defense against the application having changed the
# current working directory (which could make sys.executable "not found" if
# sys.executable is a relative-path, and/or cause other problems).
_startup_cwd = os.getcwd()

class ChannelFailures(Exception):
    delimiter = '\n'

    def __init__(self, *args, **kwargs):
        # Don't use 'super' here; Exceptions are old-style in Py2.4
        # See http://www.cherrypy.org/ticket/959
        Exception.__init__(self, *args, **kwargs)
        self._exceptions = list()

    def handle_exception(self):
        self._exceptions.append(sys.exc_info())

    def get_instances(self):
        return [instance for cls, instance, traceback in self._exceptions]

    def __str__(self):
        exception_strings = map(repr, self.get_instances())
        return self.delimiter.join(exception_strings)

    def __nonzero__(self):
        return bool(self._exceptions)

# Use a flag to indicate the state of the bus.
class _StateEnum(object):
    class State(object):
        name = None
        def __repr__(self):
            return "states.%s" % self.name

    def __setattr__(self, key, value):
        if isinstance(value, self.State):
            value.name = key
        object.__setattr__(self, key, value)
states = _StateEnum()
states.STOPPED = states.State()
states.STARTING = states.State()
states.STARTED = states.State()
states.STOPPING = states.State()
states.EXITING = states.State()

class Bus(object):
    """Process state-machine and messenger for HTTP site deployment.

    All listeners for a given channel are guaranteed to be called even
    if others at the same channel fail. Each failure is logged, but
    execution proceeds on to the next listener. The only way to stop all
    processing from inside a listener is to raise SystemExit and stop the
    whole server.
    """

    states = states
    state = states.STOPPED
    execv = False

    def __init__(self):
        self.execv = False
        self.state = states.STOPPED
        self.listeners = dict(
            [(channel, set()) for channel
             in ('start', 'stop', 'exit', 'graceful', 'log', 'main')])
        self._priorities = {}

    def subscribe(self, channel, callback, priority=None):
        """Add the given callback at the given channel (if not present)."""
        if channel not in self.listeners:
            self.listeners[channel] = set()
        self.listeners[channel].add(callback)

        if priority is None:
            priority = getattr(callback, 'priority', 50)
        self._priorities[(channel, callback)] = priority

    def unsubscribe(self, channel, callback):
        """Discard the given callback (if present)."""
        listeners = self.listeners.get(channel)
        if listeners and callback in listeners:
            listeners.discard(callback)
            del self._priorities[(channel, callback)]

    def publish(self, channel, *args, **kwargs):
        """Return output of all subscribers for the given channel."""
        if channel not in self.listeners:
            return []

        exc = ChannelFailures()
        output = []

        items = [(self._priorities[(channel, listener)], listener)
                 for listener in self.listeners[channel]]
        items.sort()
        for priority, listener in items:
            try:
                output.append(listener(*args, **kwargs))
            except KeyboardInterrupt:
                raise
            except SystemExit, e:
                # If we have previous errors ensure the exit code is non-zero
                if exc and e.code == 0:
                    e.code = 1
                raise
            except:
                exc.handle_exception()
                if channel == 'log':
                    # Assume any further messages to 'log' will fail.
                    pass
                else:
                    self.log("Error in %r listener %r" % (channel, listener),
                             level=40, traceback=True)
        if exc:
            raise exc
        return output

    def _clean_exit(self):
        """An atexit handler which asserts the Bus is not running."""
        if self.state != states.EXITING:
            warnings.warn(
                "The main thread is exiting, but the Bus is in the %r state; "
                "shutting it down automatically now. You must either call "
                "bus.block() after start(), or call bus.exit() before the "
                "main thread exits." % self.state, RuntimeWarning)
            self.exit()

    def start(self):
        """Start all services."""
        atexit.register(self._clean_exit)

        self.state = states.STARTING
        self.log('Bus STARTING')
        try:
            self.publish('start')
            self.state = states.STARTED
            self.log('Bus STARTED')
        except (KeyboardInterrupt, SystemExit):
            raise
        except:
            self.log("Shutting down due to error in start listener:",
                     level=40, traceback=True)
            e_info = sys.exc_info()
            try:
                self.exit()
            except:
                # Any stop/exit errors will be logged inside publish().
                pass
            raise e_info[0], e_info[1], e_info[2]

    def exit(self):
        """Stop all services and prepare to exit the process."""
        try:
            self.stop()

            self.state = states.EXITING
            self.log('Bus EXITING')
            self.publish('exit')
            # This isn't strictly necessary, but it's better than seeing
            # "Waiting for child threads to terminate..." and then nothing.
            self.log('Bus EXITED')
        except:
            # This method is often called asynchronously (whether thread,
            # signal handler, console handler, or atexit handler), so we
            # can't just let exceptions propagate out unhandled.
            # Assume it's been logged and just die.
            os._exit(70) # EX_SOFTWARE

    def restart(self):
        """Restart the process (may close connections).

        This method does not restart the process from the calling thread;
        instead, it stops the bus and asks the main thread to call execv.
        """
        self.execv = True
        self.exit()

    def graceful(self):
        """Advise all services to reload."""
        self.log('Bus graceful')
        self.publish('graceful')

    def block(self, interval=0.1):
        """Wait for the EXITING state, KeyboardInterrupt or SystemExit.

        This function is intended to be called only by the main thread.
        After waiting for the EXITING state, it also waits for all threads
        to terminate, and then calls os.execv if self.execv is True. This
        design allows another thread to call bus.restart, yet have the main
        thread perform the actual execv call (required on some platforms).
        """
        try:
            self.wait(states.EXITING, interval=interval, channel='main')
        except (KeyboardInterrupt, IOError):
            # The time.sleep call might raise
            # "IOError: [Errno 4] Interrupted function call" on KBInt.
            self.log('Keyboard Interrupt: shutting down bus')
            self.exit()
        except SystemExit:
            self.log('SystemExit raised: shutting down bus')
            self.exit()
            raise

        # Waiting for ALL child threads to finish is necessary on OS X.
        # See http://www.cherrypy.org/ticket/581.
        # It's also good to let them all shut down before allowing
        # the main thread to call atexit handlers.
        # See http://www.cherrypy.org/ticket/751.
        self.log("Waiting for child threads to terminate...")
        for t in threading.enumerate():
            if t != threading.currentThread() and t.isAlive():
                # Note that any dummy (external) threads are always daemonic.
                if hasattr(threading.Thread, "daemon"):
                    # Python 2.6+
                    d = t.daemon
                else:
                    d = t.isDaemon()
                if not d:
                    t.join()

        if self.execv:
            self._do_execv()

    def wait(self, state, interval=0.1, channel=None):
        """Wait for the given state(s)."""
        if isinstance(state, (tuple, list)):
            states = state
        else:
            states = [state]

        def _wait():
            while self.state not in states:
                time.sleep(interval)
                self.publish(channel)

        # From http://psyco.sourceforge.net/psycoguide/bugs.html:
        # "The compiled machine code does not include the regular polling
        # done by Python, meaning that a KeyboardInterrupt will not be
        # detected before execution comes back to the regular Python
        # interpreter. Your program cannot be interrupted if caught
        # into an infinite Psyco-compiled loop."
        try:
            sys.modules['psyco'].cannotcompile(_wait)
        except (KeyError, AttributeError):
            pass

        _wait()

    def _do_execv(self):
        """Re-execute the current process.

        This must be called from the main thread, because certain platforms
        (OS X) don't allow execv to be called in a child thread very well.
        """
        args = sys.argv[:]
        self.log('Re-spawning %s' % ' '.join(args))
        args.insert(0, sys.executable)
        if sys.platform == 'win32':
            args = ['"%s"' % arg for arg in args]

        os.chdir(_startup_cwd)
        os.execv(sys.executable, args)

    def stop(self):
        """Stop all services."""
        self.state = states.STOPPING
        self.log('Bus STOPPING')
        self.publish('stop')
        self.state = states.STOPPED
        self.log('Bus STOPPED')

    def start_with_callback(self, func, args=None, kwargs=None):
        """Start 'func' in a new thread T, then start self (and return T)."""
        if args is None:
            args = ()
        if kwargs is None:
            kwargs = {}
        args = (func,) + args

        def _callback(func, *a, **kw):
            self.wait(states.STARTED)
            func(*a, **kw)
        t = threading.Thread(target=_callback, args=args, kwargs=kwargs)
        t.setName('Bus Callback ' + t.getName())
        t.start()

        self.start()

        return t

    def log(self, msg="", level=20, traceback=False):
        """Log the given message. Append the last traceback if requested."""
        if traceback:
            exc = sys.exc_info()
            msg += "\n" + "".join(_traceback.format_exception(*exc))
        self.publish('log', msg, level)

bus = Bus()
<commit_msg>Made bus.exit call `os._exit` if called before start completes.<commit_after>"""An implementation of the Web Site Process Bus.

This module is completely standalone, depending only on the stdlib.

Web Site Process Bus
--------------------

A Bus object is used to contain and manage site-wide behavior:
daemonization, HTTP server start/stop, process reload, signal handling,
drop privileges, PID file management, logging for all of these,
and many more.

In addition, a Bus object provides a place for each web framework
to register code that runs in response to site-wide events (like
process start and stop), or which controls or otherwise interacts with
the site-wide components mentioned above. For example, a framework which
uses file-based templates would add known template filenames to an
autoreload component.

Ideally, a Bus object will be flexible enough to be useful in a variety
of invocation scenarios:

 1. The deployer starts a site from the command line via a framework-
     neutral deployment script; applications from multiple frameworks
     are mixed in a single site. Command-line arguments and configuration
     files are used to define site-wide components such as the HTTP server,
     WSGI component graph, autoreload behavior, signal handling, etc.
 2. The deployer starts a site via some other process, such as Apache;
     applications from multiple frameworks are mixed in a single site.
     Autoreload and signal handling (from Python at least) are disabled.
 3. The deployer starts a site via a framework-specific mechanism;
     for example, when running tests, exploring tutorials, or deploying
     single applications from a single framework. The framework controls
     which site-wide components are enabled as it sees fit.

The Bus object in this package uses topic-based publish-subscribe
messaging to accomplish all this. A few topic channels are built in
('start', 'stop', 'exit', 'graceful', 'log', and 'main'). Frameworks and
site containers are free to define their own. If a message is sent to a
channel that has not been defined or has no listeners, there is no effect.

In general, there should only ever be a single Bus object per process.
Frameworks and site containers share a single Bus object by publishing
messages and subscribing listeners.

The Bus object works as a finite state machine which models the current
state of the process. Bus methods move it from one state to another;
those methods then publish to subscribed listeners on the channel for
the new state.

                        O
                        |
                        V
       STOPPING --> STOPPED --> EXITING -> X
          A   A         |
          |    \___     |
          |        \    |
          |         V   V
        STARTED <-- STARTING

"""

import atexit
import os
try:
    set
except NameError:
    from sets import Set as set
import sys
import threading
import time
import traceback as _traceback
import warnings

# Here I save the value of os.getcwd(), which, if I am imported early enough,
# will be the directory from which the startup script was run.  This is needed
# by _do_execv(), to change back to the original directory before execv()ing a
# new process.  This is a defense against the application having changed the
# current working directory (which could make sys.executable "not found" if
# sys.executable is a relative-path, and/or cause other problems).
_startup_cwd = os.getcwd()

class ChannelFailures(Exception):
    delimiter = '\n'

    def __init__(self, *args, **kwargs):
        # Don't use 'super' here; Exceptions are old-style in Py2.4
        # See http://www.cherrypy.org/ticket/959
        Exception.__init__(self, *args, **kwargs)
        self._exceptions = list()

    def handle_exception(self):
        self._exceptions.append(sys.exc_info())

    def get_instances(self):
        return [instance for cls, instance, traceback in self._exceptions]

    def __str__(self):
        exception_strings = map(repr, self.get_instances())
        return self.delimiter.join(exception_strings)

    def __nonzero__(self):
        return bool(self._exceptions)

# Use a flag to indicate the state of the bus.
class _StateEnum(object):
    class State(object):
        name = None
        def __repr__(self):
            return "states.%s" % self.name

    def __setattr__(self, key, value):
        if isinstance(value, self.State):
            value.name = key
        object.__setattr__(self, key, value)
states = _StateEnum()
states.STOPPED = states.State()
states.STARTING = states.State()
states.STARTED = states.State()
states.STOPPING = states.State()
states.EXITING = states.State()

class Bus(object):
    """Process state-machine and messenger for HTTP site deployment.

    All listeners for a given channel are guaranteed to be called even
    if others at the same channel fail. Each failure is logged, but
    execution proceeds on to the next listener. The only way to stop all
    processing from inside a listener is to raise SystemExit and stop the
    whole server.
    """

    states = states
    state = states.STOPPED
    execv = False

    def __init__(self):
        self.execv = False
        self.state = states.STOPPED
        self.listeners = dict(
            [(channel, set()) for channel
             in ('start', 'stop', 'exit', 'graceful', 'log', 'main')])
        self._priorities = {}

    def subscribe(self, channel, callback, priority=None):
        """Add the given callback at the given channel (if not present)."""
        if channel not in self.listeners:
            self.listeners[channel] = set()
        self.listeners[channel].add(callback)

        if priority is None:
            priority = getattr(callback, 'priority', 50)
        self._priorities[(channel, callback)] = priority

    def unsubscribe(self, channel, callback):
        """Discard the given callback (if present)."""
        listeners = self.listeners.get(channel)
        if listeners and callback in listeners:
            listeners.discard(callback)
            del self._priorities[(channel, callback)]

    def publish(self, channel, *args, **kwargs):
        """Return output of all subscribers for the given channel."""
        if channel not in self.listeners:
            return []

        exc = ChannelFailures()
        output = []

        items = [(self._priorities[(channel, listener)], listener)
                 for listener in self.listeners[channel]]
        items.sort()
        for priority, listener in items:
            try:
                output.append(listener(*args, **kwargs))
            except KeyboardInterrupt:
                raise
            except SystemExit, e:
                # If we have previous errors ensure the exit code is non-zero
                if exc and e.code == 0:
                    e.code = 1
                raise
            except:
                exc.handle_exception()
                if channel == 'log':
                    # Assume any further messages to 'log' will fail.
                    pass
                else:
                    self.log("Error in %r listener %r" % (channel, listener),
                             level=40, traceback=True)
        if exc:
            raise exc
        return output

    def _clean_exit(self):
        """An atexit handler which asserts the Bus is not running."""
        if self.state != states.EXITING:
            warnings.warn(
                "The main thread is exiting, but the Bus is in the %r state; "
                "shutting it down automatically now. You must either call "
                "bus.block() after start(), or call bus.exit() before the "
                "main thread exits." % self.state, RuntimeWarning)
            self.exit()

    def start(self):
        """Start all services."""
        atexit.register(self._clean_exit)

        self.state = states.STARTING
        self.log('Bus STARTING')
        try:
            self.publish('start')
            self.state = states.STARTED
            self.log('Bus STARTED')
        except (KeyboardInterrupt, SystemExit):
            raise
        except:
            self.log("Shutting down due to error in start listener:",
                     level=40, traceback=True)
            e_info = sys.exc_info()
            try:
                self.exit()
            except:
                # Any stop/exit errors will be logged inside publish().
                pass
            raise e_info[0], e_info[1], e_info[2]

    def exit(self):
        """Stop all services and prepare to exit the process."""
        exitstate = self.state
        try:
            self.stop()

            self.state = states.EXITING
            self.log('Bus EXITING')
            self.publish('exit')
            # This isn't strictly necessary, but it's better than seeing
            # "Waiting for child threads to terminate..." and then nothing.
            self.log('Bus EXITED')
        except:
            # This method is often called asynchronously (whether thread,
            # signal handler, console handler, or atexit handler), so we
            # can't just let exceptions propagate out unhandled.
            # Assume it's been logged and just die.
            os._exit(70) # EX_SOFTWARE

        if exitstate == states.STARTING:
            # exit() was called before start() finished, possibly due to
            # Ctrl-C because a start listener got stuck. In this case,
            # we could get stuck in a loop where Ctrl-C never exits the
            # process, so we just call os.exit here.
            os._exit(70) # EX_SOFTWARE

    def restart(self):
        """Restart the process (may close connections).

        This method does not restart the process from the calling thread;
        instead, it stops the bus and asks the main thread to call execv.
        """
        self.execv = True
        self.exit()

    def graceful(self):
        """Advise all services to reload."""
        self.log('Bus graceful')
        self.publish('graceful')

    def block(self, interval=0.1):
        """Wait for the EXITING state, KeyboardInterrupt or SystemExit.

        This function is intended to be called only by the main thread.
        After waiting for the EXITING state, it also waits for all threads
        to terminate, and then calls os.execv if self.execv is True. This
        design allows another thread to call bus.restart, yet have the main
        thread perform the actual execv call (required on some platforms).
        """
        try:
            self.wait(states.EXITING, interval=interval, channel='main')
        except (KeyboardInterrupt, IOError):
            # The time.sleep call might raise
            # "IOError: [Errno 4] Interrupted function call" on KBInt.
            self.log('Keyboard Interrupt: shutting down bus')
            self.exit()
        except SystemExit:
            self.log('SystemExit raised: shutting down bus')
            self.exit()
            raise

        # Waiting for ALL child threads to finish is necessary on OS X.
        # See http://www.cherrypy.org/ticket/581.
        # It's also good to let them all shut down before allowing
        # the main thread to call atexit handlers.
        # See http://www.cherrypy.org/ticket/751.
        self.log("Waiting for child threads to terminate...")
        for t in threading.enumerate():
            if t != threading.currentThread() and t.isAlive():
                # Note that any dummy (external) threads are always daemonic.
                if hasattr(threading.Thread, "daemon"):
                    # Python 2.6+
                    d = t.daemon
                else:
                    d = t.isDaemon()
                if not d:
                    t.join()

        if self.execv:
            self._do_execv()

    def wait(self, state, interval=0.1, channel=None):
        """Wait for the given state(s)."""
        if isinstance(state, (tuple, list)):
            states = state
        else:
            states = [state]

        def _wait():
            while self.state not in states:
                time.sleep(interval)
                self.publish(channel)

        # From http://psyco.sourceforge.net/psycoguide/bugs.html:
        # "The compiled machine code does not include the regular polling
        # done by Python, meaning that a KeyboardInterrupt will not be
        # detected before execution comes back to the regular Python
        # interpreter. Your program cannot be interrupted if caught
        # into an infinite Psyco-compiled loop."
        try:
            sys.modules['psyco'].cannotcompile(_wait)
        except (KeyError, AttributeError):
            pass

        _wait()

    def _do_execv(self):
        """Re-execute the current process.

        This must be called from the main thread, because certain platforms
        (OS X) don't allow execv to be called in a child thread very well.
        """
        args = sys.argv[:]
        self.log('Re-spawning %s' % ' '.join(args))
        args.insert(0, sys.executable)
        if sys.platform == 'win32':
            args = ['"%s"' % arg for arg in args]

        os.chdir(_startup_cwd)
        os.execv(sys.executable, args)

    def stop(self):
        """Stop all services."""
        self.state = states.STOPPING
        self.log('Bus STOPPING')
        self.publish('stop')
        self.state = states.STOPPED
        self.log('Bus STOPPED')

    def start_with_callback(self, func, args=None, kwargs=None):
        """Start 'func' in a new thread T, then start self (and return T)."""
        if args is None:
            args = ()
        if kwargs is None:
            kwargs = {}
        args = (func,) + args

        def _callback(func, *a, **kw):
            self.wait(states.STARTED)
            func(*a, **kw)
        t = threading.Thread(target=_callback, args=args, kwargs=kwargs)
        t.setName('Bus Callback ' + t.getName())
        t.start()

        self.start()

        return t

    def log(self, msg="", level=20, traceback=False):
        """Log the given message. Append the last traceback if requested."""
        if traceback:
            exc = sys.exc_info()
            msg += "\n" + "".join(_traceback.format_exception(*exc))
        self.publish('log', msg, level)

bus = Bus()
<|endoftext|>