<commit_before>import re
import traceback
from datetime import timedelta, datetime
from math import ceil

from pywikibot import ItemPage, Page
from tools.catscan import PetScan
from tools.date_conversion import DateConversion
from tools.template_handler import TemplateHandler
from tools.bots import CanonicalBot

class AuthorList(CanonicalBot):
    def __init__(self, wiki, debug):
        CanonicalBot.__init__(self, wiki, debug)
        self.botname = 'AuthorList'
        self.searcher = PetScan()
        self.repo = self.wiki.data_repository()  # this is a DataSite object
        self.string_list = []
        self.match_property = re.compile('\{\{#property:P(\d{1,4})\}\}')
        self.number_to_month = {1: 'Januar',
                                2: 'Februar',
                                3: 'März',
                                4: 'April',
                                5: 'Mai',
                                6: 'Juni',
                                7: 'Juli',
                                8: 'August',
                                9: 'September',
                                10: 'Oktober',
                                11: 'November',
                                12: 'Dezember'}

    def __enter__(self):
        CanonicalBot.__enter__(self)
        if 1 == datetime.now().day:
            self.data = None
            self.logger.warning('The data is thrown away. It is the first of the month')
        if not self.data:
            self.data = {}

    def run(self):
        lemma_list = self._run_searcher()
        self._build_database(lemma_list)
        if self.debug:
            dump = Page(self.wiki, 'Benutzer:THEbotIT/{}'.format(self.botname))
        else:
            dump = Page(self.wiki, 'Liste der Autoren')
        old_text = dump.text
        new_text = self._convert_to_table()
        if new_text[150:] != old_text[150:]: #compare all but the date
            dump.text =  new_text
            dump.save('die Liste wurde auf den aktuellen Stand gebracht.', botflag=True)
        else:
            self.logger.info('Heute gab es keine Änderungen, daher wird die Seite nicht überschrieben.')

    def _run_searcher(self):
        # was the last run successful
        if self.debug:
        #if False
            yesterday = datetime.now() - timedelta(days=5)
            self.searcher.last_change_after(int(yesterday.strftime('%Y')),
                                            int(yesterday.strftime('%m')),
                                            int(yesterday.strftime('%d')))
        elif self.last_run and self.last_run['succes'] and self.data:
            self.create_timestamp_for_search(self.searcher)
        else:
            self.logger.warning('There was no timestamp found of the last run, so the argument "after" is not set.')
        self.searcher.add_namespace(0)  # search in main namespace
        self.searcher.add_positive_category('Autoren')
        self.searcher.add_yes_template('Personendaten')
        self.searcher.get_wikidata_items()

        self.logger.debug(self.searcher)

        entries_to_search = self.searcher.run()
        #following entries are ignored by petscan
        entries_to_search.append({'title': "Karl_Löffler", 'q': 'Q1732245', 'id':'413038'})
        entries_to_search.append({'title': "Adolf_Wüllner", 'q': 'Q106820', 'id':'413022'})
        return entries_to_search

    def _build_database(self, lemma_list):
        for idx, author in enumerate(lemma_list):
            self.logger.debug('{}/{} {}'.format(idx + 1, len(lemma_list), author['title']))
            # delete preexisting data of this author
            try:
                del self.data[str(author['id'])]
            except:
                if self.last_run and self.last_run['succes']:
                    self.logger.info("Can't delete old entry of [[{}]]".format(author['title']))

            dict_author = {'title': author['title']}
            # extract the Personendaten-block form the wikisource page
            page = Page(self.wiki, author['title'])
            try:
                try:
                    personendaten = re.search('\{\{Personendaten(?:.|\n)*?\n\}\}\n', page.text).group()
                except:
                    self.logger.error('No valid block "Personendaten" was found for [[{}]].'.format(author['title']))
                    personendaten = None
                if personendaten:
                    #personendaten = re.sub('<ref.*?>.*?<\/ref>|<ref.*?\/>', '', personendaten)
                    #personendaten = re.sub('\{\{CRef|.*?(?:\{\{.*?\}\})?}}', '', personendaten)
                    template_extractor = TemplateHandler(personendaten)
                    dict_author.update({'name': template_extractor.get_parameter('NACHNAME')['value']})
                    dict_author.update({'first_name': template_extractor.get_parameter('VORNAMEN')['value']})
                    try:
                        dict_author.update({'birth': template_extractor.get_parameter('GEBURTSDATUM')['value']})
                    except:
                        dict_author.update({'birth': ""})
                        self.logger.warning("Templatehandler couldn't find a birthdate for: [[{}]]".format(author['title']))
                    try:
                        dict_author.update({'death': template_extractor.get_parameter('STERBEDATUM')['value']})
                    except:
                        dict_author.update({'death': ""})
                        self.logger.warning("Templatehandler couldn't find a deathdate for: [[{}]]".format(author['title']))
                    try:
                        dict_author.update({'description': template_extractor.get_parameter('KURZBESCHREIBUNG')['value']})
                    except:
                        dict_author.update({'description': ""})
                        self.logger.warning("Templatehandler couldn't find a description for: [[{}]]".format(author['title']))
                    try:
                        dict_author.update({'synonyms': template_extractor.get_parameter('ALTERNATIVNAMEN')['value']})
                    except:
                        dict_author.update({'synonyms': ""})
                        self.logger.warning("Templatehandler couldn't find synonyms for: [[{}]]".format(author['title']))
                    try:
                        dict_author.update({'sortkey': template_extractor.get_parameter('SORTIERUNG')['value']})
                        if dict_author['sortkey'] == '':
                            raise ValueError
                    except:
                        self.logger.debug('there is no sortkey for [[{}]].'.format(author['title']))
                        # make a dummy key
                        dict_author['sortkey'] = dict_author['name'] + ', ' + dict_author['first_name']
                    try:
                        dict_author.update({'wikidata' : author['q']})
                    except:
                        self.logger.warning('The autor [[{}]] has no wikidata_item'.format(author['title']))
                    self.data.update({author['id']: dict_author})
            except Exception as e:
                self.logger.error(traceback.format_exc())
                self.logger.error('author {} have a problem'.format(author['title']))

    def _convert_to_table(self):
        # make a list of lists
        self.logger.info('Start compiling.')
        list_authors = []
        for key in self.data:
            author_dict = self.data[key]
            list_author = []
            list_author.append(author_dict['sortkey']) #0
            list_author.append(author_dict['title'].replace('_', ' ')) #1
            list_author.append(author_dict['name'])#2
            list_author.append(author_dict['first_name'])#3

            for event in ['birth', 'death']:
                list_author.append(self._handle_birth_and_death(event, author_dict)) #4,6
                try:
                    list_author.append(str(DateConversion(list_author[-1])))#5,7
                except:
                    self.logger.error('Can´t compile sort key for {}: {}'.format(event, author_dict[event]))
                    list_author.append('!-00-00')  # 5,7
            list_author.append(author_dict['description'])#8
            list_authors.append(list_author)

        # sorting the list
        self.logger.info('Start sorting.')
        list_authors.sort(key = lambda x: x[0])
        for i in range(len(list_authors) - 1):
            if list_authors[i][0] == list_authors[i+1][0]:
                equal_count = 2
                while True:
                    if i+equal_count <= len(list_authors):
                        if list_authors[i][0] != list_authors[i+equal_count][0]:
                            break
                        equal_count += 1
                temp_list = list_authors[i:i+equal_count]
                temp_list.sort(key=lambda x: x[5])
                list_authors[i:i + equal_count] = temp_list

        self.logger.info('Start printing.')
        self.string_list.append('Diese Liste der Autoren enthält alle {count}<ref>Stand: {dt.day}.{dt.month}.{dt.year}, {clock} (UTC)</ref> Autoren, zu denen in Wikisource eine Autorenseite existiert.'
                                .format(count = len(self.data),
                                        clock = datetime.now().strftime('%H:%M'),
                                        dt = datetime.now()))
        self.string_list.append('Die Liste kann mit den Buttons neben den Spaltenüberschriften nach der jeweiligen Spalte sortiert werden.')
        self.string_list.append('<!--')
        self.string_list.append('Diese Liste wurde durch ein Computerprogramm erstellt, das die Daten verwendet, die aus den Infoboxen auf den Autorenseiten stammen.')
        self.string_list.append('Sollten daher Fehler vorhanden sein, sollten diese jeweils dort korrigiert werden.')
        self.string_list.append('-->')
        self.string_list.append('{|class="wikitable sortable"')
        self.string_list.append('!style="width:20%"| Name')
        self.string_list.append('!data-sort-type="text" style="width:15%"| Geb.-datum')
        self.string_list.append('!data-sort-type="text" style="width:15%"| Tod.-datum')
        self.string_list.append('!class="unsortable" style="width:50%"| Beschreibung')
        for list_author in list_authors:
            self.string_list.append('|-')
            if list_author[2] and list_author[3]:
                self.string_list.append('|data-sort-value="{}"|[[{}|{}, {}]]'
                                        .format(list_author[0], list_author[1], list_author[2],list_author[3]))
            elif list_author[3]:
                self.string_list.append('|data-sort-value="{}"|[[{}|{}]]'
                                        .format(list_author[0], list_author[1], list_author[3]))
            else:
                self.string_list.append('|data-sort-value="{}"|[[{}|{}]]'
                                        .format(list_author[0], list_author[1], list_author[2]))
            self.string_list.append('|data-sort-value="{}"|{}'
                                    .format(list_author[5], list_author[4]))
            self.string_list.append('|data-sort-value="{}"|{}'
                                    .format(list_author[7], list_author[6]))
            self.string_list.append('|{}'.format(list_author[8]))
        self.string_list.append('|}')
        self.string_list.append('')
        self.string_list.append('== Anmerkungen ==')
        self.string_list.append('<references/>')
        self.string_list.append('')
        self.string_list.append('{{SORTIERUNG: Autoren  # Liste der}}')
        self.string_list.append('[[Kategorie:Listen]]')
        self.string_list.append('[[Kategorie:Autoren |!]]')

        return '\n'.join(self.string_list)

    def _handle_birth_and_death(self, event, author_dict):
        if author_dict[event] == '' or self.match_property.search(author_dict[event]):
            self.logger.debug('No valid entry in {} for [[{}]] ... Fallback to wikidata'.format(event, author_dict['title']))
            try:
                item = ItemPage(self.repo, author_dict['wikidata'])
                if event == 'birth':
                    property_label = 'P569'
                else:
                    property_label = 'P570'
                claim = item.text['claims'][property_label][0]
                date_from_data = claim.getTarget()
                if date_from_data.precision < 7:
                    self.logger.error('Precison is to low for [[{}]]'.format(author_dict['title']))
                    raise
                elif date_from_data.precision < 8:
                    date_from_data = int(ceil(float(date_from_data.year) / 100.0) * 100)
                    if date_from_data < 1000:
                        date_from_data = str(date_from_data)[0:1] + '. Jh.'
                    else:
                        date_from_data = str(date_from_data)[0:2] + '. Jh.'
                elif date_from_data.precision < 10:
                    date_from_data = str(date_from_data.year)
                elif date_from_data.precision < 11:
                    date_from_data = self.number_to_month[date_from_data.month] + ' ' + str(date_from_data.year)
                else:
                    date_from_data = str(date_from_data.day) + '. ' + self.number_to_month[date_from_data.month] + ' ' + str(date_from_data.year)
                if re.search('-', date_from_data):
                    date_from_data = date_from_data.replace('-', '') + ' v. Chr.'
                self.logger.debug('Found {} @ wikidata for {}'.format(date_from_data, event))
                return(date_from_data)  # 4,6
            except:
                self.logger.debug("Wasn't able to ge any data from wikidata")
                return('')  # 4,6
        else:
            return(author_dict[event])  # 4,6<commit_msg>better sorting for the author bot<commit_after>import re
import traceback
from datetime import timedelta, datetime
from math import ceil
import icu

from pywikibot import ItemPage, Page
from tools.catscan import PetScan
from tools.date_conversion import DateConversion
from tools.template_handler import TemplateHandler
from tools.bots import CanonicalBot

class AuthorList(CanonicalBot):
    def __init__(self, wiki, debug):
        CanonicalBot.__init__(self, wiki, debug)
        self.botname = 'AuthorList'
        self.searcher = PetScan()
        self.repo = self.wiki.data_repository()  # this is a DataSite object
        self.string_list = []
        self.match_property = re.compile('\{\{#property:P(\d{1,4})\}\}')
        self.number_to_month = {1: 'Januar',
                                2: 'Februar',
                                3: 'März',
                                4: 'April',
                                5: 'Mai',
                                6: 'Juni',
                                7: 'Juli',
                                8: 'August',
                                9: 'September',
                                10: 'Oktober',
                                11: 'November',
                                12: 'Dezember'}

    def __enter__(self):
        CanonicalBot.__enter__(self)
        if 1 == datetime.now().day:
            self.data = None
            self.logger.warning('The data is thrown away. It is the first of the month')
        if not self.data:
            self.data = {}

    def run(self):
        lemma_list = self._run_searcher()
        self._build_database(lemma_list)
        if self.debug:
            dump = Page(self.wiki, 'Benutzer:THEbotIT/{}'.format(self.botname))
        else:
            dump = Page(self.wiki, 'Liste der Autoren')
        old_text = dump.text
        new_text = self._convert_to_table()
        if new_text[150:] != old_text[150:]: #compare all but the date
            dump.text =  new_text
            dump.save('die Liste wurde auf den aktuellen Stand gebracht.', botflag=True)
        else:
            self.logger.info('Heute gab es keine Änderungen, daher wird die Seite nicht überschrieben.')

    def _run_searcher(self):
        # was the last run successful
        if self.debug:
        #if False
            yesterday = datetime.now() - timedelta(days=5)
            self.searcher.last_change_after(int(yesterday.strftime('%Y')),
                                            int(yesterday.strftime('%m')),
                                            int(yesterday.strftime('%d')))
        elif self.last_run and self.last_run['succes'] and self.data:
            self.create_timestamp_for_search(self.searcher)
        else:
            self.logger.warning('There was no timestamp found of the last run, so the argument "after" is not set.')
        self.searcher.add_namespace(0)  # search in main namespace
        self.searcher.add_positive_category('Autoren')
        self.searcher.add_yes_template('Personendaten')
        self.searcher.get_wikidata_items()

        self.logger.debug(self.searcher)

        entries_to_search = self.searcher.run()
        #following entries are ignored by petscan
        entries_to_search.append({'title': "Karl_Löffler", 'q': 'Q1732245', 'id':'413038'})
        entries_to_search.append({'title': "Adolf_Wüllner", 'q': 'Q106820', 'id':'413022'})
        return entries_to_search

    def _build_database(self, lemma_list):
        for idx, author in enumerate(lemma_list):
            self.logger.debug('{}/{} {}'.format(idx + 1, len(lemma_list), author['title']))
            # delete preexisting data of this author
            try:
                del self.data[str(author['id'])]
            except:
                if self.last_run and self.last_run['succes']:
                    self.logger.info("Can't delete old entry of [[{}]]".format(author['title']))

            dict_author = {'title': author['title']}
            # extract the Personendaten-block form the wikisource page
            page = Page(self.wiki, author['title'])
            try:
                try:
                    personendaten = re.search('\{\{Personendaten(?:.|\n)*?\n\}\}\n', page.text).group()
                except:
                    self.logger.error('No valid block "Personendaten" was found for [[{}]].'.format(author['title']))
                    personendaten = None
                if personendaten:
                    #personendaten = re.sub('<ref.*?>.*?<\/ref>|<ref.*?\/>', '', personendaten)
                    #personendaten = re.sub('\{\{CRef|.*?(?:\{\{.*?\}\})?}}', '', personendaten)
                    template_extractor = TemplateHandler(personendaten)
                    dict_author.update({'name': template_extractor.get_parameter('NACHNAME')['value']})
                    dict_author.update({'first_name': template_extractor.get_parameter('VORNAMEN')['value']})
                    try:
                        dict_author.update({'birth': template_extractor.get_parameter('GEBURTSDATUM')['value']})
                    except:
                        dict_author.update({'birth': ""})
                        self.logger.warning("Templatehandler couldn't find a birthdate for: [[{}]]".format(author['title']))
                    try:
                        dict_author.update({'death': template_extractor.get_parameter('STERBEDATUM')['value']})
                    except:
                        dict_author.update({'death': ""})
                        self.logger.warning("Templatehandler couldn't find a deathdate for: [[{}]]".format(author['title']))
                    try:
                        dict_author.update({'description': template_extractor.get_parameter('KURZBESCHREIBUNG')['value']})
                    except:
                        dict_author.update({'description': ""})
                        self.logger.warning("Templatehandler couldn't find a description for: [[{}]]".format(author['title']))
                    try:
                        dict_author.update({'synonyms': template_extractor.get_parameter('ALTERNATIVNAMEN')['value']})
                    except:
                        dict_author.update({'synonyms': ""})
                        self.logger.warning("Templatehandler couldn't find synonyms for: [[{}]]".format(author['title']))
                    try:
                        dict_author.update({'sortkey': template_extractor.get_parameter('SORTIERUNG')['value']})
                        if dict_author['sortkey'] == '':
                            raise ValueError
                    except:
                        self.logger.debug('there is no sortkey for [[{}]].'.format(author['title']))
                        # make a dummy key
                        dict_author['sortkey'] = dict_author['name'] + ', ' + dict_author['first_name']
                    try:
                        dict_author.update({'wikidata' : author['q']})
                    except:
                        self.logger.warning('The autor [[{}]] has no wikidata_item'.format(author['title']))
                    self.data.update({author['id']: dict_author})
            except Exception as e:
                self.logger.error(traceback.format_exc())
                self.logger.error('author {} have a problem'.format(author['title']))

    def _convert_to_table(self):
        # make a list of lists
        self.logger.info('Start compiling.')
        list_authors = []
        for key in self.data:
            author_dict = self.data[key]
            list_author = []
            list_author.append(author_dict['sortkey']) #0
            list_author.append(author_dict['title'].replace('_', ' ')) #1
            list_author.append(author_dict['name'])#2
            list_author.append(author_dict['first_name'])#3

            for event in ['birth', 'death']:
                list_author.append(self._handle_birth_and_death(event, author_dict)) #4,6
                try:
                    list_author.append(str(DateConversion(list_author[-1])))#5,7
                except:
                    self.logger.error('Can´t compile sort key for {}: {}'.format(event, author_dict[event]))
                    list_author.append('!-00-00')  # 5,7
            list_author.append(author_dict['description'])#8
            list_authors.append(list_author)

        # sorting the list
        self.logger.info('Start sorting.')
        collator = icu.Collator.createInstance(icu.Locale('de_DE.UTF-8'))
        list_authors.sort(key=lambda x: collator.getSortKey(x[0]))
        for i in range(len(list_authors) - 1):
            if list_authors[i][0] == list_authors[i+1][0]:
                equal_count = 2
                while True:
                    if i+equal_count <= len(list_authors):
                        if list_authors[i][0] != list_authors[i+equal_count][0]:
                            break
                        equal_count += 1
                temp_list = list_authors[i:i+equal_count]
                temp_list.sort(key=lambda x: x[5])
                list_authors[i:i + equal_count] = temp_list

        self.logger.info('Start printing.')
        self.string_list.append('Diese Liste der Autoren enthält alle {count}<ref>Stand: {dt.day}.{dt.month}.{dt.year}, {clock} (UTC)</ref> Autoren, zu denen in Wikisource eine Autorenseite existiert.'
                                .format(count = len(self.data),
                                        clock = datetime.now().strftime('%H:%M'),
                                        dt = datetime.now()))
        self.string_list.append('Die Liste kann mit den Buttons neben den Spaltenüberschriften nach der jeweiligen Spalte sortiert werden.')
        self.string_list.append('<!--')
        self.string_list.append('Diese Liste wurde durch ein Computerprogramm erstellt, das die Daten verwendet, die aus den Infoboxen auf den Autorenseiten stammen.')
        self.string_list.append('Sollten daher Fehler vorhanden sein, sollten diese jeweils dort korrigiert werden.')
        self.string_list.append('-->')
        self.string_list.append('{|class="wikitable sortable"')
        self.string_list.append('!style="width:20%"| Name')
        self.string_list.append('!data-sort-type="text" style="width:15%"| Geb.-datum')
        self.string_list.append('!data-sort-type="text" style="width:15%"| Tod.-datum')
        self.string_list.append('!class="unsortable" style="width:50%"| Beschreibung')
        for list_author in list_authors:
            self.string_list.append('|-')
            if list_author[2] and list_author[3]:
                self.string_list.append('|data-sort-value="{}"|[[{}|{}, {}]]'
                                        .format(list_author[0], list_author[1], list_author[2],list_author[3]))
            elif list_author[3]:
                self.string_list.append('|data-sort-value="{}"|[[{}|{}]]'
                                        .format(list_author[0], list_author[1], list_author[3]))
            else:
                self.string_list.append('|data-sort-value="{}"|[[{}|{}]]'
                                        .format(list_author[0], list_author[1], list_author[2]))
            self.string_list.append('|data-sort-value="{}"|{}'
                                    .format(list_author[5], list_author[4]))
            self.string_list.append('|data-sort-value="{}"|{}'
                                    .format(list_author[7], list_author[6]))
            self.string_list.append('|{}'.format(list_author[8]))
        self.string_list.append('|}')
        self.string_list.append('')
        self.string_list.append('== Anmerkungen ==')
        self.string_list.append('<references/>')
        self.string_list.append('')
        self.string_list.append('{{SORTIERUNG: Autoren  # Liste der}}')
        self.string_list.append('[[Kategorie:Listen]]')
        self.string_list.append('[[Kategorie:Autoren |!]]')

        return '\n'.join(self.string_list)

    def _handle_birth_and_death(self, event, author_dict):
        if author_dict[event] == '' or self.match_property.search(author_dict[event]):
            self.logger.debug('No valid entry in {} for [[{}]] ... Fallback to wikidata'.format(event, author_dict['title']))
            try:
                item = ItemPage(self.repo, author_dict['wikidata'])
                if event == 'birth':
                    property_label = 'P569'
                else:
                    property_label = 'P570'
                claim = item.text['claims'][property_label][0]
                date_from_data = claim.getTarget()
                if date_from_data.precision < 7:
                    self.logger.error('Precison is to low for [[{}]]'.format(author_dict['title']))
                    raise
                elif date_from_data.precision < 8:
                    date_from_data = int(ceil(float(date_from_data.year) / 100.0) * 100)
                    if date_from_data < 1000:
                        date_from_data = str(date_from_data)[0:1] + '. Jh.'
                    else:
                        date_from_data = str(date_from_data)[0:2] + '. Jh.'
                elif date_from_data.precision < 10:
                    date_from_data = str(date_from_data.year)
                elif date_from_data.precision < 11:
                    date_from_data = self.number_to_month[date_from_data.month] + ' ' + str(date_from_data.year)
                else:
                    date_from_data = str(date_from_data.day) + '. ' + self.number_to_month[date_from_data.month] + ' ' + str(date_from_data.year)
                if re.search('-', date_from_data):
                    date_from_data = date_from_data.replace('-', '') + ' v. Chr.'
                self.logger.debug('Found {} @ wikidata for {}'.format(date_from_data, event))
                return(date_from_data)  # 4,6
            except:
                self.logger.debug("Wasn't able to ge any data from wikidata")
                return('')  # 4,6
        else:
            return(author_dict[event])  # 4,6<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import contextlib
import json
import optparse
import os
import shutil
import subprocess
import sys
import tempfile

BUILD_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(
    os.path.abspath(__file__))))
sys.path.append(os.path.join(BUILD_ROOT, 'scripts'))
sys.path.append(os.path.join(BUILD_ROOT, 'third_party'))

from common import annotator
from common import chromium_utils
from common import master_cfg_utils

SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
BUILD_LIMITED_ROOT = os.path.join(
    os.path.dirname(BUILD_ROOT), 'build_internal', 'scripts', 'slave')

PACKAGE_CFG = os.path.join(
    os.path.dirname(os.path.dirname(SCRIPT_PATH)),
    'infra', 'config', 'recipes.cfg')

@contextlib.contextmanager
def namedTempFile():
  fd, name = tempfile.mkstemp()
  os.close(fd)  # let the exceptions fly
  try:
    yield name
  finally:
    try:
      os.remove(name)
    except OSError as e:
      print >> sys.stderr, "LEAK: %s: %s" % (name, e)

def get_recipe_properties(factory_properties, build_properties,
                          master_overrides_slave):
  """Constructs the recipe's properties from buildbot's properties.

  This retrieves the current factory properties from the master_config
  in the slave's checkout (the factory properties handed to us from the
  master might be out of date), and merges in the build properties.

  Using the values from the checkout allows us to do things like change
  the recipe and other factory properties for a builder without needing
  a master restart.
  """
  master_properties = factory_properties.copy()
  master_properties.update(build_properties)

  mastername = master_properties.get('mastername')
  buildername = master_properties.get('buildername')
  slave_properties = {}
  if mastername and buildername:
    try:
      slave_properties = get_factory_properties_from_disk(
          mastername, buildername)
    except LookupError as e:
      if master_overrides_slave:
        print 'WARNING in annotated_run.py (non-fatal): %s' % e
      else:
        raise e

  properties = master_properties.copy()
  conflicting_properties = {}
  for name in slave_properties:
    if master_properties.get(name) != slave_properties[name]:
      conflicting_properties[name] = (master_properties.get(name),
                                   slave_properties[name])

  if conflicting_properties:
    print 'The following build properties differ between master and slave:'
    for name, (master_value, slave_value) in conflicting_properties.items():
      print ('  "%s": master: "%s", slave: "%s"' % (
          name,
          "<unset>" if (master_value is None) else master_value,
          slave_value))
    print ("Using the values from the %s." %
           ("master" if master_overrides_slave else "slave"))

  if not master_overrides_slave:
    for name, (_, slave_value) in conflicting_properties.items():
      properties[name] = slave_value

  return properties

def get_factory_properties_from_disk(mastername, buildername):
  master_list = master_cfg_utils.GetMasters()
  master_path = None
  for name, path in master_list:
    if name == mastername:
      master_path = path

  if not master_path:
    raise LookupError('master "%s" not found.' % mastername)

  script_path = os.path.join(BUILD_ROOT, 'scripts', 'tools',
                             'dump_master_cfg.py')

  with namedTempFile() as fname:
    dump_cmd = [sys.executable,
                script_path,
                master_path, fname]
    proc = subprocess.Popen(dump_cmd, cwd=BUILD_ROOT, stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE)
    out, err = proc.communicate()
    exit_code = proc.returncode

    if exit_code:
      raise LookupError('Failed to get the master config; dump_master_cfg %s'
                        'returned %d):\n%s\n%s\n'% (
                        mastername, exit_code, out, err))

    with open(fname, 'rU') as f:
      config = json.load(f)

  # Now extract just the factory properties for the requested builder
  # from the master config.
  props = {}
  found = False
  for builder_dict in config['builders']:
    if builder_dict['name'] == buildername:
      found = True
      factory_properties = builder_dict['factory']['properties']
      for name, (value, _) in factory_properties.items():
        props[name] = value

  if not found:
    raise LookupError('builder "%s" not found on in master "%s"' %
                      (buildername, mastername))

  if 'recipe' not in props:
    raise LookupError('Cannot find recipe for %s on %s' %
                      (buildername, mastername))

  return props

def get_args(argv):
  """Process command-line arguments."""

  parser = optparse.OptionParser(
      description='Entry point for annotated builds.')
  parser.add_option('--build-properties',
                    action='callback', callback=chromium_utils.convert_json,
                    type='string', default={},
                    help='build properties in JSON format')
  parser.add_option('--factory-properties',
                    action='callback', callback=chromium_utils.convert_json,
                    type='string', default={},
                    help='factory properties in JSON format')
  parser.add_option('--build-properties-gz',
                    action='callback', callback=chromium_utils.convert_gz_json,
                    type='string', default={}, dest='build_properties',
                    help='build properties in b64 gz JSON format')
  parser.add_option('--factory-properties-gz',
                    action='callback', callback=chromium_utils.convert_gz_json,
                    type='string', default={}, dest='factory_properties',
                    help='factory properties in b64 gz JSON format')
  parser.add_option('--keep-stdin', action='store_true', default=False,
                    help='don\'t close stdin when running recipe steps')
  parser.add_option('--master-overrides-slave', action='store_true',
                    help='use the property values given on the command line '
                         'from the master, not the ones looked up on the slave')
  return parser.parse_args(argv)

def update_scripts():
  if os.environ.get('RUN_SLAVE_UPDATED_SCRIPTS'):
    os.environ.pop('RUN_SLAVE_UPDATED_SCRIPTS')
    return False

  stream = annotator.StructuredAnnotationStream()

  with stream.step('update_scripts') as s:
    gclient_name = 'gclient'
    if sys.platform.startswith('win'):
      gclient_name += '.bat'
    gclient_path = os.path.join(BUILD_ROOT, '..', 'depot_tools', gclient_name)
    gclient_cmd = [gclient_path, 'sync', '--force', '--verbose']
    try:
      fd, output_json = tempfile.mkstemp()
      os.close(fd)
      gclient_cmd += ['--output-json', output_json]
    except Exception:
      # Super paranoia try block.
      output_json = None
    cmd_dict = {
        'name': 'update_scripts',
        'cmd': gclient_cmd,
        'cwd': BUILD_ROOT,
    }
    annotator.print_step(cmd_dict, os.environ, stream)
    if subprocess.call(gclient_cmd, cwd=BUILD_ROOT) != 0:
      s.step_text('gclient sync failed!')
      s.step_warnings()
    elif output_json:
      try:
        with open(output_json, 'r') as f:
          gclient_json = json.load(f)
        for line in json.dumps(
            gclient_json, sort_keys=True,
            indent=4, separators=(',', ': ')).splitlines():
          s.step_log_line('gclient_json', line)
        s.step_log_end('gclient_json')
        revision = gclient_json['solutions']['build/']['revision']
        scm = gclient_json['solutions']['build/']['scm']
        s.step_text('%s - %s' % (scm, revision))
        s.set_build_property('build_scm', json.dumps(scm))
        s.set_build_property('build_revision', json.dumps(revision))
      except Exception as e:
        s.step_text('Unable to process gclient JSON %s' % repr(e))
        s.step_warnings()
      finally:
        try:
          os.remove(output_json)
        except Exception as e:
          print >> sys.stderr, "LEAKED:", output_json, e
    else:
      s.step_text('Unable to get SCM data')
      s.step_warnings()

    os.environ['RUN_SLAVE_UPDATED_SCRIPTS'] = '1'

    # After running update scripts, set PYTHONIOENCODING=UTF-8 for the real
    # annotated_run.
    os.environ['PYTHONIOENCODING'] = 'UTF-8'

    return True

def clean_old_recipe_engine():
  """Clean stale pycs from the old location of recipe_engine.

  This function should only be needed for a little while after the recipe
  packages rollout (2015-09-16).
  """
  for (dirpath, _, filenames) in os.walk(
      os.path.join(BUILD_ROOT, 'third_party', 'recipe_engine')):
    for filename in filenames:
      if filename.endswith('.pyc'):
        path = os.path.join(dirpath, filename)
        os.remove(path)

@contextlib.contextmanager
def build_data_directory():
  """Context manager that creates a build-specific directory.

  The directory is wiped when exiting.

  Yields:
    build_data (str or None): full path to a writeable directory. Return None if
        no directory can be found or if it's not writeable.
  """
  prefix = 'build_data'

  # TODO(pgervais): import that from infra_libs.logs instead
  if sys.platform.startswith('win'):  # pragma: no cover
    DEFAULT_LOG_DIRECTORIES = [
      'E:\\chrome-infra-logs',
      'C:\\chrome-infra-logs',
    ]
  else:
    DEFAULT_LOG_DIRECTORIES = ['/var/log/chrome-infra']

  build_data_dir = None
  for candidate in DEFAULT_LOG_DIRECTORIES:
    if os.path.isdir(candidate):
      build_data_dir = os.path.join(candidate, prefix)
      break

  # Remove any leftovers and recreate the dir.
  if build_data_dir:
    print >> sys.stderr, "Creating directory"
    # TODO(pgervais): use infra_libs.rmtree instead.
    if os.path.exists(build_data_dir):
      try:
        shutil.rmtree(build_data_dir)
      except Exception as exc:
        # Catching everything: we don't want to break any builds for that reason
        print >> sys.stderr, (
          "FAILURE: path can't be deleted: %s.\n%s" % (build_data_dir, str(exc))
        )
    print >> sys.stderr, "Creating directory"

    if not os.path.exists(build_data_dir):
      try:
        os.mkdir(build_data_dir)
      except Exception as exc:
        print >> sys.stderr, (
          "FAILURE: directory can't be created: %s.\n%s" %
          (build_data_dir, str(exc))
        )
        build_data_dir = None

  # Under this line build_data_dir should point to an existing empty dir
  # or be None.
  yield build_data_dir

  # Clean up after ourselves
  if build_data_dir:
    # TODO(pgervais): use infra_libs.rmtree instead.
    try:
      shutil.rmtree(build_data_dir)
    except Exception as exc:
      # Catching everything: we don't want to break any builds for that reason.
      print >> sys.stderr, (
        "FAILURE: path can't be deleted: %s.\n%s" % (build_data_dir, str(exc))
      )

def main(argv):
  opts, _ = get_args(argv)
  properties = get_recipe_properties(
      opts.factory_properties, opts.build_properties,
      opts.master_overrides_slave)

  clean_old_recipe_engine()

  # Find out if the recipe we intend to run is in build_internal's recipes. If
  # so, use recipes.py from there, otherwise use the one from build.
  recipe_file = properties['recipe'].replace('/', os.path.sep) + '.py'
  if os.path.exists(os.path.join(BUILD_LIMITED_ROOT, 'recipes', recipe_file)):
    recipe_runner = os.path.join(BUILD_LIMITED_ROOT, 'recipes.py')
  else:
    recipe_runner = os.path.join(SCRIPT_PATH, 'recipes.py')

  with build_data_directory() as build_data_dir:
    if build_data_dir:
      os.environ['BUILD_DATA_DIR'] = build_data_dir
    with namedTempFile() as props_file:
      with open(props_file, 'w') as fh:
        fh.write(json.dumps(properties))
      cmd = [
          sys.executable, '-u', recipe_runner,
          'run',
          '--workdir=%s' % os.getcwd(),
          '--properties-file=%s' % props_file,
          properties['recipe'] ]
      status = subprocess.call(cmd)

    # TODO(pgervais): Send events from build_data_dir to the endpoint.
  return status

def shell_main(argv):
  if update_scripts():
    return subprocess.call([sys.executable] + argv)
  else:
    return main(argv)

if __name__ == '__main__':
  sys.exit(shell_main(sys.argv))
<commit_msg>Reland of https://chromiumcodereview.appspot.com/1405393003/<commit_after>#!/usr/bin/env python
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import contextlib
import json
import optparse
import os
import shutil
import subprocess
import sys
import tempfile
import traceback

BUILD_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(
    os.path.abspath(__file__))))
sys.path.append(os.path.join(BUILD_ROOT, 'scripts'))
sys.path.append(os.path.join(BUILD_ROOT, 'third_party'))

from common import annotator
from common import chromium_utils
from common import master_cfg_utils

SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
BUILD_LIMITED_ROOT = os.path.join(
    os.path.dirname(BUILD_ROOT), 'build_internal', 'scripts', 'slave')

PACKAGE_CFG = os.path.join(
    os.path.dirname(os.path.dirname(SCRIPT_PATH)),
    'infra', 'config', 'recipes.cfg')

if sys.platform.startswith('win'):
  # TODO(pgervais): add windows support
  # QQ: Where is infra/run.py on windows machines?
  RUN_CMD = None
else:
  RUN_CMD = os.path.join('/', 'opt', 'infra-python', 'run.py')

@contextlib.contextmanager
def namedTempFile():
  fd, name = tempfile.mkstemp()
  os.close(fd)  # let the exceptions fly
  try:
    yield name
  finally:
    try:
      os.remove(name)
    except OSError as e:
      print >> sys.stderr, "LEAK: %s: %s" % (name, e)

def get_recipe_properties(factory_properties, build_properties,
                          master_overrides_slave):
  """Constructs the recipe's properties from buildbot's properties.

  This retrieves the current factory properties from the master_config
  in the slave's checkout (the factory properties handed to us from the
  master might be out of date), and merges in the build properties.

  Using the values from the checkout allows us to do things like change
  the recipe and other factory properties for a builder without needing
  a master restart.
  """
  master_properties = factory_properties.copy()
  master_properties.update(build_properties)

  mastername = master_properties.get('mastername')
  buildername = master_properties.get('buildername')
  slave_properties = {}
  if mastername and buildername:
    try:
      slave_properties = get_factory_properties_from_disk(
          mastername, buildername)
    except LookupError as e:
      if master_overrides_slave:
        print 'WARNING in annotated_run.py (non-fatal): %s' % e
      else:
        raise e

  properties = master_properties.copy()
  conflicting_properties = {}
  for name in slave_properties:
    if master_properties.get(name) != slave_properties[name]:
      conflicting_properties[name] = (master_properties.get(name),
                                   slave_properties[name])

  if conflicting_properties:
    print 'The following build properties differ between master and slave:'
    for name, (master_value, slave_value) in conflicting_properties.items():
      print ('  "%s": master: "%s", slave: "%s"' % (
          name,
          "<unset>" if (master_value is None) else master_value,
          slave_value))
    print ("Using the values from the %s." %
           ("master" if master_overrides_slave else "slave"))

  if not master_overrides_slave:
    for name, (_, slave_value) in conflicting_properties.items():
      properties[name] = slave_value

  return properties

def get_factory_properties_from_disk(mastername, buildername):
  master_list = master_cfg_utils.GetMasters()
  master_path = None
  for name, path in master_list:
    if name == mastername:
      master_path = path

  if not master_path:
    raise LookupError('master "%s" not found.' % mastername)

  script_path = os.path.join(BUILD_ROOT, 'scripts', 'tools',
                             'dump_master_cfg.py')

  with namedTempFile() as fname:
    dump_cmd = [sys.executable,
                script_path,
                master_path, fname]
    proc = subprocess.Popen(dump_cmd, cwd=BUILD_ROOT, stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE)
    out, err = proc.communicate()
    exit_code = proc.returncode

    if exit_code:
      raise LookupError('Failed to get the master config; dump_master_cfg %s'
                        'returned %d):\n%s\n%s\n'% (
                        mastername, exit_code, out, err))

    with open(fname, 'rU') as f:
      config = json.load(f)

  # Now extract just the factory properties for the requested builder
  # from the master config.
  props = {}
  found = False
  for builder_dict in config['builders']:
    if builder_dict['name'] == buildername:
      found = True
      factory_properties = builder_dict['factory']['properties']
      for name, (value, _) in factory_properties.items():
        props[name] = value

  if not found:
    raise LookupError('builder "%s" not found on in master "%s"' %
                      (buildername, mastername))

  if 'recipe' not in props:
    raise LookupError('Cannot find recipe for %s on %s' %
                      (buildername, mastername))

  return props

def get_args(argv):
  """Process command-line arguments."""

  parser = optparse.OptionParser(
      description='Entry point for annotated builds.')
  parser.add_option('--build-properties',
                    action='callback', callback=chromium_utils.convert_json,
                    type='string', default={},
                    help='build properties in JSON format')
  parser.add_option('--factory-properties',
                    action='callback', callback=chromium_utils.convert_json,
                    type='string', default={},
                    help='factory properties in JSON format')
  parser.add_option('--build-properties-gz',
                    action='callback', callback=chromium_utils.convert_gz_json,
                    type='string', default={}, dest='build_properties',
                    help='build properties in b64 gz JSON format')
  parser.add_option('--factory-properties-gz',
                    action='callback', callback=chromium_utils.convert_gz_json,
                    type='string', default={}, dest='factory_properties',
                    help='factory properties in b64 gz JSON format')
  parser.add_option('--keep-stdin', action='store_true', default=False,
                    help='don\'t close stdin when running recipe steps')
  parser.add_option('--master-overrides-slave', action='store_true',
                    help='use the property values given on the command line '
                         'from the master, not the ones looked up on the slave')
  return parser.parse_args(argv)

def update_scripts():
  if os.environ.get('RUN_SLAVE_UPDATED_SCRIPTS'):
    os.environ.pop('RUN_SLAVE_UPDATED_SCRIPTS')
    return False

  stream = annotator.StructuredAnnotationStream()

  with stream.step('update_scripts') as s:
    gclient_name = 'gclient'
    if sys.platform.startswith('win'):
      gclient_name += '.bat'
    gclient_path = os.path.join(BUILD_ROOT, '..', 'depot_tools', gclient_name)
    gclient_cmd = [gclient_path, 'sync', '--force', '--verbose']
    try:
      fd, output_json = tempfile.mkstemp()
      os.close(fd)
      gclient_cmd += ['--output-json', output_json]
    except Exception:
      # Super paranoia try block.
      output_json = None
    cmd_dict = {
        'name': 'update_scripts',
        'cmd': gclient_cmd,
        'cwd': BUILD_ROOT,
    }
    annotator.print_step(cmd_dict, os.environ, stream)
    if subprocess.call(gclient_cmd, cwd=BUILD_ROOT) != 0:
      s.step_text('gclient sync failed!')
      s.step_warnings()
    elif output_json:
      try:
        with open(output_json, 'r') as f:
          gclient_json = json.load(f)
        for line in json.dumps(
            gclient_json, sort_keys=True,
            indent=4, separators=(',', ': ')).splitlines():
          s.step_log_line('gclient_json', line)
        s.step_log_end('gclient_json')
        revision = gclient_json['solutions']['build/']['revision']
        scm = gclient_json['solutions']['build/']['scm']
        s.step_text('%s - %s' % (scm, revision))
        s.set_build_property('build_scm', json.dumps(scm))
        s.set_build_property('build_revision', json.dumps(revision))
      except Exception as e:
        s.step_text('Unable to process gclient JSON %s' % repr(e))
        s.step_warnings()
      finally:
        try:
          os.remove(output_json)
        except Exception as e:
          print >> sys.stderr, "LEAKED:", output_json, e
    else:
      s.step_text('Unable to get SCM data')
      s.step_warnings()

    os.environ['RUN_SLAVE_UPDATED_SCRIPTS'] = '1'

    # After running update scripts, set PYTHONIOENCODING=UTF-8 for the real
    # annotated_run.
    os.environ['PYTHONIOENCODING'] = 'UTF-8'

    return True

def clean_old_recipe_engine():
  """Clean stale pycs from the old location of recipe_engine.

  This function should only be needed for a little while after the recipe
  packages rollout (2015-09-16).
  """
  for (dirpath, _, filenames) in os.walk(
      os.path.join(BUILD_ROOT, 'third_party', 'recipe_engine')):
    for filename in filenames:
      if filename.endswith('.pyc'):
        path = os.path.join(dirpath, filename)
        os.remove(path)

@contextlib.contextmanager
def build_data_directory():
  """Context manager that creates a build-specific directory.

  The directory is wiped when exiting.

  Yields:
    build_data (str or None): full path to a writeable directory. Return None if
        no directory can be found or if it's not writeable.
  """
  prefix = 'build_data'

  # TODO(pgervais): import that from infra_libs.logs instead
  if sys.platform.startswith('win'):  # pragma: no cover
    DEFAULT_LOG_DIRECTORIES = [
      'E:\\chrome-infra-logs',
      'C:\\chrome-infra-logs',
    ]
  else:
    DEFAULT_LOG_DIRECTORIES = ['/var/log/chrome-infra']

  build_data_dir = None
  for candidate in DEFAULT_LOG_DIRECTORIES:
    if os.path.isdir(candidate):
      build_data_dir = os.path.join(candidate, prefix)
      break

  # Remove any leftovers and recreate the dir.
  if build_data_dir:
    print >> sys.stderr, "Creating directory"
    # TODO(pgervais): use infra_libs.rmtree instead.
    if os.path.exists(build_data_dir):
      try:
        shutil.rmtree(build_data_dir)
      except Exception as exc:
        # Catching everything: we don't want to break any builds for that reason
        print >> sys.stderr, (
          "FAILURE: path can't be deleted: %s.\n%s" % (build_data_dir, str(exc))
        )
    print >> sys.stderr, "Creating directory"

    if not os.path.exists(build_data_dir):
      try:
        os.mkdir(build_data_dir)
      except Exception as exc:
        print >> sys.stderr, (
          "FAILURE: directory can't be created: %s.\n%s" %
          (build_data_dir, str(exc))
        )
        build_data_dir = None

  # Under this line build_data_dir should point to an existing empty dir
  # or be None.
  yield build_data_dir

  # Clean up after ourselves
  if build_data_dir:
    # TODO(pgervais): use infra_libs.rmtree instead.
    try:
      shutil.rmtree(build_data_dir)
    except Exception as exc:
      # Catching everything: we don't want to break any builds for that reason.
      print >> sys.stderr, (
        "FAILURE: path can't be deleted: %s.\n%s" % (build_data_dir, str(exc))
      )

def main(argv):
  opts, _ = get_args(argv)
  properties = get_recipe_properties(
      opts.factory_properties, opts.build_properties,
      opts.master_overrides_slave)

  clean_old_recipe_engine()

  # Find out if the recipe we intend to run is in build_internal's recipes. If
  # so, use recipes.py from there, otherwise use the one from build.
  recipe_file = properties['recipe'].replace('/', os.path.sep) + '.py'
  if os.path.exists(os.path.join(BUILD_LIMITED_ROOT, 'recipes', recipe_file)):
    recipe_runner = os.path.join(BUILD_LIMITED_ROOT, 'recipes.py')
  else:
    recipe_runner = os.path.join(SCRIPT_PATH, 'recipes.py')

  with build_data_directory() as build_data_dir:
    # Create a LogRequestLite proto containing this build's information.
    if build_data_dir:
      properties['build_data_dir'] = build_data_dir

      if RUN_CMD and os.path.exists(RUN_CMD):
        try:
          subprocess.call(
            [RUN_CMD, 'infra.tools.send_monitoring_event',
             '--event-mon-output-file',
                 os.path.join(build_data_dir, 'log_request_proto'),
             '--event-mon-run-type', 'file',
             '--event-mon-service-name',
                 'buildbot/master/master.%s'
                 % properties.get('mastername', 'UNKNOWN'),
             '--build-event-build-name',
                 properties.get('buildername', 'UNKNOWN'),
             '--build-event-build-number',
                 str(properties.get('buildnumber', 0)),
             '--build-event-build-scheduling-time',
                 str(1000*int(properties.get('requestedAt', 0))),
             '--build-event-type', 'BUILD',
             '--event-mon-timestamp-kind', 'POINT',
             # And use only defaults for credentials.
           ])
        except Exception:
          print >> sys.stderr, traceback.format_exc()

      else:
        print >> sys.stderr, (
          'WARNING: Unable to find run.py at %r, no events will be sent.'
          % str(RUN_CMD)
        )

    with namedTempFile() as props_file:
      with open(props_file, 'w') as fh:
        fh.write(json.dumps(properties))
      cmd = [
          sys.executable, '-u', recipe_runner,
          'run',
          '--workdir=%s' % os.getcwd(),
          '--properties-file=%s' % props_file,
          properties['recipe'] ]
      status = subprocess.call(cmd)

    # TODO(pgervais): Send events from build_data_dir to the endpoint.
  return status

def shell_main(argv):
  if update_scripts():
    return subprocess.call([sys.executable] + argv)
  else:
    return main(argv)

if __name__ == '__main__':
  sys.exit(shell_main(sys.argv))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import contextlib
import json
import optparse
import os
import subprocess
import sys
import tempfile

BUILD_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(
    os.path.abspath(__file__))))
sys.path.append(os.path.join(BUILD_ROOT, 'scripts'))
sys.path.append(os.path.join(BUILD_ROOT, 'third_party'))

from common import annotator
from common import chromium_utils
from common import master_cfg_utils

SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
BUILD_LIMITED_ROOT = os.path.join(
    os.path.dirname(BUILD_ROOT), 'build_internal', 'scripts', 'slave')

PACKAGE_CFG = os.path.join(
    os.path.dirname(os.path.dirname(SCRIPT_PATH)),
    'infra', 'config', 'recipes.cfg')

@contextlib.contextmanager
def namedTempFile():
  fd, name = tempfile.mkstemp()
  os.close(fd)  # let the exceptions fly
  try:
    yield name
  finally:
    try:
      os.remove(name)
    except OSError as e:
      print >> sys.stderr, "LEAK: %s: %s" % (name, e)

def get_recipe_properties(factory_properties, build_properties,
                          master_overrides_slave):
  """Constructs the recipe's properties from buildbot's properties.

  This retrieves the current factory properties from the master_config
  in the slave's checkout (the factory properties handed to us from the
  master might be out of date), and merges in the build properties.

  Using the values from the checkout allows us to do things like change
  the recipe and other factory properties for a builder without needing
  a master restart.
  """
  master_properties = factory_properties.copy()
  master_properties.update(build_properties)

  mastername = master_properties.get('mastername')
  buildername = master_properties.get('buildername')
  slave_properties = {}
  if mastername and buildername:
    try:
      slave_properties = get_factory_properties_from_disk(
          mastername, buildername)
    except LookupError as e:
      if master_overrides_slave:
        print 'WARNING in annotated_run.py (non-fatal): %s' % e
      else:
        raise e

  properties = master_properties.copy()
  conflicting_properties = {}
  for name in slave_properties:
    if master_properties.get(name) != slave_properties[name]:
      conflicting_properties[name] = (master_properties.get(name),
                                   slave_properties[name])

  if conflicting_properties:
    print 'The following build properties differ between master and slave:'
    for name, (master_value, slave_value) in conflicting_properties.items():
      print ('  "%s": master: "%s", slave: "%s"' % (
          name,
          "<unset>" if (master_value is None) else master_value,
          slave_value))
    print ("Using the values from the %s." %
           ("master" if master_overrides_slave else "slave"))

  if not master_overrides_slave:
    for name, (_, slave_value) in conflicting_properties.items():
      properties[name] = slave_value

  return properties

def get_factory_properties_from_disk(mastername, buildername):
  master_list = master_cfg_utils.GetMasters()
  master_path = None
  for name, path in master_list:
    if name == mastername:
      master_path = path

  if not master_path:
    raise LookupError('master "%s" not found.' % mastername)

  script_path = os.path.join(BUILD_ROOT, 'scripts', 'tools',
                             'dump_master_cfg.py')

  with namedTempFile() as fname:
    dump_cmd = [sys.executable,
                script_path,
                master_path, fname]
    proc = subprocess.Popen(dump_cmd, cwd=BUILD_ROOT, stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE)
    out, err = proc.communicate()
    exit_code = proc.returncode

    if exit_code:
      raise LookupError('Failed to get the master config; dump_master_cfg %s'
                        'returned %d):\n%s\n%s\n'% (
                        mastername, exit_code, out, err))

    with open(fname, 'rU') as f:
      config = json.load(f)

  # Now extract just the factory properties for the requested builder
  # from the master config.
  props = {}
  found = False
  for builder_dict in config['builders']:
    if builder_dict['name'] == buildername:
      found = True
      factory_properties = builder_dict['factory']['properties']
      for name, (value, _) in factory_properties.items():
        props[name] = value

  if not found:
    raise LookupError('builder "%s" not found on in master "%s"' %
                      (buildername, mastername))

  if 'recipe' not in props:
    raise LookupError('Cannot find recipe for %s on %s' %
                      (buildername, mastername))

  return props

def get_args(argv):
  """Process command-line arguments."""

  parser = optparse.OptionParser(
      description='Entry point for annotated builds.')
  parser.add_option('--build-properties',
                    action='callback', callback=chromium_utils.convert_json,
                    type='string', default={},
                    help='build properties in JSON format')
  parser.add_option('--factory-properties',
                    action='callback', callback=chromium_utils.convert_json,
                    type='string', default={},
                    help='factory properties in JSON format')
  parser.add_option('--build-properties-gz',
                    action='callback', callback=chromium_utils.convert_gz_json,
                    type='string', default={}, dest='build_properties',
                    help='build properties in b64 gz JSON format')
  parser.add_option('--factory-properties-gz',
                    action='callback', callback=chromium_utils.convert_gz_json,
                    type='string', default={}, dest='factory_properties',
                    help='factory properties in b64 gz JSON format')
  parser.add_option('--keep-stdin', action='store_true', default=False,
                    help='don\'t close stdin when running recipe steps')
  parser.add_option('--master-overrides-slave', action='store_true',
                    help='use the property values given on the command line '
                         'from the master, not the ones looked up on the slave')
  return parser.parse_args(argv)

def update_scripts():
  if os.environ.get('RUN_SLAVE_UPDATED_SCRIPTS'):
    os.environ.pop('RUN_SLAVE_UPDATED_SCRIPTS')
    return False

  stream = annotator.StructuredAnnotationStream()

  with stream.step('update_scripts') as s:
    gclient_name = 'gclient'
    if sys.platform.startswith('win'):
      gclient_name += '.bat'
    gclient_path = os.path.join(BUILD_ROOT, '..', 'depot_tools', gclient_name)
    gclient_cmd = [gclient_path, 'sync', '--force', '--verbose']
    cmd_dict = {
        'name': 'update_scripts',
        'cmd': gclient_cmd,
        'cwd': BUILD_ROOT,
    }
    annotator.print_step(cmd_dict, os.environ, stream)
    if subprocess.call(gclient_cmd, cwd=BUILD_ROOT) != 0:
      s.step_text('gclient sync failed!')
      s.step_warnings()
    os.environ['RUN_SLAVE_UPDATED_SCRIPTS'] = '1'

    # After running update scripts, set PYTHONIOENCODING=UTF-8 for the real
    # annotated_run.
    os.environ['PYTHONIOENCODING'] = 'UTF-8'

    return True

def clean_old_recipe_engine():
  """Clean stale pycs from the old location of recipe_engine.

  This function should only be needed for a little while after the recipe
  packages rollout (2015-09-16).
  """
  for (dirpath, _, filenames) in os.walk(
      os.path.join(BUILD_ROOT, 'third_party', 'recipe_engine')):
    for filename in filenames:
      if filename.endswith('.pyc'):
        path = os.path.join(dirpath, filename)
        os.remove(path)

def main(argv):
  opts, _ = get_args(argv)
  properties = get_recipe_properties(
      opts.factory_properties, opts.build_properties,
      opts.master_overrides_slave)

  clean_old_recipe_engine()

  # Find out if the recipe we intend to run is in build_internal's recipes. If
  # so, use recipes.py from there, otherwise use the one from build.
  recipe_file = properties['recipe'].replace('/', os.path.sep) + '.py'
  if os.path.exists(os.path.join(BUILD_LIMITED_ROOT, 'recipes', recipe_file)):
    recipe_runner = os.path.join(BUILD_LIMITED_ROOT, 'recipes.py')
  else:
    recipe_runner = os.path.join(SCRIPT_PATH, 'recipes.py')

  with namedTempFile() as props_file:
    with open(props_file, 'w') as fh:
      fh.write(json.dumps(properties))
    cmd = [
        sys.executable, '-u', recipe_runner,
        'run',
        '--workdir=%s' % os.getcwd(),
        '--properties-file=%s' % props_file,
        properties['recipe'] ]
    return subprocess.call(cmd)

def shell_main(argv):
  if update_scripts():
    return subprocess.call([sys.executable] + argv)
  else:
    return main(argv)

if __name__ == '__main__':
  sys.exit(shell_main(sys.argv))
<commit_msg>Revert of Clean old recipe engine (patchset #1 id:1 of https://codereview.chromium.org/1349203002/ )<commit_after>#!/usr/bin/env python
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import contextlib
import json
import optparse
import os
import subprocess
import sys
import tempfile

BUILD_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(
    os.path.abspath(__file__))))
sys.path.append(os.path.join(BUILD_ROOT, 'scripts'))
sys.path.append(os.path.join(BUILD_ROOT, 'third_party'))

from common import annotator
from common import chromium_utils
from common import master_cfg_utils

SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
BUILD_LIMITED_ROOT = os.path.join(
    os.path.dirname(BUILD_ROOT), 'build_internal', 'scripts', 'slave')

PACKAGE_CFG = os.path.join(
    os.path.dirname(os.path.dirname(SCRIPT_PATH)),
    'infra', 'config', 'recipes.cfg')

@contextlib.contextmanager
def namedTempFile():
  fd, name = tempfile.mkstemp()
  os.close(fd)  # let the exceptions fly
  try:
    yield name
  finally:
    try:
      os.remove(name)
    except OSError as e:
      print >> sys.stderr, "LEAK: %s: %s" % (name, e)

def get_recipe_properties(factory_properties, build_properties,
                          master_overrides_slave):
  """Constructs the recipe's properties from buildbot's properties.

  This retrieves the current factory properties from the master_config
  in the slave's checkout (the factory properties handed to us from the
  master might be out of date), and merges in the build properties.

  Using the values from the checkout allows us to do things like change
  the recipe and other factory properties for a builder without needing
  a master restart.
  """
  master_properties = factory_properties.copy()
  master_properties.update(build_properties)

  mastername = master_properties.get('mastername')
  buildername = master_properties.get('buildername')
  slave_properties = {}
  if mastername and buildername:
    try:
      slave_properties = get_factory_properties_from_disk(
          mastername, buildername)
    except LookupError as e:
      if master_overrides_slave:
        print 'WARNING in annotated_run.py (non-fatal): %s' % e
      else:
        raise e

  properties = master_properties.copy()
  conflicting_properties = {}
  for name in slave_properties:
    if master_properties.get(name) != slave_properties[name]:
      conflicting_properties[name] = (master_properties.get(name),
                                   slave_properties[name])

  if conflicting_properties:
    print 'The following build properties differ between master and slave:'
    for name, (master_value, slave_value) in conflicting_properties.items():
      print ('  "%s": master: "%s", slave: "%s"' % (
          name,
          "<unset>" if (master_value is None) else master_value,
          slave_value))
    print ("Using the values from the %s." %
           ("master" if master_overrides_slave else "slave"))

  if not master_overrides_slave:
    for name, (_, slave_value) in conflicting_properties.items():
      properties[name] = slave_value

  return properties

def get_factory_properties_from_disk(mastername, buildername):
  master_list = master_cfg_utils.GetMasters()
  master_path = None
  for name, path in master_list:
    if name == mastername:
      master_path = path

  if not master_path:
    raise LookupError('master "%s" not found.' % mastername)

  script_path = os.path.join(BUILD_ROOT, 'scripts', 'tools',
                             'dump_master_cfg.py')

  with namedTempFile() as fname:
    dump_cmd = [sys.executable,
                script_path,
                master_path, fname]
    proc = subprocess.Popen(dump_cmd, cwd=BUILD_ROOT, stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE)
    out, err = proc.communicate()
    exit_code = proc.returncode

    if exit_code:
      raise LookupError('Failed to get the master config; dump_master_cfg %s'
                        'returned %d):\n%s\n%s\n'% (
                        mastername, exit_code, out, err))

    with open(fname, 'rU') as f:
      config = json.load(f)

  # Now extract just the factory properties for the requested builder
  # from the master config.
  props = {}
  found = False
  for builder_dict in config['builders']:
    if builder_dict['name'] == buildername:
      found = True
      factory_properties = builder_dict['factory']['properties']
      for name, (value, _) in factory_properties.items():
        props[name] = value

  if not found:
    raise LookupError('builder "%s" not found on in master "%s"' %
                      (buildername, mastername))

  if 'recipe' not in props:
    raise LookupError('Cannot find recipe for %s on %s' %
                      (buildername, mastername))

  return props

def get_args(argv):
  """Process command-line arguments."""

  parser = optparse.OptionParser(
      description='Entry point for annotated builds.')
  parser.add_option('--build-properties',
                    action='callback', callback=chromium_utils.convert_json,
                    type='string', default={},
                    help='build properties in JSON format')
  parser.add_option('--factory-properties',
                    action='callback', callback=chromium_utils.convert_json,
                    type='string', default={},
                    help='factory properties in JSON format')
  parser.add_option('--build-properties-gz',
                    action='callback', callback=chromium_utils.convert_gz_json,
                    type='string', default={}, dest='build_properties',
                    help='build properties in b64 gz JSON format')
  parser.add_option('--factory-properties-gz',
                    action='callback', callback=chromium_utils.convert_gz_json,
                    type='string', default={}, dest='factory_properties',
                    help='factory properties in b64 gz JSON format')
  parser.add_option('--keep-stdin', action='store_true', default=False,
                    help='don\'t close stdin when running recipe steps')
  parser.add_option('--master-overrides-slave', action='store_true',
                    help='use the property values given on the command line '
                         'from the master, not the ones looked up on the slave')
  return parser.parse_args(argv)

def update_scripts():
  if os.environ.get('RUN_SLAVE_UPDATED_SCRIPTS'):
    os.environ.pop('RUN_SLAVE_UPDATED_SCRIPTS')
    return False

  stream = annotator.StructuredAnnotationStream()

  with stream.step('update_scripts') as s:
    gclient_name = 'gclient'
    if sys.platform.startswith('win'):
      gclient_name += '.bat'
    gclient_path = os.path.join(BUILD_ROOT, '..', 'depot_tools', gclient_name)
    gclient_cmd = [gclient_path, 'sync', '--force', '--verbose']
    cmd_dict = {
        'name': 'update_scripts',
        'cmd': gclient_cmd,
        'cwd': BUILD_ROOT,
    }
    annotator.print_step(cmd_dict, os.environ, stream)
    if subprocess.call(gclient_cmd, cwd=BUILD_ROOT) != 0:
      s.step_text('gclient sync failed!')
      s.step_warnings()
    os.environ['RUN_SLAVE_UPDATED_SCRIPTS'] = '1'

    # After running update scripts, set PYTHONIOENCODING=UTF-8 for the real
    # annotated_run.
    os.environ['PYTHONIOENCODING'] = 'UTF-8'

    return True

def main(argv):
  opts, _ = get_args(argv)
  properties = get_recipe_properties(
      opts.factory_properties, opts.build_properties,
      opts.master_overrides_slave)

  # Find out if the recipe we intend to run is in build_internal's recipes. If
  # so, use recipes.py from there, otherwise use the one from build.
  recipe_file = properties['recipe'].replace('/', os.path.sep) + '.py'
  if os.path.exists(os.path.join(BUILD_LIMITED_ROOT, 'recipes', recipe_file)):
    recipe_runner = os.path.join(BUILD_LIMITED_ROOT, 'recipes.py')
  else:
    recipe_runner = os.path.join(SCRIPT_PATH, 'recipes.py')

  with namedTempFile() as props_file:
    with open(props_file, 'w') as fh:
      fh.write(json.dumps(properties))
    cmd = [
        sys.executable, '-u', recipe_runner,
        'run',
        '--workdir=%s' % os.getcwd(),
        '--properties-file=%s' % props_file,
        properties['recipe'] ]
    return subprocess.call(cmd)

def shell_main(argv):
  if update_scripts():
    return subprocess.call([sys.executable] + argv)
  else:
    return main(argv)

if __name__ == '__main__':
  sys.exit(shell_main(sys.argv))
<|endoftext|>


<commit_before>import django
django.setup()

from rdrf.models import ContextFormGroup
from rdrf.models import RDRFContext
from rdrf.models import Registry
from rdrf.mongo_client import construct_mongo_client

from registry.patients.models import Patient
from django.contrib.contenttypes.models import ContentType

class Logger(object):
    def __init__(self, patient_model):
        self.patient_model = patient_model
        self.prefix = "Patient %s (id=%s)" % (self.patient_model, self.patient_model.pk)

    def info(self, msg):
        print "%s: %s" % (self.prefix, msg)

    def error(self, msg):
        print "ERROR! %s: %s" % (self.prefix, msg)

class ScriptError(Exception):
    pass

class FHRecordSplitter(object):
    def __init__(self, registry_model, mongo_db_name):
        self.backup_data = {}

        self.logger = None
        self.mongo_db_name = mongo_db_name
        self.registry_model = registry_model
        if self.registry_model.code != 'fh':
            raise ScriptError("This script only works with FH Registry!")

        self.checkup_form_group_name = "Checkup"

        try:
            self.checkup_context_form_group = ContextFormGroup.objects.get(registry=self.registry_model,
                                                name=self.checkup_form_group_name)
        except ContextFormGroup.DoesNotExist:
            raise ScriptError("Checkup form group does not exist - are you sure you have imported the correct FH yaml file first")

        try:
            self.main_cfg = ContextFormGroup.objects.get(registry=self.registry_model,
                                           name="Main")
        except ContextFormGroup.DoesNotExist:
            raise ScriptError("No Main context form group: Make sure latest FH yaml is imported first")

        try:
            self.mongo_client = construct_mongo_client()
        except Exception, ex:
            raise ScriptError("could not create mongo client: %s" % ex)

        self.mongo_db = self.mongo_client[self.mongo_db_name]
        self.cdes_collection = self.mongo_db["cdes"]

    def _sanity_check_main_record(self, record):
        if record:
            form_names = [form["name"] for form in record["forms"]]
            assert "FollowUp" not in form_names, "FollowUp form should not be in a main record"
            assert "FollowUp_timestamp" not in record, "FollowU_timestamp should not be in a main record"
            assert "context_id" in record, "Main record should have a context_id"
            try:
                context_model = RDRFContext.objects.get(id=record["context_id"])
            except RDRFContext.DoesNotExist:
                raise ScriptError("mentioned context does not exist in record %s" % record)

            assert context_model.object_id == record["django_id"], "Linked context not correct patient"

            assert context_model.context_form_group is not None, "Main context should be linked to the Main Context form group"
            assert context_model.context_form_group.name == "Main"
            assert context_model.context_form_group.registry.code == self.registry_model.code, "Context Form Group should linked to FH"

    def rollback(self):
        pass

    def _sanity_check_followup_record(self, record):
        num_forms = len(record["forms"])
        assert len(num_forms) == 1, "There should be one FollowUp form in the record"
        form = record["forms"][0]
        assert form["name"] == "FollowUp", "The single form in a CheckUp context should be called FollowUp"
        assert "FollowUp_timestamp" in record
        assert "context_id" in record, "Record should have a context_id"
        try:
            context_model = RDRFContext.objects.get(id=record["context_id"])
        except RDRFContext.DoesNotExist:
            raise ScriptError("Linked context id does  not exist for record %s" % record)

        assert context_model.object_id == record["django_id"], "Linked context not correct patient"
        assert context_model.context_form_group is not None, "A CheckUp context must be linked to a context form group"
        cfg = context_model.context_form_group
        assert cfg.name == "CheckUp", "The linked context form group of a FollowUp form should be called CheckUp"

    def run(self):
        for patient_model in Patient.objects.filter(rdrf_registry__in=[self.registry_model]):
            error = False
            self.logger = Logger(patient_model)
            self.logger.info("processing ...")

            # We can't use the normal "Dynamic Data Wrapper" in RDRF because this tries to retrieve by context id
            # is the registry changes from not using form groups to using form groups as here, the loadi

            records = [ data for data in self.cdes_collection.find({"django_model": "Patient",
                                                      "django_id": patient_model.pk})]

            if len(records) != 1:
                self.logger.info("There are %s mongo records - skipping!" % len(records))
                continue

            dynamic_data = records[0]

            self.backup_data[patient_model.pk] = dynamic_data

            followup_form = None
            index = None

            if dynamic_data is not None:
                self.logger.info("There is a mongo record for this patient")
                # Ensure that the existing context model is linked to the correct "Main" ( default) context form group
                context_id = dynamic_data["context_id"]
                self._link_main_context_form_group(context_id)

                # Locate the existing FollowUp form data and extract
                for i, form_dict in enumerate(dynamic_data["forms"]):
                    if form_dict["name"] == "FollowUp":
                        self.logger.info("There is a FollowUp form for this patient")
                        followup_form = form_dict
                        index = i

            if followup_form is not None:
                form_timestamp = dynamic_data.get("FollowUp_timestamp", None)
                self.logger.info("FollowUp form timestamp = %s" % form_timestamp)
                try:
                    del dynamic_data["forms"][index]
                except Exception, ex:
                    self.logger.error("could not delete old FollowUp form from record: %s" % ex)
                    self.logger.info("skipping this record ...")
                    continue

                self.logger.info("deleted old FollowUp form data from original record")
                try:
                    del dynamic_data["FollowUp_timestamp"]
                    self.logger.info("deleted old FollowUp timestamp")
                except Exception, ex:
                    self.logger.error("could not delete old FollowUp form timestamp: %s" % ex)
                    self.logger.info("skipping this record ...")
                    continue

                try:
                    followup_mongo_record = self._create_additional_mongo_record(patient_model, followup_form, form_timestamp)
                    self.logger.info("created new Checkup mongo record ok")
                except Exception, ex:
                    self.logger.error("could not create new Checkup mongo record: %s" % ex)
                    self.logger.info("skipping this patient")
                    continue

                # update the existing record
                try:
                    mongo_id = dynamic_data["_id"]
                    self.logger.info("existing mongo _id = %s" % mongo_id)
                except Exception, ex:
                    self.logger.error("could not find existing _id in record: %s" % ex)
                    self.logger.info("skipping this patient")
                    continue

                try:
                    self._sanity_check_record(dynamic_data)
                    self.cdes_collection.update({'_id': mongo_id}, {"$set": dynamic_data}, upsert=False)
                    self.logger.info("update successful")
                except Exception, ex:
                    self.logger.error("error updating the existing mongo record: %s" % ex)
                    error = True

                #insert new record for the new context
                try:
                    self._sanity_check_followup_record(followup_mongo_record)
                    self.cdes_collection.insert(followup_mongo_record)
                    self.logger.info("new CheckUp context created OK")
                except Exception, ex:
                    error = True
                    self.logger.error("failed to insert new followup form context: %s" % ex)

                if error:
                    self.logger.info("finished - this patient had errors")
                else:
                    self.logger.info("finished - successfully processed")

            else:
                self.logger.info("followUp form not found in mongo record")

    def _create_additional_mongo_record(self, patient_model, followup_form, form_timestamp):
        content_type = ContentType.objects.get_for_model(patient_model)
        rdrf_context = RDRFContext(registry=self.registry_model, content_object=patient_model)
        rdrf_context.context_form_group = self.checkup_context_form_group
        rdrf_context.save()
        new_mongo_record = {"forms": [followup_form],
                            "timestamp": form_timestamp,
                            "FollowUp_timestamp": form_timestamp,
                            "django_id": patient_model.pk,
                            "django_model": "Patient",
                            "context_id": rdrf_context.pk}

        return new_mongo_record

    def _link_main_context_form_group(self, existing_context_id):

        try:
            context_model = RDRFContext.objects.get(pk=existing_context_id)
        except RDRFContext.DoesNotExist:
            raise ScriptError("error getting context model for linking to Main context form group")

        context_model.context_form_group = self.main_cfg
        context_model.save()
        self.logger.info("Linked context %s to %s" % (context_model,
                                                      context_model.context_form_group))

if __name__=='__main__':
    import sys
    db_name = sys.argv[1]
    fh = Registry.objects.get(code='fh')
    try:
        rs = FHRecordSplitter(fh, db_name)
    except ScriptError, serr:
        print "Error : %s" % serr
        sys.exit(1)

    print "Starting run..."
    try:
        with transaction.atomic():
            rs.run()
    except Exception, ex:
        print "Error running transforms: %s" % ex
        print "Rolling back mongo..."
        rs.rollback()
    print "Finished run"

<commit_msg>RDR-1211 added transactions/"rollback"<commit_after>import django
django.setup()

from rdrf.models import ContextFormGroup
from rdrf.models import RDRFContext
from rdrf.models import Registry
from rdrf.mongo_client import construct_mongo_client

from django.db import transaction

from registry.patients.models import Patient
from django.contrib.contenttypes.models import ContentType

class Logger(object):
    def __init__(self, patient_model):
        self.patient_model = patient_model
        self.prefix = "Patient %s (id=%s)" % (self.patient_model, self.patient_model.pk)

    def info(self, msg):
        print "%s: %s" % (self.prefix, msg)

    def error(self, msg):
        print "ERROR! %s: %s" % (self.prefix, msg)

class ScriptError(Exception):
    pass

class FHRecordSplitter(object):
    def __init__(self, registry_model, mongo_db_name):
        self.backup_data = {}
        self.checkup_record_ids = []

        self.logger = None
        self.mongo_db_name = mongo_db_name
        self.registry_model = registry_model
        if self.registry_model.code != 'fh':
            raise ScriptError("This script only works with FH Registry!")

        self.checkup_form_group_name = "Checkup"

        try:
            self.checkup_context_form_group = ContextFormGroup.objects.get(registry=self.registry_model,
                                                name=self.checkup_form_group_name)
        except ContextFormGroup.DoesNotExist:
            raise ScriptError("Checkup form group does not exist - are you sure you have imported the correct FH yaml file first")

        try:
            self.main_cfg = ContextFormGroup.objects.get(registry=self.registry_model,
                                           name="Main")
        except ContextFormGroup.DoesNotExist:
            raise ScriptError("No Main context form group: Make sure latest FH yaml is imported first")

        try:
            self.mongo_client = construct_mongo_client()
        except Exception, ex:
            raise ScriptError("could not create mongo client: %s" % ex)

        self.mongo_db = self.mongo_client[self.mongo_db_name]
        self.cdes_collection = self.mongo_db["cdes"]

    def _sanity_check_main_record(self, record):
        if record:
            form_names = [form["name"] for form in record["forms"]]
            assert "FollowUp" not in form_names, "FollowUp form should not be in a main record"
            assert "FollowUp_timestamp" not in record, "FollowU_timestamp should not be in a main record"
            assert "context_id" in record, "Main record should have a context_id"
            try:
                context_model = RDRFContext.objects.get(id=record["context_id"])
            except RDRFContext.DoesNotExist:
                raise ScriptError("mentioned context does not exist in record %s" % record)

            assert context_model.object_id == record["django_id"], "Linked context not correct patient"

            assert context_model.context_form_group is not None, "Main context should be linked to the Main Context form group"
            assert context_model.context_form_group.name == "Main"
            assert context_model.context_form_group.registry.code == self.registry_model.code, "Context Form Group should linked to FH"

    def rollback(self):
        """
        We can't really rollback but we can revert back to the original mongo record and delete any FollowUp form contexts that were inserted

        """
        for patient_id, original_mongo_record in self.backup_data.items():
            print "rolling back mongo data for patient id %s" % patient_id
            mongo_id = mongo_record["_id"]

            try:
                self.cdes_collection.update({'_id': mongo_id}, {"$set": original_mongo_record}, upsert=False)
            except Exception, ex:
                print "Error rolling back patient id %s: %s" % (patient_id, ex)

        for mongo_id in self.checkup_records:
            try:
                self.cdes_collection.remove({"_id": mongo_id})
                print "Removed checkup record with _id %s" % mongo_id
            except Exception, ex:
                print "could remove mongo record with _id %s" % mongo_id

    def _sanity_check_followup_record(self, record):
        num_forms = len(record["forms"])
        assert num_forms == 1, "There should be one FollowUp form in the record"
        form = record["forms"][0]
        assert form["name"] == "FollowUp", "The single form in a CheckUp context should be called FollowUp"
        assert "FollowUp_timestamp" in record
        assert "context_id" in record, "Record should have a context_id"
        try:
            context_model = RDRFContext.objects.get(id=record["context_id"])
        except RDRFContext.DoesNotExist:
            raise ScriptError("Linked context id does  not exist for record %s" % record)

        assert context_model.object_id == record["django_id"], "Linked context not correct patient"
        assert context_model.context_form_group is not None, "A CheckUp context must be linked to a context form group"
        cfg = context_model.context_form_group
        assert cfg.name == "CheckUp", "The linked context form group of a FollowUp form should be called CheckUp"

    def run(self):
        for patient_model in Patient.objects.filter(rdrf_registry__in=[self.registry_model]):
            error = False
            self.logger = Logger(patient_model)
            self.logger.info("processing ...")

            # We can't use the normal "Dynamic Data Wrapper" in RDRF because this tries to retrieve by context id
            # is the registry changes from not using form groups to using form groups as here, the loadi

            records = [ data for data in self.cdes_collection.find({"django_model": "Patient",
                                                      "django_id": patient_model.pk})]

            if len(records) != 1:
                self.logger.info("There are %s mongo records - skipping!" % len(records))
                continue

            dynamic_data = records[0]

            self.backup_data[patient_model.pk] = dynamic_data

            followup_form = None
            index = None

            if dynamic_data is not None:
                self.logger.info("There is a mongo record for this patient")
                # Ensure that the existing context model is linked to the correct "Main" ( default) context form group
                context_id = dynamic_data["context_id"]
                self._link_main_context_form_group(context_id)

                # Locate the existing FollowUp form data and extract
                for i, form_dict in enumerate(dynamic_data["forms"]):
                    if form_dict["name"] == "FollowUp":
                        self.logger.info("There is a FollowUp form for this patient")
                        followup_form = form_dict
                        index = i

            if followup_form is not None:
                form_timestamp = dynamic_data.get("FollowUp_timestamp", None)
                self.logger.info("FollowUp form timestamp = %s" % form_timestamp)
                try:
                    del dynamic_data["forms"][index]
                except Exception, ex:
                    self.logger.error("could not delete old FollowUp form from record: %s" % ex)
                    self.logger.info("skipping this record ...")
                    continue

                self.logger.info("deleted old FollowUp form data from original record")
                try:
                    del dynamic_data["FollowUp_timestamp"]
                    self.logger.info("deleted old FollowUp timestamp")
                except Exception, ex:
                    self.logger.error("could not delete old FollowUp form timestamp: %s" % ex)
                    self.logger.info("skipping this record ...")
                    continue

                try:
                    followup_mongo_record = self._create_additional_mongo_record(patient_model, followup_form, form_timestamp)
                    self.logger.info("created new Checkup mongo record ok")
                except Exception, ex:
                    self.logger.error("could not create new Checkup mongo record: %s" % ex)
                    self.logger.info("skipping this patient")
                    continue

                # update the existing record
                try:
                    mongo_id = dynamic_data["_id"]
                    self.logger.info("existing mongo _id = %s" % mongo_id)
                except Exception, ex:
                    self.logger.error("could not find existing _id in record: %s" % ex)
                    self.logger.info("skipping this patient")
                    continue

                try:
                    self._sanity_check_main_record(dynamic_data)
                    self.cdes_collection.update({'_id': mongo_id}, {"$set": dynamic_data}, upsert=False)
                    self.logger.info("update successful")
                except Exception, ex:
                    self.logger.error("error updating the existing mongo record: %s" % ex)
                    error = True

                #insert new record for the new context
                try:
                    self._sanity_check_followup_record(followup_mongo_record)
                    result = self.cdes_collection.insert(followup_mongo_record)
                    self.logger.info("new CheckUp context created OK")
                    self.checkup_records.append(result.inserted_id)

                except Exception, ex:
                    error = True
                    self.logger.error("failed to insert new followup form context: %s" % ex)

                if error:
                    self.logger.info("finished - this patient had errors")
                else:
                    self.logger.info("finished - successfully processed")

            else:
                self.logger.info("followUp form not found in mongo record")

    def _create_additional_mongo_record(self, patient_model, followup_form, form_timestamp):
        content_type = ContentType.objects.get_for_model(patient_model)
        rdrf_context = RDRFContext(registry=self.registry_model, content_object=patient_model)
        rdrf_context.context_form_group = self.checkup_context_form_group
        rdrf_context.save()
        new_mongo_record = {"forms": [followup_form],
                            "timestamp": form_timestamp,
                            "FollowUp_timestamp": form_timestamp,
                            "django_id": patient_model.pk,
                            "django_model": "Patient",
                            "context_id": rdrf_context.pk}

        return new_mongo_record

    def _link_main_context_form_group(self, existing_context_id):

        try:
            context_model = RDRFContext.objects.get(pk=existing_context_id)
        except RDRFContext.DoesNotExist:
            raise ScriptError("error getting context model for linking to Main context form group")

        context_model.context_form_group = self.main_cfg
        context_model.save()
        self.logger.info("Linked context %s to %s" % (context_model,
                                                      context_model.context_form_group))

if __name__=='__main__':
    import sys
    db_name = sys.argv[1]
    fh = Registry.objects.get(code='fh')
    try:
        rs = FHRecordSplitter(fh, db_name)
    except ScriptError, serr:
        print "Error : %s" % serr
        sys.exit(1)

    print "Starting run..."
    try:
        with transaction.atomic():
            rs.run()
    except Exception, ex:
        print "Error running transforms: %s" % ex
        try:
            rs.rollback()
            sys.exit(1)

        except Exception, ex:
            print "Oh shit could not rollback mongo: %s" % ex
            sys.exit(1)

    print "Finished run"
    sys.exit(0)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
    Inventory

    @author: Michael Howden (michael@aidiq.com)
    @date-created: 2010-08-16

    A module to record inventories of items at a location
"""

module = request.controller

response.menu_options = inventory_menu

#==============================================================================
def shn_location_rheader(jr, tabs=[]):
    if jr.representation == "html":
        rheader_tabs = shn_rheader_tabs(jr, tabs)
        inventory_location = jr.record
        rheader = DIV(TABLE(TR(
                               TH(Tstr("Location") + ": "), shn_gis_location_represent(inventory_location.location_id),
                               TH(Tstr("Description") + ": "), inventory_location.description,
                               ),
                           ),
                      rheader_tabs
                      )
        return rheader
    return None

def index():

    """ Custom View """

    module_name = deployment_settings.modules[module].name_nice
    return dict(module_name=module_name)

def location():
    "RESTful CRUD controller"
    resource = request.function
    tablename = "%s_%s" % (module, resource)
    table = db[tablename]

    def postp(jr, output):
        shn_action_buttons(jr)
        return output
    response.s3.postp = postp

    rheader = lambda jr: shn_location_rheader(jr,
                                              tabs = [(T("Edit Details"), None),
                                                      (T("Items"), "location_item"),
                                                     ]
                                              )

    return shn_rest_controller(module, resource, rheader=rheader, sticky=True)

#==============================================================================
def location_item():
    "RESTful CRUD controller"
    resource = request.function
    tablename = "%s_%s" % (module, resource)
    table = db[tablename]

    def postp(jr, output):
        shn_action_buttons(jr)
        return output
    response.s3.postp = postp

    return shn_rest_controller(module, resource)<commit_msg>Default to the inventory_location list view<commit_after># -*- coding: utf-8 -*-

"""
    Inventory

    @author: Michael Howden (michael@aidiq.com)
    @date-created: 2010-08-16

    A module to record inventories of items at a location
"""

module = request.controller

response.menu_options = inventory_menu

#==============================================================================
def shn_location_rheader(jr, tabs=[]):
    if jr.representation == "html":
        rheader_tabs = shn_rheader_tabs(jr, tabs)
        inventory_location = jr.record
        rheader = DIV(TABLE(TR(
                               TH(Tstr("Location") + ": "), shn_gis_location_represent(inventory_location.location_id),
                               TH(Tstr("Description") + ": "), inventory_location.description,
                               ),
                           ),
                      rheader_tabs
                      )
        return rheader
    return None

def location():
    "RESTful CRUD controller"
    resource = request.function
    tablename = "%s_%s" % (module, resource)
    table = db[tablename]

    def postp(jr, output):
        shn_action_buttons(jr)
        return output
    response.s3.postp = postp

    rheader = lambda jr: shn_location_rheader(jr,
                                              tabs = [(T("Edit Details"), None),
                                                      (T("Items"), "location_item"),
                                                     ]
                                              )

    return shn_rest_controller(module, resource, rheader=rheader, sticky=True)

def index():

    """ Default to the inventory_location list view """
    request.function = "location"
    return location()
    #module_name = deployment_settings.modules[module].name_nice
    #return dict(module_name=module_name)

#==============================================================================
def location_item():
    "RESTful CRUD controller"
    resource = request.function
    tablename = "%s_%s" % (module, resource)
    table = db[tablename]

    def postp(jr, output):
        shn_action_buttons(jr)
        return output
    response.s3.postp = postp

    return shn_rest_controller(module, resource)<|endoftext|>