<commit_before># -*- coding: utf-8 -*-
"""
Updates the WikiProject Directory
Copyright (C) 2015 James Hare, Betacommand, Merlijn Van Deen
Licensed under MIT License: http://mitlicense.org
"""

import re
import time
import pywikibot
import json
import operator
import mwparserfromhell as mwph
from collections import Counter
from project_index import WikiProjectTools
from category_tree import WikiProjectCategories

def listpull(wptools, projects, directoryrow, key):
    query = wptools.query('wiki', 'select distinct page.page_title from categorylinks join page on categorylinks.cl_from=page.page_id where page_namespace = 4 and cl_to = "{0}" order by page.page_title'.format(key), None)
    output = ''
    for row in query:
        proj = row[0].decode('utf-8')
        if proj in projects:  # This check is to filter against query results like "WikiProject_Stupid/talkheader" from being considered as projects.
            output += directoryrow[proj]
    if output != "":
        output = "{{WikiProject directory top}}\n" + output + "|}\n\n"

    return output

def treeiterator(wptools, tree, projects, directoryrow, key, counter=2):
    output = ''
    if len(tree) > 0:
        header = "=" * counter  # Python always finds new ways to amaze me.
        for step in tree.keys():
            output += header + step.replace('_', ' ') + header + "\n" + listpull(wptools, projects, directoryrow, step) + "\n"
            if len(tree[step]) > 0:
                output += treeiterator(wptools, tree[step], projects, directoryrow, step, counter=counter+1)
    return output

def main(rootpage):
    # Initializing...
    bot = pywikibot.Site('en', 'wikipedia')
    wptools = WikiProjectTools()
    config = json.loads(wptools.query('index', 'select json from config;', None)[0][0])
    print("Let's get this show on the road!")

    # Get list of people who opted out
    optout = pywikibot.Page(bot, 'User:Reports bot/Opt-out')
    blacklist = []
    regexes =[re.findall('\[\[User:(.*?)\|',optout.text,re.I), re.findall('\{\{user\|(.*?)\}\}',optout.text,re.I), re.findall('\[\[:User:(.*?)\]',optout.text,re.I), re.findall('\[\[:User talk:(.*?)\]',optout.text,re.I)]
    for results in regexes:
        for user in results:
            blacklist.append(user)
    print(str(len(blacklist)) + ' users opting out')
    # Bots are to be excluded
    for result in wptools.query('wiki', "select user_name from user_groups left join user on user_id = ug_user where ug_group = 'bot';", None):
        blacklist.append(result[0].decode('utf-8'))
    print('With bots, there are ' + str(len(blacklist)) + ' usernames on the blacklist.')

    # List of projects we are working on
    # Methodology: List from Project Index + List from Formal Definition, minus duplicates
    # This will cover all of our bases.
    print("Loading the Project Index...")
    articles = {}
    counter = 0
    while True:  # I am a bad man for doing this
        print("Query for IDs " + str(counter + 1) + " through " + str(counter+ 1000000))
        query = wptools.query('index', 'select pi_page, pi_project from projectindex where pi_id > {0} and pi_id <= {1};'.format(counter, counter+1000000), None)
        if len(query) == 0:
            break
        for pair in query:
            # Normalizing by getting rid of namespace
            page = pair[0]
            page = page.replace('Draft_talk:', '')
            page = page.replace('Talk:', '')
            proj = pair[1][10:]  # Normalizing by getting rid of "Wikipedia:"
            try:
                articles[proj].append(page)
            except KeyError:
                articles[proj] = [page]
        counter += 1000000

    projects = [project for project in articles.keys()]

    print("Preparing the Formal Definition index...")
    formaldefinition = wptools.query('wiki', 'select distinct page.page_title from page join categorylinks on page.page_id = categorylinks.cl_from left join redirect on page.page_id = redirect.rd_from where page_namespace = 4 and page_title not like "%/%" and rd_title is null and (cl_to in (select page.page_title from page where page_namespace = 14 and page_title like "%\_WikiProjects" and page_title not like "%\_for\_WikiProjects" and page_title not like "%\_of\_WikiProjects") or page_title like "WikiProject\_%");', None)  # http://quarry.wmflabs.org/query/3509
    for row in formaldefinition:
        row = row[0].decode('utf-8')
        if row not in projects:
            projects.append(row)
    projects.sort()
    print('There are ' + str(len(projects)) + ' total WikiProjects and task forces.')

    directories = {'All': ''}  # All projects, plus subdirectories to be defined below.
    directoryrow = {}

    # Alright! Let's run some reports!
    for project in projects:
        print("Working on: " + project)

        # Seeding directory row and profile page
        if project not in articles:
            articles[project] = []
        project_normalized = project.replace('_', ' ')

        # List of active project participants (less blacklist)
        wp_editors = []
        start_date = time.strftime('%Y%m%d000000',time.gmtime(time.time()-(60*60*24*90)))  # 90 days
        end_date = time.strftime('%Y%m%d000000',time.gmtime(time.time()))  # Today
        query = "select rev_user_text from page left join revision on page_id = rev_page where (page_namespace = 4 OR page_namespace = 5) and (page_title like \"{0}/%%\" OR page_title = \"{0}\") and rev_timestamp > {1} and rev_timestamp < {2} group by rev_user_text HAVING count(*) > 1;".format(project, start_date, end_date)
        for result in wptools.query('wiki', query, None):
            if result[0] is not None:
                user = result[0].decode('utf-8')
                if user not in blacklist:
                    wp_editors.append(user)
        wp_editors.sort()

        # List of active subject area editors (less blacklist)
        start_date = time.strftime('%Y%m%d000000',time.gmtime(time.time()-(60*60*24*30)))  # 30 days
        end_date = time.strftime('%Y%m%d000000',time.gmtime(time.time()))  # Today

        if len(articles[project]) > 0:
            subject_editors = []
            packages = []
            for i in range(0, len(articles[project]), 10000):
                packages.append(articles[project][i:i+10000])

            counter = 0
            for package in packages:
                counter += 1
                print('Executing batch query no. ' + str(counter))
                if len(package) > 1:
                    query_builder = 'select rev_user_text from page left join revision on page_id = rev_page where page_namespace in (0, 1, 118, 119) and page_title in {0} and rev_timestamp > {1} and rev_timestamp < {2} order by rev_user_text;'.format(tuple(package), start_date, end_date)
                else:
                    query_builder = 'select rev_user_text from page left join revision on page_id = rev_page where page_namespace in (0, 1, 118, 119) and page_title = "{0}" and rev_timestamp > {1} and rev_timestamp < {2} order by rev_user_text;'.format(package[0], start_date, end_date)

                for result in wptools.query('wiki', query_builder, None):
                    if result[0] is not None:
                        subject_editors.append(result[0].decode('utf-8'))

            subject_editors = dict(Counter(subject_editors))  # Convert the list to a dictionary with username as key and edit count as value
            subject_editors_filtered = []
            for user in subject_editors.keys():
                if user not in blacklist:
                    if subject_editors[user] > 4:
                        subject_editors_filtered.append(user)
            subject_editors = subject_editors_filtered   # And now assigned back.
            subject_editors.sort()

        else:
            subject_editors = []

        # Generate and Save Profile Page
        wp_editors_formatted = ""
        subject_editors_formatted = ""
        if len(wp_editors) > 0:
            for editor in wp_editors:
                wp_editors_formatted += "\n* {{{{user|1={0}}}}}".format(editor)
        else:
                wp_editors_formatted = ""
        if len(subject_editors) > 0:
            for editor in subject_editors:
                subject_editors_formatted += "\n* {{{{user|1={0}}}}}".format(editor)
        else:
                subject_editors_formatted = ""

        profilepage = "{{{{WikiProject description page | project = {0} | list_of_active_wikiproject_participants = {1} | list_of_active_subject_area_editors = {2}}}}}".format(project_normalized, wp_editors_formatted, subject_editors_formatted)
        page = pywikibot.Page(bot, rootpage + '/Description/' + project_normalized)
        if profilepage != page.text:  # Checking to see if a change was made to cut down on API queries
            page.text = profilepage
            page.save('Updating', minor=False, async=True)

        # Construct directory entry
        directoryrow[project] = "{{{{WikiProject directory entry | project = {0} | number_of_articles = {1} | wp_editors = {2} | scope_editors = {3}}}}}\n".format(project_normalized, len(articles[project]), len(wp_editors), len(subject_editors))

    # Assign directory entry to relevant directory pages ("All entries" and relevant subdirectory pages)
    print("Populating directory pages...")
    for entry in sorted(directoryrow.items(), key=operator.itemgetter(1)):  # Sorting into alphabetical order
        directories['All'] += entry[1]
    directories['All'] = "{{WikiProject directory top}}\n" + directories['All'] + "|}"

    wpcats = WikiProjectCategories()
    tree = wpcats.generate()
    index_primary = sorted([key for key in tree.keys()])
    index_secondary = {}
    indextext = "'''[[{0}/All|All WikiProjects]]'''\n\n".format(rootpage)
    for firstlevel in tree.keys():
        directories[firstlevel] = listpull(wptools, projects, directoryrow, firstlevel)  # For immmedate subcats of WikiProjects_by_area
        directories[firstlevel] += treeiterator(wptools, tree[firstlevel], projects, directoryrow, firstlevel)  # For descendants of those immediate subcats.
        index_secondary[firstlevel] = sorted([key for key in tree[firstlevel].keys()])

    # Updating the directory index
    for firstlevel in index_primary:
        firstlevel_normalized = firstlevel.replace('_', ' ')
        indextext += ";[[{0}/{1}|{1}]]".format(rootpage, firstlevel_normalized)
        if len(tree[firstlevel]) > 0:
            indextext += " : "
            for secondlevel in index_secondary[firstlevel]:
                indextext += "[[{0}/{1}#{2}|{2}]] – ".format(rootpage, firstlevel_normalized, secondlevel.replace('_', ' '))
            indextext = indextext[:-3]  # Truncates trailing dash and is also a cute smiley face
        indextext += "\n\n"
    saveindex = pywikibot.Page(bot, 'Template:WikiProject directory index')
    saveindex.text = indextext
    saveindex.save('Updating', minor=False, async=True)

    # Generate directories and save!
    for directory in directories.keys():
        contents = directories[directory]
        page = pywikibot.Page(bot, rootpage + "/" + directory)
        if contents != page.text:  # Checking to see if a change was made to cut down on API save queries
            oldcontents = page.text
            page.text = contents
            page.save('Updating', minor=False, async=True)
            # Cleanup of obsolete description pages and "Related WikiProjects" pages
            if directory == 'All':
                oldcontents = mwph.parse(oldcontents)
                oldcontents = oldcontents.filter_templates()
                oldprojectlist = []
                for t in oldcontents:
                    if t.name.strip() == "WikiProject directory entry":
                        oldprojectlist.append(str(t.get('project').value))
                for oldproject in oldprojectlist:
                    oldproject = oldproject.strip().replace(' ', '_')  # Normalizing
                    if oldproject not in projects:
                        deletethis = pywikibot.Page(bot, rootpage + '/Description/' + oldproject)
                        deletethis.text = "{{db-g6|rationale=A bot has automatically tagged this page as obsolete. This means that the WikiProject described on this page has been deleted or made into a redirect}}\n"
                        deletethis.save('Nominating page for deletion', minor=False, async=True)
                        deletethis = pywikibot.Page(bot, 'Wikipedia:Related WikiProjects/' + oldproject)
                        if deletethis.text != "":
                            deletethis.text = "{{db-g6|rationale=A bot has automatically tagged this page as obsolete. This means that the WikiProject described on this page has been deleted or made into a redirect}}\n"
                            deletethis.save('Nominating page for deletion', minor=False, async=True)

if __name__ == "__main__":
    main('Wikipedia:WikiProject Directory')<commit_msg>Summative directory uses small version of template<commit_after># -*- coding: utf-8 -*-
"""
Updates the WikiProject Directory
Copyright (C) 2015 James Hare, Betacommand, Merlijn Van Deen
Licensed under MIT License: http://mitlicense.org
"""

import re
import time
import pywikibot
import json
import operator
import mwparserfromhell as mwph
from collections import Counter
from project_index import WikiProjectTools
from category_tree import WikiProjectCategories

def listpull(wptools, projects, directoryrow, key):
    query = wptools.query('wiki', 'select distinct page.page_title from categorylinks join page on categorylinks.cl_from=page.page_id where page_namespace = 4 and cl_to = "{0}" order by page.page_title'.format(key), None)
    output = ''
    for row in query:
        proj = row[0].decode('utf-8')
        if proj in projects:  # This check is to filter against query results like "WikiProject_Stupid/talkheader" from being considered as projects.
            output += directoryrow[proj]
    if output != "":
        output = "{{WikiProject directory top}}\n" + output + "|}\n\n"

    return output

def treeiterator(wptools, tree, projects, directoryrow, key, counter=2):
    output = ''
    if len(tree) > 0:
        header = "=" * counter  # Python always finds new ways to amaze me.
        for step in tree.keys():
            output += header + step.replace('_', ' ') + header + "\n" + listpull(wptools, projects, directoryrow, step) + "\n"
            if len(tree[step]) > 0:
                output += treeiterator(wptools, tree[step], projects, directoryrow, step, counter=counter+1)
    return output

def main(rootpage):
    # Initializing...
    bot = pywikibot.Site('en', 'wikipedia')
    wptools = WikiProjectTools()
    config = json.loads(wptools.query('index', 'select json from config;', None)[0][0])
    print("Let's get this show on the road!")

    # Get list of people who opted out
    optout = pywikibot.Page(bot, 'User:Reports bot/Opt-out')
    blacklist = []
    regexes =[re.findall('\[\[User:(.*?)\|',optout.text,re.I), re.findall('\{\{user\|(.*?)\}\}',optout.text,re.I), re.findall('\[\[:User:(.*?)\]',optout.text,re.I), re.findall('\[\[:User talk:(.*?)\]',optout.text,re.I)]
    for results in regexes:
        for user in results:
            blacklist.append(user)
    print(str(len(blacklist)) + ' users opting out')
    # Bots are to be excluded
    for result in wptools.query('wiki', "select user_name from user_groups left join user on user_id = ug_user where ug_group = 'bot';", None):
        blacklist.append(result[0].decode('utf-8'))
    print('With bots, there are ' + str(len(blacklist)) + ' usernames on the blacklist.')

    # List of projects we are working on
    # Methodology: List from Project Index + List from Formal Definition, minus duplicates
    # This will cover all of our bases.
    print("Loading the Project Index...")
    articles = {}
    counter = 0
    while True:  # I am a bad man for doing this
        print("Query for IDs " + str(counter + 1) + " through " + str(counter+ 1000000))
        query = wptools.query('index', 'select pi_page, pi_project from projectindex where pi_id > {0} and pi_id <= {1};'.format(counter, counter+1000000), None)
        if len(query) == 0:
            break
        for pair in query:
            # Normalizing by getting rid of namespace
            page = pair[0]
            page = page.replace('Draft_talk:', '')
            page = page.replace('Talk:', '')
            proj = pair[1][10:]  # Normalizing by getting rid of "Wikipedia:"
            try:
                articles[proj].append(page)
            except KeyError:
                articles[proj] = [page]
        counter += 1000000

    projects = [project for project in articles.keys()]

    print("Preparing the Formal Definition index...")
    formaldefinition = wptools.query('wiki', 'select distinct page.page_title from page join categorylinks on page.page_id = categorylinks.cl_from left join redirect on page.page_id = redirect.rd_from where page_namespace = 4 and page_title not like "%/%" and rd_title is null and (cl_to in (select page.page_title from page where page_namespace = 14 and page_title like "%\_WikiProjects" and page_title not like "%\_for\_WikiProjects" and page_title not like "%\_of\_WikiProjects") or page_title like "WikiProject\_%");', None)  # http://quarry.wmflabs.org/query/3509
    for row in formaldefinition:
        row = row[0].decode('utf-8')
        if row not in projects:
            projects.append(row)
    projects.sort()
    print('There are ' + str(len(projects)) + ' total WikiProjects and task forces.')

    directories = {'All': ''}  # All projects, plus subdirectories to be defined below.
    directoryrow = {}

    # Alright! Let's run some reports!
    for project in projects:
        print("Working on: " + project)

        # Seeding directory row and profile page
        if project not in articles:
            articles[project] = []
        project_normalized = project.replace('_', ' ')

        # List of active project participants (less blacklist)
        wp_editors = []
        start_date = time.strftime('%Y%m%d000000',time.gmtime(time.time()-(60*60*24*90)))  # 90 days
        end_date = time.strftime('%Y%m%d000000',time.gmtime(time.time()))  # Today
        query = "select rev_user_text from page left join revision on page_id = rev_page where (page_namespace = 4 OR page_namespace = 5) and (page_title like \"{0}/%%\" OR page_title = \"{0}\") and rev_timestamp > {1} and rev_timestamp < {2} group by rev_user_text HAVING count(*) > 1;".format(project, start_date, end_date)
        for result in wptools.query('wiki', query, None):
            if result[0] is not None:
                user = result[0].decode('utf-8')
                if user not in blacklist:
                    wp_editors.append(user)
        wp_editors.sort()

        # List of active subject area editors (less blacklist)
        start_date = time.strftime('%Y%m%d000000',time.gmtime(time.time()-(60*60*24*30)))  # 30 days
        end_date = time.strftime('%Y%m%d000000',time.gmtime(time.time()))  # Today

        if len(articles[project]) > 0:
            subject_editors = []
            packages = []
            for i in range(0, len(articles[project]), 10000):
                packages.append(articles[project][i:i+10000])

            counter = 0
            for package in packages:
                counter += 1
                print('Executing batch query no. ' + str(counter))
                if len(package) > 1:
                    query_builder = 'select rev_user_text from page left join revision on page_id = rev_page where page_namespace in (0, 1, 118, 119) and page_title in {0} and rev_timestamp > {1} and rev_timestamp < {2} order by rev_user_text;'.format(tuple(package), start_date, end_date)
                else:
                    query_builder = 'select rev_user_text from page left join revision on page_id = rev_page where page_namespace in (0, 1, 118, 119) and page_title = "{0}" and rev_timestamp > {1} and rev_timestamp < {2} order by rev_user_text;'.format(package[0], start_date, end_date)

                for result in wptools.query('wiki', query_builder, None):
                    if result[0] is not None:
                        subject_editors.append(result[0].decode('utf-8'))

            subject_editors = dict(Counter(subject_editors))  # Convert the list to a dictionary with username as key and edit count as value
            subject_editors_filtered = []
            for user in subject_editors.keys():
                if user not in blacklist:
                    if subject_editors[user] > 4:
                        subject_editors_filtered.append(user)
            subject_editors = subject_editors_filtered   # And now assigned back.
            subject_editors.sort()

        else:
            subject_editors = []

        # Generate and Save Profile Page
        wp_editors_formatted = ""
        subject_editors_formatted = ""
        if len(wp_editors) > 0:
            for editor in wp_editors:
                wp_editors_formatted += "\n* {{{{user|1={0}}}}}".format(editor)
        else:
                wp_editors_formatted = ""
        if len(subject_editors) > 0:
            for editor in subject_editors:
                subject_editors_formatted += "\n* {{{{user|1={0}}}}}".format(editor)
        else:
                subject_editors_formatted = ""

        profilepage = "{{{{WikiProject description page | project = {0} | list_of_active_wikiproject_participants = {1} | list_of_active_subject_area_editors = {2}}}}}".format(project_normalized, wp_editors_formatted, subject_editors_formatted)
        page = pywikibot.Page(bot, rootpage + '/Description/' + project_normalized)
        if profilepage != page.text:  # Checking to see if a change was made to cut down on API queries
            page.text = profilepage
            page.save('Updating', minor=False, async=True)

        # Construct directory entry
        directoryrow[project] = "{{{{WikiProject directory entry | project = {0} | number_of_articles = {1} | wp_editors = {2} | scope_editors = {3}}}}}\n".format(project_normalized, len(articles[project]), len(wp_editors), len(subject_editors))

    # Assign directory entry to relevant directory pages ("All entries" and relevant subdirectory pages)
    print("Populating directory pages...")
    for entry in sorted(directoryrow.items(), key=operator.itemgetter(1)):  # Sorting into alphabetical order
        directories['All'] += entry[1].replace('WikiProject directory entry', 'WikiProject directory entry small')
    directories['All'] = "{{WikiProject directory top}}\n" + directories['All'] + "|}"

    wpcats = WikiProjectCategories()
    tree = wpcats.generate()
    index_primary = sorted([key for key in tree.keys()])
    index_secondary = {}
    indextext = "'''[[{0}/All|All WikiProjects]]'''\n\n".format(rootpage)
    for firstlevel in tree.keys():
        directories[firstlevel] = listpull(wptools, projects, directoryrow, firstlevel)  # For immmedate subcats of WikiProjects_by_area
        directories[firstlevel] += treeiterator(wptools, tree[firstlevel], projects, directoryrow, firstlevel)  # For descendants of those immediate subcats.
        index_secondary[firstlevel] = sorted([key for key in tree[firstlevel].keys()])

    # Updating the directory index
    for firstlevel in index_primary:
        firstlevel_normalized = firstlevel.replace('_', ' ')
        indextext += ";[[{0}/{1}|{1}]]".format(rootpage, firstlevel_normalized)
        if len(tree[firstlevel]) > 0:
            indextext += " : "
            for secondlevel in index_secondary[firstlevel]:
                indextext += "[[{0}/{1}#{2}|{2}]] – ".format(rootpage, firstlevel_normalized, secondlevel.replace('_', ' '))
            indextext = indextext[:-3]  # Truncates trailing dash and is also a cute smiley face
        indextext += "\n\n"
    saveindex = pywikibot.Page(bot, 'Template:WikiProject directory index')
    saveindex.text = indextext
    saveindex.save('Updating', minor=False, async=True)

    # Generate directories and save!
    for directory in directories.keys():
        contents = directories[directory]
        page = pywikibot.Page(bot, rootpage + "/" + directory)
        if contents != page.text:  # Checking to see if a change was made to cut down on API save queries
            oldcontents = page.text
            page.text = contents
            page.save('Updating', minor=False, async=True)
            # Cleanup of obsolete description pages and "Related WikiProjects" pages
            if directory == 'All':
                oldcontents = mwph.parse(oldcontents)
                oldcontents = oldcontents.filter_templates()
                oldprojectlist = []
                for t in oldcontents:
                    if t.name.strip() == "WikiProject directory entry":
                        oldprojectlist.append(str(t.get('project').value))
                for oldproject in oldprojectlist:
                    oldproject = oldproject.strip().replace(' ', '_')  # Normalizing
                    if oldproject not in projects:
                        deletethis = pywikibot.Page(bot, rootpage + '/Description/' + oldproject)
                        deletethis.text = "{{db-g6|rationale=A bot has automatically tagged this page as obsolete. This means that the WikiProject described on this page has been deleted or made into a redirect}}\n"
                        deletethis.save('Nominating page for deletion', minor=False, async=True)
                        deletethis = pywikibot.Page(bot, 'Wikipedia:Related WikiProjects/' + oldproject)
                        if deletethis.text != "":
                            deletethis.text = "{{db-g6|rationale=A bot has automatically tagged this page as obsolete. This means that the WikiProject described on this page has been deleted or made into a redirect}}\n"
                            deletethis.save('Nominating page for deletion', minor=False, async=True)

if __name__ == "__main__":
    main('Wikipedia:WikiProject Directory')<|endoftext|>


<commit_before>import re
import threading
import types
from contextlib import ContextDecorator

from bonobo.config.options import Option
from bonobo.errors import MissingServiceImplementationError

_service_name_re = re.compile(r"^[^\d\W]\w*(:?\.[^\d\W]\w*)*$", re.UNICODE)

def validate_service_name(name):
    if not _service_name_re.match(name):
        raise ValueError('Invalid service name {!r}.'.format(name))
    return name

class Service(Option):
    """
    A Service is a special kind of option defining a dependency to something that will be resolved at runtime, using an
    identifier. For example, you can create a Configurable that has a "database" Service in its attribute, meaning that
    you'll define which database to use, by name, when creating the instance of this class, then provide an
    implementation when running the graph using a strategy.

    Example::

        import bonobo

        class QueryExtractor(bonobo.Configurable):
            database = bonobo.Service(default='sqlalchemy.engine.default')

        graph = bonobo.Graph(
            QueryExtractor(database='sqlalchemy.engine.secondary'),
            *more_transformations,
        )

        if __name__ == '__main__':
            engine = create_engine('... dsn ...')
            bonobo.run(graph, services={
                'sqlalchemy.engine.secondary': engine
            })

    The main goal is not to tie transformations to actual dependencies, so the same can be run in different contexts
    (stages like preprod, prod, or tenants like client1, client2, or anything you want).

    .. attribute:: name

        Service name will be used to retrieve the implementation at runtime.

    """

    def __init__(self, name, __doc__=None):
        super().__init__(str, required=False, default=name, __doc__=__doc__)

    def __set__(self, inst, value):
        inst._options_values[self.name] = validate_service_name(value)

    def resolve(self, inst, services):
        try:
            name = getattr(inst, self.name)
        except AttributeError:
            name = self.name
        return services.get(name)

class Container(dict):
    def __new__(cls, *args, **kwargs):
        if len(args) == 1:
            if len(kwargs):
                raise ValueError(
                    'You can either use {} with one positional argument or with keyword arguments, not both.'
                )
            if not args[0]:
                return super().__new__(cls)
            if isinstance(args[0], cls):
                return cls
        return super().__new__(cls, *args, **kwargs)

    def kwargs_for(self, mixed):
        try:
            options = dict(mixed.__options__)
        except AttributeError:
            options = {}

        return {name: option.resolve(mixed, self) for name, option in options.items() if isinstance(option, Service)}

    def get(self, name, default=None):
        if not name in self:
            if default:
                return default
            raise MissingServiceImplementationError(
                'Cannot resolve service {!r} using provided service collection.'.format(name)
            )
        value = super().get(name)
        # XXX this is not documented and can lead to errors.
        if isinstance(value, types.LambdaType):
            value = value(self)
        return value

def create_container(services=None, factory=Container):
    """
    Create a container with reasonable default service implementations for commonly use, standard-named, services.

    Services:
    - `fs` defaults to a fs2 instance based on current working directory
    - `http`defaults to requests

    :param services:
    :return:
    """
    container = factory(services) if services else factory()

    if not 'fs' in container:
        import bonobo

        container.setdefault('fs', bonobo.open_fs())

    if not 'http' in container:
        import requests

        container.setdefault('http', requests)

    return container

class Exclusive(ContextDecorator):
    """
    Decorator and context manager used to require exclusive usage of an object, most probably a service. It's usefull
    for example if call order matters on a service implementation (think of an http api that requires a nonce or version
    parameter ...).

    Usage:

        >>> def handler(some_service):
        ...     with Exclusive(some_service):
        ...         some_service.call_1()
        ...         some_service.call_2()
        ...         some_service.call_3()

    This will ensure that nobody else is using the same service while in the "with" block, using a lock primitive to
    ensure that.

    """

    _locks = {}

    def __init__(self, wrapped):
        self._wrapped = wrapped

    def get_lock(self):
        _id = id(self._wrapped)
        if not _id in Exclusive._locks:
            Exclusive._locks[_id] = threading.RLock()
        return Exclusive._locks[_id]

    def __enter__(self):
        self.get_lock().acquire()
        return self._wrapped

    def __exit__(self, *exc):
        self.get_lock().release()

def use(*service_names):
    def decorate(mixed):
        try:
            options = mixed.__options__
        except AttributeError:
            mixed.__options__ = options = {}

        for service_name in service_names:
            service = Service(service_name)
            service.name = service_name
            options[service_name] = service
        return mixed

    return decorate
<commit_msg>fix: missing format arg<commit_after>import re
import threading
import types
from contextlib import ContextDecorator

from bonobo.config.options import Option
from bonobo.errors import MissingServiceImplementationError

_service_name_re = re.compile(r"^[^\d\W]\w*(:?\.[^\d\W]\w*)*$", re.UNICODE)

def validate_service_name(name):
    if not _service_name_re.match(name):
        raise ValueError('Invalid service name {!r}.'.format(name))
    return name

class Service(Option):
    """
    A Service is a special kind of option defining a dependency to something that will be resolved at runtime, using an
    identifier. For example, you can create a Configurable that has a "database" Service in its attribute, meaning that
    you'll define which database to use, by name, when creating the instance of this class, then provide an
    implementation when running the graph using a strategy.

    Example::

        import bonobo

        class QueryExtractor(bonobo.Configurable):
            database = bonobo.Service(default='sqlalchemy.engine.default')

        graph = bonobo.Graph(
            QueryExtractor(database='sqlalchemy.engine.secondary'),
            *more_transformations,
        )

        if __name__ == '__main__':
            engine = create_engine('... dsn ...')
            bonobo.run(graph, services={
                'sqlalchemy.engine.secondary': engine
            })

    The main goal is not to tie transformations to actual dependencies, so the same can be run in different contexts
    (stages like preprod, prod, or tenants like client1, client2, or anything you want).

    .. attribute:: name

        Service name will be used to retrieve the implementation at runtime.

    """

    def __init__(self, name, __doc__=None):
        super().__init__(str, required=False, default=name, __doc__=__doc__)

    def __set__(self, inst, value):
        inst._options_values[self.name] = validate_service_name(value)

    def resolve(self, inst, services):
        try:
            name = getattr(inst, self.name)
        except AttributeError:
            name = self.name
        return services.get(name)

class Container(dict):
    def __new__(cls, *args, **kwargs):
        if len(args) == 1:
            if len(kwargs):
                raise ValueError(
                    'You can either use {} with one positional argument or with keyword arguments, not both.'.format(
                        cls.__name__
                    )
                )
            if not args[0]:
                return super().__new__(cls)
            if isinstance(args[0], cls):
                return cls
        return super().__new__(cls, *args, **kwargs)

    def kwargs_for(self, mixed):
        try:
            options = dict(mixed.__options__)
        except AttributeError:
            options = {}

        return {name: option.resolve(mixed, self) for name, option in options.items() if isinstance(option, Service)}

    def get(self, name, default=None):
        if not name in self:
            if default:
                return default
            raise MissingServiceImplementationError(
                'Cannot resolve service {!r} using provided service collection.'.format(name)
            )
        value = super().get(name)
        # XXX this is not documented and can lead to errors.
        if isinstance(value, types.LambdaType):
            value = value(self)
        return value

def create_container(services=None, factory=Container):
    """
    Create a container with reasonable default service implementations for commonly use, standard-named, services.

    Services:
    - `fs` defaults to a fs2 instance based on current working directory
    - `http`defaults to requests

    :param services:
    :return:
    """
    container = factory(services) if services else factory()

    if not 'fs' in container:
        import bonobo

        container.setdefault('fs', bonobo.open_fs())

    if not 'http' in container:
        import requests

        container.setdefault('http', requests)

    return container

class Exclusive(ContextDecorator):
    """
    Decorator and context manager used to require exclusive usage of an object, most probably a service. It's usefull
    for example if call order matters on a service implementation (think of an http api that requires a nonce or version
    parameter ...).

    Usage:

        >>> def handler(some_service):
        ...     with Exclusive(some_service):
        ...         some_service.call_1()
        ...         some_service.call_2()
        ...         some_service.call_3()

    This will ensure that nobody else is using the same service while in the "with" block, using a lock primitive to
    ensure that.

    """

    _locks = {}

    def __init__(self, wrapped):
        self._wrapped = wrapped

    def get_lock(self):
        _id = id(self._wrapped)
        if not _id in Exclusive._locks:
            Exclusive._locks[_id] = threading.RLock()
        return Exclusive._locks[_id]

    def __enter__(self):
        self.get_lock().acquire()
        return self._wrapped

    def __exit__(self, *exc):
        self.get_lock().release()

def use(*service_names):
    def decorate(mixed):
        try:
            options = mixed.__options__
        except AttributeError:
            mixed.__options__ = options = {}

        for service_name in service_names:
            service = Service(service_name)
            service.name = service_name
            options[service_name] = service
        return mixed

    return decorate
<|endoftext|>


<commit_before>import logging
import fingerprints
from pprint import pprint  # noqa
from banal import ensure_list
from datetime import datetime
from collections import defaultdict
from followthemoney import model
from elasticsearch.helpers import scan

from aleph.core import es, cache
from aleph.model import Entity
from aleph.index.indexes import entities_write_index, entities_read_index
from aleph.index.indexes import entities_read_index_list
from aleph.index.util import unpack_result, refresh_sync
from aleph.index.util import index_safe, authz_query, bulk_actions
from aleph.index.util import MAX_PAGE

log = logging.getLogger(__name__)
EXCLUDE_DEFAULT = ['text', 'fingerprints', 'names', 'phones', 'emails',
                   'identifiers', 'addresses', 'properties.bodyText',
                   'properties.bodyHtml', 'properties.headers']

def index_entity(entity, sync=False):
    """Index an entity."""
    if entity.deleted_at is not None:
        return delete_entity(entity.id)

    entity_id, index, data = index_operation(entity.to_dict())
    refresh = refresh_sync(sync)
    # This is required if an entity changes its type:
    # delete_entity(entity_id, exclude=proxy.schema, sync=False)
    return index_safe(index, entity_id, data, refresh=refresh)

def index_collection_entities(collection, sync=False):
    """Re-index all documents in a collection in one go."""
    from aleph.index.documents import generate_collection_docs

    def _generate():
        for entity in Entity.by_collection(collection.id):
            entity_id, index, body = index_operation(entity.to_dict())
            yield {
                '_id': entity_id,
                '_index': index,
                '_type': 'doc',
                '_source': body
            }
        yield from generate_collection_docs(collection)

    bulk_actions(_generate(), sync=sync)

def _source_spec(includes, excludes):
    includes = ensure_list(includes)
    excludes = ensure_list(excludes)
    if not len(excludes):
        excludes = EXCLUDE_DEFAULT
    return {'includes': includes, 'excludes': excludes}

def iter_entities(authz=None, collection_id=None, schemata=None,
                  includes=None, excludes=None):
    """Scan all entities matching the given criteria."""
    filters = []
    if authz is not None:
        filters.append(authz_query(authz))
    if collection_id is not None:
        filters.append({'term': {'collection_id': collection_id}})
    if ensure_list(schemata):
        filters.append({'terms': {'schemata': ensure_list(schemata)}})
    query = {
        'query': {'bool': {'filter': filters}},
        '_source': _source_spec(includes, excludes)
    }
    index = entities_read_index(schema=schemata)
    for res in scan(es, index=index, query=query, scroll='1410m'):
        entity = unpack_result(res)
        if entity is not None:
            yield entity

def iter_proxies(**kw):
    includes = ['schema', 'properties']
    for data in iter_entities(includes=includes, **kw):
        schema = model.get(data.get('schema'))
        if schema is None:
            continue
        yield model.get_proxy(data)

def entities_by_ids(ids, schemata=None, cached=False,
                    includes=None, excludes=None):
    """Iterate over unpacked entities based on a search for the given
    entity IDs."""
    ids = ensure_list(ids)
    if not len(ids):
        return
    index = entities_read_index(schema=schemata)
    query = {'ids': {'values': ids}}
    # query = {'bool': {'filter': query}}
    query = {
        'query': query,
        '_source': _source_spec(includes, excludes),
        'size': MAX_PAGE
    }
    result = es.search(index=index, body=query)
    for doc in result.get('hits', {}).get('hits', []):
        entity = unpack_result(doc)
        if entity is not None:
            # Cache entities only briefly to avoid filling up the cache:
            if cached:
                key = cache.object_key(Entity, entity.get('id'))
                cache.set_complex(key, entity, expire=60 * 60)
            yield entity

def get_entity(entity_id, **kwargs):
    """Fetch an entity from the index."""
    if entity_id is None:
        return
    key = cache.object_key(Entity, entity_id)
    entity = cache.get_complex(key)
    if entity is not None:
        return entity
    log.debug("Entity [%s]: object cache miss", entity_id)
    for entity in entities_by_ids(entity_id, cached=True):
        return entity

def _index_updates(collection_id, entities, merge=True):
    """Look up existing index documents and generate an updated form.

    This is necessary to make the index accumulative, i.e. if an entity or link
    gets indexed twice with different field values, it'll add up the different
    field values into a single record. This is to avoid overwriting the
    document and losing field values. An alternative solution would be to
    implement this in Groovy on the ES.
    """
    indexes = defaultdict(list)
    timestamps = {}
    common = {
        'collection_id': collection_id,
        'updated_at': datetime.utcnow(),
        'bulk': True
    }

    if merge:
        for result in entities_by_ids(list(entities.keys())):
            existing = model.get_proxy(result)
            indexes[existing.id].append(result.get('_index'))
            entities[existing.id].merge(existing)
            timestamps[existing.id] = result.get('created_at')

    actions = []
    for entity_id, proxy in entities.items():
        entity = dict(common)
        created_at = timestamps.get(proxy.id, common.get('updated_at'))
        entity['created_at'] = created_at
        entity.update(proxy.to_full_dict())
        _, index, body = index_operation(entity)
        for other in indexes.get(entity_id, []):
            if other != index:
                # log.info("Delete ID [%s] from index: %s", entity_id, other)
                actions.append(delete_operation(other, entity_id))
        actions.append({
            '_id': entity_id,
            '_index': index,
            '_type': 'doc',
            '_source': body
        })
    return actions

def index_bulk(collection_id, entities, merge=True):
    """Index a set of entities."""
    actions = _index_updates(collection_id, entities, merge=merge)
    bulk_actions(actions, sync=merge)

def delete_entity(entity_id, exclude=None, sync=False):
    """Delete an entity from the index."""
    if exclude is not None:
        exclude = entities_write_index(exclude)

    def _generate():
        for index in entities_read_index_list():
            if index != exclude:
                yield delete_operation(index, entity_id)

    bulk_actions(_generate(), sync=sync)

def index_operation(data):
    """Apply final denormalisations to the index."""
    data['bulk'] = data.get('bulk', False)
    names = ensure_list(data.get('names'))
    fps = set([fingerprints.generate(name) for name in names])
    fps.update(names)
    data['fingerprints'] = [fp for fp in fps if fp is not None]

    # Slight hack: a magic property in followthemoney that gets taken out
    # of the properties and added straight to the index text.
    texts = data.pop('text', [])
    texts.extend(data.get('properties', {}).pop('indexText', []))
    texts.extend(fps)
    data['text'] = texts

    if not data.get('created_at'):
        data['created_at'] = data.get('updated_at')

    entity_id = str(data.pop('id'))
    data.pop('_index', None)
    index = entities_write_index(data.get('schema'))
    return entity_id, index, data

def delete_operation(index, entity_id):
    return {
        '_id': entity_id,
        '_index': index,
        '_type': 'doc',
        '_op_type': 'delete'
    }
<commit_msg>revert delete function<commit_after>import logging
import fingerprints
from pprint import pprint  # noqa
from banal import ensure_list
from datetime import datetime
from collections import defaultdict
from followthemoney import model
from elasticsearch.helpers import scan

from aleph.core import es, cache
from aleph.model import Entity
from aleph.index.indexes import entities_write_index, entities_read_index
from aleph.index.util import unpack_result, refresh_sync
from aleph.index.util import index_safe, authz_query, bulk_actions
from aleph.index.util import MAX_PAGE

log = logging.getLogger(__name__)
EXCLUDE_DEFAULT = ['text', 'fingerprints', 'names', 'phones', 'emails',
                   'identifiers', 'addresses', 'properties.bodyText',
                   'properties.bodyHtml', 'properties.headers']

def index_entity(entity, sync=False):
    """Index an entity."""
    if entity.deleted_at is not None:
        return delete_entity(entity.id)

    entity_id, index, data = index_operation(entity.to_dict())
    refresh = refresh_sync(sync)
    # This is required if an entity changes its type:
    # delete_entity(entity_id, exclude=proxy.schema, sync=False)
    return index_safe(index, entity_id, data, refresh=refresh)

def index_collection_entities(collection, sync=False):
    """Re-index all documents in a collection in one go."""
    from aleph.index.documents import generate_collection_docs

    def _generate():
        for entity in Entity.by_collection(collection.id):
            entity_id, index, body = index_operation(entity.to_dict())
            yield {
                '_id': entity_id,
                '_index': index,
                '_type': 'doc',
                '_source': body
            }
        yield from generate_collection_docs(collection)

    bulk_actions(_generate(), sync=sync)

def _source_spec(includes, excludes):
    includes = ensure_list(includes)
    excludes = ensure_list(excludes)
    if not len(excludes):
        excludes = EXCLUDE_DEFAULT
    return {'includes': includes, 'excludes': excludes}

def iter_entities(authz=None, collection_id=None, schemata=None,
                  includes=None, excludes=None):
    """Scan all entities matching the given criteria."""
    filters = []
    if authz is not None:
        filters.append(authz_query(authz))
    if collection_id is not None:
        filters.append({'term': {'collection_id': collection_id}})
    if ensure_list(schemata):
        filters.append({'terms': {'schemata': ensure_list(schemata)}})
    query = {
        'query': {'bool': {'filter': filters}},
        '_source': _source_spec(includes, excludes)
    }
    index = entities_read_index(schema=schemata)
    for res in scan(es, index=index, query=query, scroll='1410m'):
        entity = unpack_result(res)
        if entity is not None:
            yield entity

def iter_proxies(**kw):
    includes = ['schema', 'properties']
    for data in iter_entities(includes=includes, **kw):
        schema = model.get(data.get('schema'))
        if schema is None:
            continue
        yield model.get_proxy(data)

def entities_by_ids(ids, schemata=None, cached=False,
                    includes=None, excludes=None):
    """Iterate over unpacked entities based on a search for the given
    entity IDs."""
    ids = ensure_list(ids)
    if not len(ids):
        return
    index = entities_read_index(schema=schemata)
    query = {'ids': {'values': ids}}
    # query = {'bool': {'filter': query}}
    query = {
        'query': query,
        '_source': _source_spec(includes, excludes),
        'size': MAX_PAGE
    }
    result = es.search(index=index, body=query)
    for doc in result.get('hits', {}).get('hits', []):
        entity = unpack_result(doc)
        if entity is not None:
            # Cache entities only briefly to avoid filling up the cache:
            if cached:
                key = cache.object_key(Entity, entity.get('id'))
                cache.set_complex(key, entity, expire=60 * 60)
            yield entity

def get_entity(entity_id, **kwargs):
    """Fetch an entity from the index."""
    if entity_id is None:
        return
    key = cache.object_key(Entity, entity_id)
    entity = cache.get_complex(key)
    if entity is not None:
        return entity
    log.debug("Entity [%s]: object cache miss", entity_id)
    for entity in entities_by_ids(entity_id, cached=True):
        return entity

def _index_updates(collection_id, entities, merge=True):
    """Look up existing index documents and generate an updated form.

    This is necessary to make the index accumulative, i.e. if an entity or link
    gets indexed twice with different field values, it'll add up the different
    field values into a single record. This is to avoid overwriting the
    document and losing field values. An alternative solution would be to
    implement this in Groovy on the ES.
    """
    indexes = defaultdict(list)
    timestamps = {}
    common = {
        'collection_id': collection_id,
        'updated_at': datetime.utcnow(),
        'bulk': True
    }

    if merge:
        for result in entities_by_ids(list(entities.keys())):
            existing = model.get_proxy(result)
            indexes[existing.id].append(result.get('_index'))
            entities[existing.id].merge(existing)
            timestamps[existing.id] = result.get('created_at')

    actions = []
    for entity_id, proxy in entities.items():
        entity = dict(common)
        created_at = timestamps.get(proxy.id, common.get('updated_at'))
        entity['created_at'] = created_at
        entity.update(proxy.to_full_dict())
        _, index, body = index_operation(entity)
        for other in indexes.get(entity_id, []):
            if other != index:
                # log.info("Delete ID [%s] from index: %s", entity_id, other)
                actions.append(delete_operation(other, entity_id))
        actions.append({
            '_id': entity_id,
            '_index': index,
            '_type': 'doc',
            '_source': body
        })
    return actions

def index_bulk(collection_id, entities, merge=True):
    """Index a set of entities."""
    actions = _index_updates(collection_id, entities, merge=merge)
    bulk_actions(actions, sync=merge)

def delete_entity(entity_id, exclude=None, sync=False):
    """Delete an entity from the index."""
    if exclude is not None:
        exclude = entities_write_index(exclude)
    for entity in entities_by_ids(entity_id, excludes='*'):
        index = entity.get('_index')
        if index == exclude:
            continue
        es.delete(index=index, doc_type='doc', id=entity_id,
                  refresh=refresh_sync(sync))

def index_operation(data):
    """Apply final denormalisations to the index."""
    data['bulk'] = data.get('bulk', False)
    names = ensure_list(data.get('names'))
    fps = set([fingerprints.generate(name) for name in names])
    fps.update(names)
    data['fingerprints'] = [fp for fp in fps if fp is not None]

    # Slight hack: a magic property in followthemoney that gets taken out
    # of the properties and added straight to the index text.
    texts = data.pop('text', [])
    texts.extend(data.get('properties', {}).pop('indexText', []))
    texts.extend(fps)
    data['text'] = texts

    if not data.get('created_at'):
        data['created_at'] = data.get('updated_at')

    entity_id = str(data.pop('id'))
    data.pop('_index', None)
    index = entities_write_index(data.get('schema'))
    return entity_id, index, data

def delete_operation(index, entity_id):
    return {
        '_id': entity_id,
        '_index': index,
        '_type': 'doc',
        '_op_type': 'delete'
    }
<|endoftext|>


<commit_before>#!/usr/bin/python

import os.path
import subprocess
import sys
import gyp.common
import gyp.MSVSNew as MSVSNew
import gyp.MSVSProject as MSVSProject

generator_default_variables = {
    'OS': 'win',
    }

def _FixPath(path):
  """Convert paths to a form that will make sense in a vcproj file.

  Arguments:
    path: The path to convert, may contain / etc.
  Returns:
    The path with cygwinisms removed, and all slashes made into backslashes.
  """
  if sys.platform == 'cygwin':
    path = subprocess.Popen(['cygpath', '-w', path],
                            stdout=subprocess.PIPE).communicate()[0]
    path = path.replace('\n', '')
    return path
  else:
    return path.replace('/', '\\')

def _SourceInFolders(sources, prefix=None):
  """Converts a list split source file paths into a vcproj folder hierarchy.

  Arguments:
    sources: A list of source file paths split.
    prefix: A list of source file path layers meant to apply to each of sources.
  Returns:
    A hierarchy of filenames and MSVSProject.Filter objects that matches the
    layout of the source tree.
    For example:
    _SourceInFolders([['a', 'bob1.c'], ['b', 'bob2.c']], prefix=['joe'])
    -->
    [MSVSProject.Filter('a', contents=['joe\\a\\bob1.c']),
     MSVSProject.Filter('b', contents=['joe\\b\\bob2.c'])]
  """
  if not prefix: prefix = []
  result = []
  folders = dict()
  for s in sources:
    if len(s) == 1:
      filename = '\\'.join(prefix + s)
      result.append(filename)
    else:
      if not folders.get(s[0]):
        folders[s[0]] = []
      folders[s[0]].append(s[1:])
  for f in folders:
    contents = _SourceInFolders(folders[f], prefix + [f])
    contents = MSVSProject.Filter(f, contents=contents)
    result.append(contents)

  return result

def _GenerateProject(vcproj_filename, build_file, spec):
  """Generates a vcproj file.

  Arguments:
    vcproj_filename: Filename of the vcproj file to generate.
    build_file: Filename of the .gyp file that the vcproj file comes from.
    spec: The target dictionary containing the properties of the target.
  """
  print 'Generating %s' % vcproj_filename

  p = MSVSProject.Writer(vcproj_filename)
  p.Create(spec['name'])

  # Get directory project file is in.
  gyp_dir = os.path.split(vcproj_filename)[0]
  # Figure out which vsprops to use.
  vsprops_dirs = spec.get('vs_props', [])
  # Make them relative to the gyp file.
  vsprops_dirs = [os.path.join(gyp_dir, i) for i in vsprops_dirs]
  vsprops_dirs = [_FixPath(i) for i in vsprops_dirs]

  # Prepare compiler tool.
  include_dirs = spec.get('include_dirs', [])
  include_dirs = [os.path.join(gyp_dir, i) for i in include_dirs]
  include_dirs = [_FixPath(i) for i in include_dirs]
  defines = ['OS_WIN'] + spec.get('defines', [])
  disabled_warnings = [str(i) for i in spec.get('vs_disabled_warnings', [])]
  compiler_tool = MSVSProject.Tool(
      'VCCLCompilerTool', {
          'AdditionalIncludeDirectories': ';'.join(include_dirs),
          'PreprocessorDefinitions': ';'.join(defines),
          'DisableSpecificWarnings': ';'.join(disabled_warnings),
      })

  # Pre-build and Post-build.
  prebuild = MSVSProject.Tool(
      'VCPreBuildEventTool', {
          'CommandLine': spec.get('vs_prebuild', ''),
      })
  postbuild = MSVSProject.Tool(
      'VCPostBuildEventTool', {
          'CommandLine': spec.get('vs_postbuild', ''),
      })

  # Prepare tools.
  tools = [
      prebuild,
      'VCCustomBuildTool',
      'VCXMLDataGeneratorTool',
      'VCWebServiceProxyGeneratorTool',
      'VCMIDLTool',
      compiler_tool,
      'VCManagedResourceCompilerTool',
      'VCResourceCompilerTool',
      'VCPreLinkEventTool',
      'VCLibrarianTool',
      'VCALinkTool',
      'VCXDCMakeTool',
      'VCBscMakeTool',
      'VCFxCopTool',
      postbuild,
      ]

  # Pick configuration type.
  config_type = {
      'executable': '1',
      'shared_library': '2',
      'static_library': '4',
      'none': '10',
      }[spec['type']]

  # Add Debug/Release.
  p.AddConfig(
      'Debug|Win32',
      attrs={'InheritedPropertySheets':
             ';'.join(['$(SolutionDir)\\..\\build\\debug.vsprops'] +
                      vsprops_dirs),
             'ConfigurationType': config_type},
      tools=tools)
  p.AddConfig(
      'Release|Win32',
      attrs={'InheritedPropertySheets':
             ';'.join(['$(SolutionDir)\\..\\build\\release.vsprops'] +
                      vsprops_dirs),
             'ConfigurationType': config_type},
      tools=tools)
  # Prepare list of sources.
  sources = spec['sources']
  # Add in the gyp file.
  sources.append(os.path.split(build_file)[1])
  # Convert to folders and the right slashes.
  sources = [i.replace('/', '\\') for i in sources]
  sources = [i.split('\\') for i in sources]
  sources = _SourceInFolders(sources)
  # Add in files.
  p.AddFiles(sources)
  # Write it out.
  p.Write()

  # Return the guid so we can refer to it elsewhere.
  return p.guid

def _PopulateProjectObjects(projects, target_list, target_dicts,
                            qualified_target):
  # Done if this project has an object.
  if projects[qualified_target].get('obj'): return
  # Get dependencies for this project.
  spec = target_dicts[qualified_target]
  deps = spec.get('dependencies', [])
  # Populate all dependencies first.
  for d in deps:
    _PopulateProjectObjects(projects, target_list, target_dicts, d)
  # Get objects for each dependency.
  deps = [projects[d]['obj'] for d in deps]
  # Create object for this project.
  projects[qualified_target]['obj'] = MSVSNew.MSVSProject(
      _FixPath(projects[qualified_target]['vcproj_path']),
      name=spec['name'],
      guid=projects[qualified_target]['guid'],
      dependencies=deps)

def _DependentProjects(target_dicts, roots):
  dependents = set()
  for r in roots:
    spec = target_dicts[r]
    r_deps = spec.get('dependencies', [])
    for d in r_deps:
      if d not in roots:
        dependents.add(d)
    for d in _DependentProjects(target_dicts, r_deps):
      if d not in roots:
        dependents.add(d)
  return list(dependents)

def GenerateOutput(target_list, target_dicts, data):
  """Generate .sln and .vcproj files.

  This is the entry point for this generator.
  Arguments:
    target_list: List of target pairs: 'base/base.gyp:base'.
    target_dicts: Dict of target properties keyed on target pair.
    data: Dictionary containing per .gyp data.
  """
  solutions = {}
  for build_file, unused_build_file_dict in data.iteritems():
    if build_file[-4:] != '.gyp':
      # TODO(mark): Pick an exception class
      raise 'Build file name must end in .gyp'
    build_file_stem = build_file[:-4]
    solutions[build_file] = {
        'sln_path': os.path.abspath(build_file_stem + '_gyp.sln'),
    }

  projects = {}
  for qualified_target in target_list:
    build_file = gyp.common.BuildFileAndTarget('', qualified_target)[0]
    spec = target_dicts[qualified_target]
    vcproj_path = os.path.abspath(os.path.join(os.path.split(build_file)[0],
                                               spec['name'] + '_gyp.vcproj'))
    projects[qualified_target] = {
        'vcproj_path': vcproj_path,
        'guid': _GenerateProject(vcproj_path, build_file, spec),
    }

  for qualified_target in target_list:
    _PopulateProjectObjects(projects, target_list, target_dicts,
                            qualified_target)

  for s, s_dict in solutions.iteritems():
    sln_path = s_dict['sln_path']
    print 'Generating %s' % sln_path
    # Get projects in the solution, and their dependents in a separate bucket.
    sln_projects = [p for p in target_list if
                    gyp.common.BuildFileAndTarget('', p)[0] == s]
    dep_projects = _DependentProjects(target_dicts, sln_projects)
    # Convert to entries.
    entries = [projects[e]['obj'] for e in sln_projects]
    dep_entries = [projects[e]['obj'] for e in dep_projects]
    entries.append(MSVSNew.MSVSFolder('dependencies', entries=dep_entries))
    # Create solution.
    sln = MSVSNew.MSVSSolution(_FixPath(sln_path),
                               entries=entries,
                               variants=[
                                   'Debug|Win32',
                                   'Release|Win32',
                               ])
    sln.Write()
<commit_msg>Only emit Command lines for prebuild/postbuild when there is one.<commit_after>#!/usr/bin/python

import os.path
import subprocess
import sys
import gyp.common
import gyp.MSVSNew as MSVSNew
import gyp.MSVSProject as MSVSProject

generator_default_variables = {
    'OS': 'win',
    }

def _FixPath(path):
  """Convert paths to a form that will make sense in a vcproj file.

  Arguments:
    path: The path to convert, may contain / etc.
  Returns:
    The path with cygwinisms removed, and all slashes made into backslashes.
  """
  if sys.platform == 'cygwin':
    path = subprocess.Popen(['cygpath', '-w', path],
                            stdout=subprocess.PIPE).communicate()[0]
    path = path.replace('\n', '')
    return path
  else:
    return path.replace('/', '\\')

def _SourceInFolders(sources, prefix=None):
  """Converts a list split source file paths into a vcproj folder hierarchy.

  Arguments:
    sources: A list of source file paths split.
    prefix: A list of source file path layers meant to apply to each of sources.
  Returns:
    A hierarchy of filenames and MSVSProject.Filter objects that matches the
    layout of the source tree.
    For example:
    _SourceInFolders([['a', 'bob1.c'], ['b', 'bob2.c']], prefix=['joe'])
    -->
    [MSVSProject.Filter('a', contents=['joe\\a\\bob1.c']),
     MSVSProject.Filter('b', contents=['joe\\b\\bob2.c'])]
  """
  if not prefix: prefix = []
  result = []
  folders = dict()
  for s in sources:
    if len(s) == 1:
      filename = '\\'.join(prefix + s)
      result.append(filename)
    else:
      if not folders.get(s[0]):
        folders[s[0]] = []
      folders[s[0]].append(s[1:])
  for f in folders:
    contents = _SourceInFolders(folders[f], prefix + [f])
    contents = MSVSProject.Filter(f, contents=contents)
    result.append(contents)

  return result

def _GenerateProject(vcproj_filename, build_file, spec):
  """Generates a vcproj file.

  Arguments:
    vcproj_filename: Filename of the vcproj file to generate.
    build_file: Filename of the .gyp file that the vcproj file comes from.
    spec: The target dictionary containing the properties of the target.
  """
  print 'Generating %s' % vcproj_filename

  p = MSVSProject.Writer(vcproj_filename)
  p.Create(spec['name'])

  # Get directory project file is in.
  gyp_dir = os.path.split(vcproj_filename)[0]
  # Figure out which vsprops to use.
  vsprops_dirs = spec.get('vs_props', [])
  # Make them relative to the gyp file.
  vsprops_dirs = [os.path.join(gyp_dir, i) for i in vsprops_dirs]
  vsprops_dirs = [_FixPath(i) for i in vsprops_dirs]

  # Prepare compiler tool.
  include_dirs = spec.get('include_dirs', [])
  include_dirs = [os.path.join(gyp_dir, i) for i in include_dirs]
  include_dirs = [_FixPath(i) for i in include_dirs]
  defines = ['OS_WIN'] + spec.get('defines', [])
  disabled_warnings = [str(i) for i in spec.get('vs_disabled_warnings', [])]
  compiler_tool = MSVSProject.Tool(
      'VCCLCompilerTool', {
          'AdditionalIncludeDirectories': ';'.join(include_dirs),
          'PreprocessorDefinitions': ';'.join(defines),
          'DisableSpecificWarnings': ';'.join(disabled_warnings),
      })

  # Pre-build.
  if spec.get('vs_prebuild'):
    prebuild_dict = { 'CommandLine': spec.get('vs_prebuild', '') }
  else:
    prebuild_dict = {}
  prebuild = MSVSProject.Tool('VCPreBuildEventTool', prebuild_dict)
  # Post-build.
  if spec.get('vs_postbuild'):
    postbuild_dict = { 'CommandLine': spec.get('vs_postbuild', '') }
  else:
    postbuild_dict = {}
  postbuild = MSVSProject.Tool('VCPostBuildEventTool', postbuild_dict)

  # Prepare tools.
  tools = [
      prebuild,
      'VCCustomBuildTool',
      'VCXMLDataGeneratorTool',
      'VCWebServiceProxyGeneratorTool',
      'VCMIDLTool',
      compiler_tool,
      'VCManagedResourceCompilerTool',
      'VCResourceCompilerTool',
      'VCPreLinkEventTool',
      'VCLibrarianTool',
      'VCALinkTool',
      'VCXDCMakeTool',
      'VCBscMakeTool',
      'VCFxCopTool',
      postbuild,
      ]

  # Pick configuration type.
  config_type = {
      'executable': '1',
      'shared_library': '2',
      'static_library': '4',
      'none': '10',
      }[spec['type']]

  # Add Debug/Release.
  p.AddConfig(
      'Debug|Win32',
      attrs={'InheritedPropertySheets':
             ';'.join(['$(SolutionDir)\\..\\build\\debug.vsprops'] +
                      vsprops_dirs),
             'ConfigurationType': config_type},
      tools=tools)
  p.AddConfig(
      'Release|Win32',
      attrs={'InheritedPropertySheets':
             ';'.join(['$(SolutionDir)\\..\\build\\release.vsprops'] +
                      vsprops_dirs),
             'ConfigurationType': config_type},
      tools=tools)
  # Prepare list of sources.
  sources = spec['sources']
  # Add in the gyp file.
  sources.append(os.path.split(build_file)[1])
  # Convert to folders and the right slashes.
  sources = [i.replace('/', '\\') for i in sources]
  sources = [i.split('\\') for i in sources]
  sources = _SourceInFolders(sources)
  # Add in files.
  p.AddFiles(sources)
  # Write it out.
  p.Write()

  # Return the guid so we can refer to it elsewhere.
  return p.guid

def _PopulateProjectObjects(projects, target_list, target_dicts,
                            qualified_target):
  # Done if this project has an object.
  if projects[qualified_target].get('obj'): return
  # Get dependencies for this project.
  spec = target_dicts[qualified_target]
  deps = spec.get('dependencies', [])
  # Populate all dependencies first.
  for d in deps:
    _PopulateProjectObjects(projects, target_list, target_dicts, d)
  # Get objects for each dependency.
  deps = [projects[d]['obj'] for d in deps]
  # Create object for this project.
  projects[qualified_target]['obj'] = MSVSNew.MSVSProject(
      _FixPath(projects[qualified_target]['vcproj_path']),
      name=spec['name'],
      guid=projects[qualified_target]['guid'],
      dependencies=deps)

def _DependentProjects(target_dicts, roots):
  dependents = set()
  for r in roots:
    spec = target_dicts[r]
    r_deps = spec.get('dependencies', [])
    for d in r_deps:
      if d not in roots:
        dependents.add(d)
    for d in _DependentProjects(target_dicts, r_deps):
      if d not in roots:
        dependents.add(d)
  return list(dependents)

def GenerateOutput(target_list, target_dicts, data):
  """Generate .sln and .vcproj files.

  This is the entry point for this generator.
  Arguments:
    target_list: List of target pairs: 'base/base.gyp:base'.
    target_dicts: Dict of target properties keyed on target pair.
    data: Dictionary containing per .gyp data.
  """
  solutions = {}
  for build_file, unused_build_file_dict in data.iteritems():
    if build_file[-4:] != '.gyp':
      # TODO(mark): Pick an exception class
      raise 'Build file name must end in .gyp'
    build_file_stem = build_file[:-4]
    solutions[build_file] = {
        'sln_path': os.path.abspath(build_file_stem + '_gyp.sln'),
    }

  projects = {}
  for qualified_target in target_list:
    build_file = gyp.common.BuildFileAndTarget('', qualified_target)[0]
    spec = target_dicts[qualified_target]
    vcproj_path = os.path.abspath(os.path.join(os.path.split(build_file)[0],
                                               spec['name'] + '_gyp.vcproj'))
    projects[qualified_target] = {
        'vcproj_path': vcproj_path,
        'guid': _GenerateProject(vcproj_path, build_file, spec),
    }

  for qualified_target in target_list:
    _PopulateProjectObjects(projects, target_list, target_dicts,
                            qualified_target)

  for s, s_dict in solutions.iteritems():
    sln_path = s_dict['sln_path']
    print 'Generating %s' % sln_path
    # Get projects in the solution, and their dependents in a separate bucket.
    sln_projects = [p for p in target_list if
                    gyp.common.BuildFileAndTarget('', p)[0] == s]
    dep_projects = _DependentProjects(target_dicts, sln_projects)
    # Convert to entries.
    entries = [projects[e]['obj'] for e in sln_projects]
    dep_entries = [projects[e]['obj'] for e in dep_projects]
    entries.append(MSVSNew.MSVSFolder('dependencies', entries=dep_entries))
    # Create solution.
    sln = MSVSNew.MSVSSolution(_FixPath(sln_path),
                               entries=entries,
                               variants=[
                                   'Debug|Win32',
                                   'Release|Win32',
                               ])
    sln.Write()
<|endoftext|>


<commit_before>import logging
import os
import re
from subprocess import call, check_call
import threading
import time

from flask import Flask, request
from ironicclient import client, exceptions

app = Flask(__name__)

LOG = logging.getLogger("discoverd")
OS_ARGS = dict((k.lower(), v)
               for (k, v) in os.environ.items()
               if k.startswith('OS_'))
ALLOW_SEARCH_BY_MAC = True

def is_valid_mac(address):
    m = "[0-9a-f]{2}(:[0-9a-f]{2}){5}$"
    return (isinstance(address, (str, unicode))
            and re.match(m, address.lower()))

def process(node_info):
    """Process data from discovery ramdisk."""
    if node_info.get('error'):
        LOG.error('Error happened during discovery: %s',
                  node_info['error'])
        return

    keys = ('cpus', 'cpu_arch', 'memory_mb', 'local_gb', 'macs')
    missing = [key for key in keys if not node_info.get(key)]
    if missing:
        LOG.error('The following required parameters are missing: %s',
                  missing)
        return

    valid_macs = [mac.lower() for mac in node_info['macs']
                  if is_valid_mac(mac)]
    if valid_macs != node_info['macs']:
        LOG.warn('The following MACs were invalid in discovery data '
                 'for node with BMC %(ipmi_address)s and were '
                 'excluded: %(invalid)s',
                 {'invalid': set(node_info['macs']) - set(valid_macs),
                  'ipmi_address': node_info.get('ipmi_address')})

    LOG.info('Discovery data received from node with BMC '
             '%(ipmi_address)s: CPUs: %(cpus)s %(cpu_arch)s, '
             'memory %(memory_mb)s MiB, disk %(local_gb)s GiB, '
             'macs %(macs)s',
             dict((key, node_info.get(key))
                  for key in keys + ('ipmi_address',)))

    ironic = client.get_client(1, **OS_ARGS)
    bmc_known = bool(node_info.get('ipmi_address'))
    if bmc_known:
        # TODO(dtantsur): bulk loading
        nodes = ironic.node.list(maintenance=True, limit=0,
                                 sort_key='created_at',
                                 sort_dir='desc', detail=True)
        address = node_info['ipmi_address']
        for node in nodes:
            if node.driver_info.get('ipmi_address') == address:
                break
        else:
            LOG.error('Unable to find node with ipmi_address %s',
                      node_info['ipmi_address'])
            return
    elif ALLOW_SEARCH_BY_MAC:
        # In case of testing with vms and pxe_ssh driver
        LOG.warn('No BMC address provided, trying to use MAC '
                 'addresses for finding node')
        port = None
        for mac in valid_macs:
            try:
                port = ironic.port.get_by_address(mac)
            except exceptions.NotFound:
                continue
            else:
                break

        if port is not None:
            try:
                node = ironic.node.get(port.node_uuid)
            except exceptions.NotFound:
                node = None

        if port is None or node is None:
            LOG.error('Unable to find node with macs %s',
                      valid_macs)
            return
    else:
        LOG.error('No ipmi_address provided and searching by MAC is not '
                  'allowed')
        return

    if not node.maintenance:
        LOG.error('Refusing to apply discovered data to node %s '
                  'which is not in maintenance state', node.uuid)
        return

    patch = [{'op': 'add', 'path': '/extra/newly_discovered', 'value': 'true'}]
    existing = node.properties
    for key in ('cpus', 'cpu_arch', 'memory_mb', 'local_gb'):
        if not existing.get(key):
            patch.append({'op': 'add', 'path': '/properties/%s' % key,
                          'value': str(node_info[key])})
    ironic.node.update(node.uuid, patch)

    for mac in valid_macs:
        try:
            ironic.port.create(node_uuid=node.uuid, address=mac)
        except exceptions.Conflict:
            LOG.warning('MAC %(mac)s appeared in discovery data for '
                        'node %(node)s, but already exists in '
                        'database for another node - skipping',
                        {'mac': mac, 'node': node.uuid})

    LOG.info('Node %s was updated with data from discovery process, forcing '
             'power off', node.uuid)

    ironic.node.set_power_state(node.uuid, 'off')

    Firewall.unwhitelist_macs(valid_macs)

class Firewall(object):
    LOCK = threading.RLock()
    MACS_DISCOVERY = set()
    NEW_CHAIN = 'discovery_temp'
    CHAIN = 'discovery'
    INTERFACE = 'br-ctlplane'

    @staticmethod
    def _iptables(*args, **kwargs):
        cmd = ('iptables',) + args
        LOG.debug('Running iptables %s', cmd)
        if kwargs.pop('ignore', False):
            if call(cmd, **kwargs):
                LOG.warn('iptables failed: %s', args)
                return False
            else:
                return True
        else:
            return check_call(cmd, **kwargs)

    @classmethod
    def init(cls):
        cls._iptables('-F', cls.NEW_CHAIN, ignore=True)
        cls._iptables('-X', cls.NEW_CHAIN, ignore=True)
        cls._iptables('-F', cls.CHAIN, ignore=True)
        cls._iptables('-X', cls.CHAIN, ignore=True)
        # Code expects it to exist
        cls._iptables('-N', cls.CHAIN)

    @classmethod
    def whitelist_macs(cls, macs):
        with cls.LOCK:
            cls.MACS_DISCOVERY.update(macs)

    @classmethod
    def unwhitelist_macs(cls, macs):
        with cls.LOCK:
            cls.MACS_DISCOVERY.difference_update(macs)

    @classmethod
    def update_filters(cls, ironic):
        with cls.LOCK:
            macs_active = set(p.address for p in ironic.port.list(limit=0))
            to_blacklist = macs_active - cls.MACS_DISCOVERY

            # Operate on temporary chain
            cls._iptables('-N', cls.NEW_CHAIN)
            # - Blacklist active macs, so that nova can boot them
            for mac in to_blacklist:
                cls._iptables('-A', cls.NEW_CHAIN, '-m', 'mac',
                              '--mac-source', mac, '-j', 'DROP')
            # - Whitelist everything else
            cls._iptables('-A', cls.NEW_CHAIN, '-j', 'ACCEPT')

            # Swap chains
            cls._iptables('-I', 'input', '-i', cls.INTERFACE, '-p' 'udp',
                          '--dport', '67', '-j', cls.NEW_CHAIN)
            cls._iptables('-D', 'input', '-i', cls.INTERFACE, '-p' 'udp',
                          '--dport', '67', '-j', cls.CHAIN,
                          ignore=True)  # may be missing on first run
            cls._iptables('-F', cls.CHAIN)
            cls._iptables('-X', cls.CHAIN)
            cls._iptables('-E', cls.NEW_CHAIN, cls.CHAIN)

def start(uuids):
    """Initiate discovery for given node uuids."""
    ironic = client.get_client(1, **OS_ARGS)
    LOG.debug('Validating nodes %s', uuids)
    nodes = []
    for uuid in uuids:
        try:
            node = ironic.node.get(uuid)
        except exceptions.HTTPClientError:
            LOG.exception('Failed validation of node %s', uuid)
            continue

        if not node.maintenance:
            LOG.error('Node %s not in maintenance - skipping', uuid)
            continue

        nodes.append(node)

    LOG.info('Proceeding with discovery on nodes %s', [n.uuid for n in nodes])

    to_exclude = set()
    for node in nodes:
        if not node.driver.endswith('ssh'):
            continue

        LOG.warn('Driver for %s is %s, requires white-listing MAC',
                 node.uuid, node.driver)

        # TODO(dtantsur): pagination
        ports = ironic.node.list_ports(node.uuid, limit=0)
        to_exclude.update(p.address for p in ports)

    if to_exclude:
        Firewall.whitelist_macs(to_exclude)
        Firewall.update_filters(ironic)

    for node in nodes:
        ironic.node.set_power_state(node.uuid, 'on')

@app.route('/continue', methods=['POST'])
def post_continue():
    data = request.get_json(force=True)
    LOG.debug("Got JSON %s, going into processing thread", data)
    threading.Thread(target=process, args=(data,)).start()
    return "{}", 202, {"content-type": "application/json"}

@app.route('/start', methods=['POST'])
def post_start():
    data = request.get_json(force=True)
    LOG.debug("Got JSON %s, going into processing thread", data)
    threading.Thread(target=start, args=(data,)).start()
    return "{}", 202, {"content-type": "application/json"}

def periodic_update(event, ironic):
    while not event.is_set():
        LOG.info('Running periodic update of filters')
        Firewall.update_filters(ironic)
        time.sleep(30)

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    ironic = client.get_client(1, **OS_ARGS)
    Firewall.init()
    event = threading.Event()
    threading.Thread(target=periodic_update, args=(event, ironic)).start()
    try:
        app.run(debug=True, host='0.0.0.0', port=5050)
    finally:
        LOG.info('Waiting for background thread to shutdown')
        event.set()
<commit_msg>Small fixes<commit_after>import logging
import os
import re
from subprocess import call, check_call
import threading
import time

from flask import Flask, request
from ironicclient import client, exceptions

app = Flask(__name__)

LOG = logging.getLogger("discoverd")
OS_ARGS = dict((k.lower(), v)
               for (k, v) in os.environ.items()
               if k.startswith('OS_'))
ALLOW_SEARCH_BY_MAC = True

def is_valid_mac(address):
    m = "[0-9a-f]{2}(:[0-9a-f]{2}){5}$"
    return (isinstance(address, (str, unicode))
            and re.match(m, address.lower()))

def process(node_info):
    """Process data from discovery ramdisk."""
    if node_info.get('error'):
        LOG.error('Error happened during discovery: %s',
                  node_info['error'])
        return

    keys = ('cpus', 'cpu_arch', 'memory_mb', 'local_gb', 'macs')
    missing = [key for key in keys if not node_info.get(key)]
    if missing:
        LOG.error('The following required parameters are missing: %s',
                  missing)
        return

    valid_macs = [mac.lower() for mac in node_info['macs']
                  if is_valid_mac(mac)]
    if valid_macs != node_info['macs']:
        LOG.warn('The following MACs were invalid in discovery data '
                 'for node with BMC %(ipmi_address)s and were '
                 'excluded: %(invalid)s',
                 {'invalid': set(node_info['macs']) - set(valid_macs),
                  'ipmi_address': node_info.get('ipmi_address')})

    LOG.info('Discovery data received from node with BMC '
             '%(ipmi_address)s: CPUs: %(cpus)s %(cpu_arch)s, '
             'memory %(memory_mb)s MiB, disk %(local_gb)s GiB, '
             'macs %(macs)s',
             dict((key, node_info.get(key))
                  for key in keys + ('ipmi_address',)))

    ironic = client.get_client(1, **OS_ARGS)
    bmc_known = bool(node_info.get('ipmi_address'))
    if bmc_known:
        # TODO(dtantsur): bulk loading
        nodes = ironic.node.list(maintenance=True, limit=0,
                                 sort_key='created_at',
                                 sort_dir='desc', detail=True)
        address = node_info['ipmi_address']
        for node in nodes:
            if node.driver_info.get('ipmi_address') == address:
                break
        else:
            LOG.error('Unable to find node with ipmi_address %s',
                      node_info['ipmi_address'])
            return
    elif ALLOW_SEARCH_BY_MAC:
        # In case of testing with vms and pxe_ssh driver
        LOG.warn('No BMC address provided, trying to use MAC '
                 'addresses for finding node')
        port = None
        for mac in valid_macs:
            try:
                port = ironic.port.get_by_address(mac)
            except exceptions.NotFound:
                continue
            else:
                break

        if port is not None:
            try:
                node = ironic.node.get(port.node_uuid)
            except exceptions.NotFound:
                node = None

        if port is None or node is None:
            LOG.error('Unable to find node with macs %s',
                      valid_macs)
            return
    else:
        LOG.error('No ipmi_address provided and searching by MAC is not '
                  'allowed')
        return

    if not node.maintenance:
        LOG.error('Refusing to apply discovered data to node %s '
                  'which is not in maintenance state', node.uuid)
        return

    patch = [{'op': 'add', 'path': '/extra/newly_discovered', 'value': 'true'}]
    existing = node.properties
    for key in ('cpus', 'cpu_arch', 'memory_mb', 'local_gb'):
        if not existing.get(key):
            patch.append({'op': 'add', 'path': '/properties/%s' % key,
                          'value': str(node_info[key])})
    ironic.node.update(node.uuid, patch)

    for mac in valid_macs:
        try:
            ironic.port.create(node_uuid=node.uuid, address=mac)
        except exceptions.Conflict:
            LOG.warning('MAC %(mac)s appeared in discovery data for '
                        'node %(node)s, but already exists in '
                        'database for another node - skipping',
                        {'mac': mac, 'node': node.uuid})

    LOG.info('Node %s was updated with data from discovery process, forcing '
             'power off', node.uuid)

    ironic.node.set_power_state(node.uuid, 'off')

    Firewall.unwhitelist_macs(valid_macs)

class Firewall(object):
    LOCK = threading.RLock()
    MACS_DISCOVERY = set()
    NEW_CHAIN = 'discovery_temp'
    CHAIN = 'discovery'
    INTERFACE = 'br-ctlplane'

    @staticmethod
    def _iptables(*args, **kwargs):
        cmd = ('iptables',) + args
        LOG.debug('Running iptables %s', args)
        if kwargs.pop('ignore', False):
            if call(cmd, **kwargs):
                LOG.warn('iptables failed: %s', args)
                return False
            else:
                return True
        else:
            try:
                return check_call(cmd, **kwargs)
            except Exception:
                LOG.error('iptables failed: %s', args)
                raise

    @classmethod
    def init(cls):
        cls._iptables('-F', cls.NEW_CHAIN, ignore=True)
        cls._iptables('-X', cls.NEW_CHAIN, ignore=True)
        cls._iptables('-F', cls.CHAIN, ignore=True)
        cls._iptables('-X', cls.CHAIN, ignore=True)
        # Code expects it to exist
        cls._iptables('-N', cls.CHAIN)

    @classmethod
    def whitelist_macs(cls, macs):
        with cls.LOCK:
            cls.MACS_DISCOVERY.update(macs)

    @classmethod
    def unwhitelist_macs(cls, macs):
        with cls.LOCK:
            cls.MACS_DISCOVERY.difference_update(macs)

    @classmethod
    def update_filters(cls, ironic):
        with cls.LOCK:
            macs_active = set(p.address for p in ironic.port.list(limit=0))
            to_blacklist = macs_active - cls.MACS_DISCOVERY

            # Operate on temporary chain
            cls._iptables('-N', cls.NEW_CHAIN)
            # - Blacklist active macs, so that nova can boot them
            for mac in to_blacklist:
                cls._iptables('-A', cls.NEW_CHAIN, '-m', 'mac',
                              '--mac-source', mac, '-j', 'DROP')
            # - Whitelist everything else
            cls._iptables('-A', cls.NEW_CHAIN, '-j', 'ACCEPT')

            # Swap chains
            cls._iptables('-I', 'input', '-i', cls.INTERFACE, '-p', 'udp',
                          '--dport', '67', '-j', cls.NEW_CHAIN)
            cls._iptables('-D', 'input', '-i', cls.INTERFACE, '-p', 'udp',
                          '--dport', '67', '-j', cls.CHAIN,
                          ignore=True)  # may be missing on first run
            cls._iptables('-F', cls.CHAIN)
            cls._iptables('-X', cls.CHAIN)
            cls._iptables('-E', cls.NEW_CHAIN, cls.CHAIN)

def start(uuids):
    """Initiate discovery for given node uuids."""
    ironic = client.get_client(1, **OS_ARGS)
    LOG.debug('Validating nodes %s', uuids)
    nodes = []
    for uuid in uuids:
        try:
            node = ironic.node.get(uuid)
        except exceptions.HTTPClientError:
            LOG.exception('Failed validation of node %s', uuid)
            continue

        if not node.maintenance:
            LOG.error('Node %s not in maintenance - skipping', uuid)
            continue

        nodes.append(node)

    LOG.info('Proceeding with discovery on nodes %s', [n.uuid for n in nodes])

    to_exclude = set()
    for node in nodes:
        if not node.driver.endswith('ssh'):
            continue

        LOG.warn('Driver for %s is %s, requires white-listing MAC',
                 node.uuid, node.driver)

        # TODO(dtantsur): pagination
        ports = ironic.node.list_ports(node.uuid, limit=0)
        to_exclude.update(p.address for p in ports)

    if to_exclude:
        Firewall.whitelist_macs(to_exclude)
        Firewall.update_filters(ironic)

    for node in nodes:
        ironic.node.set_power_state(node.uuid, 'on')

@app.route('/continue', methods=['POST'])
def post_continue():
    data = request.get_json(force=True)
    LOG.debug("Got JSON %s, going into processing thread", data)
    threading.Thread(target=process, args=(data,)).start()
    return "{}", 202, {"content-type": "application/json"}

@app.route('/start', methods=['POST'])
def post_start():
    data = request.get_json(force=True)
    LOG.debug("Got JSON %s, going into processing thread", data)
    threading.Thread(target=start, args=(data,)).start()
    return "{}", 202, {"content-type": "application/json"}

def periodic_update(event, ironic):
    while not event.is_set():
        LOG.info('Running periodic update of filters')
        Firewall.update_filters(ironic)
        time.sleep(30)

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    ironic = client.get_client(1, **OS_ARGS)
    Firewall.init()
    event = threading.Event()
    threading.Thread(target=periodic_update, args=(event, ironic)).start()
    try:
        app.run(debug=True, host='0.0.0.0', port=5050)
    finally:
        LOG.info('Waiting for background thread to shutdown')
        event.set()
<|endoftext|>