<commit_before># -*- coding: utf-8 -*-
"""
.. module:: MyCapytain.retrievers.cts5
   :synopsis: Cts5 endpoint implementation

.. moduleauthor:: Thibault Clérice <leponteineptique@gmail.com>

"""
import MyCapytain.retrievers.prototypes
from MyCapytain.common.reference import Reference
import requests

class HttpCtsRetriever(MyCapytain.retrievers.prototypes.CtsRetriever):
    """ Basic integration of the MyCapytain.retrievers.proto.CTS abstraction
    """

    def __init__(self, endpoint, inventory=None):
        """ API Prototype object

        :param self: Object
        :type self: API
        :param endpoint: URL of the API
        :type endpoint: str
        :param inventory: Inventory to use
        :type inventory: str

        :ivar endpoint: Url of the endpoint
        :ivar inventory: Default Inventory
        """
        super(HttpCtsRetriever, self).__init__(endpoint)
        self.inventory = inventory

    def call(self, parameters):
        """ Call an endpoint given the parameters

        :param parameters: Dictionary of parameters
        :type parameters: dict
        :rtype: text
        """
        # DEV !
        parameters = {
            key: str(parameters[key]) for key in parameters if parameters[key] is not None
        }
        if self.inventory is not None and "inv" not in parameters:
            parameters["inv"] = self.inventory

        request = requests.get(self.endpoint, params=parameters)
        request.raise_for_status()
        return request.text

    def getCapabilities(self, inventory=None, urn=None):
        """ Retrieve the inventory information of an API

        :param inventory: Name of the inventory
        :type inventory: text
        :param urn: URN to filter with
        :type urn: str
        :rtype: str
        """
        return self.call({
            "inv": inventory,
            "urn": urn,
            "request": "GetCapabilities"
        })

    def getValidReff(self, urn, inventory=None, level=None):
        """ Retrieve valid urn-references for a text

        :param urn: URN identifying the text
        :type urn: text
        :param inventory: Name of the inventory
        :type inventory: text
        :param level: Depth of references expected
        :type level: int
        :return: XML Response from the API as string
        :rtype: str
        """
        return self.call({
            "inv": inventory,
            "urn": urn,
            "level": level,
            "request": "GetValidReff"
        })

    def getFirstUrn(self, urn, inventory=None):
        """ Retrieve the first passage urn of a text

        :param urn: URN identifying the text
        :type urn: text
        :param inventory: Name of the inventory
        :type inventory: text
        :rtype: str
        """
        return self.call({
            "inv": inventory,
            "urn": urn,
            "request": "GetFirstUrn"
        })

    def getPrevNextUrn(self, urn, inventory=None):
        """ Retrieve the previous and next passage urn of one passage

        :param urn: URN identifying the text's passage (Minimum depth : 1)
        :type urn: text
        :param inventory: Name of the inventory
        :type inventory: text
        :rtype: str
        """
        return self.call({
            "inv": inventory,
            "urn": urn,
            "request": "GetPrevNextUrn"
        })

    def getLabel(self, urn, inventory=None):
        """ Retrieve informations about a CTS Urn

        :param urn: URN identifying the text's passage (Minimum depth : 1)
        :type urn: text
        :param inventory: Name of the inventory
        :type inventory: text
        :rtype: str
        """
        return self.call({
            "inv": inventory,
            "urn": urn,
            "request": "GetLabel"
        })

    def getPassage(self, urn, inventory=None, context=None):
        """ Retrieve a passage

        :param urn: URN identifying the text's passage (Minimum depth : 1)
        :type urn: text
        :param inventory: Name of the inventory
        :type inventory: text
        :param context: Number of citation units at the same level of the citation hierarchy as the requested urn, immediately preceding and immediately following the requested urn to include in the reply
        :type context: int
        :rtype: str
        """
        return self.call({
            "inv": inventory,
            "urn": urn,
            "context": context,
            "request": "GetPassage"
        })

    def getPassagePlus(self, urn, inventory=None, context=None):
        """ Retrieve a passage and information about it

        :param urn: URN identifying the text's passage (Minimum depth : 1)
        :type urn: text
        :param inventory: Name of the inventory
        :type inventory: text
        :param context: Number of citation units at the same level of the citation hierarchy as the requested urn, immediately preceding and immediately following the requested urn to include in the reply
        :type context: int
        :rtype: str
        """
        return self.call({
            "inv": inventory,
            "urn": urn,
            "context": context,
            "request": "GetPassagePlus"
        })

    #
    # Common methods
    #

    def getMetadata(self, objectId=None, **filters):
        """ Request metadata about a text or a collection

        :param objectId: Filter for some object identifier
        :param filters: Kwargs parameters. URN and Inv are available
        :return: GetCapabilities CTS API request response
        """
        filters.update({"urn": objectId})
        return self.getCapabilities(**filters)

    def getTextualNode(self, textId, subreference=None, prevnext=False, metadata=False):
        """ Retrieve a text node from the API

        :param textId: CtsTextMetadata Identifier
        :param subreference: CapitainsCtsPassage Reference
        :param prevnext: Retrieve graph representing previous and next passage
        :param metadata: Retrieve metadata about the passage and the text
        :return: GetPassage or GetPassagePlus CTS API request response
        """
        if  subreference:
            textId = "{}:{}".format(textId, subreference)

        if prevnext or metadata:
            return self.getPassagePlus(urn=textId)
        else:
            return self.getPassage(urn=textId)

    def getSiblings(self, textId, subreference):
        """ Retrieve the siblings of a textual node

        :param textId: CtsTextMetadata Identifier
        :param reference: CapitainsCtsPassage Reference
        :return: GetPrevNextUrn request response from the endpoint
        """
        textId = "{}:{}".format(textId, subreference)
        return self.getPrevNextUrn(urn=textId)

    def getReffs(self, textId, level=1, subreference=None):
        """ Retrieve the siblings of a textual node

        :param textId: CtsTextMetadata Identifier
        :type textId: str
        :param level: Depth for retrieval
        :type level: int
        :param subreference: CapitainsCtsPassage Reference
        :type subreference: str
        :return: List of references
        :rtype: [str]
        """
        depth = level
        if subreference:
            textId = "{}:{}".format(textId, subreference)
        if subreference:
            if isinstance(subreference, Reference):
                depth += len(subreference)
            else:
                depth += len(Reference(subreference))
        if level:
            level = max(depth, level)
        return self.getValidReff(urn=textId, level=level)
<commit_msg>Explicitly set encoding on responses from CTS API<commit_after># -*- coding: utf-8 -*-
"""
.. module:: MyCapytain.retrievers.cts5
   :synopsis: Cts5 endpoint implementation

.. moduleauthor:: Thibault Clérice <leponteineptique@gmail.com>

"""
import MyCapytain.retrievers.prototypes
from MyCapytain.common.reference import Reference
import requests

class HttpCtsRetriever(MyCapytain.retrievers.prototypes.CtsRetriever):
    """ Basic integration of the MyCapytain.retrievers.proto.CTS abstraction
    """

    def __init__(self, endpoint, inventory=None):
        """ API Prototype object

        :param self: Object
        :type self: API
        :param endpoint: URL of the API
        :type endpoint: str
        :param inventory: Inventory to use
        :type inventory: str

        :ivar endpoint: Url of the endpoint
        :ivar inventory: Default Inventory
        """
        super(HttpCtsRetriever, self).__init__(endpoint)
        self.inventory = inventory

    def call(self, parameters):
        """ Call an endpoint given the parameters

        :param parameters: Dictionary of parameters
        :type parameters: dict
        :rtype: text
        """
        # DEV !
        parameters = {
            key: str(parameters[key]) for key in parameters if parameters[key] is not None
        }
        if self.inventory is not None and "inv" not in parameters:
            parameters["inv"] = self.inventory

        request = requests.get(self.endpoint, params=parameters)
        request.raise_for_status()
        if request.encoding is None:
            request.encoding = "utf-8"
        return request.text

    def getCapabilities(self, inventory=None, urn=None):
        """ Retrieve the inventory information of an API

        :param inventory: Name of the inventory
        :type inventory: text
        :param urn: URN to filter with
        :type urn: str
        :rtype: str
        """
        return self.call({
            "inv": inventory,
            "urn": urn,
            "request": "GetCapabilities"
        })

    def getValidReff(self, urn, inventory=None, level=None):
        """ Retrieve valid urn-references for a text

        :param urn: URN identifying the text
        :type urn: text
        :param inventory: Name of the inventory
        :type inventory: text
        :param level: Depth of references expected
        :type level: int
        :return: XML Response from the API as string
        :rtype: str
        """
        return self.call({
            "inv": inventory,
            "urn": urn,
            "level": level,
            "request": "GetValidReff"
        })

    def getFirstUrn(self, urn, inventory=None):
        """ Retrieve the first passage urn of a text

        :param urn: URN identifying the text
        :type urn: text
        :param inventory: Name of the inventory
        :type inventory: text
        :rtype: str
        """
        return self.call({
            "inv": inventory,
            "urn": urn,
            "request": "GetFirstUrn"
        })

    def getPrevNextUrn(self, urn, inventory=None):
        """ Retrieve the previous and next passage urn of one passage

        :param urn: URN identifying the text's passage (Minimum depth : 1)
        :type urn: text
        :param inventory: Name of the inventory
        :type inventory: text
        :rtype: str
        """
        return self.call({
            "inv": inventory,
            "urn": urn,
            "request": "GetPrevNextUrn"
        })

    def getLabel(self, urn, inventory=None):
        """ Retrieve informations about a CTS Urn

        :param urn: URN identifying the text's passage (Minimum depth : 1)
        :type urn: text
        :param inventory: Name of the inventory
        :type inventory: text
        :rtype: str
        """
        return self.call({
            "inv": inventory,
            "urn": urn,
            "request": "GetLabel"
        })

    def getPassage(self, urn, inventory=None, context=None):
        """ Retrieve a passage

        :param urn: URN identifying the text's passage (Minimum depth : 1)
        :type urn: text
        :param inventory: Name of the inventory
        :type inventory: text
        :param context: Number of citation units at the same level of the citation hierarchy as the requested urn, immediately preceding and immediately following the requested urn to include in the reply
        :type context: int
        :rtype: str
        """
        return self.call({
            "inv": inventory,
            "urn": urn,
            "context": context,
            "request": "GetPassage"
        })

    def getPassagePlus(self, urn, inventory=None, context=None):
        """ Retrieve a passage and information about it

        :param urn: URN identifying the text's passage (Minimum depth : 1)
        :type urn: text
        :param inventory: Name of the inventory
        :type inventory: text
        :param context: Number of citation units at the same level of the citation hierarchy as the requested urn, immediately preceding and immediately following the requested urn to include in the reply
        :type context: int
        :rtype: str
        """
        return self.call({
            "inv": inventory,
            "urn": urn,
            "context": context,
            "request": "GetPassagePlus"
        })

    #
    # Common methods
    #

    def getMetadata(self, objectId=None, **filters):
        """ Request metadata about a text or a collection

        :param objectId: Filter for some object identifier
        :param filters: Kwargs parameters. URN and Inv are available
        :return: GetCapabilities CTS API request response
        """
        filters.update({"urn": objectId})
        return self.getCapabilities(**filters)

    def getTextualNode(self, textId, subreference=None, prevnext=False, metadata=False):
        """ Retrieve a text node from the API

        :param textId: CtsTextMetadata Identifier
        :param subreference: CapitainsCtsPassage Reference
        :param prevnext: Retrieve graph representing previous and next passage
        :param metadata: Retrieve metadata about the passage and the text
        :return: GetPassage or GetPassagePlus CTS API request response
        """
        if  subreference:
            textId = "{}:{}".format(textId, subreference)

        if prevnext or metadata:
            return self.getPassagePlus(urn=textId)
        else:
            return self.getPassage(urn=textId)

    def getSiblings(self, textId, subreference):
        """ Retrieve the siblings of a textual node

        :param textId: CtsTextMetadata Identifier
        :param reference: CapitainsCtsPassage Reference
        :return: GetPrevNextUrn request response from the endpoint
        """
        textId = "{}:{}".format(textId, subreference)
        return self.getPrevNextUrn(urn=textId)

    def getReffs(self, textId, level=1, subreference=None):
        """ Retrieve the siblings of a textual node

        :param textId: CtsTextMetadata Identifier
        :type textId: str
        :param level: Depth for retrieval
        :type level: int
        :param subreference: CapitainsCtsPassage Reference
        :type subreference: str
        :return: List of references
        :rtype: [str]
        """
        depth = level
        if subreference:
            textId = "{}:{}".format(textId, subreference)
        if subreference:
            if isinstance(subreference, Reference):
                depth += len(subreference)
            else:
                depth += len(Reference(subreference))
        if level:
            level = max(depth, level)
        return self.getValidReff(urn=textId, level=level)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2016 - Gustavo Montamat
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
awreporting.py

AdWords API reporting script suitable for large scale reports.
"""

import argparse
import csv
import logging
import os
import Queue

from time import sleep

from accounts import get_account_ids
from reporting_threads import ReportDownloader, ReportDecompressor, END_SIGNAL

logger = logging.getLogger(__name__)

TEMP_DIR = 'temp'

def clear_dir(path):
    if not os.path.exists(path):
        try:
            os.makedirs(path)
        except Exception as e:
            logger.exception("Couldn't create temporal directory.")
    for file_name in os.listdir(path):
        file_path = os.path.join(path, file_name)
        try:
            os.unlink(file_path)
        except Exception, e:
            logger.exception(
                u"Couldn't delete <{name}>".format(name=file_name)
            )
    return

def read_query(query_file):
    try:
        with open(query_file, 'rb') as fin:
            query = fin.read()
            query = query.replace('\n', ' ')
            query = query.replace('\r', '')
    except Exception as e:
        logger.exception("Couldn't read query file.")
        return
    return query

def merge_output(output, path):
    first = True
    with open(output, 'wb') as fout:
        csv_writer = csv.writer(fout, delimiter=',', quotechar='"')
        for file_name in os.listdir(path):
            if file_name[-4:] == '.csv':
                file_path = os.path.join(path, file_name)
                with open(file_path, 'rb') as fin:
                    csv_reader = csv.reader(fin, delimiter=',', quotechar='"')
                    if not first:
                        # Skip column headers
                        csv_reader.next()
                    first = False
                    for row in csv_reader:
                        csv_writer.writerow(row)
    return

def get_report(token, query_file, output, threads, account_ids=None):
    logger.info("Preparing temporal directory.")
    clear_dir(TEMP_DIR)
    if not account_ids:
        logger.info("Retrieving all AdWords account ids.")
        account_ids = get_account_ids(token)
    logger.info("Loading AWQL query.")
    awql_query = read_query(query_file)
    # Create a queue with all the accounts to be queried
    queue_ids = Queue.Queue()
    for account_id in account_ids:
        queue_ids.put(account_id)
    queue_decompress = Queue.Queue()
    queue_fails = Queue.Queue()
    while True:
        # Generate decompressor threads pool
        for i in range(2):
            report_decompressor = ReportDecompressor(
                queue_decompress, queue_fails, TEMP_DIR
            )
            report_decompressor.daemon = True
            report_decompressor.start()
        # Generate downloader threads pool
        logger.info("Initializing ReportDownloader threads.")
        max_threads = min(queue_ids.qsize(), threads)
        for i in xrange(max_threads):
            if queue_ids.qsize() == 0:
                break
            report_downloader = ReportDownloader(
                token, queue_ids, queue_decompress, awql_query, TEMP_DIR
            )
            report_downloader.daemon = True
            report_downloader.start()
            sleep(0.1)
        logger.info("Used {thread_num} threads.".format(thread_num=i+1))
        # Wait until all the account ids have been processed
        queue_ids.join()
        queue_ids.put(END_SIGNAL)
        # Wait untit all gzipped reports have been extracted
        queue_decompress.join()
        queue_decompress.put(END_SIGNAL)
        if queue_fails.qsize() == 0:
            logger.info("All reports have been obtained.")
            break
        else:
            # Restart job with failed downloads
            queue_decompress = Queue.Queue()
            queue_ids = Queue.Queue()
            for account_id in queue_fails.get():
                queue_ids.put(account_id)
            queue_fails = Queue.Queue()
    merge_output(output, TEMP_DIR)
    clear_dir(TEMP_DIR)
    return

def main(token, query_file, output, threads):
    with open('run.log', 'w'):
        pass
    logging.basicConfig(
        filename='run.log', level=logging.WARNING,
        format=('%(asctime)s.%(msecs)03d\t%(threadName)s'
                '\t%(module)s.%(funcName)s\t%(levelname)s\t%(message)s'),
        datefmt='%Y-%m-%d %H:%M:%S'
    )
    get_report(token, query_file, output, threads)
    return

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="PyAwReporting")
    parser.add_argument(
        '-o', '--output', help="Output file name", default="output.csv"
    )
    parser.add_argument(
        '-n', '--numthreads', help="Number of threads", type=int, default=10
    )
    required_arguments = parser.add_argument_group("required arguments")
    required_arguments.add_argument(
        '-t', '--token', help="AdWords YAML token file", required=True
    )
    required_arguments.add_argument(
        '-q', '--query', help="AWQL query file name", required=True
    )
    args = parser.parse_args()
    main(args.token, args.query, args.output, args.numthreads)

<commit_msg>Minor fix in loop<commit_after>#!/usr/bin/env python

# Copyright 2016 - Gustavo Montamat
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
awreporting.py

AdWords API reporting script suitable for large scale reports.
"""

import argparse
import csv
import logging
import os
import Queue

from time import sleep

from accounts import get_account_ids
from reporting_threads import ReportDownloader, ReportDecompressor, END_SIGNAL

logger = logging.getLogger(__name__)

TEMP_DIR = 'temp'

def clear_dir(path):
    if not os.path.exists(path):
        try:
            os.makedirs(path)
        except Exception as e:
            logger.exception("Couldn't create temporal directory.")
    for file_name in os.listdir(path):
        file_path = os.path.join(path, file_name)
        try:
            os.unlink(file_path)
        except Exception, e:
            logger.exception(
                u"Couldn't delete <{name}>".format(name=file_name)
            )
    return

def read_query(query_file):
    try:
        with open(query_file, 'rb') as fin:
            query = fin.read()
            query = query.replace('\n', ' ')
            query = query.replace('\r', '')
    except Exception as e:
        logger.exception("Couldn't read query file.")
        return
    return query

def merge_output(output, path):
    first = True
    with open(output, 'wb') as fout:
        csv_writer = csv.writer(fout, delimiter=',', quotechar='"')
        for file_name in os.listdir(path):
            if file_name[-4:] == '.csv':
                file_path = os.path.join(path, file_name)
                with open(file_path, 'rb') as fin:
                    csv_reader = csv.reader(fin, delimiter=',', quotechar='"')
                    if not first:
                        # Skip column headers
                        csv_reader.next()
                    first = False
                    for row in csv_reader:
                        csv_writer.writerow(row)
    return

def get_report(token, query_file, output, threads, account_ids=None):
    logger.info("Preparing temporal directory.")
    clear_dir(TEMP_DIR)
    if not account_ids:
        logger.info("Retrieving all AdWords account ids.")
        account_ids = get_account_ids(token)
    logger.info("Loading AWQL query.")
    awql_query = read_query(query_file)
    # Create a queue with all the accounts to be queried
    queue_ids = Queue.Queue()
    for account_id in account_ids:
        queue_ids.put(account_id)
    queue_decompress = Queue.Queue()
    queue_fails = Queue.Queue()
    while True:
        # Generate decompressor threads pool
        for i in xrange(2):
            report_decompressor = ReportDecompressor(
                queue_decompress, queue_fails, TEMP_DIR
            )
            report_decompressor.daemon = True
            report_decompressor.start()
        # Generate downloader threads pool
        logger.info("Initializing ReportDownloader threads.")
        max_threads = min(queue_ids.qsize(), threads)
        for i in xrange(max_threads):
            if queue_ids.qsize() == 0:
                break
            report_downloader = ReportDownloader(
                token, queue_ids, queue_decompress, awql_query, TEMP_DIR
            )
            report_downloader.daemon = True
            report_downloader.start()
            sleep(0.1)
        logger.info("Used {thread_num} threads.".format(thread_num=i+1))
        # Wait until all the account ids have been processed
        queue_ids.join()
        queue_ids.put(END_SIGNAL)
        # Wait untit all gzipped reports have been extracted
        queue_decompress.join()
        queue_decompress.put(END_SIGNAL)
        if queue_fails.qsize() == 0:
            logger.info("All reports have been obtained.")
            break
        else:
            # Restart job with failed downloads
            queue_decompress = Queue.Queue()
            queue_ids = Queue.Queue()
            for account_id in queue_fails.get():
                queue_ids.put(account_id)
            queue_fails = Queue.Queue()
    merge_output(output, TEMP_DIR)
    clear_dir(TEMP_DIR)
    return

def main(token, query_file, output, threads):
    with open('run.log', 'w'):
        pass
    logging.basicConfig(
        filename='run.log', level=logging.WARNING,
        format=('%(asctime)s.%(msecs)03d\t%(threadName)s'
                '\t%(module)s.%(funcName)s\t%(levelname)s\t%(message)s'),
        datefmt='%Y-%m-%d %H:%M:%S'
    )
    get_report(token, query_file, output, threads)
    return

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="PyAwReporting")
    parser.add_argument(
        '-o', '--output', help="Output file name", default="output.csv"
    )
    parser.add_argument(
        '-n', '--numthreads', help="Number of threads", type=int, default=10
    )
    required_arguments = parser.add_argument_group("required arguments")
    required_arguments.add_argument(
        '-t', '--token', help="AdWords YAML token file", required=True
    )
    required_arguments.add_argument(
        '-q', '--query', help="AWQL query file name", required=True
    )
    args = parser.parse_args()
    main(args.token, args.query, args.output, args.numthreads)

<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Copyright (c) 2011 rPath, Inc.
#
# All rights reserved.
#

queryset_post_xml = """
<query_set>
  <filter_entries>
    <filter_entry>
      <field>system.name</field>
      <operator>LIKE</operator>
      <value>3</value>
    </filter_entry>
  </filter_entries>
  <name>New Query Set</name>
  <resource_type>system</resource_type>
  <description>System name has a 3 in it</description>
</query_set>
"""

# some hardcodes here, may need to be updated when installed
# querysets change, when that happens, subsitute the 12
# and the 5 programatically, not implementing just now
queryset_put_xml = """
<query_set id="/api/v1/query_sets/14">
  <filter_entries>
    <filter_entry>
      <operator>LIKE</operator>
      <field>system.name</field>
      <filter_entry_id>14</filter_entry_id>
      <value>3</value>
    </filter_entry>
  </filter_entries>
  <filter_descriptor id="/api/v1/query_sets/filter_descriptor"/>
  <can_modify>true</can_modify>
  <presentation_type></presentation_type>
  <modified_date>2011-08-17T14:52:58.737001+00:00</modified_date>
  <filtered_members id="/api/v1/query_sets/14/filtered"/>
  <is_top_level>True</is_top_level>
  <tagged_date></tagged_date>
  <name>New Query Set</name>
  <chosen_members id="/api/v1/query_sets/14/chosen"/>
  <query_set_id>14</query_set_id>
  <child_members id="/api/v1/query_sets/14/child"/>
  <created_date>2011-08-17T14:52:58.736901+00:00</created_date>
  <all_members id="/api/v1/query_sets/14/all"/>
  <children>
    <query_set id="/api/v1/query_sets/5">
      <filter_entries>
        <filter_entry>
          <operator>IS_NULL</operator>
          <field>system.name</field>
          <filter_entry_id>1</filter_entry_id>
          <value>false</value>
        </filter_entry>
      </filter_entries>
      <filter_descriptor id="/api/v1/query_sets/filter_descriptor"/>
      <can_modify>false</can_modify>
      <presentation_type></presentation_type>
      <modified_date>2011-08-17T14:52:28.162448+00:00</modified_date>
      <filtered_members id="/api/v1/query_sets/5/filtered"/>
      <is_top_level>False</is_top_level>
      <tagged_date></tagged_date>
      <name>All Systems</name>
      <chosen_members id="/api/v1/query_sets/5/chosen"/>
      <query_set_id>5</query_set_id>
      <child_members id="/api/v1/query_sets/5/child"/>
      <created_date>2011-08-17T14:52:28.162426+00:00</created_date>
      <all_members id="/api/v1/query_sets/5/all"/>
      <children/>
      <resource_type>system</resource_type>
      <description>All systems</description>
    </query_set>
  </children>
  <resource_type>system</resource_type>
  <description>System name has a 3 in it</description>
  <actions/>
</query_set>
"""

queryset_with_actions = """
<query_set id="http://testserver/api/v1/query_sets/5">
  <actions>
    <action>
      <description>Refresh queryset</description>
      <descriptor id="http://testserver/api/v1/query_sets/5/descriptors/14"/>
      <enabled>true</enabled>
      <job_type id="http://testserver/api/v1/inventory/event_types/14"/>
      <name>refresh queryset</name>
    </action>
  </actions>
  <all_members id="http://testserver/api/v1/query_sets/5/all"/>
  <can_modify>false</can_modify>
  <child_members id="http://testserver/api/v1/query_sets/5/child"/>
  <children/>
  <chosen_members id="http://testserver/api/v1/query_sets/5/chosen"/>
  <description>All systems</description>
  <filter_descriptor id="http://testserver/api/v1/query_sets/filter_descriptor"/>
  <filter_entries>
    <filter_entry>
      <field>system.name</field>
      <filter_entry_id>1</filter_entry_id>
      <operator>IS_NULL</operator>
      <value>false</value>
    </filter_entry>
  </filter_entries>
  <filtered_members id="http://testserver/api/v1/query_sets/5/filtered"/>
  <grants/>
  <is_top_level>True</is_top_level>
  <name>All Systems</name>
  <presentation_type/>
  <query_set_id>5</query_set_id>
  <resource_type>system</resource_type>
  <tagged_date>2011-08-29T21:44:24.358194+00:00</tagged_date>
</query_set>
"""

queryset_invalidate_post_xml = """
<job>
<job_type id='https://localhost/api/v1/inventory/event_types/14'>queryset refresh</job_type>
</job>
"""

<commit_msg>Update testsxml to reflect presentation_type now set on "All Systems" queryset.<commit_after>#!/usr/bin/python
#
# Copyright (c) 2011 rPath, Inc.
#
# All rights reserved.
#

queryset_post_xml = """
<query_set>
  <filter_entries>
    <filter_entry>
      <field>system.name</field>
      <operator>LIKE</operator>
      <value>3</value>
    </filter_entry>
  </filter_entries>
  <name>New Query Set</name>
  <resource_type>system</resource_type>
  <description>System name has a 3 in it</description>
</query_set>
"""

# some hardcodes here, may need to be updated when installed
# querysets change, when that happens, subsitute the 12
# and the 5 programatically, not implementing just now
queryset_put_xml = """
<query_set id="/api/v1/query_sets/14">
  <filter_entries>
    <filter_entry>
      <operator>LIKE</operator>
      <field>system.name</field>
      <filter_entry_id>14</filter_entry_id>
      <value>3</value>
    </filter_entry>
  </filter_entries>
  <filter_descriptor id="/api/v1/query_sets/filter_descriptor"/>
  <can_modify>true</can_modify>
  <presentation_type></presentation_type>
  <modified_date>2011-08-17T14:52:58.737001+00:00</modified_date>
  <filtered_members id="/api/v1/query_sets/14/filtered"/>
  <is_top_level>True</is_top_level>
  <tagged_date></tagged_date>
  <name>New Query Set</name>
  <chosen_members id="/api/v1/query_sets/14/chosen"/>
  <query_set_id>14</query_set_id>
  <child_members id="/api/v1/query_sets/14/child"/>
  <created_date>2011-08-17T14:52:58.736901+00:00</created_date>
  <all_members id="/api/v1/query_sets/14/all"/>
  <children>
    <query_set id="/api/v1/query_sets/5">
      <filter_entries>
        <filter_entry>
          <operator>IS_NULL</operator>
          <field>system.name</field>
          <filter_entry_id>1</filter_entry_id>
          <value>false</value>
        </filter_entry>
      </filter_entries>
      <filter_descriptor id="/api/v1/query_sets/filter_descriptor"/>
      <can_modify>false</can_modify>
      <presentation_type></presentation_type>
      <modified_date>2011-08-17T14:52:28.162448+00:00</modified_date>
      <filtered_members id="/api/v1/query_sets/5/filtered"/>
      <is_top_level>False</is_top_level>
      <tagged_date></tagged_date>
      <name>All Systems</name>
      <presentation_type>system</presentation_type>
      <chosen_members id="/api/v1/query_sets/5/chosen"/>
      <query_set_id>5</query_set_id>
      <child_members id="/api/v1/query_sets/5/child"/>
      <created_date>2011-08-17T14:52:28.162426+00:00</created_date>
      <all_members id="/api/v1/query_sets/5/all"/>
      <children/>
      <resource_type>system</resource_type>
      <description>All systems</description>
    </query_set>
  </children>
  <resource_type>system</resource_type>
  <description>System name has a 3 in it</description>
  <actions/>
</query_set>
"""

queryset_with_actions = """
<query_set id="http://testserver/api/v1/query_sets/5">
  <actions>
    <action>
      <description>Refresh queryset</description>
      <descriptor id="http://testserver/api/v1/query_sets/5/descriptors/14"/>
      <enabled>true</enabled>
      <job_type id="http://testserver/api/v1/inventory/event_types/14"/>
      <name>refresh queryset</name>
    </action>
  </actions>
  <all_members id="http://testserver/api/v1/query_sets/5/all"/>
  <can_modify>false</can_modify>
  <child_members id="http://testserver/api/v1/query_sets/5/child"/>
  <children/>
  <chosen_members id="http://testserver/api/v1/query_sets/5/chosen"/>
  <description>All systems</description>
  <filter_descriptor id="http://testserver/api/v1/query_sets/filter_descriptor"/>
  <filter_entries>
    <filter_entry>
      <field>system.name</field>
      <filter_entry_id>1</filter_entry_id>
      <operator>IS_NULL</operator>
      <value>false</value>
    </filter_entry>
  </filter_entries>
  <filtered_members id="http://testserver/api/v1/query_sets/5/filtered"/>
  <grants/>
  <is_top_level>True</is_top_level>
  <name>All Systems</name>
  <presentation_type>system</presentation_type>
  <query_set_id>5</query_set_id>
  <resource_type>system</resource_type>
  <tagged_date>2011-08-29T21:44:24.358194+00:00</tagged_date>
</query_set>
"""

queryset_invalidate_post_xml = """
<job>
<job_type id='https://localhost/api/v1/inventory/event_types/14'>queryset refresh</job_type>
</job>
"""

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Generated by Django 1.11.20 on 2019-05-30 10:02
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('builds', '0007_add-automation-rules'),
    ]

    operations = [
        migrations.AlterField(
            model_name='version',
            name='type',
            field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], default='unknown', max_length=20, verbose_name='Type'),
        ),
        migrations.AlterField(
            model_name='versionautomationrule',
            name='version_type',
            field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], max_length=32, verbose_name='Version type'),
        ),
    ]
<commit_msg>Updated migrations<commit_after># -*- coding: utf-8 -*-
# Generated by Django 1.11.20 on 2019-06-06 16:51
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('builds', '0007_add-automation-rules'),
    ]

    operations = [
        migrations.AlterField(
            model_name='version',
            name='type',
            field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], default='unknown', max_length=20, verbose_name='Type'),
        ),
        migrations.AlterField(
            model_name='versionautomationrule',
            name='version_type',
            field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], max_length=32, verbose_name='Version type'),
        ),
    ]
<|endoftext|>


<commit_before>import sys
import re
import time
import ConfigParser

from intelmq.lib.event import Event
from intelmq.lib.pipeline import Pipeline
from intelmq.lib.utils import force_decode, log

SYSTEM_CONF_FILE = "/etc/intelmq/system.conf"
PIPELINE_CONF_FILE = "/etc/intelmq/pipeline.conf"
BOTS_CONF_FILE = "/etc/intelmq/bots.conf"
LOGS_PATH = "/var/log/intelmq/"

class Bot(object):

    def __init__(self, bot_id):
        self.check_bot_id(bot_id)

        self.bot_id = bot_id

        self.logger = self.load_logger()
        self.logger.info('Bot is starting')

        self.load_configurations()

        self.src_queue, self.dest_queues = self.load_queues()
        self.interval = float(self.parameters.processing_interval)

        self.init()

    def init(self):
        pass

    def start(self):
        self.logger.info('Bot start processing')
        self.pipeline = None

        while True:
            try:
                if not self.pipeline:
                    self.logger.info("Connecting to pipeline queues")
                    self.pipeline = Pipeline(self.src_queue, self.dest_queues)
                    self.logger.info("Connected to pipeline queues. Start processing.")
                self.process()
                self.pipeline.sleep(self.interval)

            except Exception as e:
                retry_delay = 30
                self.logger.error('Pipeline connection failed (%s)' % e)
                self.logger.info('Pipeline will reconnect in %s seconds' % retry_delay)
                time.sleep(retry_delay)
                self.pipeline = None

            except KeyboardInterrupt as e:
                if self.pipeline:
                    self.pipeline.disconnect()
                    self.logger.info("Disconnecting from pipeline")
                self.logger.info("Bot is shutting down")
                break

    def stop(self):
        try:
            self.logger.error("Bot found an error. Exiting")
        except:
            pass
        finally:
            print "Bot found an error. Exiting"
        exit(-1)

    def check_bot_id(self, str):
        res = re.search('[^0-9a-zA-Z\-]+', str)
        if res:
            print "Invalid bot id."
            self.stop()

    def load_configurations(self):
        self.parameters = Parameters()
        config = ConfigParser.ConfigParser()
        config.read(BOTS_CONF_FILE)

        default_section = "default"
        self.logger.debug("Loading configuration in default section from '%s' file" % BOTS_CONF_FILE)

        if config.has_section(default_section):
            for option in config.options(default_section):
                setattr(self.parameters, option, config.get(default_section, option))
                self.logger.debug("Parameter '%s' loaded with the value '%s'" % (option, config.get(default_section, option)))

        self.logger.debug("Loading configuration in %s section from '%s' file" % (self.bot_id, BOTS_CONF_FILE))

        if config.has_section(self.bot_id):
            for option in config.options(self.bot_id):
                setattr(self.parameters, option, config.get(self.bot_id, option))
                self.logger.debug("Parameter '%s' loaded with the value '%s'" % (option, config.get(self.bot_id, option)))

    def load_logger(self):
        config = ConfigParser.ConfigParser()
        config.read(SYSTEM_CONF_FILE)
        loglevel = config.get('Logging','level')
        return log(LOGS_PATH, self.bot_id, loglevel)

    def load_pipeline(self):
        return

    def load_queues(self):
        config = ConfigParser.ConfigParser()
        config.read(PIPELINE_CONF_FILE)
        self.logger.debug("Loading pipeline queues from '%s' file" % PIPELINE_CONF_FILE)

        for option in config.options("Pipeline"):
            if option == self.bot_id:
                queues = config.get("Pipeline", self.bot_id)

                src_queue, dest_queues = self.parse_queues(queues)
                if src_queue:
                    self.logger.info("Source queue '%s'" % src_queue)
                if dest_queues:
                    self.logger.info("Destination queue(s) '%s'" % "', '".join(dest_queues))
                return [src_queue, dest_queues]

        self.logger.error("Failed to load queues")
        self.stop()

    def parse_queues(self, queues):
        queues = queues.split('|')

        if len(queues) == 2:
            src_queue = queues[0].strip()
            if len(src_queue) == 0:
                src_queue = None

            dest_queues = queues[1].strip()
            if len(dest_queues) == 0:
                dest_queues_list = None
            else:
                dest_queues_list = list()
                for queue in dest_queues.split(','):
                    dest_queues_list.append(queue.strip())

            return [src_queue, dest_queues_list]
        else:
            return [None, None]

    def send_message(self, message):
        try:
            message = unicode(message)
        except:
            message = force_decode(message)
        self.pipeline.send(message)

    def receive_message(self):
        raw_message = self.pipeline.receive()
        try:
            message = Event.from_unicode(raw_message)
        except:
            message = raw_message
        return message

    def acknowledge_message(self):
        self.pipeline.acknowledge()

class Parameters(object):
    pass

<commit_msg>Update bot.py<commit_after>import sys
import re
import time
import ConfigParser

from intelmq.lib.event import Event
from intelmq.lib.pipeline import Pipeline
from intelmq.lib.utils import force_decode, log

SYSTEM_CONF_FILE = "/etc/intelmq/system.conf"
PIPELINE_CONF_FILE = "/etc/intelmq/pipeline.conf"
BOTS_CONF_FILE = "/etc/intelmq/bots.conf"
LOGS_PATH = "/var/log/intelmq/"

class Bot(object):

    def __init__(self, bot_id):
        self.check_bot_id(bot_id)

        self.bot_id = bot_id

        self.logger = self.load_logger()
        self.logger.info('Bot is starting')

        self.load_configurations()

        self.src_queue, self.dest_queues = self.load_queues()
        self.parameters.processing_interval = float(self.parameters.processing_interval)

        self.init()

    def init(self):
        pass

    def start(self):
        self.logger.info('Bot start processing')
        self.pipeline = None

        while True:
            try:
                if not self.pipeline:
                    self.logger.info("Connecting to pipeline queues")
                    self.pipeline = Pipeline(self.src_queue, self.dest_queues)
                    self.logger.info("Connected to pipeline queues. Start processing.")
                self.process()
                self.pipeline.sleep(self.parameters.processing_interval)

            except Exception as e:
                retry_delay = 30
                self.logger.error('Pipeline connection failed (%s)' % e)
                self.logger.info('Pipeline will reconnect in %s seconds' % retry_delay)
                time.sleep(retry_delay)
                self.pipeline = None

            except KeyboardInterrupt as e:
                if self.pipeline:
                    self.pipeline.disconnect()
                    self.logger.info("Disconnecting from pipeline")
                self.logger.info("Bot is shutting down")
                break

    def stop(self):
        try:
            self.logger.error("Bot found an error. Exiting")
        except:
            pass
        finally:
            print "Bot found an error. Exiting"
        exit(-1)

    def check_bot_id(self, str):
        res = re.search('[^0-9a-zA-Z\-]+', str)
        if res:
            print "Invalid bot id."
            self.stop()

    def load_configurations(self):
        self.parameters = Parameters()
        config = ConfigParser.ConfigParser()
        config.read(BOTS_CONF_FILE)

        default_section = "default"
        self.logger.debug("Loading configuration in default section from '%s' file" % BOTS_CONF_FILE)

        if config.has_section(default_section):
            for option in config.options(default_section):
                setattr(self.parameters, option, config.get(default_section, option))
                self.logger.debug("Parameter '%s' loaded with the value '%s'" % (option, config.get(default_section, option)))

        self.logger.debug("Loading configuration in %s section from '%s' file" % (self.bot_id, BOTS_CONF_FILE))

        if config.has_section(self.bot_id):
            for option in config.options(self.bot_id):
                setattr(self.parameters, option, config.get(self.bot_id, option))
                self.logger.debug("Parameter '%s' loaded with the value '%s'" % (option, config.get(self.bot_id, option)))

    def load_logger(self):
        config = ConfigParser.ConfigParser()
        config.read(SYSTEM_CONF_FILE)
        loglevel = config.get('Logging','level')
        return log(LOGS_PATH, self.bot_id, loglevel)

    def load_pipeline(self):
        return

    def load_queues(self):
        config = ConfigParser.ConfigParser()
        config.read(PIPELINE_CONF_FILE)
        self.logger.debug("Loading pipeline queues from '%s' file" % PIPELINE_CONF_FILE)

        for option in config.options("Pipeline"):
            if option == self.bot_id:
                queues = config.get("Pipeline", self.bot_id)

                src_queue, dest_queues = self.parse_queues(queues)
                if src_queue:
                    self.logger.info("Source queue '%s'" % src_queue)
                if dest_queues:
                    self.logger.info("Destination queue(s) '%s'" % "', '".join(dest_queues))
                return [src_queue, dest_queues]

        self.logger.error("Failed to load queues")
        self.stop()

    def parse_queues(self, queues):
        queues = queues.split('|')

        if len(queues) == 2:
            src_queue = queues[0].strip()
            if len(src_queue) == 0:
                src_queue = None

            dest_queues = queues[1].strip()
            if len(dest_queues) == 0:
                dest_queues_list = None
            else:
                dest_queues_list = list()
                for queue in dest_queues.split(','):
                    dest_queues_list.append(queue.strip())

            return [src_queue, dest_queues_list]
        else:
            return [None, None]

    def send_message(self, message):
        try:
            message = unicode(message)
        except:
            message = force_decode(message)
        self.pipeline.send(message)

    def receive_message(self):
        raw_message = self.pipeline.receive()
        try:
            message = Event.from_unicode(raw_message)
        except:
            message = raw_message
        return message

    def acknowledge_message(self):
        self.pipeline.acknowledge()

class Parameters(object):
    pass

<|endoftext|>


<commit_before>

# A form cannot be submitted if the values it was originally rendered with have changed in the backend by other means before the form gets submitted.
# When presented with such an error, the user can click on a button to reset all inputs to the now-current values (if using the not-yet-there "ResetForm" in non-bootstrap stuff)
#   - this also clears all possible form (the current form) exceptions and remembered input values

# Scenario: user B changes something AFTER user A has gotten a DomainException but before user B submits correct data again.

# (move to a bootstrap-specific test:) if using FormLayout.add_alert_for_domain_exception

# Only VISIBLE PrimitiveInput value data is taken into account by default for such optimistic locking, but:
#  if an input becomes readonly (or not readonly) since last rendered, it is also considered as having changed.
#  passing ignore_concurrency=True prevents a PrimitiveInput from taking part, despite being visible, and despite its readablity
#  any VISIBLE Widget can take part, if it overrides its concurrency_hash_strings method, and append a string representing its value to the yielded values

# If the user submits to a form that is no longer present on the current view (because of changes data as well),
# the user is sent to the general error page and all form input/exceptions for all forms on the view are cleared.
# When clicking on the OK button on said error page, the user is taken back to the view, but sees it refreshed to new values.
<commit_msg>Added some optimistic concurrency tests<commit_after># Copyright 2019 Reahl Software Services (Pty) Ltd. All rights reserved.
#
#    This file is part of Reahl.
#
#    Reahl is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation; version 3 of the License.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import print_function, unicode_literals, absolute_import, division
import six

from sqlalchemy import Column, Integer, UnicodeText

from reahl.tofu import Fixture, scenario
from reahl.tofu.pytestsupport import with_fixtures, uses
from reahl.component.modelinterface import exposed, Field, Event, PatternConstraint, Action
from reahl.component.exceptions import DomainException
from reahl.web.ui import Form, ButtonInput, TextInput, Label, FormLayout
from reahl.web_dev.fixtures import WebFixture
from reahl.webdev.tools import Browser, XPath
from reahl.sqlalchemysupport_dev.fixtures import SqlAlchemyFixture
from reahl.sqlalchemysupport import Base, Session

@uses(web_fixture=WebFixture)
class OptimisticConcurrencyFixture(Fixture):
    def new_ModelObject(self):
        class ModelObject(Base):
            __tablename__ = 'test_optimistic_concurrency_model_object'
            id = Column(Integer, primary_key=True)
            some_field = Column(UnicodeText)

            @exposed
            def fields(self, fields):
                fields.some_field = Field(label='Some field', default='not set').with_validation_constraint(PatternConstraint('((?!invalidinput).)*'))

            @exposed
            def events(self, events):
                events.submit = Event(label='Submit')
                events.submit_break = Event(label='Submit break', action=Action(self.always_break))

            def always_break(self):
                raise DomainException('boo')

        return ModelObject

    def new_model_object(self):
        model_object = self.ModelObject()
        Session.add(model_object)
        return model_object

    def new_MyForm(self):
        fixture = self
        class MyForm(Form):
            def __init__(self, view):
                super(MyForm, self).__init__(view, 'myform')
                self.set_attribute('novalidate', 'novalidate')
                self.use_layout(FormLayout())

                if self.exception:
                    self.layout.add_alert_for_domain_exception(self.exception)

                self.layout.add_input(TextInput(self, fixture.model_object.fields.some_field))
                self.define_event_handler(fixture.model_object.events.submit)
                self.add_child(ButtonInput(self, fixture.model_object.events.submit))
                self.define_event_handler(fixture.model_object.events.submit_break)
                self.add_child(ButtonInput(self, fixture.model_object.events.submit_break))
        return MyForm

    def make_concurrent_change_in_backend(self):
        self.model_object.some_field = 'changed by someone else'

    def concurrent_change_is_present(self):
        return self.model_object.some_field == 'changed by someone else'

    def is_concurrency_error_displayed(self):
        error_li = XPath.li().with_text('Some data changed since you opened this page, please reset input to try again.')
        return self.web_fixture.driver_browser.is_element_present(error_li)

    def is_any_error_displayed(self):
        return self.web_fixture.driver_browser.is_element_present(XPath.div().including_class('errors'))

@with_fixtures(WebFixture, SqlAlchemyFixture, OptimisticConcurrencyFixture)
def test_optimistic_concurrency(web_fixture, sql_alchemy_fixture, concurrency_fixture):
    """A user is prompted to handle the situation where data would be overwritten when submitting a
       form that was originally rendered based on older data.
    """
    fixture = concurrency_fixture

    with sql_alchemy_fixture.persistent_test_classes(fixture.ModelObject):
        model_object = fixture.model_object

        wsgi_app = web_fixture.new_wsgi_app(child_factory=fixture.MyForm.factory())
        web_fixture.reahl_server.set_app(wsgi_app)
        browser = web_fixture.driver_browser
        browser.open('/')

        # The form submit does not overwrite any data changed by other means
        assert not fixture.is_concurrency_error_displayed()

        browser.type(XPath.input_labelled('Some field'), 'something')
        fixture.make_concurrent_change_in_backend()

        browser.click(XPath.button_labelled('Submit'))

        assert fixture.is_concurrency_error_displayed()
        assert fixture.concurrent_change_is_present()

        # When presented with such an error, the user can click on a button to reset all inputs to the now-current values
        browser.click(XPath.button_labelled('Reset input'))
        assert not fixture.is_concurrency_error_displayed()
        assert browser.get_value(XPath.input_labelled('Some field')) == 'changed by someone else'

        browser.type(XPath.input_labelled('Some field'), 'final changed value')
        browser.click(XPath.button_labelled('Submit'))
        assert not fixture.is_concurrency_error_displayed()

        assert model_object.some_field == 'final changed value'

class ExceptionScenarios(Fixture):
    @scenario
    def break_with_validation_error(self):
        def submit_changed_but_invalid_inputs(browser):
            browser.type(XPath.input_labelled('Some field'), 'invalidinput')
            browser.click(XPath.button_labelled('Submit'))
        self.cause_exception_on_submit = submit_changed_but_invalid_inputs

    @scenario
    def break_with_other_domain_error(self):
        def submit_with_unchanged_inputs(browser):
            browser.click(XPath.button_labelled('Submit break'))
        self.cause_exception_on_submit = submit_with_unchanged_inputs

@with_fixtures(WebFixture, SqlAlchemyFixture, OptimisticConcurrencyFixture, ExceptionScenarios)
def test_clear_form_inputs_on_optimistic_concurrency(web_fixture, sql_alchemy_fixture, concurrency_fixture, scenario):
    """A concurrency error is detected upon submit after an exception.
       When a user resets inputs upon such a concurrency error, previous form exceptions and input data are cleared.
    """
    fixture = concurrency_fixture

    with sql_alchemy_fixture.persistent_test_classes(fixture.ModelObject):
        wsgi_app = web_fixture.new_wsgi_app(child_factory=fixture.MyForm.factory())
        web_fixture.reahl_server.set_app(wsgi_app)
        browser = web_fixture.driver_browser
        browser.open('/')

        # Concurrency error is detected on submit after an exception
        scenario.cause_exception_on_submit(browser)
        assert fixture.is_any_error_displayed()
        assert not fixture.is_concurrency_error_displayed()

        fixture.make_concurrent_change_in_backend()

        browser.type(XPath.input_labelled('Some field'), 'valid input')
        browser.click(XPath.button_labelled('Submit'))
        assert fixture.is_concurrency_error_displayed()

        # Previous error and inputs are cleared
        import pdb; pdb.set_trace()
        browser.click(XPath.button_labelled('Reset input'))

        assert browser.get_value(XPath.input_labelled('Some field')) == 'changed by someone else'
        assert not fixture.is_any_error_displayed()

# (move to a bootstrap-specific test:) if using FormLayout.add_alert_for_domain_exception

# Only VISIBLE PrimitiveInput value data is taken into account by default for such optimistic locking, but:
#  if an input becomes readonly (or not readonly) since last rendered, it is also considered as having changed.
#  passing ignore_concurrency=True prevents a PrimitiveInput from taking part, despite being visible, and despite its readablity
#  any VISIBLE Widget can take part, if it overrides its concurrency_hash_strings method, and append a string representing its value to the yielded values

# If the user submits to a form that is no longer present on the current view (because of changes data as well),
# the user is sent to the general error page and all form input/exceptions for all forms on the view are cleared.
# When clicking on the OK button on said error page, the user is taken back to the view, but sees it refreshed to new values.

<|endoftext|>


<commit_before># Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

# pylint: disable=line-too-long
r"""A demo to run UNet 3D inference in Python.
"""
# pylint: enable=line-too-long

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import io
import os
from absl import app
from absl import flags
import numpy as np
import tensorflow.compat.v1 as tf

FLAGS = flags.FLAGS

# pylint: disable=line-too-long
flags.DEFINE_string('image_file_pattern', '', 'The glob image file pattern.')
flags.DEFINE_string('saved_model_dir', '', 'The saved model directory.')
flags.DEFINE_string('tag_set', 'serve', 'The saved model tag.')
flags.DEFINE_string('input_type', 'tf_example',
                    '`tf_example` or `image_bytes`.')
flags.DEFINE_string('input_node', 'Placeholder:0',
                    'The name of the input images.')
flags.DEFINE_bool('compressed_input', True, 'Read compressed input tfrecord')
flags.DEFINE_string('output_classes_node', 'Classes:0',
                    'The name of the input images.')
flags.DEFINE_string('output_scores_node', 'Scores:0',
                    'The name of the input images.')
flags.DEFINE_string('output_dir', '', 'The output image with detections.')
# pylint: enable=line-too-long

DESIRED_IMAGE_SHAPE = [128, 128, 128, 1]

def prepare_input_feed(image, input_type):
  """Prepares the np array to be fed to the model.

  Args:
    image: a PIL Image object representing the original input image.
    input_type: a string specifying what input type the SavedModel expects. One
      of `image_bytes`, `image_tensor`, `tf_example` and `annotated_image`.

  Returns:
    input_feed: a numpy array of shape (1,) that will be fed into the
      SavedModel.

  Raises:
    ValueError: if input_type is not supported.
  """
  if input_type == 'image_tensor':
    # Add batch size dimension.
    input_feed = np.expand_dims(image, axis=0)
  elif input_type == 'tf_example':
    input_feed = np.array([image])
  else:
    raise ValueError('Unsupported input_type!')
  return input_feed

def main(unused_argv):

  assert FLAGS.output_dir, 'Must specify --output_dir.'
  output_dir = FLAGS.output_dir
  if not tf.gfile.Exists(output_dir):
    tf.gfile.MakeDirs(output_dir)

  with tf.Session(graph=tf.Graph()) as sess:
    print(' - Loading saved model...')
    tf.saved_model.loader.load(sess, FLAGS.tag_set.split(','),
                               FLAGS.saved_model_dir)

    image_files = tf.gfile.Glob(FLAGS.image_file_pattern)
    for i, image_file in enumerate(image_files):
      print(' - Processing image %d...' % i)

      if FLAGS.compressed_input:
        options = tf.python_io.TFRecordOptions(
            tf.python_io.TFRecordCompressionType.GZIP)
      else:
        options = tf.python_io.TFRecordOptions()
      record_iterator = tf.python_io.tf_record_iterator(
          path=image_file, options=options)
      np_input = None
      if FLAGS.input_type == 'tf_example':
        for tf_example_string in record_iterator:
          image_string = tf_example_string
          break  # Only read the first tf.Example.
        np_input = prepare_input_feed(image_string, FLAGS.input_type)
      elif FLAGS.input_type == 'image_tensor':
        example = tf.train.Example()
        for tf_example_string in record_iterator:
          example.ParseFromString(tf_example_string)
          break  # Only read the first tf.Example.
        f = example.features.feature
        image = f['image/ct_image'].bytes_list.value[0]
        image = np.fromstring(
            image, dtype=np.float32).reshape(DESIRED_IMAGE_SHAPE)
        np_input = prepare_input_feed(image, FLAGS.input_type)
      else:
        raise ValueError('Unknown input type %s' % FLAGS.input_type)

      output_nodes = [
          FLAGS.output_classes_node,
          FLAGS.output_scores_node,
      ]
      outputs = sess.run(output_nodes, feed_dict={FLAGS.input_node: np_input})
      classes = outputs[0]
      scores = outputs[1]
      scores = np.squeeze(scores, axis=(0,))
      classes = np.squeeze(classes.astype(np.int32), axis=(0,))

      output_path = os.path.join(output_dir,
                                 os.path.basename(image_file) + '.npz')
      with tf.gfile.GFile(output_path, 'w') as f:
        io_buffer = io.BytesIO()
        np.savez(io_buffer, scores=scores, classes=classes)
        f.write(io_buffer.getvalue())
      print('Output to %s' % output_path)

  print(' - Done!')

if __name__ == '__main__':
  app.run()
<commit_msg>Internal change.<commit_after># Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

# pylint: disable=line-too-long
r"""A demo to run UNet 3D inference in Python.
"""
# pylint: enable=line-too-long

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import io
import os
from absl import app
from absl import flags
import numpy as np
import tensorflow.compat.v1 as tf

FLAGS = flags.FLAGS

# pylint: disable=line-too-long
flags.DEFINE_string('image_file_pattern', '', 'The glob image file pattern.')
flags.DEFINE_string('saved_model_dir', '', 'The saved model directory.')
flags.DEFINE_string('tag_set', 'serve', 'The saved model tag.')
flags.DEFINE_string('input_type', 'tf_example',
                    '`tf_example` or `image_bytes`.')
flags.DEFINE_string('input_node', 'Placeholder:0',
                    'The name of the input images.')
flags.DEFINE_bool('compressed_input', True, 'Read compressed input tfrecord')
flags.DEFINE_string('output_classes_node', 'Classes:0',
                    'The name of the input images.')
flags.DEFINE_string('output_scores_node', 'Scores:0',
                    'The name of the input images.')
flags.DEFINE_string('output_dir', '', 'The output image with detections.')
# pylint: enable=line-too-long

DESIRED_IMAGE_SHAPE = [128, 128, 128, 1]

def prepare_input_feed(image, input_type):
  """Prepares the np array to be fed to the model.

  Args:
    image: a PIL Image object representing the original input image.
    input_type: a string specifying what input type the SavedModel expects. One
      of `image_bytes`, `image_tensor`, `tf_example` and `annotated_image`.

  Returns:
    input_feed: a numpy array of shape (1,) that will be fed into the
      SavedModel.

  Raises:
    ValueError: if input_type is not supported.
  """
  if input_type == 'image_tensor':
    # Add batch size dimension.
    input_feed = np.expand_dims(image, axis=0)
  elif input_type == 'tf_example':
    input_feed = np.array([image])
  else:
    raise ValueError('Unsupported input_type!')
  return input_feed

def main(unused_argv):

  assert FLAGS.output_dir, 'Must specify --output_dir.'
  output_dir = FLAGS.output_dir
  if not tf.gfile.Exists(output_dir):
    tf.gfile.MakeDirs(output_dir)

  with tf.Session(graph=tf.Graph()) as sess:
    print(' - Loading saved model...')
    tf.saved_model.loader.load(sess, FLAGS.tag_set.split(','),
                               FLAGS.saved_model_dir)

    image_files = tf.gfile.Glob(FLAGS.image_file_pattern)
    for i, image_file in enumerate(image_files):
      print(' - Processing image %d...' % i)

      if FLAGS.compressed_input:
        options = tf.python_io.TFRecordOptions(
            tf.python_io.TFRecordCompressionType.GZIP)
      else:
        options = tf.python_io.TFRecordOptions()
      record_iterator = tf.python_io.tf_record_iterator(
          path=image_file, options=options)
      np_input = None
      if FLAGS.input_type == 'tf_example':
        for tf_example_string in record_iterator:
          image_string = tf_example_string
          break  # Only read the first tf.Example.
        np_input = prepare_input_feed(image_string, FLAGS.input_type)
      elif FLAGS.input_type == 'image_tensor':
        example = tf.train.Example()
        for tf_example_string in record_iterator:
          example.ParseFromString(tf_example_string)
          break  # Only read the first tf.Example.
        f = example.features.feature
        image = f['image/ct_image'].bytes_list.value[0]
        image = np.fromstring(
            image, dtype=np.float32).reshape(DESIRED_IMAGE_SHAPE)
        np_input = prepare_input_feed(image, FLAGS.input_type)
      else:
        raise ValueError('Unknown input type %s' % FLAGS.input_type)

      output_nodes = [
          FLAGS.output_classes_node,
          FLAGS.output_scores_node,
      ]
      outputs = sess.run(output_nodes, feed_dict={FLAGS.input_node: np_input})
      classes = outputs[0]
      scores = outputs[1]
      scores = np.squeeze(scores, axis=(0,))
      classes = np.squeeze(classes.astype(np.int32), axis=(0,))

      output_path = os.path.join(output_dir,
                                 os.path.basename(image_file) + '.npz')
      with tf.gfile.GFile(output_path, 'w') as f:
        io_buffer = io.BytesIO()
        np.savez(io_buffer, scores=scores, classes=classes)
        f.write(io_buffer.getvalue())
      print('Output to %s' % output_path)

  print(' - Done!')

if __name__ == '__main__':
  app.run(main)
<|endoftext|>


<commit_before>import json
from enum import Enum
from typing import Dict, List

from eth_utils import decode_hex, to_canonical_address, to_checksum_address
from web3.contract import Contract
from web3.utils.abi import get_abi_input_types
from web3.utils.contracts import encode_transaction_data, find_matching_fn_abi

from raiden.constants import EthClient
from raiden.exceptions import (
    InsufficientFunds,
    ReplacementTransactionUnderpriced,
    TransactionAlreadyPending,
)
from raiden.utils.filters import decode_event

class ClientErrorInspectResult(Enum):
    """Represents the action to follow after inspecting a client exception"""
    PROPAGATE_ERROR = 1
    INSUFFICIENT_FUNDS = 2
    TRANSACTION_UNDERPRICED = 3
    TRANSACTION_PENDING = 4
    ALWAYS_FAIL = 5

def inspect_client_error(val_err: ValueError, eth_node: str) -> ClientErrorInspectResult:
    # both clients return invalid json. They use single quotes while json needs double ones.
    json_response = str(val_err).replace("'", '"')
    try:
        error = json.loads(json_response)
    except json.JSONDecodeError:
        return ClientErrorInspectResult.PROPAGATE_ERROR

    if eth_node == EthClient.GETH:
        if error['code'] == -32000:
            if 'insufficient funds' in error['message']:
                return ClientErrorInspectResult.INSUFFICIENT_FUNDS
            elif 'always failing transaction' in error['message']:
                return ClientErrorInspectResult.ALWAYS_FAIL
            elif error['message'] == 'replacement transaction underpriced':
                return ClientErrorInspectResult.TRANSACTION_UNDERPRICED
            elif error['message'].startswith('known transaction:'):
                return ClientErrorInspectResult.TRANSACTION_PENDING

    elif eth_node == EthClient.PARITY:
        if error['code'] == -32010 and 'insufficient funds' in error['message']:
            return ClientErrorInspectResult.INSUFFICIENT_FUNDS
        elif error['code'] == -32010 and 'another transaction with same nonce in the queue':
            return ClientErrorInspectResult.TRANSACTION_UNDERPRICED
        elif error['code'] == -32015 and 'Transaction execution error' in error['message']:
            return ClientErrorInspectResult.ALWAYS_FAIL

    return ClientErrorInspectResult.PROPAGATE_ERROR

class ContractProxy:
    def __init__(
            self,
            jsonrpc_client,
            contract: Contract,
    ):
        if contract is None:
            raise ValueError('Contract must not be None')
        if jsonrpc_client is None:
            raise ValueError('JSONRPCClient must not be None')
        self.jsonrpc_client = jsonrpc_client
        self.contract = contract

    def transact(self, function_name: str, startgas: int, *args, **kargs):
        data = ContractProxy.get_transaction_data(self.contract.abi, function_name, args)

        try:
            txhash = self.jsonrpc_client.send_transaction(
                to=self.contract.address,
                startgas=startgas,
                value=kargs.pop('value', 0),
                data=decode_hex(data),
                **kargs,
            )
        except ValueError as e:
            action = inspect_client_error(e, self.jsonrpc_client.eth_node)
            if action == ClientErrorInspectResult.INSUFFICIENT_FUNDS:
                raise InsufficientFunds('Insufficient ETH for transaction')
            elif action == ClientErrorInspectResult.TRANSACTION_UNDERPRICED:
                raise ReplacementTransactionUnderpriced(
                    'Transaction was rejected. This is potentially '
                    'caused by the reuse of the previous transaction '
                    'nonce as well as paying an amount of gas less than or '
                    'equal to the previous transaction\'s gas amount',
                )
            elif action == ClientErrorInspectResult.TRANSACTION_PENDING:
                raise TransactionAlreadyPending(
                    'The transaction has already been submitted. Please '
                    'wait until is has been mined or increase the gas price.',
                )

            raise e

        return txhash

    @staticmethod
    def sanitize_args(abi: Dict, args: List):
        """Prepare inputs to match the ABI"""
        inputs = get_abi_input_types(abi)
        output = []
        assert len(inputs) == len(args)
        for input_type, arg in zip(inputs, args):
            if input_type == 'address':
                output.append(to_checksum_address(arg))
            elif input_type == 'bytes' and isinstance(arg, str):
                output.append(arg.encode())
            else:
                output.append(arg)
        return output

    @staticmethod
    def get_transaction_data(abi: Dict, function_name: str, args: List = None):
        """Get encoded transaction data"""
        args = args or list()
        fn_abi = find_matching_fn_abi(
            abi,
            function_name,
            args,
        )
        args = ContractProxy.sanitize_args(fn_abi, args)
        return encode_transaction_data(
            None,
            function_name,
            contract_abi=abi,
            fn_abi=fn_abi,
            args=args,
        )

    def decode_transaction_input(self, transaction_hash: bytes) -> Dict:
        """Return inputs of a method call"""
        transaction = self.contract.web3.eth.getTransaction(
            transaction_hash,
        )

        return self.contract.decode_function_input(
            transaction['input'],
        )

    def decode_event(self, log: Dict):
        return decode_event(self.contract.abi, log)

    def encode_function_call(self, function: str, args: List = None):
        return self.get_transaction_data(self.contract.abi, function, args)

    def estimate_gas(self, function: str, *args):
        fn = getattr(self.contract.functions, function)
        try:
            return fn(*args).estimateGas({
                'from': to_checksum_address(self.jsonrpc_client.address),
            })
        except ValueError as err:
            action = inspect_client_error(err, self.jsonrpc_client.eth_node)
            will_fail = action in (
                ClientErrorInspectResult.INSUFFICIENT_FUNDS,
                ClientErrorInspectResult.ALWAYS_FAIL,
            )
            if will_fail:
                return None

            raise err

    @property
    def contract_address(self):
        return to_canonical_address(self.contract.address)
<commit_msg>Properly return TRANSACTION_UNDERPRICED for parity<commit_after>import json
from enum import Enum
from typing import Dict, List

from eth_utils import decode_hex, to_canonical_address, to_checksum_address
from web3.contract import Contract
from web3.utils.abi import get_abi_input_types
from web3.utils.contracts import encode_transaction_data, find_matching_fn_abi

from raiden.constants import EthClient
from raiden.exceptions import (
    InsufficientFunds,
    ReplacementTransactionUnderpriced,
    TransactionAlreadyPending,
)
from raiden.utils.filters import decode_event

class ClientErrorInspectResult(Enum):
    """Represents the action to follow after inspecting a client exception"""
    PROPAGATE_ERROR = 1
    INSUFFICIENT_FUNDS = 2
    TRANSACTION_UNDERPRICED = 3
    TRANSACTION_PENDING = 4
    ALWAYS_FAIL = 5

def inspect_client_error(val_err: ValueError, eth_node: str) -> ClientErrorInspectResult:
    # both clients return invalid json. They use single quotes while json needs double ones.
    json_response = str(val_err).replace("'", '"')
    try:
        error = json.loads(json_response)
    except json.JSONDecodeError:
        return ClientErrorInspectResult.PROPAGATE_ERROR

    if eth_node == EthClient.GETH:
        if error['code'] == -32000:
            if 'insufficient funds' in error['message']:
                return ClientErrorInspectResult.INSUFFICIENT_FUNDS
            elif 'always failing transaction' in error['message']:
                return ClientErrorInspectResult.ALWAYS_FAIL
            elif error['message'] == 'replacement transaction underpriced':
                return ClientErrorInspectResult.TRANSACTION_UNDERPRICED
            elif error['message'].startswith('known transaction:'):
                return ClientErrorInspectResult.TRANSACTION_PENDING

    elif eth_node == EthClient.PARITY:
        if error['code'] == -32010:
            if 'insufficient funds' in error['message']:
                return ClientErrorInspectResult.INSUFFICIENT_FUNDS
            elif 'another transaction with same nonce in the queue' in error['message']:
                return ClientErrorInspectResult.TRANSACTION_UNDERPRICED
        elif error['code'] == -32015 and 'Transaction execution error' in error['message']:
            return ClientErrorInspectResult.ALWAYS_FAIL

    return ClientErrorInspectResult.PROPAGATE_ERROR

class ContractProxy:
    def __init__(
            self,
            jsonrpc_client,
            contract: Contract,
    ):
        if contract is None:
            raise ValueError('Contract must not be None')
        if jsonrpc_client is None:
            raise ValueError('JSONRPCClient must not be None')
        self.jsonrpc_client = jsonrpc_client
        self.contract = contract

    def transact(self, function_name: str, startgas: int, *args, **kargs):
        data = ContractProxy.get_transaction_data(self.contract.abi, function_name, args)

        try:
            txhash = self.jsonrpc_client.send_transaction(
                to=self.contract.address,
                startgas=startgas,
                value=kargs.pop('value', 0),
                data=decode_hex(data),
                **kargs,
            )
        except ValueError as e:
            action = inspect_client_error(e, self.jsonrpc_client.eth_node)
            if action == ClientErrorInspectResult.INSUFFICIENT_FUNDS:
                raise InsufficientFunds('Insufficient ETH for transaction')
            elif action == ClientErrorInspectResult.TRANSACTION_UNDERPRICED:
                raise ReplacementTransactionUnderpriced(
                    'Transaction was rejected. This is potentially '
                    'caused by the reuse of the previous transaction '
                    'nonce as well as paying an amount of gas less than or '
                    'equal to the previous transaction\'s gas amount',
                )
            elif action == ClientErrorInspectResult.TRANSACTION_PENDING:
                raise TransactionAlreadyPending(
                    'The transaction has already been submitted. Please '
                    'wait until is has been mined or increase the gas price.',
                )

            raise e

        return txhash

    @staticmethod
    def sanitize_args(abi: Dict, args: List):
        """Prepare inputs to match the ABI"""
        inputs = get_abi_input_types(abi)
        output = []
        assert len(inputs) == len(args)
        for input_type, arg in zip(inputs, args):
            if input_type == 'address':
                output.append(to_checksum_address(arg))
            elif input_type == 'bytes' and isinstance(arg, str):
                output.append(arg.encode())
            else:
                output.append(arg)
        return output

    @staticmethod
    def get_transaction_data(abi: Dict, function_name: str, args: List = None):
        """Get encoded transaction data"""
        args = args or list()
        fn_abi = find_matching_fn_abi(
            abi,
            function_name,
            args,
        )
        args = ContractProxy.sanitize_args(fn_abi, args)
        return encode_transaction_data(
            None,
            function_name,
            contract_abi=abi,
            fn_abi=fn_abi,
            args=args,
        )

    def decode_transaction_input(self, transaction_hash: bytes) -> Dict:
        """Return inputs of a method call"""
        transaction = self.contract.web3.eth.getTransaction(
            transaction_hash,
        )

        return self.contract.decode_function_input(
            transaction['input'],
        )

    def decode_event(self, log: Dict):
        return decode_event(self.contract.abi, log)

    def encode_function_call(self, function: str, args: List = None):
        return self.get_transaction_data(self.contract.abi, function, args)

    def estimate_gas(self, function: str, *args):
        fn = getattr(self.contract.functions, function)
        try:
            return fn(*args).estimateGas({
                'from': to_checksum_address(self.jsonrpc_client.address),
            })
        except ValueError as err:
            action = inspect_client_error(err, self.jsonrpc_client.eth_node)
            will_fail = action in (
                ClientErrorInspectResult.INSUFFICIENT_FUNDS,
                ClientErrorInspectResult.ALWAYS_FAIL,
            )
            if will_fail:
                return None

            raise err

    @property
    def contract_address(self):
        return to_canonical_address(self.contract.address)
<|endoftext|>


<commit_before>import re
import sys
import json
import time
import ConfigParser

from intelmq.lib.message import Event
from intelmq.lib.pipeline import Pipeline
from intelmq.lib.utils import decode, log

SYSTEM_CONF_FILE = "/opt/intelmq/etc/system.conf"
PIPELINE_CONF_FILE = "/opt/intelmq/etc/pipeline.conf"
RUNTIME_CONF_FILE = "/opt/intelmq/etc/runtime.conf"
DEFAULT_LOGGING_PATH = "/opt/intelmq/var/log/"
DEFAULT_LOGGING_LEVEL = "INFO"

class Bot(object):

    def __init__(self, bot_id):
        self.parameters = Parameters()

        self.current_message = None
        self.last_message = None
        self.message_counter = 0

        self.check_bot_id(bot_id)
        self.bot_id = bot_id

        self.load_system_configurations()

        self.logger = log(self.parameters.logging_path, self.bot_id, self.parameters.logging_level)
        self.logger.info('Bot is starting')

        self.load_runtime_configurations()
        self.load_pipeline_configurations()

        self.init()

    def init(self):
        pass

    def start(self):
        self.source_pipeline = None
        self.destination_pipeline = None
        local_retry_delay = 0
        self.parameters.retry_delay = 30 # Temporary fix. Need to add to BOTS conf

        self.logger.info('Bot start processing')

        while True:
            try:
                if not self.source_pipeline:
                    time.sleep(local_retry_delay)
                    self.logger.info("Connecting to source pipeline")
                    self.source_pipeline = Pipeline()
                    self.source_pipeline.source_queues(self.source_queues)
                    self.logger.info("Connected to source pipeline")

                if not self.destination_pipeline:
                    time.sleep(local_retry_delay)
                    self.logger.info("Connecting to destination pipeline")
                    self.destination_pipeline = Pipeline()
                    self.destination_pipeline.destination_queues(self.destination_queues)
                    self.logger.info("Connected to destination pipeline")

                self.process()
                self.source_pipeline.sleep(self.parameters.rate_limit)

            except Exception, ex:
                local_retry_delay = self.parameters.retry_delay
                self.logger.info("Last Correct Message(event): %r" % self.last_message)
                self.logger.info("Current Message(event): %r" % self.current_message)
                self.logger.exception("Check the following exception:")
                self.logger.error('Pipeline connection failed (%r)' % ex)
                self.logger.info('Pipeline will reconnect in %s seconds' % local_retry_delay)
                self.source_pipeline = None
                self.destination_pipeline = None

            except KeyboardInterrupt as e:
                if self.source_pipeline:
                    self.source_pipeline.disconnect()
                    self.logger.info("Disconnecting from source pipeline")
                if self.destination_pipeline:
                    self.destination_pipeline.disconnect()
                    self.logger.info("Disconnecting from destination pipeline")

                self.logger.info("Bot is shutting down")
                break

    def stop(self):
        try:
            self.logger.error("Bot found an error. Exiting")
        except:
            pass
        finally:
            print "Bot found an error. Exiting"
        exit(-1)

    def check_bot_id(self, str):
        res = re.search('[^0-9a-zA-Z\-]+', str)
        if res:
            print "Invalid bot id."
            self.stop()

    def load_system_configurations(self):

        with open(SYSTEM_CONF_FILE, 'r') as fpconfig:
            config = json.loads(fpconfig.read())

        setattr(self.parameters, 'logging_path' , DEFAULT_LOGGING_PATH)
        setattr(self.parameters, 'logging_level' , DEFAULT_LOGGING_LEVEL)

        for option, value in config.iteritems():
            setattr(self.parameters, option, value)

    def load_runtime_configurations(self):

        with open(RUNTIME_CONF_FILE, 'r') as fpconfig:
            config = json.loads(fpconfig.read())

        # Load __default__ runtime configuration section

        self.logger.debug("Runtime configuration: loading '%s' section from '%s' file" % ("__default__", RUNTIME_CONF_FILE))
        if "__default__" in config.keys():
            for option, value in config["__default__"].iteritems():
                setattr(self.parameters, option, value)
                self.logger.debug("Runtime configuration: parameter '%s' loaded with the value '%s'" % (option, value))

        # Load bot runtime configuration section

        self.logger.debug("Runtime configuration: loading '%s' section from '%s' file" % (self.bot_id, RUNTIME_CONF_FILE))
        if self.bot_id in config.keys():
            for option, value in config[self.bot_id].iteritems():
                setattr(self.parameters, option, value)
                self.logger.debug("Runtime configuration: parameter '%s' loaded with the value '%s'" % (option, value))

    def load_pipeline_configurations(self):
        with open(PIPELINE_CONF_FILE, 'r') as fpconfig:
            config = json.loads(fpconfig.read())

        self.logger.debug("Pipeline configuration: loading '%s' section from '%s' file" % (self.bot_id, PIPELINE_CONF_FILE))

        self.source_queues = None
        self.destination_queues = None

        if self.bot_id in config.keys():

            if 'source-queue' in config[self.bot_id].keys():
                self.source_queues = config[self.bot_id]['source-queue']
                self.logger.debug("Pipeline configuration: parameter 'source-queue' loaded with the value '%s'" % self.source_queues)

            if 'destination-queues' in config[self.bot_id].keys():
                self.destination_queues = config[self.bot_id]['destination-queues']
                self.logger.debug("Pipeline configuration: parameter 'destination-queues' loaded with the value '%s'" % ", ".join(self.destination_queues))

        else:
            self.logger.error("Pipeline configuration: failed to load configuration")
            self.stop()

    def send_message(self, message):
        if not message:
            self.logger.warning("Empty message found.")
            return False

        if isinstance(message, Event):
            message = unicode(message) # convert Event Object to string (UTF-8)

        self.message_counter += 1
        if self.message_counter % 500 == 0:
            self.logger.info("Processed %s messages" % self.message_counter)

        self.destination_pipeline.send(message)

    def receive_message(self):
        self.current_message = self.source_pipeline.receive()

        if not self.current_message:
            return None

        message = self.current_message.decode('utf-8')

        try:    # Event Object
            return Event.from_unicode(message)
        except: # Report Object
            return message

    def acknowledge_message(self):
        self.last_message = self.current_message
        self.source_pipeline.acknowledge()

class Parameters(object):
    pass
<commit_msg>Add docstrings to lib/bot.py and pep8 mods<commit_after>import re
import sys
import json
import time
import ConfigParser

from intelmq.lib.message import Event
from intelmq.lib.pipeline import Pipeline
from intelmq.lib.utils import decode, log

SYSTEM_CONF_FILE = "/opt/intelmq/etc/system.conf"
PIPELINE_CONF_FILE = "/opt/intelmq/etc/pipeline.conf"
RUNTIME_CONF_FILE = "/opt/intelmq/etc/runtime.conf"
DEFAULT_LOGGING_PATH = "/opt/intelmq/var/log/"
DEFAULT_LOGGING_LEVEL = "INFO"

class Bot(object):

    def __init__(self, bot_id):
        self.parameters = Parameters()

        self.current_message = None
        self.last_message = None
        self.message_counter = 0

        self.check_bot_id(bot_id)
        self.bot_id = bot_id

        self.load_system_configurations()

        self.logger = log(self.parameters.logging_path, self.bot_id, self.parameters.logging_level)
        self.logger.info('Bot is starting')

        self.load_runtime_configurations()
        self.load_pipeline_configurations()

        self.init()

    def init(self):
        """Initializes a bot"""
        pass

    def start(self):
        """Starts a bot"""
        self.source_pipeline = None
        self.destination_pipeline = None
        local_retry_delay = 0
        self.parameters.retry_delay = 30 # Temporary fix. Need to add to BOTS conf

        self.logger.info('Bot start processing')

        while True:
            try:
                if not self.source_pipeline:
                    time.sleep(local_retry_delay)
                    self.logger.info("Connecting to source pipeline")
                    self.source_pipeline = Pipeline()
                    self.source_pipeline.source_queues(self.source_queues)
                    self.logger.info("Connected to source pipeline")

                if not self.destination_pipeline:
                    time.sleep(local_retry_delay)
                    self.logger.info("Connecting to destination pipeline")
                    self.destination_pipeline = Pipeline()
                    self.destination_pipeline.destination_queues(self.destination_queues)
                    self.logger.info("Connected to destination pipeline")

                self.process()
                self.source_pipeline.sleep(self.parameters.rate_limit)

            except Exception, ex:
                local_retry_delay = self.parameters.retry_delay
                self.logger.info("Last Correct Message(event): %r" % self.last_message)
                self.logger.info("Current Message(event): %r" % self.current_message)
                self.logger.exception("Check the following exception:")
                self.logger.error('Pipeline connection failed (%r)' % ex)
                self.logger.info('Pipeline will reconnect in %s seconds' % local_retry_delay)
                self.source_pipeline = None
                self.destination_pipeline = None

            except KeyboardInterrupt as e:
                if self.source_pipeline:
                    self.source_pipeline.disconnect()
                    self.logger.info("Disconnecting from source pipeline")
                if self.destination_pipeline:
                    self.destination_pipeline.disconnect()
                    self.logger.info("Disconnecting from destination pipeline")

                self.logger.info("Bot is shutting down")
                break

    def stop(self):
        """Stops a bot"""
        try:
            self.logger.error("Bot found an error. Exiting")
        except:
            pass
        finally:
            print "Bot found an error. Exiting"
        exit(-1)

    def check_bot_id(self, str):
        """Returns True if the given str is a valid bot id"""
        res = re.search('[^0-9a-zA-Z\-]+', str)
        if res:
            print "Invalid bot id."
            self.stop()

    def load_system_configurations(self):
        """Instructs a bot to load the system configuration (json)"""

        with open(SYSTEM_CONF_FILE, 'r') as fpconfig:
            config = json.loads(fpconfig.read())

        setattr(self.parameters, 'logging_path', DEFAULT_LOGGING_PATH)
        setattr(self.parameters, 'logging_level', DEFAULT_LOGGING_LEVEL)

        for option, value in config.iteritems():
            setattr(self.parameters, option, value)

    def load_runtime_configurations(self):
        """Load runtime json configuration for a bot"""

        with open(RUNTIME_CONF_FILE, 'r') as fpconfig:
            config = json.loads(fpconfig.read())

        # Load __default__ runtime configuration section

        self.logger.debug("Runtime configuration: loading '%s' section from '%s' file" % ("__default__", RUNTIME_CONF_FILE))
        if "__default__" in config.keys():
            for option, value in config["__default__"].iteritems():
                setattr(self.parameters, option, value)
                self.logger.debug("Runtime configuration: parameter '%s' loaded with the value '%s'" % (option, value))

        # Load bot runtime configuration section

        self.logger.debug("Runtime configuration: loading '%s' section from '%s' file" % (self.bot_id, RUNTIME_CONF_FILE))
        if self.bot_id in config.keys():
            for option, value in config[self.bot_id].iteritems():
                setattr(self.parameters, option, value)
                self.logger.debug("Runtime configuration: parameter '%s' loaded with the value '%s'" % (option, value))

    def load_pipeline_configurations(self):
        """Load pipeline json configuration file"""

        with open(PIPELINE_CONF_FILE, 'r') as fpconfig:
            config = json.loads(fpconfig.read())

        self.logger.debug("Pipeline configuration: loading '%s' section from '%s' file" % (self.bot_id, PIPELINE_CONF_FILE))

        self.source_queues = None
        self.destination_queues = None

        if self.bot_id in config.keys():

            if 'source-queue' in config[self.bot_id].keys():
                self.source_queues = config[self.bot_id]['source-queue']
                self.logger.debug("Pipeline configuration: parameter 'source-queue' loaded with the value '%s'" % self.source_queues)

            if 'destination-queues' in config[self.bot_id].keys():
                self.destination_queues = config[self.bot_id]['destination-queues']
                self.logger.debug("Pipeline configuration: parameter 'destination-queues' loaded with the value '%s'" % ", ".join(self.destination_queues))

        else:
            self.logger.error("Pipeline configuration: failed to load configuration")
            self.stop()

    def send_message(self, message):
        """Sends a given message with configured destination pipeline"""

        if not message:
            self.logger.warning("Empty message found.")
            return False

        if isinstance(message, Event):
            message = unicode(message)  # convert Event Object to string (UTF-8)

        self.message_counter += 1
        if self.message_counter % 500 == 0:
            self.logger.info("Processed %s messages" % self.message_counter)

        self.destination_pipeline.send(message)

    def receive_message(self):
        """Receive a message from the configured source queue"""
        self.current_message = self.source_pipeline.receive()

        if not self.current_message:
            return None

        message = self.current_message.decode('utf-8')

        try:     # Event Object
            return Event.from_unicode(message)
        except:  # Report Object
            return message

    def acknowledge_message(self):
        """Acknowledge a message and remove it from the queue system permanently"""
        self.last_message = self.current_message
        self.source_pipeline.acknowledge()

class Parameters(object):
    pass
<|endoftext|>