<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
=============
TAP plus
=============

@author: Juan Carlos Segovia
@contact: juan.carlos.segovia@sciops.esa.int

European Space Astronomy Centre (ESAC)
European Space Agency (ESA)

Created on 30 jun. 2016

"""

import time

from astroquery.utils.tap.model import modelutils
from astroquery.utils.tap.xmlparser import utils

__all__ = ['Job']

class Job(object):
    """Job class
    """

    def __init__(self, async_job, query=None, connhandler=None):
        """Constructor

        Parameters
        ----------
        async_job : bool, mandatory
            'True' if the job is asynchronous
        query : str, optional, default None
            Query
        connhandler : TapConn, optional, default None
            Connection handler
        """
        self.__internal_init()
        self.__connHandler = connhandler
        self.__async = async_job
        self.__parameters['query'] = query

    def __internal_init(self):
        self.__connHandler = None
        self.__isFinished = None
        self.__jobid = None
        self.__remoteLocation = None
        self.__phase = None
        self.__async = None
        self.__outputFile = None
        self.__responseStatus = 0
        self.__responseMsg = None
        self.__results = None
        self.__resultInMemory = False
        self.__failed = False
        self.__runid = None
        self.__ownerid = None
        self.__startTime = None
        self.__endTime = None
        self.__creationTime = None
        self.__executionDuration = None
        self.__destruction = None
        self.__locationId = None
        self.__name = None
        self.__quote = None
        self.__parameters = {}
        # default output format
        self.set_output_format('votable')

    def set_connhandler(self, connhandler):
        self.__connHandler = connhandler

    def set_jobid(self, jobid):
        """Sets job identifier

        Parameters
        ----------
        jobid : str, mandatory
            job identifier
        """
        self.__jobid = jobid

    def get_jobid(self):
        """Returns the job identifier

        Returns
        -------
        The job identifier
        """
        return self.__jobid

    def set_failed(self, failed=False):
        """Sets the job status to failed

        Parameters
        ----------
        failed : bool, optional, default 'False'
            failed status
        """
        self.__failed = failed

    def is_failed(self):
        """Returns the job status

        Returns
        -------
        'True' if the job is failed
        """
        return self.__failed

    def set_remote_location(self, location):
        """Sets the job remote location

        Parameters
        ----------
        location : str, mandatory
            job remote location
        """
        self.__remoteLocation = location

    def get_remote_location(self):
        """Returns the job remote location

        Returns
        -------
        The job remote location
        """
        return self.__remoteLocation

    def set_phase(self, phase):
        """Sets the job phase

        Parameters
        ----------
        phase : str, mandatory
            job phase
        """
        self.__phase = phase

    def get_phase(self, update=False):
        """Returns the job phase. May optionally update the job's phase.

        Parameters
        ----------
        update : bool
            if True, the phase will by updated by querying the server before
            returning.

        Returns
        -------
        The job phase
        """
        if update:
            phase_request = "async/"+str(self.get_jobid())+"/phase"
            response = self.__connHandler.execute_get(phase_request)
            if response.status != 200:
                raise Exception(response.reason)

            self.set_phase(str(response.read().decode('utf-8')))

        return self.__phase

    def set_output_file(self, output_file):
        """Sets the job output file

        Parameters
        ----------
        output_file : str, mandatory
            job output file
        """
        self.__outputFile = output_file

    def get_output_file(self):
        """Returns the job output file

        Returns
        -------
        The results output file
        """
        return self.__outputFile

    def set_response_status(self, status, msg):
        """Sets the HTTP(s) connection status

        Parameters
        ----------
        status : int, mandatory
            HTTP(s) response status
        msg : str, mandatory
            HTTP(s) response message
        """
        self.__responseStatus = status
        self.__responseMsg = msg

    def get_response_status(self):
        """Returns the HTTP(s) connection status

        Returns
        -------
        The HTTP(s) connection response status
        """
        return self.__responseStatus

    def get_response_msg(self):
        """Returns the HTTP(s) connection message

        Returns
        -------
        The HTTP(s) connection response message
        """
        return self.__responseMsg

    def set_output_format(self, output_format):
        """Sets the job output format

        Parameters
        ----------
        output_format : str, mandatory
            job results output format
        """
        self.__parameters['format'] = output_format

    def get_output_format(self):
        """Returns the job output format

        Returns
        -------
        The job results output format
        """
        return self.__parameters['format']

    def is_sync(self):
        """Returns True if this job was executed synchronously

        Returns
        -------
        'True' if the job is synchronous
        """
        return not self.__async

    def is_async(self):
        """Returns True if this job was executed asynchronously

        Returns
        -------
        'True' if the job is synchronous
        """
        return self.__async

    def get_query(self):
        """Returns the job query

        Returns
        -------
        The job query
        """
        return self.__parameters['query']

    def get_runid(self):
        """Returns the job run identifier

        Returns
        -------
        The job run identifier
        """
        return self.__runid

    def set_runid(self, runid):
        """Sets the job run identifier

        Parameters
        ----------
        runid : str, mandatory
            job run identifier
        """
        self.__runid = runid

    def get_ownerid(self):
        """Returns the job owner identifier

        Returns
        -------
        The job owner identifier
        """
        return self.__ownerid

    def set_ownerid(self, ownerid):
        """Sets the job owner identifier

        Parameters
        ----------
        ownerid : str, mandatory
            job owner identifier
        """
        self.__ownerid = ownerid

    def set_start_time(self, starttime):
        """Sets the job start time

        Parameters
        ----------
        starttime : str, mandatory
            job start time
        """
        self.__startTime = starttime

    def get_start_time(self):
        """Returns the job start time

        Returns
        -------
        The job start time
        """
        return self.__startTime

    def set_end_time(self, endtime):
        """Sets the job end time

        Parameters
        ----------
        endtime : str, mandatory
            job end time
        """
        self.__endTime = endtime

    def get_end_time(self):
        """Returns the job end time

        Returns
        -------
        The job end time
        """
        return self.__endTime

    def set_creation_time(self, creationtime):
        """Sets the job creation time

        Parameters
        ----------
        creationtime : str, mandatory
            job creation time
        """
        self.__creationTime = creationtime

    def get_creation_time(self):
        """Returns the job creation time

        Returns
        -------
        The job creation time
        """
        return self.__creationTime

    def set_execution_duration(self, executionduration):
        """Sets the job execution duration

        Parameters
        ----------
        executionduration : int, mandatory
            job execution duration
        """
        self.__executionDuration = executionduration

    def get_execution_duration(self):
        """Returns the job execution duration

        Returns
        -------
        The job execution duration
        """
        return self.__executionDuration

    def set_destruction(self, destruction):
        """Sets the job destruction value

        Parameters
        ----------
        destruction : int, mandatory
            job destruction
        """
        self.__destruction = destruction

    def get_destruction(self):
        """Returns the job destruction value

        Returns
        -------
        The job destruction value
        """
        return self.__destruction

    def set_locationid(self, locationid):
        """Sets the job location identifier

        Parameters
        ----------
        locationid : str, mandatory
            job location identifier
        """
        self.__locationId = locationid

    def get_locationid(self):
        """Returns the job location identifier

        Returns
        -------
        The job location identifier
        """
        return self.__locationId

    def set_name(self, name):
        """Sets the job name

        Parameters
        ----------
        name : str, mandatory
            job name
        """
        self.__name = name

    def get_name(self):
        """Returns the job name

        Returns
        -------
        The job name
        """
        return self.__name

    def set_quote(self, quote):
        """Sets the job quote

        Parameters
        ----------
        quote : int, mandatory
            job quote
        """
        self.__quote = quote

    def get_quote(self):
        """Returns the job quote

        Returns
        -------
        The job quote
        """
        return self.__quote

    def set_parameter(self, key, value):
        """Sets a job parameter

        Parameters
        ----------
        key : str, mandatory
            job parameter key
        value : str, mandatory
            job parameter value
        """
        self.__parameters[key] = value

    def get_parameter(self, key):
        """Returns a job parameter

        Parameters
        ----------
        key : str, mandatory
            job parameter key

        Returns
        -------
        The job parameter value
        """
        return self.__parameters[key]

    def get_parameters(self):
        """Returns the job parameters

        Returns
        -------
        The job parameters (a list)
        """
        return self.__parameters

    def get_data(self):
        """Returns the job results (Astroquery API specification)
        This method will block if the job is asynchronous and the job has not
        finished yet.

        Returns
        -------
        The job results (astropy.table).
        """
        return self.get_results()

    def get_results(self):
        """Returns the job results
        This method will block if the job is asynchronous and the job has not
        finished yet.

        Returns
        -------
        The job results (astropy.table).
        """
        if self.__results is not None:
            return self.__results
        # try load results from file
        # read_results_table_from_file checks whether the file already exists or not
        outputFormat = self.get_output_format()
        results = modelutils.read_results_table_from_file(self.__outputFile,
                                                          outputFormat)
        if results is not None:
            self.set_results(results)
            return results
        # Try to load from server: only async
        if not self.__async:
            # sync: result is in a file
            return None
        else:
            # async: result is in the server once the job is finished
            self.__load_async_job_results()
            return self.__results

    def set_results(self, results):
        """Sets the job results

        Parameters
        ----------
        results : Table object, mandatory
            job results
        """
        self.__results = results
        self.__resultInMemory = True

    def save_results(self, verbose=False):
        """Saves job results
        If the job is asynchronous, this method will block until the results
        are available.

        Parameters
        ----------
        verbose : bool, optional, default 'False'
            flag to display information about the process
        """
        output = self.get_output_file()
        if self.__resultInMemory:
            self.__results.to_xml(output)
        else:
            if self.is_sync():
                # sync: cannot access server again
                print("No results to save")
            else:
                # Async
                self.wait_for_job_end(verbose)
                response = self.__connHandler.execute_get(
                    "async/"+str(self.__jobid)+"/results/result")
                if verbose:
                    print(response.status, response.reason)
                    print(response.getheaders())
                isError = self.__connHandler.check_launch_response_status(response,
                                                                          verbose,
                                                                          200)
                if isError:
                    print(response.reason)
                    raise Exception(response.reason)
                self.__connHandler.dump_to_file(output, response)

    def wait_for_job_end(self, verbose=False):
        """Waits until a job is finished

        Parameters
        ----------
        verbose : bool, optional, default 'False'
            flag to display information about the process
        """
        phaseRequest = "async/"+str(self.__jobid)+"/phase"
        currentResponse = None
        responseData = None
        while True:
            response = self.__connHandler.execute_get(phaseRequest)
            currentResponse = response.status
            if response.status != 200:
                raise Exception(response.reason)
                break
            responseData = str(response.read().decode('utf-8'))
            data = responseData.lower().strip()
            if verbose:
                print("Job " + self.__jobid + " status: " + data)
            if "pending" != data and "queued" != data and "executing" != data:
                break
            # PENDING, QUEUED, EXECUTING, COMPLETED, ERROR, ABORTED, UNKNOWN,
            # HELD, SUSPENDED, ARCHIVED:
            time.sleep(0.5)
        return currentResponse, responseData

    def __load_async_job_results(self, debug=False):
        wjResponse, wjData = self.wait_for_job_end()
        subContext = "async/" + str(self.__jobid) + "/results/result"
        resultsResponse = self.__connHandler.execute_get(subContext)
        # resultsResponse = self.__readAsyncResults(self.__jobid, debug)
        if debug:
            print(resultsResponse.status, resultsResponse.reason)
            print(resultsResponse.getheaders())
        isError = self.__connHandler.check_launch_response_status(resultsResponse,
                                                                  debug,
                                                                  200)
        if isError:
            print(resultsResponse.reason)
            raise Exception(resultsResponse.reason)
        else:
            outputFormat = self.get_output_format()
            results = utils.read_http_response(resultsResponse, outputFormat)
            self.set_results(results)
            self.__phase = wjData

    def __str__(self):
        if self.__results is None:
            result = "None"
        else:
            result = self.__results.info()
        return "Jobid: " + str(self.__jobid) + \
            "\nPhase: " + str(self.__phase) + \
            "\nOwner: " + str(self.__ownerid) + \
            "\nOutput file: " + str(self.__outputFile) + \
            "\nResults: " + str(result)
<commit_msg>change wait_for_job_end to use new get_phase<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
=============
TAP plus
=============

@author: Juan Carlos Segovia
@contact: juan.carlos.segovia@sciops.esa.int

European Space Astronomy Centre (ESAC)
European Space Agency (ESA)

Created on 30 jun. 2016

"""

import time

from astroquery.utils.tap.model import modelutils
from astroquery.utils.tap.xmlparser import utils

__all__ = ['Job']

class Job(object):
    """Job class
    """

    def __init__(self, async_job, query=None, connhandler=None):
        """Constructor

        Parameters
        ----------
        async_job : bool, mandatory
            'True' if the job is asynchronous
        query : str, optional, default None
            Query
        connhandler : TapConn, optional, default None
            Connection handler
        """
        self.__internal_init()
        self.__connHandler = connhandler
        self.__async = async_job
        self.__parameters['query'] = query

    def __internal_init(self):
        self.__connHandler = None
        self.__isFinished = None
        self.__jobid = None
        self.__remoteLocation = None
        self.__phase = None
        self.__async = None
        self.__outputFile = None
        self.__responseStatus = 0
        self.__responseMsg = None
        self.__results = None
        self.__resultInMemory = False
        self.__failed = False
        self.__runid = None
        self.__ownerid = None
        self.__startTime = None
        self.__endTime = None
        self.__creationTime = None
        self.__executionDuration = None
        self.__destruction = None
        self.__locationId = None
        self.__name = None
        self.__quote = None
        self.__parameters = {}
        # default output format
        self.set_output_format('votable')

    def set_connhandler(self, connhandler):
        self.__connHandler = connhandler

    def set_jobid(self, jobid):
        """Sets job identifier

        Parameters
        ----------
        jobid : str, mandatory
            job identifier
        """
        self.__jobid = jobid

    def get_jobid(self):
        """Returns the job identifier

        Returns
        -------
        The job identifier
        """
        return self.__jobid

    def set_failed(self, failed=False):
        """Sets the job status to failed

        Parameters
        ----------
        failed : bool, optional, default 'False'
            failed status
        """
        self.__failed = failed

    def is_failed(self):
        """Returns the job status

        Returns
        -------
        'True' if the job is failed
        """
        return self.__failed

    def set_remote_location(self, location):
        """Sets the job remote location

        Parameters
        ----------
        location : str, mandatory
            job remote location
        """
        self.__remoteLocation = location

    def get_remote_location(self):
        """Returns the job remote location

        Returns
        -------
        The job remote location
        """
        return self.__remoteLocation

    def set_phase(self, phase):
        """Sets the job phase

        Parameters
        ----------
        phase : str, mandatory
            job phase
        """
        self.__phase = phase

    def get_phase(self, update=False):
        """Returns the job phase. May optionally update the job's phase.

        Parameters
        ----------
        update : bool
            if True, the phase will by updated by querying the server before
            returning.

        Returns
        -------
        The job phase
        """
        if update:
            phase_request = "async/"+str(self.get_jobid())+"/phase"
            response = self.__connHandler.execute_get(phase_request)

            self.__last_phase_response_status = response.status
            if response.status != 200:
                raise Exception(response.reason)

            self.set_phase(str(response.read().decode('utf-8')))

        return self.__phase

    def set_output_file(self, output_file):
        """Sets the job output file

        Parameters
        ----------
        output_file : str, mandatory
            job output file
        """
        self.__outputFile = output_file

    def get_output_file(self):
        """Returns the job output file

        Returns
        -------
        The results output file
        """
        return self.__outputFile

    def set_response_status(self, status, msg):
        """Sets the HTTP(s) connection status

        Parameters
        ----------
        status : int, mandatory
            HTTP(s) response status
        msg : str, mandatory
            HTTP(s) response message
        """
        self.__responseStatus = status
        self.__responseMsg = msg

    def get_response_status(self):
        """Returns the HTTP(s) connection status

        Returns
        -------
        The HTTP(s) connection response status
        """
        return self.__responseStatus

    def get_response_msg(self):
        """Returns the HTTP(s) connection message

        Returns
        -------
        The HTTP(s) connection response message
        """
        return self.__responseMsg

    def set_output_format(self, output_format):
        """Sets the job output format

        Parameters
        ----------
        output_format : str, mandatory
            job results output format
        """
        self.__parameters['format'] = output_format

    def get_output_format(self):
        """Returns the job output format

        Returns
        -------
        The job results output format
        """
        return self.__parameters['format']

    def is_sync(self):
        """Returns True if this job was executed synchronously

        Returns
        -------
        'True' if the job is synchronous
        """
        return not self.__async

    def is_async(self):
        """Returns True if this job was executed asynchronously

        Returns
        -------
        'True' if the job is synchronous
        """
        return self.__async

    def get_query(self):
        """Returns the job query

        Returns
        -------
        The job query
        """
        return self.__parameters['query']

    def get_runid(self):
        """Returns the job run identifier

        Returns
        -------
        The job run identifier
        """
        return self.__runid

    def set_runid(self, runid):
        """Sets the job run identifier

        Parameters
        ----------
        runid : str, mandatory
            job run identifier
        """
        self.__runid = runid

    def get_ownerid(self):
        """Returns the job owner identifier

        Returns
        -------
        The job owner identifier
        """
        return self.__ownerid

    def set_ownerid(self, ownerid):
        """Sets the job owner identifier

        Parameters
        ----------
        ownerid : str, mandatory
            job owner identifier
        """
        self.__ownerid = ownerid

    def set_start_time(self, starttime):
        """Sets the job start time

        Parameters
        ----------
        starttime : str, mandatory
            job start time
        """
        self.__startTime = starttime

    def get_start_time(self):
        """Returns the job start time

        Returns
        -------
        The job start time
        """
        return self.__startTime

    def set_end_time(self, endtime):
        """Sets the job end time

        Parameters
        ----------
        endtime : str, mandatory
            job end time
        """
        self.__endTime = endtime

    def get_end_time(self):
        """Returns the job end time

        Returns
        -------
        The job end time
        """
        return self.__endTime

    def set_creation_time(self, creationtime):
        """Sets the job creation time

        Parameters
        ----------
        creationtime : str, mandatory
            job creation time
        """
        self.__creationTime = creationtime

    def get_creation_time(self):
        """Returns the job creation time

        Returns
        -------
        The job creation time
        """
        return self.__creationTime

    def set_execution_duration(self, executionduration):
        """Sets the job execution duration

        Parameters
        ----------
        executionduration : int, mandatory
            job execution duration
        """
        self.__executionDuration = executionduration

    def get_execution_duration(self):
        """Returns the job execution duration

        Returns
        -------
        The job execution duration
        """
        return self.__executionDuration

    def set_destruction(self, destruction):
        """Sets the job destruction value

        Parameters
        ----------
        destruction : int, mandatory
            job destruction
        """
        self.__destruction = destruction

    def get_destruction(self):
        """Returns the job destruction value

        Returns
        -------
        The job destruction value
        """
        return self.__destruction

    def set_locationid(self, locationid):
        """Sets the job location identifier

        Parameters
        ----------
        locationid : str, mandatory
            job location identifier
        """
        self.__locationId = locationid

    def get_locationid(self):
        """Returns the job location identifier

        Returns
        -------
        The job location identifier
        """
        return self.__locationId

    def set_name(self, name):
        """Sets the job name

        Parameters
        ----------
        name : str, mandatory
            job name
        """
        self.__name = name

    def get_name(self):
        """Returns the job name

        Returns
        -------
        The job name
        """
        return self.__name

    def set_quote(self, quote):
        """Sets the job quote

        Parameters
        ----------
        quote : int, mandatory
            job quote
        """
        self.__quote = quote

    def get_quote(self):
        """Returns the job quote

        Returns
        -------
        The job quote
        """
        return self.__quote

    def set_parameter(self, key, value):
        """Sets a job parameter

        Parameters
        ----------
        key : str, mandatory
            job parameter key
        value : str, mandatory
            job parameter value
        """
        self.__parameters[key] = value

    def get_parameter(self, key):
        """Returns a job parameter

        Parameters
        ----------
        key : str, mandatory
            job parameter key

        Returns
        -------
        The job parameter value
        """
        return self.__parameters[key]

    def get_parameters(self):
        """Returns the job parameters

        Returns
        -------
        The job parameters (a list)
        """
        return self.__parameters

    def get_data(self):
        """Returns the job results (Astroquery API specification)
        This method will block if the job is asynchronous and the job has not
        finished yet.

        Returns
        -------
        The job results (astropy.table).
        """
        return self.get_results()

    def get_results(self):
        """Returns the job results
        This method will block if the job is asynchronous and the job has not
        finished yet.

        Returns
        -------
        The job results (astropy.table).
        """
        if self.__results is not None:
            return self.__results
        # try load results from file
        # read_results_table_from_file checks whether the file already exists or not
        outputFormat = self.get_output_format()
        results = modelutils.read_results_table_from_file(self.__outputFile,
                                                          outputFormat)
        if results is not None:
            self.set_results(results)
            return results
        # Try to load from server: only async
        if not self.__async:
            # sync: result is in a file
            return None
        else:
            # async: result is in the server once the job is finished
            self.__load_async_job_results()
            return self.__results

    def set_results(self, results):
        """Sets the job results

        Parameters
        ----------
        results : Table object, mandatory
            job results
        """
        self.__results = results
        self.__resultInMemory = True

    def save_results(self, verbose=False):
        """Saves job results
        If the job is asynchronous, this method will block until the results
        are available.

        Parameters
        ----------
        verbose : bool, optional, default 'False'
            flag to display information about the process
        """
        output = self.get_output_file()
        if self.__resultInMemory:
            self.__results.to_xml(output)
        else:
            if self.is_sync():
                # sync: cannot access server again
                print("No results to save")
            else:
                # Async
                self.wait_for_job_end(verbose)
                response = self.__connHandler.execute_get(
                    "async/"+str(self.__jobid)+"/results/result")
                if verbose:
                    print(response.status, response.reason)
                    print(response.getheaders())
                isError = self.__connHandler.check_launch_response_status(response,
                                                                          verbose,
                                                                          200)
                if isError:
                    print(response.reason)
                    raise Exception(response.reason)
                self.__connHandler.dump_to_file(output, response)

    def wait_for_job_end(self, verbose=False):
        """Waits until a job is finished

        Parameters
        ----------
        verbose : bool, optional, default 'False'
            flag to display information about the process
        """
        currentResponse = None
        responseData = None
        while True:
            responseData = self.get_phase(update=True)
            currentResponse = self.__last_phase_response_status

            lphase = responseData.lower().strip()
            if verbose:
                print("Job " + self.__jobid + " status: " + lphase)
            if "pending" != lphase and "queued" != lphase and "executing" != lphase:
                break
            # PENDING, QUEUED, EXECUTING, COMPLETED, ERROR, ABORTED, UNKNOWN,
            # HELD, SUSPENDED, ARCHIVED:
            time.sleep(0.5)
        return currentResponse, responseData

    def __load_async_job_results(self, debug=False):
        wjResponse, wjData = self.wait_for_job_end()
        subContext = "async/" + str(self.__jobid) + "/results/result"
        resultsResponse = self.__connHandler.execute_get(subContext)
        # resultsResponse = self.__readAsyncResults(self.__jobid, debug)
        if debug:
            print(resultsResponse.status, resultsResponse.reason)
            print(resultsResponse.getheaders())
        isError = self.__connHandler.check_launch_response_status(resultsResponse,
                                                                  debug,
                                                                  200)
        if isError:
            print(resultsResponse.reason)
            raise Exception(resultsResponse.reason)
        else:
            outputFormat = self.get_output_format()
            results = utils.read_http_response(resultsResponse, outputFormat)
            self.set_results(results)
            self.__phase = wjData

    def __str__(self):
        if self.__results is None:
            result = "None"
        else:
            result = self.__results.info()
        return "Jobid: " + str(self.__jobid) + \
            "\nPhase: " + str(self.__phase) + \
            "\nOwner: " + str(self.__ownerid) + \
            "\nOutput file: " + str(self.__outputFile) + \
            "\nResults: " + str(result)
<|endoftext|>


<commit_before># -*- coding: windows-1252 -*-

__VERSION__ = '0.7.2'

import sys
if sys.version_info[:2] < (2, 3):
    print >> sys.stderr, "Sorry, xlwt requires Python 2.3 or later"
    sys.exit(1)

from Workbook import Workbook
from Worksheet import Worksheet
from Row import Row
from Column import Column
from Formatting import Font, Alignment, Borders, Pattern, Protection
from Style import XFStyle, easyxf
from ExcelFormula import *
<commit_msg>xlwt patch for compatibility reasons (may be unnecessary)<commit_after># -*- coding: windows-1252 -*-

__VERSION__ = '0.7.2'

import sys
if sys.version_info[:2] < (2, 3):
    print >> sys.stderr, "Sorry, xlwt requires Python 2.3 or later"
    sys.exit(1)

from Workbook import Workbook
from Worksheet import Worksheet
from Row import Row
from Column import Column
from Formatting import Font, Alignment, Borders, Pattern, Protection
from Style import XFStyle, easyxf
from ExcelFormula import *

import CompoundDoc
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2020 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from os import environ
import uuid

from google.api_core import exceptions
from google.cloud import servicedirectory_v1

import snippets

PROJECT_ID = environ['GOOGLE_CLOUD_PROJECT']
LOCATION_ID = 'us-east1'
NAMESPACE_ID = f'test-namespace-{uuid.uuid4().hex}'
SERVICE_ID = f'test-service-{uuid.uuid4().hex}'
ENDPOINT_ID = f'test-endpoint-{uuid.uuid4().hex}'
ADDRESS = '1.2.3.4'
PORT = 443

def teardown_module():
    client = servicedirectory_v1.RegistrationServiceClient()
    namespace_name = client.namespace_path(PROJECT_ID, LOCATION_ID, NAMESPACE_ID)
    try:
        namespace = client.get_namespace(name=namespace_name)
        client.delete_namespace(name=namespace.name)
    except exceptions.NotFound:
        pass

def test_create_namespace():
    response = snippets.create_namespace(PROJECT_ID, LOCATION_ID, NAMESPACE_ID)

    assert NAMESPACE_ID in response.name

def test_create_service():
    response = snippets.create_service(PROJECT_ID, LOCATION_ID, NAMESPACE_ID,
                                       SERVICE_ID)

    assert SERVICE_ID in response.name

def test_create_endpoint():
    response = snippets.create_endpoint(PROJECT_ID, LOCATION_ID, NAMESPACE_ID,
                                        SERVICE_ID, ENDPOINT_ID, ADDRESS, PORT)

    assert ENDPOINT_ID in response.name

def test_resolve_service():
    response = snippets.resolve_service(PROJECT_ID, LOCATION_ID, NAMESPACE_ID,
                                        SERVICE_ID)

    assert len(response.service.endpoints) == 1
    assert ENDPOINT_ID in response.service.endpoints[0].name

def test_delete_endpoint(capsys):
    snippets.delete_endpoint(PROJECT_ID, LOCATION_ID, NAMESPACE_ID, SERVICE_ID,
                             ENDPOINT_ID)

    out, _ = capsys.readouterr()
    assert ENDPOINT_ID in out

def test_delete_service(capsys):
    snippets.delete_service(PROJECT_ID, LOCATION_ID, NAMESPACE_ID, SERVICE_ID)

    out, _ = capsys.readouterr()
    assert SERVICE_ID in out

def test_delete_namespace(capsys):
    snippets.delete_namespace(PROJECT_ID, LOCATION_ID, NAMESPACE_ID)

    out, _ = capsys.readouterr()
    assert NAMESPACE_ID in out
<commit_msg>samples: fix flaky test teardown in servicedirectory (#7023)<commit_after>#!/usr/bin/env python

# Copyright 2020 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from os import environ
import uuid

from google.api_core import exceptions
from google.cloud import servicedirectory_v1

import quickstart
import snippets

PROJECT_ID = environ['GOOGLE_CLOUD_PROJECT']
LOCATION_ID = 'us-east1'
NAMESPACE_ID = f'test-namespace-{uuid.uuid4().hex}'
SERVICE_ID = f'test-service-{uuid.uuid4().hex}'
ENDPOINT_ID = f'test-endpoint-{uuid.uuid4().hex}'
ADDRESS = '1.2.3.4'
PORT = 443

def teardown_module():
    client = servicedirectory_v1.RegistrationServiceClient()
    namespace_name = client.namespace_path(PROJECT_ID, LOCATION_ID, NAMESPACE_ID)
    all_namespaces = quickstart.list_namespaces(PROJECT_ID, LOCATION_ID).namespaces

    # Delete namespace only if it exists
    for namespace in all_namespaces:
        if namespace_name in namespace.name:
            try:
                client.delete_namespace(name=namespace_name)
            except exceptions.NotFound:
                print("Namespace already deleted")
            break

def test_create_namespace():
    response = snippets.create_namespace(PROJECT_ID, LOCATION_ID, NAMESPACE_ID)

    assert NAMESPACE_ID in response.name

def test_create_service():
    response = snippets.create_service(PROJECT_ID, LOCATION_ID, NAMESPACE_ID,
                                       SERVICE_ID)

    assert SERVICE_ID in response.name

def test_create_endpoint():
    response = snippets.create_endpoint(PROJECT_ID, LOCATION_ID, NAMESPACE_ID,
                                        SERVICE_ID, ENDPOINT_ID, ADDRESS, PORT)

    assert ENDPOINT_ID in response.name

def test_resolve_service():
    response = snippets.resolve_service(PROJECT_ID, LOCATION_ID, NAMESPACE_ID,
                                        SERVICE_ID)

    assert len(response.service.endpoints) == 1
    assert ENDPOINT_ID in response.service.endpoints[0].name

def test_delete_endpoint(capsys):
    snippets.delete_endpoint(PROJECT_ID, LOCATION_ID, NAMESPACE_ID, SERVICE_ID,
                             ENDPOINT_ID)

    out, _ = capsys.readouterr()
    assert ENDPOINT_ID in out

def test_delete_service(capsys):
    snippets.delete_service(PROJECT_ID, LOCATION_ID, NAMESPACE_ID, SERVICE_ID)

    out, _ = capsys.readouterr()
    assert SERVICE_ID in out

def test_delete_namespace(capsys):
    snippets.delete_namespace(PROJECT_ID, LOCATION_ID, NAMESPACE_ID)

    out, _ = capsys.readouterr()
    assert NAMESPACE_ID in out
<|endoftext|>


<commit_before># Copyright Contributors to the Pyro project.
# SPDX-License-Identifier: Apache-2.0

from collections import namedtuple
import csv
import gzip
import os
import struct
from urllib.parse import urlparse
from urllib.request import urlretrieve

import numpy as np

from jax import device_put, lax
from jax.interpreters.xla import DeviceArray

if 'CI' in os.environ:
    DATA_DIR = os.path.expanduser('~/.data')
else:
    DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                            '.data'))
os.makedirs(DATA_DIR, exist_ok=True)

dset = namedtuple('dset', ['name', 'urls'])

BASEBALL = dset('baseball', [
    'https://d2hg8soec8ck9v.cloudfront.net/datasets/EfronMorrisBB.txt',
])

COVTYPE = dset('covtype', [
    'https://d2hg8soec8ck9v.cloudfront.net/datasets/covtype.zip',
])

MNIST = dset('mnist', [
    'https://d2hg8soec8ck9v.cloudfront.net/datasets/mnist/train-images-idx3-ubyte.gz',
    'https://d2hg8soec8ck9v.cloudfront.net/datasets/mnist/train-labels-idx1-ubyte.gz',
    'https://d2hg8soec8ck9v.cloudfront.net/datasets/mnist/t10k-images-idx3-ubyte.gz',
    'https://d2hg8soec8ck9v.cloudfront.net/datasets/mnist/t10k-labels-idx1-ubyte.gz',
])

SP500 = dset('SP500', [
    'https://d2hg8soec8ck9v.cloudfront.net/datasets/SP500.csv',
])

UCBADMIT = dset('ucbadmit', [
    'https://d2hg8soec8ck9v.cloudfront.net/datasets/UCBadmit.csv',
])

LYNXHARE = dset('lynxhare', [
    'http://people.whitman.edu/~hundledr/courses/M250F03/LynxHare.txt',
])

def _download(dset):
    for url in dset.urls:
        file = os.path.basename(urlparse(url).path)
        out_path = os.path.join(DATA_DIR, file)
        if not os.path.exists(out_path):
            print('Downloading - {}.'.format(url))
            urlretrieve(url, out_path)
            print('Download complete.')

def _load_baseball():
    _download(BASEBALL)

    def train_test_split(file):
        train, test, player_names = [], [], []
        with open(file, 'r') as f:
            csv_reader = csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE)
            for row in csv_reader:
                player_names.append(row['FirstName'] + ' ' + row['LastName'])
                at_bats, hits = row['At-Bats'], row['Hits']
                train.append(np.array([int(at_bats), int(hits)]))
                season_at_bats, season_hits = row['SeasonAt-Bats'], row['SeasonHits']
                test.append(np.array([int(season_at_bats), int(season_hits)]))
        return np.stack(train), np.stack(test), np.array(player_names)

    train, test, player_names = train_test_split(os.path.join(DATA_DIR, 'EfronMorrisBB.txt'))
    return {'train': (train, player_names),
            'test': (test, player_names)}

def _load_covtype():
    _download(COVTYPE)

    file_path = os.path.join(DATA_DIR, 'covtype.zip')
    data = np.load(file_path)

    return {
        'train': (data['data'], data['target'])
    }

def _load_mnist():
    _download(MNIST)

    def read_label(file):
        with gzip.open(file, 'rb') as f:
            f.read(8)
            data = np.frombuffer(f.read(), dtype=np.int8) / np.float32(255.)
            return device_put(data)

    def read_img(file):
        with gzip.open(file, 'rb') as f:
            _, _, nrows, ncols = struct.unpack(">IIII", f.read(16))
            data = np.frombuffer(f.read(), dtype=np.uint8) / np.float32(255.)
            return device_put(data.reshape(-1, nrows, ncols))

    files = [os.path.join(DATA_DIR, os.path.basename(urlparse(url).path))
             for url in MNIST.urls]
    return {'train': (read_img(files[0]), read_label(files[1])),
            'test': (read_img(files[2]), read_label(files[3]))}

def _load_sp500():
    _download(SP500)

    date, value = [], []
    with open(os.path.join(DATA_DIR, 'SP500.csv'), 'r') as f:
        csv_reader = csv.DictReader(f, quoting=csv.QUOTE_NONE)
        for row in csv_reader:
            date.append(row['DATE'])
            value.append(float(row['VALUE']))
    date = np.stack(date)
    value = np.stack(value)

    return {'train': (date, value)}

def _load_ucbadmit():
    _download(UCBADMIT)

    dept, male, applications, admit = [], [], [], []
    with open(os.path.join(DATA_DIR, 'UCBadmit.csv')) as f:
        csv_reader = csv.DictReader(
            f,
            delimiter=';',
            fieldnames=['index', 'dept', 'gender', 'admit', 'reject', 'applications']
        )
        next(csv_reader)  # skip the first row
        for row in csv_reader:
            dept.append(ord(row['dept']) - ord('A'))
            male.append(row['gender'] == 'male')
            applications.append(int(row['applications']))
            admit.append(int(row['admit']))

    return {'train': (np.stack(dept), np.stack(male), np.stack(applications), np.stack(admit))}

def _load_lynxhare():
    _download(LYNXHARE)

    file_path = os.path.join(DATA_DIR, 'LynxHare.txt')
    data = np.loadtxt(file_path)

    return {
        'train': (data[:, 0].astype(int), data[:, 1:])
    }

def _load(dset):
    if dset == BASEBALL:
        return _load_baseball()
    elif dset == COVTYPE:
        return _load_covtype()
    elif dset == MNIST:
        return _load_mnist()
    elif dset == SP500:
        return _load_sp500()
    elif dset == UCBADMIT:
        return _load_ucbadmit()
    elif dset == LYNXHARE:
        return _load_lynxhare()
    raise ValueError('Dataset - {} not found.'.format(dset.name))

def iter_dataset(dset, batch_size=None, split='train', shuffle=True):
    arrays = _load(dset)[split]
    num_records = len(arrays[0])
    idxs = np.arange(num_records)
    if not batch_size:
        batch_size = num_records
    if shuffle:
        idxs = np.random.permutation(idxs)
    for i in range(num_records // batch_size):
        start_idx = i * batch_size
        end_idx = min((i + 1) * batch_size, num_records)
        yield tuple(a[idxs[start_idx:end_idx]] for a in arrays)

def load_dataset(dset, batch_size=None, split='train', shuffle=True):
    arrays = _load(dset)[split]
    num_records = len(arrays[0])
    idxs = np.arange(num_records)
    if not batch_size:
        batch_size = num_records

    def init():
        return num_records // batch_size, np.random.permutation(idxs) if shuffle else idxs

    def get_batch(i=0, idxs=idxs):
        ret_idx = lax.dynamic_slice_in_dim(idxs, i * batch_size, batch_size)
        return tuple(lax.index_take(a, (ret_idx,), axes=(0,)) if isinstance(a, DeviceArray)
                     else np.take(a, ret_idx, axis=0) for a in arrays)

    return init, get_batch
<commit_msg>change lynxhare url to cloud (#525)<commit_after># Copyright Contributors to the Pyro project.
# SPDX-License-Identifier: Apache-2.0

from collections import namedtuple
import csv
import gzip
import os
import struct
from urllib.parse import urlparse
from urllib.request import urlretrieve

import numpy as np

from jax import device_put, lax
from jax.interpreters.xla import DeviceArray

if 'CI' in os.environ:
    DATA_DIR = os.path.expanduser('~/.data')
else:
    DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                            '.data'))
os.makedirs(DATA_DIR, exist_ok=True)

dset = namedtuple('dset', ['name', 'urls'])

BASEBALL = dset('baseball', [
    'https://d2hg8soec8ck9v.cloudfront.net/datasets/EfronMorrisBB.txt',
])

COVTYPE = dset('covtype', [
    'https://d2hg8soec8ck9v.cloudfront.net/datasets/covtype.zip',
])

MNIST = dset('mnist', [
    'https://d2hg8soec8ck9v.cloudfront.net/datasets/mnist/train-images-idx3-ubyte.gz',
    'https://d2hg8soec8ck9v.cloudfront.net/datasets/mnist/train-labels-idx1-ubyte.gz',
    'https://d2hg8soec8ck9v.cloudfront.net/datasets/mnist/t10k-images-idx3-ubyte.gz',
    'https://d2hg8soec8ck9v.cloudfront.net/datasets/mnist/t10k-labels-idx1-ubyte.gz',
])

SP500 = dset('SP500', [
    'https://d2hg8soec8ck9v.cloudfront.net/datasets/SP500.csv',
])

UCBADMIT = dset('ucbadmit', [
    'https://d2hg8soec8ck9v.cloudfront.net/datasets/UCBadmit.csv',
])

LYNXHARE = dset('lynxhare', [
    'https://d2hg8soec8ck9v.cloudfront.net/datasets/LynxHare.txt',
])

def _download(dset):
    for url in dset.urls:
        file = os.path.basename(urlparse(url).path)
        out_path = os.path.join(DATA_DIR, file)
        if not os.path.exists(out_path):
            print('Downloading - {}.'.format(url))
            urlretrieve(url, out_path)
            print('Download complete.')

def _load_baseball():
    _download(BASEBALL)

    def train_test_split(file):
        train, test, player_names = [], [], []
        with open(file, 'r') as f:
            csv_reader = csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE)
            for row in csv_reader:
                player_names.append(row['FirstName'] + ' ' + row['LastName'])
                at_bats, hits = row['At-Bats'], row['Hits']
                train.append(np.array([int(at_bats), int(hits)]))
                season_at_bats, season_hits = row['SeasonAt-Bats'], row['SeasonHits']
                test.append(np.array([int(season_at_bats), int(season_hits)]))
        return np.stack(train), np.stack(test), np.array(player_names)

    train, test, player_names = train_test_split(os.path.join(DATA_DIR, 'EfronMorrisBB.txt'))
    return {'train': (train, player_names),
            'test': (test, player_names)}

def _load_covtype():
    _download(COVTYPE)

    file_path = os.path.join(DATA_DIR, 'covtype.zip')
    data = np.load(file_path)

    return {
        'train': (data['data'], data['target'])
    }

def _load_mnist():
    _download(MNIST)

    def read_label(file):
        with gzip.open(file, 'rb') as f:
            f.read(8)
            data = np.frombuffer(f.read(), dtype=np.int8) / np.float32(255.)
            return device_put(data)

    def read_img(file):
        with gzip.open(file, 'rb') as f:
            _, _, nrows, ncols = struct.unpack(">IIII", f.read(16))
            data = np.frombuffer(f.read(), dtype=np.uint8) / np.float32(255.)
            return device_put(data.reshape(-1, nrows, ncols))

    files = [os.path.join(DATA_DIR, os.path.basename(urlparse(url).path))
             for url in MNIST.urls]
    return {'train': (read_img(files[0]), read_label(files[1])),
            'test': (read_img(files[2]), read_label(files[3]))}

def _load_sp500():
    _download(SP500)

    date, value = [], []
    with open(os.path.join(DATA_DIR, 'SP500.csv'), 'r') as f:
        csv_reader = csv.DictReader(f, quoting=csv.QUOTE_NONE)
        for row in csv_reader:
            date.append(row['DATE'])
            value.append(float(row['VALUE']))
    date = np.stack(date)
    value = np.stack(value)

    return {'train': (date, value)}

def _load_ucbadmit():
    _download(UCBADMIT)

    dept, male, applications, admit = [], [], [], []
    with open(os.path.join(DATA_DIR, 'UCBadmit.csv')) as f:
        csv_reader = csv.DictReader(
            f,
            delimiter=';',
            fieldnames=['index', 'dept', 'gender', 'admit', 'reject', 'applications']
        )
        next(csv_reader)  # skip the first row
        for row in csv_reader:
            dept.append(ord(row['dept']) - ord('A'))
            male.append(row['gender'] == 'male')
            applications.append(int(row['applications']))
            admit.append(int(row['admit']))

    return {'train': (np.stack(dept), np.stack(male), np.stack(applications), np.stack(admit))}

def _load_lynxhare():
    _download(LYNXHARE)

    file_path = os.path.join(DATA_DIR, 'LynxHare.txt')
    data = np.loadtxt(file_path)

    return {
        'train': (data[:, 0].astype(int), data[:, 1:])
    }

def _load(dset):
    if dset == BASEBALL:
        return _load_baseball()
    elif dset == COVTYPE:
        return _load_covtype()
    elif dset == MNIST:
        return _load_mnist()
    elif dset == SP500:
        return _load_sp500()
    elif dset == UCBADMIT:
        return _load_ucbadmit()
    elif dset == LYNXHARE:
        return _load_lynxhare()
    raise ValueError('Dataset - {} not found.'.format(dset.name))

def iter_dataset(dset, batch_size=None, split='train', shuffle=True):
    arrays = _load(dset)[split]
    num_records = len(arrays[0])
    idxs = np.arange(num_records)
    if not batch_size:
        batch_size = num_records
    if shuffle:
        idxs = np.random.permutation(idxs)
    for i in range(num_records // batch_size):
        start_idx = i * batch_size
        end_idx = min((i + 1) * batch_size, num_records)
        yield tuple(a[idxs[start_idx:end_idx]] for a in arrays)

def load_dataset(dset, batch_size=None, split='train', shuffle=True):
    arrays = _load(dset)[split]
    num_records = len(arrays[0])
    idxs = np.arange(num_records)
    if not batch_size:
        batch_size = num_records

    def init():
        return num_records // batch_size, np.random.permutation(idxs) if shuffle else idxs

    def get_batch(i=0, idxs=idxs):
        ret_idx = lax.dynamic_slice_in_dim(idxs, i * batch_size, batch_size)
        return tuple(lax.index_take(a, (ret_idx,), axes=(0,)) if isinstance(a, DeviceArray)
                     else np.take(a, ret_idx, axis=0) for a in arrays)

    return init, get_batch
<|endoftext|>


<commit_before>#! /usr/bin/env python
# Last Change: Sun Feb 10 01:00 AM 2008 J

"""This module defines some helper functions, to be used by high level
checkers."""
from copy import deepcopy

# Tools to save and restore environments construction variables (the ones often
# altered for configuration tests).
_arg2env = {'include_dirs' : 'CPPPATH',
            'cflags' : 'CFLAGS',
            'library_dirs' : 'LIBPATH',
            'libraries' : 'LIBS',
            'linkflags' : 'LINKFLAGS',
            'linkflagsend' : 'LINKFLAGSEND',
            'rpath' : 'RPATH',
            'frameworks' : 'FRAMEWORKS'}

def save_and_set(env, opts, keys = None):
    """Put informations from option configuration into a scons environment, and returns the savedkeys given as config opts args."""
    saved_keys = {}
    if keys is None:
        keys = opts.keys()
    for k in keys:
        saved_keys[k] = (env.has_key(_arg2env[k]) and\
                         deepcopy(env[_arg2env[k]])) or\
                        []
    kw = zip([_arg2env[k] for k in keys], [opts[k] for k in keys])
    kw = dict(kw)
    env.AppendUnique(**kw)
    return saved_keys

def restore(env, saved):
    keys = saved.keys()
    kw = zip([_arg2env[k] for k in keys],
             [saved[k] for k in keys])
    kw = dict(kw)
    env.Replace(**kw)

# Implementation function to check symbol in a library
def check_symbol(context, headers, sym, extra = r''):
    # XXX: add dep vars in code
    #code = [r'#include <%s>' %h for h in headers]
    code = []
    code.append(r'''
#undef %(func)s
#ifdef __cplusplus
extern "C"
#endif
char %(func)s();

int main()
{
return %(func)s();
return 0;
}
''' % {'func' : sym})
    code.append(extra)
    return context.TryLink('\n'.join(code), '.c')

def _check_headers(context, opts, headers, autoadd):
    """Try to compile code including the given headers."""
    env = context.env

    #----------------------------
    # Check headers are available
    #----------------------------
    saved = save_and_set(context.env, opts)
    try:
        # XXX: handle context
        hcode = ['#include <%s>' % h for h in headers]

        # HACK: we add cpppath in the command of the source, to add dependency of
        # the check on the cpppath.
        hcode.extend(['#if 0', str(opts), '#endif\n'])
        src = '\n'.join(hcode)
        ret = context.TryCompile(src, '.c')
    finally:
        if ret == 0 or autoadd == 0:
            restore(context.env, saved)

    return ret

def check_include_and_run(context, name, opts, headers, run_src, autoadd = 1):
    """This is a basic implementation for generic "test include and run"
    testers.

    For example, for library foo, which implements function do_foo, and with
    include header foo.h, this will:
        - test that foo.h is found and compilable by the compiler
        - test that the given source code can be compiled. The source code
          should contain a simple program with the function.

    XXX: this doc is not up-to-date
    Arguments:
        - name: name of the library
        - include_dirs: list of directories
        - headers: list of headers
        - run_src: the code for the run test
        - libs: list of libraries to link
        - library_dirs: list of library path.
        - linkflags: list of link flags to add."""

    context.Message('Checking for %s ... ' % name)
    env = context.env

    ret = _check_headers(context, opts, headers, autoadd)
    if not ret:
        context.Result('Failed: %s include not found' % name)
        return 0

    #------------------------------
    # Check a simple example works
    #------------------------------
    saved = save_and_set(env, opts)
    try:
        # HACK: we add libpath and libs at the end of the source as a comment,
        # to add dependency of the check on those.
        src = '\n'.join([r'#include <%s>' % h for h in headers] +\
                        [run_src, r'#if  0', r'%s' % str(opts), r'#endif',
                         '\n'])
        ret, out = context.TryRun(src, '.c')
    finally:
        if (not ret or not autoadd):
            # If test failed or autoadd is disabled, restore everything
            restore(env, saved)

    if not ret:
        context.Result('Failed: %s test could not be linked and run' % name)
        return 0

    context.Result(ret)
    return ret

def check_run_f77(context, name, opts, run_src, autoadd = 1):
    """This is a basic implementation for generic "run" testers.

    For example, for library foo, which implements function do_foo
        - test that the given source code can be compiled. The source code
          should contain a simple program with the function.

    Arguments:
        - name: name of the library."""

    context.Message('Checking for %s ... ' % name)
    env = context.env

    #------------------------------
    # Check a simple example works
    #------------------------------
    saved = save_and_set(env, opts)
    try:
        # HACK: we add libpath and libs at the end of the source as a comment,
        # to add dependency of the check on those.
        src = '\n'.join([run_src] + [r'* %s' % s for s in
                                     str(opts).split('\n')])
        ret, out = context.TryRun(src, '.f')
    finally:
        if (not ret or not autoadd):
            # If test failed or autoadd is disabled, restore everything
            restore(env, saved)

    if not ret:
        context.Result('Failed: %s test could not be linked and run' % name)
        return 0

    context.Result(ret)
    return ret

<commit_msg>Only save/restore options relevant for header compilation in check_header function<commit_after>#! /usr/bin/env python
# Last Change: Sun Feb 10 01:00 AM 2008 J

"""This module defines some helper functions, to be used by high level
checkers."""
from copy import deepcopy

# Tools to save and restore environments construction variables (the ones often
# altered for configuration tests).
_arg2env = {'include_dirs' : 'CPPPATH',
            'cflags' : 'CFLAGS',
            'library_dirs' : 'LIBPATH',
            'libraries' : 'LIBS',
            'linkflags' : 'LINKFLAGS',
            'linkflagsend' : 'LINKFLAGSEND',
            'rpath' : 'RPATH',
            'frameworks' : 'FRAMEWORKS'}

def save_and_set(env, opts, keys = None):
    """Put informations from option configuration into a scons environment, and returns the savedkeys given as config opts args."""
    saved_keys = {}
    if keys is None:
        keys = opts.keys()
    for k in keys:
        saved_keys[k] = (env.has_key(_arg2env[k]) and\
                         deepcopy(env[_arg2env[k]])) or\
                        []
    kw = zip([_arg2env[k] for k in keys], [opts[k] for k in keys])
    kw = dict(kw)
    env.AppendUnique(**kw)
    return saved_keys

def restore(env, saved):
    keys = saved.keys()
    kw = zip([_arg2env[k] for k in keys],
             [saved[k] for k in keys])
    kw = dict(kw)
    env.Replace(**kw)

# Implementation function to check symbol in a library
def check_symbol(context, headers, sym, extra = r''):
    # XXX: add dep vars in code
    #code = [r'#include <%s>' %h for h in headers]
    code = []
    code.append(r'''
#undef %(func)s
#ifdef __cplusplus
extern "C"
#endif
char %(func)s();

int main()
{
return %(func)s();
return 0;
}
''' % {'func' : sym})
    code.append(extra)
    return context.TryLink('\n'.join(code), '.c')

def _check_headers(context, opts, headers, autoadd):
    """Try to compile code including the given headers."""
    env = context.env

    #----------------------------
    # Check headers are available
    #----------------------------
    keys = ('include_dirs', 'cflags')
    saved = save_and_set(context.env, opts, keys)
    try:
        # XXX: handle context
        hcode = ['#include <%s>' % h for h in headers]

        # HACK: we add cpppath in the command of the source, to add dependency of
        # the check on the cpppath.
        hcode.extend(['#if 0', str(opts), '#endif\n'])
        src = '\n'.join(hcode)
        ret = context.TryCompile(src, '.c')
    finally:
        if ret == 0 or autoadd == 0:
            restore(context.env, saved)

    return ret

def check_include_and_run(context, name, opts, headers, run_src, autoadd = 1):
    """This is a basic implementation for generic "test include and run"
    testers.

    For example, for library foo, which implements function do_foo, and with
    include header foo.h, this will:
        - test that foo.h is found and compilable by the compiler
        - test that the given source code can be compiled. The source code
          should contain a simple program with the function.

    XXX: this doc is not up-to-date
    Arguments:
        - name: name of the library
        - include_dirs: list of directories
        - headers: list of headers
        - run_src: the code for the run test
        - libs: list of libraries to link
        - library_dirs: list of library path.
        - linkflags: list of link flags to add."""

    context.Message('Checking for %s ... ' % name)
    env = context.env

    ret = _check_headers(context, opts, headers, autoadd)
    if not ret:
        context.Result('Failed: %s include not found' % name)
        return 0

    #------------------------------
    # Check a simple example works
    #------------------------------
    keys = ('library_dirs', 'cflags')
    saved = save_and_set(env, opts)
    try:
        # HACK: we add libpath and libs at the end of the source as a comment,
        # to add dependency of the check on those.
        src = '\n'.join([r'#include <%s>' % h for h in headers] +\
                        [run_src, r'#if  0', r'%s' % str(opts), r'#endif',
                         '\n'])
        ret, out = context.TryRun(src, '.c')
    finally:
        if (not ret or not autoadd):
            # If test failed or autoadd is disabled, restore everything
            restore(env, saved)

    if not ret:
        context.Result('Failed: %s test could not be linked and run' % name)
        return 0

    context.Result(ret)
    return ret

def check_run_f77(context, name, opts, run_src, autoadd = 1):
    """This is a basic implementation for generic "run" testers.

    For example, for library foo, which implements function do_foo
        - test that the given source code can be compiled. The source code
          should contain a simple program with the function.

    Arguments:
        - name: name of the library."""

    context.Message('Checking for %s ... ' % name)
    env = context.env

    #------------------------------
    # Check a simple example works
    #------------------------------
    saved = save_and_set(env, opts)
    try:
        # HACK: we add libpath and libs at the end of the source as a comment,
        # to add dependency of the check on those.
        src = '\n'.join([run_src] + [r'* %s' % s for s in
                                     str(opts).split('\n')])
        ret, out = context.TryRun(src, '.f')
    finally:
        if (not ret or not autoadd):
            # If test failed or autoadd is disabled, restore everything
            restore(env, saved)

    if not ret:
        context.Result('Failed: %s test could not be linked and run' % name)
        return 0

    context.Result(ret)
    return ret

<|endoftext|>


<commit_before>"""unit tests"""

# FIXME : add logic to run numpy tests with our newly created functions?

import numpy as np
import pytest
import numpy.testing
from numpy_indexed import *
from numpy_indexed.utility import as_struct_array

def test_group_by():

    keys   = ["e", "b", "b", "c", "d", "e", "e", 'a']
    values = [1.2, 4.5, 4.3, 2.0, 5.67, 8.08, 9.01,1]

    print('two methods of splitting by group')
    print('as iterable')
    g = group_by(keys)
    for k,v in zip(g.unique, g.split_sequence_as_iterable(values)):
        print(k, list(v))
    print('as list')
    for k,v in zip(*group_by(keys)(values)):
        print(k, v)

    print('some reducing group operations')
    g = group_by(keys)
    unique_keys, reduced_values = g.median(values)
    print('per group median')
    print(reduced_values)
    unique_keys, reduced_values = g.mean(values)
    print('per group mean')
    print(reduced_values)
    unique_keys, reduced_values = g.std(values)
    print('per group std')
    print(reduced_values)
    reduced_values = g.reduce(np.array(values), np.minimum) #alternate way of calling
    print('per group min')
    print(reduced_values)
    unique_keys, reduced_values = g.max(values)
    print('per group max')
    print(reduced_values)

    print('per group sum using custom reduction')
    print(group_by(keys, values, lambda x:x.sum()))

def test_lex_median():
    """for making sure i squased all bugs related to fancy-keys and median filter implementation"""
    keys1  = ["e", "b", "b", "c", "d", "e", "e", 'a']
    keys2  = ["b", "b", "b", "d", "e", "e", 'e', 'e']
    values = [1.2, 4.5, 4.3, 2.0, 5.6, 8.8, 9.1, 1]

    unique, median = group_by((keys1, keys2)).median(values)
    for i in zip(zip(*unique), median):
        print(i)

def test_dict():
    input = [
    {'dept': '001', 'sku': 'foo', 'transId': 'uniqueId1', 'qty': 100},
    {'dept': '001', 'sku': 'bar', 'transId': 'uniqueId2', 'qty': 200},
    {'dept': '001', 'sku': 'foo', 'transId': 'uniqueId3', 'qty': 300},
    {'dept': '002', 'sku': 'baz', 'transId': 'uniqueId4', 'qty': 400},
    {'dept': '002', 'sku': 'baz', 'transId': 'uniqueId5', 'qty': 500},
    {'dept': '002', 'sku': 'qux', 'transId': 'uniqueId6', 'qty': 600},
    {'dept': '003', 'sku': 'foo', 'transId': 'uniqueId7', 'qty': 700}
    ]

    inputs = dict((k, [i[k] for i in input ]) for k in input[0].keys())
    print(group_by((inputs['dept'], inputs['sku'])).mean(inputs['qty']))

def test_fancy_keys():
    """test Index subclasses"""
    keys        = np.random.randint(0, 2, (20, 3)).astype(np.int8)
    values      = np.random.randint(-1, 2, (20, 4))

    #all these various datastructures should produce the same behavior
    #multiplicity is a nice unit test, since it draws on most of the low level functionality
    if semantics.backwards_compatible:
        assert(np.all(
            multiplicity(keys, axis=0) ==           #void object indexing
            multiplicity(tuple(keys.T))))           #lexographic indexing
        assert(np.all(
            multiplicity(keys, axis=0) ==           #void object indexing
            multiplicity(as_struct_array(keys))))   #struct array indexing
    else:
        assert(np.all(
            multiplicity(keys) ==                   #void object indexing
            multiplicity(tuple(keys.T))))           #lexographic indexing
        assert(np.all(
            multiplicity(keys) ==                   #void object indexing
            multiplicity(as_struct_array(keys))))   #struct array indexing

    #lets go mixing some dtypes!
    floatkeys   = np.zeros(len(keys))
    floatkeys[0] = 8.8
    print('sum per group of identical rows using struct key')
    g = group_by(as_struct_array(keys, floatkeys))
    for e in zip(g.count, *g.sum(values)):
        print(e)
    print('sum per group of identical rows using lex of nd-key')
    g = group_by(( keys, floatkeys))
    for e in zip(zip(*g.unique), g.sum(values)[1]):
        print(e)
    print('sum per group of identical rows using lex of struct key')
    g = group_by((as_struct_array( keys), floatkeys))
    for e in zip(zip(*g.unique), g.sum(values)[1]):
        print(e)

    #showcase enhanced unique functionality
    images = np.random.rand(4,4,4)
    #shuffle the images; this is a giant mess now; how to find the unique ones?
    shuffled = images[np.random.randint(0,4,200)]
    #there you go
    if semantics.backwards_compatible:
        print(unique(shuffled, axis=0))
    else:
        print(unique(shuffled))

def test_compact():
    """demonstrate the most functionality in the least number of lines"""
    key1 = list('abaabb')
    key2 = np.random.randint(0,2,(6,2))
    values = np.random.rand(6,3)
    (unique1, unique2), median = group_by((key1, key2)).median(values)
    print(unique1)
    print(unique2)
    print(median)

def test_timings():
    """small performance test"""
    idx = np.random.randint(0, 1000, 10000)
    g = group_by(idx)
    values = np.random.rand(10000, 100)
    assert (np.allclose(g.at(values), g.reduce(values)))

    from time import clock
    t = clock()
    for i in range(100):
        g.reduce(values)
    print('reducing with np.add over 1000 groups with 10000x100 values repeated 100 times took %f seconds' % (clock()-t))

def test_indices():
    """test indices function"""
    values = np.random.rand(20)
    idx = [1, 2, 5, 7]

    assert(np.alltrue(indices(values, values[idx]) == idx))
    assert(np.alltrue(contains(values, values[idx])))
    with pytest.raises(KeyError):
        indices(values, [-1], assume_contained=False)

def test_indices_object():
    """test indices function with objectindex"""
    A = np.array(
        [[0, 1],
         [0, 2],
         [1, 1],
         [0, 2]])
    B = np.array([[0, 2]])
    assert np.alltrue(indices(A, B) == 1)

def test_setops_edgecase():
    """test some edge cases like zero-length, etc"""
    assert np.array_equal(intersection([1], [1]), [1])
    assert np.array_equal(intersection([], []), [])

    assert np.array_equal(count_selected([], []), [])
    # test repeating values in both arguments
    assert np.array_equal(count_selected([1, 2, 3, 1], [1, 1, 2]), [2, 2, 1])

    assert np.array_equal(difference([1], []), [1])
    assert difference([1], []).dtype == np.int

def test_setops():
    """test generalized classic set operations"""
    # edges exclusive to one of three sets
    edges = np.random.randint(0, 9, size=(3, 100, 2))
    print(exclusive(*edges))

    # difference on object keys
    edges = np.arange(20).reshape(10, 2)
    assert(np.all(difference(edges[:8], edges[-8:]) == edges[:2]))

    # unique on lex keys
    key1 = list('abc')*10
    key2 = np.random.randint(0,9,30)
    print(unique( (key1, key2)))

def test_count_table():
    k = list('aababaababbbaabba')
    i = np.random.randint(0, 10, len(k))
    l, t = count_table(k, i)
    print(l)
    print(t)

    l, t = count_table(*np.random.randint(0,4,(3,1000)))
    print(l)
    print(t)

    l, t = count_table(np.random.randint(0,4,(1000,3)))
    print(l)
    print(t)
<commit_msg>minor<commit_after>"""unit tests"""

# FIXME : add logic to run numpy tests with our newly created functions?

import numpy as np
import pytest
import numpy.testing
from numpy_indexed import *
from numpy_indexed.utility import as_struct_array

def test_group_by():

    keys   = ["e", "b", "b", "c", "d", "e", "e", 'a']
    values = [1.2, 4.5, 4.3, 2.0, 5.67, 8.08, 9.01,1]

    print('two methods of splitting by group')
    print('as iterable')
    g = group_by(keys)
    for k,v in zip(g.unique, g.split_sequence_as_iterable(values)):
        print(k, list(v))
    print('as list')
    for k,v in zip(*group_by(keys)(values)):
        print(k, v)

    print('some reducing group operations')
    g = group_by(keys)
    unique_keys, reduced_values = g.median(values)
    print('per group median')
    print(reduced_values)
    unique_keys, reduced_values = g.mean(values)
    print('per group mean')
    print(reduced_values)
    unique_keys, reduced_values = g.std(values)
    print('per group std')
    print(reduced_values)
    reduced_values = g.reduce(np.array(values), np.minimum) #alternate way of calling
    print('per group min')
    print(reduced_values)
    unique_keys, reduced_values = g.max(values)
    print('per group max')
    print(reduced_values)

    print('per group sum using custom reduction')
    print(group_by(keys, values, lambda x:x.sum()))

def test_lex_median():
    """for making sure i squased all bugs related to fancy-keys and median filter implementation"""
    keys1  = ["e", "b", "b", "c", "d", "e", "e", 'a']
    keys2  = ["b", "b", "b", "d", "e", "e", 'e', 'e']
    values = [1.2, 4.5, 4.3, 2.0, 5.6, 8.8, 9.1, 1]

    unique, median = group_by((keys1, keys2)).median(values)
    for i in zip(zip(*unique), median):
        print(i)

def test_dict():
    input = [
    {'dept': '001', 'sku': 'foo', 'transId': 'uniqueId1', 'qty': 100},
    {'dept': '001', 'sku': 'bar', 'transId': 'uniqueId2', 'qty': 200},
    {'dept': '001', 'sku': 'foo', 'transId': 'uniqueId3', 'qty': 300},
    {'dept': '002', 'sku': 'baz', 'transId': 'uniqueId4', 'qty': 400},
    {'dept': '002', 'sku': 'baz', 'transId': 'uniqueId5', 'qty': 500},
    {'dept': '002', 'sku': 'qux', 'transId': 'uniqueId6', 'qty': 600},
    {'dept': '003', 'sku': 'foo', 'transId': 'uniqueId7', 'qty': 700}
    ]

    inputs = dict((k, [i[k] for i in input ]) for k in input[0].keys())
    print(group_by((inputs['dept'], inputs['sku'])).mean(inputs['qty']))

def test_fancy_keys():
    """test Index subclasses"""
    keys        = np.random.randint(0, 2, (20, 3)).astype(np.int8)
    values      = np.random.randint(-1, 2, (20, 4))

    #all these various datastructures should produce the same behavior
    #multiplicity is a nice unit test, since it draws on most of the low level functionality
    if semantics.backwards_compatible:
        assert(np.all(
            multiplicity(keys, axis=0) ==           #void object indexing
            multiplicity(tuple(keys.T))))           #lexographic indexing
        assert(np.all(
            multiplicity(keys, axis=0) ==           #void object indexing
            multiplicity(as_struct_array(keys))))   #struct array indexing
    else:
        assert(np.all(
            multiplicity(keys) ==                   #void object indexing
            multiplicity(tuple(keys.T))))           #lexographic indexing
        assert(np.all(
            multiplicity(keys) ==                   #void object indexing
            multiplicity(as_struct_array(keys))))   #struct array indexing

    #lets go mixing some dtypes!
    floatkeys   = np.zeros(len(keys))
    floatkeys[0] = 8.8
    print('sum per group of identical rows using struct key')
    g = group_by(as_struct_array(keys, floatkeys))
    for e in zip(g.count, *g.sum(values)):
        print(e)
    print('sum per group of identical rows using lex of nd-key')
    g = group_by(( keys, floatkeys))
    for e in zip(zip(*g.unique), g.sum(values)[1]):
        print(e)
    print('sum per group of identical rows using lex of struct key')
    g = group_by((as_struct_array( keys), floatkeys))
    for e in zip(zip(*g.unique), g.sum(values)[1]):
        print(e)

    #showcase enhanced unique functionality
    images = np.random.rand(4,4,4)
    #shuffle the images; this is a giant mess now; how to find the unique ones?
    shuffled = images[np.random.randint(0,4,200)]
    #there you go
    if semantics.backwards_compatible:
        print(unique(shuffled, axis=0))
    else:
        print(unique(shuffled))

def test_compact():
    """demonstrate the most functionality in the least number of lines"""
    key1 = list('abaabb')
    key2 = np.random.randint(0,2,(6,2))
    values = np.random.rand(6,3)
    (unique1, unique2), median = group_by((key1, key2)).median(values)
    print(unique1)
    print(unique2)
    print(median)

def test_timings():
    """small performance test"""
    idx = np.random.randint(0, 1000, 10000)
    g = group_by(idx)
    values = np.random.rand(10000, 100)
    assert (np.allclose(g.at(values), g.reduce(values)))

    from time import clock
    t = clock()
    for i in range(100):
        g.reduce(values)
    print('reducing with np.add over 1000 groups with 10000x100 values repeated 100 times took %f seconds' % (clock()-t))

def test_indices():
    """test indices function"""
    values = np.random.rand(20)
    idx = [1, 2, 5, 7]

    assert(np.alltrue(indices(values, values[idx]) == idx))
    assert(np.alltrue(contains(values, values[idx])))
    with pytest.raises(KeyError):
        indices(values, [-1], assume_contained=False)

def test_indices_object():
    """test indices function with objectindex"""
    A = np.array(
        [[0, 1],
         [0, 2],
         [1, 1],
         [0, 2]])
    B = np.array([[0, 2]])
    assert np.alltrue(indices(A, B) == 1)

def test_setops_edgecase():
    """test some edge cases like zero-length, etc"""
    assert np.array_equal(intersection([1], [1]), [1])
    assert np.array_equal(intersection([], []), [])

    assert np.array_equal(count_selected([], []), [])
    # test repeating values in both arguments
    assert np.array_equal(count_selected([1, 2, 3, 1], [1, 1, 2]), [2, 2, 1])

    assert np.array_equal(difference([1], []), [1])
    assert difference([1], []).dtype == np.int

    assert np.array_equal(union([], [], []), [])
    assert np.array_equal(exclusive([], []), [])

def test_setops():
    """test generalized classic set operations"""
    # edges exclusive to one of three sets
    edges = np.random.randint(0, 9, size=(3, 100, 2))
    print(exclusive(*edges))

    # difference on object keys
    edges = np.arange(20).reshape(10, 2)
    assert(np.all(difference(edges[:8], edges[-8:]) == edges[:2]))

    # unique on lex keys
    key1 = list('abc')*10
    key2 = np.random.randint(0,9,30)
    print(unique( (key1, key2)))

def test_count_table():
    k = list('aababaababbbaabba')
    i = np.random.randint(0, 10, len(k))
    l, t = count_table(k, i)
    print(l)
    print(t)

    l, t = count_table(*np.random.randint(0,4,(3,1000)))
    print(l)
    print(t)

    l, t = count_table(np.random.randint(0,4,(1000,3)))
    print(l)
    print(t)
<|endoftext|>


<commit_before>import logging
from copy import deepcopy

from motor.motor_asyncio import AsyncIOMotorClient
from pymongo.errors import ServerSelectionTimeoutError

from .triggers import TriggerCollection
from .data_processing import DataProcessingCollection
from .metadata import MetadataCollection
from .workflow_templates import WorkflowTemplatesCollection, TemplateState
from .task_templates import TaskTemplatesCollection
from .workflow_instances import WorkflowInstancesCollection
from .task_instances import TaskInstancesCollection

log = logging.getLogger(__name__)

class MongoStorage:

    def __init__(self):
        self._client = None
        self._db = None
        self._validate_on_start = False

        # Collections
        self._workflow_templates = None
        self._task_templates = None
        self._workflow_metadata = None
        self._workflow_instances = None
        self._task_instances = None
        self.regexes = None
        self.lookups = None
        self.triggers = None

    def configure(self, host, database, validate_on_start=True, **kwargs):
        log.info(
            "Setting up mongo storage with host '%s' and database '%s'",
            host, database,
        )
        self._client = AsyncIOMotorClient(host, **kwargs)
        self._db = self._client[database]
        self._validate_on_start = validate_on_start

        # Collections
        self._workflow_templates = WorkflowTemplatesCollection(self._db)
        self._task_templates = TaskTemplatesCollection(self._db)
        self._workflow_metadata = MetadataCollection(self._db)
        self._workflow_instances = WorkflowInstancesCollection(self._db)
        self._task_instances = TaskInstancesCollection(self._db)
        self.regexes = DataProcessingCollection(self._db, 'regexes')
        self.lookups = DataProcessingCollection(self._db, 'lookups')
        self.triggers = TriggerCollection(self._db)

    async def index(self):
        """
        Try to connect to mongo and index all the collections.
        """
        log.info('Trying to connect to Mongo...')
        while True:
            try:
                await self._workflow_templates.index()
                await self._task_templates.index()
                await self._workflow_metadata.index()
                await self._workflow_instances.index()
                await self._task_instances.index()
                await self.regexes.index()
                await self.lookups.index()
                await self.triggers.index()
            except ServerSelectionTimeoutError as exc:
                log.error('Could not connect to Mongo - %s', exc)
            else:
                log.info('Successfully connected to Mongo')
                break

        if self._validate_on_start is True:
            collections = await self._db.list_collection_names()
            log.info('Validating %s collections', len(collections))
            for collection in collections:
                await self._db.validate_collection(collection)
                log.info('Validated collection %s', collection)

    # Templates

    async def update_workflow_metadata(self, tid, metadata):
        """
        Update and return
        """
        return await self._workflow_metadata.update(tid, metadata)

    async def upsert_draft(self, template):
        """
        Update a template's draft and all its associated tasks.
        """
        metadata = await self._workflow_metadata.get_one(template['id'])
        if not metadata:
            metadata = {
                'workflow_template_id': template['id'],
                'title': template.pop('title'),
                'tags': template.pop('tags', []),
            }
            await self._workflow_metadata.insert(metadata)

        # Force set of values 'version' and 'draft'.
        template['version'] = await self._workflow_templates.get_last_version(
            template['id']
        ) + 1
        template['state'] = TemplateState.DRAFT.value

        # Split and insert tasks.
        tasks = template.pop('tasks')
        await self._task_templates.insert_many(deepcopy(tasks), template)

        # Insert template without tasks.
        await self._workflow_templates.insert_draft(template)
        template['tasks'] = tasks
        template.update({'title': metadata['title'], 'tags': metadata['tags']})
        return template

    async def publish_draft(self, template_id):
        """
        Publish a draft into an 'active' state, and archive the old active.
        """
        await self._workflow_templates.publish_draft(template_id)
        log.info('Draft for template %s published', template_id[:8])

    async def get_for_topic(self, topic):
        """
        Return all the templates listening on a particular topic.
        This does not append the metadata.
        """
        templates = await self._workflow_templates.get_for_topic(topic)
        if templates:
            log.info(
                'Fetched %s templates for event from "%s"',
                len(templates), topic,
            )
        for template in templates:
            template['tasks'] = await self._task_templates.get(
                template['id'], template['version']
            )
        return templates

    async def get_templates(self, template_id=None, full=False):
        """
        Return all active/draft templates
        Limited to a small set of fields if 'full' is False.
        TODO: Pagination.
        """
        templates = await self._workflow_templates.get(template_id, full)
        for template in templates:
            metadata = await self._workflow_metadata.get_one(template['id'])
            template.update(metadata)
            if full is True:
                template['tasks'] = await self._task_templates.get(
                    template['id'], template['version']
                )
        return templates

    async def get_template(self, tid, draft=False, version=None):
        """
        Return the active template.
        """
        template = await self._workflow_templates.get_one(
            tid,
            draft=draft,
            version=int(version) if version else None,
        )
        if not template:
            return

        metadata = await self._workflow_metadata.get_one(tid)
        template.update(metadata)
        template['tasks'] = await self._task_templates.get(
            template['id'], template['version']
        )
        return template

    async def delete_template(self, tid, draft=False):
        """
        Delete a whole template or only its draft.
        """
        await self._workflow_templates.delete(tid, draft)
        if draft is False:
            await self._task_templates.delete_many(tid)
            await self._workflow_metadata.delete(tid)
            await self.triggers.delete(tid)

    # Instances

    async def insert_instance(self, instance):
        """
        Insert a static workflow instance and all its tasks.
        """
        task_instances = []
        for task in instance['template'].pop('tasks'):
            task['workflow_instance_id'] = instance['id']
            task_instances.append(task)
        await self._task_instances.insert_many(task_instances)
        await self._workflow_instances.insert(instance)

    # History

    async def get_history(self, **kwargs):
        """
        Return paginated workflow history.
        """
        count, workflows = await self._workflow_instances.get(**kwargs)
        if kwargs.get('full') is True:
            for workflow in workflows:
                workflow['template']['tasks'] = await self._task_instances.get(
                    workflow['id'], True
                )
        return count, workflows

    async def get_instance(self, instance_id, full=False):
        workflow = await self._workflow_instances.get_one(instance_id, full)
        if not workflow:
            return
        workflow['template']['tasks'] = await self._task_instances.get(
            workflow['id'], full
        )
        return workflow

    async def get_instance_task(self, task_id, full=False):
        return await self._task_instances.get_one(task_id, full)

    async def get_instance_task_data(self, task_id):
        return await self._task_instances.get_data(task_id)
<commit_msg>Put tenant_id in database name for new installations<commit_after>import logging
import os
from copy import deepcopy

from motor.motor_asyncio import AsyncIOMotorClient
from pymongo.errors import ServerSelectionTimeoutError

from .triggers import TriggerCollection
from .data_processing import DataProcessingCollection
from .metadata import MetadataCollection
from .workflow_templates import WorkflowTemplatesCollection, TemplateState
from .task_templates import TaskTemplatesCollection
from .workflow_instances import WorkflowInstancesCollection
from .task_instances import TaskInstancesCollection

log = logging.getLogger(__name__)

class MongoStorage:

    def __init__(self):
        self._client = None
        self._db = None
        self._validate_on_start = False

        # Collections
        self._workflow_templates = None
        self._task_templates = None
        self._workflow_metadata = None
        self._workflow_instances = None
        self._task_instances = None
        self.regexes = None
        self.lookups = None
        self.triggers = None

    def configure(self, host, database, validate_on_start=True, **kwargs):
        log.info("Setting up mongo storage with host '%s'", host)
        self._client = AsyncIOMotorClient(host, **kwargs)
        self._validate_on_start = validate_on_start
        self._db_name = database

    async def index(self):
        """
        Try to connect to mongo and index all the collections.
        """
        if self._db_name not in await self._client.list_database_names():
            # If the old mongo DB does not exist, use the new tenant format
            self._db_name = f"{os.environ.get('TENANT_ID', 'dev')}#{self._db_name}"
        log.info("Selected database '%s'", self._db_name)

        self._db = self._client[self._db_name]
        # Collections
        self._workflow_templates = WorkflowTemplatesCollection(self._db)
        self._task_templates = TaskTemplatesCollection(self._db)
        self._workflow_metadata = MetadataCollection(self._db)
        self._workflow_instances = WorkflowInstancesCollection(self._db)
        self._task_instances = TaskInstancesCollection(self._db)
        self.regexes = DataProcessingCollection(self._db, 'regexes')
        self.lookups = DataProcessingCollection(self._db, 'lookups')
        self.triggers = TriggerCollection(self._db)

        log.info('Trying to connect to Mongo...')
        while True:
            try:
                await self._workflow_templates.index()
                await self._task_templates.index()
                await self._workflow_metadata.index()
                await self._workflow_instances.index()
                await self._task_instances.index()
                await self.regexes.index()
                await self.lookups.index()
                await self.triggers.index()
            except ServerSelectionTimeoutError as exc:
                log.error('Could not connect to Mongo - %s', exc)
            else:
                log.info('Successfully connected to Mongo')
                break

        if self._validate_on_start is True:
            collections = await self._db.list_collection_names()
            log.info('Validating %s collections', len(collections))
            for collection in collections:
                await self._db.validate_collection(collection)
                log.info('Validated collection %s', collection)

    # Templates

    async def update_workflow_metadata(self, tid, metadata):
        """
        Update and return
        """
        return await self._workflow_metadata.update(tid, metadata)

    async def upsert_draft(self, template):
        """
        Update a template's draft and all its associated tasks.
        """
        metadata = await self._workflow_metadata.get_one(template['id'])
        if not metadata:
            metadata = {
                'workflow_template_id': template['id'],
                'title': template.pop('title'),
                'tags': template.pop('tags', []),
            }
            await self._workflow_metadata.insert(metadata)

        # Force set of values 'version' and 'draft'.
        template['version'] = await self._workflow_templates.get_last_version(
            template['id']
        ) + 1
        template['state'] = TemplateState.DRAFT.value

        # Split and insert tasks.
        tasks = template.pop('tasks')
        await self._task_templates.insert_many(deepcopy(tasks), template)

        # Insert template without tasks.
        await self._workflow_templates.insert_draft(template)
        template['tasks'] = tasks
        template.update({'title': metadata['title'], 'tags': metadata['tags']})
        return template

    async def publish_draft(self, template_id):
        """
        Publish a draft into an 'active' state, and archive the old active.
        """
        await self._workflow_templates.publish_draft(template_id)
        log.info('Draft for template %s published', template_id[:8])

    async def get_for_topic(self, topic):
        """
        Return all the templates listening on a particular topic.
        This does not append the metadata.
        """
        templates = await self._workflow_templates.get_for_topic(topic)
        if templates:
            log.info(
                'Fetched %s templates for event from "%s"',
                len(templates), topic,
            )
        for template in templates:
            template['tasks'] = await self._task_templates.get(
                template['id'], template['version']
            )
        return templates

    async def get_templates(self, template_id=None, full=False):
        """
        Return all active/draft templates
        Limited to a small set of fields if 'full' is False.
        TODO: Pagination.
        """
        templates = await self._workflow_templates.get(template_id, full)
        for template in templates:
            metadata = await self._workflow_metadata.get_one(template['id'])
            template.update(metadata)
            if full is True:
                template['tasks'] = await self._task_templates.get(
                    template['id'], template['version']
                )
        return templates

    async def get_template(self, tid, draft=False, version=None):
        """
        Return the active template.
        """
        template = await self._workflow_templates.get_one(
            tid,
            draft=draft,
            version=int(version) if version else None,
        )
        if not template:
            return

        metadata = await self._workflow_metadata.get_one(tid)
        template.update(metadata)
        template['tasks'] = await self._task_templates.get(
            template['id'], template['version']
        )
        return template

    async def delete_template(self, tid, draft=False):
        """
        Delete a whole template or only its draft.
        """
        await self._workflow_templates.delete(tid, draft)
        if draft is False:
            await self._task_templates.delete_many(tid)
            await self._workflow_metadata.delete(tid)
            await self.triggers.delete(tid)

    # Instances

    async def insert_instance(self, instance):
        """
        Insert a static workflow instance and all its tasks.
        """
        task_instances = []
        for task in instance['template'].pop('tasks'):
            task['workflow_instance_id'] = instance['id']
            task_instances.append(task)
        await self._task_instances.insert_many(task_instances)
        await self._workflow_instances.insert(instance)

    # History

    async def get_history(self, **kwargs):
        """
        Return paginated workflow history.
        """
        count, workflows = await self._workflow_instances.get(**kwargs)
        if kwargs.get('full') is True:
            for workflow in workflows:
                workflow['template']['tasks'] = await self._task_instances.get(
                    workflow['id'], True
                )
        return count, workflows

    async def get_instance(self, instance_id, full=False):
        workflow = await self._workflow_instances.get_one(instance_id, full)
        if not workflow:
            return
        workflow['template']['tasks'] = await self._task_instances.get(
            workflow['id'], full
        )
        return workflow

    async def get_instance_task(self, task_id, full=False):
        return await self._task_instances.get_one(task_id, full)

    async def get_instance_task_data(self, task_id):
        return await self._task_instances.get_data(task_id)
<|endoftext|>


<commit_before>import pytest
from rest_framework.reverse import reverse

@pytest.mark.django_db
def test_search(
    api_client, units, streets, services, addresses, administrative_division
):
    # Note for yet unknown reasons postgresql trigram extension is
    # not working under pytest, to overcome all test are made so that
    # trigram search is not used.

    # Search only units and services, no munigeo data in fixtures
    url = reverse("search") + "?q=museo&use_trigram=false&extended_serializer=false"
    response = api_client.get(url)
    assert response.status_code == 200
    results = response.json()["results"]
    assert results[0]["object_type"] == "unit"
    assert results[0]["name"]["fi"] == "Museo"
    assert results[2]["object_type"] == "service"
    assert results[2]["name"]["sv"] == "Museum"

    # Test that unit Impivara is retrived from service Uimahalli
    url = reverse("search") + "?q=uimahalli&use_trigram=false&extended_serializer=false"
    response = api_client.get(url)
    results = response.json()["results"]
    assert results[0]["name"]["fi"] == "Impivaara"
    assert results[1]["name"]["fi"] == "Uimahalli"
    # Test address search
    url = reverse("search") + "?q=kurra&use_trigram=false"
    response = api_client.get(url)
    results = response.json()["results"]
    assert results[0]["name"]["fi"] == "Kurrapolku 1A"
    assert results[0]["object_type"] == "address"
    # Test administrative division search
    url = reverse("search") + "?q=hel&use_trigram=false&extended_serializer=false"
    response = api_client.get(url)
    results = response.json()["results"]
    assert results[0]["object_type"] == "administrativedivision"
    assert results[0]["name"]["fi"] == "Helsinki"
    # Test that addresses are sorted by naturalsort
    url = reverse("search") + "?q=yliopistonkatu"
    response = api_client.get(url)
    results = response.json()["results"]
    assert results[0]["name"]["fi"] == "Yliopistonkatu 5"
    assert results[1]["name"]["fi"] == "Yliopistonkatu 21"
    assert results[2]["name"]["fi"] == "Yliopistonkatu 33"
<commit_msg>Remove external_serializer-argument<commit_after>import pytest
from rest_framework.reverse import reverse

@pytest.mark.django_db
def test_search(
    api_client, units, streets, services, addresses, administrative_division
):
    # Note for yet unknown reasons postgresql trigram extension is
    # not working under pytest, to overcome all test are made so that
    # trigram search is not used.

    # Search only units and services, no munigeo data in fixtures
    url = reverse("search") + "?q=museo&use_trigram=false"
    response = api_client.get(url)
    assert response.status_code == 200
    results = response.json()["results"]
    assert results[0]["object_type"] == "unit"
    assert results[0]["name"]["fi"] == "Museo"
    assert results[2]["object_type"] == "service"
    assert results[2]["name"]["sv"] == "Museum"

    # Test that unit Impivara is retrived from service Uimahalli
    url = reverse("search") + "?q=uimahalli&use_trigram=false"
    response = api_client.get(url)
    results = response.json()["results"]
    assert results[0]["name"]["fi"] == "Impivaara"
    assert results[1]["name"]["fi"] == "Uimahalli"
    # Test address search
    url = reverse("search") + "?q=kurra&use_trigram=false"
    response = api_client.get(url)
    results = response.json()["results"]
    assert results[0]["name"]["fi"] == "Kurrapolku 1A"
    assert results[0]["object_type"] == "address"
    # Test administrative division search
    url = reverse("search") + "?q=hel&use_trigram=false"
    response = api_client.get(url)
    results = response.json()["results"]
    assert results[0]["object_type"] == "administrativedivision"
    assert results[0]["name"]["fi"] == "Helsinki"
    # Test that addresses are sorted by naturalsort
    url = reverse("search") + "?q=yliopistonkatu"
    response = api_client.get(url)
    results = response.json()["results"]
    assert results[0]["name"]["fi"] == "Yliopistonkatu 5"
    assert results[1]["name"]["fi"] == "Yliopistonkatu 21"
    assert results[2]["name"]["fi"] == "Yliopistonkatu 33"
<|endoftext|>