<commit_before># This file is part of the ISIS IBEX application.
# Copyright (C) 2012-2016 Science & Technology Facilities Council.
# All rights reserved.
#
# This program is distributed in the hope that it will be useful.
# This program and the accompanying materials are made available under the
# terms of the Eclipse Public License v1.0 which accompanies this distribution.
# EXCEPT AS EXPRESSLY SET FORTH IN THE ECLIPSE PUBLIC LICENSE V1.0, THE PROGRAM
# AND ACCOMPANYING MATERIALS ARE PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES
# OR CONDITIONS OF ANY KIND.  See the Eclipse Public License v1.0 for more details.
#
# You should have received a copy of the Eclipse Public License v1.0
# along with this program; if not, you can obtain a copy from
# https://www.eclipse.org/org/documents/epl-v10.php or
# http://opensource.org/licenses/eclipse-1.0.php

# Add root path for access to server_commons
import os
import sys
sys.path.insert(0, os.path.abspath(os.environ["MYDIRBLOCK"]))

# Standard imports
from pcaspy import Driver
from time import sleep
import argparse
from server_common.utilities import compress_and_hex, print_and_log, set_logger, convert_to_json, dehex_and_decompress
from server_common.channel_access_server import CAServer
from ioc_data import IOCData
from exp_data import ExpData
import json
from threading import Thread, RLock
from procserv_utils import ProcServWrapper
from options_holder import OptionsHolder
from options_loader import OptionsLoader
from mocks.mock_procserv_utils import MockProcServWrapper

IOCDB = 'iocdb'
IOCS_NOT_TO_STOP = ('INSTETC', 'PSCTRL', 'ISISDAE', 'BLOCKSVR', 'ARINST', 'ARBLOCK', 'GWBLOCK', 'RUNCTRL')

MACROS = {
    "$(MYPVPREFIX)": os.environ['MYPVPREFIX'],
    "$(EPICS_KIT_ROOT)": os.environ['EPICS_KIT_ROOT'],
    "$(ICPCONFIGROOT)": os.environ['ICPCONFIGROOT']
}

PVDB = {
    'IOCS': {
        # Handled by the monitor thread
        'type': 'char',
        'count': 16000,
        'value': [0],
    },
    'PVS:INTEREST:HIGH': {
        # Handled by the monitor thread
        'type': 'char',
        'count': 64000,
        'value': [0],
    },
    'PVS:INTEREST:MEDIUM': {
        # Handled by the monitor thread
        'type': 'char',
        'count': 64000,
        'value': [0],
    },
    'PVS:INTEREST:FACILITY': {
        # Handled by the monitor thread
        'type': 'char',
        'count': 64000,
        'value': [0],
    },
    'PVS:ACTIVE': {
        # Handled by the monitor thread
        'type': 'char',
        'count': 64000,
        'value': [0],
    },
    'PVS:ALL': {
        # Handled by the monitor thread
        'type': 'char',
        'count': 64000,
        'value': [0],
    },
    'SAMPLE_PARS': {
        'type': 'char',
        'count': 10000,
        'value': [0],
    },
    'BEAMLINE_PARS': {
        'type': 'char',
        'count': 10000,
        'value': [0],
    },
    'USER_PARS': {
        'type': 'char',
        'count': 10000,
        'value': [0],
    },
    'IOCS_NOT_TO_STOP': {
        'type': 'char',
        'count': 16000,
        'value': [0],
    },
}

class DatabaseServer(Driver):
    """The class for handling all the static PV access and monitors etc.
    """
    def __init__(self, ca_server, dbid, options_folder, test_mode=False):
        """Constructor.

        Args:
            ca_server (CAServer): The CA server used for generating PVs on the fly
            dbid (string): The id of the database that holds IOC information.
            options_folder (string): The location of the folder containing the config.xml file that holds IOC options
        """
        if test_mode:
            ps = MockProcServWrapper()
        else:
            super(DatabaseServer, self).__init__()
            ps = ProcServWrapper()
        self._ca_server = ca_server
        self._options_holder = OptionsHolder(options_folder, OptionsLoader())

        # Initialise database connection
        try:
            self._db = IOCData(dbid, ps, MACROS["$(MYPVPREFIX)"])
            print_and_log("Connected to database", "INFO", "DBSVR")
        except Exception as err:
            self._db = None
            print_and_log("Problem initialising DB connection: %s" % err, "MAJOR", "DBSVR")

        # Initialise experimental database connection
        try:
            self._ed = ExpData(MACROS["$(MYPVPREFIX)"])
            print_and_log("Connected to experimental details database", "INFO", "DBSVR")
        except Exception as err:
            self._ed = None
            print_and_log("Problem connecting to experimental details database: %s" % err, "MAJOR", "DBSVR")

        if self._db is not None and not test_mode:
            # Start a background thread for keeping track of running IOCs
            self.monitor_lock = RLock()
            monitor_thread = Thread(target=self.update_ioc_monitors, args=())
            monitor_thread.daemon = True  # Daemonise thread
            monitor_thread.start()

    def read(self, reason):
        """A method called by SimpleServer when a PV is read from the DatabaseServer over Channel Access.

        Args:
            reason (string): The PV that is being requested (without the PV prefix)

        Returns:
            string : A compressed and hexed JSON formatted string that gives the desired information based on reason.
        """
        if reason == 'SAMPLE_PARS':
            value = self.encode4return(self.get_sample_par_names())
        elif reason == 'BEAMLINE_PARS':
            value = self.encode4return(self.get_beamline_par_names())
        elif reason == 'USER_PARS':
            value = self.encode4return(self.get_user_par_names())
        elif reason == "IOCS_NOT_TO_STOP":
            value = self.encode4return(IOCS_NOT_TO_STOP)
        else:
            value = self.getParam(reason)
        return value

    def write(self, reason, value):
        """A method called by SimpleServer when a PV is written to the DatabaseServer over Channel Access.

        Args:
            reason (string): The PV that is being requested (without the PV prefix)
            value (string): The data being written to the 'reason' PV

        Returns:
            bool : True
        """
        status = True
        try:
            if reason == 'ED:RBNUMBER:SP':
                #print_and_log("Updating to use experiment ID: " + value, "INFO", "DBSVR")
                self._ed.updateExperimentID(value)
            elif reason == 'ED:USERNAME:SP':
                self._ed.updateUsername(dehex_and_decompress(value))
        except Exception as err:
            value = compress_and_hex(convert_to_json("Error: " + str(err)))
            print_and_log(str(err), "MAJOR")
        # store the values
        if status:
            self.setParam(reason, value)
        return status

    def update_ioc_monitors(self):
        """Updates all the PVs that hold information on the IOCS and their associated PVs
        """
        while True:
            if self._db is not None:
                self._db.update_iocs_status()
                self.setParam("IOCS", self.encode4return(self._get_iocs_info()))
                self.setParam("PVS:ALL", self.encode4return(self._get_interesting_pvs("")))
                self.setParam("PVS:ACTIVE", self.encode4return(self._get_active_pvs()))
                self.setParam("PVS:INTEREST:HIGH", self.encode4return(self._get_interesting_pvs("HIGH")))
                self.setParam("PVS:INTEREST:MEDIUM", self.encode4return(self._get_interesting_pvs("MEDIUM")))
                self.setParam("PVS:INTEREST:FACILITY", self.encode4return(self._get_interesting_pvs("FACILITY")))
                # Update them
                with self.monitor_lock:
                    self.updatePVs()
            sleep(1)

    def encode4return(self, data):
        """Converts data to JSON, compresses it and converts it to hex.

        Args:
            data (string): The data to encode

        Returns:
            string : The encoded data
        """
        return compress_and_hex(json.dumps(data).encode('ascii', 'replace'))

    def _get_iocs_info(self):
        iocs = self._db.get_iocs()
        options = self._options_holder.get_config_options()
        for iocname in iocs.keys():
            if iocname in options:
                iocs[iocname].update_pv_value(options[iocname])
        return iocs

    def _get_interesting_pvs(self, level, ioc=None):
        if self._db is not None:
            return self._db.get_interesting_pvs(level, ioc)
        else:
            return list()

    def _get_active_pvs(self):
        if self._db is not None:
            return self._db.get_active_pvs()
        else:
            return list()

    def get_sample_par_names(self):
        """Returns the sample parameters from the database, replacing the MYPVPREFIX macro

        Returns:
            list : A list of sample parameter names, an empty list if the database does not exist
        """
        if self._db is not None:
            return [p.replace(MACROS["$(MYPVPREFIX)"], "") for p in self._db.get_sample_pars()]
        else:
            return list()

    def get_beamline_par_names(self):
        """Returns the beamline parameters from the database, replacing the MYPVPREFIX macro

        Returns:
            list : A list of beamline parameter names, an empty list if the database does not exist
        """
        if self._db is not None:
            return [p.replace(MACROS["$(MYPVPREFIX)"], "") for p in self._db.get_beamline_pars()]
        else:
            return list()

    def get_user_par_names(self):
        """Returns the user parameters from the database, replacing the MYPVPREFIX macro

        Returns:
            list : A list of user parameter names, an empty list if the database does not exist
        """
        if self._db is not None:
            return [p.replace(MACROS["$(MYPVPREFIX)"], "") for p in self._db.get_user_pars()]
        else:
            return list()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument('-bs', '--blockserver_prefix', nargs=1, type=str, default=[MACROS["$(MYPVPREFIX)"]+'CS:BLOCKSERVER:'],
                        help='The prefix for PVs served by the blockserver(default=%MYPVPREFIX%CS:BLOCKSERVER:)')

    parser.add_argument('-od', '--options_dir', nargs=1, type=str, default=['.'],
                        help='The directory from which to load the configuration options(default=current directory)')

    parser.add_argument('-f', '--facility', nargs=1, type=str, default=['ISIS'],
                        help='Which facility is this being run for (default=ISIS)')

    args = parser.parse_args()

    FACILITY = args.facility[0]
    if FACILITY == "ISIS":
        from server_common.loggers.isis_logger import IsisLogger
        set_logger(IsisLogger())
    print_and_log("FACILITY = %s" % FACILITY, "INFO", "DBSVR")

    BLOCKSERVER_PREFIX = args.blockserver_prefix[0]
    if not BLOCKSERVER_PREFIX.endswith(':'):
        BLOCKSERVER_PREFIX += ":"
    BLOCKSERVER_PREFIX = BLOCKSERVER_PREFIX.replace('%MYPVPREFIX%', MACROS["$(MYPVPREFIX)"])
    print_and_log("BLOCKSERVER PREFIX = %s" % BLOCKSERVER_PREFIX, "INFO", "DBSVR")

    OPTIONS_DIR = os.path.abspath(args.options_dir[0])
    print_and_log("OPTIONS DIRECTORY = %s" % OPTIONS_DIR, "INFO", "DBSVR")
    if not os.path.isdir(os.path.abspath(OPTIONS_DIR)):
        # Create it then
        os.makedirs(os.path.abspath(OPTIONS_DIR))

    SERVER = CAServer(BLOCKSERVER_PREFIX)
    SERVER.createPV(BLOCKSERVER_PREFIX, PVDB)
    SERVER.createPV(MACROS["$(MYPVPREFIX)"], ExpData.EDPV)
    DRIVER = DatabaseServer(SERVER, IOCDB, OPTIONS_DIR)

    # Process CA transactions
    while True:
        try:
            SERVER.process(0.1)
        except Exception as err:
            print_and_log(err,"MAJOR")
            break

    DRIVER.close()
<commit_msg>Revert change to database server use correct method<commit_after># This file is part of the ISIS IBEX application.
# Copyright (C) 2012-2016 Science & Technology Facilities Council.
# All rights reserved.
#
# This program is distributed in the hope that it will be useful.
# This program and the accompanying materials are made available under the
# terms of the Eclipse Public License v1.0 which accompanies this distribution.
# EXCEPT AS EXPRESSLY SET FORTH IN THE ECLIPSE PUBLIC LICENSE V1.0, THE PROGRAM
# AND ACCOMPANYING MATERIALS ARE PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES
# OR CONDITIONS OF ANY KIND.  See the Eclipse Public License v1.0 for more details.
#
# You should have received a copy of the Eclipse Public License v1.0
# along with this program; if not, you can obtain a copy from
# https://www.eclipse.org/org/documents/epl-v10.php or
# http://opensource.org/licenses/eclipse-1.0.php

# Add root path for access to server_commons
import os
import sys
sys.path.insert(0, os.path.abspath(os.environ["MYDIRBLOCK"]))

# Standard imports
from pcaspy import Driver
from time import sleep
import argparse
from server_common.utilities import compress_and_hex, print_and_log, set_logger, convert_to_json, dehex_and_decompress
from server_common.channel_access_server import CAServer
from ioc_data import IOCData
from exp_data import ExpData
import json
from threading import Thread, RLock
from procserv_utils import ProcServWrapper
from options_holder import OptionsHolder
from options_loader import OptionsLoader
from mocks.mock_procserv_utils import MockProcServWrapper

IOCDB = 'iocdb'
IOCS_NOT_TO_STOP = ('INSTETC', 'PSCTRL', 'ISISDAE', 'BLOCKSVR', 'ARINST', 'ARBLOCK', 'GWBLOCK', 'RUNCTRL')

MACROS = {
    "$(MYPVPREFIX)": os.environ['MYPVPREFIX'],
    "$(EPICS_KIT_ROOT)": os.environ['EPICS_KIT_ROOT'],
    "$(ICPCONFIGROOT)": os.environ['ICPCONFIGROOT']
}

PVDB = {
    'IOCS': {
        # Handled by the monitor thread
        'type': 'char',
        'count': 16000,
        'value': [0],
    },
    'PVS:INTEREST:HIGH': {
        # Handled by the monitor thread
        'type': 'char',
        'count': 64000,
        'value': [0],
    },
    'PVS:INTEREST:MEDIUM': {
        # Handled by the monitor thread
        'type': 'char',
        'count': 64000,
        'value': [0],
    },
    'PVS:INTEREST:FACILITY': {
        # Handled by the monitor thread
        'type': 'char',
        'count': 64000,
        'value': [0],
    },
    'PVS:ACTIVE': {
        # Handled by the monitor thread
        'type': 'char',
        'count': 64000,
        'value': [0],
    },
    'PVS:ALL': {
        # Handled by the monitor thread
        'type': 'char',
        'count': 64000,
        'value': [0],
    },
    'SAMPLE_PARS': {
        'type': 'char',
        'count': 10000,
        'value': [0],
    },
    'BEAMLINE_PARS': {
        'type': 'char',
        'count': 10000,
        'value': [0],
    },
    'USER_PARS': {
        'type': 'char',
        'count': 10000,
        'value': [0],
    },
    'IOCS_NOT_TO_STOP': {
        'type': 'char',
        'count': 16000,
        'value': [0],
    },
}

class DatabaseServer(Driver):
    """The class for handling all the static PV access and monitors etc.
    """
    def __init__(self, ca_server, dbid, options_folder, test_mode=False):
        """Constructor.

        Args:
            ca_server (CAServer): The CA server used for generating PVs on the fly
            dbid (string): The id of the database that holds IOC information.
            options_folder (string): The location of the folder containing the config.xml file that holds IOC options
        """
        if test_mode:
            ps = MockProcServWrapper()
        else:
            super(DatabaseServer, self).__init__()
            ps = ProcServWrapper()
        self._ca_server = ca_server
        self._options_holder = OptionsHolder(options_folder, OptionsLoader())

        # Initialise database connection
        try:
            self._db = IOCData(dbid, ps, MACROS["$(MYPVPREFIX)"])
            print_and_log("Connected to database", "INFO", "DBSVR")
        except Exception as err:
            self._db = None
            print_and_log("Problem initialising DB connection: %s" % err, "MAJOR", "DBSVR")

        # Initialise experimental database connection
        try:
            self._ed = ExpData(MACROS["$(MYPVPREFIX)"])
            print_and_log("Connected to experimental details database", "INFO", "DBSVR")
        except Exception as err:
            self._ed = None
            print_and_log("Problem connecting to experimental details database: %s" % err, "MAJOR", "DBSVR")

        if self._db is not None and not test_mode:
            # Start a background thread for keeping track of running IOCs
            self.monitor_lock = RLock()
            monitor_thread = Thread(target=self.update_ioc_monitors, args=())
            monitor_thread.daemon = True  # Daemonise thread
            monitor_thread.start()

    def read(self, reason):
        """A method called by SimpleServer when a PV is read from the DatabaseServer over Channel Access.

        Args:
            reason (string): The PV that is being requested (without the PV prefix)

        Returns:
            string : A compressed and hexed JSON formatted string that gives the desired information based on reason.
        """
        if reason == 'SAMPLE_PARS':
            value = self.encode4return(self.get_sample_par_names())
        elif reason == 'BEAMLINE_PARS':
            value = self.encode4return(self.get_beamline_par_names())
        elif reason == 'USER_PARS':
            value = self.encode4return(self.get_user_par_names())
        elif reason == "IOCS_NOT_TO_STOP":
            value = self.encode4return(IOCS_NOT_TO_STOP)
        else:
            value = self.getParam(reason)
        return value

    def write(self, reason, value):
        """A method called by SimpleServer when a PV is written to the DatabaseServer over Channel Access.

        Args:
            reason (string): The PV that is being requested (without the PV prefix)
            value (string): The data being written to the 'reason' PV

        Returns:
            bool : True
        """
        status = True
        try:
            if reason == 'ED:RBNUMBER:SP':
                #print_and_log("Updating to use experiment ID: " + value, "INFO", "DBSVR")
                self._ed.updateExperimentID(value)
            elif reason == 'ED:USERNAME:SP':
                self._ed.updateUsername(dehex_and_decompress(value))
        except Exception as err:
            value = compress_and_hex(convert_to_json("Error: " + str(err)))
            print_and_log(str(err), "MAJOR")
        # store the values
        if status:
            self.setParam(reason, value)
        return status

    def update_ioc_monitors(self):
        """Updates all the PVs that hold information on the IOCS and their associated PVs
        """
        while True:
            if self._db is not None:
                self._db.update_iocs_status()
                self.setParam("IOCS", self.encode4return(self._get_iocs_info()))
                self.setParam("PVS:ALL", self.encode4return(self._get_interesting_pvs("")))
                self.setParam("PVS:ACTIVE", self.encode4return(self._get_active_pvs()))
                self.setParam("PVS:INTEREST:HIGH", self.encode4return(self._get_interesting_pvs("HIGH")))
                self.setParam("PVS:INTEREST:MEDIUM", self.encode4return(self._get_interesting_pvs("MEDIUM")))
                self.setParam("PVS:INTEREST:FACILITY", self.encode4return(self._get_interesting_pvs("FACILITY")))
                # Update them
                with self.monitor_lock:
                    self.updatePVs()
            sleep(1)

    def encode4return(self, data):
        """Converts data to JSON, compresses it and converts it to hex.

        Args:
            data (string): The data to encode

        Returns:
            string : The encoded data
        """
        return compress_and_hex(json.dumps(data).encode('ascii', 'replace'))

    def _get_iocs_info(self):
        iocs = self._db.get_iocs()
        options = self._options_holder.get_config_options()
        for iocname in iocs.keys():
            if iocname in options:
                iocs[iocname].update(options[iocname])
        return iocs

    def _get_interesting_pvs(self, level, ioc=None):
        if self._db is not None:
            return self._db.get_interesting_pvs(level, ioc)
        else:
            return list()

    def _get_active_pvs(self):
        if self._db is not None:
            return self._db.get_active_pvs()
        else:
            return list()

    def get_sample_par_names(self):
        """Returns the sample parameters from the database, replacing the MYPVPREFIX macro

        Returns:
            list : A list of sample parameter names, an empty list if the database does not exist
        """
        if self._db is not None:
            return [p.replace(MACROS["$(MYPVPREFIX)"], "") for p in self._db.get_sample_pars()]
        else:
            return list()

    def get_beamline_par_names(self):
        """Returns the beamline parameters from the database, replacing the MYPVPREFIX macro

        Returns:
            list : A list of beamline parameter names, an empty list if the database does not exist
        """
        if self._db is not None:
            return [p.replace(MACROS["$(MYPVPREFIX)"], "") for p in self._db.get_beamline_pars()]
        else:
            return list()

    def get_user_par_names(self):
        """Returns the user parameters from the database, replacing the MYPVPREFIX macro

        Returns:
            list : A list of user parameter names, an empty list if the database does not exist
        """
        if self._db is not None:
            return [p.replace(MACROS["$(MYPVPREFIX)"], "") for p in self._db.get_user_pars()]
        else:
            return list()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument('-bs', '--blockserver_prefix', nargs=1, type=str, default=[MACROS["$(MYPVPREFIX)"]+'CS:BLOCKSERVER:'],
                        help='The prefix for PVs served by the blockserver(default=%MYPVPREFIX%CS:BLOCKSERVER:)')

    parser.add_argument('-od', '--options_dir', nargs=1, type=str, default=['.'],
                        help='The directory from which to load the configuration options(default=current directory)')

    parser.add_argument('-f', '--facility', nargs=1, type=str, default=['ISIS'],
                        help='Which facility is this being run for (default=ISIS)')

    args = parser.parse_args()

    FACILITY = args.facility[0]
    if FACILITY == "ISIS":
        from server_common.loggers.isis_logger import IsisLogger
        set_logger(IsisLogger())
    print_and_log("FACILITY = %s" % FACILITY, "INFO", "DBSVR")

    BLOCKSERVER_PREFIX = args.blockserver_prefix[0]
    if not BLOCKSERVER_PREFIX.endswith(':'):
        BLOCKSERVER_PREFIX += ":"
    BLOCKSERVER_PREFIX = BLOCKSERVER_PREFIX.replace('%MYPVPREFIX%', MACROS["$(MYPVPREFIX)"])
    print_and_log("BLOCKSERVER PREFIX = %s" % BLOCKSERVER_PREFIX, "INFO", "DBSVR")

    OPTIONS_DIR = os.path.abspath(args.options_dir[0])
    print_and_log("OPTIONS DIRECTORY = %s" % OPTIONS_DIR, "INFO", "DBSVR")
    if not os.path.isdir(os.path.abspath(OPTIONS_DIR)):
        # Create it then
        os.makedirs(os.path.abspath(OPTIONS_DIR))

    SERVER = CAServer(BLOCKSERVER_PREFIX)
    SERVER.createPV(BLOCKSERVER_PREFIX, PVDB)
    SERVER.createPV(MACROS["$(MYPVPREFIX)"], ExpData.EDPV)
    DRIVER = DatabaseServer(SERVER, IOCDB, OPTIONS_DIR)

    # Process CA transactions
    while True:
        try:
            SERVER.process(0.1)
        except Exception as err:
            print_and_log(err,"MAJOR")
            break

    DRIVER.close()
<|endoftext|>


<commit_before># Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import datetime
import hmac
import json
import os
import random
import secrets
import time
import sys

from hashlib import sha1
from urllib.request import Request, urlopen

# set defaults
event_timespan = 604800
num_events = 20
num_issues = 2

# parse arguments
parser = argparse.ArgumentParser()
parser.add_argument("--event_timespan", "-t",
    help="time duration (in seconds) of timestamps of generated events (from [Now-timespan] to [Now]); default=604800 (1 week)")
parser.add_argument("--num_events", "-e", help="number of events to generate; default=20")
parser.add_argument("--num_issues", "-i", help="number of issues to generate; default=2")
parser.add_argument("--vc_system", "-v", help="version control system (e.g. 'github','gitlab')", required=True, choices=['gitlab','github'])
args = parser.parse_args()

# override defaults if specified in command line
if args.event_timespan:
    event_timespan = int(args.event_timespan)
if args.num_events:
    num_events = int(args.num_events)
if args.num_issues:
    num_issues = int(args.num_issues)

if num_issues > num_events:
    print("Error: num_issues cannot be greater than num_events")
    sys.exit()

def make_changes(num_changes):
    changes = []
    max_time = time.time() - event_timespan
    head_commit = None
    event = None

    for x in range(num_changes):
        change_id = secrets.token_hex(20)
        unix_timestamp = time.time() - random.randrange(0, event_timespan)
        change = {
            "id": change_id,
            "timestamp": datetime.datetime.fromtimestamp(unix_timestamp),
        }

        if unix_timestamp > max_time:
            max_time = unix_timestamp
            head_commit = change

        changes.append(change)

    if args.vc_system == "gitlab":
        event = {"object_kind": "push",
             "checkout_sha": head_commit["id"],
             "commits": changes}
    if args.vc_system == "github":
        event = {"head_commit": head_commit, "commits": changes}

    return event

def create_github_deploy_event(change):
    deployment = {
        "deployment_status": {
            "updated_at": change["timestamp"],
            "id": secrets.token_hex(20),
            "state": "success",
        },
        "deployment": {
           "sha": change["id"],
        }
    }
    return deployment

def create_gitlab_pipeline_event(changes):
    pipeline = None
    checkout_sha = changes["checkout_sha"]
    for c in changes["commits"]:
        if c["id"] == checkout_sha:
            pipeline = {
                "object_kind": "pipeline",
                "object_attributes": {
                    "created_at": c["timestamp"],
                    "id": random.randrange(0, 1000),
                    "status": "success"
                },
                "commit": c,
            }
    return pipeline

def make_github_issue(root_cause):

    event = {
        "issue": {
            "created_at": root_cause["timestamp"],
            "updated_at": datetime.datetime.now(),
            "closed_at": datetime.datetime.now(),
            "number": random.randrange(0, 1000),
            "labels": [{"name": "Incident"}],
            "body": "root cause: %s" % root_cause["id"],
        }
    }
    return event

def make_gitlab_issue(changes):
    issue = None
    checkout_sha = changes["checkout_sha"]
    for c in changes["commits"]:
        if c["id"] == checkout_sha:
            issue = {
                "object_kind": "issue",
                "object_attributes": {
                    "created_at": c["timestamp"],
                    "updated_at": datetime.datetime.now(),
                    "closed_at": datetime.datetime.now(),
                    "id": random.randrange(0, 1000),
                    "labels": [{"title": "Incident"}],
                    "description": "root cause: %s" % c["id"],
                }
            }
    return issue

def post_to_webhook(event_type, data):
    webhook_url = os.environ.get("WEBHOOK")
    data = json.dumps(data, default=str).encode()
    request = Request(webhook_url, data)

    if args.vc_system == "github":
        secret = os.environ.get("SECRET").encode()
        signature = hmac.new(secret, data, sha1)
        request.add_header("X-Github-Event", event_type)
        request.add_header("X-Hub-Signature", "sha1=" + signature.hexdigest())
        request.add_header("User-Agent", "GitHub-Hookshot/mock")

    if args.vc_system == "gitlab":
        secret = os.environ.get("SECRET")
        request.add_header("X-Gitlab-Event", event_type)
        request.add_header("X-Gitlab-Token", secret)

    request.add_header("Content-Type", "application/json")
    request.add_header("Mock", True)

    token = os.environ.get("TOKEN")
    if token:
        request.add_header("Authorization", f"Bearer {token}")

    response = urlopen(request)

    if response.getcode() == 204:
        return 1
    else:
        return 0

all_changesets = []
changes_sent = 0
for x in range(num_events):

    # make a change set containing a random number of changes
    changeset = make_changes(random.randrange(1,5))

    # Send individual changes data
    for c in changeset["commits"]:
        curr_change = None
        if args.vc_system == "gitlab":
            curr_change = {"object_kind": "push", "checkout_sha": c['id'], "commits": [c]}
        if args.vc_system == "github":
            curr_change = {"head_commit": c, "commits": [c]}
        changes_sent += post_to_webhook("push", curr_change)

    # Send fully associated push event
    post_to_webhook("push", changeset)

    # Make and send a deployment
    if args.vc_system == "gitlab":
        pipeline = create_gitlab_pipeline_event(changeset)
        post_to_webhook("pipeline",pipeline)

    if args.vc_system == "github":
        deploy = create_github_deploy_event(changeset["head_commit"])
        post_to_webhook("deployment_status", deploy)

    all_changesets.append(changeset)

# randomly create incidents associated to changes
changesets_with_issues = random.sample(all_changesets,num_issues)
for changeset in changesets_with_issues:
    issue = None
    if args.vc_system == "gitlab":
        issue = make_gitlab_issue(changeset)
    if args.vc_system == "github":
        issue = make_github_issue(changeset["head_commit"])
    post_to_webhook("issues",issue)

print(f"{changes_sent} changes successfully sent to event-handler")<commit_msg>fix lint errors<commit_after># Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import datetime
import hmac
import json
import os
import random
import secrets
import time
import sys

from hashlib import sha1
from urllib.request import Request, urlopen

# set defaults
event_timespan = 604800
num_events = 20
num_issues = 2

# parse arguments
parser = argparse.ArgumentParser()
parser.add_argument("--event_timespan", "-t",
                    help="time duration (in seconds) of timestamps of generated events \
                    (from [Now-timespan] to [Now]); default=604800 (1 week)")
parser.add_argument("--num_events", "-e", help="number of events to generate; default=20")
parser.add_argument("--num_issues", "-i", help="number of issues to generate; default=2")
parser.add_argument("--vc_system", "-v", help="version control system (e.g. 'github', 'gitlab')",
                    required=True, choices=['gitlab', 'github'])
args = parser.parse_args()

# override defaults if specified in command line
if args.event_timespan:
    event_timespan = int(args.event_timespan)
if args.num_events:
    num_events = int(args.num_events)
if args.num_issues:
    num_issues = int(args.num_issues)

if num_issues > num_events:
    print("Error: num_issues cannot be greater than num_events")
    sys.exit()

def make_changes(num_changes):
    changes = []
    max_time = time.time() - event_timespan
    head_commit = None
    event = None

    for x in range(num_changes):
        change_id = secrets.token_hex(20)
        unix_timestamp = time.time() - random.randrange(0, event_timespan)
        change = {
            "id": change_id,
            "timestamp": datetime.datetime.fromtimestamp(unix_timestamp),
        }

        if unix_timestamp > max_time:
            max_time = unix_timestamp
            head_commit = change

        changes.append(change)

    if args.vc_system == "gitlab":
        event = {"object_kind": "push",
                 "checkout_sha": head_commit["id"], "commits": changes}
    if args.vc_system == "github":
        event = {"head_commit": head_commit, "commits": changes}

    return event

def create_github_deploy_event(change):
    deployment = {
        "deployment_status": {
            "updated_at": change["timestamp"],
            "id": secrets.token_hex(20),
            "state": "success",
        },
        "deployment": {
           "sha": change["id"],
        }
    }
    return deployment

def create_gitlab_pipeline_event(changes):
    pipeline = None
    checkout_sha = changes["checkout_sha"]
    for c in changes["commits"]:
        if c["id"] == checkout_sha:
            pipeline = {
                "object_kind": "pipeline",
                "object_attributes": {
                    "created_at": c["timestamp"],
                    "id": random.randrange(0, 1000),
                    "status": "success"
                },
                "commit": c,
            }
    return pipeline

def make_github_issue(root_cause):
    event = {
        "issue": {
            "created_at": root_cause["timestamp"],
            "updated_at": datetime.datetime.now(),
            "closed_at": datetime.datetime.now(),
            "number": random.randrange(0, 1000),
            "labels": [{"name": "Incident"}],
            "body": "root cause: %s" % root_cause["id"],
        }
    }
    return event

def make_gitlab_issue(changes):
    issue = None
    checkout_sha = changes["checkout_sha"]
    for c in changes["commits"]:
        if c["id"] == checkout_sha:
            issue = {
                "object_kind": "issue",
                "object_attributes": {
                    "created_at": c["timestamp"],
                    "updated_at": datetime.datetime.now(),
                    "closed_at": datetime.datetime.now(),
                    "id": random.randrange(0, 1000),
                    "labels": [{"title": "Incident"}],
                    "description": "root cause: %s" % c["id"],
                }
            }
    return issue

def post_to_webhook(event_type, data):
    webhook_url = os.environ.get("WEBHOOK")
    data = json.dumps(data, default=str).encode()
    request = Request(webhook_url, data)

    if args.vc_system == "github":
        secret = os.environ.get("SECRET").encode()
        signature = hmac.new(secret, data, sha1)
        request.add_header("X-Github-Event", event_type)
        request.add_header("X-Hub-Signature", "sha1=" + signature.hexdigest())
        request.add_header("User-Agent", "GitHub-Hookshot/mock")

    if args.vc_system == "gitlab":
        secret = os.environ.get("SECRET")
        request.add_header("X-Gitlab-Event", event_type)
        request.add_header("X-Gitlab-Token", secret)

    request.add_header("Content-Type", "application/json")
    request.add_header("Mock", True)

    token = os.environ.get("TOKEN")
    if token:
        request.add_header("Authorization", f"Bearer {token}")

    response = urlopen(request)

    if response.getcode() == 204:
        return 1
    else:
        return 0

all_changesets = []
changes_sent = 0
for x in range(num_events):

    # make a change set containing a random number of changes
    changeset = make_changes(random.randrange(1, 5))

    # Send individual changes data
    for c in changeset["commits"]:
        curr_change = None
        if args.vc_system == "gitlab":
            curr_change = {"object_kind": "push", "checkout_sha": c['id'], "commits": [c]}
        if args.vc_system == "github":
            curr_change = {"head_commit": c, "commits": [c]}
        changes_sent += post_to_webhook("push", curr_change)

    # Send fully associated push event
    post_to_webhook("push", changeset)

    # Make and send a deployment
    if args.vc_system == "gitlab":
        pipeline = create_gitlab_pipeline_event(changeset)
        post_to_webhook("pipeline", pipeline)

    if args.vc_system == "github":
        deploy = create_github_deploy_event(changeset["head_commit"])
        post_to_webhook("deployment_status", deploy)

    all_changesets.append(changeset)

# randomly create incidents associated to changes
changesets_with_issues = random.sample(all_changesets, num_issues)
for changeset in changesets_with_issues:
    issue = None
    if args.vc_system == "gitlab":
        issue = make_gitlab_issue(changeset)
    if args.vc_system == "github":
        issue = make_github_issue(changeset["head_commit"])
    post_to_webhook("issues", issue)

print(f"{changes_sent} changes successfully sent to event-handler")
<|endoftext|>


<commit_before>from tiling_cpp import (fill_mesh, fill_mesh_function, fill_mesh_valuecollection,
                        fill_mf_from_mvc)
from test_periodic import compute_vertex_periodicity

from dolfin import (CompiledSubDomain, Mesh, MeshEditor, MeshFunction, MeshValueCollection,
                    Timer, info, SubsetIterator)
from collections import defaultdict
from itertools import izip
import numpy as np
import operator

# FIXME:
#
# mesh connectivity to identify entities takes the most time
#
# if make_mesh stored MeshValueCollections and there was way to
# make mvc to MeshFunctions then a lot of space could be saved

def TileMesh(tile, shape, mesh_data={}, TOL=1E-9):
    '''
    [tile tile;
     tile tile;
     tile tile;
     tile tile]

    The shape is an ntuple describing the number of pieces put next
    to each other in the i-th axis. mesh_data : (tdim, tag) -> [entities]
    is the way to encode mesh data of the tile.
    '''
    # All the axis shapes needs to be power of two
    assert all((((v & (v - 1)) == 0) and v > 0) for v in shape)
    # Sanity for glueing
    gdim = tile.geometry().dim()
    assert len(shape) <= gdim
    # While evolve is general mesh writing is limited to simplices only (FIXME)
    # so we bail out early
    assert str(tile.ufl_cell()) in ('interval', 'triangle', 'tetrahedron')

    t = Timer('evolve')
    # Do nothing
    if all(v == 1 for v in shape):
        return tile, mesh_data

    # We want to evolve cells, vertices of the mesh using geometry information
    # and periodicity info
    x = tile.coordinates()
    min_x = np.min(x, axis=0)
    max_x = np.max(x, axis=0)
    shifts = max_x - min_x

    shifts_x = []  # Geometry
    vertex_mappings = []  # Periodicity
    # Compute geometrical shift for EVERY direction:
    for axis in range(len(shape)):
        shift = shifts[axis]
        # Vector to shift cell vertices
        shift_x = np.zeros(gdim); shift_x[axis] = shift
        shifts_x.append(shift_x)

        # Compute periodicity in the vertices
        to_master = lambda x, shift=shift_x: x - shift
        # Mapping facets
        master = CompiledSubDomain('near(x[i], A, tol)', i=axis, A=min_x[axis], tol=TOL)
        slave = CompiledSubDomain('near(x[i], A, tol)', i=axis, A=max_x[axis], tol=TOL)

        error, vertex_mapping = compute_vertex_periodicity(tile, master, slave, to_master)
        # Fail when exended direction is no periodic
        assert error < 10*TOL, error

        vertex_mappings.append(vertex_mapping)
    # The final piece of information is cells
    cells = np.fromiter(tile.cells().flat, dtype='uintp').reshape(tile.cells().shape)

    # Evolve
    while shape:
        # Evolve is a bang method on vertex_mappings, shifts_x
        x, cells, shape = evolve(x, cells, vertex_mappings, shifts_x, shape, mesh_data=mesh_data)
    info('\tEvolve took %g s ' % t.stop())

    # Mesh data is evolved, (x cells) -> to mesh
    mesh = make_mesh(x, cells, tdim=tile.topology().dim(), gdim=gdim)

    return mesh, mesh_data

def evolve(x, cells, vertex_mappings, shifts_x, shape, mesh_data={}):
    '''Evolve tile along the last exis'''
    axis, gdim = len(shape) - 1, x.shape[1]
    assert gdim > axis >= 0

    # We're done evolving if only one tile is to be plae in the axis dir
    if shape[axis] == 1:
        vertex_mappings.pop()  # No longer needed
        shifts_x.pop()  # Do not touch x and cells
        return x, cells, shape[:-1]

    # Use the axis's periodicity and shifting.
    # NOTE: used only here and discarded
    vertex_mapping, shift_x = vertex_mappings.pop(), shifts_x.pop()

    master_vertices = vertex_mapping.values()
    slave_vertices = vertex_mapping.keys()

    refine = shape[axis]
    while refine > 1:
        n = len(x)
        # To make the tile piece we add all but the master vertices
        #new_vertices = np.fromiter(sorted(set(range(n)) - set(master_vertices)),
        #                           dtype=int, count=n-len(master_vertices))
        new_vertices = set(range(n))
        new_vertices.difference_update(master_vertices)
        new_vertices = np.fromiter(new_vertices, dtype=int)

        assert np.all(np.diff(new_vertices) > 0)
        # print '>>', new_vertices
        # Verices of the glued tiles
        x = np.vstack([x, x[new_vertices] + shift_x])

        # NOTE: using getitem and arrays seems to be on par in efficiency
        # with dicts. So then I keep translate as array because efficiency
        # Offset the free
        translate = np.empty(n, dtype=int)
        translate[new_vertices] = n + np.arange(len(new_vertices))
        # Those at master positions take slave values
        translate[master_vertices] = slave_vertices

        # Cells of the glued tiles
        new_cells = np.zeros_like(cells)
        new_cells.ravel()[:] = translate[cells.flatten()]

        cells = np.vstack([cells, new_cells])
        # Update the periodicty mapping - slaves are new
        slave_vertices = translate[slave_vertices]
        # For the directions that do not evolve we add the periodic pairs
        for vm in vertex_mappings:
            vm.update(dict(izip(translate[vm.keys()], translate[vm.values()])))
        # Add the entities defined in terms of the vertices
        if mesh_data:
            evolve_data(mesh_data, translate)

        # Iterate
        refine /= 2
        shift_x *= 2
    # Discard data not needed in next evolution
    return x, cells, shape[:-1]

def evolve_data(data, mapping):
    '''
    If mapping holds (tdim, tag) -> [tuple of indices]) where indices are
    w.r.t of old numbering and mapping is old to new we simply add the mapped
    entities.
    '''
    for key in data.keys():
        old = data[key]

        new = np.zeros_like(old)
        new.ravel()[:] = mapping[old.flatten()]
        data[key] = np.vstack([old, new])
    return data

def make_mesh(coordinates, cells, tdim, gdim):
    '''Mesh by MeshEditor from vertices and cells'''
    mesh = Mesh()
    assert mesh.mpi_comm().tompi4py().size == 1

    fill_mesh(coordinates.flatten(), cells.flatten(), tdim, gdim, mesh)

    return mesh

def mf_from_data(mesh, data):
    '''Build tdim -> mesh function from the data of TileMesh'''
    return _mx_from_data(mesh, data,
                         fill=fill_mesh_function,
                         init_container=lambda m, t: MeshFunction('size_t', m, t, 0))

def mvc_from_data(mesh, data):
    '''Build tdim -> mesh value collection from data of TileMesh'''
    return _mx_from_data(mesh, data,
                         fill=fill_mesh_valuecollection,
                         init_container=lambda m, t: MeshValueCollection('size_t', m, t))

def groupby(pairs, index):
    '''Organize pairs by pairs[index]'''
    groups = defaultdict(list)
    for pair in pairs: groups[pair[index]].append(pair)

    for item in groups.iteritems():
        yield item

def _mx_from_data(mesh, data, fill, init_container):
    '''Fill the contained over mesh by data'''
    assert mesh.mpi_comm().tompi4py().size == 1

    containers = {}
    # We have define entities in terms of vertex numbering
    # Order keys such by tdim (the first key)
    for tdim, keys in groupby(data.keys(), 0):
        # So we'll be getting the entity index by lookup
        mesh.init(tdim)
        mesh.init(0, tdim)
        # Build the meshfunction from data
        f = init_container(mesh, tdim)
        for key in keys:
            indices = data[key]
            # These entity indices get the 'color'
            fill(mesh, indices.flatten(), tdim, key[1], f)
        containers[tdim] = f

    return containers

def as_meshf(mvc, init_value=0):
    '''Make a mesh function out of mesh value collection'''
    if isinstance(mvc, (tuple, list)):
        return [as_meshf(x, init_value) for x in mvc]

    if isinstance(mvc, dict):
        return dict(zip(mvc.keys(), as_meshf(mvc.values())))

    # Base case
    mesh_f = MeshFunction('size_t', mvc.mesh(), mvc.dim(), init_value)
    fill_mf_from_mvc(mvc, mesh_f)

    return mesh_f

def load_data(mesh, h5_file, data_set, dim, data):
    '''
    Fill the data dictionary with data_set representing mesh function with
    dim over mesh read from h5_file according to key spec expected by tiling
    algorithm.
    '''
    mf = MeshFunction('size_t', mesh, dim, 0)
    h5_file.read(mf, data_set)

    # Data to evolve
    mesh.init(dim, 0)
    e2v = tile.topology()(dim, 0)

    tags = set(mf.array())
    # Don't evolve zero - we initialize to it
    if 0 in tags: tags.remove(0)
    info('%s evolves tags %r' % (data_set, tags))

    for tag in tags:
        data[(dim, tag)] = np.array([e2v(e.index()) for e in SubsetIterator(mf, tag)],
                                    dtype='uintp')
    return data

# ------------------------------------------------------------------------------

if __name__ == '__main__':
    from dolfin import mpi_comm_world, HDF5File, Timer, File
    import argparse, os

    parser = argparse.ArgumentParser(description='Put n tiles in x axis, m in y axis.')
    parser.add_argument('tile', type=str, help='H5 file that is the file')
    parser.add_argument('-n', type=int, default=1)
    parser.add_argument('-m', type=int, default=1)
    parser.add_argument('-facet_tags', type=str, default='surfaces',
                        help='name under which H5 stores facet tags')
    parser.add_argument('-cell_tags', type=str, default='volumes',
                        help='name under which H5 stores volume tags')

    save_pvd_parser = parser.add_mutually_exclusive_group(required=False)
    save_pvd_parser.add_argument('--save_pvd', dest='save_pvd', action='store_true')
    save_pvd_parser.add_argument('--no_save_pvd', dest='save_pvd', action='store_false')
    parser.set_defaults(save_pvd=False)

    args = parser.parse_args()

    # Some sanity
    root, ext = os.path.splitext(args.tile)
    assert ext == '.h5'

    shape = (args.n, args.m)
    assert all((((v & (v - 1)) == 0) and v > 0) for v in shape)

    # Load the tile mesh
    comm = mpi_comm_world()
    h5 = HDF5File(comm, args.tile, 'r')
    tile = Mesh()
    h5.read(tile, 'mesh', False)

    data = {}
    cell_dim = tile.topology().dim()
    facet_dim = cell_dim - 1

    if args.facet_tags:
        data = load_data(tile, h5, args.facet_tags, facet_dim, data)

    if args.cell_tags:
        data = load_data(tile, h5, args.cell_tags, cell_dim, data)

    t = Timer('tile')
    mesh, mesh_data = TileMesh(tile, shape, mesh_data=data)
    info('\nTiling took %g s; nvertices %d, ncells %d' % (t.stop(),
                                                          mesh.num_vertices(),
                                                          mesh.num_cells()))

    # Saving
    t = Timer('save')
    h5_file = '%s_%d_%d.h5' % (root, shape[0], shape[1])

    out = HDF5File(mesh.mpi_comm(), h5_file, 'w')
    out.write(mesh, 'mesh')

    tt = Timer('data')
    # To mesh functions
    if mesh_data:
        mfs = mf_from_data(mesh, mesh_data)

        for dim, name in (zip((facet_dim, cell_dim), (args.facet_tags, args.cell_tags))):
            if name:
                out.write(mfs[dim], name)

                if args.save_pvd:
                    File('%s_%d_%d_%s.pvd' % (root, shape[0], shape[1], name)) << mfs[dim]

    info('\t\tGetting data as MeshFoo took %g s' % tt.stop())

    info('\tSaving took %g' % t.stop())
<commit_msg>some pow2 speedup<commit_after>from tiling_cpp import (fill_mesh, fill_mesh_function, fill_mesh_valuecollection,
                        fill_mf_from_mvc)
from test_periodic import compute_vertex_periodicity

from dolfin import (CompiledSubDomain, Mesh, MeshEditor, MeshFunction, MeshValueCollection,
                    Timer, info, SubsetIterator)
from collections import defaultdict
from itertools import izip
import numpy as np
import operator

# FIXME:
#
# mesh connectivity to identify entities takes the most time
#
# if make_mesh stored MeshValueCollections and there was way to
# make mvc to MeshFunctions then a lot of space could be saved

def TileMesh(tile, shape, mesh_data={}, TOL=1E-9):
    '''
    [tile tile;
     tile tile;
     tile tile;
     tile tile]

    The shape is an ntuple describing the number of pieces put next
    to each other in the i-th axis. mesh_data : (tdim, tag) -> [entities]
    is the way to encode mesh data of the tile.
    '''
    # All the axis shapes needs to be power of two
    assert all((((v & (v - 1)) == 0) and v > 0) for v in shape)
    # Sanity for glueing
    gdim = tile.geometry().dim()
    assert len(shape) <= gdim
    # While evolve is general mesh writing is limited to simplices only (FIXME)
    # so we bail out early
    assert str(tile.ufl_cell()) in ('interval', 'triangle', 'tetrahedron')

    t = Timer('evolve')
    # Do nothing
    if all(v == 1 for v in shape):
        return tile, mesh_data

    # We want to evolve cells, vertices of the mesh using geometry information
    # and periodicity info
    x = tile.coordinates()
    min_x = np.min(x, axis=0)
    max_x = np.max(x, axis=0)
    shifts = max_x - min_x

    shifts_x = []  # Geometry
    vertex_mappings = []  # Periodicity
    # Compute geometrical shift for EVERY direction:
    for axis in range(len(shape)):
        shift = shifts[axis]
        # Vector to shift cell vertices
        shift_x = np.zeros(gdim); shift_x[axis] = shift
        shifts_x.append(shift_x)

        # Compute periodicity in the vertices
        to_master = lambda x, shift=shift_x: x - shift
        # Mapping facets
        master = CompiledSubDomain('near(x[i], A, tol)', i=axis, A=min_x[axis], tol=TOL)
        slave = CompiledSubDomain('near(x[i], A, tol)', i=axis, A=max_x[axis], tol=TOL)

        error, vertex_mapping = compute_vertex_periodicity(tile, master, slave, to_master)
        # Fail when exended direction is no periodic
        assert error < 10*TOL, error

        vertex_mappings.append(vertex_mapping)
    # The final piece of information is cells
    cells = np.empty((tile.num_cells(), tile.ufl_cell().num_vertices()), dtype='uintp')
    cells.ravel()[:] = tile.cells().flat
    #cells = np.fromiter(tile.cells().flat, dtype='uintp').reshape(tile.cells().shape)

    # Evolve
    while shape:
        # Evolve is a bang method on vertex_mappings, shifts_x
        x, cells, shape = evolve(x, cells, vertex_mappings, shifts_x, shape, mesh_data=mesh_data)
    info('\tEvolve took %g s ' % t.stop())

    # Mesh data is evolved, (x cells) -> to mesh
    mesh = make_mesh(x, cells, tdim=tile.topology().dim(), gdim=gdim)

    return mesh, mesh_data

def evolve(x, cells, vertex_mappings, shifts_x, shape, mesh_data={}):
    '''Evolve tile along the last exis'''
    axis, gdim = len(shape) - 1, x.shape[1]
    assert gdim > axis >= 0

    # We're done evolving if only one tile is to be plae in the axis dir
    if shape[axis] == 1:
        vertex_mappings.pop()  # No longer needed
        shifts_x.pop()  # Do not touch x and cells
        return x, cells, shape[:-1]

    # Use the axis's periodicity and shifting.
    # NOTE: used only here and discarded
    vertex_mapping, shift_x = vertex_mappings.pop(), shifts_x.pop()

    master_vertices = vertex_mapping.values()
    slave_vertices = vertex_mapping.keys()

    refine = shape[axis]
    while refine > 1:
        n = len(x)
        # To make the tile piece we add all but the master vertices
        #new_vertices = np.fromiter(sorted(set(range(n)) - set(master_vertices)),
        #                           dtype=int, count=n-len(master_vertices))
        new_vertices = set(range(n))
        new_vertices.difference_update(master_vertices)
        new_vertices = np.fromiter(new_vertices, dtype=int)
        assert np.all(np.diff(new_vertices) > 0)

        # Verices of the glued tiles
        x = np.vstack([x, x[new_vertices] + shift_x])

        # Offset the free
        translate = np.empty(n, dtype=int)
        translate[new_vertices] = n + np.arange(len(new_vertices))
        # Those at master positions take slave values
        translate[master_vertices] = slave_vertices

        # Cells of the glued tiles
        new_cells = np.empty_like(cells)
        new_cells.ravel()[:] = translate[cells.flat]

        cells = np.vstack([cells, new_cells])
        # Update the periodicty mapping - slaves are new
        slave_vertices = translate[slave_vertices]
        # For the directions that do not evolve we add the periodic pairs
        for vm in vertex_mappings:
            keys, values = np.array(vm.items()).T
            vm.update(dict(izip(translate[keys], translate[values])))
        # Add the entities defined in terms of the vertices
        if mesh_data:
            evolve_data(mesh_data, translate)

        # Iterate
        refine /= 2
        shift_x *= 2
    # Discard data not needed in next evolution
    return x, cells, shape[:-1]

def evolve_data(data, mapping):
    '''
    If mapping holds (tdim, tag) -> [tuple of indices]) where indices are
    w.r.t of old numbering and mapping is old to new we simply add the mapped
    entities.
    '''
    for key in data.keys():
        old = data[key]

        new = np.empty_like(old)
        new.ravel()[:] = mapping[old.flat]
        data[key] = np.vstack([old, new])
    return data

def make_mesh(coordinates, cells, tdim, gdim):
    '''Mesh by MeshEditor from vertices and cells'''
    mesh = Mesh()
    assert mesh.mpi_comm().tompi4py().size == 1

    fill_mesh(coordinates.flatten(), cells.flatten(), tdim, gdim, mesh)

    return mesh

def mf_from_data(mesh, data):
    '''Build tdim -> mesh function from the data of TileMesh'''
    return _mx_from_data(mesh, data,
                         fill=fill_mesh_function,
                         init_container=lambda m, t: MeshFunction('size_t', m, t, 0))

def mvc_from_data(mesh, data):
    '''Build tdim -> mesh value collection from data of TileMesh'''
    return _mx_from_data(mesh, data,
                         fill=fill_mesh_valuecollection,
                         init_container=lambda m, t: MeshValueCollection('size_t', m, t))

def groupby(pairs, index):
    '''Organize pairs by pairs[index]'''
    groups = defaultdict(list)
    for pair in pairs: groups[pair[index]].append(pair)

    for item in groups.iteritems():
        yield item

def _mx_from_data(mesh, data, fill, init_container):
    '''Fill the contained over mesh by data'''
    assert mesh.mpi_comm().tompi4py().size == 1

    containers = {}
    # We have define entities in terms of vertex numbering
    # Order keys such by tdim (the first key)
    for tdim, keys in groupby(data.keys(), 0):
        # So we'll be getting the entity index by lookup
        mesh.init(tdim)
        mesh.init(0, tdim)
        # Build the meshfunction from data
        f = init_container(mesh, tdim)
        for key in keys:
            indices = data[key]
            # These entity indices get the 'color'
            fill(mesh, indices.flatten(), tdim, key[1], f)
        containers[tdim] = f

    return containers

def as_meshf(mvc, init_value=0):
    '''Make a mesh function out of mesh value collection'''
    if isinstance(mvc, (tuple, list)):
        return [as_meshf(x, init_value) for x in mvc]

    if isinstance(mvc, dict):
        return dict(zip(mvc.keys(), as_meshf(mvc.values())))

    # Base case
    mesh_f = MeshFunction('size_t', mvc.mesh(), mvc.dim(), init_value)
    fill_mf_from_mvc(mvc, mesh_f)

    return mesh_f

def load_data(mesh, h5_file, data_set, dim, data):
    '''
    Fill the data dictionary with data_set representing mesh function with
    dim over mesh read from h5_file according to key spec expected by tiling
    algorithm.
    '''
    mf = MeshFunction('size_t', mesh, dim, 0)
    h5_file.read(mf, data_set)

    # Data to evolve
    mesh.init(dim, 0)
    e2v = tile.topology()(dim, 0)

    tags = set(mf.array())
    # Don't evolve zero - we initialize to it
    if 0 in tags: tags.remove(0)
    info('%s evolves tags %r' % (data_set, tags))

    for tag in tags:
        data[(dim, tag)] = np.array([e2v(e.index()) for e in SubsetIterator(mf, tag)],
                                    dtype='uintp')
    return data

# ------------------------------------------------------------------------------

if __name__ == '__main__':
    from dolfin import mpi_comm_world, HDF5File, Timer, File
    import argparse, os

    parser = argparse.ArgumentParser(description='Put n tiles in x axis, m in y axis.')
    parser.add_argument('tile', type=str, help='H5 file that is the file')
    parser.add_argument('-n', type=int, default=1)
    parser.add_argument('-m', type=int, default=1)
    parser.add_argument('-facet_tags', type=str, default='surfaces',
                        help='name under which H5 stores facet tags')
    parser.add_argument('-cell_tags', type=str, default='volumes',
                        help='name under which H5 stores volume tags')

    save_pvd_parser = parser.add_mutually_exclusive_group(required=False)
    save_pvd_parser.add_argument('--save_pvd', dest='save_pvd', action='store_true')
    save_pvd_parser.add_argument('--no_save_pvd', dest='save_pvd', action='store_false')
    parser.set_defaults(save_pvd=False)

    args = parser.parse_args()

    # Some sanity
    root, ext = os.path.splitext(args.tile)
    assert ext == '.h5'

    shape = (args.n, args.m)
    assert all((((v & (v - 1)) == 0) and v > 0) for v in shape)

    # Load the tile mesh
    comm = mpi_comm_world()
    h5 = HDF5File(comm, args.tile, 'r')
    tile = Mesh()
    h5.read(tile, 'mesh', False)

    data = {}
    cell_dim = tile.topology().dim()
    facet_dim = cell_dim - 1

    if args.facet_tags:
        data = load_data(tile, h5, args.facet_tags, facet_dim, data)

    if args.cell_tags:
        data = load_data(tile, h5, args.cell_tags, cell_dim, data)

    t = Timer('tile')
    mesh, mesh_data = TileMesh(tile, shape, mesh_data=data)
    info('\nTiling took %g s; nvertices %d, ncells %d' % (t.stop(),
                                                          mesh.num_vertices(),
                                                          mesh.num_cells()))

    # Saving
    t = Timer('save')
    h5_file = '%s_%d_%d.h5' % (root, shape[0], shape[1])

    out = HDF5File(mesh.mpi_comm(), h5_file, 'w')
    out.write(mesh, 'mesh')

    tt = Timer('data')
    # To mesh functions
    if mesh_data:
        mfs = mf_from_data(mesh, mesh_data)

        for dim, name in (zip((facet_dim, cell_dim), (args.facet_tags, args.cell_tags))):
            if name:
                out.write(mfs[dim], name)

                if args.save_pvd:
                    File('%s_%d_%d_%s.pvd' % (root, shape[0], shape[1], name)) << mfs[dim]

    info('\t\tGetting data as MeshFoo took %g s' % tt.stop())

    info('\tSaving took %g' % t.stop())
<|endoftext|>


<commit_before>import asyncio
import pytest
import socket
import subprocess
import sys
import re
import contextlib
import os
import ssl
import time
import unittest.case

from collections import namedtuple

import aioredis

# Public fixtures

@pytest.yield_fixture
def loop():
    """Creates new event loop."""
    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(None)

    yield loop

    if hasattr(loop, 'is_closed'):
        closed = loop.is_closed()
    else:
        closed = loop._closed   # XXX
    if not closed:
        loop.call_soon(loop.stop)
        loop.run_forever()
        loop.close()

@pytest.fixture(scope='session')
def unused_port():
    """Gets random free port."""
    def fun():
        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
            s.bind(('127.0.0.1', 0))
            return s.getsockname()[1]
    return fun

@pytest.fixture
def create_connection(_closable):
    """Wrapper around aioredis.create_connection."""

    @asyncio.coroutine
    def f(*args, **kw):
        conn = yield from aioredis.create_connection(*args, **kw)
        _closable(conn)
        return conn
    return f

@pytest.fixture
def create_redis(_closable, loop):
    """Wrapper around aioredis.create_redis."""

    @asyncio.coroutine
    def f(*args, **kw):
        redis = yield from aioredis.create_redis(*args, **kw)
        _closable(redis)
        return redis
    return f

@pytest.fixture
def create_pool(_closable):
    """Wrapper around aioredis.create_pool."""

    @asyncio.coroutine
    def f(*args, **kw):
        redis = yield from aioredis.create_pool(*args, **kw)
        _closable(redis)
        return redis
    return f

@pytest.fixture
def pool(create_pool, server, loop):
    """Returns RedisPool instance."""
    pool = loop.run_until_complete(
        create_pool(server.tcp_address, loop=loop))
    return pool

@pytest.fixture
def redis(create_redis, server, loop):
    """Returns Redis client instance."""
    redis = loop.run_until_complete(
        create_redis(server.tcp_address, loop=loop))
    loop.run_until_complete(redis.flushall())
    return redis

@pytest.yield_fixture
def _closable(loop):
    conns = []

    yield conns.append

    waiters = []
    while conns:
        conn = conns.pop(0)
        conn.close()
        waiters.append(conn.wait_closed())
    if waiters:
        loop.run_until_complete(asyncio.gather(*waiters, loop=loop))

@pytest.fixture(scope='session')
def server(start_server):
    """Starts redis-server instance."""
    return start_server('A')

@pytest.fixture(scope='session')
def serverB(start_server):
    """Starts redis-server instance."""
    return start_server('B')

# Internal stuff #

def pytest_addoption(parser):
    parser.addoption('--redis-server', default='/usr/bin/redis-server',
                     help="Path to redis-server executable,"
                          " defaults to `%(default)s`")
    parser.addoption('--ssl-cafile', default='tests/ssl/test.crt',
                     help="Path to testing SSL certificate")
    parser.addoption('--uvloop', default=False,
                     action='store_true',
                     help="Run tests with uvloop")

def _read_server_version(config):
    args = [config.getoption('--redis-server'), '--version']
    with subprocess.Popen(args, stdout=subprocess.PIPE) as proc:
        version = proc.stdout.readline().decode('utf-8')
    for part in version.split():
        if part.startswith('v='):
            break
    else:
        raise RuntimeError(
            "No version info can be found in {}".format(version))
    return tuple(map(int, part[2:].split('.')))

@pytest.yield_fixture(scope='session')
def start_server(request, unused_port):

    TCPAddress = namedtuple('TCPAddress', 'host port')
    RedisServer = namedtuple(
        'RedisServer', 'name tcp_address unixsocket version')
    processes = []
    servers = {}

    version = _read_server_version(request.config)

    def maker(name):
        if name in servers:
            return servers[name]

        port = unused_port()
        tcp_address = TCPAddress('localhost', port)
        unixsocket = '/tmp/redis.{}.sock'.format(port)

        proc = subprocess.Popen([request.config.getoption('--redis-server'),
                                 '--daemonize', 'no',
                                 '--save', '""',
                                 '--port', str(port),
                                 '--unixsocket', unixsocket,
                                 ], stdout=subprocess.PIPE)
        processes.append(proc)
        log = b''
        while b'The server is now ready to accept connections ' not in log:
            log = proc.stdout.readline()

        info = RedisServer(name, tcp_address, unixsocket, version)
        servers.setdefault(name, info)
        return info

    yield maker

    while processes:
        proc = processes.pop(0)
        proc.terminate()
        proc.wait()

@pytest.yield_fixture(scope='session')
def ssl_proxy(request, unused_port):
    processes = []
    by_port = {}

    cafile = os.path.abspath(request.config.getoption('--ssl-cafile'))
    pemfile = os.path.splitext(cafile)[0] + '.pem'
    assert os.path.exists(cafile), \
        "No test ssl certificate, run `make certificate`"
    assert os.path.exists(pemfile), \
        "No test ssl certificate, run `make certificate`"

    if hasattr(ssl, 'create_default_context'):
        ssl_ctx = ssl.create_default_context(cafile=cafile)
    else:
        ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
        ssl_ctx.load_verify_locations(cafile=cafile)
    if hasattr(ssl_ctx, 'check_hostname'):
        # available since python 3.4
        ssl_ctx.check_hostname = False
    ssl_ctx.verify_mode = ssl.CERT_NONE

    def sockat(unsecure_port):
        if unsecure_port in by_port:
            return by_port[unsecure_port]

        secure_port = unused_port()
        proc = subprocess.Popen(['/usr/bin/socat',
                                 'openssl-listen:{},'
                                 'cert={},'
                                 'verify=0,fork'
                                 .format(secure_port, pemfile),
                                 'tcp-connect:localhost:{}'
                                 .format(unsecure_port)
                                 ])
        processes.append(proc)
        time.sleep(1)   # XXX
        by_port[unsecure_port] = secure_port, ssl_ctx
        return secure_port, ssl_ctx

    yield sockat

    while processes:
        proc = processes.pop(0)
        proc.terminate()
        proc.wait()

@pytest.mark.tryfirst
def pytest_pycollect_makeitem(collector, name, obj):
    if collector.funcnamefilter(name):
        if not callable(obj):
            return
        item = pytest.Function(name, parent=collector)
        if 'run_loop' in item.keywords:
            # TODO: re-wrap with asyncio.coroutine if not native coroutine
            return list(collector._genfunctions(name, obj))

@pytest.mark.tryfirst
def pytest_pyfunc_call(pyfuncitem):
    """
    Run asyncio marked test functions in an event loop instead of a normal
    function call.
    """
    if 'run_loop' in pyfuncitem.keywords:
        funcargs = pyfuncitem.funcargs
        loop = funcargs['loop']
        testargs = {arg: funcargs[arg]
                    for arg in pyfuncitem._fixtureinfo.argnames}
        loop.run_until_complete(
            asyncio.wait_for(pyfuncitem.obj(**testargs),
                             15, loop=loop))
        return True

def pytest_runtest_setup(item):
    if 'run_loop' in item.keywords and 'loop' not in item.fixturenames:
        # inject an event loop fixture for all async tests
        item.fixturenames.append('loop')

def pytest_ignore_collect(path, config):
    if 'py35' in str(path):
        if sys.version_info < (3, 5, 0):
            return True

def pytest_collection_modifyitems(session, config, items):
    version = _read_server_version(config)
    for item in items:
        if 'redis_version' in item.keywords:
            marker = item.keywords['redis_version']
            if version < marker.kwargs['version']:
                item.add_marker(pytest.mark.skip(
                    reason=marker.kwargs['reason']))
        if 'ssl_proxy' in item.fixturenames:
            item.add_marker(pytest.mark.skipif(
                "not os.path.exists('/usr/bin/socat')",
                reason="socat package required (apt-get install socat)"))

def pytest_configure(config):
    if config.getoption('--uvloop'):
        try:
            import uvloop
        except ImportError:
            raise RuntimeError(
                "Can not import uvloop, make sure it is installed")
        asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())

@contextlib.contextmanager
def raises_regex(exc_type, message):
    with pytest.raises(exc_type) as exc_info:
        yield exc_info
    match = re.search(message, str(exc_info.value))
    assert match is not None, (
        "Pattern {!r} does not match {!r}"
        .format(message, str(exc_info.value)))

def logs(logger, level=None):
    """Catches logs for given logger and level.

    See unittest.TestCase.assertLogs for details.
    """
    return _AssertLogsContext(None, logger, level)

if sys.version_info > (3, 3):
    class _AssertLogsContext(unittest.case._AssertLogsContext):
        """Standard unittest's _AssertLogsContext context manager
        adopted to raise pytest failure.
        """

        def _raiseFailure(self, standardMsg):
            pytest.fail(standardMsg)
else:
    def _AssertLogsContext(self, test, logger, level):
        raise NotImplementedError("Available since Python3.4")

def redis_version(*version, reason):
    assert 1 < len(version) <= 3, version
    assert all(isinstance(v, int) for v in version), version
    return pytest.mark.redis_version(version=version, reason=reason)

def assert_almost_equal(first, second, places=None, msg=None, delta=None):
    assert not (places is None and delta is None)
    if delta is not None:
        assert abs(first - second) <= delta
    else:
        assert round(abs(first - second), places) == 0

def pytest_namespace():
    return {
        'raises_regex': raises_regex,
        'assert_almost_equal': assert_almost_equal,
        'redis_version': redis_version,
        'logs': logs,
        }
<commit_msg>another try to fix tests<commit_after>import asyncio
import pytest
import socket
import subprocess
import sys
import re
import contextlib
import os
import ssl
import time
import unittest.case

from collections import namedtuple

import aioredis

# Public fixtures

@pytest.yield_fixture
def loop():
    """Creates new event loop."""
    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(None)

    yield loop

    if hasattr(loop, 'is_closed'):
        closed = loop.is_closed()
    else:
        closed = loop._closed   # XXX
    if not closed:
        loop.call_soon(loop.stop)
        loop.run_forever()
        loop.close()

@pytest.fixture(scope='session')
def unused_port():
    """Gets random free port."""
    def fun():
        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
            s.bind(('127.0.0.1', 0))
            return s.getsockname()[1]
    return fun

@pytest.fixture
def create_connection(_closable):
    """Wrapper around aioredis.create_connection."""

    @asyncio.coroutine
    def f(*args, **kw):
        conn = yield from aioredis.create_connection(*args, **kw)
        _closable(conn)
        return conn
    return f

@pytest.fixture
def create_redis(_closable, loop):
    """Wrapper around aioredis.create_redis."""

    @asyncio.coroutine
    def f(*args, **kw):
        redis = yield from aioredis.create_redis(*args, **kw)
        _closable(redis)
        return redis
    return f

@pytest.fixture
def create_pool(_closable):
    """Wrapper around aioredis.create_pool."""

    @asyncio.coroutine
    def f(*args, **kw):
        redis = yield from aioredis.create_pool(*args, **kw)
        _closable(redis)
        return redis
    return f

@pytest.fixture
def pool(create_pool, server, loop):
    """Returns RedisPool instance."""
    pool = loop.run_until_complete(
        create_pool(server.tcp_address, loop=loop))
    return pool

@pytest.fixture
def redis(create_redis, server, loop):
    """Returns Redis client instance."""
    redis = loop.run_until_complete(
        create_redis(server.tcp_address, loop=loop))
    loop.run_until_complete(redis.flushall())
    return redis

@pytest.yield_fixture
def _closable(loop):
    conns = []

    yield conns.append

    waiters = []
    while conns:
        conn = conns.pop(0)
        conn.close()
        waiters.append(conn.wait_closed())
    if waiters:
        loop.run_until_complete(asyncio.gather(*waiters, loop=loop))

@pytest.fixture(scope='session')
def server(start_server):
    """Starts redis-server instance."""
    return start_server('A')

@pytest.fixture(scope='session')
def serverB(start_server):
    """Starts redis-server instance."""
    return start_server('B')

# Internal stuff #

def pytest_addoption(parser):
    parser.addoption('--redis-server', default='/usr/bin/redis-server',
                     help="Path to redis-server executable,"
                          " defaults to `%(default)s`")
    parser.addoption('--ssl-cafile', default='tests/ssl/test.crt',
                     help="Path to testing SSL certificate")
    parser.addoption('--uvloop', default=False,
                     action='store_true',
                     help="Run tests with uvloop")

def _read_server_version(config):
    args = [config.getoption('--redis-server'), '--version']
    with subprocess.Popen(args, stdout=subprocess.PIPE) as proc:
        version = proc.stdout.readline().decode('utf-8')
    for part in version.split():
        if part.startswith('v='):
            break
    else:
        raise RuntimeError(
            "No version info can be found in {}".format(version))
    return tuple(map(int, part[2:].split('.')))

@pytest.yield_fixture(scope='session')
def start_server(request, unused_port):

    TCPAddress = namedtuple('TCPAddress', 'host port')
    RedisServer = namedtuple(
        'RedisServer', 'name tcp_address unixsocket version')
    processes = []
    servers = {}

    version = _read_server_version(request.config)

    def maker(name):
        if name in servers:
            return servers[name]

        port = unused_port()
        tcp_address = TCPAddress('localhost', port)
        unixsocket = '/tmp/redis.{}.sock'.format(port)

        proc = subprocess.Popen([request.config.getoption('--redis-server'),
                                 '--daemonize', 'no',
                                 '--save', '""',
                                 '--port', str(port),
                                 '--unixsocket', unixsocket,
                                 ], stdout=subprocess.PIPE)
        processes.append(proc)
        log = b''
        while b'The server is now ready to accept connections ' not in log:
            log = proc.stdout.readline()

        info = RedisServer(name, tcp_address, unixsocket, version)
        servers.setdefault(name, info)
        return info

    yield maker

    while processes:
        proc = processes.pop(0)
        proc.terminate()
        proc.wait()

@pytest.yield_fixture(scope='session')
def ssl_proxy(request, unused_port):
    processes = []
    by_port = {}

    cafile = os.path.abspath(request.config.getoption('--ssl-cafile'))
    pemfile = os.path.splitext(cafile)[0] + '.pem'
    assert os.path.exists(cafile), \
        "No test ssl certificate, run `make certificate`"
    assert os.path.exists(pemfile), \
        "No test ssl certificate, run `make certificate`"

    if hasattr(ssl, 'create_default_context'):
        ssl_ctx = ssl.create_default_context(cafile=cafile)
    else:
        ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
        ssl_ctx.load_verify_locations(cafile=cafile)
    if hasattr(ssl_ctx, 'check_hostname'):
        # available since python 3.4
        ssl_ctx.check_hostname = False
    ssl_ctx.verify_mode = ssl.CERT_NONE

    def sockat(unsecure_port):
        if unsecure_port in by_port:
            return by_port[unsecure_port]

        secure_port = unused_port()
        proc = subprocess.Popen(['/usr/bin/socat',
                                 'openssl-listen:{},'
                                 'cert={},'
                                 'verify=0,fork'
                                 .format(secure_port, pemfile),
                                 'tcp-connect:localhost:{}'
                                 .format(unsecure_port)
                                 ])
        processes.append(proc)
        time.sleep(1)   # XXX
        by_port[unsecure_port] = secure_port, ssl_ctx
        return secure_port, ssl_ctx

    yield sockat

    while processes:
        proc = processes.pop(0)
        proc.terminate()
        proc.wait()

@pytest.mark.tryfirst
def pytest_pycollect_makeitem(collector, name, obj):
    if collector.funcnamefilter(name):
        if not callable(obj):
            return
        item = pytest.Function(name, parent=collector)
        if 'run_loop' in item.keywords:
            # TODO: re-wrap with asyncio.coroutine if not native coroutine
            return list(collector._genfunctions(name, obj))

@pytest.mark.tryfirst
def pytest_pyfunc_call(pyfuncitem):
    """
    Run asyncio marked test functions in an event loop instead of a normal
    function call.
    """
    if 'run_loop' in pyfuncitem.keywords:
        funcargs = pyfuncitem.funcargs
        loop = funcargs['loop']
        testargs = {arg: funcargs[arg]
                    for arg in pyfuncitem._fixtureinfo.argnames}
        loop.run_until_complete(
            asyncio.wait_for(pyfuncitem.obj(**testargs),
                             15, loop=loop))
        return True

def pytest_runtest_setup(item):
    if 'run_loop' in item.keywords and 'loop' not in item.fixturenames:
        # inject an event loop fixture for all async tests
        item.fixturenames.append('loop')

def pytest_ignore_collect(path, config):
    if 'py35' in str(path):
        if sys.version_info < (3, 5, 0):
            return True

def pytest_collection_modifyitems(session, config, items):
    version = _read_server_version(config)
    for item in items:
        if 'redis_version' in item.keywords:
            marker = item.keywords['redis_version']
            if version < marker.kwargs['version']:
                item.add_marker(pytest.mark.skip(
                    reason=marker.kwargs['reason']))
        if 'ssl_proxy' in item.fixturenames:
            item.add_marker(pytest.mark.skipif(
                "not os.path.exists('/usr/bin/socat')",
                reason="socat package required (apt-get install socat)"))

def pytest_configure(config):
    if config.getoption('--uvloop'):
        try:
            import uvloop
        except ImportError:
            raise RuntimeError(
                "Can not import uvloop, make sure it is installed")
        asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())

@contextlib.contextmanager
def raises_regex(exc_type, message):
    with pytest.raises(exc_type) as exc_info:
        yield exc_info
    match = re.search(message, str(exc_info.value))
    assert match is not None, (
        "Pattern {!r} does not match {!r}"
        .format(message, str(exc_info.value)))

def logs(logger, level=None):
    """Catches logs for given logger and level.

    See unittest.TestCase.assertLogs for details.
    """
    return _AssertLogsContext(None, logger, level)

if sys.version_info >= (3, 4):
    class _AssertLogsContext(unittest.case._AssertLogsContext):
        """Standard unittest's _AssertLogsContext context manager
        adopted to raise pytest failure.
        """

        def _raiseFailure(self, standardMsg):
            pytest.fail(standardMsg)
else:
    def _AssertLogsContext(self, test, logger, level):
        raise NotImplementedError("Available since Python3.4")

def redis_version(*version, reason):
    assert 1 < len(version) <= 3, version
    assert all(isinstance(v, int) for v in version), version
    return pytest.mark.redis_version(version=version, reason=reason)

def assert_almost_equal(first, second, places=None, msg=None, delta=None):
    assert not (places is None and delta is None)
    if delta is not None:
        assert abs(first - second) <= delta
    else:
        assert round(abs(first - second), places) == 0

def pytest_namespace():
    return {
        'raises_regex': raises_regex,
        'assert_almost_equal': assert_almost_equal,
        'redis_version': redis_version,
        'logs': logs,
        }
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Base classes for our unit tests.
Allows overriding of flags for use of fakes,
and some black magic for inline callbacks.
"""

import datetime
import functools
import os
import shutil
import uuid
import unittest

import mox
import shutil
import stubout
from eventlet import greenpool
from eventlet import greenthread

from nova import context
from nova import db
from nova import fakerabbit
from nova import flags
from nova import rpc
from nova import service
from nova import wsgi

FLAGS = flags.FLAGS
flags.DEFINE_string('sqlite_clean_db', 'clean.sqlite',
                    'File name of clean sqlite db')
flags.DEFINE_bool('fake_tests', True,
                  'should we use everything for testing')

def skip_if_fake(func):
    """Decorator that skips a test if running in fake mode"""
    def _skipper(*args, **kw):
        """Wrapped skipper function"""
        if FLAGS.fake_tests:
            raise unittest.SkipTest('Test cannot be run in fake mode')
        else:
            return func(*args, **kw)
    return _skipper

class TestCase(unittest.TestCase):
    """Test case base class for all unit tests"""
    def setUp(self):
        """Run before each test method to initialize test environment"""
        super(TestCase, self).setUp()
        # NOTE(vish): We need a better method for creating fixtures for tests
        #             now that we have some required db setup for the system
        #             to work properly.
        self.start = datetime.datetime.utcnow()
        shutil.copyfile(os.path.join(FLAGS.state_path, FLAGS.sqlite_clean_db),
                        os.path.join(FLAGS.state_path, FLAGS.sqlite_db))

        # emulate some of the mox stuff, we can't use the metaclass
        # because it screws with our generators
        self.mox = mox.Mox()
        self.stubs = stubout.StubOutForTesting()
        self.flag_overrides = {}
        self.injected = []
        self._services = []
        self._monkey_patch_attach()
        self._monkey_patch_wsgi()
        self._original_flags = FLAGS.FlagValuesDict()

    def tearDown(self):
        """Runs after each test method to finalize/tear down test
        environment."""
        try:
            self.mox.UnsetStubs()
            self.stubs.UnsetAll()
            self.stubs.SmartUnsetAll()
            self.mox.VerifyAll()
            super(TestCase, self).tearDown()
        finally:
            # Clean out fake_rabbit's queue if we used it
            if FLAGS.fake_rabbit:
                fakerabbit.reset_all()

            # Reset any overriden flags
            self.reset_flags()

            # Reset our monkey-patches
            rpc.Consumer.attach_to_eventlet = self.original_attach
            wsgi.Server.start = self.original_start

            # Stop any timers
            for x in self.injected:
                try:
                    x.stop()
                except AssertionError:
                    pass

            # Kill any services
            for x in self._services:
                try:
                    x.kill()
                except Exception:
                    pass

    def flags(self, **kw):
        """Override flag variables for a test"""
        for k, v in kw.iteritems():
            if k in self.flag_overrides:
                self.reset_flags()
                raise Exception(
                        'trying to override already overriden flag: %s' % k)
            self.flag_overrides[k] = getattr(FLAGS, k)
            setattr(FLAGS, k, v)

    def reset_flags(self):
        """Resets all flag variables for the test.  Runs after each test"""
        FLAGS.Reset()
        for k, v in self._original_flags.iteritems():
            setattr(FLAGS, k, v)

    def start_service(self, name, host=None, **kwargs):
        host = host and host or uuid.uuid4().hex
        kwargs.setdefault('host', host)
        kwargs.setdefault('binary', 'nova-%s' % name)
        svc = service.Service.create(**kwargs)
        svc.start()
        self._services.append(svc)
        return svc

    def _monkey_patch_attach(self):
        self.original_attach = rpc.Consumer.attach_to_eventlet

        def _wrapped(inner_self):
            rv = self.original_attach(inner_self)
            self.injected.append(rv)
            return rv

        _wrapped.func_name = self.original_attach.func_name
        rpc.Consumer.attach_to_eventlet = _wrapped

    def _monkey_patch_wsgi(self):
        """Allow us to kill servers spawned by wsgi.Server."""
        # TODO(termie): change these patterns to use functools
        self.original_start = wsgi.Server.start

        @functools.wraps(self.original_start)
        def _wrapped_start(inner_self, *args, **kwargs):
            original_spawn_n = inner_self.pool.spawn_n

            @functools.wraps(original_spawn_n)
            def _wrapped_spawn_n(*args, **kwargs):
                rv = greenthread.spawn(*args, **kwargs)
                self._services.append(rv)

            inner_self.pool.spawn_n = _wrapped_spawn_n
            self.original_start(inner_self, *args, **kwargs)
            inner_self.pool.spawn_n = original_spawn_n

        _wrapped_start.func_name = self.original_start.func_name
        wsgi.Server.start = _wrapped_start

    # Useful assertions
    def assertDictMatch(self, d1, d2):
        """Assert two dicts are equivalent.

        This is a 'deep' match in the sense that it handles nested
        dictionaries appropriately.

        NOTE:

            If you don't care (or don't know) a given value, you can specify
            the string DONTCARE as the value. This will cause that dict-item
            to be skipped.
        """
        def raise_assertion(msg):
            d1str = str(d1)
            d2str = str(d2)
            base_msg = ("Dictionaries do not match. %(msg)s d1: %(d1str)s "
                        "d2: %(d2str)s" % locals())
            raise AssertionError(base_msg)

        d1keys = set(d1.keys())
        d2keys = set(d2.keys())
        if d1keys != d2keys:
            d1only = d1keys - d2keys
            d2only = d2keys - d1keys
            raise_assertion("Keys in d1 and not d2: %(d1only)s. "
                            "Keys in d2 and not d1: %(d2only)s" % locals())

        for key in d1keys:
            d1value = d1[key]
            d2value = d2[key]
            if hasattr(d1value, 'keys') and hasattr(d2value, 'keys'):
                self.assertDictMatch(d1value, d2value)
            elif 'DONTCARE' in (d1value, d2value):
                continue
            elif d1value != d2value:
                raise_assertion("d1['%(key)s']=%(d1value)s != "
                                "d2['%(key)s']=%(d2value)s" % locals())

    def assertDictListMatch(self, L1, L2):
        """Assert a list of dicts are equivalent"""
        def raise_assertion(msg):
            L1str = str(L1)
            L2str = str(L2)
            base_msg = ("List of dictionaries do not match: %(msg)s "
                        "L1: %(L1str)s L2: %(L2str)s" % locals())
            raise AssertionError(base_msg)

        L1count = len(L1)
        L2count = len(L2)
        if L1count != L2count:
            raise_assertion("Length mismatch: len(L1)=%(L1count)d != "
                            "len(L2)=%(L2count)d" % locals())

        for d1, d2 in zip(L1, L2):
            self.assertDictMatch(d1, d2)
<commit_msg>port the objectstore tests to the new tests<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Base classes for our unit tests.
Allows overriding of flags for use of fakes,
and some black magic for inline callbacks.
"""

import datetime
import functools
import os
import shutil
import uuid
import unittest

import mox
import shutil
import stubout
from eventlet import greenthread

from nova import context
from nova import db
from nova import fakerabbit
from nova import flags
from nova import rpc
from nova import service
from nova import wsgi

FLAGS = flags.FLAGS
flags.DEFINE_string('sqlite_clean_db', 'clean.sqlite',
                    'File name of clean sqlite db')
flags.DEFINE_bool('fake_tests', True,
                  'should we use everything for testing')

def skip_if_fake(func):
    """Decorator that skips a test if running in fake mode"""
    def _skipper(*args, **kw):
        """Wrapped skipper function"""
        if FLAGS.fake_tests:
            raise unittest.SkipTest('Test cannot be run in fake mode')
        else:
            return func(*args, **kw)
    return _skipper

class TestCase(unittest.TestCase):
    """Test case base class for all unit tests"""
    def setUp(self):
        """Run before each test method to initialize test environment"""
        super(TestCase, self).setUp()
        # NOTE(vish): We need a better method for creating fixtures for tests
        #             now that we have some required db setup for the system
        #             to work properly.
        self.start = datetime.datetime.utcnow()
        shutil.copyfile(os.path.join(FLAGS.state_path, FLAGS.sqlite_clean_db),
                        os.path.join(FLAGS.state_path, FLAGS.sqlite_db))

        # emulate some of the mox stuff, we can't use the metaclass
        # because it screws with our generators
        self.mox = mox.Mox()
        self.stubs = stubout.StubOutForTesting()
        self.flag_overrides = {}
        self.injected = []
        self._services = []
        self._monkey_patch_attach()
        self._monkey_patch_wsgi()
        self._original_flags = FLAGS.FlagValuesDict()

    def tearDown(self):
        """Runs after each test method to finalize/tear down test
        environment."""
        try:
            self.mox.UnsetStubs()
            self.stubs.UnsetAll()
            self.stubs.SmartUnsetAll()
            self.mox.VerifyAll()
            super(TestCase, self).tearDown()
        finally:
            # Clean out fake_rabbit's queue if we used it
            if FLAGS.fake_rabbit:
                fakerabbit.reset_all()

            # Reset any overriden flags
            self.reset_flags()

            # Reset our monkey-patches
            rpc.Consumer.attach_to_eventlet = self.original_attach
            wsgi.Server.start = self.original_start

            # Stop any timers
            for x in self.injected:
                try:
                    x.stop()
                except AssertionError:
                    pass

            # Kill any services
            for x in self._services:
                try:
                    x.kill()
                except Exception:
                    pass

    def flags(self, **kw):
        """Override flag variables for a test"""
        for k, v in kw.iteritems():
            if k in self.flag_overrides:
                self.reset_flags()
                raise Exception(
                        'trying to override already overriden flag: %s' % k)
            self.flag_overrides[k] = getattr(FLAGS, k)
            setattr(FLAGS, k, v)

    def reset_flags(self):
        """Resets all flag variables for the test.  Runs after each test"""
        FLAGS.Reset()
        for k, v in self._original_flags.iteritems():
            setattr(FLAGS, k, v)

    def start_service(self, name, host=None, **kwargs):
        host = host and host or uuid.uuid4().hex
        kwargs.setdefault('host', host)
        kwargs.setdefault('binary', 'nova-%s' % name)
        svc = service.Service.create(**kwargs)
        svc.start()
        self._services.append(svc)
        return svc

    def _monkey_patch_attach(self):
        self.original_attach = rpc.Consumer.attach_to_eventlet

        def _wrapped(inner_self):
            rv = self.original_attach(inner_self)
            self.injected.append(rv)
            return rv

        _wrapped.func_name = self.original_attach.func_name
        rpc.Consumer.attach_to_eventlet = _wrapped

    def _monkey_patch_wsgi(self):
        """Allow us to kill servers spawned by wsgi.Server."""
        # TODO(termie): change these patterns to use functools
        self.original_start = wsgi.Server.start

        @functools.wraps(self.original_start)
        def _wrapped_start(inner_self, *args, **kwargs):
            original_spawn_n = inner_self.pool.spawn_n

            @functools.wraps(original_spawn_n)
            def _wrapped_spawn_n(*args, **kwargs):
                rv = greenthread.spawn(*args, **kwargs)
                self._services.append(rv)

            inner_self.pool.spawn_n = _wrapped_spawn_n
            self.original_start(inner_self, *args, **kwargs)
            inner_self.pool.spawn_n = original_spawn_n

        _wrapped_start.func_name = self.original_start.func_name
        wsgi.Server.start = _wrapped_start

    # Useful assertions
    def assertDictMatch(self, d1, d2):
        """Assert two dicts are equivalent.

        This is a 'deep' match in the sense that it handles nested
        dictionaries appropriately.

        NOTE:

            If you don't care (or don't know) a given value, you can specify
            the string DONTCARE as the value. This will cause that dict-item
            to be skipped.
        """
        def raise_assertion(msg):
            d1str = str(d1)
            d2str = str(d2)
            base_msg = ("Dictionaries do not match. %(msg)s d1: %(d1str)s "
                        "d2: %(d2str)s" % locals())
            raise AssertionError(base_msg)

        d1keys = set(d1.keys())
        d2keys = set(d2.keys())
        if d1keys != d2keys:
            d1only = d1keys - d2keys
            d2only = d2keys - d1keys
            raise_assertion("Keys in d1 and not d2: %(d1only)s. "
                            "Keys in d2 and not d1: %(d2only)s" % locals())

        for key in d1keys:
            d1value = d1[key]
            d2value = d2[key]
            if hasattr(d1value, 'keys') and hasattr(d2value, 'keys'):
                self.assertDictMatch(d1value, d2value)
            elif 'DONTCARE' in (d1value, d2value):
                continue
            elif d1value != d2value:
                raise_assertion("d1['%(key)s']=%(d1value)s != "
                                "d2['%(key)s']=%(d2value)s" % locals())

    def assertDictListMatch(self, L1, L2):
        """Assert a list of dicts are equivalent"""
        def raise_assertion(msg):
            L1str = str(L1)
            L2str = str(L2)
            base_msg = ("List of dictionaries do not match: %(msg)s "
                        "L1: %(L1str)s L2: %(L2str)s" % locals())
            raise AssertionError(base_msg)

        L1count = len(L1)
        L2count = len(L2)
        if L1count != L2count:
            raise_assertion("Length mismatch: len(L1)=%(L1count)d != "
                            "len(L2)=%(L2count)d" % locals())

        for d1, d2 in zip(L1, L2):
            self.assertDictMatch(d1, d2)
<|endoftext|>