<commit_before>"""Parallel workflow execution via SGE
"""

import os
import sys

from .base import (GraphPluginBase, logger)

from ...interfaces.base import CommandLine

def node_completed_status( checknode):
    """
    A function to determine if a node has previously completed it's work
    :param checknode: The node to check the run status
    :return: boolean value True indicates that the node does not need to be run.
    """
    """ TODO: place this in the base.py file and refactor """
    node_state_does_not_require_overwrite = ( checknode.overwrite == False or
                       (checknode.overwrite == None and
                        not checknode._interface.always_run )
    )
    hash_exists = False
    try:
        hash_exists, _, _, _ = checknode.hash_exists()
    except Exception:
        hash_exists = False
    return (hash_exists and node_state_does_not_require_overwrite )

class SLURMGraphPlugin(GraphPluginBase):
    """Execute using SGE

    The plugin_args input to run can be used to control the SGE execution.
    Currently supported options are:

    - template : template to use for batch job submission
    - qsub_args : arguments to be prepended to the job execution script in the
                  qsub call

    """
    _template = """
#!/bin/bash
"""

    def __init__(self, **kwargs):
        self._qsub_args = ''
        if 'plugin_args' in kwargs and kwargs['plugin_args']:
            if 'retry_timeout' in kwargs['plugin_args']:
                self._retry_timeout = kwargs['plugin_args']['retry_timeout']
            if  'max_tries' in kwargs['plugin_args']:
                self._max_tries = kwargs['plugin_args']['max_tries']
            if 'template' in kwargs['plugin_args']:
                self._template = kwargs['plugin_args']['template']
                if os.path.isfile(self._template):
                    self._template = open(self._template).read()
            if 'sbatch_args' in kwargs['plugin_args']:
                self._sbatch_args = kwargs['plugin_args']['sbatch_args']
            if 'dont_resubmit_completed_jobs' in kwargs['plugin_args']:
                self._dont_resubmit_completed_jobs = plugin_args['dont_resubmit_completed_jobs']
            else:
                self._dont_resubmit_completed_jobs = False
        super(SGEGraphPlugin, self).__init__(**kwargs)

    def _submit_graph(self, pyfiles, dependencies, nodes):
        def make_job_name(jobnumber, nodeslist):
            """
            - jobnumber: The index number of the job to create
            - nodeslist: The name of the node being processed
            - return: A string representing this job to be displayed by SGE
            """
            job_name='j{0}_{1}'.format(jobnumber, nodeslist[jobnumber]._id)
            # Condition job_name to be a valid bash identifier (i.e. - is invalid)
            job_name=job_name.replace('-','_').replace('.','_').replace(':','_')
            return job_name
        batch_dir, _ = os.path.split(pyfiles[0])
        submitjobsfile = os.path.join(batch_dir, 'submit_jobs.sh')

        cache_doneness_per_node = dict()
        if self._dont_resubmit_completed_jobs: ## A future parameter for controlling this behavior could be added here
            for idx, pyscript in enumerate(pyfiles):
                node = nodes[idx]
                node_status_done = node_completed_status(node)
                ## If a node has no dependencies, and it is requested to run_without_submitting
                ## then run this node in place
                if (not node_status_done) and (len(dependencies[idx]) == 0 ) and (node.run_without_submitting == True):
                    try:
                        node.run()
                    except Exception:
                        node._clean_queue(idx, nodes)
                    node_status_done = True # if successfully run locally, then claim true

                #if the node itself claims done, then check to ensure all
                #dependancies are also done
                if node_status_done and idx in dependencies:
                    for child_idx in dependencies[idx]:
                        if child_idx in cache_doneness_per_node:
                            child_status_done = cache_doneness_per_node[child_idx]
                        else:
                            child_status_done = node_completed_status(nodes[child_idx])
                        node_status_done = node_status_done and child_status_done

                cache_doneness_per_node[idx] = node_status_done

        with open(submitjobsfile, 'wt') as fp:
            fp.writelines('#!/usr/bin/env bash\n')
            fp.writelines('# Condense format attempted\n')
            for idx, pyscript in enumerate(pyfiles):
                node = nodes[idx]
                if cache_doneness_per_node.get(idx,False):
                    continue
                else:
                    template, sbatch_args = self._get_args(
                        node, ["template", "sbatch_args"])

                    batch_dir, name = os.path.split(pyscript)
                    name = '.'.join(name.split('.')[:-1])
                    batchscript = '\n'.join((template,
                                             '%s %s' % (sys.executable, pyscript)))
                    batchscriptfile = os.path.join(batch_dir,
                                                   'batchscript_%s.sh' % name)

                    batchscriptoutfile = batchscriptfile + '.o'
                    batchscripterrfile = batchscriptfile + '.e'

                    with open(batchscriptfile, 'wt') as batchfp:
                        batchfp.writelines(batchscript)
                        batchfp.close()
                    deps = ''
                    if idx in dependencies:
                        values = ' '
                        for jobid in dependencies[idx]:
                            ## Avoid dependancies of done jobs
                            if cache_doneness_per_node[jobid] == False:
                                values += "${{{0}}}:".format(make_job_name(jobid, nodes))
                        if values != ' ': # i.e. if some jobs were added to dependency list
                            values = values.rstrip(',')
                            deps = '--dependency=afterok:%s' % values
                    jobname = make_job_name(idx, nodes)
                    # Do not use default output locations if they are set in self._sbatch_args
                    stderrFile = ''
                    if self._sbatch_args.count('-e ') == 0:
                        stderrFile = '-e {errFile}'.format(
                            errFile=batchscripterrfile)
                    stdoutFile = ''
                    if self._sbatch_args.count('-o ') == 0:
                        stdoutFile = '-o {outFile}'.format(
                            outFile=batchscriptoutfile)
                    full_line = '{jobNm}=$(sbatch {outFileOption} {errFileOption} {extraSBatchArgs} {dependantIndex} -J {jobNm} {batchscript} | awk \'{{print $4}}\')\n'.format(
                        jobNm=jobname,
                        outFileOption=stdoutFile,
                        errFileOption=stderrFile,
                        extraSBatchArgs=sbatch_args,
                        dependantIndex=deps,
                        batchscript=batchscriptfile)
                    fp.writelines(full_line)
        cmd = CommandLine('bash', environ=os.environ.data,
                          terminal_output='allatonce')
        cmd.inputs.args = '%s' % submitjobsfile
        cmd.run()
        logger.info('submitted all jobs to queue')
<commit_msg>remove SGE remnants<commit_after>"""Parallel workflow execution via SLURM
"""

import os
import sys

from .base import (GraphPluginBase, logger)

from ...interfaces.base import CommandLine

def node_completed_status( checknode):
    """
    A function to determine if a node has previously completed it's work
    :param checknode: The node to check the run status
    :return: boolean value True indicates that the node does not need to be run.
    """
    """ TODO: place this in the base.py file and refactor """
    node_state_does_not_require_overwrite = ( checknode.overwrite == False or
                       (checknode.overwrite == None and
                        not checknode._interface.always_run )
    )
    hash_exists = False
    try:
        hash_exists, _, _, _ = checknode.hash_exists()
    except Exception:
        hash_exists = False
    return (hash_exists and node_state_does_not_require_overwrite )

class SLURMGraphPlugin(GraphPluginBase):
    """Execute using SLURM

    The plugin_args input to run can be used to control the SGE execution.
    Currently supported options are:

    - template : template to use for batch job submission
    - qsub_args : arguments to be prepended to the job execution script in the
                  qsub call

    """
    _template = """
#!/bin/bash
"""

    def __init__(self, **kwargs):
        self._qsub_args = ''
        if 'plugin_args' in kwargs and kwargs['plugin_args']:
            if 'retry_timeout' in kwargs['plugin_args']:
                self._retry_timeout = kwargs['plugin_args']['retry_timeout']
            if  'max_tries' in kwargs['plugin_args']:
                self._max_tries = kwargs['plugin_args']['max_tries']
            if 'template' in kwargs['plugin_args']:
                self._template = kwargs['plugin_args']['template']
                if os.path.isfile(self._template):
                    self._template = open(self._template).read()
            if 'sbatch_args' in kwargs['plugin_args']:
                self._sbatch_args = kwargs['plugin_args']['sbatch_args']
            if 'dont_resubmit_completed_jobs' in kwargs['plugin_args']:
                self._dont_resubmit_completed_jobs = plugin_args['dont_resubmit_completed_jobs']
            else:
                self._dont_resubmit_completed_jobs = False
        super(SLURMGraphPlugin, self).__init__(**kwargs)

    def _submit_graph(self, pyfiles, dependencies, nodes):
        def make_job_name(jobnumber, nodeslist):
            """
            - jobnumber: The index number of the job to create
            - nodeslist: The name of the node being processed
            - return: A string representing this job to be displayed by SLURM
            """
            job_name='j{0}_{1}'.format(jobnumber, nodeslist[jobnumber]._id)
            # Condition job_name to be a valid bash identifier (i.e. - is invalid)
            job_name=job_name.replace('-','_').replace('.','_').replace(':','_')
            return job_name
        batch_dir, _ = os.path.split(pyfiles[0])
        submitjobsfile = os.path.join(batch_dir, 'submit_jobs.sh')

        cache_doneness_per_node = dict()
        if self._dont_resubmit_completed_jobs: ## A future parameter for controlling this behavior could be added here
            for idx, pyscript in enumerate(pyfiles):
                node = nodes[idx]
                node_status_done = node_completed_status(node)
                ## If a node has no dependencies, and it is requested to run_without_submitting
                ## then run this node in place
                if (not node_status_done) and (len(dependencies[idx]) == 0 ) and (node.run_without_submitting == True):
                    try:
                        node.run()
                    except Exception:
                        node._clean_queue(idx, nodes)
                    node_status_done = True # if successfully run locally, then claim true

                #if the node itself claims done, then check to ensure all
                #dependancies are also done
                if node_status_done and idx in dependencies:
                    for child_idx in dependencies[idx]:
                        if child_idx in cache_doneness_per_node:
                            child_status_done = cache_doneness_per_node[child_idx]
                        else:
                            child_status_done = node_completed_status(nodes[child_idx])
                        node_status_done = node_status_done and child_status_done

                cache_doneness_per_node[idx] = node_status_done

        with open(submitjobsfile, 'wt') as fp:
            fp.writelines('#!/usr/bin/env bash\n')
            fp.writelines('# Condense format attempted\n')
            for idx, pyscript in enumerate(pyfiles):
                node = nodes[idx]
                if cache_doneness_per_node.get(idx,False):
                    continue
                else:
                    template, sbatch_args = self._get_args(
                        node, ["template", "sbatch_args"])

                    batch_dir, name = os.path.split(pyscript)
                    name = '.'.join(name.split('.')[:-1])
                    batchscript = '\n'.join((template,
                                             '%s %s' % (sys.executable, pyscript)))
                    batchscriptfile = os.path.join(batch_dir,
                                                   'batchscript_%s.sh' % name)

                    batchscriptoutfile = batchscriptfile + '.o'
                    batchscripterrfile = batchscriptfile + '.e'

                    with open(batchscriptfile, 'wt') as batchfp:
                        batchfp.writelines(batchscript)
                        batchfp.close()
                    deps = ''
                    if idx in dependencies:
                        values = ' '
                        for jobid in dependencies[idx]:
                            ## Avoid dependancies of done jobs
                            if cache_doneness_per_node[jobid] == False:
                                values += "${{{0}}}:".format(make_job_name(jobid, nodes))
                        if values != ' ': # i.e. if some jobs were added to dependency list
                            values = values.rstrip(',')
                            deps = '--dependency=afterok:%s' % values
                    jobname = make_job_name(idx, nodes)
                    # Do not use default output locations if they are set in self._sbatch_args
                    stderrFile = ''
                    if self._sbatch_args.count('-e ') == 0:
                        stderrFile = '-e {errFile}'.format(
                            errFile=batchscripterrfile)
                    stdoutFile = ''
                    if self._sbatch_args.count('-o ') == 0:
                        stdoutFile = '-o {outFile}'.format(
                            outFile=batchscriptoutfile)
                    full_line = '{jobNm}=$(sbatch {outFileOption} {errFileOption} {extraSBatchArgs} {dependantIndex} -J {jobNm} {batchscript} | awk \'{{print $4}}\')\n'.format(
                        jobNm=jobname,
                        outFileOption=stdoutFile,
                        errFileOption=stderrFile,
                        extraSBatchArgs=sbatch_args,
                        dependantIndex=deps,
                        batchscript=batchscriptfile)
                    fp.writelines(full_line)
        cmd = CommandLine('bash', environ=os.environ.data,
                          terminal_output='allatonce')
        cmd.inputs.args = '%s' % submitjobsfile
        cmd.run()
        logger.info('submitted all jobs to queue')
<|endoftext|>


<commit_before>""" Menu, oor no? """

# pylint: disable=too-few-public-methods,line-too-long

from flask import Flask, render_template, request
import wasthereannhlgamelastnight
import NHLHelpers

class MainPage(webapp2.RequestHandler):
    """ Menu Page Class"""

    def get(self):
        """Return a menu where one can choose a team

        """

        self.response.headers["Content-Type"] = "text/html"
        self.response.write(
            '<!DOCTYPE html>\n\
    \n\
    <title>Which NHL team do you choose?</title>\n'
        )
        self.response.write(wasthereannhlgamelastnight.COMMON_META)
        self.response.write(
            '<link href="/stylesheets/menu.css" rel="stylesheet">\n\
        <link type="text/css" href="/menu_team.css" rel="stylesheet">\n\
        \n\
        \n\
    \n\
        \n\

'
        )
        # Loop through and write all the teams like:
        allteams = self.get_teams()
        for ateam in allteams:
            # https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Grid_Layout/Auto-placement_in_CSS_Grid_Layout
            # https://css-tricks.com/snippets/jquery/make-entire-div-clickable/
            longteamname = NHLHelpers.get_team(ateam)
            # https://davidwalsh.name/html5-storage
            # Note the use of %r instead of %s in the onClick to have it print 'DET' instead of " det" ..
            # Used to store the chosen team in a local browser variable
            self.response.write(
                '
%s
'
                % (longteamname.replace(" ", ""), ateam, longteamname, ateam, ateam)
            )
        self.response.write(
            """
Black BG Color
"""
        )
        self.response.write(
            """
Clear Team
& Color Selection
"""
        )
        self.response.write(
            """
Get Schedule

"""
        )
        self.response.write(
            """
Last Update
of Schedule

"""
        )
        self.response.write(
            """
Update Schedule
(admin)

"""
        )
        self.response.write("
\n")
        self.response.write(wasthereannhlgamelastnight.DISCLAIMER)

        self.response.write(
            """

        """
        )

    @staticmethod
    def get_teams():
        """ Return a sorted list of teams """
        allteams = sorted(list(NHLHelpers.get_all_teams().keys()))
        return allteams

APPLICATION = webapp2.WSGIApplication([("/.*", MainPage)], debug=True)

########

class CSSPage(webapp2.RequestHandler):
    """ CSS Page Class """

    def get(self):
        """Return CSS used to color boxes """

        self.response.headers["Content-Type"] = "text/css"
        # Loop through and write all the teams:
        allteams = sorted(list(NHLHelpers.get_all_teams().keys()))
        # Recreate give_me_a_color classmethod because I couldn't figure out how to call it
        colordict = {}
        # If we use
        # https://raw.githubusercontent.com/jimniels/teamcolors/master/static/data/teams.json
        # we would need to pick which of the colors to show. Sometimes it's 3rd, 2nd, first...
        for ateam in allteams:
            # Loop through colors and don't pick black as background for the box
            colors = NHLHelpers.get_team_colors(ateam)
            backgroundcolor = colors[0]
            try:
                backgroundcolor2 = colors[1]
            except IndexError:
                backgroundcolor2 = colors[0]
            if backgroundcolor == "000000":
                backgroundcolor = backgroundcolor2
            colordict[ateam] = backgroundcolor
        # Make CSS
        # Default font color is black.
        # With some backgrounds black is not so readable so we change it to white.
        # https://en.wikipedia.org/wiki/Template:NHL_team_color might be good, it talks about contrast at least..
        whitetext = [
            "ARI",
            "BUF",
            "CBJ",
            "DET",
            "EDM",
            "NSH",
            "NYI",
            "NYR",
            "TBL",
            "TOR",
            "VAN",
            "WPG",
        ]
        yellowtext = ["STL"]
        for ateam in allteams:
            # https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Grid_Layout/Auto-placement_in_CSS_Grid_Layout
            self.response.write(
                ".wrapper > a.%s { \n\
                background-color: #%s; \n"
                % (ateam, colordict[ateam])
            )
            if ateam in whitetext:
                self.response.write("\t\tcolor: white \n }\n")
            elif ateam in yellowtext:
                self.response.write("\t\tcolor: yellow \n }\n")
            else:
                self.response.write("}\n")

########

MENU_CSS = webapp2.WSGIApplication([("/.*", CSSPage)], debug=True)
<commit_msg>remove menu.py - it's in main.py now<commit_after><|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010-2011 OpenStack LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from nova import utils
from nova.api.openstack import common

class ViewBuilder(object):
    ''' Models a server addresses response as a python dictionary.'''

    def build(self, inst):
        raise NotImplementedError()

class ViewBuilderV10(ViewBuilder):
    def build(self, inst):
        private_ips = self.build_private_parts(inst)
        public_ips = self.build_public_parts(inst)
        return dict(public=public_ips, private=private_ips)

    def build_public_parts(self, inst):
        return utils.get_from_path(inst, 'fixed_ip/floating_ips/address')

    def build_private_parts(self, inst):
        return utils.get_from_path(inst, 'fixed_ip/address')

class ViewBuilderV11(ViewBuilder):
    def build(self, inst):
        private_ips = utils.get_from_path(inst, 'fixed_ips/address')
        private_ips = [dict(version=4, addr=a) for a in private_ips]
        public_ips = utils.get_from_path(inst,
                                         'fixed_ips/floating_ips/address')
        public_ips = [dict(version=4, addr=a) for a in public_ips]
        return dict(public=public_ips, private=private_ips)
<commit_msg>updated osapi 1.0 addresses view to work with multiple fixed ips<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010-2011 OpenStack LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from nova import utils
from nova.api.openstack import common

class ViewBuilder(object):
    ''' Models a server addresses response as a python dictionary.'''

    def build(self, inst):
        raise NotImplementedError()

class ViewBuilderV10(ViewBuilder):
    def build(self, inst):
        private_ips = self.build_private_parts(inst)
        public_ips = self.build_public_parts(inst)
        return dict(public=public_ips, private=private_ips)

    def build_public_parts(self, inst):
        return utils.get_from_path(inst, 'fixed_ips/floating_ips/address')

    def build_private_parts(self, inst):
        return utils.get_from_path(inst, 'fixed_ips/address')

class ViewBuilderV11(ViewBuilder):
    def build(self, inst):
        # TODO(tr3buchet) - this shouldn't be hard coded to 4...
        private_ips = utils.get_from_path(inst, 'fixed_ips/address')
        private_ips = [dict(version=4, addr=a) for a in private_ips]
        public_ips = utils.get_from_path(inst,
                                         'fixed_ips/floating_ips/address')
        public_ips = [dict(version=4, addr=a) for a in public_ips]
        return dict(public=public_ips, private=private_ips)
<|endoftext|>


<commit_before># Python Version: 3.x
__author__  = 'Kimiyuki Onaka'
__email__ = 'kimiyuki95@gmail.com'
__license__ = 'MIT License'
__url__ = 'https://github.com/kmyk/online-judge-tools'
__version_info__ = ( 0, 1, 34, 'final', 0 )
__version__ = '.'.join(map(str, __version_info__[: 3]))
<commit_msg>RELEASE: 0.1.35<commit_after># Python Version: 3.x
__author__  = 'Kimiyuki Onaka'
__email__ = 'kimiyuki95@gmail.com'
__license__ = 'MIT License'
__url__ = 'https://github.com/kmyk/online-judge-tools'
__version_info__ = ( 0, 1, 35, 'final', 0 )
__version__ = '.'.join(map(str, __version_info__[: 3]))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-TODAY OpenERP S.A. <http://www.openerp.com>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import time

import openerp

class ir_sequence_type(openerp.osv.osv.osv):
    _name = 'ir.sequence.type'
    _order = 'name'
    _columns = {
        'name': openerp.osv.fields.char('Name', size=64, required=True),
        'code': openerp.osv.fields.char('Code', size=32, required=True),
    }

    _sql_constraints = [
        ('code_unique', 'unique(code)', '`code` must be unique.'),
    ]

def _code_get(self, cr, uid, context={}):
    cr.execute('select code, name from ir_sequence_type')
    return cr.fetchall()

IMPLEMENTATION_SELECTION = \
    [('standard', 'Standard'), ('no_gap', 'No gap')]

class ir_sequence(openerp.osv.osv.osv):
    """ Sequence model.

    The sequence model allows to define and use so-called sequence objects.
    Such objects are used to generate unique identifiers in a transaction-safe
    way.

    """
    _name = 'ir.sequence'
    _order = 'name'
    _columns = {
        'name': openerp.osv.fields.char('Name', size=64, required=True),
        'code': openerp.osv.fields.selection(_code_get, 'Code', size=64, required=True), # TODO should it be unique?
        'implementation': openerp.osv.fields.selection( # TODO update the view
            IMPLEMENTATION_SELECTION, 'Implementation', required=True,
            help="Two sequence object implementations are offered: Standard "
            "and 'No gap'. The later is slower than the former but forbids any"
            " gap in the sequence (while they are possible in the former)."),
        'active': openerp.osv.fields.boolean('Active'),
        'prefix': openerp.osv.fields.char('Prefix', size=64, help="Prefix value of the record for the sequence"),
        'suffix': openerp.osv.fields.char('Suffix', size=64, help="Suffix value of the record for the sequence"),
        'number_next': openerp.osv.fields.integer('Next Number', required=True, help="Next number of this sequence"),
        'number_increment': openerp.osv.fields.integer('Increment Number', required=True, help="The next number of the sequence will be incremented by this number"),
        'padding' : openerp.osv.fields.integer('Number padding', required=True, help="OpenERP will automatically adds some '0' on the left of the 'Next Number' to get the required padding size."),
        'company_id': openerp.osv.fields.many2one('res.company', 'Company'),
    }
    _defaults = {
        'implementation': 'standard',
        'active': True,
        'company_id': lambda s,cr,uid,c: s.pool.get('res.company')._company_default_get(cr, uid, 'ir.sequence', context=c),
        'number_increment': 1,
        'number_next': 1,
        'padding' : 0,
    }

    def create(self, cr, uid, values, context=None):
        values = self._add_missing_default_values(cr, uid, values, context)
        go = super(ir_sequence, self).create \
            if values['implementation'] == 'standard' else self.create_nogap
        return go(cr, uid, values, context)

    def _interpolate(self, s, d):
        return s % d if s else ''

    def _interpolation_dict(self):
        t = time.localtime() # Actually, the server is always in UTC.
        return {
            'year': time.strftime('%Y', t),
            'month': time.strftime('%m', t),
            'day': time.strftime('%d', t),
            'y': time.strftime('%y', t),
            'doy': time.strftime('%j', t),
            'woy': time.strftime('%W', t),
            'weekday': time.strftime('%w', t),
            'h24': time.strftime('%H', t),
            'h12': time.strftime('%I', t),
            'min': time.strftime('%M', t),
            'sec': time.strftime('%S', t),
        }

    # TODO rename 'test' to 'code_or_id' in account/sequence.
    def get_id(self, cr, uid, sequence_code_or_id, code_or_id='id', context=None):
        """ Draw an interpolated string using the specified sequence.

        The sequence to use is specified by the ``sequence_code_or_id``
        argument, which can be a code or an id (as controlled by the
        ``code_or_id`` argument.
        """
        assert code_or_id in ('code', 'id')
        company_ids = self.pool.get('res.company').search(cr, uid, [], context=context)
        cr.execute('''
            SELECT id, number_next, prefix, suffix, padding
            FROM ir_sequence
            WHERE %s=%%s
              AND active=true
              AND (company_id in %%s or company_id is NULL)
            ORDER BY company_id, id
            LIMIT 1
            FOR UPDATE NOWAIT''' % code_or_id,
            (sequence_code_or_id, tuple(company_ids)))
        res = cr.dictfetchone()
        if res:
            d = self._interpolation_dict()
            interpolated_prefix = self._interpolate(res['prefix'], d)
            interpolated_suffix = self._interpolate(res['suffix'], d)
            cr.execute('UPDATE ir_sequence SET number_next=number_next+number_increment WHERE id=%s', (res['id'],))
            if res['number_next']:
                return interpolated_prefix + '%%0%sd' % res['padding'] % res['number_next'] + interpolated_suffix
            else:
                # TODO what is this case used for ?
                return interpolated_prefix + interpolated_suffix
        return False

    def get(self, cr, uid, code, context=None):
        """ Draw an interpolated string using the specified sequence.

        The sequence to use is specified by its code.
        """
        return self.get_id(cr, uid, code, 'code', context)

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>[IMP] ir_sequence: add support for PostgreSQL sequence. This is partly inspired by openlabs' Sequence Postgres module. Untested yet.<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-TODAY OpenERP S.A. <http://www.openerp.com>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import time

import openerp

class ir_sequence_type(openerp.osv.osv.osv):
    _name = 'ir.sequence.type'
    _order = 'name'
    _columns = {
        'name': openerp.osv.fields.char('Name', size=64, required=True),
        'code': openerp.osv.fields.char('Code', size=32, required=True),
    }

    _sql_constraints = [
        ('code_unique', 'unique(code)', '`code` must be unique.'),
    ]

def _code_get(self, cr, uid, context={}):
    cr.execute('select code, name from ir_sequence_type')
    return cr.fetchall()

IMPLEMENTATION_SELECTION = \
    [('standard', 'Standard'), ('no_gap', 'No gap')]

class ir_sequence(openerp.osv.osv.osv):
    """ Sequence model.

    The sequence model allows to define and use so-called sequence objects.
    Such objects are used to generate unique identifiers in a transaction-safe
    way.

    """
    _name = 'ir.sequence'
    _order = 'name'
    _columns = {
        'name': openerp.osv.fields.char('Name', size=64, required=True),
        'code': openerp.osv.fields.selection(_code_get, 'Code', size=64, required=True), # TODO should it be unique?
        'implementation': openerp.osv.fields.selection( # TODO update the view
            IMPLEMENTATION_SELECTION, 'Implementation', required=True,
            help="Two sequence object implementations are offered: Standard "
            "and 'No gap'. The later is slower than the former but forbids any"
            " gap in the sequence (while they are possible in the former)."),
        'active': openerp.osv.fields.boolean('Active'),
        'prefix': openerp.osv.fields.char('Prefix', size=64, help="Prefix value of the record for the sequence"),
        'suffix': openerp.osv.fields.char('Suffix', size=64, help="Suffix value of the record for the sequence"),
        'number_next': openerp.osv.fields.integer('Next Number', required=True, help="Next number of this sequence"),
        'number_increment': openerp.osv.fields.integer('Increment Number', required=True, help="The next number of the sequence will be incremented by this number"),
        'padding' : openerp.osv.fields.integer('Number padding', required=True, help="OpenERP will automatically adds some '0' on the left of the 'Next Number' to get the required padding size."),
        'company_id': openerp.osv.fields.many2one('res.company', 'Company'),
    }
    _defaults = {
        'implementation': 'standard',
        'active': True,
        'company_id': lambda s,cr,uid,c: s.pool.get('res.company')._company_default_get(cr, uid, 'ir.sequence', context=c),
        'number_increment': 1,
        'number_next': 1,
        'padding' : 0,
    }

    def create(self, cr, uid, values, context=None):
        values = self._add_missing_default_values(cr, uid, values, context)
        go = super(ir_sequence, self).create_postgres \
            if values['implementation'] == 'standard' else self.create
        return go(cr, uid, values, context)

    def create_postgres(self, cr, uid, values, context=None):
        """ Create a fast, gaps-allowed PostgreSQL sequence.

        :param values: same argument than for ``create()`` but the keys
            ``number_increment`` and ``number_next`` must be present.
            ``_add_missing_default_values()`` can be used to this effect.
        :return: id of the newly created record
        """
        id = super(ir_sequence, self).create(cr, uid, values, context)
        self._create_sequence(cr,
            values['number_increment'], values['number_next'])
        return id

    def unlink(self, cr, uid, ids, context=None):
        super(ir_sequence, self).unlink(cr, uid, ids, context)
        self._drop_sequence(cr, ids)

    def write(self, cr, uid, ids, values, context=None):
        ids = ids if isinstance(ids, (list, tuple)) else [ids]
        new_implementation = values.get('implementation')
        rows = self.read(cr, uid, ids, ['id,' 'implementation',
            'number_increment', 'number_next'], context)
        super(ir_sequence, self).write(cr, uid, ids, values, context)

        for row in rows:
            # 4 cases: we test the previous impl. against the new one.
            if row['implementation'] == 'standard':
                i = values.get('number_increment', row['number_increment'])
                n = values.get('number_next', row['number_next'])
                if new_implementation in ('standard', None):
                    self._alter_sequence(cr, row['id'], i, n)
                else:
                    self._drop_sequence(cr, row['id'])
            else:
                if new_implementation in ('no_gap', None):
                    pass
                else:
                    self._create_sequence(cr, row['id'], i, n)

        return True

    def _interpolate(self, s, d):
        return s % d if s else ''

    def _interpolation_dict(self):
        t = time.localtime() # Actually, the server is always in UTC.
        return {
            'year': time.strftime('%Y', t),
            'month': time.strftime('%m', t),
            'day': time.strftime('%d', t),
            'y': time.strftime('%y', t),
            'doy': time.strftime('%j', t),
            'woy': time.strftime('%W', t),
            'weekday': time.strftime('%w', t),
            'h24': time.strftime('%H', t),
            'h12': time.strftime('%I', t),
            'min': time.strftime('%M', t),
            'sec': time.strftime('%S', t),
        }

    # TODO rename 'test' to 'code_or_id' in account/sequence.
    def get_id(self, cr, uid, sequence_code_or_id, code_or_id='id', context=None):
        """ Draw an interpolated string using the specified sequence.

        The sequence to use is specified by the ``sequence_code_or_id``
        argument, which can be a code or an id (as controlled by the
        ``code_or_id`` argument.
        """
        # TODO there is no access rights check on the sequence itself.
        res = self._select_by_code_or_id(cr, uid, sequence_code_or_id,
            code_or_id, context)

        if not res:
            return False

        if res['implementation'] == 'standard':
            cr.execute("""
                SELECT nextval('ir_sequence_%03d')
                """, (res['id'],))
            res['number_next'] = cr.fetchone()
        else:
            cr.execute("""
                UPDATE ir_sequence
                SET number_next=number_next+number_increment
                WHERE id=%s
                """, (res['id'],))

        d = self._interpolation_dict()
        interpolated_prefix = self._interpolate(res['prefix'], d)
        interpolated_suffix = self._interpolate(res['suffix'], d)
        if res['number_next']:
            return interpolated_prefix + '%%0%sd' % res['padding'] % \
                res['number_next'] + interpolated_suffix
        else:
            # TODO what is this case used for ?
            return interpolated_prefix + interpolated_suffix

    def get(self, cr, uid, code, context=None):
        """ Draw an interpolated string using the specified sequence.

        The sequence to use is specified by its code.
        """
        return self.get_id(cr, uid, code, 'code', context)

    def _select_by_code_or_id(self, cr, uid, sequence_code_or_id, code_or_id, context=None):
        """ Read a sequence object.

        There is no access rights check on the sequence itself.
        """
        assert code_or_id in ('code', 'id')
        res_company = self.pool.get('res.company')
        company_ids = res_company.search(cr, uid, [], context=context)
        cr.execute("""
            SELECT id, number_next, prefix, suffix, padding, implementation
            FROM ir_sequence
            WHERE %s=%%s
              AND active=true
              AND (company_id in %%s or company_id is NULL)
            ORDER BY company_id, id
            LIMIT 1
            FOR UPDATE NOWAIT
            """ % code_or_id, (sequence_code_or_id, tuple(company_ids)))
        return cr.dictfetchone()

    def _create_sequence(self, cr, number_increment, number_next):
        """ Create a PostreSQL sequence.

        There is no access rights check.
        """
        cr.execute("""
            CREATE SEQUENCE ir_sequence_%03d INCREMENT BY %s START WITH %s
            """, (id, number_increment, number_next))

    def _drop_sequence(self, cr, ids):
        """ Drop the PostreSQL sequence if it exists.

        There is no access rights check.
        """

        ids = ids if isinstance(ids, (list, tuple)) else [ids]
        assert all(isinstance(i, (int, long)) for i in ids), \
            "Only ids in (int, long) allowed."
        names = ','.join('ir_sequence_%03d' % i for i in ids)

        # RESTRICT is the default; it prevents dropping the sequence if an
        # object depends on it.
        cr.execute("""
            DROP SEQUENCE IF EXISTS %s RESTRICT
            """ % names)

    def _alter_sequence(self, cr, id, number_increment, number_next):
        """ Alter a PostreSQL sequence.

        There is no access rights check.
        """
        cr.execute("""
            ALTER SEQUENCE ir_sequence_%03d INCREMENT BY %s RESTART WITH %s
            """, (id, number_increment, number_next))

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before>#  -*- coding: utf-8 -*-
#  vim: tabstop=4 shiftwidth=4 softtabstop=4

#  Copyright (c) 2014, GEM Foundation

#  OpenQuake is free software: you can redistribute it and/or modify it
#  under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.

#  OpenQuake is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.

#  You should have received a copy of the GNU Affero General Public License
#  along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import unittest
from openquake.commonlib import sap

def f(a, b, c, d=1):
    return [a, b, c, d]

class SapTestCase(unittest.TestCase):
    def test_ok(self):
        p = sap.Parser(f)
        p.arg('a', 'first argument')
        p.opt('b', 'second argument')
        p.flg('c', 'third argument')
        p.opt('d', 'fourth argument')
        self.assertEqual(
            ['1', '2', False, '3'], p.callfunc('1 -b=2 -d=3'.split()))

    def test_group(self):
        p = sap.Parser(f, version=None)
        p.arg('a', 'first argument')
        p.opt('b', 'second argument')
        p.group('other arguments')
        p.flg('c', 'third argument')
        p.opt('d', 'fourth argument')
        self.assertEqual(p.help(), '''\
usage: nosetests [-h] [-b B] [-c] [-d 1] a

positional arguments:
  a            first argument

optional arguments:
  -h, --help   show this help message and exit
  -b B, --b B  second argument

other arguments:
  -c, --c      third argument
  -d 1, --d 1  fourth argument
''')

    def test_NameError(self):
        p = sap.Parser(f)
        p.arg('a', 'first argument')
        with self.assertRaises(NameError):
            p.flg('c', 'third argument')

    def test_no_help(self):
        p = sap.Parser(f, help=None, version=None)
        p.arg('a', 'first argument')
        p.opt('b', 'second argument')
        self.assertEqual(p.help(), '''\
usage: nosetests [-b B] a

positional arguments:
  a            first argument

optional arguments:
  -b B, --b B  second argument
''')
        # missing argparse specification for 'c'
        with self.assertRaises(NameError):
            p.check_arguments()

    def test_long_argument(self):
        # test the replacement '_' -> '-' in variable names
        p = sap.Parser(lambda a_long_argument: None)
        p.opt('a_long_argument', 'a long argument')
        self.assertEqual(p.help(), '''\
usage: nosetests [-h] [-v] [-a A_LONG_ARGUMENT]

optional arguments:
  -h, --help            show this help message and exit
  -v, --version         show program's version number and exit
  -a A_LONG_ARGUMENT, --a-long-argument A_LONG_ARGUMENT
                        a long argument
''')
<commit_msg>Fixed sap_test<commit_after>#  -*- coding: utf-8 -*-
#  vim: tabstop=4 shiftwidth=4 softtabstop=4

#  Copyright (c) 2014, GEM Foundation

#  OpenQuake is free software: you can redistribute it and/or modify it
#  under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.

#  OpenQuake is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.

#  You should have received a copy of the GNU Affero General Public License
#  along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import unittest
from openquake.commonlib import sap

def f(a, b, c, d=1):
    return [a, b, c, d]

class SapTestCase(unittest.TestCase):
    def test_ok(self):
        p = sap.Parser(f)
        p.arg('a', 'first argument')
        p.opt('b', 'second argument')
        p.flg('c', 'third argument')
        p.opt('d', 'fourth argument')
        self.assertEqual(
            ['1', '2', False, '3'], p.callfunc('1 -b=2 -d=3'.split()))

    def test_group(self):
        p = sap.Parser(f)
        p.arg('a', 'first argument')
        p.opt('b', 'second argument')
        p.group('other arguments')
        p.flg('c', 'third argument')
        p.opt('d', 'fourth argument')
        self.assertEqual(p.help(), '''\
usage: nosetests [-h] [-b B] [-c] [-d 1] a

positional arguments:
  a            first argument

optional arguments:
  -h, --help   show this help message and exit
  -b B, --b B  second argument

other arguments:
  -c, --c      third argument
  -d 1, --d 1  fourth argument
''')

    def test_NameError(self):
        p = sap.Parser(f)
        p.arg('a', 'first argument')
        with self.assertRaises(NameError):
            p.flg('c', 'third argument')

    def test_no_help(self):
        p = sap.Parser(f, help=None)
        p.arg('a', 'first argument')
        p.opt('b', 'second argument')
        self.assertEqual(p.help(), '''\
usage: nosetests [-b B] a

positional arguments:
  a            first argument

optional arguments:
  -b B, --b B  second argument
''')
        # missing argparse specification for 'c'
        with self.assertRaises(NameError):
            p.check_arguments()

    def test_long_argument(self):
        # test the replacement '_' -> '-' in variable names
        p = sap.Parser(lambda a_long_argument: None)
        p.opt('a_long_argument', 'a long argument')
        self.assertEqual(p.help(), '''\
usage: nosetests [-h] [-a A_LONG_ARGUMENT]

optional arguments:
  -h, --help            show this help message and exit
  -a A_LONG_ARGUMENT, --a-long-argument A_LONG_ARGUMENT
                        a long argument
''')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2010-2016 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import os
import sys
import getpass
import subprocess
import unittest
import tempfile
import mock

from django.db import connection
from openquake.server.db import models, actions, upgrade_manager
from openquake.server.settings import DATABASE
from openquake.server.tests import helpers

def setup_module():
    global tmpfile
    fh, tmpfile = tempfile.mkstemp()
    os.close(fh)
    sys.stderr.write('Using the database %s\n' % tmpfile)
    DATABASE['name'] = tmpfile
    DATABASE['user'] = getpass.getuser()
    connection.close()  # if already open
    connection.cursor()  # reconnect to the db
    upgrade_manager.upgrade_db(connection.connection)

def teardown_module():
    connection.close()

def get_job(cfg, username, hazard_calculation_id=None):
    job_id, oq = actions.job_from_file(cfg, username, hazard_calculation_id)
    return models.OqJob.objects.get(pk=job_id)

class FakeJob(object):
    def __init__(self, job_type, calculation_mode):
        self.job_type = job_type
        self.calculation_mode = calculation_mode

    def get_oqparam(self):
        m = mock.Mock()
        m.calculation_mode = self.calculation_mode
        return m

class CheckHazardRiskConsistencyTestCase(unittest.TestCase):
    def test_ok(self):
        haz_job = FakeJob('hazard', 'scenario')
        actions.check_hazard_risk_consistency(
            haz_job, 'scenario_risk')

    def test_obsolete_mode(self):
        haz_job = FakeJob('hazard', 'scenario')
        with self.assertRaises(ValueError) as ctx:
            actions.check_hazard_risk_consistency(
                haz_job, 'scenario')
        msg = str(ctx.exception)
        self.assertEqual(msg, 'Please change calculation_mode=scenario into '
                         'scenario_risk in the .ini file')

    def test_inconsistent_mode(self):
        haz_job = FakeJob('hazard', 'scenario')
        with self.assertRaises(actions.InvalidCalculationID) as ctx:
            actions.check_hazard_risk_consistency(
                haz_job, 'classical_risk')
        msg = str(ctx.exception)
        self.assertEqual(
            msg, "In order to run a risk calculation of kind "
            "'classical_risk', you need to provide a "
            "calculation of kind ['classical', 'classical_risk'], "
            "but you provided a 'scenario' instead")

class JobFromFileTestCase(unittest.TestCase):

    def test_create_job_default_user(self):
        jid = actions.create_job('classical', 'test_create_job_default_user')
        job = models.OqJob.objects.get(pk=jid)
        self.assertEqual('openquake', job.user_name)
        self.assertEqual('executing', job.status)

    def test_create_job_specified_user(self):
        user_name = helpers.random_string()
        jid = actions.create_job(
            'classical', 'test_create_job_specified_user',
            user_name=user_name)
        job = models.OqJob.objects.get(pk=jid)
        self.assertEqual(user_name, job.user_name)
        self.assertEqual('executing', job.status)

class OpenquakeCliTestCase(unittest.TestCase):
    """
    Run "oq-engine --version" as a separate process using `subprocess`.
    """

    def test_run_version(self):
        args = [helpers.RUNNER, "--version"]

        print 'Running:', ' '.join(args)  # this is useful for debugging
        return subprocess.check_call(args)

class DeleteHazCalcTestCase(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.hazard_cfg = helpers.get_data_path(
            'simple_fault_demo_hazard/job.ini')
        cls.risk_cfg = helpers.get_data_path(
            'classical_psha_based_risk/job.ini')
        cls.job = get_job(cls.hazard_cfg, getpass.getuser())

    def test_del_calc(self):
        hazard_job = get_job(self.hazard_cfg, getpass.getuser())

        models.Output.objects.create_output(
            hazard_job, 'test_curves_1', ds_key='hcurve'
        )
        models.Output.objects.create_output(
            hazard_job, 'test_curves_2', ds_key='hcurve'
        )

        # Sanity check: make sure the hazard calculation and outputs exist in
        # the database:
        hazard_jobs = models.OqJob.objects.filter(id=hazard_job.id)
        self.assertEqual(1, hazard_jobs.count())

        outputs = models.Output.objects.filter(oq_job=hazard_job.id)
        self.assertEqual(2, outputs.count())

        # Delete the calculation
        actions.del_calc(hazard_job.id)

        # Check that the hazard calculation and its outputs were deleted:
        outputs = models.Output.objects.filter(oq_job=hazard_job.id)
        self.assertEqual(0, outputs.count())

        hazard_jobs = models.OqJob.objects.filter(id=hazard_job.id)
        self.assertEqual(0, hazard_jobs.count())

    def test_del_calc_does_not_exist(self):
        self.assertRaises(models.NotFound, actions.del_calc, -1)

    def test_del_calc_no_access(self):
        # Test the case where we try to delete a hazard calculation which does
        # not belong to current user.
        # In this case, deletion is now allowed and should raise an exception.
        hazard_job = get_job(self.hazard_cfg, helpers.random_string())
        self.assertRaises(RuntimeError, actions.del_calc, hazard_job.id)

    def test_del_calc_referenced_by_risk_calc(self):
        # Test the case where a risk calculation is referencing the hazard
        # calculation we want to delete.
        # In this case, deletion is not allowed and should raise an exception.
        risk_job = get_job(self.risk_cfg, getpass.getuser(),
                           hazard_calculation_id=self.job.id)
        hc = risk_job.hazard_calculation
        self.assertRaises(RuntimeError, actions.del_calc, hc.id)

    def test_del_calc_output_referenced_by_risk_calc(self):
        # Test the case where a risk calculation is referencing one of the
        # belonging to the hazard calculation we want to delete.
        # In this case, deletion is not allowed and should raise an exception.
        risk_job = get_job(self.risk_cfg, getpass.getuser(),
                           hazard_calculation_id=self.job.id)
        hc = risk_job.hazard_calculation
        self.assertRaises(RuntimeError, actions.del_calc, hc.id)

class DeleteRiskCalcTestCase(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.hazard_cfg = helpers.get_data_path(
            'simple_fault_demo_hazard/job.ini')
        cls.risk_cfg = helpers.get_data_path(
            'classical_psha_based_risk/job.ini')
        cls.job = get_job(cls.hazard_cfg, getpass.getuser())

    def test_del_calc(self):
        risk_job = get_job(self.risk_cfg, getpass.getuser(),
                           hazard_calculation_id=self.job.id)
        models.Output.objects.create_output(
            risk_job, 'test_curves_1', ds_key='rcurves-rlzs'
        )
        models.Output.objects.create_output(
            risk_job, 'test_curves_2', ds_key='rcurves-rlzs'
        )

        # Sanity check: make sure the risk calculation and outputs exist in
        # the database:
        risk_calcs = models.OqJob.objects.filter(id=risk_job.id)
        self.assertEqual(1, risk_calcs.count())

        outputs = models.Output.objects.filter(oq_job=risk_job.id)
        self.assertEqual(2, outputs.count())

        # Delete the calculation
        actions.del_calc(risk_job.id)

        # Check that the risk calculation and its outputs were deleted:
        outputs = models.Output.objects.filter(oq_job=risk_job.id)
        self.assertEqual(0, outputs.count())

        risk_calcs = models.OqJob.objects.filter(id=risk_job.id)
        self.assertEqual(0, risk_calcs.count())

    def test_del_calc_does_not_exist(self):
        self.assertRaises(models.NotFound, actions.del_calc, -1)

    def test_del_calc_no_access(self):
        # Test the case where we try to delete a risk calculation which does
        # not belong to current user.
        # In this case, deletion is now allowed and should raise an exception.
        risk_job = get_job(self.risk_cfg, helpers.random_string(),
                           hazard_calculation_id=self.job.id)
        self.assertRaises(RuntimeError, actions.del_calc, risk_job.id)

class FakeOutput(object):
    def __init__(self, id, display_name):
        self.id = id
        self.display_name = display_name

    def get_output_type_display(self):
        return self.display_name + str(self.id)

class PrintSummaryTestCase(unittest.TestCase):
    outputs = [FakeOutput(i, 'gmf') for i in range(1, 12)]

    def print_outputs_summary(self, full):
        got = actions.print_outputs_summary(self.outputs, full)
        return '\n'.join(got)

    def test_print_outputs_summary_full(self):
        self.assertEqual(self.print_outputs_summary(full=True), '''\
  id | name
   1 | gmf
   2 | gmf
   3 | gmf
   4 | gmf
   5 | gmf
   6 | gmf
   7 | gmf
   8 | gmf
   9 | gmf
  10 | gmf
  11 | gmf''')

    def test_print_outputs_summary_short(self):
        self.assertEqual(
            self.print_outputs_summary(full=False), '''\
  id | name
   1 | gmf
   2 | gmf
   3 | gmf
   4 | gmf
   5 | gmf
   6 | gmf
   7 | gmf
   8 | gmf
   9 | gmf
  10 | gmf
 ... | 1 additional output(s)
Some outputs where not shown. You can see the full list with the command
`oq-engine --list-outputs`''')
<commit_msg>Fixed bug in engine_test.setup_module<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2010-2016 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import os
import getpass
import subprocess
import unittest
import tempfile
import mock

from django.db import connection
from openquake.server.db import models, actions, upgrade_manager
from openquake.server.settings import DATABASE
from openquake.server.tests import helpers

def setup_module():
    global tmpfile
    fh, tmpfile = tempfile.mkstemp()
    os.close(fh)
    DATABASE['NAME'] = tmpfile
    DATABASE['USER'] = getpass.getuser()
    connection.cursor()  # connect to the db
    # sanity check: make sure we are using the right db
    fname = connection.connection.execute(
        'PRAGMA database_list').fetchone()[-1]
    assert fname == tmpfile, (fname, tmpfile)
    upgrade_manager.upgrade_db(connection.connection)

def teardown_module():
    connection.close()

def get_job(cfg, username, hazard_calculation_id=None):
    job_id, oq = actions.job_from_file(cfg, username, hazard_calculation_id)
    return models.OqJob.objects.get(pk=job_id)

class FakeJob(object):
    def __init__(self, job_type, calculation_mode):
        self.job_type = job_type
        self.calculation_mode = calculation_mode

    def get_oqparam(self):
        m = mock.Mock()
        m.calculation_mode = self.calculation_mode
        return m

class CheckHazardRiskConsistencyTestCase(unittest.TestCase):
    def test_ok(self):
        haz_job = FakeJob('hazard', 'scenario')
        actions.check_hazard_risk_consistency(
            haz_job, 'scenario_risk')

    def test_obsolete_mode(self):
        haz_job = FakeJob('hazard', 'scenario')
        with self.assertRaises(ValueError) as ctx:
            actions.check_hazard_risk_consistency(
                haz_job, 'scenario')
        msg = str(ctx.exception)
        self.assertEqual(msg, 'Please change calculation_mode=scenario into '
                         'scenario_risk in the .ini file')

    def test_inconsistent_mode(self):
        haz_job = FakeJob('hazard', 'scenario')
        with self.assertRaises(actions.InvalidCalculationID) as ctx:
            actions.check_hazard_risk_consistency(
                haz_job, 'classical_risk')
        msg = str(ctx.exception)
        self.assertEqual(
            msg, "In order to run a risk calculation of kind "
            "'classical_risk', you need to provide a "
            "calculation of kind ['classical', 'classical_risk'], "
            "but you provided a 'scenario' instead")

class JobFromFileTestCase(unittest.TestCase):

    def test_create_job_default_user(self):
        jid = actions.create_job('classical', 'test_create_job_default_user')
        job = models.OqJob.objects.get(pk=jid)
        self.assertEqual('openquake', job.user_name)
        self.assertEqual('executing', job.status)

    def test_create_job_specified_user(self):
        user_name = helpers.random_string()
        jid = actions.create_job(
            'classical', 'test_create_job_specified_user',
            user_name=user_name)
        job = models.OqJob.objects.get(pk=jid)
        self.assertEqual(user_name, job.user_name)
        self.assertEqual('executing', job.status)

class OpenquakeCliTestCase(unittest.TestCase):
    """
    Run "oq-engine --version" as a separate process using `subprocess`.
    """

    def test_run_version(self):
        args = [helpers.RUNNER, "--version"]

        print 'Running:', ' '.join(args)  # this is useful for debugging
        return subprocess.check_call(args)

class DeleteHazCalcTestCase(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.hazard_cfg = helpers.get_data_path(
            'simple_fault_demo_hazard/job.ini')
        cls.risk_cfg = helpers.get_data_path(
            'classical_psha_based_risk/job.ini')
        cls.job = get_job(cls.hazard_cfg, getpass.getuser())

    def test_del_calc(self):
        hazard_job = get_job(self.hazard_cfg, getpass.getuser())

        models.Output.objects.create_output(
            hazard_job, 'test_curves_1', ds_key='hcurve'
        )
        models.Output.objects.create_output(
            hazard_job, 'test_curves_2', ds_key='hcurve'
        )

        # Sanity check: make sure the hazard calculation and outputs exist in
        # the database:
        hazard_jobs = models.OqJob.objects.filter(id=hazard_job.id)
        self.assertEqual(1, hazard_jobs.count())

        outputs = models.Output.objects.filter(oq_job=hazard_job.id)
        self.assertEqual(2, outputs.count())

        # Delete the calculation
        actions.del_calc(hazard_job.id)

        # Check that the hazard calculation and its outputs were deleted:
        outputs = models.Output.objects.filter(oq_job=hazard_job.id)
        self.assertEqual(0, outputs.count())

        hazard_jobs = models.OqJob.objects.filter(id=hazard_job.id)
        self.assertEqual(0, hazard_jobs.count())

    def test_del_calc_does_not_exist(self):
        self.assertRaises(models.NotFound, actions.del_calc, -1)

    def test_del_calc_no_access(self):
        # Test the case where we try to delete a hazard calculation which does
        # not belong to current user.
        # In this case, deletion is now allowed and should raise an exception.
        hazard_job = get_job(self.hazard_cfg, helpers.random_string())
        self.assertRaises(RuntimeError, actions.del_calc, hazard_job.id)

    def test_del_calc_referenced_by_risk_calc(self):
        # Test the case where a risk calculation is referencing the hazard
        # calculation we want to delete.
        # In this case, deletion is not allowed and should raise an exception.
        risk_job = get_job(self.risk_cfg, getpass.getuser(),
                           hazard_calculation_id=self.job.id)
        hc = risk_job.hazard_calculation
        self.assertRaises(RuntimeError, actions.del_calc, hc.id)

    def test_del_calc_output_referenced_by_risk_calc(self):
        # Test the case where a risk calculation is referencing one of the
        # belonging to the hazard calculation we want to delete.
        # In this case, deletion is not allowed and should raise an exception.
        risk_job = get_job(self.risk_cfg, getpass.getuser(),
                           hazard_calculation_id=self.job.id)
        hc = risk_job.hazard_calculation
        self.assertRaises(RuntimeError, actions.del_calc, hc.id)

class DeleteRiskCalcTestCase(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.hazard_cfg = helpers.get_data_path(
            'simple_fault_demo_hazard/job.ini')
        cls.risk_cfg = helpers.get_data_path(
            'classical_psha_based_risk/job.ini')
        cls.job = get_job(cls.hazard_cfg, getpass.getuser())

    def test_del_calc(self):
        risk_job = get_job(self.risk_cfg, getpass.getuser(),
                           hazard_calculation_id=self.job.id)
        models.Output.objects.create_output(
            risk_job, 'test_curves_1', ds_key='rcurves-rlzs'
        )
        models.Output.objects.create_output(
            risk_job, 'test_curves_2', ds_key='rcurves-rlzs'
        )

        # Sanity check: make sure the risk calculation and outputs exist in
        # the database:
        risk_calcs = models.OqJob.objects.filter(id=risk_job.id)
        self.assertEqual(1, risk_calcs.count())

        outputs = models.Output.objects.filter(oq_job=risk_job.id)
        self.assertEqual(2, outputs.count())

        # Delete the calculation
        actions.del_calc(risk_job.id)

        # Check that the risk calculation and its outputs were deleted:
        outputs = models.Output.objects.filter(oq_job=risk_job.id)
        self.assertEqual(0, outputs.count())

        risk_calcs = models.OqJob.objects.filter(id=risk_job.id)
        self.assertEqual(0, risk_calcs.count())

    def test_del_calc_does_not_exist(self):
        self.assertRaises(models.NotFound, actions.del_calc, -1)

    def test_del_calc_no_access(self):
        # Test the case where we try to delete a risk calculation which does
        # not belong to current user.
        # In this case, deletion is now allowed and should raise an exception.
        risk_job = get_job(self.risk_cfg, helpers.random_string(),
                           hazard_calculation_id=self.job.id)
        self.assertRaises(RuntimeError, actions.del_calc, risk_job.id)

class FakeOutput(object):
    def __init__(self, id, display_name):
        self.id = id
        self.display_name = display_name

    def get_output_type_display(self):
        return self.display_name + str(self.id)

class PrintSummaryTestCase(unittest.TestCase):
    outputs = [FakeOutput(i, 'gmf') for i in range(1, 12)]

    def print_outputs_summary(self, full):
        got = actions.print_outputs_summary(self.outputs, full)
        return '\n'.join(got)

    def test_print_outputs_summary_full(self):
        self.assertEqual(self.print_outputs_summary(full=True), '''\
  id | name
   1 | gmf
   2 | gmf
   3 | gmf
   4 | gmf
   5 | gmf
   6 | gmf
   7 | gmf
   8 | gmf
   9 | gmf
  10 | gmf
  11 | gmf''')

    def test_print_outputs_summary_short(self):
        self.assertEqual(
            self.print_outputs_summary(full=False), '''\
  id | name
   1 | gmf
   2 | gmf
   3 | gmf
   4 | gmf
   5 | gmf
   6 | gmf
   7 | gmf
   8 | gmf
   9 | gmf
  10 | gmf
 ... | 1 additional output(s)
Some outputs where not shown. You can see the full list with the command
`oq-engine --list-outputs`''')
<|endoftext|>


<commit_before>from urllib.parse import unquote
from urllib.parse import urlparse

import click
from azure.servicebus import ServiceBusClient
from libcloud.storage.providers import get_driver

from opwen_email_server import config

_STORAGES = (
    (
        config.BLOBS_ACCOUNT,
        config.BLOBS_KEY,
        config.BLOBS_HOST,
        config.BLOBS_SECURE,
    ),
    (
        config.TABLES_ACCOUNT,
        config.TABLES_KEY,
        config.TABLES_HOST,
        config.TABLES_SECURE,
    ),
    (
        config.CLIENT_STORAGE_ACCOUNT,
        config.CLIENT_STORAGE_KEY,
        config.CLIENT_STORAGE_HOST,
        config.CLIENT_STORAGE_SECURE,
    ),
)

@click.group()
def cli():
    pass

@cli.command()
@click.option('--separator', '-s', default='\n')
def print_queues(separator):
    click.echo(
        separator.join((
            config.REGISTER_CLIENT_QUEUE,
            config.INBOUND_STORE_QUEUE,
            config.WRITTEN_STORE_QUEUE,
            config.SEND_QUEUE,
            config.MAILBOX_RECEIVED_QUEUE,
            config.MAILBOX_SENT_QUEUE,
        )))

@cli.command()
@click.option('-s', '--suffix', default='')
def delete_queues(suffix):
    suffix = suffix.replace('-', '_')

    queue_broker = urlparse(config.QUEUE_BROKER)
    if queue_broker.scheme != 'azureservicebus':
        click.echo(f'Skipping queue cleanup for {queue_broker.scheme}')
        return

    client = ServiceBusClient(
        service_namespace=queue_broker.hostname,
        shared_access_key_name=unquote(queue_broker.username),
        shared_access_key_value=unquote(queue_broker.password),
    )

    for queue in client.list_queues():
        if queue.name.endswith(suffix):
            click.echo(f'Deleting queue {queue.name}')
            client.delete_queue(queue.name)

@cli.command()
@click.option('-s', '--suffix')
def delete_containers(suffix):
    for account, key, host, secure in _STORAGES:
        client = get_driver(config.STORAGE_PROVIDER)(account, key, host=host, secure=secure)

        for container in client.iterate_containers():
            if container.name.endswith(suffix):
                click.echo(f'Deleting container {container.name}')

                response = client.connection.request(
                    f'/{container.name}',
                    params={'restype': 'container'},
                    method='DELETE',
                )

                if response.status != 202:
                    raise ValueError(f'Unable to delete container {container.name}')

if __name__ == '__main__':
    cli()
<commit_msg>Ensure service email queue gets consumed (#431)<commit_after>from urllib.parse import unquote
from urllib.parse import urlparse

import click
from azure.servicebus import ServiceBusClient
from libcloud.storage.providers import get_driver

from opwen_email_server import config

_STORAGES = (
    (
        config.BLOBS_ACCOUNT,
        config.BLOBS_KEY,
        config.BLOBS_HOST,
        config.BLOBS_SECURE,
    ),
    (
        config.TABLES_ACCOUNT,
        config.TABLES_KEY,
        config.TABLES_HOST,
        config.TABLES_SECURE,
    ),
    (
        config.CLIENT_STORAGE_ACCOUNT,
        config.CLIENT_STORAGE_KEY,
        config.CLIENT_STORAGE_HOST,
        config.CLIENT_STORAGE_SECURE,
    ),
)

@click.group()
def cli():
    pass

@cli.command()
@click.option('--separator', '-s', default='\n')
def print_queues(separator):
    click.echo(
        separator.join((
            config.REGISTER_CLIENT_QUEUE,
            config.INBOUND_STORE_QUEUE,
            config.WRITTEN_STORE_QUEUE,
            config.PROCESS_SERVICE_QUEUE,
            config.SEND_QUEUE,
            config.MAILBOX_RECEIVED_QUEUE,
            config.MAILBOX_SENT_QUEUE,
        )))

@cli.command()
@click.option('-s', '--suffix', default='')
def delete_queues(suffix):
    suffix = suffix.replace('-', '_')

    queue_broker = urlparse(config.QUEUE_BROKER)
    if queue_broker.scheme != 'azureservicebus':
        click.echo(f'Skipping queue cleanup for {queue_broker.scheme}')
        return

    client = ServiceBusClient(
        service_namespace=queue_broker.hostname,
        shared_access_key_name=unquote(queue_broker.username),
        shared_access_key_value=unquote(queue_broker.password),
    )

    for queue in client.list_queues():
        if queue.name.endswith(suffix):
            click.echo(f'Deleting queue {queue.name}')
            client.delete_queue(queue.name)

@cli.command()
@click.option('-s', '--suffix')
def delete_containers(suffix):
    for account, key, host, secure in _STORAGES:
        client = get_driver(config.STORAGE_PROVIDER)(account, key, host=host, secure=secure)

        for container in client.iterate_containers():
            if container.name.endswith(suffix):
                click.echo(f'Deleting container {container.name}')

                response = client.connection.request(
                    f'/{container.name}',
                    params={'restype': 'container'},
                    method='DELETE',
                )

                if response.status != 202:
                    raise ValueError(f'Unable to delete container {container.name}')

if __name__ == '__main__':
    cli()
<|endoftext|>


<commit_before>#
# __COPYRIGHT__
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#

__doc__ = """
SCons compatibility package for old Python versions

This subpackage holds modules that provide backwards-compatible
implementations of various things that we'd like to use in SCons but which
only show up in later versions of Python than the early, old version(s)
we still support.

Other code will not generally reference things in this package through
the SCons.compat namespace.  The modules included here add things to
the builtins namespace or the global module list so that the rest
of our code can use the objects and names imported here regardless of
Python version.

Simply enough, things that go in the builtins name space come from
our _scons_builtins module.

The rest of the things here will be in individual compatibility modules
that are either: 1) suitably modified copies of the future modules that
we want to use; or 2) backwards compatible re-implementations of the
specific portions of a future module's API that we want to use.

GENERAL WARNINGS:  Implementations of functions in the SCons.compat
modules are *NOT* guaranteed to be fully compliant with these functions in
later versions of Python.  We are only concerned with adding functionality
that we actually use in SCons, so be wary if you lift this code for
other uses.  (That said, making these more nearly the same as later,
official versions is still a desirable goal, we just don't need to be
obsessive about it.)

We name the compatibility modules with an initial '_scons_' (for example,
_scons_subprocess.py is our compatibility module for subprocess) so
that we can still try to import the real module name and fall back to
our compatibility module if we get an ImportError.  The import_as()
function defined below loads the module as the "real" name (without the
'_scons'), after which all of the "import {module}" statements in the
rest of our code will find our pre-loaded compatibility module.
"""

__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"

def import_as(module, name):
    """
    Imports the specified module (from our local directory) as the
    specified name, returning the loaded module object.
    """
    import imp
    import os.path
    dir = os.path.split(__file__)[0]
    file, filename, suffix_mode_type = imp.find_module(module, [dir])
    return imp.load_module(name, file, filename, suffix_mode_type)

try:
    import builtins
except ImportError:
    # Use the "imp" module to protect the import from fixers.
    import imp
    import sys
    _builtin = imp.load_module('__builtin__',
                               *imp.find_module('__builtin__'))
    sys.modules['builtins'] = _builtin
    del _builtin

import _scons_builtins

try:
    import hashlib
except ImportError:
    # Pre-2.5 Python has no hashlib module.
    try:
        import_as('_scons_hashlib', 'hashlib')
    except ImportError:
        # If we failed importing our compatibility module, it probably
        # means this version of Python has no md5 module.  Don't do
        # anything and let the higher layer discover this fact, so it
        # can fall back to using timestamp.
        pass

try:
    set
except NameError:
    # Pre-2.4 Python has no native set type
    import_as('_scons_sets', 'sets')
    import builtins, sets
    builtins.set = sets.Set

try:
    import collections
except ImportError:
    # Pre-2.4 Python has no collections module.
    import_as('_scons_collections', 'collections')
else:
    # Use the "imp" module to protect the imports below from fixers.
    import imp
    try:
        collections.UserDict
    except AttributeError:
        _UserDict = imp.load_module('UserDict', *imp.find_module('UserDict'))
        collections.UserDict = _UserDict.UserDict
        del _UserDict
    try:
        collections.UserList
    except AttributeError:
        _UserList = imp.load_module('UserList', *imp.find_module('UserList'))
        collections.UserList = _UserList.UserList
        del _UserList
    try:
        collections.UserString
    except AttributeError:
        _UserString = imp.load_module('UserString',
                                      *imp.find_module('UserString'))
        collections.UserString = _UserString.UserString
        del _UserString

try:
    import dbm
except ImportError:
    dbm = import_as('_scons_dbm', 'dbm')
try:
    dbm.whichdb
except AttributeError:
    # Pre-3.0 Python has no dbm.whichdb function.
    import whichdb
    dbm.whichdb = whichdb.whichdb
    del whichdb

try:
    import io
except ImportError:
    # Pre-2.6 Python has no io module.
    import_as('_scons_io', 'io')

import os
try:
    os.devnull
except AttributeError:
    # Pre-2.4 Python has no os.devnull attribute
    import sys
    _names = sys.builtin_module_names
    if 'posix' in _names:
        os.devnull = '/dev/null'
    elif 'nt' in _names:
        os.devnull = 'nul'
    os.path.devnull = os.devnull
try:
    os.path.lexists
except AttributeError:
    # Pre-2.4 Python has no os.path.lexists function
    def lexists(path):
        return os.path.exists(path) or os.path.islink(path)
    os.path.lexists = lexists

# When we're using the '-3' option during regression tests, importing
# cPickle gives a warning no matter how it's done, so always use the
# real profile module, whether it's fast or not.
if os.environ.get('SCONS_HORRIBLE_REGRESSION_TEST_HACK') is None:
    # Not a regression test with '-3', so try to use faster version.
    try:
        # Use the "imp" module to protect the import from fixers.
        import imp
        _cPickle = imp.load_module('cPickle', *imp.find_module('cPickle'))
    except ImportError, e:
        # The "cPickle" module has already been eliminated in favor of
        # having "import pickle" import the fast version when available.
        pass
    else:
        import sys
        sys.modules['pickle'] = _cPickle
        del _cPickle

try:
    # Use the "imp" module to protect the import from fixers.
    import imp
    _cProfile = imp.load_module('cProfile', *imp.find_module('cProfile'))
except ImportError:
    # The "cProfile" module has already been eliminated in favor of
    # having "import profile" import the fast version when available.
    pass
else:
    import sys
    sys.modules['profile'] = _cProfile
    del _cProfile

try:
    import platform
except ImportError:
    # Pre-2.3 Python has no platform module.
    import_as('_scons_platform', 'platform')

try:
    import queue
except ImportError:
    # Before Python 3.0, the 'queue' module was named 'Queue'.
    import imp
    file, filename, suffix_mode_type = imp.find_module('Queue')
    imp.load_module('queue', file, filename, suffix_mode_type)

import shutil
try:
    shutil.move
except AttributeError:
    # Pre-2.3 Python has no shutil.move() function.
    #
    # Cribbed from Python 2.5.
    import os

    def move(src, dst):
        """Recursively move a file or directory to another location.

        If the destination is on our current filesystem, then simply use
        rename.  Otherwise, copy src to the dst and then remove src.
        A lot more could be done here...  A look at a mv.c shows a lot of
        the issues this implementation glosses over.

        """
        try:
            os.rename(src, dst)
        except OSError:
            if os.path.isdir(src):
                if shutil.destinsrc(src, dst):
                    raise Error("Cannot move a directory '%s' into itself '%s'." % (src, dst))
                shutil.copytree(src, dst, symlinks=True)
                shutil.rmtree(src)
            else:
                shutil.copy2(src,dst)
                os.unlink(src)
    shutil.move = move
    del move

    def destinsrc(src, dst):
        src = os.path.abspath(src)
        return os.path.abspath(dst)[:len(src)] == src
    shutil.destinsrc = destinsrc
    del destinsrc

try:
    import subprocess
except ImportError:
    # Pre-2.4 Python has no subprocess module.
    import_as('_scons_subprocess', 'subprocess')

import sys
try:
    sys.intern
except AttributeError:
    # Pre-2.6 Python has no sys.intern() function.
    import builtins
    try:
        sys.intern = builtins.intern
    except AttributeError:
        # Pre-2.x Python has no builtin intern() function.
        def intern(x):
           return x
        sys.intern = intern
        del intern
try:
    sys.maxsize
except AttributeError:
    # Pre-2.6 Python has no sys.maxsize attribute
    # Wrapping sys in () is silly, but protects it from 2to3 renames fixer
    sys.maxsize = (sys).maxint

import tempfile
try:
    tempfile.mkstemp
except AttributeError:
    # Pre-2.3 Python has no tempfile.mkstemp function, so try to simulate it.
    # adapted from the mkstemp implementation in python 3.
    import os
    import errno
    def mkstemp(*args, **kw):
        text = False
        # TODO (1.5)
        #if 'text' in kw :
        if 'text' in kw.keys() :
            text = kw['text']
            del kw['text']
        elif len( args ) == 4 :
            text = args[3]
            args = args[:3]
        flags = os.O_RDWR | os.O_CREAT | os.O_EXCL
        if not text and hasattr( os, 'O_BINARY' ) :
            flags = flags | os.O_BINARY
        while True:
            try :
                name = tempfile.mktemp(*args, **kw)
                fd = os.open( name, flags, 0600 )
                return (fd, os.path.abspath(name))
            except OSError, e:
                if e.errno == errno.EEXIST:
                    continue
                raise

    tempfile.mkstemp = mkstemp
    del mkstemp

if os.environ.get('SCONS_HORRIBLE_REGRESSION_TEST_HACK') is not None:
    # We can't apply the 'callable' fixer until the floor is 2.6, but the
    # '-3' option to Python 2.6 and 2.7 generates almost ten thousand
    # warnings.  This hack allows us to run regression tests with the '-3'
    # option by replacing the callable() built-in function with a hack
    # that performs the same function but doesn't generate the warning.
    # Note that this hack is ONLY intended to be used for regression
    # testing, and should NEVER be used for real runs.
    from types import ClassType
    def callable(obj):
        if hasattr(obj, '__call__'): return True
        if isinstance(obj, (ClassType, type)): return True
        return False
    import builtins
    builtins.callable = callable
    del callable

# Local Variables:
# tab-width:4
# indent-tabs-mode:nil
# End:
# vim: set expandtab tabstop=4 shiftwidth=4:
<commit_msg>Remove the compat shutil.move() function.<commit_after>#
# __COPYRIGHT__
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#

__doc__ = """
SCons compatibility package for old Python versions

This subpackage holds modules that provide backwards-compatible
implementations of various things that we'd like to use in SCons but which
only show up in later versions of Python than the early, old version(s)
we still support.

Other code will not generally reference things in this package through
the SCons.compat namespace.  The modules included here add things to
the builtins namespace or the global module list so that the rest
of our code can use the objects and names imported here regardless of
Python version.

Simply enough, things that go in the builtins name space come from
our _scons_builtins module.

The rest of the things here will be in individual compatibility modules
that are either: 1) suitably modified copies of the future modules that
we want to use; or 2) backwards compatible re-implementations of the
specific portions of a future module's API that we want to use.

GENERAL WARNINGS:  Implementations of functions in the SCons.compat
modules are *NOT* guaranteed to be fully compliant with these functions in
later versions of Python.  We are only concerned with adding functionality
that we actually use in SCons, so be wary if you lift this code for
other uses.  (That said, making these more nearly the same as later,
official versions is still a desirable goal, we just don't need to be
obsessive about it.)

We name the compatibility modules with an initial '_scons_' (for example,
_scons_subprocess.py is our compatibility module for subprocess) so
that we can still try to import the real module name and fall back to
our compatibility module if we get an ImportError.  The import_as()
function defined below loads the module as the "real" name (without the
'_scons'), after which all of the "import {module}" statements in the
rest of our code will find our pre-loaded compatibility module.
"""

__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"

def import_as(module, name):
    """
    Imports the specified module (from our local directory) as the
    specified name, returning the loaded module object.
    """
    import imp
    import os.path
    dir = os.path.split(__file__)[0]
    file, filename, suffix_mode_type = imp.find_module(module, [dir])
    return imp.load_module(name, file, filename, suffix_mode_type)

try:
    import builtins
except ImportError:
    # Use the "imp" module to protect the import from fixers.
    import imp
    import sys
    _builtin = imp.load_module('__builtin__',
                               *imp.find_module('__builtin__'))
    sys.modules['builtins'] = _builtin
    del _builtin

import _scons_builtins

try:
    import hashlib
except ImportError:
    # Pre-2.5 Python has no hashlib module.
    try:
        import_as('_scons_hashlib', 'hashlib')
    except ImportError:
        # If we failed importing our compatibility module, it probably
        # means this version of Python has no md5 module.  Don't do
        # anything and let the higher layer discover this fact, so it
        # can fall back to using timestamp.
        pass

try:
    set
except NameError:
    # Pre-2.4 Python has no native set type
    import_as('_scons_sets', 'sets')
    import builtins, sets
    builtins.set = sets.Set

try:
    import collections
except ImportError:
    # Pre-2.4 Python has no collections module.
    import_as('_scons_collections', 'collections')
else:
    # Use the "imp" module to protect the imports below from fixers.
    import imp
    try:
        collections.UserDict
    except AttributeError:
        _UserDict = imp.load_module('UserDict', *imp.find_module('UserDict'))
        collections.UserDict = _UserDict.UserDict
        del _UserDict
    try:
        collections.UserList
    except AttributeError:
        _UserList = imp.load_module('UserList', *imp.find_module('UserList'))
        collections.UserList = _UserList.UserList
        del _UserList
    try:
        collections.UserString
    except AttributeError:
        _UserString = imp.load_module('UserString',
                                      *imp.find_module('UserString'))
        collections.UserString = _UserString.UserString
        del _UserString

try:
    import dbm
except ImportError:
    dbm = import_as('_scons_dbm', 'dbm')
try:
    dbm.whichdb
except AttributeError:
    # Pre-3.0 Python has no dbm.whichdb function.
    import whichdb
    dbm.whichdb = whichdb.whichdb
    del whichdb

try:
    import io
except ImportError:
    # Pre-2.6 Python has no io module.
    import_as('_scons_io', 'io')

import os
try:
    os.devnull
except AttributeError:
    # Pre-2.4 Python has no os.devnull attribute
    import sys
    _names = sys.builtin_module_names
    if 'posix' in _names:
        os.devnull = '/dev/null'
    elif 'nt' in _names:
        os.devnull = 'nul'
    os.path.devnull = os.devnull
try:
    os.path.lexists
except AttributeError:
    # Pre-2.4 Python has no os.path.lexists function
    def lexists(path):
        return os.path.exists(path) or os.path.islink(path)
    os.path.lexists = lexists

# When we're using the '-3' option during regression tests, importing
# cPickle gives a warning no matter how it's done, so always use the
# real profile module, whether it's fast or not.
if os.environ.get('SCONS_HORRIBLE_REGRESSION_TEST_HACK') is None:
    # Not a regression test with '-3', so try to use faster version.
    try:
        # Use the "imp" module to protect the import from fixers.
        import imp
        _cPickle = imp.load_module('cPickle', *imp.find_module('cPickle'))
    except ImportError, e:
        # The "cPickle" module has already been eliminated in favor of
        # having "import pickle" import the fast version when available.
        pass
    else:
        import sys
        sys.modules['pickle'] = _cPickle
        del _cPickle

try:
    # Use the "imp" module to protect the import from fixers.
    import imp
    _cProfile = imp.load_module('cProfile', *imp.find_module('cProfile'))
except ImportError:
    # The "cProfile" module has already been eliminated in favor of
    # having "import profile" import the fast version when available.
    pass
else:
    import sys
    sys.modules['profile'] = _cProfile
    del _cProfile

try:
    import platform
except ImportError:
    # Pre-2.3 Python has no platform module.
    import_as('_scons_platform', 'platform')

try:
    import queue
except ImportError:
    # Before Python 3.0, the 'queue' module was named 'Queue'.
    import imp
    file, filename, suffix_mode_type = imp.find_module('Queue')
    imp.load_module('queue', file, filename, suffix_mode_type)

try:
    import subprocess
except ImportError:
    # Pre-2.4 Python has no subprocess module.
    import_as('_scons_subprocess', 'subprocess')

import sys
try:
    sys.intern
except AttributeError:
    # Pre-2.6 Python has no sys.intern() function.
    import builtins
    try:
        sys.intern = builtins.intern
    except AttributeError:
        # Pre-2.x Python has no builtin intern() function.
        def intern(x):
           return x
        sys.intern = intern
        del intern
try:
    sys.maxsize
except AttributeError:
    # Pre-2.6 Python has no sys.maxsize attribute
    # Wrapping sys in () is silly, but protects it from 2to3 renames fixer
    sys.maxsize = (sys).maxint

import tempfile
try:
    tempfile.mkstemp
except AttributeError:
    # Pre-2.3 Python has no tempfile.mkstemp function, so try to simulate it.
    # adapted from the mkstemp implementation in python 3.
    import os
    import errno
    def mkstemp(*args, **kw):
        text = False
        # TODO (1.5)
        #if 'text' in kw :
        if 'text' in kw.keys() :
            text = kw['text']
            del kw['text']
        elif len( args ) == 4 :
            text = args[3]
            args = args[:3]
        flags = os.O_RDWR | os.O_CREAT | os.O_EXCL
        if not text and hasattr( os, 'O_BINARY' ) :
            flags = flags | os.O_BINARY
        while True:
            try :
                name = tempfile.mktemp(*args, **kw)
                fd = os.open( name, flags, 0600 )
                return (fd, os.path.abspath(name))
            except OSError, e:
                if e.errno == errno.EEXIST:
                    continue
                raise

    tempfile.mkstemp = mkstemp
    del mkstemp

if os.environ.get('SCONS_HORRIBLE_REGRESSION_TEST_HACK') is not None:
    # We can't apply the 'callable' fixer until the floor is 2.6, but the
    # '-3' option to Python 2.6 and 2.7 generates almost ten thousand
    # warnings.  This hack allows us to run regression tests with the '-3'
    # option by replacing the callable() built-in function with a hack
    # that performs the same function but doesn't generate the warning.
    # Note that this hack is ONLY intended to be used for regression
    # testing, and should NEVER be used for real runs.
    from types import ClassType
    def callable(obj):
        if hasattr(obj, '__call__'): return True
        if isinstance(obj, (ClassType, type)): return True
        return False
    import builtins
    builtins.callable = callable
    del callable

# Local Variables:
# tab-width:4
# indent-tabs-mode:nil
# End:
# vim: set expandtab tabstop=4 shiftwidth=4:
<|endoftext|>