<commit_before># -*- coding: utf-8 -*-
u"""SRW execution template.

:copyright: Copyright (c) 2015 RadiaSoft LLC.  All Rights Reserved.
:license: http://www.apache.org/licenses/LICENSE-2.0.html
"""
from __future__ import absolute_import, division, print_function

from pykern import pkio
from pykern import pkjinja
from . import template_common
import h5py
import numpy as np
import os
import re

#: How long before killing WARP process
MAX_SECONDS = 60 * 60

_MODE_TEXT = {
    '0': '0',
    '1': '1 (real part)',
    '2': '1 (imaginary part)',
}

_PARTICLE_ARG_PATH = {
    'x' : 'position/x',
    'y' : 'position/y',
    'z' : 'position/z',
    'ux' : 'momentum/x',
    'uy' : 'momentum/y',
    'uz' : 'momentum/z',
}

def background_percent_complete(data, persistent_files_dir, is_running):
    simulation_id = data['models']['simulation']['simulationId']
    files = _h5_file_list(str(persistent_files_dir.join('diags/hdf5')))
    if len(files) < 2:
        return {
            'percent_complete': 0,
            'frame_count': 0,
            'total_frames': 0,
        }
    file_index = len(files) - 1
    # look at 2nd to last file if running, last one may be incomplete
    if is_running:
        file_index -= 1
    field = 'E'
    coordinate = 'r'
    Lplasma_lab = float(data['models']['electronPlasma']['length']) / 1e3
    zmmin = float(data['models']['simulationGrid']['zMin']) / 1e6
    dfile = h5py.File(files[file_index], 'r')
    dset = dfile['fields/{}/{}'.format(field, coordinate)]
    dz = dset.attrs['dx']
    zmin = dset.attrs['xmin']
    percent_complete = (zmin - zmmin) / (Lplasma_lab - zmmin)
    if percent_complete < 0:
        percent_complete = 0.0
    elif percent_complete > 1.0:
        percent_complete = 1.0
    return {
        'percent_complete': percent_complete * 100,
        'frame_count': file_index + 1,
        #TODO(pjm): this isn't a great calculation ...
        'total_frames': int((file_index + 1) / percent_complete + 0.3),
    }

def fixup_old_data(data):
    if 'laserPreviewReport' not in data['models']:
        data['models']['laserPreviewReport'] = {}
    if 'particleAnimation' not in data['models']:
        data['models']['particleAnimation'] = {
            'x': 'z',
            'y': 'x',
            'histogramBins': 100,
        }
    if 'simulationStatus' not in data['models']:
        data['models']['simulationStatus'] = {
            'startTime': 0,
            'state': 'initial',
        }
    if 'startTime' not in data['models']['simulationStatus']:
        data['models']['simulationStatus']['startTime'] = 0
    if 'histogramBins' not in data['models']['particleAnimation']:
        data['models']['particleAnimation']['histogramBins'] = 100
    if 'framesPerSecond' not in data['models']['fieldAnimation']:
        data['models']['fieldAnimation']['framesPerSecond'] = 20
        data['models']['particleAnimation']['framesPerSecond'] = 20
    if 'rScale' not in data['models']['simulationGrid']:
        grid = data['models']['simulationGrid']
        grid['rScale'] = 4
        grid['rLength'] = '20.324980154380'
        grid['rMin'] = 0
        grid['rMax'] = '20.324980154380'
        grid['rCellsPerSpotSize'] = 8
        grid['rCount'] = 32
        grid['zScale'] = 2
        grid['zLength'] = '10.162490077189'
        grid['zMin'] = '-10.162490077189'
        grid['zMax'] = '1.64'
        grid['zCellsPerWavelength'] = 8
        grid['zCount'] = 115
        del grid['xMin']
        del grid['xMax']
        del grid['xCount']
        del grid['zLambda']

def generate_parameters_file(data, schema, persistent_files_dir=None):
    _validate_data(data, schema)
    v = template_common.flatten_data(data['models'], {})
    if persistent_files_dir:
        if not persistent_files_dir.check():
            pkio.mkdir_parent(persistent_files_dir)
        v['outputDir'] = '"{}"'.format(persistent_files_dir)
    else:
        v['outputDir'] = None
    if 'report' in data and data['report'] == 'laserPreviewReport':
        v['enablePlasma'] = 0
    else:
        v['enablePlasma'] = 1
    return pkjinja.render_resource('warp.py', v)

def get_simulation_frame(persistent_files_dir, data):
    frame_index = int(data['frame_index'])
    files = _h5_file_list(str(persistent_files_dir.join('diags/hdf5')))
    filename = files[frame_index]
    iteration = int(re.search(r'data(\d+)', filename).group(1))

    #TODO(pjm): consolidate with pykern_cli/warp.py
    dfile = h5py.File(filename, "r")

    args = data['animation_args'].split('_')

    if data['model_name'] == 'fieldAnimation':
        return _field_animation(args, dfile, iteration, len(files))
    if data['model_name'] == 'particleAnimation':
        return _particle_animation(args, dfile, iteration, len(files))
    raise RuntimeError('{}: unknown simulation frame model'.format(data['model_name']))

def prepare_aux_files(wd):
    pass

def remove_last_frame(persistent_files_dir):
    files = _h5_file_list(str(persistent_files_dir.join('diags/hdf5')))
    if len(files) > 0:
        os.remove(files[-1])

def run_all_text():
    return '''
doit = True
while(doit):
    step(10)
    doit = ( w3d.zmmin + top.zgrid < Lplasma )
'''

def _field_animation(args, dfile, iteration, frame_count):
    field = args[0]
    coordinate = args[1]
    mode = int(args[2])

    if field == 'rho' :
        dset = dfile['fields/rho']
        coordinate = ''
    else:
        dset = dfile['fields/{}/{}'.format(field, coordinate)]
    F = np.flipud(np.array(dset[mode,:,:]).T)
    Nr, Nz = F.shape[0], F.shape[1]
    dz = dset.attrs['dx']
    dr = dset.attrs['dy']
    zmin = dset.attrs['xmin']
    extent = np.array([zmin-0.5*dz, zmin+0.5*dz+dz*Nz, 0., (Nr+1)*dr])
    return {
        'x_range': [extent[0], extent[1], len(F[0])],
        'y_range': [extent[2], extent[3], len(F)],
        'x_label': 'z [m]',
        'y_label': 'r [m]',
        'title': "{} {} in the mode {} at {}".format(
            field, coordinate, _MODE_TEXT[str(mode)], _iteration_title(dfile, iteration)),
        'z_matrix': np.flipud(F).tolist(),
        'frameCount': frame_count,
    }

def _h5_file_list(path_to_dir):
    if not os.path.isdir(path_to_dir):
        return []
    all_files = os.listdir(path_to_dir)

    # Select the hdf5 files
    h5_files = []
    for filename in all_files :
        if filename[-3:] == '.h5' :
            h5_files.append( os.path.join( path_to_dir, filename) )
    # Sort them
    h5_files.sort()
    return h5_files

def _iteration_title(dfile, iteration):
    return '{:.1f} fs (iteration {})'.format(
        iteration * 1e15 * float(dfile.attrs['timeStepUnitSI']), iteration)

def _particle_animation(args, dfile, iteration, frame_count):
    xarg = args[0]
    yarg = args[1]
    histogramBins = args[2]
    x = dfile['particles/electrons/{}'.format(_PARTICLE_ARG_PATH[xarg])][:]
    y = dfile['particles/electrons/{}'.format(_PARTICLE_ARG_PATH[yarg])][:]
    hist, edges = np.histogramdd([x, y], int(histogramBins))
    return {
        'x_range': [float(edges[0][0]), float(edges[0][-1]), len(hist)],
        'y_range': [float(edges[1][0]), float(edges[1][-1]), len(hist[0])],
        'x_label': '{} [m]'.format(xarg),
        'y_label': '{} [m]'.format(yarg),
        'title': 't = {}'.format(_iteration_title(dfile, iteration)),
        'z_matrix': hist.T.tolist(),
        'frameCount': frame_count,
    }

def _validate_data(data, schema):
    # ensure enums match, convert ints/floats, apply scaling
    enum_info = template_common.parse_enums(schema['enum'])
    for k in data['models']:
        if k in schema['model']:
            template_common.validate_model(data['models'][k], schema['model'][k], enum_info)
<commit_msg>added fixups for rParticlesPerCell and zParticlesPerCell, share field report run with cli<commit_after># -*- coding: utf-8 -*-
u"""SRW execution template.

:copyright: Copyright (c) 2015 RadiaSoft LLC.  All Rights Reserved.
:license: http://www.apache.org/licenses/LICENSE-2.0.html
"""
from __future__ import absolute_import, division, print_function

from pykern import pkio
from pykern import pkjinja
from . import template_common
import h5py
import numpy as np
import os
import re

#: How long before killing WARP process
MAX_SECONDS = 60 * 60

_MODE_TEXT = {
    '0': '0',
    '1': '1 (real part)',
    '2': '1 (imaginary part)',
}

_PARTICLE_ARG_PATH = {
    'x' : 'position/x',
    'y' : 'position/y',
    'z' : 'position/z',
    'ux' : 'momentum/x',
    'uy' : 'momentum/y',
    'uz' : 'momentum/z',
}

def background_percent_complete(data, persistent_files_dir, is_running):
    simulation_id = data['models']['simulation']['simulationId']
    files = _h5_file_list(str(persistent_files_dir.join('diags/hdf5')))
    if len(files) < 2:
        return {
            'percent_complete': 0,
            'frame_count': 0,
            'total_frames': 0,
        }
    file_index = len(files) - 1
    # look at 2nd to last file if running, last one may be incomplete
    if is_running:
        file_index -= 1
    field = 'E'
    coordinate = 'r'
    Lplasma_lab = float(data['models']['electronPlasma']['length']) / 1e3
    zmmin = float(data['models']['simulationGrid']['zMin']) / 1e6
    dfile = h5py.File(files[file_index], 'r')
    dset = dfile['fields/{}/{}'.format(field, coordinate)]
    dz = dset.attrs['dx']
    zmin = dset.attrs['xmin']
    percent_complete = (zmin - zmmin) / (Lplasma_lab - zmmin)
    if percent_complete < 0:
        percent_complete = 0.0
    elif percent_complete > 1.0:
        percent_complete = 1.0
    return {
        'percent_complete': percent_complete * 100,
        'frame_count': file_index + 1,
        #TODO(pjm): this isn't a great calculation ...
        'total_frames': int((file_index + 1) / percent_complete + 0.3),
    }

def extract_field_report(field, coordinate, mode, dfile, iteration):
    if field == 'rho' :
        dset = dfile['fields/rho']
        coordinate = ''
    else:
        dset = dfile['fields/{}/{}'.format(field, coordinate)]
    F = np.flipud(np.array(dset[mode,:,:]).T)
    Nr, Nz = F.shape[0], F.shape[1]
    dz = dset.attrs['dx']
    dr = dset.attrs['dy']
    zmin = dset.attrs['xmin']
    extent = np.array([zmin-0.5*dz, zmin+0.5*dz+dz*Nz, 0., (Nr+1)*dr])
    return {
        'x_range': [extent[0], extent[1], len(F[0])],
        'y_range': [extent[2], extent[3], len(F)],
        'x_label': 'z [m]',
        'y_label': 'r [m]',
        'title': "{} {} in the mode {} at {}".format(
            field, coordinate, _MODE_TEXT[str(mode)], _iteration_title(dfile, iteration)),
        'z_matrix': np.flipud(F).tolist(),
    }

def fixup_old_data(data):
    if 'laserPreviewReport' not in data['models']:
        data['models']['laserPreviewReport'] = {}
    if 'particleAnimation' not in data['models']:
        data['models']['particleAnimation'] = {
            'x': 'z',
            'y': 'x',
            'histogramBins': 100,
        }
    if 'simulationStatus' not in data['models']:
        data['models']['simulationStatus'] = {
            'startTime': 0,
            'state': 'initial',
        }
    if 'startTime' not in data['models']['simulationStatus']:
        data['models']['simulationStatus']['startTime'] = 0
    if 'histogramBins' not in data['models']['particleAnimation']:
        data['models']['particleAnimation']['histogramBins'] = 100
    if 'framesPerSecond' not in data['models']['fieldAnimation']:
        data['models']['fieldAnimation']['framesPerSecond'] = 20
        data['models']['particleAnimation']['framesPerSecond'] = 20
    if 'rScale' not in data['models']['simulationGrid']:
        grid = data['models']['simulationGrid']
        grid['rScale'] = 4
        grid['rLength'] = '20.324980154380'
        grid['rMin'] = 0
        grid['rMax'] = '20.324980154380'
        grid['rCellsPerSpotSize'] = 8
        grid['rCount'] = 32
        grid['zScale'] = 2
        grid['zLength'] = '10.162490077189'
        grid['zMin'] = '-10.162490077189'
        grid['zMax'] = '1.64'
        grid['zCellsPerWavelength'] = 8
        grid['zCount'] = 115
        del grid['xMin']
        del grid['xMax']
        del grid['xCount']
        del grid['zLambda']
    if 'rParticlesPerCell' not in data['models']['simulationGrid']:
        data['models']['simulationGrid']['rParticlesPerCell'] = 1
        data['models']['simulationGrid']['zParticlesPerCell'] = 2
    if 'field' not in data['models']['laserPreviewReport']:
        laserPreview = data['models']['laserPreviewReport']
        laserPreview['field'] = 'E'
        laserPreview['coordinate'] = 'r'
        laserPreview['mode'] = '1'

def generate_parameters_file(data, schema, persistent_files_dir=None):
    _validate_data(data, schema)
    v = template_common.flatten_data(data['models'], {})
    if persistent_files_dir:
        if not persistent_files_dir.check():
            pkio.mkdir_parent(persistent_files_dir)
        v['outputDir'] = '"{}"'.format(persistent_files_dir)
    else:
        v['outputDir'] = None
    v['enablePlasma'] = 1
    return pkjinja.render_resource('warp.py', v)

def get_simulation_frame(persistent_files_dir, data):
    frame_index = int(data['frame_index'])
    files = _h5_file_list(str(persistent_files_dir.join('diags/hdf5')))
    filename = files[frame_index]
    iteration = int(re.search(r'data(\d+)', filename).group(1))
    dfile = h5py.File(filename, "r")
    args = data['animation_args'].split('_')

    if data['model_name'] == 'fieldAnimation':
        return _field_animation(args, dfile, iteration, len(files))
    if data['model_name'] == 'particleAnimation':
        return _particle_animation(args, dfile, iteration, len(files))
    raise RuntimeError('{}: unknown simulation frame model'.format(data['model_name']))

def prepare_aux_files(wd):
    pass

def remove_last_frame(persistent_files_dir):
    files = _h5_file_list(str(persistent_files_dir.join('diags/hdf5')))
    if len(files) > 0:
        os.remove(files[-1])

def run_all_text():
    return '''
doit = True
while(doit):
    step(10)
    doit = ( w3d.zmmin + top.zgrid < Lplasma )
'''

def _field_animation(args, dfile, iteration, frame_count):
    field = args[0]
    coordinate = args[1]
    mode = int(args[2])
    res = extract_field_report(field, coordinate, mode, dfile, iteration)
    res['frameCount'] = frame_count
    return res

def _h5_file_list(path_to_dir):
    if not os.path.isdir(path_to_dir):
        return []
    all_files = os.listdir(path_to_dir)

    # Select the hdf5 files
    h5_files = []
    for filename in all_files :
        if filename[-3:] == '.h5' :
            h5_files.append( os.path.join( path_to_dir, filename) )
    # Sort them
    h5_files.sort()
    return h5_files

def _iteration_title(dfile, iteration):
    return '{:.1f} fs (iteration {})'.format(
        iteration * 1e15 * float(dfile.attrs['timeStepUnitSI']), iteration)

def _particle_animation(args, dfile, iteration, frame_count):
    xarg = args[0]
    yarg = args[1]
    histogramBins = args[2]
    x = dfile['particles/electrons/{}'.format(_PARTICLE_ARG_PATH[xarg])][:]
    y = dfile['particles/electrons/{}'.format(_PARTICLE_ARG_PATH[yarg])][:]
    hist, edges = np.histogramdd([x, y], int(histogramBins))
    return {
        'x_range': [float(edges[0][0]), float(edges[0][-1]), len(hist)],
        'y_range': [float(edges[1][0]), float(edges[1][-1]), len(hist[0])],
        'x_label': '{} [m]'.format(xarg),
        'y_label': '{} [m]'.format(yarg),
        'title': 't = {}'.format(_iteration_title(dfile, iteration)),
        'z_matrix': hist.T.tolist(),
        'frameCount': frame_count,
    }

def _validate_data(data, schema):
    # ensure enums match, convert ints/floats, apply scaling
    enum_info = template_common.parse_enums(schema['enum'])
    for k in data['models']:
        if k in schema['model']:
            template_common.validate_model(data['models'][k], schema['model'][k], enum_info)
<|endoftext|>


<commit_before># Copyright 2014
# The Cloudscaling Group, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Starting point for routing EC2 requests.
"""
import hashlib
import sys

import botocore
from keystoneclient import access as keystone_access
from keystoneclient.auth.identity import access as keystone_identity_access
from keystoneclient import session as keystone_session
from oslo_config import cfg
from oslo_context import context as common_context
from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils import timeutils
import requests
import six
import webob
import webob.dec
import webob.exc

from ec2api.api import apirequest
from ec2api.api import ec2utils
from ec2api.api import faults
from ec2api import clients
from ec2api import context
from ec2api import exception
from ec2api.i18n import _
from ec2api import wsgi

LOG = logging.getLogger(__name__)

ec2_opts = [
    cfg.StrOpt('keystone_url',
               default='http://localhost:5000/',
               deprecated_for_removal=True,
               deprecated_reason='code was switched to common section '
                                 '"keystone_authtoken"',
               deprecated_since='Newton',
               help='URL for getting admin session.'),
    cfg.StrOpt('keystone_ec2_tokens_url',
               default='http://localhost:5000/v3/ec2tokens',
               help='URL to authenticate token from ec2 request.'),
    cfg.IntOpt('ec2_timestamp_expiry',
               default=300,
               help='Time in seconds before ec2 timestamp expires'),
]

CONF = cfg.CONF
CONF.register_opts(ec2_opts)
CONF.import_opt('use_forwarded_for', 'ec2api.api.auth')

# Fault Wrapper around all EC2 requests #
class FaultWrapper(wsgi.Middleware):

    """Calls the middleware stack, captures any exceptions into faults."""

    @webob.dec.wsgify(RequestClass=wsgi.Request)
    def __call__(self, req):
        try:
            return req.get_response(self.application)
        except Exception:
            LOG.exception("FaultWrapper catches error")
            return faults.Fault(webob.exc.HTTPInternalServerError())

class RequestLogging(wsgi.Middleware):

    """Access-Log akin logging for all EC2 API requests."""

    @webob.dec.wsgify(RequestClass=wsgi.Request)
    def __call__(self, req):
        start = timeutils.utcnow()
        rv = req.get_response(self.application)
        self.log_request_completion(rv, req, start)
        return rv

    def log_request_completion(self, response, request, start):
        apireq = request.environ.get('ec2.request', None)
        if apireq:
            action = apireq.action
        else:
            action = None
        ctxt = request.environ.get('ec2api.context', None)
        delta = timeutils.utcnow() - start
        seconds = delta.seconds
        microseconds = delta.microseconds
        LOG.info(
            "%s.%ss %s %s %s %s %s [%s] %s %s",
            seconds,
            microseconds,
            request.remote_addr,
            request.method,
            "%s%s" % (request.script_name, request.path_info),
            action,
            response.status_int,
            request.user_agent,
            request.content_type,
            response.content_type,
            context=ctxt)

class EC2KeystoneAuth(wsgi.Middleware):

    """Authenticate an EC2 request with keystone and convert to context."""

    def _get_signature(self, req):
        """Extract the signature from the request.

        This can be a get/post variable or for version 4 also in a header
        called 'Authorization'.
        - params['Signature'] == version 0,1,2,3
        - params['X-Amz-Signature'] == version 4
        - header 'Authorization' == version 4
        """
        sig = req.params.get('Signature') or req.params.get('X-Amz-Signature')
        if sig is not None:
            return sig

        if 'Authorization' not in req.headers:
            return None

        auth_str = req.headers['Authorization']
        if not auth_str.startswith('AWS4-HMAC-SHA256'):
            return None

        return auth_str.partition("Signature=")[2].split(',')[0]

    def _get_access(self, req):
        """Extract the access key identifier.

        For version 0/1/2/3 this is passed as the AccessKeyId parameter, for
        version 4 it is either an X-Amz-Credential parameter or a Credential=
        field in the 'Authorization' header string.
        """
        access = req.params.get('AWSAccessKeyId')
        if access is not None:
            return access

        cred_param = req.params.get('X-Amz-Credential')
        if cred_param:
            access = cred_param.split("/")[0]
            if access is not None:
                return access

        if 'Authorization' not in req.headers:
            return None
        auth_str = req.headers['Authorization']
        if not auth_str.startswith('AWS4-HMAC-SHA256'):
            return None
        cred_str = auth_str.partition("Credential=")[2].split(',')[0]
        return cred_str.split("/")[0]

    @webob.dec.wsgify(RequestClass=wsgi.Request)
    def __call__(self, req):
        request_id = common_context.generate_request_id()

        # NOTE(alevine) We need to calculate the hash here because
        # subsequent access to request modifies the req.body so the hash
        # calculation will yield invalid results.
        body_hash = hashlib.sha256(req.body).hexdigest()

        signature = self._get_signature(req)
        if not signature:
            msg = _("Signature not provided")
            return faults.ec2_error_response(request_id, "AuthFailure", msg,
                                             status=400)
        access = self._get_access(req)
        if not access:
            msg = _("Access key not provided")
            return faults.ec2_error_response(request_id, "AuthFailure", msg,
                                             status=400)

        if 'X-Amz-Signature' in req.params or 'Authorization' in req.headers:
            params = {}
        else:
            # Make a copy of args for authentication and signature verification
            params = dict(req.params)
            # Not part of authentication args
            params.pop('Signature', None)

        cred_dict = {
            'access': access,
            'signature': signature,
            'host': req.host,
            'verb': req.method,
            'path': req.path,
            'params': params,
            # python3 takes only keys for json from headers object
            'headers': {k: req.headers[k] for k in req.headers},
            'body_hash': body_hash
        }

        token_url = CONF.keystone_ec2_tokens_url
        if "ec2" in token_url:
            creds = {'ec2Credentials': cred_dict}
        else:
            creds = {'auth': {'OS-KSEC2:ec2Credentials': cred_dict}}
        creds_json = jsonutils.dumps(creds)
        headers = {'Content-Type': 'application/json'}
        params = {'data': creds_json, 'headers': headers}
        clients.update_request_params_with_ssl(params)
        response = requests.request('POST', token_url, **params)
        status_code = response.status_code
        if status_code != 200:
            msg = response.reason
            return faults.ec2_error_response(request_id, "AuthFailure", msg,
                                             status=status_code)

        try:
            auth_ref = keystone_access.AccessInfo.factory(resp=response,
                                                          body=response.json())
        except (NotImplementedError, KeyError):
            LOG.exception("Keystone failure")
            msg = _("Failure communicating with keystone")
            return faults.ec2_error_response(request_id, "AuthFailure", msg,
                                             status=400)
        auth = keystone_identity_access.AccessInfoPlugin(auth_ref)
        params = {'auth': auth}
        clients.update_request_params_with_ssl(params)
        session = keystone_session.Session(**params)
        remote_address = req.remote_addr
        if CONF.use_forwarded_for:
            remote_address = req.headers.get('X-Forwarded-For',
                                             remote_address)

        ctxt = context.RequestContext(auth_ref.user_id, auth_ref.project_id,
                                      request_id=request_id,
                                      user_name=auth_ref.username,
                                      project_name=auth_ref.project_name,
                                      remote_address=remote_address,
                                      session=session,
                                      api_version=req.params.get('Version'))

        req.environ['ec2api.context'] = ctxt

        return self.application

class Requestify(wsgi.Middleware):

    @webob.dec.wsgify(RequestClass=wsgi.Request)
    def __call__(self, req):
        non_args = ['Action', 'Signature', 'AWSAccessKeyId', 'SignatureMethod',
                    'SignatureVersion', 'Version', 'Timestamp']
        args = dict(req.params)
        try:
            expired = ec2utils.is_ec2_timestamp_expired(
                req.params,
                expires=CONF.ec2_timestamp_expiry)
            if expired:
                msg = _("Timestamp failed validation.")
                LOG.exception(msg)
                raise webob.exc.HTTPForbidden(explanation=msg)

            # Raise KeyError if omitted
            action = req.params['Action']
            # Fix bug lp:720157 for older (version 1) clients
            version = req.params.get('SignatureVersion')
            if version and int(version) == 1:
                non_args.remove('SignatureMethod')
                if 'SignatureMethod' in args:
                    args.pop('SignatureMethod')
            for non_arg in non_args:
                args.pop(non_arg, None)
        except KeyError:
            raise webob.exc.HTTPBadRequest()
        except exception.InvalidRequest as err:
            raise webob.exc.HTTPBadRequest(explanation=unicode(err))

        LOG.debug('action: %s', action)
        for key, value in args.items():
            LOG.debug('arg: %(key)s\t\tval: %(value)s',
                      {'key': key, 'value': value})

        # Success!
        api_request = apirequest.APIRequest(
            action, req.params['Version'], args)
        req.environ['ec2.request'] = api_request
        return self.application

def exception_to_ec2code(ex):
    """Helper to extract EC2 error code from exception.

    For other than EC2 exceptions (those without ec2_code attribute),
    use exception name.
    """
    if hasattr(ex, 'ec2_code'):
        code = ex.ec2_code
    else:
        code = type(ex).__name__
    return code

def ec2_error_ex(ex, req, unexpected=False):
    """Return an EC2 error response.

    Return an EC2 error response based on passed exception and log
    the exception on an appropriate log level:

        * DEBUG: expected errors
        * ERROR: unexpected errors

    All expected errors are treated as client errors and 4xx HTTP
    status codes are always returned for them.

    Unexpected 5xx errors may contain sensitive information,
    suppress their messages for security.
    """
    code = exception_to_ec2code(ex)
    for status_name in ('code', 'status', 'status_code', 'http_status'):
        status = getattr(ex, status_name, None)
        if isinstance(status, int):
            break
    else:
        status = 500

    if unexpected:
        log_fun = LOG.error
        log_msg = _("Unexpected %(ex_name)s raised: %(ex_str)s")
        exc_info = sys.exc_info()
    else:
        log_fun = LOG.debug
        log_msg = _("%(ex_name)s raised: %(ex_str)s")
        exc_info = None

    context = req.environ['ec2api.context']
    request_id = context.request_id
    log_msg_args = {
        'ex_name': type(ex).__name__,
        'ex_str': ex
    }
    log_fun(log_msg % log_msg_args, context=context, exc_info=exc_info)

    if unexpected and status >= 500:
        message = _('Unknown error occurred.')
    elif getattr(ex, 'message', None):
        message = six.text_type(ex.message)
    elif ex.args and any(arg for arg in ex.args):
        message = " ".join(map(six.text_type, ex.args))
    else:
        message = six.text_type(ex)
    if unexpected:
        # Log filtered environment for unexpected errors.
        env = req.environ.copy()
        for k in list(env.keys()):
            if not isinstance(env[k], six.string_types):
                env.pop(k)
        log_fun(_('Environment: %s') % jsonutils.dumps(env))
    return faults.ec2_error_response(request_id, code, message, status=status)

class Executor(wsgi.Application):

    """Execute an EC2 API request.

    Executes 'ec2.action', passing 'ec2api.context' and
    'ec2.action_args' (all variables in WSGI environ.)  Returns an XML
    response, or a 400 upon failure.
    """

    @webob.dec.wsgify(RequestClass=wsgi.Request)
    def __call__(self, req):
        context = req.environ['ec2api.context']
        api_request = req.environ['ec2.request']
        try:
            result = api_request.invoke(context)
        except botocore.exceptions.ClientError as ex:
            error = ex.response.get('Error', {})
            code = ex.response.get('Code', error.get('Code'))
            message = ex.response.get('Message', error.get('Message'))
            # the early versions of botocore didn't provide HTTPStatusCode
            # for 400 errors
            status = ex.response.get('ResponseMetadata', {}).get(
                'HTTPStatusCode', 400)
            if status < 400 or status > 499:
                LOG.exception("Exception from remote server")
            return faults.ec2_error_response(
                context.request_id, code, message, status=status)
        except Exception as ex:
            return ec2_error_ex(
                ex, req, unexpected=not isinstance(ex, exception.EC2Exception))
        else:
            resp = webob.Response()
            resp.status = 200
            resp.headers['Content-Type'] = 'text/xml'
            resp.body = six.binary_type(result)

            return resp
<commit_msg>Correct import of keystoneauth1 session<commit_after># Copyright 2014
# The Cloudscaling Group, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Starting point for routing EC2 requests.
"""
import hashlib
import sys

import botocore
from keystoneauth1 import session as keystone_session
from keystoneclient import access as keystone_access
from keystoneclient.auth.identity import access as keystone_identity_access
from oslo_config import cfg
from oslo_context import context as common_context
from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils import timeutils
import requests
import six
import webob
import webob.dec
import webob.exc

from ec2api.api import apirequest
from ec2api.api import ec2utils
from ec2api.api import faults
from ec2api import clients
from ec2api import context
from ec2api import exception
from ec2api.i18n import _
from ec2api import wsgi

LOG = logging.getLogger(__name__)

ec2_opts = [
    cfg.StrOpt('keystone_url',
               default='http://localhost:5000/',
               deprecated_for_removal=True,
               deprecated_reason='code was switched to common section '
                                 '"keystone_authtoken"',
               deprecated_since='Newton',
               help='URL for getting admin session.'),
    cfg.StrOpt('keystone_ec2_tokens_url',
               default='http://localhost:5000/v3/ec2tokens',
               help='URL to authenticate token from ec2 request.'),
    cfg.IntOpt('ec2_timestamp_expiry',
               default=300,
               help='Time in seconds before ec2 timestamp expires'),
]

CONF = cfg.CONF
CONF.register_opts(ec2_opts)
CONF.import_opt('use_forwarded_for', 'ec2api.api.auth')

# Fault Wrapper around all EC2 requests #
class FaultWrapper(wsgi.Middleware):

    """Calls the middleware stack, captures any exceptions into faults."""

    @webob.dec.wsgify(RequestClass=wsgi.Request)
    def __call__(self, req):
        try:
            return req.get_response(self.application)
        except Exception:
            LOG.exception("FaultWrapper catches error")
            return faults.Fault(webob.exc.HTTPInternalServerError())

class RequestLogging(wsgi.Middleware):

    """Access-Log akin logging for all EC2 API requests."""

    @webob.dec.wsgify(RequestClass=wsgi.Request)
    def __call__(self, req):
        start = timeutils.utcnow()
        rv = req.get_response(self.application)
        self.log_request_completion(rv, req, start)
        return rv

    def log_request_completion(self, response, request, start):
        apireq = request.environ.get('ec2.request', None)
        if apireq:
            action = apireq.action
        else:
            action = None
        ctxt = request.environ.get('ec2api.context', None)
        delta = timeutils.utcnow() - start
        seconds = delta.seconds
        microseconds = delta.microseconds
        LOG.info(
            "%s.%ss %s %s %s %s %s [%s] %s %s",
            seconds,
            microseconds,
            request.remote_addr,
            request.method,
            "%s%s" % (request.script_name, request.path_info),
            action,
            response.status_int,
            request.user_agent,
            request.content_type,
            response.content_type,
            context=ctxt)

class EC2KeystoneAuth(wsgi.Middleware):

    """Authenticate an EC2 request with keystone and convert to context."""

    def _get_signature(self, req):
        """Extract the signature from the request.

        This can be a get/post variable or for version 4 also in a header
        called 'Authorization'.
        - params['Signature'] == version 0,1,2,3
        - params['X-Amz-Signature'] == version 4
        - header 'Authorization' == version 4
        """
        sig = req.params.get('Signature') or req.params.get('X-Amz-Signature')
        if sig is not None:
            return sig

        if 'Authorization' not in req.headers:
            return None

        auth_str = req.headers['Authorization']
        if not auth_str.startswith('AWS4-HMAC-SHA256'):
            return None

        return auth_str.partition("Signature=")[2].split(',')[0]

    def _get_access(self, req):
        """Extract the access key identifier.

        For version 0/1/2/3 this is passed as the AccessKeyId parameter, for
        version 4 it is either an X-Amz-Credential parameter or a Credential=
        field in the 'Authorization' header string.
        """
        access = req.params.get('AWSAccessKeyId')
        if access is not None:
            return access

        cred_param = req.params.get('X-Amz-Credential')
        if cred_param:
            access = cred_param.split("/")[0]
            if access is not None:
                return access

        if 'Authorization' not in req.headers:
            return None
        auth_str = req.headers['Authorization']
        if not auth_str.startswith('AWS4-HMAC-SHA256'):
            return None
        cred_str = auth_str.partition("Credential=")[2].split(',')[0]
        return cred_str.split("/")[0]

    @webob.dec.wsgify(RequestClass=wsgi.Request)
    def __call__(self, req):
        request_id = common_context.generate_request_id()

        # NOTE(alevine) We need to calculate the hash here because
        # subsequent access to request modifies the req.body so the hash
        # calculation will yield invalid results.
        body_hash = hashlib.sha256(req.body).hexdigest()

        signature = self._get_signature(req)
        if not signature:
            msg = _("Signature not provided")
            return faults.ec2_error_response(request_id, "AuthFailure", msg,
                                             status=400)
        access = self._get_access(req)
        if not access:
            msg = _("Access key not provided")
            return faults.ec2_error_response(request_id, "AuthFailure", msg,
                                             status=400)

        if 'X-Amz-Signature' in req.params or 'Authorization' in req.headers:
            params = {}
        else:
            # Make a copy of args for authentication and signature verification
            params = dict(req.params)
            # Not part of authentication args
            params.pop('Signature', None)

        cred_dict = {
            'access': access,
            'signature': signature,
            'host': req.host,
            'verb': req.method,
            'path': req.path,
            'params': params,
            # python3 takes only keys for json from headers object
            'headers': {k: req.headers[k] for k in req.headers},
            'body_hash': body_hash
        }

        token_url = CONF.keystone_ec2_tokens_url
        if "ec2" in token_url:
            creds = {'ec2Credentials': cred_dict}
        else:
            creds = {'auth': {'OS-KSEC2:ec2Credentials': cred_dict}}
        creds_json = jsonutils.dumps(creds)
        headers = {'Content-Type': 'application/json'}
        params = {'data': creds_json, 'headers': headers}
        clients.update_request_params_with_ssl(params)
        response = requests.request('POST', token_url, **params)
        status_code = response.status_code
        if status_code != 200:
            msg = response.reason
            return faults.ec2_error_response(request_id, "AuthFailure", msg,
                                             status=status_code)

        try:
            auth_ref = keystone_access.AccessInfo.factory(resp=response,
                                                          body=response.json())
        except (NotImplementedError, KeyError):
            LOG.exception("Keystone failure")
            msg = _("Failure communicating with keystone")
            return faults.ec2_error_response(request_id, "AuthFailure", msg,
                                             status=400)
        auth = keystone_identity_access.AccessInfoPlugin(auth_ref)
        params = {'auth': auth}
        clients.update_request_params_with_ssl(params)
        session = keystone_session.Session(**params)
        remote_address = req.remote_addr
        if CONF.use_forwarded_for:
            remote_address = req.headers.get('X-Forwarded-For',
                                             remote_address)

        ctxt = context.RequestContext(auth_ref.user_id, auth_ref.project_id,
                                      request_id=request_id,
                                      user_name=auth_ref.username,
                                      project_name=auth_ref.project_name,
                                      remote_address=remote_address,
                                      session=session,
                                      api_version=req.params.get('Version'))

        req.environ['ec2api.context'] = ctxt

        return self.application

class Requestify(wsgi.Middleware):

    @webob.dec.wsgify(RequestClass=wsgi.Request)
    def __call__(self, req):
        non_args = ['Action', 'Signature', 'AWSAccessKeyId', 'SignatureMethod',
                    'SignatureVersion', 'Version', 'Timestamp']
        args = dict(req.params)
        try:
            expired = ec2utils.is_ec2_timestamp_expired(
                req.params,
                expires=CONF.ec2_timestamp_expiry)
            if expired:
                msg = _("Timestamp failed validation.")
                LOG.exception(msg)
                raise webob.exc.HTTPForbidden(explanation=msg)

            # Raise KeyError if omitted
            action = req.params['Action']
            # Fix bug lp:720157 for older (version 1) clients
            version = req.params.get('SignatureVersion')
            if version and int(version) == 1:
                non_args.remove('SignatureMethod')
                if 'SignatureMethod' in args:
                    args.pop('SignatureMethod')
            for non_arg in non_args:
                args.pop(non_arg, None)
        except KeyError:
            raise webob.exc.HTTPBadRequest()
        except exception.InvalidRequest as err:
            raise webob.exc.HTTPBadRequest(explanation=unicode(err))

        LOG.debug('action: %s', action)
        for key, value in args.items():
            LOG.debug('arg: %(key)s\t\tval: %(value)s',
                      {'key': key, 'value': value})

        # Success!
        api_request = apirequest.APIRequest(
            action, req.params['Version'], args)
        req.environ['ec2.request'] = api_request
        return self.application

def exception_to_ec2code(ex):
    """Helper to extract EC2 error code from exception.

    For other than EC2 exceptions (those without ec2_code attribute),
    use exception name.
    """
    if hasattr(ex, 'ec2_code'):
        code = ex.ec2_code
    else:
        code = type(ex).__name__
    return code

def ec2_error_ex(ex, req, unexpected=False):
    """Return an EC2 error response.

    Return an EC2 error response based on passed exception and log
    the exception on an appropriate log level:

        * DEBUG: expected errors
        * ERROR: unexpected errors

    All expected errors are treated as client errors and 4xx HTTP
    status codes are always returned for them.

    Unexpected 5xx errors may contain sensitive information,
    suppress their messages for security.
    """
    code = exception_to_ec2code(ex)
    for status_name in ('code', 'status', 'status_code', 'http_status'):
        status = getattr(ex, status_name, None)
        if isinstance(status, int):
            break
    else:
        status = 500

    if unexpected:
        log_fun = LOG.error
        log_msg = _("Unexpected %(ex_name)s raised: %(ex_str)s")
        exc_info = sys.exc_info()
    else:
        log_fun = LOG.debug
        log_msg = _("%(ex_name)s raised: %(ex_str)s")
        exc_info = None

    context = req.environ['ec2api.context']
    request_id = context.request_id
    log_msg_args = {
        'ex_name': type(ex).__name__,
        'ex_str': ex
    }
    log_fun(log_msg % log_msg_args, context=context, exc_info=exc_info)

    if unexpected and status >= 500:
        message = _('Unknown error occurred.')
    elif getattr(ex, 'message', None):
        message = six.text_type(ex.message)
    elif ex.args and any(arg for arg in ex.args):
        message = " ".join(map(six.text_type, ex.args))
    else:
        message = six.text_type(ex)
    if unexpected:
        # Log filtered environment for unexpected errors.
        env = req.environ.copy()
        for k in list(env.keys()):
            if not isinstance(env[k], six.string_types):
                env.pop(k)
        log_fun(_('Environment: %s') % jsonutils.dumps(env))
    return faults.ec2_error_response(request_id, code, message, status=status)

class Executor(wsgi.Application):

    """Execute an EC2 API request.

    Executes 'ec2.action', passing 'ec2api.context' and
    'ec2.action_args' (all variables in WSGI environ.)  Returns an XML
    response, or a 400 upon failure.
    """

    @webob.dec.wsgify(RequestClass=wsgi.Request)
    def __call__(self, req):
        context = req.environ['ec2api.context']
        api_request = req.environ['ec2.request']
        try:
            result = api_request.invoke(context)
        except botocore.exceptions.ClientError as ex:
            error = ex.response.get('Error', {})
            code = ex.response.get('Code', error.get('Code'))
            message = ex.response.get('Message', error.get('Message'))
            # the early versions of botocore didn't provide HTTPStatusCode
            # for 400 errors
            status = ex.response.get('ResponseMetadata', {}).get(
                'HTTPStatusCode', 400)
            if status < 400 or status > 499:
                LOG.exception("Exception from remote server")
            return faults.ec2_error_response(
                context.request_id, code, message, status=status)
        except Exception as ex:
            return ec2_error_ex(
                ex, req, unexpected=not isinstance(ex, exception.EC2Exception))
        else:
            resp = webob.Response()
            resp.status = 200
            resp.headers['Content-Type'] = 'text/xml'
            resp.body = six.binary_type(result)

            return resp
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

from qtpy import QtWidgets

from .connect import (connect_checkable_button,
                      connect_value,
                      connect_combo_data,
                      connect_combo_text,
                      connect_float_text,
                      connect_text,
                      connect_button)

__all__ = ['autoconnect_callbacks_to_qt']

HANDLERS = {}
HANDLERS['value'] = connect_value
HANDLERS['valuetext'] = connect_float_text
HANDLERS['bool'] = connect_checkable_button
HANDLERS['text'] = connect_text
HANDLERS['combodata'] = connect_combo_data
HANDLERS['combotext'] = connect_combo_text
HANDLERS['button'] = connect_button

def autoconnect_callbacks_to_qt(instance, widget, connect_kwargs={}):
    """
    Given a class instance with callback properties and a Qt widget/window,
    connect callback properties to Qt widgets automatically.

    The matching is done based on the objectName of the Qt widgets. Qt widgets
    that need to be connected should be named using the syntax ``type_name``
    where ``type`` describes the kind of matching to be done, and ``name``
    matches the name of a callback property. By default, the types can be:

    * ``value``: the callback property is linked to a Qt widget that has
      ``value`` and ``setValue`` methods. Note that for this type, two
      additional keyword arguments can be specified using ``connect_kwargs``
      (see below): these are ``value_range``, which is used for cases where
      the Qt widget is e.g. a slider which has a range of values, and you want
      to map this range of values onto a different range for the callback
      property, and the second is ``log``, which can be set to `True` if this
      mapping should be done in log space.

    * ``valuetext``: the callback property is linked to a Qt widget that has
      ``text`` and ``setText`` methods, and the text is set to a string
      representation of the value. Note that for this type, an additional
      argument ``fmt`` can be provided, which gives either the format to use
      using the ``{}`` syntax, or should be a function that takes a value
      and returns a string. Optionally, if the Qt widget supports
      the ``editingFinished`` signal, this signal is connected to the callback
      property too.

    * ``bool``: the callback property is linked to a Qt widget that has
      ``isChecked`` and ``setChecked`` methods, such as a checkable button.

    * ``text``: the callback property is linked to a Qt widget that has
      ``text`` and ``setText`` methods. Optionally, if the Qt widget supports
      the ``editingFinished`` signal, this signal is connected to the callback
      property too.

    * ``combodata``: the callback property is linked to a QComboBox based on
      the ``userData`` of the entries in the combo box.

    * ``combotext``: the callback property is linked to a QComboBox based on
      the label of the entries in the combo box.

    Applications can also define additional mappings between type and
    auto-linking. To do this, simply add a new entry to the ``HANDLERS`` object::

        >>> echo.qt.autoconnect import HANDLERS
        >>> HANDLERS['color'] = connect_color

    The handler function (``connect_color`` in the example above) should take
    the following arguments: the instance the callback property is attached to,
    the name of the callback property, the Qt widget, and optionally some
    keyword arguments.

    When calling ``autoconnect_callbacks_to_qt``, you can specify
    ``connect_kwargs``, where each key should be a valid callback property name,
    and which gives any additional keyword arguments that can be taken by the
    connect functions, as described above. These include for example
    ``value_range``, ``log``, and ``fmt``.

    This function is especially useful when defining ui files, since widget
    objectNames can be easily set during the editing process.
    """

    if not hasattr(widget, 'children'):
        return

    for child in widget.findChildren(QtWidgets.QWidget):
        full_name = child.objectName()
        if '_' in full_name:
            wtype, wname = full_name.split('_', 1)
            kwargs = connect_kwargs.get(wname, {})
            if hasattr(instance, wname):
                if wtype in HANDLERS:
                    HANDLERS[wtype](instance, wname, child, **kwargs)
<commit_msg>Allow connect_kwargs to have full name instead of short name for widgets in case there are any ambiguities (e.g. two widgets connected to one callback property)<commit_after>from __future__ import absolute_import, division, print_function

from qtpy import QtWidgets

from .connect import (connect_checkable_button,
                      connect_value,
                      connect_combo_data,
                      connect_combo_text,
                      connect_float_text,
                      connect_text,
                      connect_button)

__all__ = ['autoconnect_callbacks_to_qt']

HANDLERS = {}
HANDLERS['value'] = connect_value
HANDLERS['valuetext'] = connect_float_text
HANDLERS['bool'] = connect_checkable_button
HANDLERS['text'] = connect_text
HANDLERS['combodata'] = connect_combo_data
HANDLERS['combotext'] = connect_combo_text
HANDLERS['button'] = connect_button

def autoconnect_callbacks_to_qt(instance, widget, connect_kwargs={}):
    """
    Given a class instance with callback properties and a Qt widget/window,
    connect callback properties to Qt widgets automatically.

    The matching is done based on the objectName of the Qt widgets. Qt widgets
    that need to be connected should be named using the syntax ``type_name``
    where ``type`` describes the kind of matching to be done, and ``name``
    matches the name of a callback property. By default, the types can be:

    * ``value``: the callback property is linked to a Qt widget that has
      ``value`` and ``setValue`` methods. Note that for this type, two
      additional keyword arguments can be specified using ``connect_kwargs``
      (see below): these are ``value_range``, which is used for cases where
      the Qt widget is e.g. a slider which has a range of values, and you want
      to map this range of values onto a different range for the callback
      property, and the second is ``log``, which can be set to `True` if this
      mapping should be done in log space.

    * ``valuetext``: the callback property is linked to a Qt widget that has
      ``text`` and ``setText`` methods, and the text is set to a string
      representation of the value. Note that for this type, an additional
      argument ``fmt`` can be provided, which gives either the format to use
      using the ``{}`` syntax, or should be a function that takes a value
      and returns a string. Optionally, if the Qt widget supports
      the ``editingFinished`` signal, this signal is connected to the callback
      property too.

    * ``bool``: the callback property is linked to a Qt widget that has
      ``isChecked`` and ``setChecked`` methods, such as a checkable button.

    * ``text``: the callback property is linked to a Qt widget that has
      ``text`` and ``setText`` methods. Optionally, if the Qt widget supports
      the ``editingFinished`` signal, this signal is connected to the callback
      property too.

    * ``combodata``: the callback property is linked to a QComboBox based on
      the ``userData`` of the entries in the combo box.

    * ``combotext``: the callback property is linked to a QComboBox based on
      the label of the entries in the combo box.

    Applications can also define additional mappings between type and
    auto-linking. To do this, simply add a new entry to the ``HANDLERS`` object::

        >>> echo.qt.autoconnect import HANDLERS
        >>> HANDLERS['color'] = connect_color

    The handler function (``connect_color`` in the example above) should take
    the following arguments: the instance the callback property is attached to,
    the name of the callback property, the Qt widget, and optionally some
    keyword arguments.

    When calling ``autoconnect_callbacks_to_qt``, you can specify
    ``connect_kwargs``, where each key should be a valid callback property name,
    and which gives any additional keyword arguments that can be taken by the
    connect functions, as described above. These include for example
    ``value_range``, ``log``, and ``fmt``.

    This function is especially useful when defining ui files, since widget
    objectNames can be easily set during the editing process.
    """

    if not hasattr(widget, 'children'):
        return

    for child in widget.findChildren(QtWidgets.QWidget):
        full_name = child.objectName()
        if '_' in full_name:
            wtype, wname = full_name.split('_', 1)
            if full_name in connect_kwargs:
                kwargs = connect_kwargs[full_name]
            elif wname in connect_kwargs:
                kwargs = connect_kwargs[wname]
            else:
                kwargs = {}
            if hasattr(instance, wname):
                if wtype in HANDLERS:
                    HANDLERS[wtype](instance, wname, child, **kwargs)
<|endoftext|>


<commit_before>#!/usr/bin/env python2
# coding: utf-8

import copy

from collections import defaultdict

from pykit.dictutil import FixedKeysDict

from .block_id import BlockID
from .block_desc import BlockDesc
from .block_group_id import BlockGroupID
from .block_index import BlockIndex
from .replication_config import ReplicationConfig

class BlockGroupBaseError(Exception):
    pass

class BlockNotFoundError(BlockGroupBaseError):
    pass

class BlockExists(BlockGroupBaseError):
    pass

class BlockTypeNotSupported(BlockGroupBaseError):
    pass

class BlockTypeNotSupportReplica(BlockGroupBaseError):
    pass

def _idcs(lst):
    return list(lst)

class BlockGroup(FixedKeysDict):

    keys_default = dict(
        block_group_id=BlockGroupID,
        config=ReplicationConfig,
        idcs=_idcs,
        blocks=dict,
    )

    ident_keys = ('block_group_id',)

    def __init__(self, *args, **kwargs):
        super(BlockGroup, self).__init__(*args, **kwargs)
        self.type_map = self.make_type_map()

    def get_block_type(self, block_index):

        mp = self.type_map
        bi = BlockIndex(block_index)

        try:
            return mp[bi.i][bi.j]
        except IndexError:
            raise BlockTypeNotSupported('invalid index at {bi}'.format(bi=bi))

    def make_type_map(self):

        cnf = self['config']

        nr_data, nr_parity = cnf['in_idc']
        nr_in_idc, nr_xor_idc = cnf['cross_idc']
        data_replica = cnf['data_replica']

        rst = []

        prefixes = ('d' * nr_in_idc
                    + 'x' * nr_xor_idc)

        for pref in prefixes:

            o = [pref + '0'] * nr_data
            o += [pref + 'p'] * nr_parity

            for j in range(1, data_replica):
                o += ['%s%d' % (pref, j)] * nr_data

            rst.append(o)

        return rst

    def mark_delete_block(self, block_index):
        block = self.get_block(block_index, raise_error=True)
        block['is_del'] = 1

    def delete_block(self, block_index):
        block = self.get_block(block_index)
        if block is not None:
            del self['blocks'][str(block_index)]

    def mark_delete_block_byid(self, block_id):
        block = self.get_block_byid(block_id)
        if block is not None:
            block['is_del'] = 1

    def delete_block_byid(self, block_id):
        block = self.get_block_byid(block_id)
        if block is not None:
            del self['blocks'][block_id.block_index]

    def has(self, block):
        bid = block['block_id']
        bidx = bid.block_index

        existent = self['blocks'].get(bidx)
        return existent == block

    def add_block(self, new_block, replace=False, allow_exist=False):

        if self.has(new_block) and allow_exist:
            return new_block

        bid = new_block['block_id']
        bidx = bid.block_index

        prev = self['blocks'].get(bidx)
        if not replace and prev is not None:
            raise BlockExists(
                'there is already a block at {bid}'.format(bid=bid))

        self['blocks'][bidx] = new_block

        if prev is None:
            return None
        else:
            return BlockDesc(prev)

    def get_free_block_indexes(self, block_type=None, get_all=False):

        free_block_index = defaultdict(list)

        cnf = self['config']
        n = sum(cnf['cross_idc'])
        m = sum(cnf['in_idc'])

        for i in range(n):
            for j in range(m):
                bi = BlockIndex(i, j)
                typ = self.get_block_type(bi)

                idc = self.get_block_idc(bi)

                if get_all:
                    # set the key 'idc' with default if key not set
                    free_block_index[idc]

                if block_type is not None and typ != block_type:
                    continue

                if self.get_block(bi) is None:
                    free_block_index[idc].append(str(bi))

        return free_block_index

    def get_block(self, block_index, raise_error=False):

        bi = BlockIndex(block_index)
        b = self['blocks'].get(str(bi))

        if raise_error and b is None:
            raise BlockNotFoundError(
                'block_index:{bi}'
                ' not found in block_group:{block_group_id}'.format(bi=bi, **self))

        return b

    def get_block_idc(self, block_index):

        bi = BlockIndex(block_index)

        return self['idcs'][bi.i]

    def get_primary_index(self, block_index):

        nr_data, nr_parity = self['config']['in_idc']

        bi = BlockIndex(block_index)

        j = bi.j
        if j >= nr_data:
            j -= nr_parity
            j %= nr_data

        return BlockIndex(bi.i, j)

    def get_replica_indexes(self, block_index, include_me=True):

        nr_data, nr_parity = self['config']['in_idc']
        data_replica = self['config']['data_replica']

        bi = BlockIndex(block_index)
        typ = self.get_block_type(bi)

        if typ.endswith('p'):
            raise BlockTypeNotSupportReplica(
                'block type {typ}'
                ' does not support replica'.format(typ=typ))

        pbi = self.get_primary_index(block_index)

        rst = [str(pbi)]

        for j in range(1, data_replica):
            rbi = BlockIndex(pbi.i,
                             pbi.j + nr_parity + j * nr_data)
            rst.append(str(rbi))

        # if not include_me and str(block_index) in rst:
        if not include_me:
            rst.remove(str(block_index))

        return rst

    def classify_blocks(self, idc_index, only_primary=True):

        nr_data, nr_parity = self['config']['in_idc']

        ec = []
        replica = []
        mark_del = []

        for i in range(0, nr_data):

            bi = BlockIndex(idc_index, i)

            blk = self.get_block(bi)
            if blk is None:
                continue

            if blk['is_del'] == 1:
                mark_del.append(blk)
                continue

            replica_idxes = self.get_replica_indexes(bi, include_me=False)
            rblks = self.indexes_to_blocks(replica_idxes)

            if None in rblks:
                ec.append(blk)
                continue

            replica.append(blk)
            if only_primary:
                continue

            replica.extend(rblks)

        return {'ec': ec, 'replica': replica, 'mark_del': mark_del}

    def indexes_to_blocks(self, indexes):

        blks = []

        for idx in indexes:
            bi = BlockIndex(idx)

            blk = self.get_block(bi)
            blks.append(blk)

        return blks

    def get_parity_indexes(self, idc_index):

        indexes = []
        nr_data, nr_parity = self['config']['in_idc']

        for i in range(nr_data, nr_data + nr_parity):

            bi = BlockIndex(idc_index, i)
            indexes.append(bi)

        return indexes

    def get_parities(self, idc_index):

        idxes = self.get_parity_indexes(idc_index)

        blks = self.indexes_to_blocks(idxes)

        return [blk for blk in blks if blk is not None]

    def is_ec_block(self, block_id):
        block_id = BlockID(block_id)

        blk = self.get_block(block_id.block_index)
        if blk is None or blk['block_id'] != block_id:
            raise BlockNotFoundError(
                'block_id:{bid}'
                ' not found in block_group:{block_group_id}'.format(bid=block_id, **self))

        if block_id.type.endswith('p'):
            blk = self.get_block(block_id.block_index)
            return True

        r_indexes = self.get_replica_indexes(block_id.block_index)
        r_blks = [self.get_block(x) for x in r_indexes]

        return None in r_blks

    def get_blocks(self):
        blks = []

        for idx in sorted(self['blocks'].keys()):
            blk = self['blocks'][idx]
            blks.append(blk)

        return blks

    def get_ec_blocks(self, idc_idx):
        nr_data, nr_parity = self['config']['in_idc']

        blks = []
        for i in range(0, nr_data + nr_parity):
            blk = self.get_block(BlockIndex(idc_idx, i))

            if blk is None:
                continue

            if self.is_ec_block(blk['block_id']):
                blks.append(blk)

        return blks

    def get_ec_broken_blocks(self, idc_idx, broken_bids):
        broken_blks = []

        for blk in self.get_ec_blocks(idc_idx):
            if blk['block_id'] in broken_bids:
                broken_blks.append(blk)

        return broken_blks

    def get_ec_block_ids(self, idc_idx):
        bids = []

        for blk in self.get_ec_blocks(idc_idx):
            bids.append(blk['block_id'])

        return bids

    def get_replica_blocks(self, block_id, include_me=True):
        block_id = BlockID(block_id)
        r_indexes = self.get_replica_indexes(block_id.block_index, include_me)

        blks = []
        for idx in r_indexes:
            blk = self.get_block(idx)

            if blk is not None:
                blks.append(blk)

        return blks

    def get_block_byid(self, block_id):
        block_id = BlockID(block_id)

        blk = self.get_block(block_id.block_index)
        if blk is None or blk['block_id'] != block_id:
            return None

        return blk

    def get_idc_blocks(self, idc_idx, is_del=None):
        blks = []

        for idx in sorted(self['blocks'].keys()):
            blk = self['blocks'][idx]

            idx = BlockIndex(idx)

            if idx.i != idc_idx:
                continue

            if is_del is not None and blk['is_del'] != is_del:
                continue

            blks.append(blk)

        return blks

    def get_idc_block_ids(self, idc_idx, is_del=None):
        blks = self.get_idc_blocks(idc_idx, is_del=is_del)
        return [BlockID(b['block_id']) for b in blks]
<commit_msg>convert block to block_desc when new block_group<commit_after>#!/usr/bin/env python2
# coding: utf-8

from collections import defaultdict

from pykit.dictutil import FixedKeysDict

from .block_id import BlockID
from .block_desc import BlockDesc
from .block_group_id import BlockGroupID
from .block_index import BlockIndex
from .replication_config import ReplicationConfig

class BlockGroupBaseError(Exception):
    pass

class BlockNotFoundError(BlockGroupBaseError):
    pass

class BlockExists(BlockGroupBaseError):
    pass

class BlockTypeNotSupported(BlockGroupBaseError):
    pass

class BlockTypeNotSupportReplica(BlockGroupBaseError):
    pass

def _idcs(lst):
    return list(lst)

def _blocks(blocks=None):
    if blocks is None:
        return {}

    for idx, blk in blocks.items():
        blocks[idx] = BlockDesc(blk)

    return blocks

class BlockGroup(FixedKeysDict):

    keys_default = dict(
        block_group_id=BlockGroupID,
        config=ReplicationConfig,
        idcs=_idcs,
        blocks=_blocks,
    )

    ident_keys = ('block_group_id',)

    def __init__(self, *args, **kwargs):
        super(BlockGroup, self).__init__(*args, **kwargs)
        self.type_map = self.make_type_map()

    def get_block_type(self, block_index):

        mp = self.type_map
        bi = BlockIndex(block_index)

        try:
            return mp[bi.i][bi.j]
        except IndexError:
            raise BlockTypeNotSupported('invalid index at {bi}'.format(bi=bi))

    def make_type_map(self):

        cnf = self['config']

        nr_data, nr_parity = cnf['in_idc']
        nr_in_idc, nr_xor_idc = cnf['cross_idc']
        data_replica = cnf['data_replica']

        rst = []

        prefixes = ('d' * nr_in_idc
                    + 'x' * nr_xor_idc)

        for pref in prefixes:

            o = [pref + '0'] * nr_data
            o += [pref + 'p'] * nr_parity

            for j in range(1, data_replica):
                o += ['%s%d' % (pref, j)] * nr_data

            rst.append(o)

        return rst

    def mark_delete_block(self, block_index):
        block = self.get_block(block_index, raise_error=True)
        block['is_del'] = 1

    def delete_block(self, block_index):
        block = self.get_block(block_index)
        if block is not None:
            del self['blocks'][str(block_index)]

    def mark_delete_block_byid(self, block_id):
        block = self.get_block_byid(block_id)
        if block is not None:
            block['is_del'] = 1

    def delete_block_byid(self, block_id):
        block = self.get_block_byid(block_id)
        if block is not None:
            del self['blocks'][block_id.block_index]

    def has(self, block):
        bid = block['block_id']
        bidx = bid.block_index

        existent = self['blocks'].get(bidx)
        return existent == block

    def add_block(self, new_block, replace=False, allow_exist=False):

        if self.has(new_block) and allow_exist:
            return new_block

        bid = new_block['block_id']
        bidx = bid.block_index

        prev = self['blocks'].get(bidx)
        if not replace and prev is not None:
            raise BlockExists(
                'there is already a block at {bid}'.format(bid=bid))

        self['blocks'][bidx] = new_block

        if prev is None:
            return None
        else:
            return BlockDesc(prev)

    def get_free_block_indexes(self, block_type=None, get_all=False):

        free_block_index = defaultdict(list)

        cnf = self['config']
        n = sum(cnf['cross_idc'])
        m = sum(cnf['in_idc'])

        for i in range(n):
            for j in range(m):
                bi = BlockIndex(i, j)
                typ = self.get_block_type(bi)

                idc = self.get_block_idc(bi)

                if get_all:
                    # set the key 'idc' with default if key not set
                    free_block_index[idc]

                if block_type is not None and typ != block_type:
                    continue

                if self.get_block(bi) is None:
                    free_block_index[idc].append(str(bi))

        return free_block_index

    def get_block(self, block_index, raise_error=False):

        bi = BlockIndex(block_index)
        b = self['blocks'].get(str(bi))

        if raise_error and b is None:
            raise BlockNotFoundError(
                'block_index:{bi}'
                ' not found in block_group:{block_group_id}'.format(bi=bi, **self))

        return b

    def get_block_idc(self, block_index):

        bi = BlockIndex(block_index)

        return self['idcs'][bi.i]

    def get_primary_index(self, block_index):

        nr_data, nr_parity = self['config']['in_idc']

        bi = BlockIndex(block_index)

        j = bi.j
        if j >= nr_data:
            j -= nr_parity
            j %= nr_data

        return BlockIndex(bi.i, j)

    def get_replica_indexes(self, block_index, include_me=True):

        nr_data, nr_parity = self['config']['in_idc']
        data_replica = self['config']['data_replica']

        bi = BlockIndex(block_index)
        typ = self.get_block_type(bi)

        if typ.endswith('p'):
            raise BlockTypeNotSupportReplica(
                'block type {typ}'
                ' does not support replica'.format(typ=typ))

        pbi = self.get_primary_index(block_index)

        rst = [str(pbi)]

        for j in range(1, data_replica):
            rbi = BlockIndex(pbi.i,
                             pbi.j + nr_parity + j * nr_data)
            rst.append(str(rbi))

        # if not include_me and str(block_index) in rst:
        if not include_me:
            rst.remove(str(block_index))

        return rst

    def classify_blocks(self, idc_index, only_primary=True):

        nr_data, nr_parity = self['config']['in_idc']

        ec = []
        replica = []
        mark_del = []

        for i in range(0, nr_data):

            bi = BlockIndex(idc_index, i)

            blk = self.get_block(bi)
            if blk is None:
                continue

            if blk['is_del'] == 1:
                mark_del.append(blk)
                continue

            replica_idxes = self.get_replica_indexes(bi, include_me=False)
            rblks = self.indexes_to_blocks(replica_idxes)

            if None in rblks:
                ec.append(blk)
                continue

            replica.append(blk)
            if only_primary:
                continue

            replica.extend(rblks)

        return {'ec': ec, 'replica': replica, 'mark_del': mark_del}

    def indexes_to_blocks(self, indexes):

        blks = []

        for idx in indexes:
            bi = BlockIndex(idx)

            blk = self.get_block(bi)
            blks.append(blk)

        return blks

    def get_parity_indexes(self, idc_index):

        indexes = []
        nr_data, nr_parity = self['config']['in_idc']

        for i in range(nr_data, nr_data + nr_parity):

            bi = BlockIndex(idc_index, i)
            indexes.append(bi)

        return indexes

    def get_parities(self, idc_index):

        idxes = self.get_parity_indexes(idc_index)

        blks = self.indexes_to_blocks(idxes)

        return [blk for blk in blks if blk is not None]

    def is_ec_block(self, block_id):
        block_id = BlockID(block_id)

        blk = self.get_block(block_id.block_index)
        if blk is None or blk['block_id'] != block_id:
            raise BlockNotFoundError(
                'block_id:{bid}'
                ' not found in block_group:{block_group_id}'.format(bid=block_id, **self))

        if block_id.type.endswith('p'):
            blk = self.get_block(block_id.block_index)
            return True

        r_indexes = self.get_replica_indexes(block_id.block_index)
        r_blks = [self.get_block(x) for x in r_indexes]

        return None in r_blks

    def get_blocks(self):
        blks = []

        for idx in sorted(self['blocks'].keys()):
            blk = self['blocks'][idx]
            blks.append(blk)

        return blks

    def get_ec_blocks(self, idc_idx):
        nr_data, nr_parity = self['config']['in_idc']

        blks = []
        for i in range(0, nr_data + nr_parity):
            blk = self.get_block(BlockIndex(idc_idx, i))

            if blk is None:
                continue

            if self.is_ec_block(blk['block_id']):
                blks.append(blk)

        return blks

    def get_ec_broken_blocks(self, idc_idx, broken_bids):
        broken_blks = []

        for blk in self.get_ec_blocks(idc_idx):
            if blk['block_id'] in broken_bids:
                broken_blks.append(blk)

        return broken_blks

    def get_ec_block_ids(self, idc_idx):
        bids = []

        for blk in self.get_ec_blocks(idc_idx):
            bids.append(blk['block_id'])

        return bids

    def get_replica_blocks(self, block_id, include_me=True):
        block_id = BlockID(block_id)
        r_indexes = self.get_replica_indexes(block_id.block_index, include_me)

        blks = []
        for idx in r_indexes:
            blk = self.get_block(idx)

            if blk is not None:
                blks.append(blk)

        return blks

    def get_block_byid(self, block_id):
        block_id = BlockID(block_id)

        blk = self.get_block(block_id.block_index)
        if blk is None or blk['block_id'] != block_id:
            return None

        return blk

    def get_idc_blocks(self, idc_idx, is_del=None):
        blks = []

        for idx in sorted(self['blocks'].keys()):
            blk = self['blocks'][idx]

            idx = BlockIndex(idx)

            if idx.i != idc_idx:
                continue

            if is_del is not None and blk['is_del'] != is_del:
                continue

            blks.append(blk)

        return blks

    def get_idc_block_ids(self, idc_idx, is_del=None):
        blks = self.get_idc_blocks(idc_idx, is_del=is_del)
        return [BlockID(b['block_id']) for b in blks]
<|endoftext|>


<commit_before># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

"""
This script creates a release candidate and relevant files.

(How to run)
python release.py <reef_home> <reef_version> <rc candidate number> <public key id (8 hex numbers)>

You can also see how to run the script with 'python release.py -h'

(Examples)
python release.py ~/reef 0.14.0 1 E488F925

"""

import subprocess
import fnmatch
import tarfile
import hashlib
import os
import argparse

"""
Get list from .gitignore
"""
def get_ignore_list(reef_home):
    f = open(reef_home + "/.gitignore", "r")
    if f is None:
        return []

    ignore_list = list()

    while True:
        line = f.readline()[:-1]
        if not line:
            break
        if "#" not in line:
            ignore_list.insert(0, "*/" + line)

    return ignore_list

"""
Make text for e-mail
"""
def get_mail_text(reef_version, rc_num):
    file_name = "apache-reef-" + reef_version + "-rc" + str(rc_num) + ".tar.gz"

    return_str = ""
    return_str += "This is to call for a new vote for the source release of Apache REEF " \
        + reef_version + " (rc" + str(rc_num) + ").\n\n"
    return_str += "The source tar ball, including signatures, digests, etc can be found at:\n" + "<yours>\n\n"
    return_str += "The Git tag is release-" + reef_version + "-rc" + str(rc_num) + "\n"
    return_str += "The Git commit ID is <Git Commit ID>\n\n\n"

    return_str += "Checksums of apache-reef-" + reef_version + "-rc" + str(rc_num) + ".tar.gz:\n\n"

    md5 = open(file_name + ".md5").read().split(" ")[0]
    return_str += "MD5: " + md5 + "\n"

    sha = open(file_name + ".sha512.txt").read().split(" ")[0]
    return_str += "SHA: " + sha + "\n"

    return_str += "\nRelease artifacts are signed with a key found in the KEYS file available here:\n"
    return_str += "\nhttps://dist.apache.org/repos/dist/release/reef/KEYS\n\n\n\n"

    return_str += "<Issue Things>\n\n\n\n"

    return_str += "The vote will be open for 72 hours. Please download the release\n" \
        + "candidate, check the hashes/signature, build it and test it, and then\nplease vote:\n\n" \
        + "[ ] +1 Release this package as Apache REEF " + reef_version + "\n" \
        + "[ ] +0 no opinion\n" \
        + "[ ] -1 Do not release this package because ...\n\n" \
        + "Thanks!"

    return return_str

"""
Function to exclude files in .gitignore when making tar.gz
Return true when a file is in .gitignore
"""
def exclude_git_ignore(file_name):
    ignore_list = get_ignore_list(reef_home)
    for e in ignore_list:
        if fnmatch.fnmatch(file_name, e):
            return True

    return fnmatch.fnmatch(file_name, "*/.git") or fnmatch.fnmatch(file_name, "*/.gitignore") or \
        fnmatch.fnmatch(file_name, "*/.gitattributes")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Script for release versioning, packaging, checksum")

    parser.add_argument("reef_home", type=str, help="REEF home")
    parser.add_argument("reef_version", type=str, help="REEF version")
    parser.add_argument("rc_num", type=int, help ="rc candidate number")
    parser.add_argument("key_id", type=str, help = "public key id (8 hex numbers)")
    args = parser.parse_args()

    reef_home = os.path.abspath(args.reef_home)
    reef_version = args.reef_version
    rc_num = args.rc_num
    key_id = args.key_id

    build_result = subprocess.call("cd " + reef_home + " && " + "mvn apache-rat:check", shell=True)

    if build_result == 0:
        file_name = "apache-reef-" + reef_version + "-rc" + str(rc_num) + ".tar.gz"

        # Make tar.gz
        tar = tarfile.open(file_name, "w:gz")
        tar.add(reef_home, arcname="apache-reef-"+reef_version, exclude=exclude_git_ignore)
        tar.close()

        gpg_str = "gpg --armor -u " + str(key_id) + " --output " + file_name + ".asc " + "--detach-sig " + file_name
        gpg_result = subprocess.call(gpg_str, shell=True)

        if gpg_result == 0:
            md5 = hashlib.md5(open(file_name, 'rb').read()).hexdigest()
            sha = hashlib.sha512(open(file_name, 'rb').read()).hexdigest()

            md5_file = open(file_name + ".md5", "w")
            md5_file.write(md5 + " *" + file_name + "\n")
            sha_file = open(file_name + ".sha512.txt", "w")
            sha_file.write(sha + " *" + file_name + "\n")
            md5_file.close()
            sha_file.close()

            # Make a Text for an e-mail
            print "\n==================================Result==================================="
            print get_mail_text(reef_version, rc_num)
            print "===========================================================================\n"

        else:
            print "gpg error"

    else:
        print "build error - mvn apache-rat:check failed"
<commit_msg>[REEF-1867] Generate .sha512 instead of .sha512.txt in the release script<commit_after># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

"""
This script creates a release candidate and relevant files.

(How to run)
python release.py <reef_home> <reef_version> <rc candidate number> <public key id (8 hex numbers)>

You can also see how to run the script with 'python release.py -h'

(Examples)
python release.py ~/reef 0.14.0 1 E488F925

"""

import subprocess
import fnmatch
import tarfile
import hashlib
import os
import argparse

"""
Get list from .gitignore
"""
def get_ignore_list(reef_home):
    f = open(reef_home + "/.gitignore", "r")
    if f is None:
        return []

    ignore_list = list()

    while True:
        line = f.readline()[:-1]
        if not line:
            break
        if "#" not in line:
            ignore_list.insert(0, "*/" + line)

    return ignore_list

"""
Make text for e-mail
"""
def get_mail_text(reef_version, rc_num):
    file_name = "apache-reef-" + reef_version + "-rc" + str(rc_num) + ".tar.gz"

    return_str = ""
    return_str += "This is to call for a new vote for the source release of Apache REEF " \
        + reef_version + " (rc" + str(rc_num) + ").\n\n"
    return_str += "The source tar ball, including signatures, digests, etc can be found at:\n" + "<yours>\n\n"
    return_str += "The Git tag is release-" + reef_version + "-rc" + str(rc_num) + "\n"
    return_str += "The Git commit ID is <Git Commit ID>\n\n\n"

    return_str += "Checksums of apache-reef-" + reef_version + "-rc" + str(rc_num) + ".tar.gz:\n\n"

    md5 = open(file_name + ".md5").read().split(" ")[0]
    return_str += "MD5: " + md5 + "\n"

    sha = open(file_name + ".sha512").read().split(" ")[0]
    return_str += "SHA: " + sha + "\n"

    return_str += "\nRelease artifacts are signed with a key found in the KEYS file available here:\n"
    return_str += "\nhttps://dist.apache.org/repos/dist/release/reef/KEYS\n\n\n\n"

    return_str += "<Issue Things>\n\n\n\n"

    return_str += "The vote will be open for 72 hours. Please download the release\n" \
        + "candidate, check the hashes/signature, build it and test it, and then\nplease vote:\n\n" \
        + "[ ] +1 Release this package as Apache REEF " + reef_version + "\n" \
        + "[ ] +0 no opinion\n" \
        + "[ ] -1 Do not release this package because ...\n\n" \
        + "Thanks!"

    return return_str

"""
Function to exclude files in .gitignore when making tar.gz
Return true when a file is in .gitignore
"""
def exclude_git_ignore(file_name):
    ignore_list = get_ignore_list(reef_home)
    for e in ignore_list:
        if fnmatch.fnmatch(file_name, e):
            return True

    return fnmatch.fnmatch(file_name, "*/.git") or fnmatch.fnmatch(file_name, "*/.gitignore") or \
        fnmatch.fnmatch(file_name, "*/.gitattributes")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Script for release versioning, packaging, checksum")

    parser.add_argument("reef_home", type=str, help="REEF home")
    parser.add_argument("reef_version", type=str, help="REEF version")
    parser.add_argument("rc_num", type=int, help ="rc candidate number")
    parser.add_argument("key_id", type=str, help = "public key id (8 hex numbers)")
    args = parser.parse_args()

    reef_home = os.path.abspath(args.reef_home)
    reef_version = args.reef_version
    rc_num = args.rc_num
    key_id = args.key_id

    build_result = subprocess.call("cd " + reef_home + " && " + "mvn apache-rat:check", shell=True)

    if build_result == 0:
        file_name = "apache-reef-" + reef_version + "-rc" + str(rc_num) + ".tar.gz"

        # Make tar.gz
        tar = tarfile.open(file_name, "w:gz")
        tar.add(reef_home, arcname="apache-reef-"+reef_version, exclude=exclude_git_ignore)
        tar.close()

        gpg_str = "gpg --armor -u " + str(key_id) + " --output " + file_name + ".asc " + "--detach-sig " + file_name
        gpg_result = subprocess.call(gpg_str, shell=True)

        if gpg_result == 0:
            md5 = hashlib.md5(open(file_name, 'rb').read()).hexdigest()
            sha = hashlib.sha512(open(file_name, 'rb').read()).hexdigest()

            md5_file = open(file_name + ".md5", "w")
            md5_file.write(md5 + " *" + file_name + "\n")
            sha_file = open(file_name + ".sha512", "w")
            sha_file.write(sha + " *" + file_name + "\n")
            md5_file.close()
            sha_file.close()

            # Make a Text for an e-mail
            print "\n==================================Result==================================="
            print get_mail_text(reef_version, rc_num)
            print "===========================================================================\n"

        else:
            print "gpg error"

    else:
        print "build error - mvn apache-rat:check failed"
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

""" give lists entry-wise products, powers, division, addition and
subtraction by other lists, as though they were vectors. This code
is probably way over the top, but it's simple and it works """

from warnings import warn

class entrylist(list):
    """ An iterable with useful attributes from lists and entrylists.

    Example usage:

    >>> from numpy.random import randn
    >>> X = entrylist(randn(5,6).shape)
    >>> X
    [5, 6]
    >>> X // 2
    [2, 3]
    >>> X / 2
    [2, 3]
    >>> X * 2
    [10, 12]
    >>> X + 2
    [7, 8]
    >>> X
    [5, 6]
    >>> X - 2
    [3, 4]
    >>> X > 2
    True
    >>> X < 2
    False
    >>> X <= 2
    False
    >>> X >= 2
    True
    """
    def __floordiv__(self, other):
        if (isinstance(other, (int, float))):
            return entrylist(i // other for i in self)
        elif (isinstance(other, (list, entrylist))):
            if (other.__len__() != self.__len__()):
                warn('len(other) != len(self): Possible data loss')
            return entrylist(i // j for i, j in zip(self, other))
        else:
            raise TypeError

    def __div__(self, other):
        if (isinstance(other, (int, float))):
            return entrylist(i / other for i in self)
        elif (isinstance(other, (list, entrylist))):
            if (other.__len__() != self.__len__()):
                warn('len(other) != len(self): Possible data loss')
            return entrylist(i / j for i, j in zip(self, other))
        else:
            raise TypeError

    def __mul__(self, other):
        if (isinstance(other, int)):
            return entrylist(i * other for i in self)
        elif (isinstance(other, (list, entrylist))):
            # multiply entry-wise
            if (other.__len__() != self.__len__()):
                warn('len(other) != len(self): Possible data loss')
            return entrylist(i * j for i, j in zip(self, other))
        else:
            raise TypeError

    def __pow__(self, other):
        if (isinstance(other, (int, float))):
            return entrylist(i ** other for i in self)
        elif (isinstance(other, (list, entrylist))):
            if (other.__len__() != self.__len__()):
                warn('len(other) != len(self): Possible data loss')
            return entrylist(i ** j for i, j in zip(self, other))
        else:
            raise TypeError

    def __add__(self, other):
        if (isinstance(other, (int, float))):
            return entrylist(i + other for i in self)
        elif (isinstance(other, (list, entrylist))):
            if (other.__len__() != self.__len__()):
                warn('len(other) != len(self): Possible data loss')
            return entrylist(i + j for i, j in zip(self, other))
        else:
            raise TypeError

    def __sub__(self, other):
        return self.__add__(-other)

    def __le__(self, other):
        if (isinstance(other, (int, float))):
            return all(entrylist(i <= other for i in self))
        elif (isinstance(other, (list, entrylist))):
            if (other.__len__() != self.__len__()):
                warn('len(other) != len(self): Possible data loss')
            return all(entrylist(i <= j for i, j in zip(self, other)))
        else:
            raise TypeError

    def __ge__(self, other):
        if (isinstance(other, (int, float))):
            return all(entrylist(i >= other for i in self))
        elif (isinstance(other, (list, entrylist))):
            if (other.__len__() != self.__len__()):
                warn('len(other) != len(self): Possible data loss')
            return all(entrylist(i >= j for i, j in zip(self, other)))
        else:
            raise TypeError

    def __eq__(self, other):
        if (isinstance(other, (int, float))):
            return all(entrylist(i == other for i in self))
        elif (isinstance(other, (list, entrylist))):
            if (other.__len__() != self.__len__()):
                warn('len(other) != len(self): Possible data loss')
            return all(entrylist(i == j for i, j in zip(self, other)))
        else:
            raise TypeError

    def __ne__(self, other):
        return not self.__eq__(other)

    def __lt__(self, other):
        return not self.__ge__(other)

    def __gt__(self, other):
        return not self.__le__(other)<commit_msg>Simplified comment<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

""" give lists entry-wise products, powers, division, addition and
subtraction by other lists, as though they were vectors. This code
is probably way over the top, but it's simple and it works """

from warnings import warn

class entrylist(list):
    """ An iterable with useful attributes from lists and entrylists.

    Example usage:

    >>> from numpy.random import randn
    >>> X = entrylist(randn(5,6).shape)
    >>> X
    [5, 6]
    >>> X // 2
    [2, 3]
    >>> X / 2
    [2, 3]
    >>> X * 2
    [10, 12]
    >>> X + 2
    [7, 8]
    >>> X - 2
    [3, 4]
    >>> X > 2
    True
    >>> X < 2
    False
    >>> X <= 2
    False
    >>> X >= 2
    True
    """
    def __floordiv__(self, other):
        if (isinstance(other, (int, float))):
            return entrylist(i // other for i in self)
        elif (isinstance(other, (list, entrylist))):
            if (other.__len__() != self.__len__()):
                warn('len(other) != len(self): Possible data loss')
            return entrylist(i // j for i, j in zip(self, other))
        else:
            raise TypeError

    def __div__(self, other):
        if (isinstance(other, (int, float))):
            return entrylist(i / other for i in self)
        elif (isinstance(other, (list, entrylist))):
            if (other.__len__() != self.__len__()):
                warn('len(other) != len(self): Possible data loss')
            return entrylist(i / j for i, j in zip(self, other))
        else:
            raise TypeError

    def __mul__(self, other):
        if (isinstance(other, int)):
            return entrylist(i * other for i in self)
        elif (isinstance(other, (list, entrylist))):
            # multiply entry-wise
            if (other.__len__() != self.__len__()):
                warn('len(other) != len(self): Possible data loss')
            return entrylist(i * j for i, j in zip(self, other))
        else:
            raise TypeError

    def __pow__(self, other):
        if (isinstance(other, (int, float))):
            return entrylist(i ** other for i in self)
        elif (isinstance(other, (list, entrylist))):
            if (other.__len__() != self.__len__()):
                warn('len(other) != len(self): Possible data loss')
            return entrylist(i ** j for i, j in zip(self, other))
        else:
            raise TypeError

    def __add__(self, other):
        if (isinstance(other, (int, float))):
            return entrylist(i + other for i in self)
        elif (isinstance(other, (list, entrylist))):
            if (other.__len__() != self.__len__()):
                warn('len(other) != len(self): Possible data loss')
            return entrylist(i + j for i, j in zip(self, other))
        else:
            raise TypeError

    def __sub__(self, other):
        return self.__add__(-other)

    def __le__(self, other):
        if (isinstance(other, (int, float))):
            return all(entrylist(i <= other for i in self))
        elif (isinstance(other, (list, entrylist))):
            if (other.__len__() != self.__len__()):
                warn('len(other) != len(self): Possible data loss')
            return all(entrylist(i <= j for i, j in zip(self, other)))
        else:
            raise TypeError

    def __ge__(self, other):
        if (isinstance(other, (int, float))):
            return all(entrylist(i >= other for i in self))
        elif (isinstance(other, (list, entrylist))):
            if (other.__len__() != self.__len__()):
                warn('len(other) != len(self): Possible data loss')
            return all(entrylist(i >= j for i, j in zip(self, other)))
        else:
            raise TypeError

    def __eq__(self, other):
        if (isinstance(other, (int, float))):
            return all(entrylist(i == other for i in self))
        elif (isinstance(other, (list, entrylist))):
            if (other.__len__() != self.__len__()):
                warn('len(other) != len(self): Possible data loss')
            return all(entrylist(i == j for i, j in zip(self, other)))
        else:
            raise TypeError

    def __ne__(self, other):
        return not self.__eq__(other)

    def __lt__(self, other):
        return not self.__ge__(other)

    def __gt__(self, other):
        return not self.__le__(other)<|endoftext|>