<commit_before>#!/usr/bin/env python
import argparse
from cStringIO import StringIO
import logging
from math import ceil
from multiprocessing import Pool
import sys
import time
import urlparse

import boto

parser = argparse.ArgumentParser(description="Transfer large files to S3",
        prog="s3-mp-upload")
parser.add_argument("src", type=file, help="The file to transfer")
parser.add_argument("dest", help="The S3 destination object")
parser.add_argument("-np", "--num-processes", help="Number of processors to use",
        type=int, default=2)
parser.add_argument("-f", "--force", help="Overwrite an existing S3 key",
        action="store_true")
parser.add_argument("-s", "--split", help="Split size, in Mb", type=int, default=50)
parser.add_argument("-rrs", "--reduced-redundancy", help="Use reduced redundancy storage. Default is standard.", default=False,  action="store_true")
parser.add_argument("-v", "--verbose", help="Be more verbose", default=False, action="store_true")

logger = logging.getLogger("s3-mp-upload")

def do_part_upload(args):
    """
    Upload a part of a MultiPartUpload

    Open the target file and read in a chunk. Since we can't pickle
    S3Connection or MultiPartUpload objects, we have to reconnect and lookup
    the MPU object with each part upload.

    :type args: tuple of (string, string, string, int, int, int)
    :param args: The actual arguments of this method. Due to lameness of
                 multiprocessing, we have to extract these outside of the
                 function definition.

                 The arguments are: S3 Bucket name, MultiPartUpload id, file
                 name, the part number, part offset, part size
    """
    # Multiprocessing args lameness
    bucket_name, mpu_id, fname, i, start, size = args
    logger.debug("do_part_upload got args: %s" % (args,))

    # Connect to S3, get the MultiPartUpload
    s3 = boto.connect_s3()
    bucket = s3.lookup(bucket_name)
    mpu = None
    for mp in bucket.list_multipart_uploads():
        if mp.id == mpu_id:
            mpu = mp
            break
    if mpu is None:
        raise Exception("Could not find MultiPartUpload %s" % mpu_id)

    # Read the chunk from the file
    fp = open(fname, 'rb')
    fp.seek(start)
    data = fp.read(size)
    fp.close()
    if not data:
        raise Exception("Unexpectedly tried to read an empty chunk")

    def progress(x,y):
        logger.info("Part %d: %0.2f%%" % (i+1, 1.*x/y))

    # Do the upload
    t1 = time.time()
    mpu.upload_part_from_file(StringIO(data), i+1, cb=progress)

    # Print some timings
    t2 = time.time() - t1
    s = len(data)/1024./1024.
    logger.info("Uploaded part %s (%0.2fM) in %0.2fs at %0.2fMbps" % (i+1, s, t2, s/t2))

def main(src, dest, num_processes=2, split=50, force=False, reduced_redundancy=False, verbose=False):
    # Check that dest is a valid S3 url
    split_rs = urlparse.urlsplit(dest)
    if split_rs.scheme != "s3":
        raise ValueError("'%s' is not an S3 url" % dest)

    s3 = boto.connect_s3()
    bucket = s3.lookup(split_rs.netloc)
    key = bucket.get_key(split_rs.path)
    # See if we're overwriting an existing key
    if key is not None:
        if not force:
            raise ValueError("'%s' already exists. Specify -f to overwrite it" % dest)

    # Determine the splits
    part_size = max(5*1024*1024, 1024*1024*split)
    src.seek(0,2)
    size = src.tell()
    num_parts = int(ceil(size / part_size))

    # If file is less than 5M, just upload it directly
    if size < 5*1024*1024:
        src.seek(0)
        t1 = time.time()
	k = boto.s3.key.Key(bucket,split_rs.path)
        k.set_contents_from_file(src)
        t2 = time.time() - t1
        s = size/1024./1024.
        logger.info("Finished uploading %0.2fM in %0.2fs (%0.2fMbps)" % (s, t2, s/t2))
        return

    # Create the multi-part upload object
    mpu = bucket.initiate_multipart_upload(split_rs.path, reduced_redundancy=reduced_redundancy)
    logger.info("Initialized upload: %s" % mpu.id)

    # Generate arguments for invocations of do_part_upload
    def gen_args(num_parts, fold_last):
        for i in range(num_parts+1):
            part_start = part_size*i
            if i == (num_parts-1) and fold_last is True:
                yield (bucket.name, mpu.id, src.name, i, part_start, part_size*2)
                break
            else:
                yield (bucket.name, mpu.id, src.name, i, part_start, part_size)

    # If the last part is less than 5M, just fold it into the previous part
    fold_last = ((size % part_size) < 5*1024*1024)

    # Do the thing
    try:
        # Create a pool of workers
        pool = Pool(processes=num_processes)
        t1 = time.time()
        pool.map_async(do_part_upload, gen_args(num_parts, fold_last)).get(9999999)
        # Print out some timings
        t2 = time.time() - t1
        s = size/1024./1024.
        # Finalize
        src.close()
        mpu.complete_upload()
        logger.info("Finished uploading %0.2fM in %0.2fs (%0.2fMbps)" % (s, t2, s/t2))
    except KeyboardInterrupt:
        logger.warn("Received KeyboardInterrupt, canceling upload")
        pool.terminate()
        mpu.cancel_upload()
    except Exception, err:
        logger.error("Encountered an error, canceling upload")
        logger.error(err)
        mpu.cancel_upload()

if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)
    args = parser.parse_args()
    arg_dict = vars(args)
    if arg_dict['verbose'] == True:
        logger.setLevel(logging.DEBUG)
    logger.debug("CLI args: %s" % args)
    main(**arg_dict)
<commit_msg>Changing a tab to spaces from #12<commit_after>#!/usr/bin/env python
import argparse
from cStringIO import StringIO
import logging
from math import ceil
from multiprocessing import Pool
import sys
import time
import urlparse

import boto

parser = argparse.ArgumentParser(description="Transfer large files to S3",
        prog="s3-mp-upload")
parser.add_argument("src", type=file, help="The file to transfer")
parser.add_argument("dest", help="The S3 destination object")
parser.add_argument("-np", "--num-processes", help="Number of processors to use",
        type=int, default=2)
parser.add_argument("-f", "--force", help="Overwrite an existing S3 key",
        action="store_true")
parser.add_argument("-s", "--split", help="Split size, in Mb", type=int, default=50)
parser.add_argument("-rrs", "--reduced-redundancy", help="Use reduced redundancy storage. Default is standard.", default=False,  action="store_true")
parser.add_argument("-v", "--verbose", help="Be more verbose", default=False, action="store_true")

logger = logging.getLogger("s3-mp-upload")

def do_part_upload(args):
    """
    Upload a part of a MultiPartUpload

    Open the target file and read in a chunk. Since we can't pickle
    S3Connection or MultiPartUpload objects, we have to reconnect and lookup
    the MPU object with each part upload.

    :type args: tuple of (string, string, string, int, int, int)
    :param args: The actual arguments of this method. Due to lameness of
                 multiprocessing, we have to extract these outside of the
                 function definition.

                 The arguments are: S3 Bucket name, MultiPartUpload id, file
                 name, the part number, part offset, part size
    """
    # Multiprocessing args lameness
    bucket_name, mpu_id, fname, i, start, size = args
    logger.debug("do_part_upload got args: %s" % (args,))

    # Connect to S3, get the MultiPartUpload
    s3 = boto.connect_s3()
    bucket = s3.lookup(bucket_name)
    mpu = None
    for mp in bucket.list_multipart_uploads():
        if mp.id == mpu_id:
            mpu = mp
            break
    if mpu is None:
        raise Exception("Could not find MultiPartUpload %s" % mpu_id)

    # Read the chunk from the file
    fp = open(fname, 'rb')
    fp.seek(start)
    data = fp.read(size)
    fp.close()
    if not data:
        raise Exception("Unexpectedly tried to read an empty chunk")

    def progress(x,y):
        logger.info("Part %d: %0.2f%%" % (i+1, 1.*x/y))

    # Do the upload
    t1 = time.time()
    mpu.upload_part_from_file(StringIO(data), i+1, cb=progress)

    # Print some timings
    t2 = time.time() - t1
    s = len(data)/1024./1024.
    logger.info("Uploaded part %s (%0.2fM) in %0.2fs at %0.2fMbps" % (i+1, s, t2, s/t2))

def main(src, dest, num_processes=2, split=50, force=False, reduced_redundancy=False, verbose=False):
    # Check that dest is a valid S3 url
    split_rs = urlparse.urlsplit(dest)
    if split_rs.scheme != "s3":
        raise ValueError("'%s' is not an S3 url" % dest)

    s3 = boto.connect_s3()
    bucket = s3.lookup(split_rs.netloc)
    key = bucket.get_key(split_rs.path)
    # See if we're overwriting an existing key
    if key is not None:
        if not force:
            raise ValueError("'%s' already exists. Specify -f to overwrite it" % dest)

    # Determine the splits
    part_size = max(5*1024*1024, 1024*1024*split)
    src.seek(0,2)
    size = src.tell()
    num_parts = int(ceil(size / part_size))

    # If file is less than 5M, just upload it directly
    if size < 5*1024*1024:
        src.seek(0)
        t1 = time.time()
        k = boto.s3.key.Key(bucket,split_rs.path)
        k.set_contents_from_file(src)
        t2 = time.time() - t1
        s = size/1024./1024.
        logger.info("Finished uploading %0.2fM in %0.2fs (%0.2fMbps)" % (s, t2, s/t2))
        return

    # Create the multi-part upload object
    mpu = bucket.initiate_multipart_upload(split_rs.path, reduced_redundancy=reduced_redundancy)
    logger.info("Initialized upload: %s" % mpu.id)

    # Generate arguments for invocations of do_part_upload
    def gen_args(num_parts, fold_last):
        for i in range(num_parts+1):
            part_start = part_size*i
            if i == (num_parts-1) and fold_last is True:
                yield (bucket.name, mpu.id, src.name, i, part_start, part_size*2)
                break
            else:
                yield (bucket.name, mpu.id, src.name, i, part_start, part_size)

    # If the last part is less than 5M, just fold it into the previous part
    fold_last = ((size % part_size) < 5*1024*1024)

    # Do the thing
    try:
        # Create a pool of workers
        pool = Pool(processes=num_processes)
        t1 = time.time()
        pool.map_async(do_part_upload, gen_args(num_parts, fold_last)).get(9999999)
        # Print out some timings
        t2 = time.time() - t1
        s = size/1024./1024.
        # Finalize
        src.close()
        mpu.complete_upload()
        logger.info("Finished uploading %0.2fM in %0.2fs (%0.2fMbps)" % (s, t2, s/t2))
    except KeyboardInterrupt:
        logger.warn("Received KeyboardInterrupt, canceling upload")
        pool.terminate()
        mpu.cancel_upload()
    except Exception, err:
        logger.error("Encountered an error, canceling upload")
        logger.error(err)
        mpu.cancel_upload()

if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)
    args = parser.parse_args()
    arg_dict = vars(args)
    if arg_dict['verbose'] == True:
        logger.setLevel(logging.DEBUG)
    logger.debug("CLI args: %s" % args)
    main(**arg_dict)
<|endoftext|>


<commit_before>import os
import sys
import collections
import numpy as np

import d4rl.infos
from d4rl.offline_env import set_dataset_path, get_keys

SUPPRESS_MESSAGES = bool(os.environ.get('D4RL_SUPPRESS_IMPORT_ERROR', 0))

_ERROR_MESSAGE = 'Warning: %s failed to import. Set the environment variable D4RL_SUPPRESS_IMPORT_ERROR=1 to suppress this message.'

try:
    import d4rl.locomotion
    import d4rl.hand_manipulation_suite
    import d4rl.pointmaze
    import d4rl.gym_minigrid
    import d4rl.gym_mujoco
except ImportError as e:
    if not SUPPRESS_MESSAGES:
        print(_ERROR_MESSAGE % 'Mujoco-based envs', file=sys.stderr)
        print(e, file=sys.stderr)

try:
    import d4rl.flow
except ImportError as e:
    if not SUPPRESS_MESSAGES:
        print(_ERROR_MESSAGE % 'Flow', file=sys.stderr)
        print(e, file=sys.stderr)

try:
    import d4rl.kitchen
except ImportError as e:
    if not SUPPRESS_MESSAGES:
        print(_ERROR_MESSAGE % 'FrankaKitchen', file=sys.stderr)
        print(e, file=sys.stderr)

try:
    import d4rl.carla
except ImportError as e:
    if not SUPPRESS_MESSAGES:
        print(_ERROR_MESSAGE % 'CARLA', file=sys.stderr)
        print(e, file=sys.stderr)

try:
    import d4rl.gym_bullet
    import d4rl.pointmaze_bullet
except ImportError as e:
    if not SUPPRESS_MESSAGES:
        print(_ERROR_MESSAGE % 'GymBullet', file=sys.stderr)
        print(e, file=sys.stderr)

def reverse_normalized_score(env_name, normalized_score):
    ref_min_score = d4rl.infos.REF_MIN_SCORE[env_name]
    ref_max_score = d4rl.infos.REF_MAX_SCORE[env_name]
    # return (score - ref_min_score) / (ref_max_score - ref_min_score)
    return (score * (ref_max_score - ref_min_score)) + ref_min_score

def get_normalized_score(env_name, score):
    ref_min_score = d4rl.infos.REF_MIN_SCORE[env_name]
    ref_max_score = d4rl.infos.REF_MAX_SCORE[env_name]
    return (score - ref_min_score) / (ref_max_score - ref_min_score)

def qlearning_dataset(env, dataset=None, terminate_on_end=False, **kwargs):
    """
    Returns datasets formatted for use by standard Q-learning algorithms,
    with observations, actions, next_observations, rewards, and a terminal
    flag.

    Args:
        env: An OfflineEnv object.
        dataset: An optional dataset to pass in for processing. If None,
            the dataset will default to env.get_dataset()
        terminate_on_end (bool): Set done=True on the last timestep
            in a trajectory. Default is False, and will discard the
            last timestep in each trajectory.
        **kwargs: Arguments to pass to env.get_dataset().

    Returns:
        A dictionary containing keys:
            observations: An N x dim_obs array of observations.
            actions: An N x dim_action array of actions.
            next_observations: An N x dim_obs array of next observations.
            rewards: An N-dim float array of rewards.
            terminals: An N-dim boolean array of "done" or episode termination flags.
    """
    if dataset is None:
        dataset = env.get_dataset(**kwargs)

    N = dataset['rewards'].shape[0]
    obs_ = []
    next_obs_ = []
    action_ = []
    reward_ = []
    done_ = []

    # The newer version of the dataset adds an explicit
    # timeouts field. Keep old method for backwards compatability.
    use_timeouts = False
    if 'timeouts' in dataset:
        use_timeouts = True

    obs_shape = (N-1,)+ dataset['observations'][0].shape
    observations_ = np.zeros(obs_shape, dtype=np.float32)
    next_observations_ = np.zeros(obs_shape, dtype=np.float32)

    episode_step = 0
    for i in range(N-1):
        obs = dataset['observations'][i].astype(np.float32)
        new_obs = dataset['observations'][i+1].astype(np.float32)
        action = dataset['actions'][i].astype(np.float32)
        reward = dataset['rewards'][i].astype(np.float32)
        done_bool = bool(dataset['terminals'][i])

        if use_timeouts:
            final_timestep = dataset['timeouts'][i]
        else:
            final_timestep = (episode_step == env._max_episode_steps - 1)
        if (not terminate_on_end) and final_timestep:
            # Skip this transition and don't apply terminals on the last step of an episode
            episode_step = 0
            continue
        if done_bool or final_timestep:
            episode_step = 0

        #obs_.append(obs)
        #next_obs_.append(new_obs)
        observations_[i] = obs
        next_observations_[i] = new_obs
        action_.append(action)
        reward_.append(reward)
        done_.append(done_bool)
        episode_step += 1

    return {
        #'observations': np.array(obs_),
        'observations': observations_,
        'actions': np.array(action_),
        #'next_observations': np.array(next_obs_),
        'next_observations': next_observations_,
        'rewards': np.array(reward_),
        'terminals': np.array(done_),
    }

def sequence_dataset(env, dataset=None, **kwargs):
    """
    Returns an iterator through trajectories.

    Args:
        env: An OfflineEnv object.
        dataset: An optional dataset to pass in for processing. If None,
            the dataset will default to env.get_dataset()
        **kwargs: Arguments to pass to env.get_dataset().

    Returns:
        An iterator through dictionaries with keys:
            observations
            actions
            rewards
            terminals
    """
    if dataset is None:
        dataset = env.get_dataset(**kwargs)

    N = dataset['rewards'].shape[0]
    data_ = collections.defaultdict(list)

    # The newer version of the dataset adds an explicit
    # timeouts field. Keep old method for backwards compatability.
    use_timeouts = False
    if 'timeouts' in dataset:
        use_timeouts = True

    episode_step = 0
    for i in range(N):
        done_bool = bool(dataset['terminals'][i])
        if use_timeouts:
            final_timestep = dataset['timeouts'][i]
        else:
            final_timestep = (episode_step == env._max_episode_steps - 1)

        if done_bool or final_timestep:
            episode_step = 0
            episode_data = {}
            for k in data_:
                episode_data[k] = np.array(data_[k])
            yield episode_data
            data_ = collections.defaultdict(list)

        for k in dataset:
            data_[k].append(dataset[k][i])
        episode_step += 1

<commit_msg>fix qlearning dataset<commit_after>import os
import sys
import collections
import numpy as np

import d4rl.infos
from d4rl.offline_env import set_dataset_path, get_keys

SUPPRESS_MESSAGES = bool(os.environ.get('D4RL_SUPPRESS_IMPORT_ERROR', 0))

_ERROR_MESSAGE = 'Warning: %s failed to import. Set the environment variable D4RL_SUPPRESS_IMPORT_ERROR=1 to suppress this message.'

try:
    import d4rl.locomotion
    import d4rl.hand_manipulation_suite
    import d4rl.pointmaze
    import d4rl.gym_minigrid
    import d4rl.gym_mujoco
except ImportError as e:
    if not SUPPRESS_MESSAGES:
        print(_ERROR_MESSAGE % 'Mujoco-based envs', file=sys.stderr)
        print(e, file=sys.stderr)

try:
    import d4rl.flow
except ImportError as e:
    if not SUPPRESS_MESSAGES:
        print(_ERROR_MESSAGE % 'Flow', file=sys.stderr)
        print(e, file=sys.stderr)

try:
    import d4rl.kitchen
except ImportError as e:
    if not SUPPRESS_MESSAGES:
        print(_ERROR_MESSAGE % 'FrankaKitchen', file=sys.stderr)
        print(e, file=sys.stderr)

try:
    import d4rl.carla
except ImportError as e:
    if not SUPPRESS_MESSAGES:
        print(_ERROR_MESSAGE % 'CARLA', file=sys.stderr)
        print(e, file=sys.stderr)

try:
    import d4rl.gym_bullet
    import d4rl.pointmaze_bullet
except ImportError as e:
    if not SUPPRESS_MESSAGES:
        print(_ERROR_MESSAGE % 'GymBullet', file=sys.stderr)
        print(e, file=sys.stderr)

def reverse_normalized_score(env_name, normalized_score):
    ref_min_score = d4rl.infos.REF_MIN_SCORE[env_name]
    ref_max_score = d4rl.infos.REF_MAX_SCORE[env_name]
    # return (score - ref_min_score) / (ref_max_score - ref_min_score)
    return (score * (ref_max_score - ref_min_score)) + ref_min_score

def get_normalized_score(env_name, score):
    ref_min_score = d4rl.infos.REF_MIN_SCORE[env_name]
    ref_max_score = d4rl.infos.REF_MAX_SCORE[env_name]
    return (score - ref_min_score) / (ref_max_score - ref_min_score)

def qlearning_dataset(env, dataset=None, terminate_on_end=False, **kwargs):
    """
    Returns datasets formatted for use by standard Q-learning algorithms,
    with observations, actions, next_observations, rewards, and a terminal
    flag.

    Args:
        env: An OfflineEnv object.
        dataset: An optional dataset to pass in for processing. If None,
            the dataset will default to env.get_dataset()
        terminate_on_end (bool): Set done=True on the last timestep
            in a trajectory. Default is False, and will discard the
            last timestep in each trajectory.
        **kwargs: Arguments to pass to env.get_dataset().

    Returns:
        A dictionary containing keys:
            observations: An N x dim_obs array of observations.
            actions: An N x dim_action array of actions.
            next_observations: An N x dim_obs array of next observations.
            rewards: An N-dim float array of rewards.
            terminals: An N-dim boolean array of "done" or episode termination flags.
    """
    if dataset is None:
        dataset = env.get_dataset(**kwargs)

    N = dataset['rewards'].shape[0]
    obs_ = []
    next_obs_ = []
    action_ = []
    reward_ = []
    done_ = []

    # The newer version of the dataset adds an explicit
    # timeouts field. Keep old method for backwards compatability.
    use_timeouts = False
    if 'timeouts' in dataset:
        use_timeouts = True

    episode_step = 0
    for i in range(N-1):
        obs = dataset['observations'][i].astype(np.float32)
        new_obs = dataset['observations'][i+1].astype(np.float32)
        action = dataset['actions'][i].astype(np.float32)
        reward = dataset['rewards'][i].astype(np.float32)
        done_bool = bool(dataset['terminals'][i])

        if use_timeouts:
            final_timestep = dataset['timeouts'][i]
        else:
            final_timestep = (episode_step == env._max_episode_steps - 1)
        if (not terminate_on_end) and final_timestep:
            # Skip this transition and don't apply terminals on the last step of an episode
            episode_step = 0
            continue
        if done_bool or final_timestep:
            episode_step = 0

        obs_.append(obs)
        next_obs_.append(new_obs)
        action_.append(action)
        reward_.append(reward)
        done_.append(done_bool)
        episode_step += 1

    return {
        'observations': np.array(obs_),
        'actions': np.array(action_),
        'next_observations': np.array(next_obs_),
        'rewards': np.array(reward_),
        'terminals': np.array(done_),
    }

def sequence_dataset(env, dataset=None, **kwargs):
    """
    Returns an iterator through trajectories.

    Args:
        env: An OfflineEnv object.
        dataset: An optional dataset to pass in for processing. If None,
            the dataset will default to env.get_dataset()
        **kwargs: Arguments to pass to env.get_dataset().

    Returns:
        An iterator through dictionaries with keys:
            observations
            actions
            rewards
            terminals
    """
    if dataset is None:
        dataset = env.get_dataset(**kwargs)

    N = dataset['rewards'].shape[0]
    data_ = collections.defaultdict(list)

    # The newer version of the dataset adds an explicit
    # timeouts field. Keep old method for backwards compatability.
    use_timeouts = False
    if 'timeouts' in dataset:
        use_timeouts = True

    episode_step = 0
    for i in range(N):
        done_bool = bool(dataset['terminals'][i])
        if use_timeouts:
            final_timestep = dataset['timeouts'][i]
        else:
            final_timestep = (episode_step == env._max_episode_steps - 1)

        if done_bool or final_timestep:
            episode_step = 0
            episode_data = {}
            for k in data_:
                episode_data[k] = np.array(data_[k])
            yield episode_data
            data_ = collections.defaultdict(list)

        for k in dataset:
            data_[k].append(dataset[k][i])
        episode_step += 1

<|endoftext|>


<commit_before>from grokcore.view.util import url
from persistent import Persistent
from zope.interface import implements
from zope.component import getUtility, getMultiAdapter
from zope.intid.interfaces import IIntIds
from zope.globalrequest import getRequest
from zope.location.interfaces import ILocation
from zope.container.interfaces import INameChooser
from datetime import datetime, timedelta
import pytz

from .interfaces import IParameterDefinition, IProcessDefinition, IApplicationDefinition, IActivity, IBusinessAction, IRuntime
from .core import EventHandler, WorkItemBehavior, AlreadyLocked

class Parameter(object):

    implements(IParameterDefinition)

    input = output = False

    def __init__(self, name):
        self.__name__ = name

class OutputParameter(Parameter):

    output = True

class InputParameter(Parameter):

    input = True

class InputOutputParameter(InputParameter, OutputParameter):

    pass

class Application:

    implements(IApplicationDefinition)

    def __init__(self, *parameters):
        self.parameters = parameters

    def defineParameters(self, *parameters):
        self.parameters += parameters

    def __repr__(self):
        input = u', '.join([param.__name__ for param in self.parameters
                           if param.input == True])
        output = u', '.join([param.__name__ for param in self.parameters
                           if param.output == True])
        return "<Application %r: (%s) --> (%s)>" %(self.id, input, output)

class Activity(WorkItemBehavior, EventHandler):
    implements(IActivity)

class SubProcess(Activity):

    def __init__(self, process, definition):
        super(SubProcess, self).__init__(process, definition)
        self.wi = None
        self.subProcess = None

    def start_subprocess(self):
        pd = getUtility(
                IProcessDefinition,
                self.definition.processDefinition)
        proc = pd()
        runtime = getUtility(IRuntime)
        chooser = INameChooser(runtime)
        name = chooser.chooseName(self.definition.processDefinition, proc)
        runtime[name] = proc
        proc.attachedTo = self
        proc.start()
        self.subProcess = proc

        # unindex wi, but dont delete it
        self.wi.remove()

from zeam.form.base.interfaces import IForm

class ActionType:
    automatic = 1
    manual = 2

class BusinessAction(Persistent):
    implements(ILocation, IBusinessAction)

    context = NotImplemented
    action =  NotImplemented
    report =  NotImplemented
    study =  NotImplemented
    relation_validation = NotImplemented
    roles_validation = NotImplemented
    processsecurity_validation = NotImplemented
    state_validation = NotImplemented
    title = NotImplemented
    actionType = NotImplemented
    steps = {}

    def __init__(self, parent):
        super(BusinessAction, self).__init__()
        self.__parent__ = parent
        self.isexecuted = False

    @property
    def __name__(self):
        return self.action.__view_name__

    @property
    def process(self):
        return self.__parent__.process

    @property
    def view_name(self):
        return self.action.__view_name__

    @property
    def process_id(self):
        return self.__parent__.process_id

    @property
    def node_id(self):
        return self.__parent__.node_id

    @property
    def request(self):
        request = getRequest()
        if self.process is not None:
            ids = getUtility(IIntIds)
            uid = ids.getId(self.process)
            request.form[u'p_uid'] = uid
        return request

    def url(self, obj):
        if self.process is None:
            return url(getRequest(), obj, self.view_name)

        intids = getUtility(IIntIds)
        puid = intids.getId(self.process)
        return url(getRequest(), obj, self.view_name, {'p_uid': puid})

    def content(self, obj):
        content = u''
        view = getMultiAdapter((obj, self.request), name=self.view_name)

        if not view.__providedBy__(IForm):
            return None

        view.update()
        content = (content + view.content() )
        return content

    def studyContent(self, obj):
        content = u''
        view = getMultiAdapter((obj, self.request), name=self.study.__view_name__)

        if not view.__providedBy__(IForm):
            return None

        view.update()
        content = (content + view.content())
        return content

    def reportContent(self, obj):
        content = u''
        view = getMultiAdapter((obj, self.request), name=self.report.__view_name__)

        if not view.__providedBy__(IForm):
            return None

        view.update()
        content = (content + view.content())
        return content

    def validate(self,obj):
        if  self.isexecuted and not obj.__provides__(self.context) or not self.__parent__.validate():
            return False

        if self.relation_validation and not self.relation_validation.im_func(self.process, obj):
            return False

        if self.roles_validation and not self.roles_validation.im_func(self.process, obj):
            return False

        if self.processsecurity_validation and not self.processsecurity_validation.im_func(self.process, obj):
            return False

        if self.state_validation and not self.state_validation.im_func(self.process, obj):
            return False

        return True

    def beforexecution(self, resuest):
        self.lock(resuest)
        self.__parent__.lock(request)

    def start(self, context, request, appstruct, args = None):
        # il y a probablement un moyen plus simple en cherchant la méthode par son nom dans self par exemple..
        steps[appstruct['stepid']].im_func(self, context, request, appstruct)

    def execut(self, context, request, appstruct):
        pass

    def afterexecution(self, resuest):
        self.unlock(resuest)
        self.__parent__.node.workItemFinished(self.__parent__)
        self.isexecuted = True

    def redirect(self, context, request, appstruct):
        pass

    _lock = (None, None)

    def lock(self, request):
        """Raise AlreadyLocked if the activity was already locked by someone
        else.
        """
        if self.is_locked(request):
            raise AlreadyLocked(_(u"Already locked by ${user} at ${datetime}",
                mapping={'user': self._lock[0], 'datetime': self._lock[1]}))
        self._lock = (request.principal.id, datetime.now(pytz.utc))

    def unlock(self, request):
        """Raise AlreadyLocked if the activity was already locked by someone
        else.
        """
        if self.is_locked(request):
            raise AlreadyLocked(_(u"Already locked by ${user} at ${datetime}",
                mapping={'user': self._lock[0], 'datetime': self._lock[1]}))
        self._lock = (None, None)

    def is_locked(self, request):
        """If the activity was locked by the same user, return False.
        """
        if self._lock[1] is None:
            return False
        if self._lock[1] + LOCK_DURATION <= datetime.now(pytz.utc):
            return False
        if self._lock[0] == request.principal.id:
            return False
        return True

class ElementaryAction(BusinessAction):

    def execut(self, context, request, appstruct):
        isFinished = self.start(context, request, appstruct)
        if isFinished:
            self.afterexecution(request)
            self.redirect(context, request, appstruct)

# Une loopAction ne peut être une action avec des steps. Cela n'a pas de sens
class LoopActionCardinality(BusinessAction):

    loopMaximum = None
    loopCondition = None
    testBefore = False

    def _executBefore(self, context, request, appstruct):
        nbloop = 0
        while self.loopCondition.im_func(context, self.process, appstruct) and nbloop < loopMaximum:
            self.start(context, request, appstruct)
            nbloop += 1

    def _executAfter(self, context, request, appstruct):
        nbloop = 0
        while nbloop < loopMaximum:
            self.start(context, request, appstruct)
            nbloop += 1
            if self.loopCondition.im_func(context, self.process, appstruct):
                break

    def execut(self, context, request, appstruct):
        if testBefore:
            self._executBefore(context, request, appstruct)
        else:
            self._executAfter(context, request, appstruct)

        self.afterexecution(request)
        self.redirect(context, request, appstruct)

class LoopActionDataInput(BusinessAction):

    loopDataInputRef = None

    def execut(self, context, request, appstruct):
        instances = self.loopDataInputRef.im_func(context, self.process, appstruct)
        for item in instances:
            args = {'item': item}
            self.start(context, request, appstruct, args)

         self.afterexecution(request)
         self.redirect(context, request, appstruct)

class MultiInstanceActionLimitedCardinality(BusinessAction):

    loopCardinality = None
    isSequential = False

    def __init__(self, parent):
        super(MultiInstanceActionLimitedCardinality, self).__init__(parent)
        self.numberOfInstances = self.loopCardinality.im_func(None, self.process, None)
        for instance in range(self.numberOfInstances):
            self.__parent__.actions.append(ActionInstance(instance, self, parent))

        if self.numberOfInstances == 0:
            self.__parent__.node.workItemFinished(self.__parent__)

        self.isexecuted = True

class MultiInstanceActionInfiniteCardinality(BusinessAction):

    isSequential = False

    def beforexecution(self, resuest):
        if isSequential:
            self.lock(resuest)
            self.__parent__.lock(request)

    def afterexecution(self, resuest):
        if isSequential:
            self.unlock(resuest)
            self.__parent__.unlock(request)

    def execut(self, context, request, appstruct):
        isFinished = self.start(context, resuest, appstruct)
        if isFinished :
            self.afterexecution(request)
            self.redirect(context, request, appstruct)

class MultiInstanceActionDataInput(BusinessAction):

    loopDataInputRef = None
    isSequential = False

    def __init__(self, parent):
        super(MultiInstanceActionCardinality, self).__init__(parent)
        self.instances = PersistentList()
        # loopDataInputRef renvoie une liste d'éléments identifiables
        self.instances = self.loopDataInputRef.im_func(None, self.process, None)
        for instance in self.instances:
            if dataIsPrincipal:
                self.__parent__.actions.append(ActionInstanceAsPrincipal(instance, self, parent))
            else:
                self.__parent__.actions.append(ActionInstance(instance, self, parent))

            self.isexecuted = True

class ActionInstance(BusinessAction):

    # mia = multi instance action
    def __init__(self, item, mia, parent)
        super(ActionInstance, self).__init__(parent)
        self.mia = mia
        self.item = item

    def beforexecution(self, resuest):
        self.lock(resuest)
        if self.mia.isSequential:
            self.__parent__.lock(request)

    def afterexecution(self, resuest):
        if self.mia.isSequential:
            self.__parent__.unlock(request)

        if  not self.mia.instances:
            self.__parent__.node.workItemFinished(self.__parent__)

        self.isexecuted = True

    def start(self, context, resuest, appstruct, args = None):
        args = {'item': self.item}
        return self.mia.start(context, resuest, appstruct, args)

    def redirect(self, context, resuest, appstruct):
        args = {'item': self.item}
        self.mia.redirect(context, resuest, appstruct, args = None)

    def execut(self, context, resuest, appstruct):
        isFinished = self.start(context, resuest, appstruct)
        if isFinished :
            self.mia.instances.pop(self.item)
            self.afterexecution(request)
            self.redirect(context, request, appstruct)

class ActionInstanceAsPrincipal(ActionInstance):

    def validate(self,obj):
        return (obj is self.item) and super(ActionInstanceAsPrincipal, self).validate(obj)

# La question est comment faire pour les actions multiStep?
# Je pense que cela peut être géré dans l'action start de l'action. La fonction start se charge d'exécuter la bonne
# Step (voir la méthode start de la class BusinessAction (comportement par défaut en plutistep)).
# dans ce cas le nom du step doit être ajouté dans le appstruct comme paramètre de l'action: chaque step (Vue, généralement un form)
# demande à l'action de s'exécuter action.execut(...). Avant cela, la step rajoute dans le appstruct une clef 'stepid' et sa valeur
# correspondant à la méthode qui doit être exécuté. Le code associé aux steps et le start ainsi que les vues seront générés.

# exemple:

#class Monaction(ElementaryAction):
#
#    steps = {'s1':s1,'s2':s2}
#
#
#    def s1(self, contex, resuest, appstruct):
#        pass
#
#    def s2(self, contex, resuest, appstruct):
#        pass

<commit_msg>execution policy<commit_after>from grokcore.view.util import url
from persistent import Persistent
from zope.interface import implements
from zope.component import getUtility, getMultiAdapter
from zope.intid.interfaces import IIntIds
from zope.globalrequest import getRequest
from zope.location.interfaces import ILocation
from zope.container.interfaces import INameChooser
from datetime import datetime, timedelta
import pytz

from .interfaces import IParameterDefinition, IProcessDefinition, IApplicationDefinition, IActivity, IBusinessAction, IRuntime
from .core import EventHandler, WorkItemBehavior, AlreadyLocked

class Parameter(object):

    implements(IParameterDefinition)

    input = output = False

    def __init__(self, name):
        self.__name__ = name

class OutputParameter(Parameter):

    output = True

class InputParameter(Parameter):

    input = True

class InputOutputParameter(InputParameter, OutputParameter):

    pass

class Application:

    implements(IApplicationDefinition)

    def __init__(self, *parameters):
        self.parameters = parameters

    def defineParameters(self, *parameters):
        self.parameters += parameters

    def __repr__(self):
        input = u', '.join([param.__name__ for param in self.parameters
                           if param.input == True])
        output = u', '.join([param.__name__ for param in self.parameters
                           if param.output == True])
        return "<Application %r: (%s) --> (%s)>" %(self.id, input, output)

class Activity(WorkItemBehavior, EventHandler):
    implements(IActivity)

class SubProcess(Activity):

    def __init__(self, process, definition):
        super(SubProcess, self).__init__(process, definition)
        self.wi = None
        self.subProcess = None

    def start_subprocess(self):
        pd = getUtility(
                IProcessDefinition,
                self.definition.processDefinition)
        proc = pd()
        runtime = getUtility(IRuntime)
        chooser = INameChooser(runtime)
        name = chooser.chooseName(self.definition.processDefinition, proc)
        runtime[name] = proc
        proc.attachedTo = self
        proc.start()
        self.subProcess = proc

        # unindex wi, but dont delete it
        self.wi.remove()

from zeam.form.base.interfaces import IForm

class ActionType:
    automatic = 1
    manual = 2

class BusinessAction(Persistent):
    implements(ILocation, IBusinessAction)

    context = NotImplemented
    action =  NotImplemented
    report =  NotImplemented
    study =  NotImplemented
    relation_validation = NotImplemented
    roles_validation = NotImplemented
    processsecurity_validation = NotImplemented
    state_validation = NotImplemented
    title = NotImplemented
    actionType = NotImplemented
    steps = {}

    def __init__(self, parent):
        super(BusinessAction, self).__init__()
        self.__parent__ = parent
        self.isexecuted = False

    @property
    def __name__(self):
        return self.action.__view_name__

    @property
    def process(self):
        return self.__parent__.process

    @property
    def view_name(self):
        return self.action.__view_name__

    @property
    def process_id(self):
        return self.__parent__.process_id

    @property
    def node_id(self):
        return self.__parent__.node_id

    @property
    def request(self):
        request = getRequest()
        if self.process is not None:
            ids = getUtility(IIntIds)
            uid = ids.getId(self.process)
            request.form[u'p_uid'] = uid
        return request

    def url(self, obj):
        if self.process is None:
            return url(getRequest(), obj, self.view_name)

        intids = getUtility(IIntIds)
        puid = intids.getId(self.process)
        return url(getRequest(), obj, self.view_name, {'p_uid': puid})

    def content(self, obj):
        content = u''
        view = getMultiAdapter((obj, self.request), name=self.view_name)

        if not view.__providedBy__(IForm):
            return None

        view.update()
        content = (content + view.content() )
        return content

    def studyContent(self, obj):
        content = u''
        view = getMultiAdapter((obj, self.request), name=self.study.__view_name__)

        if not view.__providedBy__(IForm):
            return None

        view.update()
        content = (content + view.content())
        return content

    def reportContent(self, obj):
        content = u''
        view = getMultiAdapter((obj, self.request), name=self.report.__view_name__)

        if not view.__providedBy__(IForm):
            return None

        view.update()
        content = (content + view.content())
        return content

    def validate(self,obj):
        if  self.isexecuted and not obj.__provides__(self.context) or not self.__parent__.validate():
            return False

        if self.relation_validation and not self.relation_validation.im_func(self.process, obj):
            return False

        if self.roles_validation and not self.roles_validation.im_func(self.process, obj):
            return False

        if self.processsecurity_validation and not self.processsecurity_validation.im_func(self.process, obj):
            return False

        if self.state_validation and not self.state_validation.im_func(self.process, obj):
            return False

        return True

    def beforexecution(self, resuest):
        self.lock(resuest)
        self.__parent__.lock(request)

    def start(self, context, request, appstruct, args = None):
        # il y a probablement un moyen plus simple en cherchant la méthode par son nom dans self par exemple..
        return steps[appstruct['stepid']].im_func(self, context, request, appstruct, args)

    def execut(self, context, request, appstruct):
        pass

    def afterexecution(self, resuest):
        self.unlock(resuest)
        self.__parent__.node.workItemFinished(self.__parent__)
        self.isexecuted = True

    def redirect(self, context, request, appstruct):
        pass

    _lock = (None, None)

    def lock(self, request):
        """Raise AlreadyLocked if the activity was already locked by someone
        else.
        """
        if self.is_locked(request):
            raise AlreadyLocked(_(u"Already locked by ${user} at ${datetime}",
                mapping={'user': self._lock[0], 'datetime': self._lock[1]}))
        self._lock = (request.principal.id, datetime.now(pytz.utc))

    def unlock(self, request):
        """Raise AlreadyLocked if the activity was already locked by someone
        else.
        """
        if self.is_locked(request):
            raise AlreadyLocked(_(u"Already locked by ${user} at ${datetime}",
                mapping={'user': self._lock[0], 'datetime': self._lock[1]}))
        self._lock = (None, None)

    def is_locked(self, request):
        """If the activity was locked by the same user, return False.
        """
        if self._lock[1] is None:
            return False
        if self._lock[1] + LOCK_DURATION <= datetime.now(pytz.utc):
            return False
        if self._lock[0] == request.principal.id:
            return False
        return True

class ElementaryAction(BusinessAction):

    def execut(self, context, request, appstruct):
        isFinished = self.start(context, request, appstruct)
        if isFinished:
            self.afterexecution(request)
            self.redirect(context, request, appstruct)

# Une loopAction ne peut être une action avec des steps. Cela n'a pas de sens
class LoopActionCardinality(BusinessAction):

    loopMaximum = None
    loopCondition = None
    testBefore = False

    def _executBefore(self, context, request, appstruct):
        nbloop = 0
        while self.loopCondition.im_func(context, self.process, appstruct) and nbloop < loopMaximum:
            self.start(context, request, appstruct)
            nbloop += 1

    def _executAfter(self, context, request, appstruct):
        nbloop = 0
        while nbloop < loopMaximum:
            self.start(context, request, appstruct)
            nbloop += 1
            if self.loopCondition.im_func(context, self.process, appstruct):
                break

    def execut(self, context, request, appstruct):
        if testBefore:
            self._executBefore(context, request, appstruct)
        else:
            self._executAfter(context, request, appstruct)

        self.afterexecution(request)
        self.redirect(context, request, appstruct)

class LoopActionDataInput(BusinessAction):

    loopDataInputRef = None

    def execut(self, context, request, appstruct):
        instances = self.loopDataInputRef.im_func(context, self.process, appstruct)
        for item in instances:
            args = {'item': item}
            self.start(context, request, appstruct, args)

         self.afterexecution(request)
         self.redirect(context, request, appstruct)

class MultiInstanceActionLimitedCardinality(BusinessAction):

    loopCardinality = None
    isSequential = False

    def __init__(self, parent):
        super(MultiInstanceActionLimitedCardinality, self).__init__(parent)
        self.numberOfInstances = self.loopCardinality.im_func(None, self.process, None)
        for instance in range(self.numberOfInstances):
            self.__parent__.actions.append(ActionInstance(instance, self, parent))

        if self.numberOfInstances == 0:
            self.__parent__.node.workItemFinished(self.__parent__)

        self.isexecuted = True

class MultiInstanceActionInfiniteCardinality(BusinessAction):

    isSequential = False

    def beforexecution(self, resuest):
        if isSequential:
            self.lock(resuest)
            self.__parent__.lock(request)

    def afterexecution(self, resuest):
        if isSequential:
            self.unlock(resuest)
            self.__parent__.unlock(request)

    def execut(self, context, request, appstruct):
        isFinished = self.start(context, resuest, appstruct)
        if isFinished :
            self.afterexecution(request)
            self.redirect(context, request, appstruct)

class MultiInstanceActionDataInput(BusinessAction):

    loopDataInputRef = None
    isSequential = False

    def __init__(self, parent):
        super(MultiInstanceActionCardinality, self).__init__(parent)
        self.instances = PersistentList()
        # loopDataInputRef renvoie une liste d'éléments identifiables
        self.instances = self.loopDataInputRef.im_func(None, self.process, None)
        for instance in self.instances:
            if dataIsPrincipal:
                self.__parent__.actions.append(ActionInstanceAsPrincipal(instance, self, parent))
            else:
                self.__parent__.actions.append(ActionInstance(instance, self, parent))

            self.isexecuted = True

class ActionInstance(BusinessAction):

    # mia = multi instance action
    def __init__(self, item, mia, parent)
        super(ActionInstance, self).__init__(parent)
        self.mia = mia
        self.item = item

    def beforexecution(self, resuest):
        self.lock(resuest)
        if self.mia.isSequential:
            self.__parent__.lock(request)

    def afterexecution(self, resuest):
        if self.mia.isSequential:
            self.__parent__.unlock(request)

        if  not self.mia.instances:
            self.__parent__.node.workItemFinished(self.__parent__)

        self.isexecuted = True

    def start(self, context, resuest, appstruct, args = None):
        args = {'item': self.item}
        return self.mia.start(context, resuest, appstruct, args)

    def redirect(self, context, resuest, appstruct):
        args = {'item': self.item}
        self.mia.redirect(context, resuest, appstruct, args = None)

    def execut(self, context, resuest, appstruct):
        isFinished = self.start(context, resuest, appstruct)
        if isFinished :
            self.mia.instances.pop(self.item)
            self.afterexecution(request)
            self.redirect(context, request, appstruct)

class ActionInstanceAsPrincipal(ActionInstance):

    def validate(self,obj):
        return (obj is self.item) and super(ActionInstanceAsPrincipal, self).validate(obj)

# La question est comment faire pour les actions multiStep?
# Je pense que cela peut être géré dans l'action start de l'action. La fonction start se charge d'exécuter la bonne
# Step (voir la méthode start de la class BusinessAction (comportement par défaut en plutistep)).
# dans ce cas le nom du step doit être ajouté dans le appstruct comme paramètre de l'action: chaque step (Vue, généralement un form)
# demande à l'action de s'exécuter action.execut(...). Avant cela, la step rajoute dans le appstruct une clef 'stepid' et sa valeur
# correspondant à la méthode qui doit être exécuté. Le code associé aux steps et le start ainsi que les vues seront générés.

# exemple:

#class Monaction(ElementaryAction):
#
#    steps = {'s1':s1,'s2':s2}
#
#
#    def s1(self, contex, resuest, appstruct):
#        pass
#
#    def s2(self, contex, resuest, appstruct):
#        pass

<|endoftext|>


<commit_before>'''
Many aspects of the salt payload need to be managed, from the return of
encrypted keys to general payload dynamics and packaging, these happen
in here
'''

# Import python libs
import sys

# Import salt libs
import salt.log
import salt.crypt
from salt.exceptions import SaltReqTimeoutError
from salt._compat import pickle

# Import zeromq
import zmq

log = salt.log.logging.getLogger(__name__)

try:
    # Attempt to import msgpack
    import msgpack
    # There is a serialization issue on ARM and potentially other platforms
    # for some msgpack bindings, check for it
    if msgpack.loads(msgpack.dumps([1, 2, 3])) is None:
        raise ImportError
except ImportError:
    # Fall back to msgpack_pure
    try:
        import msgpack_pure as msgpack
    except ImportError:
        # TODO: Come up with a sane way to get a configured logfile
        #       and write to the logfile when this error is hit also
        log_format = '[%(levelname)-8s] %(message)s'
        salt.log.setup_console_logger(log_format=log_format)
        log.fatal('Unable to import msgpack or msgpack_pure python modules')
        sys.exit(1)

def package(payload):
    '''
    This method for now just wraps msgpack.dumps, but it is here so that
    we can make the serialization a custom option in the future with ease.
    '''
    return msgpack.dumps(payload)

def unpackage(package_):
    '''
    Unpackages a payload
    '''
    return msgpack.loads(package_, use_list=True)

def format_payload(enc, **kwargs):
    '''
    Pass in the required arguments for a payload, the enc type and the cmd,
    then a list of keyword args to generate the body of the load dict.
    '''
    payload = {'enc': enc}
    load = {}
    for key in kwargs:
        load[key] = kwargs[key]
    payload['load'] = load
    return package(payload)

class Serial(object):
    '''
    Create a serialization object, this object manages all message
    serialization in Salt
    '''
    def __init__(self, opts):
        if isinstance(opts, dict):
            self.serial = opts.get('serial', 'msgpack')
        elif isinstance(opts, str):
            self.serial = opts
        else:
            self.serial = 'msgpack'

    def loads(self, msg):
        '''
        Run the correct loads serialization format
        '''
        if self.serial == 'msgpack':
            return msgpack.loads(msg, use_list=True)
        elif self.serial == 'pickle':
            try:
                return pickle.loads(msg)
            except Exception:
                return msgpack.loads(msg, use_list=True)

    def load(self, fn_):
        '''
        Run the correct serialization to load a file
        '''
        data = fn_.read()
        fn_.close()
        return self.loads(data)

    def dumps(self, msg):
        '''
        Run the correct dumps serialization format
        '''
        if self.serial == 'pickle':
            return pickle.dumps(msg)
        else:
            return msgpack.dumps(msg)

    def dump(self, msg, fn_):
        '''
        Serialize the correct data into the named file object
        '''
        fn_.write(self.dumps(msg))
        fn_.close()

class SREQ(object):
    '''
    Create a generic interface to wrap salt zeromq req calls.
    '''
    def __init__(self, master, id_='', serial='msgpack', linger=0):
        self.master = master
        self.serial = Serial(serial)
        self.context = zmq.Context()
        self.socket = self.context.socket(zmq.REQ)
        self.socket.linger = linger
        if id_:
            self.socket.setsockopt(zmq.IDENTITY, id_)
        self.socket.connect(master)
        self.poller = zmq.Poller()

    def send(self, enc, load, tries=1, timeout=60):
        '''
        Takes two arguments, the encryption type and the base payload
        '''
        payload = {'enc': enc}
        payload['load'] = load
        package = self.serial.dumps(payload)
        self.socket.send(package)
        self.poller.register(self.socket, zmq.POLLIN)
        tried = 0
        while True:
            polled = self.poller.poll(timeout * 1000)
            if polled:
                break
            elif tried >= tries:
                raise SaltReqTimeoutError('Waited {0} seconds'.format(timeout))
            tried += 1
        try:
            return self.serial.loads(self.socket.recv())
        finally:
            self.poller.unregister(self.socket)

    def send_auto(self, payload):
        '''
        Detect the encryption type based on the payload
        '''
        enc = payload.get('enc', 'clear')
        load = payload.get('load', {})
        return self.send(enc, load)

    def destroy(self):
        for socket in self.poller.sockets.keys():
            if not socket.closed:
                socket.close()
            self.poller.unregister(socket)
        if not self.socket.closed:
            self.socket.close()
        self.context.term()

    def __del__(self):
        self.destroy()
<commit_msg>Fix `SaltReqTimeoutError` message to show the actual passed seconds. Refs #2676.<commit_after>'''
Many aspects of the salt payload need to be managed, from the return of
encrypted keys to general payload dynamics and packaging, these happen
in here
'''

# Import python libs
import sys

# Import salt libs
import salt.log
import salt.crypt
from salt.exceptions import SaltReqTimeoutError
from salt._compat import pickle

# Import zeromq
import zmq

log = salt.log.logging.getLogger(__name__)

try:
    # Attempt to import msgpack
    import msgpack
    # There is a serialization issue on ARM and potentially other platforms
    # for some msgpack bindings, check for it
    if msgpack.loads(msgpack.dumps([1, 2, 3])) is None:
        raise ImportError
except ImportError:
    # Fall back to msgpack_pure
    try:
        import msgpack_pure as msgpack
    except ImportError:
        # TODO: Come up with a sane way to get a configured logfile
        #       and write to the logfile when this error is hit also
        log_format = '[%(levelname)-8s] %(message)s'
        salt.log.setup_console_logger(log_format=log_format)
        log.fatal('Unable to import msgpack or msgpack_pure python modules')
        sys.exit(1)

def package(payload):
    '''
    This method for now just wraps msgpack.dumps, but it is here so that
    we can make the serialization a custom option in the future with ease.
    '''
    return msgpack.dumps(payload)

def unpackage(package_):
    '''
    Unpackages a payload
    '''
    return msgpack.loads(package_, use_list=True)

def format_payload(enc, **kwargs):
    '''
    Pass in the required arguments for a payload, the enc type and the cmd,
    then a list of keyword args to generate the body of the load dict.
    '''
    payload = {'enc': enc}
    load = {}
    for key in kwargs:
        load[key] = kwargs[key]
    payload['load'] = load
    return package(payload)

class Serial(object):
    '''
    Create a serialization object, this object manages all message
    serialization in Salt
    '''
    def __init__(self, opts):
        if isinstance(opts, dict):
            self.serial = opts.get('serial', 'msgpack')
        elif isinstance(opts, str):
            self.serial = opts
        else:
            self.serial = 'msgpack'

    def loads(self, msg):
        '''
        Run the correct loads serialization format
        '''
        if self.serial == 'msgpack':
            return msgpack.loads(msg, use_list=True)
        elif self.serial == 'pickle':
            try:
                return pickle.loads(msg)
            except Exception:
                return msgpack.loads(msg, use_list=True)

    def load(self, fn_):
        '''
        Run the correct serialization to load a file
        '''
        data = fn_.read()
        fn_.close()
        return self.loads(data)

    def dumps(self, msg):
        '''
        Run the correct dumps serialization format
        '''
        if self.serial == 'pickle':
            return pickle.dumps(msg)
        else:
            return msgpack.dumps(msg)

    def dump(self, msg, fn_):
        '''
        Serialize the correct data into the named file object
        '''
        fn_.write(self.dumps(msg))
        fn_.close()

class SREQ(object):
    '''
    Create a generic interface to wrap salt zeromq req calls.
    '''
    def __init__(self, master, id_='', serial='msgpack', linger=0):
        self.master = master
        self.serial = Serial(serial)
        self.context = zmq.Context()
        self.socket = self.context.socket(zmq.REQ)
        self.socket.linger = linger
        if id_:
            self.socket.setsockopt(zmq.IDENTITY, id_)
        self.socket.connect(master)
        self.poller = zmq.Poller()

    def send(self, enc, load, tries=1, timeout=60):
        '''
        Takes two arguments, the encryption type and the base payload
        '''
        payload = {'enc': enc}
        payload['load'] = load
        package = self.serial.dumps(payload)
        self.socket.send(package)
        self.poller.register(self.socket, zmq.POLLIN)
        tried = 0
        while True:
            polled = self.poller.poll(timeout * 1000)
            if polled:
                break
            elif tried >= tries:
                raise SaltReqTimeoutError(
                    'Waited {0} seconds'.format(
                        timeout * tried
                    )
                )
            tried += 1
        try:
            return self.serial.loads(self.socket.recv())
        finally:
            self.poller.unregister(self.socket)

    def send_auto(self, payload):
        '''
        Detect the encryption type based on the payload
        '''
        enc = payload.get('enc', 'clear')
        load = payload.get('load', {})
        return self.send(enc, load)

    def destroy(self):
        for socket in self.poller.sockets.keys():
            if not socket.closed:
                socket.close()
            self.poller.unregister(socket)
        if not self.socket.closed:
            self.socket.close()
        self.context.term()

    def __del__(self):
        self.destroy()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Set up the version of Salt
'''

# Import python libs
import re
import sys

# Import salt libs
try:
    from salt._compat import string_types
except ImportError:
    if sys.version_info[0] == 3:
        string_types = str
    else:
        string_types = basestring

# ----- ATTENTION ----------------------------------------------------------->
#
# For version bumps, please update `__saltstack_version__` below
#
# <---- ATTENTION ------------------------------------------------------------

class SaltStackVersion(object):
    '''
    Handle SaltStack versions class.

    Knows how to parse ``git describe`` output, knows about release candidates
    and also supports version comparison.
    '''

    __slots__ = ('major', 'minor', 'bugfix', 'rc', 'noc', 'sha')

    git_describe_regex = re.compile(
        r'(?:[^\d]+)?(?P<major>[\d]{1,2})\.(?P<minor>[\d]{1,2})'
        r'(?:\.(?P<bugfix>[\d]{0,2}))?(?:rc(?P<rc>[\d]{1}))?'
        r'(?:(?:.*)-(?P<noc>[\d]+)-(?P<sha>[a-z0-9]{8}))?'
    )

    def __init__(self,              # pylint: disable=C0103
                 major,
                 minor,
                 bugfix=0,
                 rc=0,              # pylint: disable=C0103
                 noc=0,
                 sha=None):

        if isinstance(major, string_types):
            major = int(major)

        if isinstance(minor, string_types):
            minor = int(minor)

        if bugfix is None:
            bugfix = 0
        elif isinstance(bugfix, string_types):
            bugfix = int(bugfix)

        if rc is None:
            rc = 0
        elif isinstance(rc, string_types):
            rc = int(rc)

        if noc is None:
            noc = 0
        elif isinstance(noc, string_types):
            noc = int(noc)

        self.major = major
        self.minor = minor
        self.bugfix = bugfix
        self.rc = rc  # pylint: disable=C0103
        self.noc = noc
        self.sha = sha

    @classmethod
    def parse(cls, version_string):
        match = cls.git_describe_regex.match(version_string)
        if not match:
            raise ValueError(
                'Unable to parse version string: {0!r}'.format(version_string)
            )
        return cls(*match.groups())

    @property
    def info(self):
        return (
            self.major,
            self.minor,
            self.bugfix
        )

    @property
    def rc_info(self):
        return (
            self.major,
            self.minor,
            self.bugfix,
            self.rc
        )

    @property
    def noc_info(self):
        return (
            self.major,
            self.minor,
            self.bugfix,
            self.rc,
            self.noc
        )

    @property
    def full_info(self):
        return (
            self.major,
            self.minor,
            self.bugfix,
            self.rc,
            self.noc,
            self.sha
        )

    @property
    def string(self):
        version_string = '{0}.{1}.{2}'.format(
            self.major,
            self.minor,
            self.bugfix
        )
        if self.rc:
            version_string += 'rc{0}'.format(self.rc)
        if self.noc and self.sha:
            version_string += '-{0}-{1}'.format(self.noc, self.sha)
        return version_string

    def __str__(self):
        return self.string

    def __cmp__(self, other):
        if not isinstance(other, SaltStackVersion):
            if isinstance(other, string_types):
                other = SaltStackVersion.parse(other)
            elif isinstance(other, (list, tuple)):
                other = SaltStackVersion(*other)
            else:
                raise ValueError(
                    'Cannot instantiate Version from type {0!r}'.format(
                        type(other)
                    )
                )

        if (self.rc and other.rc) or (not self.rc and not other.rc):
            # Both have rc information, regular compare is ok
            return cmp(self.noc_info, other.noc_info)

        # RC's are always lower versions than non RC's
        if self.rc > 0 and other.rc <= 0:
            noc_info = list(self.noc_info)
            noc_info[3] = -1
            return cmp(tuple(noc_info), other.noc_info)

        if self.rc <= 0 and other.rc > 0:
            other_noc_info = list(other.noc_info)
            other_noc_info[3] = -1
            return cmp(self.noc_info, tuple(other_noc_info))

# ----- Hardcoded Salt Version Information ---------------------------------->
#
# Please bump version information for __saltstack_version__ on new releases
# ----------------------------------------------------------------------------
__saltstack_version__ = SaltStackVersion(0, 17, 0)
__version_info__ = __saltstack_version__.info
__version__ = __saltstack_version__.string
# <---- Hardcoded Salt Version Information -----------------------------------

# ----- Dynamic/Runtime Salt Version Information ---------------------------->
def __get_version(version, version_info):
    '''
    If we can get a version provided at installation time or from Git, use
    that instead, otherwise we carry on.
    '''
    try:
        # Try to import the version information provided at install time
        from salt._version import __version__, __version_info__  # pylint: disable=E0611
        return __version__, __version_info__
    except ImportError:
        pass

    # This might be a 'python setup.py develop' installation type. Let's
    # discover the version information at runtime.
    import os
    import warnings
    import subprocess

    try:
        cwd = os.path.abspath(os.path.dirname(__file__))
    except NameError:
        # We're most likely being frozen and __file__ triggered this NameError
        # Let's work around that
        import inspect
        cwd = os.path.abspath(
            os.path.dirname(inspect.getsourcefile(__get_version))
        )

    try:
        kwargs = dict(
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=cwd
        )

        if not sys.platform.startswith('win'):
            # Let's not import `salt.utils` for the above check
            kwargs['close_fds'] = True

        process = subprocess.Popen(
                ['git', 'describe', '--tags', '--match', 'v[0-9]*'], **kwargs)
        out, err = process.communicate()
        out = out.strip()
        err = err.strip()

        if not out or err:
            return version, version_info

        parsed_version = SaltStackVersion.parse(out)

        if parsed_version.info > version_info:
            warnings.warn(
                'The parsed version info, `{0}`, is bigger than the one '
                'defined in the file, `{1}`. Missing version bump?'.format(
                    parsed_version.info,
                    version_info
                ),
                UserWarning,
                stacklevel=2
            )
            return version, version_info
        elif parsed_version.info < version_info:
            warnings.warn(
                'The parsed version info, `{0}`, is lower than the one '
                'defined in the file, `{1}`.'
                'In order to get the proper salt version with the git hash '
                'you need to update salt\'s local git tags. Something like: '
                '\'git fetch --tags\' or \'git fetch --tags upstream\' if '
                'you followed salt\'s contribute documentation. The version '
                'string WILL NOT include the git hash.'.format(
                    parsed_version.info,
                    version_info
                ),
                UserWarning,
                stacklevel=2
            )
            return version, version_info
        return parsed_version.string, parsed_version.info
    except OSError as os_err:
        if os_err.errno != 2:
            # If the errno is not 2(The system cannot find the file
            # specified), raise the exception so it can be catch by the
            # developers
            raise
    return version, version_info

# Get additional version information if available
__version__, __version_info__ = __get_version(__version__, __version_info__)
# This function has executed once, we're done with it. Delete it!
del __get_version
# <---- Dynamic/Runtime Salt Version Information -----------------------------

def versions_information():
    '''
    Report on all of the versions for dependent software
    '''
    libs = (
        ('Salt', None, __version__),
        ('Python', None, sys.version.rsplit('\n')[0].strip()),
        ('Jinja2', 'jinja2', '__version__'),
        ('M2Crypto', 'M2Crypto', 'version'),
        ('msgpack-python', 'msgpack', 'version'),
        ('msgpack-pure', 'msgpack_pure', 'version'),
        ('pycrypto', 'Crypto', '__version__'),
        ('PyYAML', 'yaml', '__version__'),
        ('PyZMQ', 'zmq', '__version__'),
        ('ZMQ', 'zmq', 'zmq_version')
    )
    for name, imp, attr in libs:
        if imp is None:
            yield name, attr
            continue
        try:
            imp = __import__(imp)
            version = getattr(imp, attr)
            if callable(version):
                version = version()
            if isinstance(version, (tuple, list)):
                version = '.'.join(map(str, version))
            yield name, version
        except ImportError:
            yield name, None

def versions_report():
    '''
    Yield each library properly formatted for a console clean output.
    '''
    libs = list(versions_information())

    padding = max(len(lib[0]) for lib in libs) + 1

    fmt = '{0:>{pad}}: {1}'

    for name, version in libs:
        yield fmt.format(name, version or 'Not Installed', pad=padding)

if __name__ == '__main__':
    print(__version__)
<commit_msg>Support named salt versions<commit_after># -*- coding: utf-8 -*-
'''
Set up the version of Salt
'''

# Import python libs
import re
import sys

# Import salt libs
try:
    from salt._compat import string_types
except ImportError:
    if sys.version_info[0] == 3:
        string_types = str
    else:
        string_types = basestring

# ----- ATTENTION ----------------------------------------------------------->
#
# For version bumps, please update `__saltstack_version__` below
#
# <---- ATTENTION ------------------------------------------------------------

class SaltStackVersion(object):
    '''
    Handle SaltStack versions class.

    Knows how to parse ``git describe`` output, knows about release candidates
    and also supports version comparison.
    '''

    __slots__ = ('name', 'major', 'minor', 'bugfix', 'rc', 'noc', 'sha')

    git_describe_regex = re.compile(
        r'(?:[^\d]+)?(?P<major>[\d]{1,2})\.(?P<minor>[\d]{1,2})'
        r'(?:\.(?P<bugfix>[\d]{0,2}))?(?:rc(?P<rc>[\d]{1}))?'
        r'(?:(?:.*)-(?P<noc>[\d]+)-(?P<sha>[a-z0-9]{8}))?'
    )

    # Salt versions after 0.17.0 will be numbered like:
    #   <2-digit-year>.<month>.<bugfix>
    #
    # Since the actual version numbers will only be know on release dates, the
    # periodic table element names will be what's going to be used to name
    # versions and to be able to mention them.

    NAMES = {
        # Let's keep at least 3 version names uncommented counting from the
        # latest release so we can map deprecation warnings to versions.

        # ----- Please refrain from fixing PEP-8 E203 ----------------------->
        # The idea is keep this readable
        # --------------------------------------------------------------------
        'Hydrogen'     : (sys.maxint - 108, 0, 0, 0),
        'Helium'       : (sys.maxint - 107, 0, 0, 0),
        'Lithium'      : (sys.maxint - 106, 0, 0, 0),
        #'Beryllium'    : (sys.maxint - 105, 0, 0, 0),
        #'Boron'        : (sys.maxint - 104, 0, 0, 0),
        #'Carbon'       : (sys.maxint - 103, 0, 0, 0),
        #'Nitrogen'     : (sys.maxint - 102, 0, 0, 0),
        #'Oxygen'       : (sys.maxint - 101, 0, 0, 0),
        #'Fluorine'     : (sys.maxint - 100, 0, 0, 0),
        #'Neon'         : (sys.maxint - 99 , 0, 0, 0),
        #'Sodium'       : (sys.maxint - 98 , 0, 0, 0),
        #'Magnesium'    : (sys.maxint - 97 , 0, 0, 0),
        #'Aluminium'    : (sys.maxint - 96 , 0, 0, 0),
        #'Silicon'      : (sys.maxint - 95 , 0, 0, 0),
        #'Phosphorus'   : (sys.maxint - 94 , 0, 0, 0),
        #'Sulfur'       : (sys.maxint - 93 , 0, 0, 0),
        #'Chlorine'     : (sys.maxint - 92 , 0, 0, 0),
        #'Argon'        : (sys.maxint - 91 , 0, 0, 0),
        #'Potassium'    : (sys.maxint - 90 , 0, 0, 0),
        #'Calcium'      : (sys.maxint - 89 , 0, 0, 0),
        #'Scandium'     : (sys.maxint - 88 , 0, 0, 0),
        #'Titanium'     : (sys.maxint - 87 , 0, 0, 0),
        #'Vanadium'     : (sys.maxint - 86 , 0, 0, 0),
        #'Chromium'     : (sys.maxint - 85 , 0, 0, 0),
        #'Manganese'    : (sys.maxint - 84 , 0, 0, 0),
        #'Iron'         : (sys.maxint - 83 , 0, 0, 0),
        #'Cobalt'       : (sys.maxint - 82 , 0, 0, 0),
        #'Nickel'       : (sys.maxint - 81 , 0, 0, 0),
        #'Copper'       : (sys.maxint - 80 , 0, 0, 0),
        #'Zinc'         : (sys.maxint - 79 , 0, 0, 0),
        #'Gallium'      : (sys.maxint - 78 , 0, 0, 0),
        #'Germanium'    : (sys.maxint - 77 , 0, 0, 0),
        #'Arsenic'      : (sys.maxint - 76 , 0, 0, 0),
        #'Selenium'     : (sys.maxint - 75 , 0, 0, 0),
        #'Bromine'      : (sys.maxint - 74 , 0, 0, 0),
        #'Krypton'      : (sys.maxint - 73 , 0, 0, 0),
        #'Rubidium'     : (sys.maxint - 72 , 0, 0, 0),
        #'Strontium'    : (sys.maxint - 71 , 0, 0, 0),
        #'Yttrium'      : (sys.maxint - 70 , 0, 0, 0),
        #'Zirconium'    : (sys.maxint - 69 , 0, 0, 0),
        #'Niobium'      : (sys.maxint - 68 , 0, 0, 0),
        #'Molybdenum'   : (sys.maxint - 67 , 0, 0, 0),
        #'Technetium'   : (sys.maxint - 66 , 0, 0, 0),
        #'Ruthenium'    : (sys.maxint - 65 , 0, 0, 0),
        #'Rhodium'      : (sys.maxint - 64 , 0, 0, 0),
        #'Palladium'    : (sys.maxint - 63 , 0, 0, 0),
        #'Silver'       : (sys.maxint - 62 , 0, 0, 0),
        #'Cadmium'      : (sys.maxint - 61 , 0, 0, 0),
        #'Indium'       : (sys.maxint - 60 , 0, 0, 0),
        #'Tin'          : (sys.maxint - 59 , 0, 0, 0),
        #'Antimony'     : (sys.maxint - 58 , 0, 0, 0),
        #'Tellurium'    : (sys.maxint - 57 , 0, 0, 0),
        #'Iodine'       : (sys.maxint - 56 , 0, 0, 0),
        #'Xenon'        : (sys.maxint - 55 , 0, 0, 0),
        #'Caesium'      : (sys.maxint - 54 , 0, 0, 0),
        #'Barium'       : (sys.maxint - 53 , 0, 0, 0),
        #'Lanthanum'    : (sys.maxint - 52 , 0, 0, 0),
        #'Cerium'       : (sys.maxint - 51 , 0, 0, 0),
        #'Praseodymium' : (sys.maxint - 50 , 0, 0, 0),
        #'Neodymium'    : (sys.maxint - 49 , 0, 0, 0),
        #'Promethium'   : (sys.maxint - 48 , 0, 0, 0),
        #'Samarium'     : (sys.maxint - 47 , 0, 0, 0),
        #'Europium'     : (sys.maxint - 46 , 0, 0, 0),
        #'Gadolinium'   : (sys.maxint - 45 , 0, 0, 0),
        #'Terbium'      : (sys.maxint - 44 , 0, 0, 0),
        #'Dysprosium'   : (sys.maxint - 43 , 0, 0, 0),
        #'Holmium'      : (sys.maxint - 42 , 0, 0, 0),
        #'Erbium'       : (sys.maxint - 41 , 0, 0, 0),
        #'Thulium'      : (sys.maxint - 40 , 0, 0, 0),
        #'Ytterbium'    : (sys.maxint - 39 , 0, 0, 0),
        #'Lutetium'     : (sys.maxint - 38 , 0, 0, 0),
        #'Hafnium'      : (sys.maxint - 37 , 0, 0, 0),
        #'Tantalum'     : (sys.maxint - 36 , 0, 0, 0),
        #'Tungsten'     : (sys.maxint - 35 , 0, 0, 0),
        #'Rhenium'      : (sys.maxint - 34 , 0, 0, 0),
        #'Osmium'       : (sys.maxint - 33 , 0, 0, 0),
        #'Iridium'      : (sys.maxint - 32 , 0, 0, 0),
        #'Platinum'     : (sys.maxint - 31 , 0, 0, 0),
        #'Gold'         : (sys.maxint - 30 , 0, 0, 0),
        #'Mercury'      : (sys.maxint - 29 , 0, 0, 0),
        #'Thallium'     : (sys.maxint - 28 , 0, 0, 0),
        #'Lead'         : (sys.maxint - 27 , 0, 0, 0),
        #'Bismuth'      : (sys.maxint - 26 , 0, 0, 0),
        #'Polonium'     : (sys.maxint - 25 , 0, 0, 0),
        #'Astatine'     : (sys.maxint - 24 , 0, 0, 0),
        #'Radon'        : (sys.maxint - 23 , 0, 0, 0),
        #'Francium'     : (sys.maxint - 22 , 0, 0, 0),
        #'Radium'       : (sys.maxint - 21 , 0, 0, 0),
        #'Actinium'     : (sys.maxint - 20 , 0, 0, 0),
        #'Thorium'      : (sys.maxint - 19 , 0, 0, 0),
        #'Protactinium' : (sys.maxint - 18 , 0, 0, 0),
        #'Uranium'      : (sys.maxint - 17 , 0, 0, 0),
        #'Neptunium'    : (sys.maxint - 16 , 0, 0, 0),
        #'Plutonium'    : (sys.maxint - 15 , 0, 0, 0),
        #'Americium'    : (sys.maxint - 14 , 0, 0, 0),
        #'Curium'       : (sys.maxint - 13 , 0, 0, 0),
        #'Berkelium'    : (sys.maxint - 12 , 0, 0, 0),
        #'Californium'  : (sys.maxint - 11 , 0, 0, 0),
        #'Einsteinium'  : (sys.maxint - 10 , 0, 0, 0),
        #'Fermium'      : (sys.maxint - 9  , 0, 0, 0),
        #'Mendelevium'  : (sys.maxint - 8  , 0, 0, 0),
        #'Nobelium'     : (sys.maxint - 7  , 0, 0, 0),
        #'Lawrencium'   : (sys.maxint - 6  , 0, 0, 0),
        #'Rutherfordium': (sys.maxint - 5  , 0, 0, 0),
        #'Dubnium'      : (sys.maxint - 4  , 0, 0, 0),
        #'Seaborgium'   : (sys.maxint - 3  , 0, 0, 0),
        #'Bohrium'      : (sys.maxint - 2  , 0, 0, 0),
        #'Hassium'      : (sys.maxint - 1  , 0, 0, 0),
        #'Meitnerium'   : (sys.maxint - 0  , 0, 0, 0),
        # <---- Please refrain from fixing PEP-8 E203 ------------------------
    }

    LNAMES = dict((k.lower(), v) for (k, v) in NAMES.iteritems())
    VNAMES = dict((v, k) for (k, v) in NAMES.iteritems())

    def __init__(self,              # pylint: disable=C0103
                 major,
                 minor,
                 bugfix=0,
                 rc=0,              # pylint: disable=C0103
                 noc=0,
                 sha=None):

        if isinstance(major, string_types):
            major = int(major)

        if isinstance(minor, string_types):
            minor = int(minor)

        if bugfix is None:
            bugfix = 0
        elif isinstance(bugfix, string_types):
            bugfix = int(bugfix)

        if rc is None:
            rc = 0
        elif isinstance(rc, string_types):
            rc = int(rc)

        if noc is None:
            noc = 0
        elif isinstance(noc, string_types):
            noc = int(noc)

        self.major = major
        self.minor = minor
        self.bugfix = bugfix
        self.rc = rc  # pylint: disable=C0103
        self.name = self.VNAMES.get((major, minor, bugfix, rc), None)
        self.noc = noc
        self.sha = sha

    @classmethod
    def parse(cls, version_string):
        match = cls.git_describe_regex.match(version_string)
        if not match:
            raise ValueError(
                'Unable to parse version string: {0!r}'.format(version_string)
            )
        return cls(*match.groups())

    @classmethod
    def from_name(cls, name):
        if name.lower() not in cls.LNAMES:
            raise ValueError(
                'Named version {0!r} is not know'.format(name)
            )
        return cls(*cls.LNAMES[name.lower()])

    @property
    def info(self):
        return (
            self.major,
            self.minor,
            self.bugfix
        )

    @property
    def rc_info(self):
        return (
            self.major,
            self.minor,
            self.bugfix,
            self.rc
        )

    @property
    def noc_info(self):
        return (
            self.major,
            self.minor,
            self.bugfix,
            self.rc,
            self.noc
        )

    @property
    def full_info(self):
        return (
            self.major,
            self.minor,
            self.bugfix,
            self.rc,
            self.noc,
            self.sha
        )

    @property
    def string(self):
        version_string = '{0}.{1}.{2}'.format(
            self.major,
            self.minor,
            self.bugfix
        )
        if self.rc:
            version_string += 'rc{0}'.format(self.rc)
        if self.noc and self.sha:
            version_string += '-{0}-{1}'.format(self.noc, self.sha)
        return version_string

    def __str__(self):
        return self.string

    def __cmp__(self, other):
        if not isinstance(other, SaltStackVersion):
            if isinstance(other, string_types):
                other = SaltStackVersion.parse(other)
            elif isinstance(other, (list, tuple)):
                other = SaltStackVersion(*other)
            else:
                raise ValueError(
                    'Cannot instantiate Version from type {0!r}'.format(
                        type(other)
                    )
                )

        if (self.rc and other.rc) or (not self.rc and not other.rc):
            # Both have rc information, regular compare is ok
            return cmp(self.noc_info, other.noc_info)

        # RC's are always lower versions than non RC's
        if self.rc > 0 and other.rc <= 0:
            noc_info = list(self.noc_info)
            noc_info[3] = -1
            return cmp(tuple(noc_info), other.noc_info)

        if self.rc <= 0 and other.rc > 0:
            other_noc_info = list(other.noc_info)
            other_noc_info[3] = -1
            return cmp(self.noc_info, tuple(other_noc_info))

    def __repr__(self):
        parts = []
        if self.name:
            parts.append('name={0!r}'.format(self.name))
        parts.extend([
            'major={0}'.format(self.major),
            'minor={0}'.format(self.minor),
            'bugfix={0}'.format(self.bugfix)
        ])
        if self.rc:
            parts.append('rc={0}'.format(self.rc))
        return '<{0} {1}>'.format(self.__class__.__name__, ' '.join(parts))

# ----- Hardcoded Salt Version Information ---------------------------------->
#
# Please bump version information for __saltstack_version__ on new releases
# ----------------------------------------------------------------------------
__saltstack_version__ = SaltStackVersion(0, 17, 0)
__version_info__ = __saltstack_version__.info
__version__ = __saltstack_version__.string
# <---- Hardcoded Salt Version Information -----------------------------------

# ----- Dynamic/Runtime Salt Version Information ---------------------------->
def __get_version(version, version_info):
    '''
    If we can get a version provided at installation time or from Git, use
    that instead, otherwise we carry on.
    '''
    try:
        # Try to import the version information provided at install time
        from salt._version import __version__, __version_info__  # pylint: disable=E0611
        return __version__, __version_info__
    except ImportError:
        pass

    # This might be a 'python setup.py develop' installation type. Let's
    # discover the version information at runtime.
    import os
    import warnings
    import subprocess

    try:
        cwd = os.path.abspath(os.path.dirname(__file__))
    except NameError:
        # We're most likely being frozen and __file__ triggered this NameError
        # Let's work around that
        import inspect
        cwd = os.path.abspath(
            os.path.dirname(inspect.getsourcefile(__get_version))
        )

    try:
        kwargs = dict(
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=cwd
        )

        if not sys.platform.startswith('win'):
            # Let's not import `salt.utils` for the above check
            kwargs['close_fds'] = True

        process = subprocess.Popen(
                ['git', 'describe', '--tags', '--match', 'v[0-9]*'], **kwargs)
        out, err = process.communicate()
        out = out.strip()
        err = err.strip()

        if not out or err:
            return version, version_info

        parsed_version = SaltStackVersion.parse(out)

        if parsed_version.info > version_info:
            warnings.warn(
                'The parsed version info, `{0}`, is bigger than the one '
                'defined in the file, `{1}`. Missing version bump?'.format(
                    parsed_version.info,
                    version_info
                ),
                UserWarning,
                stacklevel=2
            )
            return version, version_info
        elif parsed_version.info < version_info:
            warnings.warn(
                'The parsed version info, `{0}`, is lower than the one '
                'defined in the file, `{1}`.'
                'In order to get the proper salt version with the git hash '
                'you need to update salt\'s local git tags. Something like: '
                '\'git fetch --tags\' or \'git fetch --tags upstream\' if '
                'you followed salt\'s contribute documentation. The version '
                'string WILL NOT include the git hash.'.format(
                    parsed_version.info,
                    version_info
                ),
                UserWarning,
                stacklevel=2
            )
            return version, version_info
        return parsed_version.string, parsed_version.info
    except OSError as os_err:
        if os_err.errno != 2:
            # If the errno is not 2(The system cannot find the file
            # specified), raise the exception so it can be catch by the
            # developers
            raise
    return version, version_info

# Get additional version information if available
__version__, __version_info__ = __get_version(__version__, __version_info__)
# This function has executed once, we're done with it. Delete it!
del __get_version
# <---- Dynamic/Runtime Salt Version Information -----------------------------

def versions_information():
    '''
    Report on all of the versions for dependent software
    '''
    libs = (
        ('Salt', None, __version__),
        ('Python', None, sys.version.rsplit('\n')[0].strip()),
        ('Jinja2', 'jinja2', '__version__'),
        ('M2Crypto', 'M2Crypto', 'version'),
        ('msgpack-python', 'msgpack', 'version'),
        ('msgpack-pure', 'msgpack_pure', 'version'),
        ('pycrypto', 'Crypto', '__version__'),
        ('PyYAML', 'yaml', '__version__'),
        ('PyZMQ', 'zmq', '__version__'),
        ('ZMQ', 'zmq', 'zmq_version')
    )
    for name, imp, attr in libs:
        if imp is None:
            yield name, attr
            continue
        try:
            imp = __import__(imp)
            version = getattr(imp, attr)
            if callable(version):
                version = version()
            if isinstance(version, (tuple, list)):
                version = '.'.join(map(str, version))
            yield name, version
        except ImportError:
            yield name, None

def versions_report():
    '''
    Yield each library properly formatted for a console clean output.
    '''
    libs = list(versions_information())

    padding = max(len(lib[0]) for lib in libs) + 1

    fmt = '{0:>{pad}}: {1}'

    for name, version in libs:
        yield fmt.format(name, version or 'Not Installed', pad=padding)

if __name__ == '__main__':
    print(__version__)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8
from __future__ import print_function

import click

from .__init__ import __version__

class SaltCli(object):
    """
    The CLI implementation.
    """

    def run_cli(self):
        """
        Run the main loop
        """
        print(u'Version:', __version__)
        print(u'Home: https://github.com/glasslion/saltcli')

@click.command()
def cli():
    """
    Create and call the CLI
    """
    saltcli = SaltCli()
    saltcli.run_cli()

if __name__ == "__main__":
    cli()
<commit_msg>dummy completer<commit_after>#!/usr/bin/env python
# -*- coding: utf-8
from __future__ import print_function
import logging

import click
from prompt_toolkit.contrib.completers import WordCompleter
from prompt_toolkit.shortcuts import get_input

from .__init__ import __version__

class SaltCli(object):
    """
    The CLI implementation.
    """
    def __init__(self):
        self.logger = logging.getLogger(__name__)
        self.id_completer = WordCompleter([
            'id1', 'id2', 'id3'
        ])

    def run_cli(self):
        """
        Run the main loop
        """
        print(u'Version:', __version__)
        print(u'Home: https://github.com/glasslion/saltcli')

        while True:
            text = get_input(u"> ", completer=self.id_completer)

            print(u'You entered:', text)

@click.command()
def cli():
    """
    Create and call the CLI
    """
    saltcli = SaltCli()
    saltcli.run_cli()

if __name__ == "__main__":
    cli()
<|endoftext|>


<commit_before># coding: utf-8
from __future__ import print_function

# greenify
import greenify
greenify.greenify()

# python patch
import gevent
import gevent.monkey
gevent.monkey.patch_all()

import sys
import time
import mod_http_head
assert greenify.patch_lib(mod_http_head.__file__)
import fake_slow_http_server

stack = []

def c_http_head_check(addr):
    stack.append(('begin', addr, 'c'))
    print('%.5f head %s begin' % (time.time(), addr), file=sys.stderr)
    ret = mod_http_head.http_head(*addr)
    print('%.5f head %s end' % (time.time(), addr), file=sys.stderr)
    stack.append(('end', addr, 'c'))
    assert ret == 1

def python_http_head_check(addr):
    try:
        from httplib import HTTPConnection
    except ImportError:
        from http.client import HTTPConnection

    stack.append(('begin', addr, 'python'))
    print('%.5f head %s begin' % (time.time(), addr), file=sys.stderr)
    conn = HTTPConnection(*addr)
    conn.request("HEAD", "/")
    resp = conn.getresponse()
    status_code = resp.status
    print('%.5f head %s end' % (time.time(), addr), file=sys.stderr)
    stack.append(('end', addr, 'python'))
    assert 200 <= status_code < 400

def sleeper():
    stack.append(('begin', 'sleeper'))
    print('%.5f sleeper begin' % time.time(), file=sys.stderr)
    time.sleep(fake_slow_http_server.BLOCKING_SECONDS/2)
    print('%.5f sleeper end' % time.time(), file=sys.stderr)
    stack.append(('end', 'sleeper'))

def main():
    global stack
    local_addr = ("localhost", str(fake_slow_http_server.PORT))
    test_sites = (local_addr, ("msn.com", "80"), ("taobao.com", "80"),
                  ("douban.com", "80"), ("github.com", "80"))
    for fn in [python_http_head_check, c_http_head_check]:
        stack = []
        print('test %s' % fn.__name__)
        t0 = time.time()
        workers = [
            gevent.spawn(fn, addr)
            for addr in test_sites
        ]
        workers.append(gevent.spawn(sleeper))

        gevent.joinall(workers)
        time_elapsed = time.time() - t0
        print('done in %.5fs' % (time_elapsed))
        print()
        # HEAD to fake_slow_http_server is expected to
        # be the slowest worker
        assert stack[-1][:2] == ('end', local_addr), \
            "unexpected: %r" % (stack[-1][:2], )
        assert time_elapsed < fake_slow_http_server.BLOCKING_SECONDS * 1.5

if __name__ == '__main__':
    main()
<commit_msg>ci: update test_sites (#17)<commit_after># coding: utf-8
from __future__ import print_function

# greenify
import greenify
greenify.greenify()

# python patch
import gevent
import gevent.monkey
gevent.monkey.patch_all()

import sys
import time
import mod_http_head
assert greenify.patch_lib(mod_http_head.__file__)
import fake_slow_http_server

stack = []

def c_http_head_check(addr):
    stack.append(('begin', addr, 'c'))
    print('%.5f head %s begin' % (time.time(), addr), file=sys.stderr)
    ret = mod_http_head.http_head(*addr)
    print('%.5f head %s end' % (time.time(), addr), file=sys.stderr)
    stack.append(('end', addr, 'c'))
    assert ret == 1

def python_http_head_check(addr):
    try:
        from httplib import HTTPConnection
    except ImportError:
        from http.client import HTTPConnection

    stack.append(('begin', addr, 'python'))
    print('%.5f head %s begin' % (time.time(), addr), file=sys.stderr)
    conn = HTTPConnection(*addr)
    conn.request("HEAD", "/")
    resp = conn.getresponse()
    status_code = resp.status
    print('%.5f head %s end' % (time.time(), addr), file=sys.stderr)
    stack.append(('end', addr, 'python'))
    assert 200 <= status_code < 400

def sleeper():
    stack.append(('begin', 'sleeper'))
    print('%.5f sleeper begin' % time.time(), file=sys.stderr)
    time.sleep(fake_slow_http_server.BLOCKING_SECONDS/2)
    print('%.5f sleeper end' % time.time(), file=sys.stderr)
    stack.append(('end', 'sleeper'))

def main():
    global stack
    local_addr = ("localhost", str(fake_slow_http_server.PORT))
    test_sites = (local_addr, ("google.com", "80"), ("twitter.com", "80"),
                  ("douban.com", "80"), ("github.com", "80"))
    for fn in [python_http_head_check, c_http_head_check]:
        stack = []
        print('test %s' % fn.__name__)
        t0 = time.time()
        workers = [
            gevent.spawn(fn, addr)
            for addr in test_sites
        ]
        workers.append(gevent.spawn(sleeper))

        gevent.joinall(workers)
        time_elapsed = time.time() - t0
        print('done in %.5fs' % (time_elapsed))
        print()
        # HEAD to fake_slow_http_server is expected to
        # be the slowest worker
        assert stack[-1][:2] == ('end', local_addr), \
            "unexpected: %r" % (stack[-1][:2], )
        assert time_elapsed < fake_slow_http_server.BLOCKING_SECONDS * 1.5

if __name__ == '__main__':
    main()
<|endoftext|>