<commit_before>#!/usr/bin/env python

import json
import logging
import os
import Queue
from functools import wraps
from threading import Thread
from time import sleep

import requests
from pprint import pprint
from urlparse import urljoin

import cmdtree as cmd
from tqdm import tqdm

IMAGE_URL_TPL = "http://img.hb.aicdn.com/{file_key}"
XHR_HEADERS = {
    "X-Requested-With": "XMLHttpRequest",
    "User-Agent":
        "Mozilla/5.0 (Windows NT 10.0; WOW64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/56.0.2924.87 Safari/537.36",
}

def _safe_file_name(file_name):
    return file_name.replace("/", "_")

def _get_file_ext(mime_type):
    return mime_type.split("/")[-1]

def get_pins(board_dict):
    board = board_dict
    pins = []
    for info in board['pins']:
        ext = _get_file_ext(info['file']['type'])
        file_name = "%s.%s" % (info['pin_id'], ext)
        meta = {
            "pin_id": info['pin_id'],
            "url": IMAGE_URL_TPL.format(file_key=info['file']['key']),
            'type': info['file']['type'],
            'ext': ext,
            "title": info['raw_text'],
            "link": info['link'],
            "source": info['source'],
            "file_name": file_name
        }
        pins.append(meta)
    return pins

def get_boards(user_meta):
    boards = []
    for board in user_meta['boards']:
        meta = {
            "board_id": board['board_id'],
            "title": board['title'],
            "pins": None,
            "dir_name": _safe_file_name(board['title']),
        }
        boards.append(meta)
    return boards

def retry(max_retries=3):

    def wrapper(func):
        @wraps(func)
        def wrapped(*args, **kwargs):
            retries = 0
            while True:
                retries += 1
                try:
                    return func(*args, **kwargs)
                except Exception:
                    if retries > max_retries:
                        logging.exception("Error occurs while execute function\n")
                        break
            return None
        return wrapped

    return wrapper

@retry()
def do_request(method, *args, **kwargs):
    is_json = kwargs.pop('is_json', True)
    response = getattr(requests, method)(*args, timeout=(2, 10), **kwargs)
    if is_json:
        response.json()
    return response

class User(object):
    def __init__(self, user_url):
        self.session = requests.session()
        self.base_url = user_url
        self.further_url_tpl = urljoin(
            self.base_url,
            "?iyyi5hr3"
            "&max={board_id}"
            "&limit=10"
            "&wfl=1"
        )

        self.username = None
        self.board_count = None
        self.pin_count = None
        self.boards = []

    def _fetch_home(self):
        resp = do_request("get", self.base_url, headers=XHR_HEADERS)
        user_meta = resp.json()['user']
        self.username = user_meta['username']
        self.board_count = user_meta['board_count']
        self.pin_count = user_meta['pin_count']
        return get_boards(user_meta)

    def _fetch_further(self, prev_boards):
        max_id = prev_boards[-1]['board_id']
        further_url = self.further_url_tpl.format(
            board_id=max_id,
        )
        resp = do_request(
            "get",
            further_url,
            headers=XHR_HEADERS,
        )
        content = resp.json()
        return get_boards(content['user'])

    def fetch_boards(self):
        self.boards.extend(self._fetch_home())
        while self.board_count > len(self.boards):
            further_boards = self._fetch_further(self.boards)
            self.boards.extend(further_boards)
        return self.boards

    def as_dict(self):
        return {
            "username": self.username,
            "board_count": self.board_count,
            "boards": self.boards,
        }

class Board(object):
    def __init__(self, board_url_or_id):
        board_url_or_id = str(board_url_or_id)
        self.session = requests.session()
        if "http" in board_url_or_id:
            self.base_url = board_url_or_id
        else:
            self.base_url = "http://huaban.com/boards/{board_id}/".format(
                board_id=board_url_or_id
            )
        self.further_pin_url_tpl = urljoin(
            self.base_url,
            "?iyqrlr0z"
            "&max={pin_id}"
            "&limit=20"
            "&wfl=1"
        )

        # uninitialized properties
        self.pin_count = None
        self.title = None
        self.description = None
        self._pins = []

    def _fetch_home(self):
        resp = do_request(
            "get",
            self.base_url,
            headers=XHR_HEADERS,
        )
        resp = resp.json()
        board = resp['board']
        self.pin_count = board['pin_count']
        self.title = board['title']
        self.description = board['description']
        return get_pins(board)

    def _fetch_further(self, prev_pins):
        max_id = prev_pins[-1]['pin_id']
        further_url = self.further_pin_url_tpl.format(
            pin_id=max_id,
        )

        resp = do_request(
            "get",
            further_url,
            headers=XHR_HEADERS,
        )
        content = resp.json()
        return get_pins(content['board'])

    def fetch_pins(self):
        self._pins.extend(self._fetch_home())
        yield self._pins
        while self.pin_count > len(self._pins):
            further_pins = self._fetch_further(self._pins)
            self._pins.extend(further_pins)
            yield further_pins

    @property
    def pins(self):
        for pin_group in self.fetch_pins():
            for pin in pin_group:
                yield pin

    def as_dict(self):
        return {
            "pins": self.pins,
            "title": self.title,
            "description": self.description,
            "pin_count": self.pin_count,
        }

class Pin(object):
    def __init__(self, pin_meta, dir_to_save):
        self.url = pin_meta["url"]
        filename = u"{title}.{ext}".format(
            title=pin_meta['pin_id'],
            ext=pin_meta['ext'],
        )
        self.file_to_save = os.path.join(
            dir_to_save,
            filename,
        )

class HuaBan(object):
    def __init__(self, user_url):
        self.meta = None
        self.base_url = user_url
        self.user = User(user_url)
        self._boards = []

    def fetch_meta(self, sleep_time=0.1):
        self.user.fetch_boards()
        for meta in self.user.boards:
            self._boards.append(Board(meta['board_id']))

    @property
    def boards_pins(self):
        for board in self._boards:
            for pin in board.pins:
                yield board, pin

    def as_dict(self):
        meta = self.user.as_dict()
        meta['boards'] = [
            board.as_dict() for board in self._boards
        ]
        return meta

    def save_meta(self, file_name):
        meta = self.as_dict()
        json.dump(meta, open(file_name, "wb"))

class Worker(Thread):

    def __init__(
            self, queue, target
    ):
        """
        :type queue: Queue.Queue
        """
        super(Worker, self).__init__()
        self.task_func = target
        self.queue = queue
        self.daemon = True
        self._stopped = False

    def run(self):
        while not self._stopped:
            try:
                task = self.queue.get(timeout=60)
            except Queue.Empty:
                break
            else:
                self.task_func(*task)

    def stop(self):
        self._stopped = True

class Downloader(object):

    def __init__(self, user_url, workers=5):
        self.huaban = HuaBan(user_url)
        self.huaban.fetch_meta()
        self.root_dir = _safe_file_name(self.huaban.user.username)
        self.progress_bar = None
        self.queue = Queue.Queue()
        self.workers = tuple(
            Worker(self.queue, self.download_one)
            for x in xrange(workers)
        )

    def download_one(self, pin, dir_to_save):
        import logging
        pin = Pin(pin, dir_to_save)
        response = do_request("get", pin.url, is_json=False)
        if response is None:
            logging.error("Failed to download image: %s" % pin.url)
        with open(pin.file_to_save, "wb") as f:
            f.write(response.content)
            self.progress_bar.update(1)

    def get_board_dir(self, board):
        """
        :type board: Board
        """
        board_title = board.title.replace("/", "_")
        return os.path.join(self.root_dir, board_title)

    def start(self):
        self.progress_bar = tqdm(total=self.huaban.user.pin_count)

        if not os.path.exists(self.root_dir):
            os.mkdir(self.root_dir)

        for worker in self.workers:
            worker.start()

        self._fetch_boards_meta()

    def _fetch_boards_meta(self):
        for board, pin in self.huaban.boards_pins:
            path = self.get_board_dir(board)
            if not os.path.exists(path):
                os.mkdir(path)
            self.queue.put(
                (pin, path)
            )
        self.save()

    def save(self):
        meta_file = os.path.join(self.root_dir, "meta.json")
        self.huaban.save_meta(meta_file)

    def stop(self):
        for worker in self.workers:
            worker.stop()

    def join(self):
        for worker in self.workers:
            worker.join()

def start_download(user_url, workers):
    print("Fetching initial meta data from huaban...")
    downloader = Downloader(user_url, workers=workers)
    print("Downloading pins from HuaBan with %s workers..." % workers)
    downloader.start()

    while not downloader.queue.empty():
        try:
            sleep(10)
        except KeyboardInterrupt:
            print("User exit")
            break
    downloader.save()
    print("Meta data download completed and saved in meta.json")
    downloader.stop()
    downloader.join()

@cmd.argument("board-url")
@cmd.command("fetch-board")
def fetch_board(board_url):
    board = Board(board_url)
    pins = board.fetch_pins()
    pprint(pins)

@cmd.argument("user-url")
@cmd.command("fetch-user")
def fetch_user(user_url):
    user = User(user_url)
    boards = user.fetch_boards()
    pprint(boards)

@cmd.argument("user-url")
@cmd.command("fetch-meta")
def fetch_meta(user_url):
    huaban = HuaBan(user_url)
    huaban.fetch_meta()
    pprint(huaban.as_dict())

@cmd.argument("user-url")
@cmd.option("workers", type=cmd.INT, default=5, help="Number of download workers.")
@cmd.command("download")
def download(user_url, workers):
    start_download(user_url, workers=workers)

if __name__ == "__main__":
    cmd.entry()
<commit_msg>Fix: Bug fix for as_dict method for Board<commit_after>#!/usr/bin/env python

import json
import logging
import os
import Queue
from functools import wraps
from threading import Thread
from time import sleep

import requests
from pprint import pprint
from urlparse import urljoin

import cmdtree as cmd
from tqdm import tqdm

IMAGE_URL_TPL = "http://img.hb.aicdn.com/{file_key}"
XHR_HEADERS = {
    "X-Requested-With": "XMLHttpRequest",
    "User-Agent":
        "Mozilla/5.0 (Windows NT 10.0; WOW64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/56.0.2924.87 Safari/537.36",
}

def _safe_file_name(file_name):
    return file_name.replace("/", "_")

def _get_file_ext(mime_type):
    return mime_type.split("/")[-1]

def get_pins(board_dict):
    board = board_dict
    pins = []
    for info in board['pins']:
        ext = _get_file_ext(info['file']['type'])
        file_name = "%s.%s" % (info['pin_id'], ext)
        meta = {
            "pin_id": info['pin_id'],
            "url": IMAGE_URL_TPL.format(file_key=info['file']['key']),
            'type': info['file']['type'],
            'ext': ext,
            "title": info['raw_text'],
            "link": info['link'],
            "source": info['source'],
            "file_name": file_name
        }
        pins.append(meta)
    return pins

def get_boards(user_meta):
    boards = []
    for board in user_meta['boards']:
        meta = {
            "board_id": board['board_id'],
            "title": board['title'],
            "pins": None,
            "dir_name": _safe_file_name(board['title']),
        }
        boards.append(meta)
    return boards

def retry(max_retries=3):

    def wrapper(func):
        @wraps(func)
        def wrapped(*args, **kwargs):
            retries = 0
            while True:
                retries += 1
                try:
                    return func(*args, **kwargs)
                except Exception:
                    if retries > max_retries:
                        logging.exception("Error occurs while execute function\n")
                        break
            return None
        return wrapped

    return wrapper

@retry()
def do_request(method, *args, **kwargs):
    is_json = kwargs.pop('is_json', True)
    response = getattr(requests, method)(*args, timeout=(2, 10), **kwargs)
    if is_json:
        response.json()
    return response

class User(object):
    def __init__(self, user_url):
        self.session = requests.session()
        self.base_url = user_url
        self.further_url_tpl = urljoin(
            self.base_url,
            "?iyyi5hr3"
            "&max={board_id}"
            "&limit=10"
            "&wfl=1"
        )

        self.username = None
        self.board_count = None
        self.pin_count = None
        self.boards = []

    def _fetch_home(self):
        resp = do_request("get", self.base_url, headers=XHR_HEADERS)
        user_meta = resp.json()['user']
        self.username = user_meta['username']
        self.board_count = user_meta['board_count']
        self.pin_count = user_meta['pin_count']
        return get_boards(user_meta)

    def _fetch_further(self, prev_boards):
        max_id = prev_boards[-1]['board_id']
        further_url = self.further_url_tpl.format(
            board_id=max_id,
        )
        resp = do_request(
            "get",
            further_url,
            headers=XHR_HEADERS,
        )
        content = resp.json()
        return get_boards(content['user'])

    def fetch_boards(self):
        self.boards.extend(self._fetch_home())
        while self.board_count > len(self.boards):
            further_boards = self._fetch_further(self.boards)
            self.boards.extend(further_boards)
        return self.boards

    def as_dict(self):
        return {
            "username": self.username,
            "board_count": self.board_count,
            "boards": self.boards,
        }

class Board(object):
    def __init__(self, board_url_or_id):
        board_url_or_id = str(board_url_or_id)
        self.session = requests.session()
        if "http" in board_url_or_id:
            self.base_url = board_url_or_id
        else:
            self.base_url = "http://huaban.com/boards/{board_id}/".format(
                board_id=board_url_or_id
            )
        self.further_pin_url_tpl = urljoin(
            self.base_url,
            "?iyqrlr0z"
            "&max={pin_id}"
            "&limit=20"
            "&wfl=1"
        )

        # uninitialized properties
        self.pin_count = None
        self.title = None
        self.description = None
        self._pins = []

    def _fetch_home(self):
        resp = do_request(
            "get",
            self.base_url,
            headers=XHR_HEADERS,
        )
        resp = resp.json()
        board = resp['board']
        self.pin_count = board['pin_count']
        self.title = board['title']
        self.description = board['description']
        return get_pins(board)

    def _fetch_further(self, prev_pins):
        max_id = prev_pins[-1]['pin_id']
        further_url = self.further_pin_url_tpl.format(
            pin_id=max_id,
        )

        resp = do_request(
            "get",
            further_url,
            headers=XHR_HEADERS,
        )
        content = resp.json()
        return get_pins(content['board'])

    def fetch_pins(self):
        self._pins.extend(self._fetch_home())
        yield self._pins
        while self.pin_count > len(self._pins):
            further_pins = self._fetch_further(self._pins)
            self._pins.extend(further_pins)
            yield further_pins

    @property
    def pins(self):
        for pin_group in self.fetch_pins():
            for pin in pin_group:
                yield pin

    def as_dict(self):
        return {
            "pins": self._pins,
            "title": self.title,
            "description": self.description,
            "pin_count": self.pin_count,
        }

class Pin(object):
    def __init__(self, pin_meta, dir_to_save):
        self.url = pin_meta["url"]
        filename = u"{title}.{ext}".format(
            title=pin_meta['pin_id'],
            ext=pin_meta['ext'],
        )
        self.file_to_save = os.path.join(
            dir_to_save,
            filename,
        )

class HuaBan(object):
    def __init__(self, user_url):
        self.meta = None
        self.base_url = user_url
        self.user = User(user_url)
        self._boards = []

    def fetch_meta(self, sleep_time=0.1):
        self.user.fetch_boards()
        for meta in self.user.boards:
            self._boards.append(Board(meta['board_id']))

    @property
    def boards_pins(self):
        for board in self._boards:
            for pin in board.pins:
                yield board, pin

    def as_dict(self):
        meta = self.user.as_dict()
        meta['boards'] = [
            board.as_dict() for board in self._boards
        ]
        return meta

    def save_meta(self, file_name):
        meta = self.as_dict()
        json.dump(meta, open(file_name, "wb"))

class Worker(Thread):

    def __init__(
            self, queue, target
    ):
        """
        :type queue: Queue.Queue
        """
        super(Worker, self).__init__()
        self.task_func = target
        self.queue = queue
        self.daemon = True
        self._stopped = False

    def run(self):
        while not self._stopped:
            try:
                task = self.queue.get(timeout=60)
            except Queue.Empty:
                break
            else:
                self.task_func(*task)

    def stop(self):
        self._stopped = True

class Downloader(object):

    def __init__(self, user_url, workers=5):
        self.huaban = HuaBan(user_url)
        self.huaban.fetch_meta()
        self.root_dir = _safe_file_name(self.huaban.user.username)
        self.progress_bar = None
        self.queue = Queue.Queue()
        self.workers = tuple(
            Worker(self.queue, self.download_one)
            for x in xrange(workers)
        )

    def download_one(self, pin, dir_to_save):
        import logging
        pin = Pin(pin, dir_to_save)
        response = do_request("get", pin.url, is_json=False)
        if response is None:
            logging.error("Failed to download image: %s" % pin.url)
        with open(pin.file_to_save, "wb") as f:
            f.write(response.content)
            self.progress_bar.update(1)

    def get_board_dir(self, board):
        """
        :type board: Board
        """
        board_title = board.title.replace("/", "_")
        return os.path.join(self.root_dir, board_title)

    def start(self):
        self.progress_bar = tqdm(total=self.huaban.user.pin_count)

        if not os.path.exists(self.root_dir):
            os.mkdir(self.root_dir)

        for worker in self.workers:
            worker.start()

        self._fetch_boards_meta()

    def _fetch_boards_meta(self):
        for board, pin in self.huaban.boards_pins:
            path = self.get_board_dir(board)
            if not os.path.exists(path):
                os.mkdir(path)
            self.queue.put(
                (pin, path)
            )
        self.save()

    def save(self):
        meta_file = os.path.join(self.root_dir, "meta.json")
        self.huaban.save_meta(meta_file)

    def stop(self):
        for worker in self.workers:
            worker.stop()

    def join(self):
        for worker in self.workers:
            worker.join()

def start_download(user_url, workers):
    print("Fetching initial meta data from huaban...")
    downloader = Downloader(user_url, workers=workers)
    print("Downloading pins from HuaBan with %s workers..." % workers)
    downloader.start()

    while not downloader.queue.empty():
        try:
            sleep(10)
        except KeyboardInterrupt:
            print("User exit")
            break
    downloader.save()
    print("Meta data download completed and saved in meta.json")
    downloader.stop()
    downloader.join()

@cmd.argument("board-url")
@cmd.command("fetch-board")
def fetch_board(board_url):
    board = Board(board_url)
    pins = board.fetch_pins()
    pprint(pins)

@cmd.argument("user-url")
@cmd.command("fetch-user")
def fetch_user(user_url):
    user = User(user_url)
    boards = user.fetch_boards()
    pprint(boards)

@cmd.argument("user-url")
@cmd.command("fetch-meta")
def fetch_meta(user_url):
    huaban = HuaBan(user_url)
    huaban.fetch_meta()
    pprint(huaban.as_dict())

@cmd.argument("user-url")
@cmd.option("workers", type=cmd.INT, default=5, help="Number of download workers.")
@cmd.command("download")
def download(user_url, workers):
    start_download(user_url, workers=workers)

if __name__ == "__main__":
    cmd.entry()
<|endoftext|>


<commit_before>###############################################################################
# The MIT License (MIT)
#
# Copyright (c) 2013 Cássio Paixão
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
###############################################################################

import logging

from core.virtual_machine import VirtualMachine
from core.strategies import PredictionStrategy

class RBFPrediction(PredictionStrategy):

    def initialize(self):
        self.measurement_reader = self._config.module['MeasurementReader']
        self.rbf_prediction = self._config.module['RBFTimeSeriesPrediction']
        self.rbf_window_size = int(self._config.params['rbf_time_series_prediction_window_size'])

    @PredictionStrategy.predict_strategy
    def predict(self, vm_name):
        last_measurements = self._get_last(vm_name, self.rbf_window_size)
        if self._logger.level <= logging.DEBUG:
            self._logger.debug('Prediction called for vm %s. Last measurements: [%s]',
                                vm_name,
                                ', '.join('({},{})'.format(m[self.measurement_reader.CPU], m[self.measurement_reader.MEM]) for m in last_measurements))

        if len(last_measurements) < self.rbf_window_size:
            self._logger.debug('No prediction. %d measurements found.', len(last_measurements))
            return None

        return self._new_demands(last_measurements)

    def _get_last(self, vm_name, window_size):
        return self.measurement_reader.n_measurements_till(
            vm_name,
            window_size,
            self._config.simulation_info.current_timestamp)

    def _new_demands(self, measurements):
        try:
            new_cpu = min(self.rbf_prediction.predict(m[self.measurement_reader.CPU] for m in measurements), 1)
            new_mem = min(self.rbf_prediction.predict(m[self.measurement_reader.MEM] for m in measurements), 1)

            new_demands = VirtualMachine('', new_cpu, new_mem)
            return new_demands
        except Exception as e:
            self._logger.info('RBF exception error: %s', e)
            self._logger.info('RBF exception values: [%s]',
                                               ', '.join('({},{})'.format(m[self.measurement_reader.CPU], m[self.measurement_reader.MEM]) for m in measurements))
            return None
<commit_msg>Prevents nan values at RBF Prediction<commit_after>###############################################################################
# The MIT License (MIT)
#
# Copyright (c) 2013 Cássio Paixão
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
###############################################################################

import logging
import math

from core.virtual_machine import VirtualMachine
from core.strategies import PredictionStrategy

class RBFPrediction(PredictionStrategy):

    def initialize(self):
        self.measurement_reader = self._config.module['MeasurementReader']
        self.rbf_prediction = self._config.module['RBFTimeSeriesPrediction']
        self.rbf_window_size = int(self._config.params['rbf_time_series_prediction_window_size'])

    @PredictionStrategy.predict_strategy
    def predict(self, vm_name):
        last_measurements = self._get_last(vm_name, self.rbf_window_size)
        if self._logger.level <= logging.DEBUG:
            self._logger.debug('Prediction called for vm %s. Last measurements: [%s]',
                                vm_name,
                                ', '.join('({},{})'.format(m[self.measurement_reader.CPU], m[self.measurement_reader.MEM]) for m in last_measurements))

        if len(last_measurements) < self.rbf_window_size:
            self._logger.debug('No prediction. %d measurements found.', len(last_measurements))
            return None

        return self._new_demands(last_measurements)

    def _get_last(self, vm_name, window_size):
        return self.measurement_reader.n_measurements_till(
            vm_name,
            window_size,
            self._config.simulation_info.current_timestamp)

    def _new_demands(self, measurements):
        try:
            new_cpu = max(min(self.rbf_prediction.predict(m[self.measurement_reader.CPU] for m in measurements), 1), 0)
            new_mem = max(min(self.rbf_prediction.predict(m[self.measurement_reader.MEM] for m in measurements), 1), 0)

            if math.isnan(new_cpu) or math.isnan(new_mem):
                raise Exception('NaN at prediction: cpu( {} ) mem( {} )'.format(new_cpu, new_mem))

            new_demands = VirtualMachine('', new_cpu, new_mem)
            return new_demands
        except Exception as e:
            self._logger.info('RBF exception error: %s', e)
            self._logger.info('RBF exception values: [%s]',
                                               ', '.join('({},{})'.format(m[self.measurement_reader.CPU], m[self.measurement_reader.MEM]) for m in measurements))
            return None
<|endoftext|>


<commit_before># Copyright 2012 Locaweb.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import fcntl
import glob
import grp
import os
import pwd
import shlex
import socket
import struct
import tempfile
import threading

from debtcollector import removals
import eventlet
from eventlet.green import subprocess
from eventlet import greenthread
from oslo_config import cfg
from oslo_log import log as logging
from oslo_log import loggers
from oslo_rootwrap import client
from oslo_utils import excutils
import six
from six.moves import http_client as httplib

from neutron.agent.common import config
from neutron.common import constants
from neutron.common import utils
from neutron.i18n import _LE
from neutron import wsgi

LOG = logging.getLogger(__name__)
config.register_root_helper(cfg.CONF)

class RootwrapDaemonHelper(object):
    __client = None
    __lock = threading.Lock()

    def __new__(cls):
        """There is no reason to instantiate this class"""
        raise NotImplementedError()

    @classmethod
    def get_client(cls):
        with cls.__lock:
            if cls.__client is None:
                cls.__client = client.Client(
                    shlex.split(cfg.CONF.AGENT.root_helper_daemon))
            return cls.__client

def addl_env_args(addl_env):
    """Build arugments for adding additional environment vars with env"""

    # NOTE (twilson) If using rootwrap, an EnvFilter should be set up for the
    # command instead of a CommandFilter.
    if addl_env is None:
        return []
    return ['env'] + ['%s=%s' % pair for pair in addl_env.items()]

def create_process(cmd, run_as_root=False, addl_env=None):
    """Create a process object for the given command.

    The return value will be a tuple of the process object and the
    list of command arguments used to create it.
    """
    cmd = list(map(str, addl_env_args(addl_env) + cmd))
    if run_as_root:
        cmd = shlex.split(config.get_root_helper(cfg.CONF)) + cmd
    LOG.debug("Running command: %s", cmd)
    obj = utils.subprocess_popen(cmd, shell=False,
                                 stdin=subprocess.PIPE,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)

    return obj, cmd

def execute_rootwrap_daemon(cmd, process_input, addl_env):
    cmd = list(map(str, addl_env_args(addl_env) + cmd))
    # NOTE(twilson) oslo_rootwrap.daemon will raise on filter match
    # errors, whereas oslo_rootwrap.cmd converts them to return codes.
    # In practice, no neutron code should be trying to execute something that
    # would throw those errors, and if it does it should be fixed as opposed to
    # just logging the execution error.
    LOG.debug("Running command (rootwrap daemon): %s", cmd)
    client = RootwrapDaemonHelper.get_client()
    return client.execute(cmd, process_input)

def execute(cmd, process_input=None, addl_env=None,
            check_exit_code=True, return_stderr=False, log_fail_as_error=True,
            extra_ok_codes=None, run_as_root=False):
    try:
        if (process_input is None or
            isinstance(process_input, six.binary_type)):
            _process_input = process_input
        else:
            _process_input = process_input.encode('utf-8')
        if run_as_root and cfg.CONF.AGENT.root_helper_daemon:
            returncode, _stdout, _stderr = (
                execute_rootwrap_daemon(cmd, process_input, addl_env))
        else:
            obj, cmd = create_process(cmd, run_as_root=run_as_root,
                                      addl_env=addl_env)
            _stdout, _stderr = obj.communicate(_process_input)
            returncode = obj.returncode
            obj.stdin.close()
        if six.PY3:
            if isinstance(_stdout, bytes):
                try:
                    _stdout = _stdout.decode(encoding='utf-8')
                except UnicodeError:
                    pass
            if isinstance(_stderr, bytes):
                try:
                    _stderr = _stderr.decode(encoding='utf-8')
                except UnicodeError:
                    pass

        m = _("\nCommand: {cmd}\nExit code: {code}\nStdin: {stdin}\n"
              "Stdout: {stdout}\nStderr: {stderr}").format(
                  cmd=cmd,
                  code=returncode,
                  stdin=process_input or '',
                  stdout=_stdout,
                  stderr=_stderr)

        extra_ok_codes = extra_ok_codes or []
        if returncode and returncode in extra_ok_codes:
            returncode = None

        if returncode and log_fail_as_error:
            LOG.error(m)
        else:
            LOG.debug(m)

        if returncode and check_exit_code:
            raise RuntimeError(m)
    finally:
        # NOTE(termie): this appears to be necessary to let the subprocess
        #               call clean something up in between calls, without
        #               it two execute calls in a row hangs the second one
        greenthread.sleep(0)

    return (_stdout, _stderr) if return_stderr else _stdout

def get_interface_mac(interface):
    MAC_START = 18
    MAC_END = 24
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    dev = interface[:constants.DEVICE_NAME_MAX_LEN]
    if isinstance(dev, six.text_type):
        dev = dev.encode('utf-8')
    info = fcntl.ioctl(s.fileno(), 0x8927, struct.pack('256s', dev))
    return ''.join(['%02x:' % ord(char)
                    for char in info[MAC_START:MAC_END]])[:-1]

def replace_file(file_name, data):
    """Replaces the contents of file_name with data in a safe manner.

    First write to a temp file and then rename. Since POSIX renames are
    atomic, the file is unlikely to be corrupted by competing writes.

    We create the tempfile on the same device to ensure that it can be renamed.
    """

    base_dir = os.path.dirname(os.path.abspath(file_name))
    tmp_file = tempfile.NamedTemporaryFile('w+', dir=base_dir, delete=False)
    tmp_file.write(data)
    tmp_file.close()
    os.chmod(tmp_file.name, 0o644)
    os.rename(tmp_file.name, file_name)

def find_child_pids(pid):
    """Retrieve a list of the pids of child processes of the given pid."""

    try:
        raw_pids = execute(['ps', '--ppid', pid, '-o', 'pid='],
                           log_fail_as_error=False)
    except RuntimeError as e:
        # Unexpected errors are the responsibility of the caller
        with excutils.save_and_reraise_exception() as ctxt:
            # Exception has already been logged by execute
            no_children_found = 'Exit code: 1' in str(e)
            if no_children_found:
                ctxt.reraise = False
                return []
    return [x.strip() for x in raw_pids.split('\n') if x.strip()]

@removals.remove(message='Use neutron.common.utils.ensure_dir instead.')
def ensure_dir(*args, **kwargs):
    return utils.ensure_dir(*args, **kwargs)

def _get_conf_base(cfg_root, uuid, ensure_conf_dir):
    #TODO(mangelajo): separate responsibilities here, ensure_conf_dir
    #                 should be a separate function
    conf_dir = os.path.abspath(os.path.normpath(cfg_root))
    conf_base = os.path.join(conf_dir, uuid)
    if ensure_conf_dir:
        utils.ensure_dir(conf_dir)
    return conf_base

def get_conf_file_name(cfg_root, uuid, cfg_file, ensure_conf_dir=False):
    """Returns the file name for a given kind of config file."""
    conf_base = _get_conf_base(cfg_root, uuid, ensure_conf_dir)
    return "%s.%s" % (conf_base, cfg_file)

def get_value_from_file(filename, converter=None):

    try:
        with open(filename, 'r') as f:
            try:
                return converter(f.read()) if converter else f.read()
            except ValueError:
                LOG.error(_LE('Unable to convert value in %s'), filename)
    except IOError:
        LOG.debug('Unable to access %s', filename)

def get_value_from_conf_file(cfg_root, uuid, cfg_file, converter=None):
    """A helper function to read a value from one of a config file."""
    file_name = get_conf_file_name(cfg_root, uuid, cfg_file)
    return get_value_from_file(file_name, converter)

def remove_conf_files(cfg_root, uuid):
    conf_base = _get_conf_base(cfg_root, uuid, False)
    for file_path in glob.iglob("%s.*" % conf_base):
        os.unlink(file_path)

def get_root_helper_child_pid(pid, run_as_root=False):
    """
    Get the lowest child pid in the process hierarchy

    If root helper was used, two or more processes would be created:

     - a root helper process (e.g. sudo myscript)
     - possibly a rootwrap script (e.g. neutron-rootwrap)
     - a child process (e.g. myscript)

    Killing the root helper process will leave the child process
    running, re-parented to init, so the only way to ensure that both
    die is to target the child process directly.
    """
    pid = str(pid)
    if run_as_root:
        try:
            pid = find_child_pids(pid)[0]
        except IndexError:
            # Process is already dead
            return None
        while True:
            try:
                # We shouldn't have more than one child per process
                # so keep getting the children of the first one
                pid = find_child_pids(pid)[0]
            except IndexError:
                # Last process in the tree, return it
                break
    return pid

def remove_abs_path(cmd):
    """Remove absolute path of executable in cmd

    Note: New instance of list is returned

    :param cmd: parsed shlex command (e.g. ['/bin/foo', 'param1', 'param two'])

    """
    if cmd and os.path.isabs(cmd[0]):
        cmd = list(cmd)
        cmd[0] = os.path.basename(cmd[0])

    return cmd

def get_cmdline_from_pid(pid):
    if pid is None or not os.path.exists('/proc/%s' % pid):
        return []
    with open('/proc/%s/cmdline' % pid, 'r') as f:
        return f.readline().split('\0')[:-1]

def cmd_matches_expected(cmd, expected_cmd):
    abs_cmd = remove_abs_path(cmd)
    abs_expected_cmd = remove_abs_path(expected_cmd)
    if abs_cmd != abs_expected_cmd:
        # Commands executed with #! are prefixed with the script
        # executable. Check for the expected cmd being a subset of the
        # actual cmd to cover this possibility.
        abs_cmd = remove_abs_path(abs_cmd[1:])
    return abs_cmd == abs_expected_cmd

def pid_invoked_with_cmdline(pid, expected_cmd):
    """Validate process with given pid is running with provided parameters

    """
    cmd = get_cmdline_from_pid(pid)
    return cmd_matches_expected(cmd, expected_cmd)

def wait_until_true(predicate, timeout=60, sleep=1, exception=None):
    """
    Wait until callable predicate is evaluated as True

    :param predicate: Callable deciding whether waiting should continue.
    Best practice is to instantiate predicate with functools.partial()
    :param timeout: Timeout in seconds how long should function wait.
    :param sleep: Polling interval for results in seconds.
    :param exception: Exception class for eventlet.Timeout.
    (see doc for eventlet.Timeout for more information)
    """
    with eventlet.timeout.Timeout(timeout, exception):
        while not predicate():
            eventlet.sleep(sleep)

def ensure_directory_exists_without_file(path):
    dirname = os.path.dirname(path)
    if os.path.isdir(dirname):
        try:
            os.unlink(path)
        except OSError:
            with excutils.save_and_reraise_exception() as ctxt:
                if not os.path.exists(path):
                    ctxt.reraise = False
    else:
        utils.ensure_dir(dirname)

def is_effective_user(user_id_or_name):
    """Returns True if user_id_or_name is effective user (id/name)."""
    euid = os.geteuid()
    if str(user_id_or_name) == str(euid):
        return True
    effective_user_name = pwd.getpwuid(euid).pw_name
    return user_id_or_name == effective_user_name

def is_effective_group(group_id_or_name):
    """Returns True if group_id_or_name is effective group (id/name)."""
    egid = os.getegid()
    if str(group_id_or_name) == str(egid):
        return True
    effective_group_name = grp.getgrgid(egid).gr_name
    return group_id_or_name == effective_group_name

class UnixDomainHTTPConnection(httplib.HTTPConnection):
    """Connection class for HTTP over UNIX domain socket."""
    def __init__(self, host, port=None, strict=None, timeout=None,
                 proxy_info=None):
        httplib.HTTPConnection.__init__(self, host, port, strict)
        self.timeout = timeout
        self.socket_path = cfg.CONF.metadata_proxy_socket

    def connect(self):
        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        if self.timeout:
            self.sock.settimeout(self.timeout)
        self.sock.connect(self.socket_path)

class UnixDomainHttpProtocol(eventlet.wsgi.HttpProtocol):
    def __init__(self, request, client_address, server):
        if client_address == '':
            client_address = ('<local>', 0)
        # base class is old-style, so super does not work properly
        eventlet.wsgi.HttpProtocol.__init__(self, request, client_address,
                                            server)

class UnixDomainWSGIServer(wsgi.Server):
    def __init__(self, name):
        self._socket = None
        self._launcher = None
        self._server = None
        super(UnixDomainWSGIServer, self).__init__(name)

    def start(self, application, file_socket, workers, backlog, mode=None):
        self._socket = eventlet.listen(file_socket,
                                       family=socket.AF_UNIX,
                                       backlog=backlog)
        if mode is not None:
            os.chmod(file_socket, mode)

        self._launch(application, workers=workers)

    def _run(self, application, socket):
        """Start a WSGI service in a new green thread."""
        logger = logging.getLogger('eventlet.wsgi.server')
        eventlet.wsgi.server(socket,
                             application,
                             max_size=self.num_threads,
                             protocol=UnixDomainHttpProtocol,
                             log=loggers.WritableLogger(logger))
<commit_msg>Stop logging STDOUT and STDERR on every shell out<commit_after># Copyright 2012 Locaweb.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import fcntl
import glob
import grp
import os
import pwd
import shlex
import socket
import struct
import tempfile
import threading

from debtcollector import removals
import eventlet
from eventlet.green import subprocess
from eventlet import greenthread
from oslo_config import cfg
from oslo_log import log as logging
from oslo_log import loggers
from oslo_rootwrap import client
from oslo_utils import excutils
import six
from six.moves import http_client as httplib

from neutron.agent.common import config
from neutron.common import constants
from neutron.common import utils
from neutron.i18n import _LE
from neutron import wsgi

LOG = logging.getLogger(__name__)
config.register_root_helper(cfg.CONF)

class RootwrapDaemonHelper(object):
    __client = None
    __lock = threading.Lock()

    def __new__(cls):
        """There is no reason to instantiate this class"""
        raise NotImplementedError()

    @classmethod
    def get_client(cls):
        with cls.__lock:
            if cls.__client is None:
                cls.__client = client.Client(
                    shlex.split(cfg.CONF.AGENT.root_helper_daemon))
            return cls.__client

def addl_env_args(addl_env):
    """Build arugments for adding additional environment vars with env"""

    # NOTE (twilson) If using rootwrap, an EnvFilter should be set up for the
    # command instead of a CommandFilter.
    if addl_env is None:
        return []
    return ['env'] + ['%s=%s' % pair for pair in addl_env.items()]

def create_process(cmd, run_as_root=False, addl_env=None):
    """Create a process object for the given command.

    The return value will be a tuple of the process object and the
    list of command arguments used to create it.
    """
    cmd = list(map(str, addl_env_args(addl_env) + cmd))
    if run_as_root:
        cmd = shlex.split(config.get_root_helper(cfg.CONF)) + cmd
    LOG.debug("Running command: %s", cmd)
    obj = utils.subprocess_popen(cmd, shell=False,
                                 stdin=subprocess.PIPE,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)

    return obj, cmd

def execute_rootwrap_daemon(cmd, process_input, addl_env):
    cmd = list(map(str, addl_env_args(addl_env) + cmd))
    # NOTE(twilson) oslo_rootwrap.daemon will raise on filter match
    # errors, whereas oslo_rootwrap.cmd converts them to return codes.
    # In practice, no neutron code should be trying to execute something that
    # would throw those errors, and if it does it should be fixed as opposed to
    # just logging the execution error.
    LOG.debug("Running command (rootwrap daemon): %s", cmd)
    client = RootwrapDaemonHelper.get_client()
    return client.execute(cmd, process_input)

def execute(cmd, process_input=None, addl_env=None,
            check_exit_code=True, return_stderr=False, log_fail_as_error=True,
            extra_ok_codes=None, run_as_root=False):
    try:
        if (process_input is None or
            isinstance(process_input, six.binary_type)):
            _process_input = process_input
        else:
            _process_input = process_input.encode('utf-8')
        if run_as_root and cfg.CONF.AGENT.root_helper_daemon:
            returncode, _stdout, _stderr = (
                execute_rootwrap_daemon(cmd, process_input, addl_env))
        else:
            obj, cmd = create_process(cmd, run_as_root=run_as_root,
                                      addl_env=addl_env)
            _stdout, _stderr = obj.communicate(_process_input)
            returncode = obj.returncode
            obj.stdin.close()
        if six.PY3:
            if isinstance(_stdout, bytes):
                try:
                    _stdout = _stdout.decode(encoding='utf-8')
                except UnicodeError:
                    pass
            if isinstance(_stderr, bytes):
                try:
                    _stderr = _stderr.decode(encoding='utf-8')
                except UnicodeError:
                    pass

        m = _("\nCommand: {cmd}\nExit code: {code}\n").format(
                  cmd=cmd,
                  code=returncode)

        extra_ok_codes = extra_ok_codes or []
        if returncode and returncode in extra_ok_codes:
            returncode = None

        if returncode and log_fail_as_error:
            m += ("Stdin: {stdin}\n"
                  "Stdout: {stdout}\nStderr: {stderr}").format(
                stdin=process_input or '',
                stdout=_stdout,
                stderr=_stderr)
            LOG.error(m)
        else:
            LOG.debug(m)

        if returncode and check_exit_code:
            raise RuntimeError(m)
    finally:
        # NOTE(termie): this appears to be necessary to let the subprocess
        #               call clean something up in between calls, without
        #               it two execute calls in a row hangs the second one
        greenthread.sleep(0)

    return (_stdout, _stderr) if return_stderr else _stdout

def get_interface_mac(interface):
    MAC_START = 18
    MAC_END = 24
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    dev = interface[:constants.DEVICE_NAME_MAX_LEN]
    if isinstance(dev, six.text_type):
        dev = dev.encode('utf-8')
    info = fcntl.ioctl(s.fileno(), 0x8927, struct.pack('256s', dev))
    return ''.join(['%02x:' % ord(char)
                    for char in info[MAC_START:MAC_END]])[:-1]

def replace_file(file_name, data):
    """Replaces the contents of file_name with data in a safe manner.

    First write to a temp file and then rename. Since POSIX renames are
    atomic, the file is unlikely to be corrupted by competing writes.

    We create the tempfile on the same device to ensure that it can be renamed.
    """

    base_dir = os.path.dirname(os.path.abspath(file_name))
    tmp_file = tempfile.NamedTemporaryFile('w+', dir=base_dir, delete=False)
    tmp_file.write(data)
    tmp_file.close()
    os.chmod(tmp_file.name, 0o644)
    os.rename(tmp_file.name, file_name)

def find_child_pids(pid):
    """Retrieve a list of the pids of child processes of the given pid."""

    try:
        raw_pids = execute(['ps', '--ppid', pid, '-o', 'pid='],
                           log_fail_as_error=False)
    except RuntimeError as e:
        # Unexpected errors are the responsibility of the caller
        with excutils.save_and_reraise_exception() as ctxt:
            # Exception has already been logged by execute
            no_children_found = 'Exit code: 1' in str(e)
            if no_children_found:
                ctxt.reraise = False
                return []
    return [x.strip() for x in raw_pids.split('\n') if x.strip()]

@removals.remove(message='Use neutron.common.utils.ensure_dir instead.')
def ensure_dir(*args, **kwargs):
    return utils.ensure_dir(*args, **kwargs)

def _get_conf_base(cfg_root, uuid, ensure_conf_dir):
    #TODO(mangelajo): separate responsibilities here, ensure_conf_dir
    #                 should be a separate function
    conf_dir = os.path.abspath(os.path.normpath(cfg_root))
    conf_base = os.path.join(conf_dir, uuid)
    if ensure_conf_dir:
        utils.ensure_dir(conf_dir)
    return conf_base

def get_conf_file_name(cfg_root, uuid, cfg_file, ensure_conf_dir=False):
    """Returns the file name for a given kind of config file."""
    conf_base = _get_conf_base(cfg_root, uuid, ensure_conf_dir)
    return "%s.%s" % (conf_base, cfg_file)

def get_value_from_file(filename, converter=None):

    try:
        with open(filename, 'r') as f:
            try:
                return converter(f.read()) if converter else f.read()
            except ValueError:
                LOG.error(_LE('Unable to convert value in %s'), filename)
    except IOError:
        LOG.debug('Unable to access %s', filename)

def get_value_from_conf_file(cfg_root, uuid, cfg_file, converter=None):
    """A helper function to read a value from one of a config file."""
    file_name = get_conf_file_name(cfg_root, uuid, cfg_file)
    return get_value_from_file(file_name, converter)

def remove_conf_files(cfg_root, uuid):
    conf_base = _get_conf_base(cfg_root, uuid, False)
    for file_path in glob.iglob("%s.*" % conf_base):
        os.unlink(file_path)

def get_root_helper_child_pid(pid, run_as_root=False):
    """
    Get the lowest child pid in the process hierarchy

    If root helper was used, two or more processes would be created:

     - a root helper process (e.g. sudo myscript)
     - possibly a rootwrap script (e.g. neutron-rootwrap)
     - a child process (e.g. myscript)

    Killing the root helper process will leave the child process
    running, re-parented to init, so the only way to ensure that both
    die is to target the child process directly.
    """
    pid = str(pid)
    if run_as_root:
        try:
            pid = find_child_pids(pid)[0]
        except IndexError:
            # Process is already dead
            return None
        while True:
            try:
                # We shouldn't have more than one child per process
                # so keep getting the children of the first one
                pid = find_child_pids(pid)[0]
            except IndexError:
                # Last process in the tree, return it
                break
    return pid

def remove_abs_path(cmd):
    """Remove absolute path of executable in cmd

    Note: New instance of list is returned

    :param cmd: parsed shlex command (e.g. ['/bin/foo', 'param1', 'param two'])

    """
    if cmd and os.path.isabs(cmd[0]):
        cmd = list(cmd)
        cmd[0] = os.path.basename(cmd[0])

    return cmd

def get_cmdline_from_pid(pid):
    if pid is None or not os.path.exists('/proc/%s' % pid):
        return []
    with open('/proc/%s/cmdline' % pid, 'r') as f:
        return f.readline().split('\0')[:-1]

def cmd_matches_expected(cmd, expected_cmd):
    abs_cmd = remove_abs_path(cmd)
    abs_expected_cmd = remove_abs_path(expected_cmd)
    if abs_cmd != abs_expected_cmd:
        # Commands executed with #! are prefixed with the script
        # executable. Check for the expected cmd being a subset of the
        # actual cmd to cover this possibility.
        abs_cmd = remove_abs_path(abs_cmd[1:])
    return abs_cmd == abs_expected_cmd

def pid_invoked_with_cmdline(pid, expected_cmd):
    """Validate process with given pid is running with provided parameters

    """
    cmd = get_cmdline_from_pid(pid)
    return cmd_matches_expected(cmd, expected_cmd)

def wait_until_true(predicate, timeout=60, sleep=1, exception=None):
    """
    Wait until callable predicate is evaluated as True

    :param predicate: Callable deciding whether waiting should continue.
    Best practice is to instantiate predicate with functools.partial()
    :param timeout: Timeout in seconds how long should function wait.
    :param sleep: Polling interval for results in seconds.
    :param exception: Exception class for eventlet.Timeout.
    (see doc for eventlet.Timeout for more information)
    """
    with eventlet.timeout.Timeout(timeout, exception):
        while not predicate():
            eventlet.sleep(sleep)

def ensure_directory_exists_without_file(path):
    dirname = os.path.dirname(path)
    if os.path.isdir(dirname):
        try:
            os.unlink(path)
        except OSError:
            with excutils.save_and_reraise_exception() as ctxt:
                if not os.path.exists(path):
                    ctxt.reraise = False
    else:
        utils.ensure_dir(dirname)

def is_effective_user(user_id_or_name):
    """Returns True if user_id_or_name is effective user (id/name)."""
    euid = os.geteuid()
    if str(user_id_or_name) == str(euid):
        return True
    effective_user_name = pwd.getpwuid(euid).pw_name
    return user_id_or_name == effective_user_name

def is_effective_group(group_id_or_name):
    """Returns True if group_id_or_name is effective group (id/name)."""
    egid = os.getegid()
    if str(group_id_or_name) == str(egid):
        return True
    effective_group_name = grp.getgrgid(egid).gr_name
    return group_id_or_name == effective_group_name

class UnixDomainHTTPConnection(httplib.HTTPConnection):
    """Connection class for HTTP over UNIX domain socket."""
    def __init__(self, host, port=None, strict=None, timeout=None,
                 proxy_info=None):
        httplib.HTTPConnection.__init__(self, host, port, strict)
        self.timeout = timeout
        self.socket_path = cfg.CONF.metadata_proxy_socket

    def connect(self):
        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        if self.timeout:
            self.sock.settimeout(self.timeout)
        self.sock.connect(self.socket_path)

class UnixDomainHttpProtocol(eventlet.wsgi.HttpProtocol):
    def __init__(self, request, client_address, server):
        if client_address == '':
            client_address = ('<local>', 0)
        # base class is old-style, so super does not work properly
        eventlet.wsgi.HttpProtocol.__init__(self, request, client_address,
                                            server)

class UnixDomainWSGIServer(wsgi.Server):
    def __init__(self, name):
        self._socket = None
        self._launcher = None
        self._server = None
        super(UnixDomainWSGIServer, self).__init__(name)

    def start(self, application, file_socket, workers, backlog, mode=None):
        self._socket = eventlet.listen(file_socket,
                                       family=socket.AF_UNIX,
                                       backlog=backlog)
        if mode is not None:
            os.chmod(file_socket, mode)

        self._launch(application, workers=workers)

    def _run(self, application, socket):
        """Start a WSGI service in a new green thread."""
        logger = logging.getLogger('eventlet.wsgi.server')
        eventlet.wsgi.server(socket,
                             application,
                             max_size=self.num_threads,
                             protocol=UnixDomainHttpProtocol,
                             log=loggers.WritableLogger(logger))
<|endoftext|>


<commit_before>"""
Type checking primitives. Traits is an overloaded term. Came up with Typelets
ala IPython Traitlets.
"""
from collections import deque, OrderedDict
import inspect
import uuid
import inspect
import datetime
import numbers

from ..class_util import _get_name

class TypeletError(Exception):
    pass

class KeyTypeError(TypeletError):
    pass

def _gather_typelets(dct, key='_earthdragon_typelets'):
    if key in dct:
        return dct[key]

    typelets = OrderedDict()
    for k, v in dct.items():
        if isinstance(v, Typelet):
            typelets[k] = v
    return typelets

def gather_typelets(dct, bases=[]):
    class_dicts = [base.__dict__ for base in bases] + [dct]
    typelet_dicts = map(_gather_typelets, class_dicts)

    typelets = OrderedDict()
    for tdict in typelet_dicts:
        typelets.update(tdict)
    return typelets

def typelet_repr(self, typelets=None):
    """
    Repr utility that will default to printing the
    class and its typelets
    """
    class_ = self.__class__
    class_name = class_.__name__

    attrs = typelets or []
    if hasattr(class_, '_earthdragon_typelets'):
        attrs = getattr(class_, '_earthdragon_typelets')
    if hasattr(class_, '__repr_attrs__'):
        attrs = getattr(class_, '__repr_attrs__')
    return _typelet_repr(self, attrs)

def _typelet_repr(self, attrs):
    class_ = self.__class__
    class_name = class_.__name__

    bits = []
    for k in attrs:
        v = getattr(self, k)
        bits.append("{k}={v}".format(k=k, v=v))
    attr_string = ', '.join(bits)
    return "{class_name}({attr_string})".format(**locals())

class Typelet:
    def __init__(self, **kwargs):
        self.value = None
        self.name = None
        self.unique = kwargs.pop('unique', False)

        default = None
        if 'default' in kwargs:
            default = kwargs.pop('default')
            self.validate(default)
        self.default = default

    def __get__(self, obj, cls=None):
        if obj is None:
            obj = cls
        name = self.get_name(obj)
        return obj.__dict__.get(name, self.default)

    def __set__(self, obj, value):
        new_value = self._validate(obj, value)
        name = self.get_name(obj)
        obj.__dict__[name] = new_value

    def get_name(self, obj):
        if self.name is None:
            self.name = _get_name(obj, self)
        return self.name

    def _validate(self, obj, value):
        try:
            value = self.validate(value)
        except TypeletError as err:
            raise err # pass through typelet errors we handle
        except Exception as err:
            self.error(value, obj)

        return value

    def validate(self, value):
        return value

    def error(self, value, obj=None):
        class_name = type(self).__name__
        value_name = type(value).__name__
        msg = "TypeletError: Required {class_name}. Got {value_name}".format(**locals())
        raise TypeletError(msg)

class Unicode(Typelet):
    def validate(self, value):
        if isinstance(value, str):
            return value
        self.error(value)

class Int(Typelet):
    def __init__(self, min=None, max=None, **kwargs):
        self.min = min
        self.max = max
        super().__init__(**kwargs)

    def check_min(self, value):
        if self.min is None:
            return True
        return self.min <= value

    def check_max(self, value):
        if self.max is None:
            return True
        return self.max >= value

    def validate(self, value):
        if isinstance(value, int) and self.check_min(value)\
            and self.check_max(value):
            return value
        self.error(value)

class Float(Typelet):
    def validate(self, value):
        if isinstance(value, numbers.Number):
            return value
        self.error(value)

class Bool(Typelet):
    def validate(self, value):
        if isinstance(value, bool):
            return value
        self.error(value)

class UUID(Typelet):
    def validate(self, value):
        if isinstance(value, uuid.UUID):
            return value
        self.error(value)

class UUID(Typelet):
    def validate(self, value):
        if isinstance(value, uuid.UUID):
            return value
        self.error(value)

class Datetime(Typelet):
    def validate(self, value):
        if isinstance(value, datetime.datetime):
            return value
        self.error(value)

class Type(Typelet):
    def __init__(self, _class, **kwargs):
        self.check_class = _class
        super().__init__(**kwargs)

    def validate(self, value):
        if isinstance(value, self.check_class):
            return value
        self.error(value)

def grab_class_reference(obj, class_name):
    """ Grab class ref from the module that object is defined in """
    modname = obj.__class__.__module__
    import sys
    mod = sys.modules[modname]
    check_class = getattr(mod, class_name, None)
    return check_class

class Collection(Typelet):

    # TODO replace list with a type checking list
    container_class = list

    def __init__(self, _class, **kwargs):
        self._class = _class
        self.check_class = None
        if inspect.isclass(_class):
            self.check_class = _class
        super().__init__(**kwargs)

    def _check_container_class(self, value):
        if not isinstance(value, self.container_class):
            raise TypeletError("Must be a {container_class}".format(
                container_class=str(self.container_class)
            ))

    def _check_collection_values(self, values):
        check_class = self.check_class
        for v in values:
            if not isinstance(v, check_class):
                raise TypeletError("Collection can only contain {type}")

    def values(self, obj):
        return obj

    def _validate(self, obj, value):
        if self.check_class is None:
            self.check_class = grab_class_reference(obj, self._class)
        return super()._validate(obj, value)

    def validate(self, value):
        self._check_container_class(value)
        self._check_collection_values(self.values(value))
        return value

class List(Collection):
    container_class = list

class Deque(Collection):
    container_class = deque

class Dict(Collection):
    container_class = dict

    def __init__(self, _class, **kwargs):
        self.key_class = kwargs.pop('key_class', None)
        super().__init__(_class, **kwargs)

    def values(self, obj):
        return obj.values()

    def _validate_keys(self, value):
        if self.key_class is None:
            return
        for k in value:
            if not isinstance(k, self.key_class):
                raise KeyTypeError("Keys must be {type} type".format(
                    type=str(self.key_class)
                ))

    def validate(self, value):
        value = super().validate(value)
        self._validate_keys(value)
        return value
<commit_msg>Remove dupe class def<commit_after>"""
Type checking primitives. Traits is an overloaded term. Came up with Typelets
ala IPython Traitlets.
"""
from collections import deque, OrderedDict
import inspect
import uuid
import inspect
import datetime
import numbers

from ..class_util import _get_name

class TypeletError(Exception):
    pass

class KeyTypeError(TypeletError):
    pass

def _gather_typelets(dct, key='_earthdragon_typelets'):
    if key in dct:
        return dct[key]

    typelets = OrderedDict()
    for k, v in dct.items():
        if isinstance(v, Typelet):
            typelets[k] = v
    return typelets

def gather_typelets(dct, bases=[]):
    class_dicts = [base.__dict__ for base in bases] + [dct]
    typelet_dicts = map(_gather_typelets, class_dicts)

    typelets = OrderedDict()
    for tdict in typelet_dicts:
        typelets.update(tdict)
    return typelets

def typelet_repr(self, typelets=None):
    """
    Repr utility that will default to printing the
    class and its typelets
    """
    class_ = self.__class__
    class_name = class_.__name__

    attrs = typelets or []
    if hasattr(class_, '_earthdragon_typelets'):
        attrs = getattr(class_, '_earthdragon_typelets')
    if hasattr(class_, '__repr_attrs__'):
        attrs = getattr(class_, '__repr_attrs__')
    return _typelet_repr(self, attrs)

def _typelet_repr(self, attrs):
    class_ = self.__class__
    class_name = class_.__name__

    bits = []
    for k in attrs:
        v = getattr(self, k)
        bits.append("{k}={v}".format(k=k, v=v))
    attr_string = ', '.join(bits)
    return "{class_name}({attr_string})".format(**locals())

class Typelet:
    def __init__(self, **kwargs):
        self.value = None
        self.name = None
        self.unique = kwargs.pop('unique', False)

        default = None
        if 'default' in kwargs:
            default = kwargs.pop('default')
            self.validate(default)
        self.default = default

    def __get__(self, obj, cls=None):
        if obj is None:
            obj = cls
        name = self.get_name(obj)
        return obj.__dict__.get(name, self.default)

    def __set__(self, obj, value):
        new_value = self._validate(obj, value)
        name = self.get_name(obj)
        obj.__dict__[name] = new_value

    def get_name(self, obj):
        if self.name is None:
            self.name = _get_name(obj, self)
        return self.name

    def _validate(self, obj, value):
        try:
            value = self.validate(value)
        except TypeletError as err:
            raise err # pass through typelet errors we handle
        except Exception as err:
            self.error(value, obj)

        return value

    def validate(self, value):
        return value

    def error(self, value, obj=None):
        class_name = type(self).__name__
        value_name = type(value).__name__
        msg = "TypeletError: Required {class_name}. Got {value_name}".format(**locals())
        raise TypeletError(msg)

class Unicode(Typelet):
    def validate(self, value):
        if isinstance(value, str):
            return value
        self.error(value)

class Int(Typelet):
    def __init__(self, min=None, max=None, **kwargs):
        self.min = min
        self.max = max
        super().__init__(**kwargs)

    def check_min(self, value):
        if self.min is None:
            return True
        return self.min <= value

    def check_max(self, value):
        if self.max is None:
            return True
        return self.max >= value

    def validate(self, value):
        if isinstance(value, int) and self.check_min(value)\
            and self.check_max(value):
            return value
        self.error(value)

class Float(Typelet):
    def validate(self, value):
        if isinstance(value, numbers.Number):
            return value
        self.error(value)

class Bool(Typelet):
    def validate(self, value):
        if isinstance(value, bool):
            return value
        self.error(value)

class UUID(Typelet):
    def validate(self, value):
        if isinstance(value, uuid.UUID):
            return value
        self.error(value)

class Datetime(Typelet):
    def validate(self, value):
        if isinstance(value, datetime.datetime):
            return value
        self.error(value)

class Type(Typelet):
    def __init__(self, _class, **kwargs):
        self.check_class = _class
        super().__init__(**kwargs)

    def validate(self, value):
        if isinstance(value, self.check_class):
            return value
        self.error(value)

def grab_class_reference(obj, class_name):
    """ Grab class ref from the module that object is defined in """
    modname = obj.__class__.__module__
    import sys
    mod = sys.modules[modname]
    check_class = getattr(mod, class_name, None)
    return check_class

class Collection(Typelet):

    # TODO replace list with a type checking list
    container_class = list

    def __init__(self, _class, **kwargs):
        self._class = _class
        self.check_class = None
        if inspect.isclass(_class):
            self.check_class = _class
        super().__init__(**kwargs)

    def _check_container_class(self, value):
        if not isinstance(value, self.container_class):
            raise TypeletError("Must be a {container_class}".format(
                container_class=str(self.container_class)
            ))

    def _check_collection_values(self, values):
        check_class = self.check_class
        for v in values:
            if not isinstance(v, check_class):
                raise TypeletError("Collection can only contain {type}")

    def values(self, obj):
        return obj

    def _validate(self, obj, value):
        if self.check_class is None:
            self.check_class = grab_class_reference(obj, self._class)
        return super()._validate(obj, value)

    def validate(self, value):
        self._check_container_class(value)
        self._check_collection_values(self.values(value))
        return value

class List(Collection):
    container_class = list

class Deque(Collection):
    container_class = deque

class Dict(Collection):
    container_class = dict

    def __init__(self, _class, **kwargs):
        self.key_class = kwargs.pop('key_class', None)
        super().__init__(_class, **kwargs)

    def values(self, obj):
        return obj.values()

    def _validate_keys(self, value):
        if self.key_class is None:
            return
        for k in value:
            if not isinstance(k, self.key_class):
                raise KeyTypeError("Keys must be {type} type".format(
                    type=str(self.key_class)
                ))

    def validate(self, value):
        value = super().validate(value)
        self._validate_keys(value)
        return value
<|endoftext|>