<commit_before>import sys, os, time, tempfile, shutil, socket, fcntl
from subprocess import Popen, PIPE

try:
    socket.getaddrinfo('non-existing-host', 80)
    NON_EXISTING_RESOLVABLE = True
except socket.gaierror:
    NON_EXISTING_RESOLVABLE = False

def get_testenv():
    env = os.environ.copy()
    env['PYTHONPATH'] = os.getcwd()
    return env

def get_ephemeral_port():
    s = socket.socket()
    s.bind(("", 0))
    return s.getsockname()[1]

def _non_block_read(output):
    fd = output.fileno()
    fl = fcntl.fcntl(fd, fcntl.F_GETFL)
    fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
    try:
        return output.read()
    except Exception:
        return ""

def _wait_for_port(portnum, delay=0.1, attempts=30):
    while attempts > 0:
        s = socket.socket()
        if s.connect_ex(('127.0.0.1', portnum)) == 0:
            s.close()
            return
        time.sleep(delay)
        attempts -= 1
    raise RuntimeError("Port %d is not open" % portnum)

class SplashServer(object):

    def __init__(self, logfile=None, proxy_profiles_path=None,
                 js_profiles_path=None, portnum=None,
                 proxy_portnum=None, extra_args=None):
        self.logfile = logfile
        self.proxy_profiles_path = proxy_profiles_path
        self.js_profiles_path = js_profiles_path
        self.portnum = portnum if portnum is not None else get_ephemeral_port()
        self.proxy_portnum = proxy_portnum if proxy_portnum is not None else get_ephemeral_port()
        self.tempdir = tempfile.mkdtemp()
        self.extra_args = extra_args or []

    def __enter__(self):
        args = [sys.executable, '-u', '-m', 'splash.server']
        args += ['--cache-path', self.tempdir]
        args += ['--port', str(self.portnum)]
        if self.logfile:
            args += ['-f', self.logfile]
        if self.proxy_profiles_path:
            args += ['--proxy-profiles-path', self.proxy_profiles_path]
        if self.js_profiles_path:
            args += ['--js-profiles-path', self.js_profiles_path]
        if self.proxy_portnum:
            args += ['--proxy-portnum', str(self.proxy_portnum)]

        args.extend(self.extra_args)

        self.proc = Popen(args, stderr=PIPE, env=get_testenv())
        self.proc.poll()
        if self.proc.returncode:
            msg = "unable to start splash server. error code: %d - stderr follows: \n%s" % \
                (self.proc.returncode, self.proc.stderr.read())
            raise RuntimeError(msg)

        try:
            _wait_for_port(self.portnum)
        finally:
            print(_non_block_read(self.proc.stderr))

        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.proc.kill()
        self.proc.wait()
        time.sleep(0.2)
        shutil.rmtree(self.tempdir)

    def url(self, path):
        return "http://localhost:%s/%s" % (self.portnum, path.lstrip('/'))

    def proxy_url(self, path):
        return "http://localhost:%s/%s" % (self.proxy_portnum, path.lstrip('/'))

class MockServer(object):

    def __init__(self, http_port=None, https_port=None, proxy_port=None):
        self.http_port = http_port if http_port is not None else get_ephemeral_port()
        self.https_port = https_port if https_port is not None else get_ephemeral_port()
        self.proxy_port = proxy_port if proxy_port is not None else get_ephemeral_port()

    def __enter__(self):
        self.proc = Popen([
                sys.executable,
                '-u', '-m', 'splash.tests.mockserver',
                '--http-port', str(self.http_port),
                '--https-port', str(self.https_port),
                '--proxy-port', str(self.proxy_port),
            ],
            stdout=PIPE,
            env=get_testenv()
        )
        for port in (self.http_port, self.https_port, self.proxy_port):
            _wait_for_port(port)
        print(_non_block_read(self.proc.stdout))

    def __exit__(self, exc_type, exc_value, traceback):
        self.proc.kill()
        self.proc.wait()
        time.sleep(0.2)

    def url(self, path):
        return "http://localhost:%s/%s" % (self.http_port, path.lstrip('/'))

    def https_url(self, path):
        return "https://localhost:%s/%s" % (self.https_port, path.lstrip('/'))

class TestServers(object):

    def __init__(self, logfile=None):
        self.logfile = logfile
        self.tmp_folder = tempfile.mkdtemp("splash-tests-tmp")
        self.proxy_profiles_path = self._copy_test_folder('proxy_profiles')
        self.js_profiles_path = self._copy_test_folder('js_profiles')

        self.mock_http_port = get_ephemeral_port()
        self.mock_https_port = get_ephemeral_port()
        self.mock_proxy_port = get_ephemeral_port()

        print("TestServers mock ports: %s http, %s https, %s proxy" % (
            self.mock_http_port, self.mock_https_port, self.mock_proxy_port))

        self._fix_testproxy_port()

    def _copy_test_folder(self, src, dst=None):
        src_path = os.path.join(os.path.dirname(__file__), src)
        dst_path = os.path.join(self.tmp_folder, dst or src)
        shutil.copytree(src_path, dst_path)
        return dst_path

    def _fix_testproxy_port(self):
        filename = os.path.join(self.proxy_profiles_path, 'test.ini')
        with open(filename, 'rb') as f:
            data = f.read()
        data = data.replace('8990', str(self.mock_proxy_port))
        with open(filename, 'wb') as f:
            f.write(data)

    def __enter__(self):
        self.mockserver = MockServer(
            self.mock_http_port,
            self.mock_https_port,
            self.mock_proxy_port,
        )
        self.mockserver.__enter__()
        self.splashserver = SplashServer(self.logfile, self.proxy_profiles_path,
                                         self.js_profiles_path)
        self.splashserver.__enter__()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.splashserver.__exit__(None, None, None)
        self.mockserver.__exit__(None, None, None)
        shutil.rmtree(self.tmp_folder)

    def print_output(self):
        print(_non_block_read(self.splashserver.proc.stderr))
        print(_non_block_read(self.mockserver.proc.stdout))

<commit_msg>TST increase default port waiting timeout to 10s<commit_after>import sys, os, time, tempfile, shutil, socket, fcntl
from subprocess import Popen, PIPE

try:
    socket.getaddrinfo('non-existing-host', 80)
    NON_EXISTING_RESOLVABLE = True
except socket.gaierror:
    NON_EXISTING_RESOLVABLE = False

def get_testenv():
    env = os.environ.copy()
    env['PYTHONPATH'] = os.getcwd()
    return env

def get_ephemeral_port():
    s = socket.socket()
    s.bind(("", 0))
    return s.getsockname()[1]

def _non_block_read(output):
    fd = output.fileno()
    fl = fcntl.fcntl(fd, fcntl.F_GETFL)
    fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
    try:
        return output.read()
    except Exception:
        return ""

def _wait_for_port(portnum, delay=0.1, attempts=100):
    while attempts > 0:
        s = socket.socket()
        if s.connect_ex(('127.0.0.1', portnum)) == 0:
            s.close()
            return
        time.sleep(delay)
        attempts -= 1
    raise RuntimeError("Port %d is not open" % portnum)

class SplashServer(object):

    def __init__(self, logfile=None, proxy_profiles_path=None,
                 js_profiles_path=None, portnum=None,
                 proxy_portnum=None, extra_args=None):
        self.logfile = logfile
        self.proxy_profiles_path = proxy_profiles_path
        self.js_profiles_path = js_profiles_path
        self.portnum = portnum if portnum is not None else get_ephemeral_port()
        self.proxy_portnum = proxy_portnum if proxy_portnum is not None else get_ephemeral_port()
        self.tempdir = tempfile.mkdtemp()
        self.extra_args = extra_args or []

    def __enter__(self):
        args = [sys.executable, '-u', '-m', 'splash.server']
        args += ['--cache-path', self.tempdir]
        args += ['--port', str(self.portnum)]
        if self.logfile:
            args += ['-f', self.logfile]
        if self.proxy_profiles_path:
            args += ['--proxy-profiles-path', self.proxy_profiles_path]
        if self.js_profiles_path:
            args += ['--js-profiles-path', self.js_profiles_path]
        if self.proxy_portnum:
            args += ['--proxy-portnum', str(self.proxy_portnum)]

        args.extend(self.extra_args)

        self.proc = Popen(args, stderr=PIPE, env=get_testenv())
        self.proc.poll()
        if self.proc.returncode:
            msg = "unable to start splash server. error code: %d - stderr follows: \n%s" % \
                (self.proc.returncode, self.proc.stderr.read())
            raise RuntimeError(msg)

        try:
            _wait_for_port(self.portnum)
        finally:
            print(_non_block_read(self.proc.stderr))

        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.proc.kill()
        self.proc.wait()
        time.sleep(0.2)
        shutil.rmtree(self.tempdir)

    def url(self, path):
        return "http://localhost:%s/%s" % (self.portnum, path.lstrip('/'))

    def proxy_url(self, path):
        return "http://localhost:%s/%s" % (self.proxy_portnum, path.lstrip('/'))

class MockServer(object):

    def __init__(self, http_port=None, https_port=None, proxy_port=None):
        self.http_port = http_port if http_port is not None else get_ephemeral_port()
        self.https_port = https_port if https_port is not None else get_ephemeral_port()
        self.proxy_port = proxy_port if proxy_port is not None else get_ephemeral_port()

    def __enter__(self):
        self.proc = Popen([
                sys.executable,
                '-u', '-m', 'splash.tests.mockserver',
                '--http-port', str(self.http_port),
                '--https-port', str(self.https_port),
                '--proxy-port', str(self.proxy_port),
            ],
            stdout=PIPE,
            env=get_testenv()
        )
        for port in (self.http_port, self.https_port, self.proxy_port):
            _wait_for_port(port)
        print(_non_block_read(self.proc.stdout))

    def __exit__(self, exc_type, exc_value, traceback):
        self.proc.kill()
        self.proc.wait()
        time.sleep(0.2)

    def url(self, path):
        return "http://localhost:%s/%s" % (self.http_port, path.lstrip('/'))

    def https_url(self, path):
        return "https://localhost:%s/%s" % (self.https_port, path.lstrip('/'))

class TestServers(object):

    def __init__(self, logfile=None):
        self.logfile = logfile
        self.tmp_folder = tempfile.mkdtemp("splash-tests-tmp")
        self.proxy_profiles_path = self._copy_test_folder('proxy_profiles')
        self.js_profiles_path = self._copy_test_folder('js_profiles')

        self.mock_http_port = get_ephemeral_port()
        self.mock_https_port = get_ephemeral_port()
        self.mock_proxy_port = get_ephemeral_port()

        print("TestServers mock ports: %s http, %s https, %s proxy" % (
            self.mock_http_port, self.mock_https_port, self.mock_proxy_port))

        self._fix_testproxy_port()

    def _copy_test_folder(self, src, dst=None):
        src_path = os.path.join(os.path.dirname(__file__), src)
        dst_path = os.path.join(self.tmp_folder, dst or src)
        shutil.copytree(src_path, dst_path)
        return dst_path

    def _fix_testproxy_port(self):
        filename = os.path.join(self.proxy_profiles_path, 'test.ini')
        with open(filename, 'rb') as f:
            data = f.read()
        data = data.replace('8990', str(self.mock_proxy_port))
        with open(filename, 'wb') as f:
            f.write(data)

    def __enter__(self):
        self.mockserver = MockServer(
            self.mock_http_port,
            self.mock_https_port,
            self.mock_proxy_port,
        )
        self.mockserver.__enter__()
        self.splashserver = SplashServer(self.logfile, self.proxy_profiles_path,
                                         self.js_profiles_path)
        self.splashserver.__enter__()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.splashserver.__exit__(None, None, None)
        self.mockserver.__exit__(None, None, None)
        shutil.rmtree(self.tmp_folder)

    def print_output(self):
        print(_non_block_read(self.splashserver.proc.stderr))
        print(_non_block_read(self.mockserver.proc.stdout))

<|endoftext|>


<commit_before>
from .base import Node
from datetime import timedelta
from ebu_tt_live.bindings._ebuttdt import LimitedClockTimingType, FullClockTimingType
import time
import random

class FixedDelayNode(Node):

    _reference_clock = None
    _sequence = None

    def __init__(self, node_id, carriage_impl, reference_clock):
        super(FixedDelayNode, self).__init__(node_id, carriage_impl)
        self._reference_clock = reference_clock

        random_int = random.randint(0, 9999999999)
        new_identifier = 'sequence_{0}'.format(str(random_int))
        self._sequence = new_identifier

    def process_document(self, document):

        # change the sequence identifier
        document.sequence_identifier = self._sequence

        # TODO: add an ebuttm:trace element to the document metadata

        fixed_delay_int = 2

        # document is explicitly timed: modify the document
        if is_explicitly_timed(document.binding):

            update_children_timing(document.binding, document.time_base, fixed_delay_int)
            self._carriage_impl.emit_document(document)

        # document is implicitly timed: pause a while, re-emit later
        else:

            time.sleep(fixed_delay_int)
            self._carriage_impl.emit_document(document)

def update_children_timing(element, timebase, delay_int):

    # if the element has a child
    if hasattr(element, 'orderedContent'):

        children = element.orderedContent()

        for child in children:

            if hasattr(child.value, 'begin') and child.value.begin != None:

                if timebase == 'clock':
                    delay = LimitedClockTimingType(timedelta(seconds=delay_int))
                    child.value.begin = LimitedClockTimingType(child.value.begin.timedelta + delay.timedelta)
                elif timebase == 'media':
                    delay = FullClockTimingType(timedelta(seconds=delay_int))
                    child.value.begin = FullClockTimingType(child.value.begin.timedelta + delay.timedelta)
                # TODO: permit timebase = "smpte" with clockMode="continuous"

            if hasattr(child.value, 'end') and child.value.end != None:

                if timebase == 'clock':
                    delay = LimitedClockTimingType(timedelta(seconds=delay_int))
                    child.value.end = LimitedClockTimingType(child.value.end.timedelta + delay.timedelta)
                elif timebase == 'media':
                    delay = FullClockTimingType(timedelta(seconds=delay_int))
                    child.value.end = FullClockTimingType(child.value.end.timedelta + delay.timedelta)
                # TODO: smpte

            if not (hasattr(child.value, 'begin') and child.value.begin != None) and \
                not (hasattr(child.value, 'end') and child.value.end != None):

                update_children_timing(child.value, timebase, delay_int)

def is_explicitly_timed(element):

    # if element has begin or end attribute
    if hasattr(element, 'begin') and element.begin != None or hasattr(element, 'end') and element.end != None:
        return True

    else:

        # if element has children
        if hasattr(element, 'orderedContent'):

            children = element.orderedContent()

            for child in children:
                res = is_explicitly_timed(child.value)
                if res:
                    return res
<commit_msg>update children only is parent begin time unmodified<commit_after>
from .base import Node
from datetime import timedelta
from ebu_tt_live.bindings._ebuttdt import LimitedClockTimingType, FullClockTimingType
import time
import random

class FixedDelayNode(Node):

    _reference_clock = None
    _sequence = None

    def __init__(self, node_id, carriage_impl, reference_clock):
        super(FixedDelayNode, self).__init__(node_id, carriage_impl)
        self._reference_clock = reference_clock

        random_int = random.randint(0, 9999999999)
        new_identifier = 'sequence_{0}'.format(str(random_int))
        self._sequence = new_identifier

    def process_document(self, document):

        # change the sequence identifier
        document.sequence_identifier = self._sequence

        # TODO: add an ebuttm:trace element to the document metadata

        fixed_delay_int = 2

        # document is explicitly timed: modify the document
        if is_explicitly_timed(document.binding):

            update_children_timing(document.binding, document.time_base, fixed_delay_int)
            self._carriage_impl.emit_document(document)

        # document is implicitly timed: pause a while, re-emit later
        else:

            time.sleep(fixed_delay_int)
            self._carriage_impl.emit_document(document)

def update_children_timing(element, timebase, delay_int):

    # if the element has a child
    if hasattr(element, 'orderedContent'):

        children = element.orderedContent()

        for child in children:

            if hasattr(child.value, 'end') and child.value.end != None:

                if timebase == 'clock':
                    delay = LimitedClockTimingType(timedelta(seconds=delay_int))
                    child.value.end = LimitedClockTimingType(child.value.end.timedelta + delay.timedelta)
                elif timebase == 'media':
                    delay = FullClockTimingType(timedelta(seconds=delay_int))
                    child.value.end = FullClockTimingType(child.value.end.timedelta + delay.timedelta)
                    # TODO: smpte

            if hasattr(child.value, 'begin') and child.value.begin != None:

                if timebase == 'clock':
                    delay = LimitedClockTimingType(timedelta(seconds=delay_int))
                    child.value.begin = LimitedClockTimingType(child.value.begin.timedelta + delay.timedelta)
                elif timebase == 'media':
                    delay = FullClockTimingType(timedelta(seconds=delay_int))
                    child.value.begin = FullClockTimingType(child.value.begin.timedelta + delay.timedelta)
                # TODO: permit timebase = "smpte" with clockMode="continuous"

            else:
                update_children_timing(child.value, timebase, delay_int)

def is_explicitly_timed(element):

    # if element has begin or end attribute
    if hasattr(element, 'begin') and element.begin != None or hasattr(element, 'end') and element.end != None:
        return True

    else:

        # if element has children
        if hasattr(element, 'orderedContent'):

            children = element.orderedContent()

            for child in children:
                res = is_explicitly_timed(child.value)
                if res:
                    return res
<|endoftext|>


<commit_before>import re
import os
import time
import socket
import collectd
import traceback

MEMCACHED_HOST = 'localhost'
MEMCACHED_PORTS = []
COLLECTION_MODE = 'stat'

TYPE_COMMENTS_RE = '^[#\\s]'
TYPE_CUT_RE = '^([^\s]+)\s+(.*)'

TYPES_DB = {}

TYPES_CONV = {
  'kv_get' : 'get',
  'kv_hit' : 'hit',
  'kv_set' : 'set',
  'kv_del' : 'del',
  'lop_create' : 'lcs',
  'lop_insert' : 'lis',
  'lop_insert_hit' : 'lih',
  'lop_delete' : 'lds',
  'lop_delete_hit' : 'ldh',
  'lop_get' : 'lgs',
  'lop_get_hit' : 'lgh',
  'sop_create' : 'scs',
  'sop_insert' : 'sis',
  'sop_insert_hit' : 'sih',
  'sop_delete' : 'sds',
  'sop_delete_hit' : 'sdh',
  'sop_get' : 'sgs',
  'sop_get_hit' : 'sgh',
  'sop_exist' : 'ses',
  'sop_exist_hit' : 'seh',
  'bop_create' : 'bcs',
  'bop_insert' : 'bis',
  'bop_insert_hit' : 'bih',
  'bop_update' : 'bus',
  'bop_update_hit' : 'buh',
  'bop_incr' : 'bps',
  'bop_incr_hit' : 'bph',
  'bop_decr' : 'bms',
  'bop_decr_hit' : 'bmh',
  'bop_delete' : 'bds',
  'bop_delete_hit' : 'bdh',
  'bop_get' : 'bgs',
  'bop_get_hit' : 'bgh',
  'bop_count' : 'bns',
  'bop_count_hit' : 'bnh',
  'getattr' : 'gas',
  'setattr' : 'sas'
}

QOS_OPS = {
  'qos_kv_set' : 'set arcus-ping:qos_kv 0 3 4\r\nDATA\r\n',
  'qos_kv_get' : 'get arcus-ping:qos_kv\r\n',
  'qos_lop_insert' : 'lop insert arcus-ping:qos_lop 0 4 create 0 3 10\r\nDATA\r\n',
  'qos_lop_get' : 'lop get arcus-ping:qos_lop 0..10\r\n',
  'qos_lop_delete' : 'lop delete arcus-ping:qos_lop 0..10 drop\r\n',
  'qos_sop_insert' : 'sop insert arcus-ping:qos_sop 4 create 0 3 10\r\nDATA\r\n',
  'qos_sop_exist' : 'sop exist arcus-ping:qos_sop 4\r\nDATA\r\n',
  'qos_sop_delete' : 'sop delete arcus-ping:qos_sop 4 drop\r\nDATA\r\n',
  'qos_bop_insert' : 'bop insert arcus-ping:qos_bop 1 4 create 0 3 10\r\nDATA\r\n',
  'qos_bop_get' : 'bop get arcus-ping:qos_bop 1..10\r\n',
  'qos_bop_delete' : 'bop delete arcus-ping:qos_bop 1..10 10 drop\r\n'
}

QOS_RES = {
  'qos_kv_set' : 'STORED\r\n',
  'qos_kv_get' : 'END\r\n',
  'qos_lop_insert' : 'CREATED_STORED\r\n',
  'qos_lop_get' : 'END\r\n',
  'qos_lop_delete' : 'DELETED_DROPPED\r\n',
  'qos_sop_insert' : 'CREATED_STORED\r\n',
  'qos_sop_exist' : 'EXIST\r\n',
  'qos_sop_delete' : 'DELETED_DROPPED\r\n',
  'qos_bop_insert' : 'CREATED_STORED\r\n',
  'qos_bop_get' : 'END\r\n',
  'qos_bop_delete' : 'DELETED_DROPPED\r\n'
}

def str_to_num(s):
  try:
    n = float(s)
  except ValueError:
    n = 0
  return n

def get_memcached_ports():
  result = []
  pids = [pid for pid in os.listdir('/proc') if pid.isdigit()]

  for pid in pids:
    try:
      cmd = open(os.path.join('/proc', pid, 'cmdline'), 'r').read()
      if 'memcached' in cmd and 'default_engine.so' in cmd:
        port = re.findall('-p\x00([\d]+)', cmd)
        if len(port) > 0:
          result.append(int(port[0]))
    except Exception, e:
      continue

  return result

def get_type_instances(dbfile):
  result = {}
  types = []
  types_file = file(dbfile, 'r')
  for line in types_file.readlines():
    types.append(line.replace('\r\n', ''))
  types_file.close()

  for type in types:
    if not re.search(TYPE_COMMENTS_RE, type):
      typeobj = re.findall(TYPE_CUT_RE, type)[0]
      typename = typeobj[0]
      result[typename] = {}
      result[typename]['dsnames'] = []
      eachtype = typeobj[1].split(', ')
      for each in eachtype:
        # lop_get:COUNTER:0:U
        props = each.split(':')

        if typename.startswith('arcus_stats'):
          # FIXME fix for rrdtool's dsname character limit(=19 chars)
          props[0] = props[0].replace('ins', 'insert').replace('upd', 'update').replace('del', 'delete')
        elif typename.startswith('arcus_prefixes'):
          # FIXME make 'stats detail dump' metrics readable
          for fr, to in TYPES_CONV.iteritems():
            if props[0] == fr: props[0] = to

        result[typename][props[0]] = { 'type' : props[1].lower(), 'min' : props[2], 'max' : props[3] }
        result[typename]['dsnames'].append(props[0])

  return result

#def get_latency_ms(sock, operation):
#  begin = time.time()
#  sock.sendall(operation)
#  return (time.time() - begin) * 1000

def get_latency_ms(sock, file, qos_key):
  begin = time.time()
  sock.sendall(QOS_OPS[qos_key])

  while (1):
    line = file.readline()
    if not line or QOS_RES[qos_key] == line or 'NOT_FOUND\r\n' == line:
      break

  return (time.time() - begin) * 1000

def fetch_stat(host, port):
  result_stats = {}
  result_stats_detail = {}

  try:
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.connect((host, port))
  except:
    collectd.error('memcached_stat plugin: error connecting to %s:%d'%(MEMCACHED_HOST, port))
    return None

  fp = s.makefile('r')

  # stats
  s.sendall('stats\r\n')

  while (1):
    line = fp.readline()
    if not line or 'END\r\n' == line:
      break
    stat = line.replace('\r\n', '').split(' ')
    result_stats[stat[1]] = stat[2]

  # stats slabs
  s.sendall('stats slabs\r\n')

  while (1):
    line = fp.readline()
    if not line or 'END\r\n' == line:
      break
    stat = line.replace('\r\n', '').split(' ')
    result_stats[stat[1]] = stat[2]

  # stats detail dump
  s.sendall('stats detail dump\r\n')

  while (1):
    line = fp.readline()
    if not line or 'END\r\n' == line:
      break
    stat = line.replace('\r\n', '').split(' ')
    prefix_name = stat[1]
    prefix_stats = stat[2:]
    result_stats_detail[prefix_name] = {}

    for i in range(0, len(prefix_stats), 2):
      prefix_stat_name = prefix_stats[i]
      result_stats_detail[prefix_name][prefix_stat_name] = prefix_stats[i+1]

  # QoS (get operation latencies)
  result_stats['qos_kv_set']     = get_latency_ms(s, fp, 'qos_kv_set')
  result_stats['qos_kv_get']     = get_latency_ms(s, fp, 'qos_kv_get')
  result_stats['qos_lop_insert'] = get_latency_ms(s, fp, 'qos_lop_insert')
  result_stats['qos_lop_get']    = get_latency_ms(s, fp, 'qos_lop_get')
  result_stats['qos_lop_delete'] = get_latency_ms(s, fp, 'qos_lop_delete')
  result_stats['qos_sop_insert'] = get_latency_ms(s, fp, 'qos_sop_insert')
  result_stats['qos_sop_exist']  = get_latency_ms(s, fp, 'qos_sop_exist')
  result_stats['qos_sop_delete'] = get_latency_ms(s, fp, 'qos_sop_delete')
  result_stats['qos_bop_insert'] = get_latency_ms(s, fp, 'qos_bop_insert')
  result_stats['qos_bop_get']    = get_latency_ms(s, fp, 'qos_bop_get')
  result_stats['qos_bop_delete'] = get_latency_ms(s, fp, 'qos_bop_delete')

  s.close()

  return result_stats, result_stats_detail

def config_callback(conf):
  global MEMCACHED_HOST, TYPES_DB, MEMCACHED_PORTS, COLLECTION_MODE
  for node in conf.children:
    if node.key == 'Host':
      MEMCACHED_HOST = node.values[0]
    elif node.key == 'Port':
      if node.values[0] == 'DETECT':
        MEMCACHED_PORTS = get_memcached_ports()
      else:
        MEMCACHED_PORTS.append(str_to_num(node.values[0]))
    elif node.key == 'TypesDB':
      TYPES_DB = get_type_instances(node.values[0])
    elif node.key == 'Mode':
      COLLECTION_MODE = node.values[0]
    else:
      collectd.warning('arcus_stat plugin: Unknown config key: %s'%node.key)

def read_callback():
  global MEMCACHED_HOST, TYPES_DB, MEMCACHED_PORTS, COLLECTION_MODE
  # FIXME always check available arcus ports
  MEMCACHED_PORTS = get_memcached_ports()
  for port in MEMCACHED_PORTS:
    try:
      stats, stats_detail = fetch_stat(MEMCACHED_HOST, port)

      if COLLECTION_MODE == 'stat':
        # stats
        for type, entries in TYPES_DB.iteritems():
          if not type.startswith('arcus_stats'): continue
          varray = []
          for dsname in entries['dsnames']:
            if stats.has_key(dsname):
              varray.append(str_to_num(stats[dsname]))
            else:
              varray.append(0)
          value = collectd.Values(plugin='arcus_stat-%d'%port)
          value.type = type
          value.values = varray
          value.dispatch()
      elif COLLECTION_MODE == 'prefix':
        # stats detail dump
        for prefix, props in stats_detail.iteritems():
          # TODO refine prefix
          for type, entries in TYPES_DB.iteritems():
            if not type.startswith('arcus_prefixes'): continue
            if type.startswith('arcus_prefixes_meta'): continue
            varray = []
            for dsname in entries['dsnames']:
              if props.has_key(dsname):
                varray.append(str_to_num(props[dsname]))
              else:
                varray.append(0)
            value = collectd.Values(plugin='arcus_prefix-%d'%port)
            value.type_instance = prefix
            value.type = type
            value.values = varray
            value.dispatch()
        # number of prefixes
        nprefixes = len(stats_detail)
        value = collectd.Values(plugin='arcus_prefix-%d'%port)
        value.type_instance = 'arcus_prefixes_meta'
        value.type = 'arcus_prefixes_meta'
        value.values = [nprefixes]
        value.dispatch()
      else:
        collectd.warning('invalid mode : '%COLLECTION_MODE)

    except Exception, e:
      #collectd.warning('arcus_stat plugin: %s : type=%s, entries=%s'%(e, type, entries))
      collectd.error('arcus_stat plugin: %s'%(traceback.format_exc()))

if __name__ == '__main__':
  read_callback()

collectd.register_config(config_callback)
collectd.register_read(read_callback)

<commit_msg>Change the key string of collectd qos item: arcus-ping:qos_xxx => arcus:qos-xxx<commit_after>import re
import os
import time
import socket
import collectd
import traceback

MEMCACHED_HOST = 'localhost'
MEMCACHED_PORTS = []
COLLECTION_MODE = 'stat'

TYPE_COMMENTS_RE = '^[#\\s]'
TYPE_CUT_RE = '^([^\s]+)\s+(.*)'

TYPES_DB = {}

TYPES_CONV = {
  'kv_get' : 'get',
  'kv_hit' : 'hit',
  'kv_set' : 'set',
  'kv_del' : 'del',
  'lop_create' : 'lcs',
  'lop_insert' : 'lis',
  'lop_insert_hit' : 'lih',
  'lop_delete' : 'lds',
  'lop_delete_hit' : 'ldh',
  'lop_get' : 'lgs',
  'lop_get_hit' : 'lgh',
  'sop_create' : 'scs',
  'sop_insert' : 'sis',
  'sop_insert_hit' : 'sih',
  'sop_delete' : 'sds',
  'sop_delete_hit' : 'sdh',
  'sop_get' : 'sgs',
  'sop_get_hit' : 'sgh',
  'sop_exist' : 'ses',
  'sop_exist_hit' : 'seh',
  'bop_create' : 'bcs',
  'bop_insert' : 'bis',
  'bop_insert_hit' : 'bih',
  'bop_update' : 'bus',
  'bop_update_hit' : 'buh',
  'bop_incr' : 'bps',
  'bop_incr_hit' : 'bph',
  'bop_decr' : 'bms',
  'bop_decr_hit' : 'bmh',
  'bop_delete' : 'bds',
  'bop_delete_hit' : 'bdh',
  'bop_get' : 'bgs',
  'bop_get_hit' : 'bgh',
  'bop_count' : 'bns',
  'bop_count_hit' : 'bnh',
  'getattr' : 'gas',
  'setattr' : 'sas'
}

QOS_OPS = {
  'qos_kv_set' : 'set arcus:qos-kv 0 3 4\r\nDATA\r\n',
  'qos_kv_get' : 'get arcus:qos-kv\r\n',
  'qos_lop_insert' : 'lop insert arcus:qos-lop 0 4 create 0 3 10\r\nDATA\r\n',
  'qos_lop_get' : 'lop get arcus:qos-lop 0..10\r\n',
  'qos_lop_delete' : 'lop delete arcus:qos-lop 0..10 drop\r\n',
  'qos_sop_insert' : 'sop insert arcus:qos-sop 4 create 0 3 10\r\nDATA\r\n',
  'qos_sop_exist' : 'sop exist arcus:qos-sop 4\r\nDATA\r\n',
  'qos_sop_delete' : 'sop delete arcus:qos-sop 4 drop\r\nDATA\r\n',
  'qos_bop_insert' : 'bop insert arcus:qos-bop 1 4 create 0 3 10\r\nDATA\r\n',
  'qos_bop_get' : 'bop get arcus:qos-bop 1..10\r\n',
  'qos_bop_delete' : 'bop delete arcus:qos-bop 1..10 10 drop\r\n'
}

QOS_RES = {
  'qos_kv_set' : 'STORED\r\n',
  'qos_kv_get' : 'END\r\n',
  'qos_lop_insert' : 'CREATED_STORED\r\n',
  'qos_lop_get' : 'END\r\n',
  'qos_lop_delete' : 'DELETED_DROPPED\r\n',
  'qos_sop_insert' : 'CREATED_STORED\r\n',
  'qos_sop_exist' : 'EXIST\r\n',
  'qos_sop_delete' : 'DELETED_DROPPED\r\n',
  'qos_bop_insert' : 'CREATED_STORED\r\n',
  'qos_bop_get' : 'END\r\n',
  'qos_bop_delete' : 'DELETED_DROPPED\r\n'
}

def str_to_num(s):
  try:
    n = float(s)
  except ValueError:
    n = 0
  return n

def get_memcached_ports():
  result = []
  pids = [pid for pid in os.listdir('/proc') if pid.isdigit()]

  for pid in pids:
    try:
      cmd = open(os.path.join('/proc', pid, 'cmdline'), 'r').read()
      if 'memcached' in cmd and 'default_engine.so' in cmd:
        port = re.findall('-p\x00([\d]+)', cmd)
        if len(port) > 0:
          result.append(int(port[0]))
    except Exception, e:
      continue

  return result

def get_type_instances(dbfile):
  result = {}
  types = []
  types_file = file(dbfile, 'r')
  for line in types_file.readlines():
    types.append(line.replace('\r\n', ''))
  types_file.close()

  for type in types:
    if not re.search(TYPE_COMMENTS_RE, type):
      typeobj = re.findall(TYPE_CUT_RE, type)[0]
      typename = typeobj[0]
      result[typename] = {}
      result[typename]['dsnames'] = []
      eachtype = typeobj[1].split(', ')
      for each in eachtype:
        # lop_get:COUNTER:0:U
        props = each.split(':')

        if typename.startswith('arcus_stats'):
          # FIXME fix for rrdtool's dsname character limit(=19 chars)
          props[0] = props[0].replace('ins', 'insert').replace('upd', 'update').replace('del', 'delete')
        elif typename.startswith('arcus_prefixes'):
          # FIXME make 'stats detail dump' metrics readable
          for fr, to in TYPES_CONV.iteritems():
            if props[0] == fr: props[0] = to

        result[typename][props[0]] = { 'type' : props[1].lower(), 'min' : props[2], 'max' : props[3] }
        result[typename]['dsnames'].append(props[0])

  return result

#def get_latency_ms(sock, operation):
#  begin = time.time()
#  sock.sendall(operation)
#  return (time.time() - begin) * 1000

def get_latency_ms(sock, file, qos_key):
  begin = time.time()
  sock.sendall(QOS_OPS[qos_key])

  while (1):
    line = file.readline()
    if not line or QOS_RES[qos_key] == line or 'NOT_FOUND\r\n' == line:
      break

  return (time.time() - begin) * 1000

def fetch_stat(host, port):
  result_stats = {}
  result_stats_detail = {}

  try:
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.connect((host, port))
  except:
    collectd.error('memcached_stat plugin: error connecting to %s:%d'%(MEMCACHED_HOST, port))
    return None

  fp = s.makefile('r')

  # stats
  s.sendall('stats\r\n')

  while (1):
    line = fp.readline()
    if not line or 'END\r\n' == line:
      break
    stat = line.replace('\r\n', '').split(' ')
    result_stats[stat[1]] = stat[2]

  # stats slabs
  s.sendall('stats slabs\r\n')

  while (1):
    line = fp.readline()
    if not line or 'END\r\n' == line:
      break
    stat = line.replace('\r\n', '').split(' ')
    result_stats[stat[1]] = stat[2]

  # stats detail dump
  s.sendall('stats detail dump\r\n')

  while (1):
    line = fp.readline()
    if not line or 'END\r\n' == line:
      break
    stat = line.replace('\r\n', '').split(' ')
    prefix_name = stat[1]
    prefix_stats = stat[2:]
    result_stats_detail[prefix_name] = {}

    for i in range(0, len(prefix_stats), 2):
      prefix_stat_name = prefix_stats[i]
      result_stats_detail[prefix_name][prefix_stat_name] = prefix_stats[i+1]

  # QoS (get operation latencies)
  result_stats['qos_kv_set']     = get_latency_ms(s, fp, 'qos_kv_set')
  result_stats['qos_kv_get']     = get_latency_ms(s, fp, 'qos_kv_get')
  result_stats['qos_lop_insert'] = get_latency_ms(s, fp, 'qos_lop_insert')
  result_stats['qos_lop_get']    = get_latency_ms(s, fp, 'qos_lop_get')
  result_stats['qos_lop_delete'] = get_latency_ms(s, fp, 'qos_lop_delete')
  result_stats['qos_sop_insert'] = get_latency_ms(s, fp, 'qos_sop_insert')
  result_stats['qos_sop_exist']  = get_latency_ms(s, fp, 'qos_sop_exist')
  result_stats['qos_sop_delete'] = get_latency_ms(s, fp, 'qos_sop_delete')
  result_stats['qos_bop_insert'] = get_latency_ms(s, fp, 'qos_bop_insert')
  result_stats['qos_bop_get']    = get_latency_ms(s, fp, 'qos_bop_get')
  result_stats['qos_bop_delete'] = get_latency_ms(s, fp, 'qos_bop_delete')

  s.close()

  return result_stats, result_stats_detail

def config_callback(conf):
  global MEMCACHED_HOST, TYPES_DB, MEMCACHED_PORTS, COLLECTION_MODE
  for node in conf.children:
    if node.key == 'Host':
      MEMCACHED_HOST = node.values[0]
    elif node.key == 'Port':
      if node.values[0] == 'DETECT':
        MEMCACHED_PORTS = get_memcached_ports()
      else:
        MEMCACHED_PORTS.append(str_to_num(node.values[0]))
    elif node.key == 'TypesDB':
      TYPES_DB = get_type_instances(node.values[0])
    elif node.key == 'Mode':
      COLLECTION_MODE = node.values[0]
    else:
      collectd.warning('arcus_stat plugin: Unknown config key: %s'%node.key)

def read_callback():
  global MEMCACHED_HOST, TYPES_DB, MEMCACHED_PORTS, COLLECTION_MODE
  # FIXME always check available arcus ports
  MEMCACHED_PORTS = get_memcached_ports()
  for port in MEMCACHED_PORTS:
    try:
      stats, stats_detail = fetch_stat(MEMCACHED_HOST, port)

      if COLLECTION_MODE == 'stat':
        # stats
        for type, entries in TYPES_DB.iteritems():
          if not type.startswith('arcus_stats'): continue
          varray = []
          for dsname in entries['dsnames']:
            if stats.has_key(dsname):
              varray.append(str_to_num(stats[dsname]))
            else:
              varray.append(0)
          value = collectd.Values(plugin='arcus_stat-%d'%port)
          value.type = type
          value.values = varray
          value.dispatch()
      elif COLLECTION_MODE == 'prefix':
        # stats detail dump
        for prefix, props in stats_detail.iteritems():
          # TODO refine prefix
          for type, entries in TYPES_DB.iteritems():
            if not type.startswith('arcus_prefixes'): continue
            if type.startswith('arcus_prefixes_meta'): continue
            varray = []
            for dsname in entries['dsnames']:
              if props.has_key(dsname):
                varray.append(str_to_num(props[dsname]))
              else:
                varray.append(0)
            value = collectd.Values(plugin='arcus_prefix-%d'%port)
            value.type_instance = prefix
            value.type = type
            value.values = varray
            value.dispatch()
        # number of prefixes
        nprefixes = len(stats_detail)
        value = collectd.Values(plugin='arcus_prefix-%d'%port)
        value.type_instance = 'arcus_prefixes_meta'
        value.type = 'arcus_prefixes_meta'
        value.values = [nprefixes]
        value.dispatch()
      else:
        collectd.warning('invalid mode : '%COLLECTION_MODE)

    except Exception, e:
      #collectd.warning('arcus_stat plugin: %s : type=%s, entries=%s'%(e, type, entries))
      collectd.error('arcus_stat plugin: %s'%(traceback.format_exc()))

if __name__ == '__main__':
  read_callback()

collectd.register_config(config_callback)
collectd.register_read(read_callback)

<|endoftext|>


<commit_before>"""
directory.py stores methodology to install various files and
packages to different locations.

"""
import logging
import os
import shutil
import stat

rc_template = """
export PATH=%s:$PATH
"""

class DirectoryException(Exception):
    """ An exception to specify it's a directory """

class Directory(object):

    root_dir = None  # path to the root directory
    manifest_path = None  # path to the manifest file
    new = False  # determines if the directory is for a new environment or not
    rewrite_rc = True  # if set to false, the existing rc file will be
                       # preserved, and will not be modifiable
    rc_file = None  # file handler for rc file

    def __init__(self, namespace, rewrite_rc=True, sprinter_root=os.path.join("~", ".sprinter"),
                 logger=logging.getLogger('sprinter')):
        """ takes in a namespace directory to initialize, defaults to .sprinter otherwise."""
        self.root_dir = os.path.expanduser(os.path.join(sprinter_root, namespace))
        self.logger = logger
        self.new = not os.path.exists(self.root_dir)
        self.manifest_path = os.path.join(self.root_dir, "manifest.cfg")
        self.rewrite_rc = rewrite_rc

    def __del__(self):
        if self.rc_file:
            self.rc_file.close()

    def initialize(self):
        """ Generate the root directory root if it doesn't already exist """
        if not os.path.exists(self.root_dir):
            os.makedirs(self.root_dir)
        assert os.path.isdir(self.root_dir), "%s is not a directory! Please move or remove it." % self.root_dir
        for d in ["bin", "lib", "include"]:
            target_path = os.path.join(self.root_dir, d)
            if not os.path.exists(target_path):
                os.makedirs(target_path)
        if not os.path.exists(self.manifest_path):
            open(self.manifest_path, "w+").close()
        self.new = False

    def remove(self):
        """ Removes the sprinter directory, if it exists """
        if self.rc_file:
            self.rc_file.close()
        shutil.rmtree(self.root_dir)

    def symlink_to_bin(self, name, path):
        """ Symlink an object at path to name in the bin folder. """
        self.__symlink_dir("bin", name, path)
        os.chmod(os.path.join(self.root_dir, "bin", name), os.stat(path).st_mode | stat.S_IXUSR | stat.S_IRUSR)

    def remove_from_bin(self, name):
        """ Remove an object from the bin folder. """
        self.__remove_path(os.path.join(self.root_dir, "bin", name))

    def remove_feature(self, feature_name):
        """ Remove an feature from the environment root folder. """
        if os.path.exists(self.install_directory(feature_name)):
            self.__remove_path(self.install_directory(feature_name))

    def symlink_to_lib(self, name, path):
        """ Symlink an object at path to name in the lib folder. """
        self.__symlink_dir("lib", name, path)

    def symlink_to_include(self, name, path):
        """ Symlink an object at path to name in the lib folder. """
        self.__symlink_dir("include", name, path)

    def bin_path(self):
        """ return the bin directory path """
        return os.path.join(self.root_dir, "bin")

    def lib_path(self):
        """ return the lib directory path """
        return os.path.join(self.root_dir, "lib")

    def include_path(self):
        """ return the include directory path """
        return os.path.join(self.root_dir, "include")

    def install_directory(self, feature_name):
        """
        return a path to the install directory that the feature should install to.
        """
        return os.path.join(self.root_dir, "features", feature_name)

    def add_to_rc(self, content):
        """
        add content to the rc script.
        """
        if not self.rewrite_rc:
            raise DirectoryException("Error! Directory was not intialized w/ rewrite_rc.")
        if not self.rc_file:
            self.rc_path, self.rc_file = self.__get_rc_handle(self.root_dir)
        self.rc_file.write(content + '\n')

    def __remove_path(self, path):
        """ Remove an object """
        if not os.path.exists(path):
            self.logger.warn("Attempted to remove a non-existent path %s" % path)
            return
        try:
            if os.path.isdir(path):
                shutil.rmtree(path)
            else:
                os.unlink(path)
        except OSError:
            self.logger.error("Unable to remove object at path %s" % path)
            raise DirectoryException("Unable to remove object at path %s" % path)

    def __get_rc_handle(self, root_dir):
        """ get the filepath and filehandle to the rc file for the environment """
        rc_path = os.path.join(root_dir, '.rc')
        if not os.path.exists(rc_path):
            fh = open(rc_path, "w+")
            fh.write(rc_template % os.path.join(root_dir, "bin"))
        return (rc_path, open(rc_path, "w+"))

    def __symlink_dir(self, dir_name, name, path):
        """
        Symlink an object at path to name in the dir_name folder. remove it if it already exists.
        """
        target_path = os.path.join(self.root_dir, dir_name, name)
        self.logger.debug("Attempting to symlink %s to %s..." % (path, target_path))
        if os.path.exists(target_path):
            if os.path.islink(target_path):
                os.remove(target_path)
            else:
                self.logger.warn("%s is not a symlink! please remove it manually." % target_path)
                return
        os.symlink(path, target_path)
<commit_msg>Adding check for the directory already existing<commit_after>"""
directory.py stores methodology to install various files and
packages to different locations.

"""
import logging
import os
import shutil
import stat

rc_template = """
export PATH=%s:$PATH
"""

class DirectoryException(Exception):
    """ An exception to specify it's a directory """

class Directory(object):

    root_dir = None  # path to the root directory
    manifest_path = None  # path to the manifest file
    new = False  # determines if the directory is for a new environment or not
    rewrite_rc = True  # if set to false, the existing rc file will be
                       # preserved, and will not be modifiable
    rc_file = None  # file handler for rc file

    def __init__(self, namespace, rewrite_rc=True, sprinter_root=os.path.join("~", ".sprinter"),
                 logger=logging.getLogger('sprinter')):
        """ takes in a namespace directory to initialize, defaults to .sprinter otherwise."""
        self.root_dir = os.path.expanduser(os.path.join(sprinter_root, namespace))
        self.logger = logger
        self.new = not os.path.exists(self.root_dir)
        self.manifest_path = os.path.join(self.root_dir, "manifest.cfg")
        self.rewrite_rc = rewrite_rc

    def __del__(self):
        if self.rc_file:
            self.rc_file.close()

    def initialize(self):
        """ Generate the root directory root if it doesn't already exist """
        if not os.path.exists(self.root_dir):
            os.makedirs(self.root_dir)
        assert os.path.isdir(self.root_dir), "%s is not a directory! Please move or remove it." % self.root_dir
        for d in ["bin", "lib", "include"]:
            target_path = os.path.join(self.root_dir, d)
            if not os.path.exists(target_path):
                os.makedirs(target_path)
        if not os.path.exists(self.manifest_path):
            open(self.manifest_path, "w+").close()
        self.new = False

    def remove(self):
        """ Removes the sprinter directory, if it exists """
        if self.rc_file:
            self.rc_file.close()
        shutil.rmtree(self.root_dir)

    def symlink_to_bin(self, name, path):
        """ Symlink an object at path to name in the bin folder. """
        self.__symlink_dir("bin", name, path)
        os.chmod(os.path.join(self.root_dir, "bin", name), os.stat(path).st_mode | stat.S_IXUSR | stat.S_IRUSR)

    def remove_from_bin(self, name):
        """ Remove an object from the bin folder. """
        self.__remove_path(os.path.join(self.root_dir, "bin", name))

    def remove_feature(self, feature_name):
        """ Remove an feature from the environment root folder. """
        if os.path.exists(self.install_directory(feature_name)):
            self.__remove_path(self.install_directory(feature_name))

    def symlink_to_lib(self, name, path):
        """ Symlink an object at path to name in the lib folder. """
        self.__symlink_dir("lib", name, path)

    def symlink_to_include(self, name, path):
        """ Symlink an object at path to name in the lib folder. """
        self.__symlink_dir("include", name, path)

    def bin_path(self):
        """ return the bin directory path """
        return os.path.join(self.root_dir, "bin")

    def lib_path(self):
        """ return the lib directory path """
        return os.path.join(self.root_dir, "lib")

    def include_path(self):
        """ return the include directory path """
        return os.path.join(self.root_dir, "include")

    def install_directory(self, feature_name):
        """
        return a path to the install directory that the feature should install to.
        """
        return os.path.join(self.root_dir, "features", feature_name)

    def add_to_rc(self, content):
        """
        add content to the rc script.
        """
        if not self.rewrite_rc:
            raise DirectoryException("Error! Directory was not intialized w/ rewrite_rc.")
        if not self.rc_file:
            self.rc_path, self.rc_file = self.__get_rc_handle(self.root_dir)
        self.rc_file.write(content + '\n')

    def __remove_path(self, path):
        """ Remove an object """
        if not os.path.exists(path):
            self.logger.warn("Attempted to remove a non-existent path %s" % path)
            return
        try:
            if os.path.isdir(path):
                shutil.rmtree(path)
            else:
                os.unlink(path)
        except OSError:
            self.logger.error("Unable to remove object at path %s" % path)
            raise DirectoryException("Unable to remove object at path %s" % path)

    def __get_rc_handle(self, root_dir):
        """ get the filepath and filehandle to the rc file for the environment """
        rc_path = os.path.join(root_dir, '.rc')
        if not os.path.exists(rc_path):
            fh = open(rc_path, "w+")
            fh.write(rc_template % os.path.join(root_dir, "bin"))
        return (rc_path, open(rc_path, "w+"))

    def __symlink_dir(self, dir_name, name, path):
        """
        Symlink an object at path to name in the dir_name folder. remove it if it already exists.
        """
        target_dir = os.path.join(self.root_dir, dir_name)
        if not os.path.exists(target_dir):
            os.makedirs(target_dir)
        target_path = os.path.join(self.root_dir, dir_name, name)
        self.logger.debug("Attempting to symlink %s to %s..." % (path, target_path))
        if os.path.exists(target_path):
            if os.path.islink(target_path):
                os.remove(target_path)
            else:
                self.logger.warn("%s is not a symlink! please remove it manually." % target_path)
                return
        os.symlink(path, target_path)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Published by Hazard-SJ (https://wikitech.wikimedia.org/wiki/User:Hazard-SJ)
# under the terms of Creative Commons Attribution-ShareAlike 3.0 Unported (CC BY-SA 3.0)
# https://creativecommons.org/licenses/by-sa/3.0/

import MySQLdb
import pywikibot

pywikibot.config.family = 'wikipedia'
pywikibot.config.mylang = 'en'

site = pywikibot.Site()
site.login()
page = pywikibot.Page(site, "User:Hazard-Bot/Google Analytics report")

connection = MySQLdb.connect(
    host = 'enwiki.labsdb',
    db = 'enwiki_p',
    read_default_file = '~/replica.my.cnf'
)
cursor = connection.cursor()
query = '''\
SELECT page_title, el_to
FROM externallinks
JOIN page
ON page_id = el_from
WHERE el_to LIKE "%&utm_%=%"
AND page_namespace = 0
LIMIT 500;
'''
cursor.execute(query)

text = "Pages containing links tracked by Google Analytics:\n\n"
for title, link in cursor.fetchall():
    text += "# [[%s]]: %s\n" % (title, link)

pywikibot.data.api.Request(
    action="edit",
    title=page.title(),
    text=text,
    token=site.token(page, "edit"),
    summary="Robot: generated list of pages with links tracked by Google Analytics",
    bot=1
).submit()
#page.put(text, comment="Robot: generated list of pages with links tracked by Google Analytics")<commit_msg>Minor changes<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Published by Hazard-SJ (https://wikitech.wikimedia.org/wiki/User:Hazard-SJ)
# under the terms of Creative Commons Attribution-ShareAlike 3.0 Unported (CC BY-SA 3.0)
# https://creativecommons.org/licenses/by-sa/3.0/

import MySQLdb
import pywikibot

pywikibot.config.family = "wikipedia"
pywikibot.config.mylang = "en"

site = pywikibot.Site()
site.login()
page = pywikibot.Page(site, "User:Hazard-Bot/Google Analytics report")

connection = MySQLdb.connect(
    host = "enwiki.labsdb",
    db = "enwiki_p",
    read_default_file = "~/replica.my.cnf"
)
cursor = connection.cursor()
query = '''\
SELECT page_title, el_to
FROM externallinks
JOIN page
ON page_id = el_from
WHERE el_to LIKE "%&utm_%=%"
AND page_namespace = 0
LIMIT 500;
'''
cursor.execute(query)

text = "Pages containing links tracked by Google Analytics:\n\n"
for title, link in cursor.fetchall():
    text += "# [[%s]]: %s\n" % (title, link)

pywikibot.data.api.Request(
    action="edit",
    title=page.title(),
    text=text,
    token=site.token(page, "edit"),
    summary="Robot: generated list of pages with links tracked by Google Analytics",
    bot=1
).submit()
# page.put(text, comment="Robot: generated list of pages with links tracked by Google Analytics")<|endoftext|>


<commit_before>#!/usr/bin/env python
'''
CREATED:2014-03-18 by Justin Salamon <justin.salamon@nyu.edu>

Compute melody extraction evaluation measures

Usage:

./melody_eval.py TRUTH.TXT PREDICTION.TXT
(CSV files also accepted)

For a detailed explanation of the measures please refer to:

J. Salamon, E. Gomez, D. P. W. Ellis and G. Richard, "Melody Extraction
from Polyphonic Music Signals: Approaches, Applications and Challenges",
IEEE Signal Processing Magazine, 31(2):118-134, Mar. 2014.
'''

import numpy as np
import argparse
import sys
import os
from collections import OrderedDict
import mir_eval

def evaluate(reference_file, estimated_file):
    '''
    Evaluate two melody (predominant f0) transcriptions, where the first is
    treated as the reference (ground truth) and the second as the estimate to
    be evaluated (prediction).

    Input:
        truth_file - path to reference file. File should contain 2 columns:
              column 1 with timestamps and column 2 the corresponding reference
              frequency values in Hz (see *).
        prediction_file - path to estimate file. File should contain with 2 columns:
              column 1 with timestamps and column 2 the corresponding estimate
              frequency values in Hz (see **).
        hop - the desired hop size (in seconds) to compare the reference and
              estimate sequences (see ***)

    Output:
        M - ordered dictionary containing 5 evaluation measures:
        voicing recall rate -       Fraction of voiced frames in ref estimated as voiced in est
        voicing false alarm rate -  Fraction of unvoiced frames in ref estimated as voiced in est
        raw pitch -                 Fraction of voiced frames in ref for which est gives a correct pitch estimate (within 50 cents)
        raw chroma -                Same as raw pitch, but ignores octave errors
        overall accuracy -          Overall performance measure combining pitch and voicing

    *    Unvoiced frames should be indicated by 0 Hz.
    **   Unvoiced frames can be indicated either by 0 Hz or by a negative Hz
         value - negative values represent the algorithm's pitch estimate for
         frames it has determined as unvoiced, in case they are in fact voiced.
    ***  For a frame-by-frame comparison, both sequences are resampled using
         the provided hop size (in seconds), the default being 10 ms. The
         frequency values of the resampled sequences are obtained via linear
         interpolation of the original frequency values converted to a cent
         scale.
    '''

    # load the data
    ref_time, ref_freq = mir_eval.io.load_time_series(reference_file)
    est_time, est_freq = mir_eval.io.load_time_series(estimated_file)
    # Check if missing sample at time 0 and if so add one
    if ref_time[0] > 0:
        ref_time = np.insert(ref_time, 0, 0)
        ref_freq = np.insert(ref_freq, 0, ref_voicing[0])
    if est_time[0] > 0:
        est_time = np.insert(est_time, 0, 0)
        est_freq = np.insert(est_freq, 0, est_voicing[0])
    # Get separated frequency array and voicing boolean array
    ref_freq, ref_voicing = mir_eval.melody.freq_to_voicing(ref_freq)
    est_freq, est_voicing = mir_eval.melody.freq_to_voicing(est_freq)
    # convert both sequences to cents
    ref_cent = mir_eval.melody.hz2cents(ref_freq)
    est_cent = mir_eval.melody.hz2cents(est_freq)
    # Resample to common time base
    ref_time_grid, ref_cent, ref_voicing = mir_eval.melody.resample_melody_series(ref_time,
                                                                                ref_cent,
                                                                                ref_voicing)
    est_time_grid, est_cent, est_voicing = mir_eval.melody.resample_melody_series(est_time,
                                                                                est_cent,
                                                                                est_voicing)
    # ensure the estimated sequence is the same length as the reference
    len_diff = ref_cent.shape[0] - est_cent.shape[0]
    if len_diff >= 0:
        est_cent = np.append(est_cent, np.zeros(len_diff))
        est_voicing = np.append(est_voicing, np.zeros(len_diff))
    else:
        est_cent = est_cent[:ref_cent.shape[0]]
        est_voicing = est_voicing[ref_voicing.shape[0]]

    # Compute metrics
    M = OrderedDict()

    # F-Measure
    M['vx_recall'], M['vx_false_alarm'] = mir_eval.melody.voicing_measures(ref_voicing, est_voicing)

    M['raw_pitch'] = mir_eval.melody.raw_pitch_accuracy(ref_cent, ref_voicing, est_cent, est_voicing)

    M['raw_chroma'] = mir_eval.melody.raw_chroma_accuracy(ref_cent, ref_voicing, est_cent, est_voicing)

    M['overall_accuracy'] = mir_eval.melody.overall_accuracy(ref_cent, ref_voicing, est_cent, est_voicing)

    return M

def print_evaluation(prediction_file, M):
    '''
    Pretty print the melody extraction evaluation measures
    '''
    keys = M.keys()
    keys.append(os.path.basename(prediction_file))
    print '%s\t%s\t%s\t%s\t%s\t(%s)' % tuple(keys)
    print '%.3f\t\t%.3f\t\t%.3f\t\t%.3f\t\t%.3f' % tuple(M.values())

def process_arguments():
    '''Argparse function to get the program parameters'''

    parser = argparse.ArgumentParser(description='mir_eval melody extraction evaluation')

    parser.add_argument('reference_file',
                        action = 'store',
                        help = 'path to the ground truth annotation')

    parser.add_argument('estimated_file',
                        action = 'store',
                        help = 'path to the estimation file')

    return vars(parser.parse_args(sys.argv[1:]))

if __name__ == '__main__':
    # Get the parameters
    parameters = process_arguments()

    # Compute all the scores
    scores = evaluate(**parameters)

    # Print the scores
    print_evaluation(parameters['reference_file'], scores)
<commit_msg>corrected value inserted into time 0 when missing<commit_after>#!/usr/bin/env python
'''
CREATED:2014-03-18 by Justin Salamon <justin.salamon@nyu.edu>

Compute melody extraction evaluation measures

Usage:

./melody_eval.py TRUTH.TXT PREDICTION.TXT
(CSV files also accepted)

For a detailed explanation of the measures please refer to:

J. Salamon, E. Gomez, D. P. W. Ellis and G. Richard, "Melody Extraction
from Polyphonic Music Signals: Approaches, Applications and Challenges",
IEEE Signal Processing Magazine, 31(2):118-134, Mar. 2014.
'''

import numpy as np
import argparse
import sys
import os
from collections import OrderedDict
import mir_eval

def evaluate(reference_file, estimated_file):
    '''
    Evaluate two melody (predominant f0) transcriptions, where the first is
    treated as the reference (ground truth) and the second as the estimate to
    be evaluated (prediction).

    Input:
        truth_file - path to reference file. File should contain 2 columns:
              column 1 with timestamps and column 2 the corresponding reference
              frequency values in Hz (see *).
        prediction_file - path to estimate file. File should contain with 2 columns:
              column 1 with timestamps and column 2 the corresponding estimate
              frequency values in Hz (see **).
        hop - the desired hop size (in seconds) to compare the reference and
              estimate sequences (see ***)

    Output:
        M - ordered dictionary containing 5 evaluation measures:
        voicing recall rate -       Fraction of voiced frames in ref estimated as voiced in est
        voicing false alarm rate -  Fraction of unvoiced frames in ref estimated as voiced in est
        raw pitch -                 Fraction of voiced frames in ref for which est gives a correct pitch estimate (within 50 cents)
        raw chroma -                Same as raw pitch, but ignores octave errors
        overall accuracy -          Overall performance measure combining pitch and voicing

    *    Unvoiced frames should be indicated by 0 Hz.
    **   Unvoiced frames can be indicated either by 0 Hz or by a negative Hz
         value - negative values represent the algorithm's pitch estimate for
         frames it has determined as unvoiced, in case they are in fact voiced.
    ***  For a frame-by-frame comparison, both sequences are resampled using
         the provided hop size (in seconds), the default being 10 ms. The
         frequency values of the resampled sequences are obtained via linear
         interpolation of the original frequency values converted to a cent
         scale.
    '''

    # load the data
    ref_time, ref_freq = mir_eval.io.load_time_series(reference_file)
    est_time, est_freq = mir_eval.io.load_time_series(estimated_file)
    # Check if missing sample at time 0 and if so add one
    if ref_time[0] > 0:
        ref_time = np.insert(ref_time, 0, 0)
        ref_freq = np.insert(ref_freq, 0, ref_freq[0])
    if est_time[0] > 0:
        est_time = np.insert(est_time, 0, 0)
        est_freq = np.insert(est_freq, 0, est_freq[0])
    # Get separated frequency array and voicing boolean array
    ref_freq, ref_voicing = mir_eval.melody.freq_to_voicing(ref_freq)
    est_freq, est_voicing = mir_eval.melody.freq_to_voicing(est_freq)
    # convert both sequences to cents
    ref_cent = mir_eval.melody.hz2cents(ref_freq)
    est_cent = mir_eval.melody.hz2cents(est_freq)
    # Resample to common time base
    ref_time_grid, ref_cent, ref_voicing = mir_eval.melody.resample_melody_series(ref_time,
                                                                                ref_cent,
                                                                                ref_voicing)
    est_time_grid, est_cent, est_voicing = mir_eval.melody.resample_melody_series(est_time,
                                                                                est_cent,
                                                                                est_voicing)
    # ensure the estimated sequence is the same length as the reference
    len_diff = ref_cent.shape[0] - est_cent.shape[0]
    if len_diff >= 0:
        est_cent = np.append(est_cent, np.zeros(len_diff))
        est_voicing = np.append(est_voicing, np.zeros(len_diff))
    else:
        est_cent = est_cent[:ref_cent.shape[0]]
        est_voicing = est_voicing[ref_voicing.shape[0]]

    # Compute metrics
    M = OrderedDict()

    # F-Measure
    M['vx_recall'], M['vx_false_alarm'] = mir_eval.melody.voicing_measures(ref_voicing, est_voicing)

    M['raw_pitch'] = mir_eval.melody.raw_pitch_accuracy(ref_cent, ref_voicing, est_cent, est_voicing)

    M['raw_chroma'] = mir_eval.melody.raw_chroma_accuracy(ref_cent, ref_voicing, est_cent, est_voicing)

    M['overall_accuracy'] = mir_eval.melody.overall_accuracy(ref_cent, ref_voicing, est_cent, est_voicing)

    return M

def print_evaluation(prediction_file, M):
    '''
    Pretty print the melody extraction evaluation measures
    '''
    keys = M.keys()
    keys.append(os.path.basename(prediction_file))
    print '%s\t%s\t%s\t%s\t%s\t(%s)' % tuple(keys)
    print '%.3f\t\t%.3f\t\t%.3f\t\t%.3f\t\t%.3f' % tuple(M.values())

def process_arguments():
    '''Argparse function to get the program parameters'''

    parser = argparse.ArgumentParser(description='mir_eval melody extraction evaluation')

    parser.add_argument('reference_file',
                        action = 'store',
                        help = 'path to the ground truth annotation')

    parser.add_argument('estimated_file',
                        action = 'store',
                        help = 'path to the estimation file')

    return vars(parser.parse_args(sys.argv[1:]))

if __name__ == '__main__':
    # Get the parameters
    parameters = process_arguments()

    # Compute all the scores
    scores = evaluate(**parameters)

    # Print the scores
    print_evaluation(parameters['reference_file'], scores)
<|endoftext|>


<commit_before>from asyncore import dispatcher
from asynchat import async_chat
import socket, asyncore, time, sys
import cmdhandler
from apps.objects.models import Object
from django.contrib.auth.models import User
import commands_general
import functions_db
import session_mgr

class PlayerSession(async_chat):
   """
   This class represents a player's sesssion. From here we branch down into
   other various classes, please try to keep this one tidy!
   """
   def __init__(self, server, sock, addr):
      async_chat.__init__(self, sock)
      self.server = server
      self.address = addr
      self.set_terminator("\n")
      self.name = None
      self.data = []
      self.uid = None
      self.sock = sock
      self.logged_in = False
      # The time the user last issued a command.
      self.cmd_last = time.time()
      # Total number of commands issued.
      self.cmd_total = 0
      # The time when the user connected.
      self.conn_time = time.time()

   def collect_incoming_data(self, data):
      """
      Stuff any incoming data into our buffer, self.data
      """
      self.data.append(data)

   def found_terminator(self):
      """
      Any line return indicates a command for the purpose of a MUD. So we take
      the user input and pass it to our command handler.
      """
      line = (''.join(self.data))
      line = line.strip('\r')
      uinput = line
      self.data = []

      # Increment our user's command counter.
      self.cmd_total += 1
      # Store the timestamp of the user's last command.
      self.cmd_last = time.time()
      # Stuff anything we need to pass in this dictionary.
      cdat = {"server": self.server, "uinput": uinput, "session": self}
      cmdhandler.handle(cdat)

   def handle_close(self):
      """
      Break the connection and do some accounting.
      """
      self.get_pobject().set_flag("CONNECTED", False)
      async_chat.handle_close(self)
      self.logged_in = False
      session_mgr.remove_session(self)
      print 'Sessions active:', len(session_mgr.get_session_list())

   def get_pobject(self):
      """
      Returns the object associated with a session.
      """
      result = Object.objects.get(id=self.uid)
      #print 'RES', result
      return result

   def game_connect_screen(self, session):
      """
      Show our banner screen.
      """
      buffer =  '-'*50
      buffer += ' \n\rWelcome to Evennia!\n\r'
      buffer += '-'*50 + '\n\r'
      buffer += """Please type one of the following to begin:\n\r
         connect <username> <password>\n\r
         create <username> <email> <password>\n\r"""
      buffer += '-'*50
      session.msg(buffer)

   def login(self, user):
      """
      After the user has authenticated, handle logging him in.
      """
      self.uid = user.id
      self.name = user.username
      self.logged_in = True
      self.conn_time = time.time()
      self.get_pobject().set_flag("CONNECTED", True)

      self.msg("You are now logged in as %s." % (self.name,))
      cdat = {"session": self, "uinput":'look', "server": self.server}
      cmdhandler.handle(cdat)
      print "Login: %s" % (self,)

   def msg(self, message):
      """
      Sends a message with the newline/return included. Use this instead of
      directly calling push().
      """
      self.push("%s\n\r" % (message,))

   def msg_no_nl(self, message):
      """
      Sends a message without the newline/return included. Use this instead of
      directly calling push().
      """
      self.push("%s" % (message,))

   def __str__(self):
      """
      String representation of the user session class. We use
      this a lot in the server logs and stuff.
      """
      if self.logged_in:
         symbol = '#'
      else:
         symbol = '?'
      return "<%s> %s@%s" % (symbol, self.name, self.address,)

#   def handle_error(self):
#      self.handle_close()
<commit_msg>Don't need this anymore.<commit_after><|endoftext|>


<commit_before>"""
Copyright 2009 Chris Tarttelin and Point2 Technologies

Redistribution and use in source and binary forms, with or without modification, are
permitted provided that the following conditions are met:

Redistributions of source code must retain the above copyright notice, this list of
conditions and the following disclaimer.

Redistributions in binary form must reproduce the above copyright notice, this list
of conditions and the following disclaimer in the documentation and/or other materials
provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE FREEBSD PROJECT ``AS IS'' AND ANY EXPRESS OR IMPLIED
WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FREEBSD PROJECT OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

The views and conclusions contained in the software and documentation are those of the
authors and should not be interpreted as representing official policies, either expressed
or implied, of the FreeBSD Project.
"""

__doc__="A REST client, supporting GET, PUT, POST and DELETE"

import requests

class Client(object):
    """
    A new Client takes a base_url e.g. http://www.mysite.com:8765/rest and
    optionally a tuple containing username and password for use as basic
    auth.
    """
    def __init__(self, base_url, credentials=(None, None)):
        self.base_url = base_url or ""
        self._creds = credentials

    def GET(self, url, headers={}):
        return self._make_request(url, 'GET', None, headers)

    def PUT(self, url, payload=None, headers={}):
        return self._make_request(url, 'PUT', payload, headers)

    def POST(self, url, payload=None, headers={}):
        return self._make_request(url, 'POST', payload, headers)

    def DELETE(self, url, payload=None, headers={}):
            return self._make_request(url, 'DELETE', payload, headers)

    def _make_request(self, url, method, payload, headers):
        response = getattr(requests, method)(self.base_url + url, headers=headers, data=payload, auth=self._cred)
        return Response(self.base_url + url, response.status_code, response.headers, response.text)

class Response(object):
    """Encapsulates the response from a client GET/PUT/POST/DELETE call"""

    def __init__(self, url, response_code, headers, content):
        self._url = url
        self._response_code = response_code
        self._headers = dict(headers)
        self._content = content

    url = property(fget=lambda : self._url, doc="The url this response was returned from")
    response_code = property(fget=lambda self : self._response_code, doc="The response code returned from the call")
    headers = property(fget=lambda self : self._headers, doc="The headers returned in the response")
    content = property(fget=lambda self : self._content, doc="The response body, as a string, returned from the call")

    def expect(self, response_code):
        "If the actual response code does not match the expected response code, raises a HTTPError"
        if self.response_code != response_code:
            from requests.exceptions import HTTPError
            raise HTTPError(self.url, self.response_code, "Expected response code: %s, but was %s" % (response_code, self.response_code), None, None)

    def __getattr__(self, attr_name):
        if self.headers.has_key(attr_name):
            return self.headers[attr_name]
        raise AttributeError

    def __str__(self):
        "returns the content of the response as a string"
        return self.content<commit_msg>http methods in request should be get, post, delete, put<commit_after>"""
Copyright 2009 Chris Tarttelin and Point2 Technologies

Redistribution and use in source and binary forms, with or without modification, are
permitted provided that the following conditions are met:

Redistributions of source code must retain the above copyright notice, this list of
conditions and the following disclaimer.

Redistributions in binary form must reproduce the above copyright notice, this list
of conditions and the following disclaimer in the documentation and/or other materials
provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE FREEBSD PROJECT ``AS IS'' AND ANY EXPRESS OR IMPLIED
WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FREEBSD PROJECT OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

The views and conclusions contained in the software and documentation are those of the
authors and should not be interpreted as representing official policies, either expressed
or implied, of the FreeBSD Project.
"""

__doc__="A REST client, supporting GET, PUT, POST and DELETE"

import requests

class Client(object):
    """
    A new Client takes a base_url e.g. http://www.mysite.com:8765/rest and
    optionally a tuple containing username and password for use as basic
    auth.
    """
    def __init__(self, base_url, credentials=(None, None)):
        self.base_url = base_url or ""
        self._creds = credentials

    def GET(self, url, headers={}):
        return self._make_request(url, 'get', None, headers)

    def PUT(self, url, payload=None, headers={}):
        return self._make_request(url, 'put', payload, headers)

    def POST(self, url, payload=None, headers={}):
        return self._make_request(url, 'post', payload, headers)

    def DELETE(self, url, payload=None, headers={}):
        return self._make_request(url, 'delete', payload, headers)

    def _make_request(self, url, method, payload, headers):
        response = getattr(requests, method)(self.base_url + url, headers=headers, data=payload, auth=self._cred)
        return Response(self.base_url + url, response.status_code, response.headers, response.text)

class Response(object):
    """Encapsulates the response from a client GET/PUT/POST/DELETE call"""

    def __init__(self, url, response_code, headers, content):
        self._url = url
        self._response_code = response_code
        self._headers = dict(headers)
        self._content = content

    url = property(fget=lambda self: self._url, doc="The url this response was returned from")
    response_code = property(fget=lambda self : self._response_code, doc="The response code returned from the call")
    headers = property(fget=lambda self : self._headers, doc="The headers returned in the response")
    content = property(fget=lambda self : self._content, doc="The response body, as a string, returned from the call")

    def expect(self, response_code):
        "If the actual response code does not match the expected response code, raises a HTTPError"
        if self.response_code != response_code:
            from requests.exceptions import HTTPError
            raise HTTPError(self.url, self.response_code, "Expected response code: %s, but was %s" % (response_code, self.response_code), None, None)

    def __getattr__(self, attr_name):
        if self.headers.has_key(attr_name):
            return self.headers[attr_name]
        raise AttributeError

    def __str__(self):
        "returns the content of the response as a string"
        return self.content<|endoftext|>