<commit_before># -*- coding: utf-8 -*-

import time
import json
import email
import urllib
import urllib2

from contextlib import closing
from lxml import html

class HTML(object):
    def __init__(self, url, timeout):
        self.url = url
        self.timeout = timeout
        self.content = None

    @property
    def root(self):
        if not self.content:
            with closing(urllib2.urlopen(self.url, timeout=self.timeout)) as f:
                self.content = html.parse(f)
        return self.content.getroot()

    def getcontent(self, xpath):
        try:
            p = self.root.xpath(xpath)[0]
            return dict(text_content=p.text_content().strip(), **p.attrib)
        except:
            return {}

class HTTPClient(object):
    def __init__(self, host, port, https=False, charset='utf-8'):
        self.host = str(host)
        self.port = str(port)
        self.https = bool(https)
        self.charset = str(charset)
        self.content_handler = {}
        self.set_content_handler('application/json', json.loads)

    def set_content_handler(self, ctype, func):
        self.content_handler[ctype] = func

    def get_content_handler(self, ctype, default=lambda x: x):
        return self.content_handler.get(ctype, default)

    def del_content_handler(self, ctype):
        self.content_handler.pop(ctype, None)

    def get(self, path, **query):
        return self.request('GET', path, query=query)

    def put(self, path, body, **query):
        return self.request('PUT', path, body=body, query=query)

    def post(self, path, body, **query):
        return self.request('POST', path, body=body, query=query)

    def request(self, method, path, body=None, query={}, headers={}):
        if body is not None:
            body = json.dumps(body, ensure_ascii=False).encode(self.charset)
            ctype = 'application/json; charset=%s' % self.charset.upper()
            headers = headers.copy()
            headers['Content-Type'] = ctype

        try:
            url = self.url(path, query)
            code, info, body = self._request(method, url, body, headers)
            return code, self._parsebody(''.join(info.headers), body)
        except urllib2.HTTPError as e:
            return e.getcode(), ''

    def _request(self, method, url, body, headers={}):
        req = urllib2.Request(url, body, headers)
        req.get_method = lambda: str(method)
        obj = urllib2.urlopen(req)
        return obj.getcode(), obj.info(), obj.read()

    def _parsebody(self, strhdr, body):
        message = email.message_from_string(strhdr)
        ctype = message.get_content_type()
        charset = message.get_content_charset() or self.charset
        body = unicode(body, charset, 'replace')
        return self.get_content_handler(ctype)(body)

    def url(self, path, query={}):
        if self.https:
            scheme = 'https'
            default_port = '443'
        else:
            scheme = 'http'
            default_port = '80'

        if default_port == self.port:
            parts = [scheme, '://', self.host, path]
        else:
            parts = [scheme, '://', self.host, ':', self.port, path]

        if query:
            parts.extend(['?', self.querystring(query)])

        return str(''.join(parts))

    def querystring(self, query):
        q = {}
        for k, v in query.iteritems():
            if type(k) == unicode:
                k = k.encode(self.charset)
            if type(v) == unicode:
                v = v.encode(self.charset)
            q[k] = v
        return urllib.urlencode(q)

class GoogleClient(HTTPClient):
    def __init__(self, **kw):
        super(GoogleClient, self).__init__('www.google.com', 443, True, **kw)
        self.set_content_handler('text/javascript', json.loads)

    def complete(self, query, locale='en', nr_retry=0, retry_interval=0.2):
        def isvalid(code, body):
            if code != 200:
                return False
            if type(body) is not list:
                return False
            if len(body) < 2:
                return False
            if type(body[1]) is not list:
                return False
            return True

        for x in xrange(nr_retry + 1):
            code, body = self.get('/complete/search',
                                  client='firefox', hl=locale, q=query)

            if isvalid(code, body):
                return body[1]

            time.sleep(retry_interval)

        return None

class APIClientV01(HTTPClient):
    PATH_PREFIX = '/v0.1'

    def learn(self, instance, lines, nr_retry=0, retry_interval=0.2):
        def isvalid(code, body):
            if code != 204:
                return False
            return True

        path = '/'.join((self.PATH_PREFIX, instance))
        for x in xrange(nr_retry + 1):
            code, body = self.put(path, {'text': lines})

            if isvalid(code, body):
                return True

            time.sleep(retry_interval)

        return False

    def generate(self, instance, nr_retry=0, retry_interval=0.2):
        def isvalid(code, body):
            if code != 200:
                return False
            if type(body) is not dict:
                return False
            if 'text' not in body:
                return False
            if 'score' not in body:
                return False
            if type(body['text']) is not unicode:
                return False
            if not isinstance(body['score'], (int, long, float)):
                return False
            return True

        path = '/'.join((self.PATH_PREFIX, instance))
        for x in xrange(nr_retry + 1):
            code, body = self.get(path)

            if isvalid(code, body):
                return body['score'], body['text']

            time.sleep(retry_interval)

        return None, None

    def entries(self, instance, nr_retry=0, retry_interval=0.2):
        def isvalid(code, body):
            if code != 200:
                return False
            if type(body) is not dict:
                return False
            if 'keys' not in body:
                return False
            if type(body['keys']) is not list:
                return False
            return True

        path = '/'.join((self.PATH_PREFIX, instance, 'entrypoints'))
        for x in xrange(nr_retry + 1):
            code, body = self.get(path)

            if isvalid(code, body):
                return body['keys']

            time.sleep(retry_interval)

        return None

    def recent_entries(self, instance, nr_retry=0, retry_interval=0.2):
        def isvalid(code, body):
            if code != 200:
                return False
            if type(body) is not dict:
                return False
            if 'keys' not in body:
                return False
            if type(body['keys']) is not list:
                return False
            return True

        path = '/'.join((self.PATH_PREFIX, instance, 'recent-entrypoints'))
        for x in xrange(nr_retry + 1):
            code, body = self.get(path)

            if isvalid(code, body):
                return body['keys']

            time.sleep(retry_interval)

        return None

    def keys(self, instance, nr_retry=0, retry_interval=0.2):
        def isvalid(code, body):
            if code != 200:
                return False
            if type(body) is not dict:
                return False
            if 'keys' not in body:
                return False
            if type(body['keys']) is not list:
                return False
            return True

        path = '/'.join((self.PATH_PREFIX, instance, 'keys'))
        for x in xrange(nr_retry + 1):
            code, body = self.get(path)

            if isvalid(code, body):
                return body['keys']

            time.sleep(retry_interval)

        return None

    def values(self, instance, keys, nr_retry=0, retry_interval=0.2):
        def isvalid(code, body):
            if code != 200:
                return False
            if type(body) is not dict:
                return False
            if 'values' not in body:
                return False
            if type(body['values']) is not list:
                return False
            return True

        keys = json.dumps(keys, ensure_ascii=False).encode('utf-8')
        keys = urllib.quote(keys, safe='')
        path = '/'.join((self.PATH_PREFIX, instance, 'keys', keys))
        for x in xrange(nr_retry + 1):
            code, body = self.get(path)

            if isvalid(code, body):
                return body['values']

            time.sleep(retry_interval)

        return None

    def histories(self, instance, nr_retry=0, retry_interval=0.2):
        def isvalid(code, body):
            if code != 200:
                return False
            if type(body) is not dict:
                return False
            if 'histories' not in body:
                return False
            if type(body['histories']) is not list:
                return False
            return True

        path = '/'.join((self.PATH_PREFIX, instance, 'histories'))
        for x in xrange(nr_retry + 1):
            code, body = self.get(path)

            if isvalid(code, body):
                return body['histories']

            time.sleep(retry_interval)

        return None

    def stats(self, instance, nr_retry=0, retry_interval=0.2):
        def isvalid(code, body):
            if code != 200:
                return False
            if type(body) is not dict:
                return False
            for x in ('threshold', 'maxchain', 'keys', 'entrypoints'):
                if x not in body:
                    return False
                if not isinstance(body[x], (int, long, float)):
                    return False
            return True

        path = '/'.join((self.PATH_PREFIX, instance, 'stats'))
        for x in xrange(nr_retry + 1):
            code, body = self.get(path)

            if isvalid(code, body):
                return body

            time.sleep(retry_interval)

        return None
<commit_msg>util: http: add helper for HTML charset detection<commit_after># -*- coding: utf-8 -*-

import time
import json
import email
import urllib
import urllib2

from contextlib import closing
from lxml import html

class HTML(object):
    def __init__(self, url, timeout):
        self.url = url
        self.timeout = timeout
        self.content = None

    @staticmethod
    def getparser(headers):
        for p in headers.getplist():
            if p.lower().startswith('charset='):
                return html.HTMLParser(encoding=p.split('=')[1])
        return html.HTMLParser()

    @property
    def root(self):
        if not self.content:
            with closing(urllib2.urlopen(self.url, timeout=self.timeout)) as f:
                self.content = html.parse(f, self.getparser(f.headers))
        return self.content.getroot()

    def getcontent(self, xpath):
        try:
            p = self.root.xpath(xpath)[0]
            return dict(text_content=p.text_content().strip(), **p.attrib)
        except:
            return {}

class HTTPClient(object):
    def __init__(self, host, port, https=False, charset='utf-8'):
        self.host = str(host)
        self.port = str(port)
        self.https = bool(https)
        self.charset = str(charset)
        self.content_handler = {}
        self.set_content_handler('application/json', json.loads)

    def set_content_handler(self, ctype, func):
        self.content_handler[ctype] = func

    def get_content_handler(self, ctype, default=lambda x: x):
        return self.content_handler.get(ctype, default)

    def del_content_handler(self, ctype):
        self.content_handler.pop(ctype, None)

    def get(self, path, **query):
        return self.request('GET', path, query=query)

    def put(self, path, body, **query):
        return self.request('PUT', path, body=body, query=query)

    def post(self, path, body, **query):
        return self.request('POST', path, body=body, query=query)

    def request(self, method, path, body=None, query={}, headers={}):
        if body is not None:
            body = json.dumps(body, ensure_ascii=False).encode(self.charset)
            ctype = 'application/json; charset=%s' % self.charset.upper()
            headers = headers.copy()
            headers['Content-Type'] = ctype

        try:
            url = self.url(path, query)
            code, info, body = self._request(method, url, body, headers)
            return code, self._parsebody(''.join(info.headers), body)
        except urllib2.HTTPError as e:
            return e.getcode(), ''

    def _request(self, method, url, body, headers={}):
        req = urllib2.Request(url, body, headers)
        req.get_method = lambda: str(method)
        obj = urllib2.urlopen(req)
        return obj.getcode(), obj.info(), obj.read()

    def _parsebody(self, strhdr, body):
        message = email.message_from_string(strhdr)
        ctype = message.get_content_type()
        charset = message.get_content_charset() or self.charset
        body = unicode(body, charset, 'replace')
        return self.get_content_handler(ctype)(body)

    def url(self, path, query={}):
        if self.https:
            scheme = 'https'
            default_port = '443'
        else:
            scheme = 'http'
            default_port = '80'

        if default_port == self.port:
            parts = [scheme, '://', self.host, path]
        else:
            parts = [scheme, '://', self.host, ':', self.port, path]

        if query:
            parts.extend(['?', self.querystring(query)])

        return str(''.join(parts))

    def querystring(self, query):
        q = {}
        for k, v in query.iteritems():
            if type(k) == unicode:
                k = k.encode(self.charset)
            if type(v) == unicode:
                v = v.encode(self.charset)
            q[k] = v
        return urllib.urlencode(q)

class GoogleClient(HTTPClient):
    def __init__(self, **kw):
        super(GoogleClient, self).__init__('www.google.com', 443, True, **kw)
        self.set_content_handler('text/javascript', json.loads)

    def complete(self, query, locale='en', nr_retry=0, retry_interval=0.2):
        def isvalid(code, body):
            if code != 200:
                return False
            if type(body) is not list:
                return False
            if len(body) < 2:
                return False
            if type(body[1]) is not list:
                return False
            return True

        for x in xrange(nr_retry + 1):
            code, body = self.get('/complete/search',
                                  client='firefox', hl=locale, q=query)

            if isvalid(code, body):
                return body[1]

            time.sleep(retry_interval)

        return None

class APIClientV01(HTTPClient):
    PATH_PREFIX = '/v0.1'

    def learn(self, instance, lines, nr_retry=0, retry_interval=0.2):
        def isvalid(code, body):
            if code != 204:
                return False
            return True

        path = '/'.join((self.PATH_PREFIX, instance))
        for x in xrange(nr_retry + 1):
            code, body = self.put(path, {'text': lines})

            if isvalid(code, body):
                return True

            time.sleep(retry_interval)

        return False

    def generate(self, instance, nr_retry=0, retry_interval=0.2):
        def isvalid(code, body):
            if code != 200:
                return False
            if type(body) is not dict:
                return False
            if 'text' not in body:
                return False
            if 'score' not in body:
                return False
            if type(body['text']) is not unicode:
                return False
            if not isinstance(body['score'], (int, long, float)):
                return False
            return True

        path = '/'.join((self.PATH_PREFIX, instance))
        for x in xrange(nr_retry + 1):
            code, body = self.get(path)

            if isvalid(code, body):
                return body['score'], body['text']

            time.sleep(retry_interval)

        return None, None

    def entries(self, instance, nr_retry=0, retry_interval=0.2):
        def isvalid(code, body):
            if code != 200:
                return False
            if type(body) is not dict:
                return False
            if 'keys' not in body:
                return False
            if type(body['keys']) is not list:
                return False
            return True

        path = '/'.join((self.PATH_PREFIX, instance, 'entrypoints'))
        for x in xrange(nr_retry + 1):
            code, body = self.get(path)

            if isvalid(code, body):
                return body['keys']

            time.sleep(retry_interval)

        return None

    def recent_entries(self, instance, nr_retry=0, retry_interval=0.2):
        def isvalid(code, body):
            if code != 200:
                return False
            if type(body) is not dict:
                return False
            if 'keys' not in body:
                return False
            if type(body['keys']) is not list:
                return False
            return True

        path = '/'.join((self.PATH_PREFIX, instance, 'recent-entrypoints'))
        for x in xrange(nr_retry + 1):
            code, body = self.get(path)

            if isvalid(code, body):
                return body['keys']

            time.sleep(retry_interval)

        return None

    def keys(self, instance, nr_retry=0, retry_interval=0.2):
        def isvalid(code, body):
            if code != 200:
                return False
            if type(body) is not dict:
                return False
            if 'keys' not in body:
                return False
            if type(body['keys']) is not list:
                return False
            return True

        path = '/'.join((self.PATH_PREFIX, instance, 'keys'))
        for x in xrange(nr_retry + 1):
            code, body = self.get(path)

            if isvalid(code, body):
                return body['keys']

            time.sleep(retry_interval)

        return None

    def values(self, instance, keys, nr_retry=0, retry_interval=0.2):
        def isvalid(code, body):
            if code != 200:
                return False
            if type(body) is not dict:
                return False
            if 'values' not in body:
                return False
            if type(body['values']) is not list:
                return False
            return True

        keys = json.dumps(keys, ensure_ascii=False).encode('utf-8')
        keys = urllib.quote(keys, safe='')
        path = '/'.join((self.PATH_PREFIX, instance, 'keys', keys))
        for x in xrange(nr_retry + 1):
            code, body = self.get(path)

            if isvalid(code, body):
                return body['values']

            time.sleep(retry_interval)

        return None

    def histories(self, instance, nr_retry=0, retry_interval=0.2):
        def isvalid(code, body):
            if code != 200:
                return False
            if type(body) is not dict:
                return False
            if 'histories' not in body:
                return False
            if type(body['histories']) is not list:
                return False
            return True

        path = '/'.join((self.PATH_PREFIX, instance, 'histories'))
        for x in xrange(nr_retry + 1):
            code, body = self.get(path)

            if isvalid(code, body):
                return body['histories']

            time.sleep(retry_interval)

        return None

    def stats(self, instance, nr_retry=0, retry_interval=0.2):
        def isvalid(code, body):
            if code != 200:
                return False
            if type(body) is not dict:
                return False
            for x in ('threshold', 'maxchain', 'keys', 'entrypoints'):
                if x not in body:
                    return False
                if not isinstance(body[x], (int, long, float)):
                    return False
            return True

        path = '/'.join((self.PATH_PREFIX, instance, 'stats'))
        for x in xrange(nr_retry + 1):
            code, body = self.get(path)

            if isvalid(code, body):
                return body

            time.sleep(retry_interval)

        return None
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import json
import argparse

from utils import db_utils as db
from scripts.processing import clan_competition as cc

# ----------------------------------------------------------------

def parse_outlist_file(outlist_file):
    """

    :param outlist_file:
    :return:
    """
    outlist_info = {}

    seen_ga = False

    fp = open(outlist_file, 'r')

    for line in fp:
        # if not a comment line
        if line[0] != '#' and not seen_ga:
            line = line.strip().split()
            unique_accession = "_".join([line[3], line[5], line[6]])
            if unique_accession not in outlist_info:
                outlist_info[unique_accession] = {"evalue": float(line[1]),
                                                  "bit_score": float(line[0]),
                                                  "accession": line[3],
                                                  "start": int(line[5]),
                                                  "end": int(line[6])}

        elif line.find("CURRENT GA THRESHOLD:") != -1:
            seen_ga = True

    fp.close()

    return outlist_info

# ----------------------------------------------------------------

def extract_outlist_hits_to_list(outlist_file, sort=True):
    """

    :param outlist_file:

    :return:
    """

    outlist_hits = {}

    seen_ga = False

    fp = open(outlist_file, 'r')

    for line in fp:
        # if not a comment line
        if line[0] != '#' and not seen_ga:
            line = line.strip().split()

            if line[3] not in outlist_hits:
                outlist_hits[line[3]] = [(int(line[5]), int(line[6]))]
            else:
                outlist_hits[line[3]].append((int(line[5]), int(line[6])))

        elif line.find("CURRENT GA THRESHOLD:") != -1:
            seen_ga = True

    fp.close()

    if sort is True:
        # sorts hits by starting points
        for accession in outlist_hits:
            outlist_hits[accession].sort(key=lambda tup: tup[1])

    return outlist_hits

# ----------------------------------------------------------------

def extract_tax_ids_from_species(species_file):
    """

    :param species_file:
    :return:
    """

    tax_ids = {}

    fp = open(species_file, 'r')

    for line in fp:
        # if not a comment line
        if line[0] != '#':
            line = line.strip().split()

            if line[3] not in tax_ids:
                tax_ids[line[3]] = int(line[5])

    fp.close()

    return tax_ids

# ----------------------------------------------------------------

def get_family_location(accession):
    """

    :param family_label:
    :return:
    """

    search_dirs = ["/hps/nobackup/production/xfam/rfam/RELEASES/14.3/miRNA_relabelled/batch1_chunk1_searches",
                   "/hps/nobackup/production/xfam/rfam/RELEASES/14.3/miRNA_relabelled/batch1_chunk2_searches",
                   "/hps/nobackup/production/xfam/rfam/RELEASES/14.3/miRNA_relabelled/batch2/searches"]

    dir_label = ''
    if accession.find("_relabelled") == -1:
        dir_label = accession + "_relabelled"

    for search_dir in search_dirs:
        family_dir_loc = os.path.join(search_dir, dir_label)
        if os.path.exists(family_dir_loc):
            return family_dir_loc

    return None

# ----------------------------------------------------------------

def count_total_num_hits(outlist_hits):

    num_hits = 0

    for accession in outlist_hits.keys():

        num_hits += len(outlist_hits[accession])

    return num_hits

# ----------------------------------------------------------------

def parse_arguments():
    """

    :return:
    """

    parser = argparse.ArgumentParser()

    parser.add_argument("--family-dir", help="Path to family directory")
    parser.add_argument("--accessions", help="A json file with old/new family mapppings")

    return parser

# ----------------------------------------------------------------

if __name__ == "__main__":

    parser = parse_arguments()
    args = parser.parse_args()

    # load accessions
    fp = open(args.accessions, 'r')
    accessions = json.load(fp)
    fp.close()

    family_overlap_counts = {}

    for rfam_acc in accessions.keys():

	# skip iteration if not a valid Rfam family accession
        if rfam_acc[0:2] != "RF":
            continue

	# fetch family full region hits
        old_family_full_hits = db.fetch_family_full_regions(rfam_acc, sort=True)

	# now work on the miRNA family
        # 1. Detect family dir location
        mirna_id = accessions[rfam_acc]["mirbase_id"]
        family_dir = get_family_location(accessions[rfam_acc]["mirbase_id"])

        # 2. Find outlist path and parse it
        outlist_file_loc = os.path.join(family_dir, "outlist")
        if not os.path.exists(outlist_file_loc):
		continue
	outlist_hits = extract_outlist_hits_to_list(outlist_file_loc)

	# 3. Find overlaps between the two families
        # only check new family hits
        overlap = 0
        overlap_count = 0

        for accession in outlist_hits:
            old_hits = None
            if accession in old_family_full_hits:
                for region in outlist_hits[accession]:
                    for f_region in old_family_full_hits[accession]:
			overlap = cc.calc_seq_overlap(region[0], region[1], f_region[0], f_region[1])

			# this ensures each region in the new family is only checked once
                        # if no overlap found, it iterates over all superfamily hits
                        if overlap >= 50:
                            overlap_count += 1
                            break

        family_overlap_counts[mirna_id] = overlap_count
        num_outlist_hits = count_total_num_hits(outlist_hits)
        # compute family overlap percentage
        overlap_percentage = (family_overlap_counts[mirna_id] * 100) / num_outlist_hits
        print "\t".join([rfam_acc, mirna_id, str(overlap_percentage)])

<commit_msg>Implements code to compare additional values of the families<commit_after>#!/usr/bin/env python

import os
import json
import argparse

from utils import db_utils as db
from scripts.processing import clan_competition as cc

# ----------------------------------------------------------------

def parse_outlist_file(outlist_file):
    """

    :param outlist_file:
    :return:
    """
    outlist_info = {}

    seen_ga = False

    fp = open(outlist_file, 'r')

    for line in fp:
        # if not a comment line
        if line[0] != '#' and not seen_ga:
            line = line.strip().split()
            unique_accession = "_".join([line[3], line[5], line[6]])
            if unique_accession not in outlist_info:
                outlist_info[unique_accession] = {"evalue": float(line[1]),
                                                  "bit_score": float(line[0]),
                                                  "accession": line[3],
                                                  "start": int(line[5]),
                                                  "end": int(line[6])}

        elif line.find("CURRENT GA THRESHOLD:") != -1:
            seen_ga = True

    fp.close()

    return outlist_info

# ----------------------------------------------------------------

def extract_outlist_hits_to_list(outlist_file, sort=True):
    """

    :param outlist_file:

    :return:
    """

    outlist_hits = {}

    seen_ga = False

    fp = open(outlist_file, 'r')

    for line in fp:
        # if not a comment line
        if line[0] != '#' and not seen_ga:
            line = line.strip().split()

            if line[3] not in outlist_hits:
                outlist_hits[line[3]] = [(int(line[5]), int(line[6]))]
            else:
                outlist_hits[line[3]].append((int(line[5]), int(line[6])))

        elif line.find("CURRENT GA THRESHOLD:") != -1:
            seen_ga = True

    fp.close()

    if sort is True:
        # sorts hits by starting points
        for accession in outlist_hits:
            outlist_hits[accession].sort(key=lambda tup: tup[1])

    return outlist_hits

# ----------------------------------------------------------------

def extract_tax_ids_from_species(species_file):
    """

    :param species_file:
    :return:
    """

    tax_ids = {}

    fp = open(species_file, 'r')

    for line in fp:
        # if not a comment line
        if line[0] != '#':
            line = line.strip().split()

            if line[3] not in tax_ids:
                tax_ids[line[3]] = int(line[5])

    fp.close()

    return tax_ids

# ----------------------------------------------------------------

def get_family_location(accession):
    """

    :param family_label:
    :return:
    """

    search_dirs = ["/hps/nobackup/production/xfam/rfam/RELEASES/14.3/miRNA_relabelled/batch1_chunk1_searches",
                   "/hps/nobackup/production/xfam/rfam/RELEASES/14.3/miRNA_relabelled/batch1_chunk2_searches",
                   "/hps/nobackup/production/xfam/rfam/RELEASES/14.3/miRNA_relabelled/batch2/searches"]

    dir_label = ''
    if accession.find("_relabelled") == -1:
        dir_label = accession + "_relabelled"

    for search_dir in search_dirs:
        family_dir_loc = os.path.join(search_dir, dir_label)
        if os.path.exists(family_dir_loc):
            return family_dir_loc

    return None

# ----------------------------------------------------------------

def count_total_num_hits(outlist_hits):

    num_hits = 0

    for accession in outlist_hits.keys():

        num_hits += len(outlist_hits[accession])

    return num_hits

# ----------------------------------------------------------------

def parse_arguments():
    """

    :return:
    """

    parser = argparse.ArgumentParser()

    parser.add_argument("--family-dir", help="Path to family directory")
    parser.add_argument("--accessions", help="A json file with old/new family mapppings")

    return parser

# ----------------------------------------------------------------

if __name__ == "__main__":

    parser = parse_arguments()
    args = parser.parse_args()

    # load accessions
    fp = open(args.accessions, 'r')
    accessions = json.load(fp)
    fp.close()

    family_overlap_counts = {}

    for rfam_acc in accessions.keys():

	# skip iteration if not a valid Rfam family accession
        if rfam_acc[0:2] != "RF":
            continue

	# fetch family full region hits
        old_family_full_hits = db.fetch_family_full_regions(rfam_acc, sort=True)

	# now work on the miRNA family
        # 1. Detect family dir location
        mirna_id = accessions[rfam_acc]["mirbase_id"]
        family_dir = get_family_location(accessions[rfam_acc]["mirbase_id"])

        # 2. Find outlist path and parse it
        outlist_file_loc = os.path.join(family_dir, "outlist")
        if not os.path.exists(outlist_file_loc):
		continue
	outlist_hits = extract_outlist_hits_to_list(outlist_file_loc)

	# 3. Find overlaps between the two families
        # only check new family hits
        overlap = 0
        overlap_count = 0

        for accession in outlist_hits:
            old_hits = None
            if accession in old_family_full_hits:
                for region in outlist_hits[accession]:
                    for f_region in old_family_full_hits[accession]:
			overlap = cc.calc_seq_overlap(region[0], region[1], f_region[0], f_region[1])

			# this ensures each region in the new family is only checked once
                        # if no overlap found, it iterates over all superfamily hits
                        if overlap >= 50:
                            overlap_count += 1
                            break

        family_overlap_counts[mirna_id] = overlap_count
        num_outlist_hits = count_total_num_hits(outlist_hits)
        num_old_family_hits = count_total_num_hits(old_family_full_hits)

	species_file_loc = os.path.join(family_dir, "species")
	num_new_ncbi_ids = extract_tax_ids_from_species(species_file_loc)
	num_old_ncbi_ids = db.fetch_family_tax_ids(rfam_acc)

	# compute family overlap percentage
        overlap_percentage = (family_overlap_counts[mirna_id] * 100) / num_outlist_hits
        print "\t".join([rfam_acc, mirna_id, str(overlap_percentage), num_old_family_hits, num_outlist_hits, num_old_ncbi_ids, num_new_ncbi_ids])

<|endoftext|>


<commit_before>import unittest
from unittest import mock
from botologist import error

class ErrorHandlerTest(unittest.TestCase):
	def test_exception_is_formatted_correctly(self):
		eh = error.ErrorHandler(bot)
		try:
			raise RuntimeError('test')
		except:
			short_msg, medium_msg, long_msg = eh.format_error()

		self.assertEqual('Uncaught exception', short_msg)
		self.assertEqual('Uncaught exception - RuntimeError: test', medium_msg)
		self.assertEqual('Traceback (most recent call last):\n'
			'  File "'+__file__+'"'
			', line 10, in test_exception_is_formatted_correctly\n'
			'    raise RuntimeError(\'test\')\nRuntimeError: test', long_msg)

	def test_error_message_is_formatted_correctly(self):
		eh = error.ErrorHandler(bot)
		try:
			raise RuntimeError('test')
		except:
			short_msg, medium_msg, long_msg = eh.format_error('doing stuff')

		self.assertEqual('doing stuff', short_msg)
		self.assertEqual('doing stuff - RuntimeError: test', medium_msg)
		self.assertEqual('Traceback (most recent call last):\n'
			'  File "'+__file__+'"'
			', line 25, in test_error_message_is_formatted_correctly\n'
			'    raise RuntimeError(\'test\')\nRuntimeError: test', long_msg)
<commit_msg>fix error handler tests<commit_after>import unittest
from unittest import mock
from botologist import error

class ErrorHandlerTest(unittest.TestCase):
	def test_exception_is_formatted_correctly(self):
		eh = error.ErrorHandler(mock.MagicMock())
		try:
			raise RuntimeError('test')
		except:
			short_msg, medium_msg, long_msg = eh.format_error()

		self.assertEqual('Uncaught exception', short_msg)
		self.assertEqual('Uncaught exception - RuntimeError: test', medium_msg)
		self.assertEqual('Traceback (most recent call last):\n'
			'  File "'+__file__+'"'
			', line 9, in test_exception_is_formatted_correctly\n'
			'    raise RuntimeError(\'test\')\nRuntimeError: test', long_msg)

	def test_error_message_is_formatted_correctly(self):
		eh = error.ErrorHandler(mock.MagicMock())
		try:
			raise RuntimeError('test')
		except:
			short_msg, medium_msg, long_msg = eh.format_error('doing stuff')

		self.assertEqual('doing stuff', short_msg)
		self.assertEqual('doing stuff - RuntimeError: test', medium_msg)
		self.assertEqual('Traceback (most recent call last):\n'
			'  File "'+__file__+'"'
			', line 23, in test_error_message_is_formatted_correctly\n'
			'    raise RuntimeError(\'test\')\nRuntimeError: test', long_msg)
<|endoftext|>


<commit_before>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Redirect related methods.
"""

__authors__ = [
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  '"Lennard de Rijk" <ljvderijk@gmail.com>',
  ]

from google.appengine.ext import db

def getApplyRedirect(entity, params):
  """Returns the apply redirect for the specified entity.
  """

  result ='/%s/apply/%s' % (
      params['url_name'], entity.key().name())

  return result

def getInviteRedirect(entity, params):
  """Returns the invitation redirect for the specified entity.
  """

  result ='/%s/invite/%s' % (
      params['url_name'], entity.key().name())

  return result

def getCreateRedirect(entity, params):
  """Returns the create redirect for the specified entity.
  """

  result ='/%s/create/%s' % (
      params['url_name'], entity.key().name())

  return result

def getEditRedirect(entity, params):
  """Returns the edit redirect for the specified entity.
  """

  return '/%s/edit/%s' % (
      params['url_name'], entity.key().name())

def getPublicRedirect(entity, params):
  """Returns the public redirect for the specified entity.
  """

  return '/%s/show/%s' % (
      params['url_name'], entity.key().name())

def getExportRedirect(entity, params):
  """Returns the export redirect for the specified entity.
  """

  return '/%s/export/%s' % (
      params['url_name'], entity.key().name())

def getHomeRedirect(entity, params):
  """Returns the home redirect for the specified entity.
  """

  return '/%s/home/%s' % (
      params['url_name'], entity.key().name())

def getReviewRedirect(entity, params):
  """Returns the redirect to review the specified entity.
  """

  return '/%s/review/%s' % (
      params['url_name'], entity.key().name())

def getReviewOverviewRedirect(entity, params):
  """Returns the redirect to the review_overview using the
     keyname of the specified entity.
  """
  return '/%s/review_overview/%s' % (
      params['url_name'], entity.key().name())

def getCreateRequestRedirect(entity, params):
  """Returns the create request redirect for the specified entity.
  """

  result ='/request/create/%s/%s/%s' % (
      params['group_scope'], params['url_name'], entity.key().name())

  return result

def getRequestRedirectForRole(entity, role_name):
  """Returns the redirect to create a request for a specific role.
  """

  result ='/%s/request/%s' % (
      role_name, entity.key().name())

  return result

def getInviteRedirectForRole(entity, role_name):
  """Returns the redirect to create an invite for a specific role.
  """

  result ='/%s/invite/%s' % (
      role_name, entity.key().name())

  return result

def getListRequestsRedirect(entity, params):
  """Returns the redirect for the List Requests paged for the given
  Group entity and Group View params.
  """

  result = '/%s/list_requests/%s' % (
      params['url_name'], entity.key().name())

  return result

def getListRolesRedirect(entity, params):
  """Returns the redirect for the List Roles paged for the given
  Group entity and Group View params.
  """

  result = '/%s/list_roles/%s' % (
      params['url_name'], entity.key().name())

  return result

def getProcessRequestRedirect(entity, _):
  """Returns the redirect for processing the specified request entity.
  """

  result = '/%s/process_request/%s/%s' % (
      entity.role, entity.scope_path, entity.link_id)

  return result

def getManageRedirect(entity, params):
  """Returns the redirect for managing the given entity.
  """

  result = '/%s/manage/%s' % (
      params['url_name'], entity.key().name())

  return result

def getSelectRedirect(params):
  """Returns the pick redirect for the specified entity.
  """

  if params.get('args'):
    return '/%(url_name)s/pick?%(args)s' % params
  else:
    return '/%(url_name)s/pick' % params

def getInviteAcceptedRedirect(entity, _):
  """Returns the redirect for accepting an invite.
  """

  return '/%s/accept_invite/%s/%s' % (
      entity.role, entity.scope_path, entity.link_id)

def getInviteProcessRedirect(entity, _):
  """Returns the redirect for processing an invite.
  """

  return '/request/process_invite/%s/%s/%s' % (
      entity.scope_path, entity.role, entity.link_id)

def getApplicantRedirect(entity, params):
  """Returns the redirect for processing accepted Applications.
  """

  return '/%s/applicant/%s' % (
      params['url_name'], entity.key().name())

def getCreateDocumentRedirect(entity, prefix):
  """Returns the redirect for new documents.
  """

  return '/document/create/%s/%s' % (prefix, entity.key().name())

def getToSRedirect(presence):
  """Returns link to 'show' the ToS Document if it exists, None otherwise.

  Args:
    presence: Presence entity that may or may not have a tos property
  """
  if not presence:
    return None

  try:
    tos_doc = presence.tos
  except db.Error:
    return None

  if not tos_doc:
    return None

  return getPublicRedirect(tos_doc, {'url_name': 'document'})
<commit_msg>Added new redirect for the new list views for group_app's.<commit_after>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Redirect related methods.
"""

__authors__ = [
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  '"Lennard de Rijk" <ljvderijk@gmail.com>',
  ]

from google.appengine.ext import db

def getApplyRedirect(entity, params):
  """Returns the apply redirect for the specified entity.
  """

  result ='/%s/apply/%s' % (
      params['url_name'], entity.key().name())

  return result

def getInviteRedirect(entity, params):
  """Returns the invitation redirect for the specified entity.
  """

  result ='/%s/invite/%s' % (
      params['url_name'], entity.key().name())

  return result

def getCreateRedirect(entity, params):
  """Returns the create redirect for the specified entity.
  """

  result ='/%s/create/%s' % (
      params['url_name'], entity.key().name())

  return result

def getEditRedirect(entity, params):
  """Returns the edit redirect for the specified entity.
  """

  return '/%s/edit/%s' % (
      params['url_name'], entity.key().name())

def getPublicRedirect(entity, params):
  """Returns the public redirect for the specified entity.
  """

  return '/%s/show/%s' % (
      params['url_name'], entity.key().name())

def getExportRedirect(entity, params):
  """Returns the export redirect for the specified entity.
  """

  return '/%s/export/%s' % (
      params['url_name'], entity.key().name())

def getHomeRedirect(entity, params):
  """Returns the home redirect for the specified entity.
  """

  return '/%s/home/%s' % (
      params['url_name'], entity.key().name())

def getReviewRedirect(entity, params):
  """Returns the redirect to review the specified entity.
  """

  return '/%s/review/%s' % (
      params['url_name'], entity.key().name())

def getReviewOverviewRedirect(entity, params):
  """Returns the redirect to the review_overview using the
     keyname of the specified entity.
  """

  return '/%s/review_overview/%s' % (
      params['url_name'], entity.key().name())

def getCreateRequestRedirect(entity, params):
  """Returns the create request redirect for the specified entity.
  """

  result ='/request/create/%s/%s/%s' % (
      params['group_scope'], params['url_name'], entity.key().name())

  return result

def getRequestRedirectForRole(entity, role_name):
  """Returns the redirect to create a request for a specific role.
  """

  result ='/%s/request/%s' % (
      role_name, entity.key().name())

  return result

def getInviteRedirectForRole(entity, role_name):
  """Returns the redirect to create an invite for a specific role.
  """

  result ='/%s/invite/%s' % (
      role_name, entity.key().name())

  return result

def getListRequestsRedirect(entity, params):
  """Returns the redirect for the List Requests paged for the given
  Group entity and Group View params.
  """

  result = '/%s/list_requests/%s' % (
      params['url_name'], entity.key().name())

  return result

def getListSelfRedirect(entity, params):
  """Returns the redirect for list_self access type.
  """

  result = '/%s/list_self/%s' % (
      params['url_name'], entity.key().name())

  return result

def getListRolesRedirect(entity, params):
  """Returns the redirect for the List Roles paged for the given
  Group entity and Group View params.
  """

  result = '/%s/list_roles/%s' % (
      params['url_name'], entity.key().name())

  return result

def getProcessRequestRedirect(entity, _):
  """Returns the redirect for processing the specified request entity.
  """

  result = '/%s/process_request/%s/%s' % (
      entity.role, entity.scope_path, entity.link_id)

  return result

def getManageRedirect(entity, params):
  """Returns the redirect for managing the given entity.
  """

  result = '/%s/manage/%s' % (
      params['url_name'], entity.key().name())

  return result

def getSelectRedirect(params):
  """Returns the pick redirect for the specified entity.
  """

  if params.get('args'):
    return '/%(url_name)s/pick?%(args)s' % params
  else:
    return '/%(url_name)s/pick' % params

def getInviteAcceptedRedirect(entity, _):
  """Returns the redirect for accepting an invite.
  """

  return '/%s/accept_invite/%s/%s' % (
      entity.role, entity.scope_path, entity.link_id)

def getInviteProcessRedirect(entity, _):
  """Returns the redirect for processing an invite.
  """

  return '/request/process_invite/%s/%s/%s' % (
      entity.scope_path, entity.role, entity.link_id)

def getApplicantRedirect(entity, params):
  """Returns the redirect for processing accepted Applications.
  """

  return '/%s/applicant/%s' % (
      params['url_name'], entity.key().name())

def getCreateDocumentRedirect(entity, prefix):
  """Returns the redirect for new documents.
  """

  return '/document/create/%s/%s' % (prefix, entity.key().name())

def getToSRedirect(presence):
  """Returns link to 'show' the ToS Document if it exists, None otherwise.

  Args:
    presence: Presence entity that may or may not have a tos property
  """
  if not presence:
    return None

  try:
    tos_doc = presence.tos
  except db.Error:
    return None

  if not tos_doc:
    return None

  return getPublicRedirect(tos_doc, {'url_name': 'document'})
<|endoftext|>


<commit_before>""" Make outside spending aggregates w/r/t candidates """

from datetime import date
from django.core.management.base import BaseCommand, CommandError
from django.db.models import Sum
from formdata.models import SkedE
from summary_data.models import Pac_Candidate, Candidate_Overlay

cycle_start = date(2013,1,1)

class Command(BaseCommand):
    help = "Create race aggregates"
    requires_model_validation = False

    def handle(self, *args, **options):
        summary = SkedE.objects.all().exclude(superceded_by_amendment=True).exclude(candidate_checked__isnull=True).exclude(support_oppose_checked__isnull=True).exclude(contribution_date__lt=cycle_start).values('candidate_checked', 'support_oppose_checked','filer_committee_id_number').annotate(total=Sum('expenditure_amount'))
        for summary_line in summary:
            print "Candidate: %s s/o: %s amt: %s committee_id %s" % (summary_line['candidate_checked'], summary_line['support_oppose_checked'], summary_line['total'], summary_line['filer_committee_id_number'])
            pc, created = Pac_Candidate.objects.get_or_create(candidate__pk=summary_line['candidate_checked'], committee__pk=summary_line['filer_committee_id_number'], support_oppose=summary_line['support_oppose_checked'])
            pc.total_ind_exp = summary_line['expenditure_amount']
            pc.save()
        <commit_msg>fix date field<commit_after>""" Make outside spending aggregates w/r/t candidates """

from datetime import date
from django.core.management.base import BaseCommand, CommandError
from django.db.models import Sum
from formdata.models import SkedE
from summary_data.models import Pac_Candidate, Candidate_Overlay

cycle_start = date(2013,1,1)

class Command(BaseCommand):
    help = "Create race aggregates"
    requires_model_validation = False

    def handle(self, *args, **options):
        summary = SkedE.objects.all().exclude(superceded_by_amendment=True).exclude(candidate_checked__isnull=True).exclude(support_oppose_checked__isnull=True).exclude(expenditure_date_formatted__lt=cycle_start).values('candidate_checked', 'support_oppose_checked','filer_committee_id_number').annotate(total=Sum('expenditure_amount'))
        for summary_line in summary:
            print "Candidate: %s s/o: %s amt: %s committee_id %s" % (summary_line['candidate_checked'], summary_line['support_oppose_checked'], summary_line['total'], summary_line['filer_committee_id_number'])
            pc, created = Pac_Candidate.objects.get_or_create(candidate__pk=summary_line['candidate_checked'], committee__pk=summary_line['filer_committee_id_number'], support_oppose=summary_line['support_oppose_checked'])
            pc.total_ind_exp = summary_line['expenditure_amount']
            pc.save()
        <|endoftext|>


<commit_before>"""
Django settings for searchlogger project.

Generated by 'django-admin startproject' using Django 1.9.6.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# Load the secret key from a file
SECRET_KEY_FILENAME = os.path.join(
    os.path.abspath(os.sep),  # root directory
    'etc', 'django', 'searchlogger', 'secret.key'
)
with open(SECRET_KEY_FILENAME) as secret_key_file:
    SECRET_KEY = secret_key_file.read()

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'tastypie',
    'searchlogger',
]

MIDDLEWARE_CLASSES = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'searchlogger.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'searchlogger.wsgi.application'

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'
<commit_msg>Bugfix: Settings base directory link fixed<commit_after>"""
Django settings for searchlogger project.

Generated by 'django-admin startproject' using Django 1.9.6.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# Load the secret key from a file
SECRET_KEY_FILENAME = os.path.join(
    os.path.abspath(os.sep),  # root directory
    'etc', 'django', 'searchlogger', 'secret.key'
)
with open(SECRET_KEY_FILENAME) as secret_key_file:
    SECRET_KEY = secret_key_file.read()

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'tastypie',
    'searchlogger',
]

MIDDLEWARE_CLASSES = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'searchlogger.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'searchlogger.wsgi.application'

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'
<|endoftext|>


<commit_before>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Redirect related methods.
"""

__authors__ = [
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  '"Lennard de Rijk" <ljvderijk@gmail.com>',
  ]

from google.appengine.ext import db

def getApplyRedirect(entity, params):
  """Returns the apply redirect for the specified entity.
  """

  result ='/%s/apply/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getInviteRedirect(entity, params):
  """Returns the invitation redirect for the specified entity.
  """

  result ='/%s/invite/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getCreateRedirect(entity, params):
  """Returns the create redirect for the specified entity.
  """

  result ='/%s/create/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getEditRedirect(entity, params):
  """Returns the edit redirect for the specified entity.
  """

  return '/%s/edit/%s' % (
      params['url_name'], entity.key().id_or_name())

def getPublicRedirect(entity, params):
  """Returns the public redirect for the specified entity.
  """

  return '/%s/show/%s' % (
      params['url_name'], entity.key().id_or_name())

def getAdminRedirect(entity, params):
  """Returns the public redirect for the specified entity.
  """

  return '/%s/admin/%s' % (
      params['url_name'], entity.key().id_or_name())

def getListRedirect(entity, params):
  """Returns the public redirect for the specified entity.
  """

  return '/%s/list/%s' % (
      params['url_name'], entity.key().id_or_name())

def getPublicListRedirect(entity, params):
  """Returns the public redirect for the specified entity.
  """

  return '/%s/list_public/%s' % (
      params['url_name'], entity.key().id_or_name())

def getExportRedirect(entity, params):
  """Returns the export redirect for the specified entity.
  """

  return '/%s/export/%s' % (
      params['url_name'], entity.key().id_or_name())

def getHomeRedirect(entity, params):
  """Returns the home redirect for the specified entity.
  """

  return '/%s/home/%s' % (
      params['url_name'], entity.key().id_or_name())

def getReviewRedirect(entity, params):
  """Returns the redirect to review the specified entity.
  """

  return '/%s/review/%s' % (
      params['url_name'], entity.key().id_or_name())

def getReviewOverviewRedirect(entity, params):
  """Returns the redirect to the review_overview using the
     keyname of the specified entity.
  """

  return '/%s/review_overview/%s' % (
      params['url_name'], entity.key().id_or_name())

def getCreateRequestRedirect(entity, params):
  """Returns the create request redirect for the specified entity.
  """

  result ='/request/create/%s/%s/%s' % (
      params['group_scope'], params['url_name'], entity.key().id_or_name())

  return result

def getRequestRedirectForRole(entity, role_name):
  """Returns the redirect to create a request for a specific role.
  """

  result ='/%s/request/%s' % (
      role_name, entity.key().id_or_name())

  return result

def getInviteRedirectForRole(entity, role_name):
  """Returns the redirect to create an invite for a specific role.
  """

  result ='/%s/invite/%s' % (
      role_name, entity.key().id_or_name())

  return result

def getListProposalsRedirect(entity, params):
  """Returns the redirect for the List page for the given
  Org entity and Org View params.
  """

  result = '/%s/list_proposals/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getAcceptedOrgsRedirect(entity, params):
  """Returns the redirect for the List of accepted orgs.
  """

  result = '/%s/accepted_orgs/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getListProjectsRedirect(entity, params):
  """Returns the redirect for the List Projects page for the given entity.
  """

  result = '/%s/list_projects/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getListRequestsRedirect(entity, params):
  """Returns the redirect for the List Requests paged for the given
  Group entity and Group View params.
  """

  result = '/%s/list_requests/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getListSelfRedirect(entity, params):
  """Returns the redirect for list_self access type.
  """

  result = '/%s/list_self/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getListRolesRedirect(entity, params):
  """Returns the redirect for the List Roles paged for the given
  Group entity and Group View params.
  """

  result = '/%s/list_roles/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getUserRolesRedirect(_, __):
  """Returns the redirect to the users Roles page.
  """

  return '/user/roles'

def getProcessRequestRedirect(entity, _):
  """Returns the redirect for processing the specified request entity.
  """

  result = '/%s/process_request/%s/%s' % (
      entity.role, entity.scope_path, entity.link_id)

  return result

def getManageRedirect(entity, params):
  """Returns the redirect for managing the given entity.
  """

  result = '/%s/manage/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getManageOverviewRedirect(entity, params):
  """Returns the redirect for the manage overview view of the given entity.
  """

  result = '/%s/manage_overview/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getSelectRedirect(params):
  """Returns the pick redirect for the specified entity.
  """

  if params.get('args'):
    return '/%(url_name)s/pick?%(args)s' % params
  else:
    return '/%(url_name)s/pick' % params

def getInviteAcceptedRedirect(entity, _):
  """Returns the redirect for accepting an invite.
  """

  return '/%s/accept_invite/%s/%s' % (
      entity.role, entity.scope_path, entity.link_id)

def getInviteProcessRedirect(entity, _):
  """Returns the redirect for processing an invite.
  """

  return '/request/process_invite/%s/%s/%s' % (
      entity.scope_path, entity.role, entity.link_id)

def getApplicantRedirect(entity, params):
  """Returns the redirect for processing accepted Applications.
  """

  return '/%s/applicant/%s' % (
      params['url_name'], entity.key().id_or_name())

def getStudentEditRedirect(entity, params):
  """Returns the redirect for Students to edit their Projects.
  """

  return '/%s/st_edit/%s' % (
      params['url_name'], entity.key().id_or_name())

def getStudentProposalRedirect(entity, params):
  """Returns the student proposal redirect for the given org and student.
  """

  result ='/%s/apply/%s?organization=%s' % (
      params['url_name'], params['student_key'], entity.link_id)

  return result

def getShowDuplicatesRedirect(entity, params):
  """Returns the show duplicates redirect for the specified entity.
  """

  return'/%s/show_duplicates/%s' % (
      params['url_name'], entity.key().name())

def getSlotsRedirect(entity, params):
  """Returns the slots redirect for the specified entity.
  """

  return'/%s/slots/%s' % (
      params['url_name'], entity.key().id_or_name())

def getAssignSlotsRedirect(entity, params):
  """Returns the assign slots redirect for the specified entity.
  """

  return'/%s/assign_slots/%s' % (
      params['url_name'], entity.key().id_or_name())

def getCreateDocumentRedirect(entity, prefix):
  """Returns the redirect for new documents.
  """

  return '/document/create/%s/%s' % (prefix, entity.key().id_or_name())

def getListDocumentsRedirect(entity, prefix):
  """Returns the redirect for listing documents.
  """

  return '/document/list/%s/%s' % (prefix, entity.key().id_or_name())

def getToSRedirect(presence):
  """Returns link to 'show' the ToS Document if it exists, None otherwise.

  Args:
    presence: Presence entity that may or may not have a tos property
  """
  if not presence:
    return None

  try:
    tos_doc = presence.tos
  except db.Error:
    return None

  if not tos_doc:
    return None

  return getPublicRedirect(tos_doc, {'url_name': 'document'})
<commit_msg>Added redirect for use in Surveys.<commit_after>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Redirect related methods.
"""

__authors__ = [
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  '"Lennard de Rijk" <ljvderijk@gmail.com>',
  ]

from google.appengine.ext import db

def getApplyRedirect(entity, params):
  """Returns the apply redirect for the specified entity.
  """

  result ='/%s/apply/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getInviteRedirect(entity, params):
  """Returns the invitation redirect for the specified entity.
  """

  result ='/%s/invite/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getCreateRedirect(entity, params):
  """Returns the create redirect for the specified entity.
  """

  result ='/%s/create/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getEditRedirect(entity, params):
  """Returns the edit redirect for the specified entity.
  """

  return '/%s/edit/%s' % (
      params['url_name'], entity.key().id_or_name())

def getPublicRedirect(entity, params):
  """Returns the public redirect for the specified entity.
  """

  return '/%s/show/%s' % (
      params['url_name'], entity.key().id_or_name())

def getAdminRedirect(entity, params):
  """Returns the public redirect for the specified entity.
  """

  return '/%s/admin/%s' % (
      params['url_name'], entity.key().id_or_name())

def getListRedirect(entity, params):
  """Returns the public redirect for the specified entity.
  """

  return '/%s/list/%s' % (
      params['url_name'], entity.key().id_or_name())

def getPublicListRedirect(entity, params):
  """Returns the public redirect for the specified entity.
  """

  return '/%s/list_public/%s' % (
      params['url_name'], entity.key().id_or_name())

def getExportRedirect(entity, params):
  """Returns the export redirect for the specified entity.
  """

  return '/%s/export/%s' % (
      params['url_name'], entity.key().id_or_name())

def getHomeRedirect(entity, params):
  """Returns the home redirect for the specified entity.
  """

  return '/%s/home/%s' % (
      params['url_name'], entity.key().id_or_name())

def getReviewRedirect(entity, params):
  """Returns the redirect to review the specified entity.
  """

  return '/%s/review/%s' % (
      params['url_name'], entity.key().id_or_name())

def getReviewOverviewRedirect(entity, params):
  """Returns the redirect to the review_overview using the
     keyname of the specified entity.
  """

  return '/%s/review_overview/%s' % (
      params['url_name'], entity.key().id_or_name())

def getCreateRequestRedirect(entity, params):
  """Returns the create request redirect for the specified entity.
  """

  result ='/request/create/%s/%s/%s' % (
      params['group_scope'], params['url_name'], entity.key().id_or_name())

  return result

def getRequestRedirectForRole(entity, role_name):
  """Returns the redirect to create a request for a specific role.
  """

  result ='/%s/request/%s' % (
      role_name, entity.key().id_or_name())

  return result

def getInviteRedirectForRole(entity, role_name):
  """Returns the redirect to create an invite for a specific role.
  """

  result ='/%s/invite/%s' % (
      role_name, entity.key().id_or_name())

  return result

def getListProposalsRedirect(entity, params):
  """Returns the redirect for the List page for the given
  Org entity and Org View params.
  """

  result = '/%s/list_proposals/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getAcceptedOrgsRedirect(entity, params):
  """Returns the redirect for the List of accepted orgs.
  """

  result = '/%s/accepted_orgs/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getListProjectsRedirect(entity, params):
  """Returns the redirect for the List Projects page for the given entity.
  """

  result = '/%s/list_projects/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getListRequestsRedirect(entity, params):
  """Returns the redirect for the List Requests paged for the given
  Group entity and Group View params.
  """

  result = '/%s/list_requests/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getListSelfRedirect(entity, params):
  """Returns the redirect for list_self access type.
  """

  result = '/%s/list_self/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getListRolesRedirect(entity, params):
  """Returns the redirect for the List Roles paged for the given
  Group entity and Group View params.
  """

  result = '/%s/list_roles/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getUserRolesRedirect(_, __):
  """Returns the redirect to the users Roles page.
  """

  return '/user/roles'

def getProcessRequestRedirect(entity, _):
  """Returns the redirect for processing the specified request entity.
  """

  result = '/%s/process_request/%s/%s' % (
      entity.role, entity.scope_path, entity.link_id)

  return result

def getManageRedirect(entity, params):
  """Returns the redirect for managing the given entity.
  """

  result = '/%s/manage/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getManageOverviewRedirect(entity, params):
  """Returns the redirect for the manage overview view of the given entity.
  """

  result = '/%s/manage_overview/%s' % (
      params['url_name'], entity.key().id_or_name())

  return result

def getSelectRedirect(params):
  """Returns the pick redirect for the specified entity.
  """

  if params.get('args'):
    return '/%(url_name)s/pick?%(args)s' % params
  else:
    return '/%(url_name)s/pick' % params

def getInviteAcceptedRedirect(entity, _):
  """Returns the redirect for accepting an invite.
  """

  return '/%s/accept_invite/%s/%s' % (
      entity.role, entity.scope_path, entity.link_id)

def getInviteProcessRedirect(entity, _):
  """Returns the redirect for processing an invite.
  """

  return '/request/process_invite/%s/%s/%s' % (
      entity.scope_path, entity.role, entity.link_id)

def getApplicantRedirect(entity, params):
  """Returns the redirect for processing accepted Applications.
  """

  return '/%s/applicant/%s' % (
      params['url_name'], entity.key().id_or_name())

def getStudentEditRedirect(entity, params):
  """Returns the redirect for Students to edit their Projects.
  """

  return '/%s/st_edit/%s' % (
      params['url_name'], entity.key().id_or_name())

def getStudentProposalRedirect(entity, params):
  """Returns the student proposal redirect for the given org and student.
  """

  result ='/%s/apply/%s?organization=%s' % (
      params['url_name'], params['student_key'], entity.link_id)

  return result

def getShowDuplicatesRedirect(entity, params):
  """Returns the show duplicates redirect for the specified entity.
  """

  return'/%s/show_duplicates/%s' % (
      params['url_name'], entity.key().name())

def getSlotsRedirect(entity, params):
  """Returns the slots redirect for the specified entity.
  """

  return'/%s/slots/%s' % (
      params['url_name'], entity.key().id_or_name())

def getAssignSlotsRedirect(entity, params):
  """Returns the assign slots redirect for the specified entity.
  """

  return'/%s/assign_slots/%s' % (
      params['url_name'], entity.key().id_or_name())

def getCreateDocumentRedirect(entity, prefix):
  """Returns the redirect for new documents.
  """

  return '/document/create/%s/%s' % (prefix, entity.key().id_or_name())

def getListDocumentsRedirect(entity, prefix):
  """Returns the redirect for listing documents.
  """

  return '/document/list/%s/%s' % (prefix, entity.key().id_or_name())

def getCreateSurveyRedirect(entity, prefix):
  """Returns the redirect for new documents.
  """

  return '/survey/create/%s/%s' % (prefix, entity.key().id_or_name())

def getListSurveysRedirect(entity, prefix):
  """Returns the redirect for listing documents.
  """

  return '/survey/list/%s/%s' % (prefix, entity.key().id_or_name())

def getListSurveyResultsRedirect(entity, prefix):
  """Returns the redirect for listing documents.
  """

  return '/survey/list_results/%s/%s' % (prefix, entity.key().id_or_name())

def getToSRedirect(presence):
  """Returns link to 'show' the ToS Document if it exists, None otherwise.

  Args:
    presence: Presence entity that may or may not have a tos property
  """
  if not presence:
    return None

  try:
    tos_doc = presence.tos
  except db.Error:
    return None

  if not tos_doc:
    return None

  return getPublicRedirect(tos_doc, {'url_name': 'document'})
<|endoftext|>


<commit_before>__version_info__ = (1, 1, 0)
__version__ = '.'.join(str(v) for v in __version_info__)
__bzr_revno__ = 2112
<commit_msg>Bump version number so we know this is the one with different defaults<commit_after>__version_info__ = (1, 2, 0)
__version__ = '.'.join(str(v) for v in __version_info__)
__bzr_revno__ = 2095
<|endoftext|>


<commit_before>#!/usr/bin/python
import os, sys, socket, random, time

sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, "common")))
from test_common import *
from line import *

key_padding = ''.zfill(20)
def gen_key(prefix, num):
    return prefix + key_padding + str(num).zfill(6)

value_padding = ''.zfill(25)
large_value_padding = ''.zfill(2000)
def gen_value(prefix, num):
    if num % 5 == 4:
        return prefix + large_value_padding + str(num).zfill(6)
    else:
        return prefix + value_padding + str(num).zfill(6)

def sock_readline(sock_file):
    ls = []
    while True:
        l = sock_file.readline()
        ls.append(l)
        if len(l) >= 2 and l[-2:] == '\r\n':
            break
    return ''.join(ls)

value_line = line("^VALUE\s+([^\s]+)\s+(\d+)\s+(\d+)\r\n$", [('key', 's'), ('flags', 'd'), ('length', 'd')])
def is_sorted_output(kvs):
    k = None
    for kv in kvs:
        if not k:
            k = kv['key']
            continue

        if k >= kv['key']:
            return False

        k = kv['key']
    return True

def get_results(s):
    res = []

    f = s.makefile()
    while True:
        l = sock_readline(f)
        if l == 'END\r\n':
            break
        val_def = value_line.parse_line(l)
        if not val_def:
            raise ValueError("received unexpected line from rget: %s" % l)
        val = sock_readline(f).rstrip()
        if len(val) != val_def['length']:
            raise ValueError("received value of unexpected length (expected %d, got %d: '%s')" % (val_def['length'], len(val), val))

        res.append({'key': val_def['key'], 'value': val})
    return res

def check_results(res, expected_count):
    count = len(res)
    if count < expected_count:
        raise ValueError("received less rget results than expected (expected: %d, got: %d)" % (expected_count, count))
    if count > expected_count:
        raise ValueError("received more rget results than expected (expected: %d, got: %d)" % (expected_count, count))
    if not is_sorted_output(res):
        raise ValueError("received unsorted rget output")

def test_function(opts, port, test_dir):
    foo_count = 100
    fop_count = 1000
    max_results = foo_count+fop_count

    print "Creating test data"
    mc = connect_to_port(opts, port)
    mc.set('z1', 'bar', time=1) # we expect it to expire before we check it later
    for i in range(0,foo_count):
        mc.set(gen_key('foo', i), gen_value('foo', i))
    for i in range(0,fop_count):
        mc.set(gen_key('fop', i), gen_value('fop', i))
    mc.disconnect_all()

    print "Testing rget"
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.connect(("localhost", port))

    print "Checking simple rget requests with open/closed boundaries"
    s.send('rget %s %s %d %d %d\r\n' % (gen_key('foo', 0), gen_key('fop', 0), 0, 1, max_results))
    res = get_results(s)
    check_results(res, foo_count)

    s.send('rget %s %s %d %d %d\r\n' % (gen_key('foo', 0), gen_key('fop', 0), 1, 0, max_results))
    res = get_results(s)
    check_results(res, foo_count - 1 + 1)

    s.send('rget %s %s %d %d %d\r\n' % (gen_key('foo', 0), gen_key('fop', 0), 1, 1, max_results))
    res = get_results(s)
    check_results(res, foo_count - 1)

    print "Checking that rget works when the boundares are not real keys"
    s.send('rget %s %s %d %d %d\r\n' % ('a', 'fop', 0, 0, max_results))
    res = get_results(s)
    check_results(res, foo_count)

    print "Checking larger number of results"
    s.send('rget %s %s %d %d %d\r\n' % ('a', gen_key('goo', 0), 0, 1, max_results))
    res = get_results(s)
    check_results(res, foo_count + fop_count)

    print "Checking simple paging"
    page_size = 13
    s.send('rget %s %s %d %d %d\r\n' % ('a', gen_key('goo', 0), 0, 1, page_size))
    res = get_results(s)
    check_results(res, page_size)

    print "Checking contiguous paging"
    page_size = 13
    res = []
    from_key = 'a'
    while True:
        s.send('rget %s %s %d %d %d\r\n' % (from_key, gen_key('fop', 0), 1, 1, page_size))
        cur_res = get_results(s)
        res.extend(cur_res)
        if len(res) < foo_count:
            check_results(cur_res, page_size)
            from_key = cur_res[-1]['key']
        else:
            check_results(cur_res, len(cur_res))
            break
    check_results(res, foo_count)

    for i in range(1,len(res)):
        kv = res[i]
        expected_key = gen_key('foo', i)
        expected_value = gen_value('foo', i)
        if kv['key'] != expected_key:
            raise ValueError("received wrong key (expected: '%s', got: '%s')" % (expected_key, kv['key']))
        if kv['value'] != expected_value:
            raise ValueError("received wrong value (expected: '%s', got: '%s')" % (expected_key, kv['value']))

    print "Checking empty results being returned when no keys match"
    s.send('rget %s %s %d %d %d\r\n' % ('a', 'b', 0, 0, max_results))
    res = get_results(s)
    check_results(res, 0)

    print "Checking that expired values are not returned"
    s.send('rget z0 z2 1 1 100\r\n')
    res = get_results(s)
    check_results(res, 0)

    s.close()

if __name__ == "__main__":
    op = make_option_parser()
    auto_server_test_main(test_function, op.parse(sys.argv))

<commit_msg>Add a sleep before checking for expired value in case the server is too fast<commit_after>#!/usr/bin/python
import os, sys, socket, random, time

sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, "common")))
from test_common import *
from line import *

key_padding = ''.zfill(20)
def gen_key(prefix, num):
    return prefix + key_padding + str(num).zfill(6)

value_padding = ''.zfill(25)
large_value_padding = ''.zfill(2000)
def gen_value(prefix, num):
    if num % 5 == 4:
        return prefix + large_value_padding + str(num).zfill(6)
    else:
        return prefix + value_padding + str(num).zfill(6)

def sock_readline(sock_file):
    ls = []
    while True:
        l = sock_file.readline()
        ls.append(l)
        if len(l) >= 2 and l[-2:] == '\r\n':
            break
    return ''.join(ls)

value_line = line("^VALUE\s+([^\s]+)\s+(\d+)\s+(\d+)\r\n$", [('key', 's'), ('flags', 'd'), ('length', 'd')])
def is_sorted_output(kvs):
    k = None
    for kv in kvs:
        if not k:
            k = kv['key']
            continue

        if k >= kv['key']:
            return False

        k = kv['key']
    return True

def get_results(s):
    res = []

    f = s.makefile()
    while True:
        l = sock_readline(f)
        if l == 'END\r\n':
            break
        val_def = value_line.parse_line(l)
        if not val_def:
            raise ValueError("received unexpected line from rget: %s" % l)
        val = sock_readline(f).rstrip()
        if len(val) != val_def['length']:
            raise ValueError("received value of unexpected length (expected %d, got %d: '%s')" % (val_def['length'], len(val), val))

        res.append({'key': val_def['key'], 'value': val})
    return res

def check_results(res, expected_count):
    count = len(res)
    if count < expected_count:
        raise ValueError("received less rget results than expected (expected: %d, got: %d)" % (expected_count, count))
    if count > expected_count:
        raise ValueError("received more rget results than expected (expected: %d, got: %d)" % (expected_count, count))
    if not is_sorted_output(res):
        raise ValueError("received unsorted rget output")

def test_function(opts, port, test_dir):
    foo_count = 100
    fop_count = 1000
    max_results = foo_count+fop_count

    print "Creating test data"
    mc = connect_to_port(opts, port)
    mc.set('z1', 'bar', time=1) # we expect it to expire before we check it later
    for i in range(0,foo_count):
        mc.set(gen_key('foo', i), gen_value('foo', i))
    for i in range(0,fop_count):
        mc.set(gen_key('fop', i), gen_value('fop', i))
    mc.disconnect_all()

    print "Testing rget"
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.connect(("localhost", port))

    print "Checking simple rget requests with open/closed boundaries"
    s.send('rget %s %s %d %d %d\r\n' % (gen_key('foo', 0), gen_key('fop', 0), 0, 1, max_results))
    res = get_results(s)
    check_results(res, foo_count)

    s.send('rget %s %s %d %d %d\r\n' % (gen_key('foo', 0), gen_key('fop', 0), 1, 0, max_results))
    res = get_results(s)
    check_results(res, foo_count - 1 + 1)

    s.send('rget %s %s %d %d %d\r\n' % (gen_key('foo', 0), gen_key('fop', 0), 1, 1, max_results))
    res = get_results(s)
    check_results(res, foo_count - 1)

    print "Checking that rget works when the boundares are not real keys"
    s.send('rget %s %s %d %d %d\r\n' % ('a', 'fop', 0, 0, max_results))
    res = get_results(s)
    check_results(res, foo_count)

    print "Checking larger number of results"
    s.send('rget %s %s %d %d %d\r\n' % ('a', gen_key('goo', 0), 0, 1, max_results))
    res = get_results(s)
    check_results(res, foo_count + fop_count)

    print "Checking simple paging"
    page_size = 13
    s.send('rget %s %s %d %d %d\r\n' % ('a', gen_key('goo', 0), 0, 1, page_size))
    res = get_results(s)
    check_results(res, page_size)

    print "Checking contiguous paging"
    page_size = 13
    res = []
    from_key = 'a'
    while True:
        s.send('rget %s %s %d %d %d\r\n' % (from_key, gen_key('fop', 0), 1, 1, page_size))
        cur_res = get_results(s)
        res.extend(cur_res)
        if len(res) < foo_count:
            check_results(cur_res, page_size)
            from_key = cur_res[-1]['key']
        else:
            check_results(cur_res, len(cur_res))
            break
    check_results(res, foo_count)

    for i in range(1,len(res)):
        kv = res[i]
        expected_key = gen_key('foo', i)
        expected_value = gen_value('foo', i)
        if kv['key'] != expected_key:
            raise ValueError("received wrong key (expected: '%s', got: '%s')" % (expected_key, kv['key']))
        if kv['value'] != expected_value:
            raise ValueError("received wrong value (expected: '%s', got: '%s')" % (expected_key, kv['value']))

    print "Checking empty results being returned when no keys match"
    s.send('rget %s %s %d %d %d\r\n' % ('a', 'b', 0, 0, max_results))
    res = get_results(s)
    check_results(res, 0)

    print "Checking that expired values are not returned"
    time.sleep(1)
    s.send('rget z0 z2 1 1 100\r\n')
    res = get_results(s)
    check_results(res, 0)

    s.close()

if __name__ == "__main__":
    op = make_option_parser()
    auto_server_test_main(test_function, op.parse(sys.argv))

<|endoftext|>


<commit_before>#  Copyright 2008-2011 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import sys

from robot import utils

from .highlighting import Highlighter, NoHighlighting
from .loggerhelper import IsLogged

class CommandLineMonitor(object):

    def __init__(self, width=78, colors='AUTO', stdout=None, stderr=None):
        self._writer = CommandLineWriter(width, colors, stdout, stderr)
        self._is_logged = IsLogged('WARN')
        self._started = False
        self._started_keywords = 0
        self._running_test = False

    def start_suite(self, suite):
        if not self._started:
            self._writer.separator('=')
            self._started = True
        self._writer.info(suite.longname, suite.doc, start_suite=True)
        self._writer.separator('=')

    def end_suite(self, suite):
        self._writer.info(suite.longname, suite.doc)
        self._writer.status(suite.status)
        self._writer.message(suite.get_full_message())
        self._writer.separator('=')

    def start_test(self, test):
        self._writer.info(test.name, test.doc)
        self._running_test = True

    def end_test(self, test):
        self._writer.status(test.status, clear=True)
        self._writer.message(test.message)
        self._writer.separator('-')
        self._running_test = False

    def start_keyword(self, kw):
        self._started_keywords += 1

    def end_keyword(self, kw):
        self._started_keywords -= 1
        if self._running_test and not self._started_keywords:
            self._writer.keyword_marker(kw)

    def message(self, msg):
        if self._is_logged(msg.level):
            self._writer.error(msg.message, msg.level, clear=self._running_test)

    def output_file(self, name, path):
        self._writer.output(name, path)

class CommandLineWriter(object):
    _status_length = len('| PASS |')

    def __init__(self, width=78, colors='AUTO', stdout=None, stderr=None):
        self._width = width
        self._stdout = stdout or sys.__stdout__
        self._stderr = stderr or sys.__stderr__
        self._highlighter = StatusHighlighter(colors, self._stdout, self._stderr)
        self._keyword_marker_count = 0

    def info(self, name, doc, start_suite=False):
        width, separator = self._get_info_width_and_separator(start_suite)
        self._info = self._get_info(name, doc, width) + separator
        self._write(self._info, newline=False)
        self._keyword_marker_count = 0

    def _get_info_width_and_separator(self, start_suite):
        if start_suite:
            return self._width, '\n'
        return self._width - self._status_length - 1, ' '

    def _get_info(self, name, doc, width):
        if utils.get_console_length(name) > width:
            return utils.pad_console_length(name, width)
        info = name if not doc else '%s :: %s' % (name, doc.splitlines()[0])
        return utils.pad_console_length(info, width)

    def separator(self, char):
        self._write(char * self._width)

    def status(self, status, clear=False):
        if clear and isatty(self._stdout):
            self._clear_status()
        self._highlight('| ', status, ' |')

    def _clear_status(self):
        self._clear_info_line()
        self._rewrite_info()

    def _clear_info_line(self):
        self._overwrite(' ' * self._width)
        self._overwrite('')

    def _overwrite(self, text):
        self._write('\r' + text, newline=False)

    def _rewrite_info(self):
        self._write(self._info, newline=False)
        self._keyword_marker_count = 0

    def message(self, message):
        if message:
            self._write(message.strip())

    def keyword_marker(self, kw):
        if not isatty(self._stdout):
            return
        if self._keyword_marker_count == self._status_length:
            self._clear_status()
        marker, color = ('.', 'green') if kw.status == 'PASS' else ('F', 'red')
        self._highlighter.highlight(marker, color, self._stdout)
        self._keyword_marker_count += 1

    def error(self, message, level, clear=False):
        if clear and isatty(self._stdout):
            self._clear_info_line()
        self._highlight('[ ', level, ' ] ' + message, error=True)
        if clear and isatty(self._stdout):
            self._rewrite_info()

    def output(self, name, path):
        self._write('%-8s %s' % (name+':', path))

    def _write(self, text, newline=True, error=False):
        stream = self._stdout if not error else self._stderr
        if newline:
            text += '\n'
        stream.write(utils.encode_output(text).replace('\t', ' '*8))
        stream.flush()

    def _highlight(self, before, status, after, newline=True, error=False):
        stream = self._stdout if not error else self._stderr
        self._write(before, newline=False, error=error)
        self._highlighter.highlight_status(status, stream)
        self._write(after, newline=newline, error=error)

class StatusHighlighter(object):

    def __init__(self, colors, *streams):
        self._highlighters = dict((stream, self._get_highlighter(stream, colors))
                                  for stream in streams)

    def _get_highlighter(self, stream, colors):
        auto = isatty(stream)
        enable = {'AUTO': auto,
                  'ON': True,
                  'FORCE': True,   # compatibility with 2.5.5 and earlier
                  'OFF': False}.get(colors.upper(), auto)
        return Highlighter(stream) if enable else NoHighlighting(stream)

    def highlight_status(self, status, stream):
        highlighter = self._start_status_highlighting(status, stream)
        stream.write(status)
        highlighter.reset()

    def _start_status_highlighting(self, status, stream):
        highlighter = self._highlighters[stream]
        {'PASS': highlighter.green,
         'FAIL': highlighter.red,
         'ERROR': highlighter.red,
         'WARN': highlighter.yellow}[status]()
        return highlighter

    def highlight(self, text, color, stream):
        highlighter = self._highlighters[stream]
        getattr(highlighter, color)()
        stream.write(text)
        stream.flush()
        highlighter.reset()

def isatty(stream):
    return hasattr(stream, 'isatty') and stream.isatty()
<commit_msg>CommandLineMonitor: don't expand tabs. they'll expand themselves.<commit_after>#  Copyright 2008-2011 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import sys

from robot import utils

from .highlighting import Highlighter, NoHighlighting
from .loggerhelper import IsLogged

class CommandLineMonitor(object):

    def __init__(self, width=78, colors='AUTO', stdout=None, stderr=None):
        self._writer = CommandLineWriter(width, colors, stdout, stderr)
        self._is_logged = IsLogged('WARN')
        self._started = False
        self._started_keywords = 0
        self._running_test = False

    def start_suite(self, suite):
        if not self._started:
            self._writer.separator('=')
            self._started = True
        self._writer.info(suite.longname, suite.doc, start_suite=True)
        self._writer.separator('=')

    def end_suite(self, suite):
        self._writer.info(suite.longname, suite.doc)
        self._writer.status(suite.status)
        self._writer.message(suite.get_full_message())
        self._writer.separator('=')

    def start_test(self, test):
        self._writer.info(test.name, test.doc)
        self._running_test = True

    def end_test(self, test):
        self._writer.status(test.status, clear=True)
        self._writer.message(test.message)
        self._writer.separator('-')
        self._running_test = False

    def start_keyword(self, kw):
        self._started_keywords += 1

    def end_keyword(self, kw):
        self._started_keywords -= 1
        if self._running_test and not self._started_keywords:
            self._writer.keyword_marker(kw)

    def message(self, msg):
        if self._is_logged(msg.level):
            self._writer.error(msg.message, msg.level, clear=self._running_test)

    def output_file(self, name, path):
        self._writer.output(name, path)

class CommandLineWriter(object):
    _status_length = len('| PASS |')

    def __init__(self, width=78, colors='AUTO', stdout=None, stderr=None):
        self._width = width
        self._stdout = stdout or sys.__stdout__
        self._stderr = stderr or sys.__stderr__
        self._highlighter = StatusHighlighter(colors, self._stdout, self._stderr)
        self._keyword_marker_count = 0

    def info(self, name, doc, start_suite=False):
        width, separator = self._get_info_width_and_separator(start_suite)
        self._info = self._get_info(name, doc, width) + separator
        self._write(self._info, newline=False)
        self._keyword_marker_count = 0

    def _get_info_width_and_separator(self, start_suite):
        if start_suite:
            return self._width, '\n'
        return self._width - self._status_length - 1, ' '

    def _get_info(self, name, doc, width):
        if utils.get_console_length(name) > width:
            return utils.pad_console_length(name, width)
        info = name if not doc else '%s :: %s' % (name, doc.splitlines()[0])
        return utils.pad_console_length(info, width)

    def separator(self, char):
        self._write(char * self._width)

    def status(self, status, clear=False):
        if clear and isatty(self._stdout):
            self._clear_status()
        self._highlight('| ', status, ' |')

    def _clear_status(self):
        self._clear_info_line()
        self._rewrite_info()

    def _clear_info_line(self):
        self._overwrite(' ' * self._width)
        self._overwrite('')

    def _overwrite(self, text):
        self._write('\r' + text, newline=False)

    def _rewrite_info(self):
        self._write(self._info, newline=False)
        self._keyword_marker_count = 0

    def message(self, message):
        if message:
            self._write(message.strip())

    def keyword_marker(self, kw):
        if not isatty(self._stdout):
            return
        if self._keyword_marker_count == self._status_length:
            self._clear_status()
        marker, color = ('.', 'green') if kw.status == 'PASS' else ('F', 'red')
        self._highlighter.highlight(marker, color, self._stdout)
        self._keyword_marker_count += 1

    def error(self, message, level, clear=False):
        if clear and isatty(self._stdout):
            self._clear_info_line()
        self._highlight('[ ', level, ' ] ' + message, error=True)
        if clear and isatty(self._stdout):
            self._rewrite_info()

    def output(self, name, path):
        self._write('%-8s %s' % (name+':', path))

    def _write(self, text, newline=True, error=False):
        stream = self._stdout if not error else self._stderr
        if newline:
            text += '\n'
        stream.write(utils.encode_output(text))
        stream.flush()

    def _highlight(self, before, status, after, newline=True, error=False):
        stream = self._stdout if not error else self._stderr
        self._write(before, newline=False, error=error)
        self._highlighter.highlight_status(status, stream)
        self._write(after, newline=newline, error=error)

class StatusHighlighter(object):

    def __init__(self, colors, *streams):
        self._highlighters = dict((stream, self._get_highlighter(stream, colors))
                                  for stream in streams)

    def _get_highlighter(self, stream, colors):
        auto = isatty(stream)
        enable = {'AUTO': auto,
                  'ON': True,
                  'FORCE': True,   # compatibility with 2.5.5 and earlier
                  'OFF': False}.get(colors.upper(), auto)
        return Highlighter(stream) if enable else NoHighlighting(stream)

    def highlight_status(self, status, stream):
        highlighter = self._start_status_highlighting(status, stream)
        stream.write(status)
        highlighter.reset()

    def _start_status_highlighting(self, status, stream):
        highlighter = self._highlighters[stream]
        {'PASS': highlighter.green,
         'FAIL': highlighter.red,
         'ERROR': highlighter.red,
         'WARN': highlighter.yellow}[status]()
        return highlighter

    def highlight(self, text, color, stream):
        highlighter = self._highlighters[stream]
        getattr(highlighter, color)()
        stream.write(text)
        stream.flush()
        highlighter.reset()

def isatty(stream):
    return hasattr(stream, 'isatty') and stream.isatty()
<|endoftext|>