<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function
"""

get_image_size.py
====================

    :Name:        get_image_size
    :Purpose:     extract image dimensions given a file path

    :Author:      Paulo Scardine (based on code from Emmanuel VAÏSSE)

    :Created:     26/09/2013
    :Copyright:   (c) Paulo Scardine 2013
    :Licence:     MIT

"""
import collections
import json
import os
import struct

FILE_UNKNOWN = "Sorry, don't know how to get size for this file."

class UnknownImageFormat(Exception):
    pass

types = collections.OrderedDict()
BMP = types['BMP'] = 'BMP'
GIF = types['GIF'] = 'GIF'
ICO = types['ICO'] = 'ICO'
JPEG = types['JPEG'] = 'JPEG'
PNG = types['PNG'] = 'PNG'
TIFF = types['TIFF'] = 'TIFF'

image_fields = ['path', 'type', 'file_size', 'width', 'height']

class Image(collections.namedtuple('Image', image_fields)):

    def to_str_row(self):
        return ("%d\t%d\t%d\t%s\t%s" % (
            self.width,
            self.height,
            self.file_size,
            self.type,
            self.path.replace('\t', '\\t'),
        ))

    def to_str_row_verbose(self):
        return ("%d\t%d\t%d\t%s\t%s\t##%s" % (
            self.width,
            self.height,
            self.file_size,
            self.type,
            self.path.replace('\t', '\\t'),
            self))

    def to_str_json(self, indent=None):
        return json.dumps(self._asdict(), indent=indent)

def get_image_size(file_path):
    """
    Return (width, height) for a given img file content - no external
    dependencies except the os and struct builtin modules
    """
    img = get_image_metadata(file_path)
    return (img.width, img.height)

def get_image_metadata(file_path):
    """
    Return an `Image` object for a given img file content - no external
    dependencies except the os and struct builtin modules

    Args:
        file_path (str): path to an image file

    Returns:
        Image: (path, type, file_size, width, height)
    """
    size = os.path.getsize(file_path)

    # be explicit with open arguments - we need binary mode
    with open(file_path, "rb") as input:
        height = -1
        width = -1
        data = input.read(26)
        msg = " raised while trying to decode as JPEG."

        if (size >= 10) and data[:6] in ('GIF87a', 'GIF89a'):
            # GIFs
            imgtype = GIF
            w, h = struct.unpack("<HH", data[6:10])
            width = int(w)
            height = int(h)
        elif ((size >= 24) and data.startswith(b'\211PNG\r\n\032\n')
              and (data[12:16] == 'IHDR')):
            # PNGs
            imgtype = PNG
            w, h = struct.unpack(">LL", data[16:24])
            width = int(w)
            height = int(h)
        elif (size >= 16) and data.startswith(b'\211PNG\r\n\032\n'):
            # older PNGs
            imgtype = PNG
            w, h = struct.unpack(">LL", data[8:16])
            width = int(w)
            height = int(h)
        elif (size >= 2) and data.startswith(b'\377\330'):
            # JPEG
            imgtype = JPEG
            input.seek(0)
            input.read(2)
            b = input.read(1)
            try:
                while (b and ord(b) != 0xDA):
                    while (ord(b) != 0xFF):
                        b = input.read(1)
                    while (ord(b) == 0xFF):
                        b = input.read(1)
                    if (ord(b) >= 0xC0 and ord(b) <= 0xC3):
                        input.read(3)
                        h, w = struct.unpack(">HH", input.read(4))
                        break
                    else:
                        input.read(
                            int(struct.unpack(">H", input.read(2))[0]) - 2)
                    b = input.read(1)
                width = int(w)
                height = int(h)
            except struct.error:
                raise UnknownImageFormat("StructError" + msg)
            except ValueError:
                raise UnknownImageFormat("ValueError" + msg)
            except Exception as e:
                raise UnknownImageFormat(e.__class__.__name__ + msg)
        elif (size >= 26) and data.startswith(b'BM'):
            # BMP
            imgtype = 'BMP'
            headersize = struct.unpack("<I", data[14:18])[0]
            if headersize == 12:
                w, h = struct.unpack("<HH", data[18:22])
                width = int(w)
                height = int(h)
            elif headersize >= 40:
                w, h = struct.unpack("<ii", data[18:26])
                width = int(w)
                # as h is negative when stored upside down
                height = abs(int(h))
            else:
                raise UnknownImageFormat(
                    "Unkown DIB header size:" +
                    str(headersize))
        elif (size >= 8) and data[:4] in ("II\052\000", "MM\000\052"):
            # Standard TIFF, big- or little-endian
            # BigTIFF and other different but TIFF-like formats are not
            # supported currently
            imgtype = TIFF
            byteOrder = data[:2]
            boChar = ">" if byteOrder == "MM" else "<"
            # maps TIFF type id to size (in bytes)
            # and python format char for struct
            tiffTypes = {
                1: (1, boChar + "B"),  # BYTE
                2: (1, boChar + "c"),  # ASCII
                3: (2, boChar + "H"),  # SHORT
                4: (4, boChar + "L"),  # LONG
                5: (8, boChar + "LL"),  # RATIONAL
                6: (1, boChar + "b"),  # SBYTE
                7: (1, boChar + "c"),  # UNDEFINED
                8: (2, boChar + "h"),  # SSHORT
                9: (4, boChar + "l"),  # SLONG
                10: (8, boChar + "ll"),  # SRATIONAL
                11: (4, boChar + "f"),  # FLOAT
                12: (8, boChar + "d")   # DOUBLE
            }
            ifdOffset = struct.unpack(boChar + "L", data[4:8])[0]
            try:
                countSize = 2
                input.seek(ifdOffset)
                ec = input.read(countSize)
                ifdEntryCount = struct.unpack(boChar + "H", ec)[0]
                # 2 bytes: TagId + 2 bytes: type + 4 bytes: count of values + 4
                # bytes: value offset
                ifdEntrySize = 12
                for i in range(ifdEntryCount):
                    entryOffset = ifdOffset + countSize + i * ifdEntrySize
                    input.seek(entryOffset)
                    tag = input.read(2)
                    tag = struct.unpack(boChar + "H", tag)[0]
                    if(tag == 256 or tag == 257):
                        # if type indicates that value fits into 4 bytes, value
                        # offset is not an offset but value itself
                        type = input.read(2)
                        type = struct.unpack(boChar + "H", type)[0]
                        if type not in tiffTypes:
                            raise UnknownImageFormat(
                                "Unkown TIFF field type:" +
                                str(type))
                        typeSize = tiffTypes[type][0]
                        typeChar = tiffTypes[type][1]
                        input.seek(entryOffset + 8)
                        value = input.read(typeSize)
                        value = int(struct.unpack(typeChar, value)[0])
                        if tag == 256:
                            width = value
                        else:
                            height = value
                    if width > -1 and height > -1:
                        break
            except Exception as e:
                raise UnknownImageFormat(str(e))
        elif size >= 2:
                # see http://en.wikipedia.org/wiki/ICO_(file_format)
            imgtype = 'ICO'
            input.seek(0)
            reserved = input.read(2)
            if 0 != struct.unpack("<H", reserved)[0]:
                raise UnknownImageFormat(FILE_UNKNOWN)
            format = input.read(2)
            assert 1 == struct.unpack("<H", format)[0]
            num = input.read(2)
            num = struct.unpack("<H", num)[0]
            if num > 1:
                import warnings
                warnings.warn("ICO File contains more than one image")
            # http://msdn.microsoft.com/en-us/library/ms997538.aspx
            w = input.read(1)
            h = input.read(1)
            width = ord(w)
            height = ord(h)
        else:
            raise UnknownImageFormat(FILE_UNKNOWN)

    return Image(path=file_path,
                 type=imgtype,
                 file_size=size,
                 width=width,
                 height=height)

import unittest

class Test_get_image_size(unittest.TestCase):
    data = [{
        'path': 'lookmanodeps.png',
        'width': 251,
        'height': 208,
        'file_size': 22228,
        'type': 'PNG'}]

    def setUp(self):
        pass

    def test_get_image_metadata(self):
        img = self.data[0]
        output = get_image_metadata(img['path'])
        self.assertTrue(output)
        self.assertEqual(output.path, img['path'])
        self.assertEqual(output.width, img['width'])
        self.assertEqual(output.height, img['height'])
        self.assertEqual(output.type, img['type'])
        self.assertEqual(output.file_size, img['file_size'])
        for field in image_fields:
            self.assertEqual(getattr(output, field), img[field])

    def test_get_image_metadata__ENOENT_OSError(self):
        with self.assertRaises(OSError):
            get_image_metadata('THIS_DOES_NOT_EXIST')

    def test_get_image_metadata__not_an_image_UnknownImageFormat(self):
        with self.assertRaises(UnknownImageFormat):
            get_image_metadata('README.rst')

    def test_get_image_size(self):
        img = self.data[0]
        output = get_image_size(img['path'])
        self.assertTrue(output)
        self.assertEqual(output,
                         (img['width'],
                          img['height']))

    def tearDown(self):
        pass

def main(argv=None):
    """
    Print image metadata fields for the given file path.

    Keyword Arguments:
        argv (list): commandline arguments (e.g. sys.argv[1:])
    Returns:
        int: zero for OK
    """
    import logging
    import optparse
    import sys

    prs = optparse.OptionParser(
        usage="%prog [-v|--verbose] [--json|--json-indent] <path0> [<pathN>]",
        description="Print metadata for the given image paths "
                    "(without image library bindings).")

    prs.add_option('--json',
                   dest='json',
                   action='store_true')
    prs.add_option('--json-indent',
                   dest='json_indent',
                   action='store_true')

    prs.add_option('-v', '--verbose',
                   dest='verbose',
                   action='store_true',)
    prs.add_option('-q', '--quiet',
                   dest='quiet',
                   action='store_true',)
    prs.add_option('-t', '--test',
                   dest='run_tests',
                   action='store_true',)

    argv = list(argv) if argv is not None else sys.argv[1:]
    (opts, args) = prs.parse_args(args=argv)
    loglevel = logging.INFO
    if opts.verbose:
        loglevel = logging.DEBUG
    elif opts.quiet:
        loglevel = logging.ERROR
    logging.basicConfig(level=loglevel)
    log = logging.getLogger()
    log.debug('argv: %r', argv)
    log.debug('opts: %r', opts)
    log.debug('args: %r', args)

    if opts.run_tests:
        import sys
        sys.argv = [sys.argv[0]] + args
        import unittest
        return unittest.main()

    output_func = Image.to_str_row
    if opts.json_indent:
        import functools
        output_func = functools.partial(Image.to_str_json, indent=2)
    elif opts.json:
        output_func = Image.to_str_json
    elif opts.verbose:
        output_func = Image.to_str_row_verbose

    EX_OK = 0
    EX_NOT_OK = 2

    if len(args) < 1:
        prs.print_help()
        print('')
        prs.error("You must specify one or more paths to image files")

    errors = []
    for path_arg in args:
        try:
            img = get_image_metadata(path_arg)
            print(output_func(img))
        except KeyboardInterrupt:
            raise
        except OSError as e:
            log.error((path_arg, e))
            errors.append((path_arg, e))
        except Exception as e:
            log.exception(e)
            errors.append((path_arg, e))
            pass
    if len(errors):
        import pprint
        print("ERRORS", file=sys.stderr)
        print("======", file=sys.stderr)
        print(pprint.pformat(errors, indent=2), file=sys.stderr)
        return EX_NOT_OK
    return EX_OK

if __name__ == "__main__":
    import sys
    sys.exit(main(argv=sys.argv[1:]))
<commit_msg>Fix #8<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function
"""

get_image_size.py
====================

    :Name:        get_image_size
    :Purpose:     extract image dimensions given a file path

    :Author:      Paulo Scardine (based on code from Emmanuel VAÏSSE)

    :Created:     26/09/2013
    :Copyright:   (c) Paulo Scardine 2013
    :Licence:     MIT

"""
import collections
import json
import os
import struct

FILE_UNKNOWN = "Sorry, don't know how to get size for this file."

class UnknownImageFormat(Exception):
    pass

types = collections.OrderedDict()
BMP = types['BMP'] = 'BMP'
GIF = types['GIF'] = 'GIF'
ICO = types['ICO'] = 'ICO'
JPEG = types['JPEG'] = 'JPEG'
PNG = types['PNG'] = 'PNG'
TIFF = types['TIFF'] = 'TIFF'

image_fields = ['path', 'type', 'file_size', 'width', 'height']

class Image(collections.namedtuple('Image', image_fields)):

    def to_str_row(self):
        return ("%d\t%d\t%d\t%s\t%s" % (
            self.width,
            self.height,
            self.file_size,
            self.type,
            self.path.replace('\t', '\\t'),
        ))

    def to_str_row_verbose(self):
        return ("%d\t%d\t%d\t%s\t%s\t##%s" % (
            self.width,
            self.height,
            self.file_size,
            self.type,
            self.path.replace('\t', '\\t'),
            self))

    def to_str_json(self, indent=None):
        return json.dumps(self._asdict(), indent=indent)

def get_image_size(file_path):
    """
    Return (width, height) for a given img file content - no external
    dependencies except the os and struct builtin modules
    """
    img = get_image_metadata(file_path)
    return (img.width, img.height)

def get_image_metadata(file_path):
    """
    Return an `Image` object for a given img file content - no external
    dependencies except the os and struct builtin modules

    Args:
        file_path (str): path to an image file

    Returns:
        Image: (path, type, file_size, width, height)
    """
    size = os.path.getsize(file_path)

    # be explicit with open arguments - we need binary mode
    with open(file_path, "rb") as input:
        height = -1
        width = -1
        data = input.read(26)
        msg = " raised while trying to decode as JPEG."

        if (size >= 10) and data[:6] in ('GIF87a', 'GIF89a'):
            # GIFs
            imgtype = GIF
            w, h = struct.unpack("<HH", data[6:10])
            width = int(w)
            height = int(h)
        elif ((size >= 24) and data.startswith(b'\211PNG\r\n\032\n')
              and (data[12:16] == 'IHDR')):
            # PNGs
            imgtype = PNG
            w, h = struct.unpack(">LL", data[16:24])
            width = int(w)
            height = int(h)
        elif (size >= 16) and data.startswith(b'\211PNG\r\n\032\n'):
            # older PNGs
            imgtype = PNG
            w, h = struct.unpack(">LL", data[8:16])
            width = int(w)
            height = int(h)
        elif (size >= 2) and data.startswith(b'\377\330'):
            # JPEG
            imgtype = JPEG
            input.seek(0)
            input.read(2)
            b = input.read(1)
            try:
                while (b and ord(b) != 0xDA):
                    while (ord(b) != 0xFF):
                        b = input.read(1)
                    while (ord(b) == 0xFF):
                        b = input.read(1)
                    if (ord(b) >= 0xC0 and ord(b) <= 0xC3):
                        input.read(3)
                        h, w = struct.unpack(">HH", input.read(4))
                        break
                    else:
                        input.read(
                            int(struct.unpack(">H", input.read(2))[0]) - 2)
                    b = input.read(1)
                width = int(w)
                height = int(h)
            except struct.error:
                raise UnknownImageFormat("StructError" + msg)
            except ValueError:
                raise UnknownImageFormat("ValueError" + msg)
            except Exception as e:
                raise UnknownImageFormat(e.__class__.__name__ + msg)
        elif (size >= 26) and data.startswith(b'BM'):
            # BMP
            imgtype = 'BMP'
            headersize = struct.unpack("<I", data[14:18])[0]
            if headersize == 12:
                w, h = struct.unpack("<HH", data[18:22])
                width = int(w)
                height = int(h)
            elif headersize >= 40:
                w, h = struct.unpack("<ii", data[18:26])
                width = int(w)
                # as h is negative when stored upside down
                height = abs(int(h))
            else:
                raise UnknownImageFormat(
                    "Unkown DIB header size:" +
                    str(headersize))
        elif (size >= 8) and data[:4] in (b"II\052\000", b"MM\000\052"):
            # Standard TIFF, big- or little-endian
            # BigTIFF and other different but TIFF-like formats are not
            # supported currently
            imgtype = TIFF
            byteOrder = data[:2]
            boChar = ">" if byteOrder == "MM" else "<"
            # maps TIFF type id to size (in bytes)
            # and python format char for struct
            tiffTypes = {
                1: (1, boChar + "B"),  # BYTE
                2: (1, boChar + "c"),  # ASCII
                3: (2, boChar + "H"),  # SHORT
                4: (4, boChar + "L"),  # LONG
                5: (8, boChar + "LL"),  # RATIONAL
                6: (1, boChar + "b"),  # SBYTE
                7: (1, boChar + "c"),  # UNDEFINED
                8: (2, boChar + "h"),  # SSHORT
                9: (4, boChar + "l"),  # SLONG
                10: (8, boChar + "ll"),  # SRATIONAL
                11: (4, boChar + "f"),  # FLOAT
                12: (8, boChar + "d")   # DOUBLE
            }
            ifdOffset = struct.unpack(boChar + "L", data[4:8])[0]
            try:
                countSize = 2
                input.seek(ifdOffset)
                ec = input.read(countSize)
                ifdEntryCount = struct.unpack(boChar + "H", ec)[0]
                # 2 bytes: TagId + 2 bytes: type + 4 bytes: count of values + 4
                # bytes: value offset
                ifdEntrySize = 12
                for i in range(ifdEntryCount):
                    entryOffset = ifdOffset + countSize + i * ifdEntrySize
                    input.seek(entryOffset)
                    tag = input.read(2)
                    tag = struct.unpack(boChar + "H", tag)[0]
                    if(tag == 256 or tag == 257):
                        # if type indicates that value fits into 4 bytes, value
                        # offset is not an offset but value itself
                        type = input.read(2)
                        type = struct.unpack(boChar + "H", type)[0]
                        if type not in tiffTypes:
                            raise UnknownImageFormat(
                                "Unkown TIFF field type:" +
                                str(type))
                        typeSize = tiffTypes[type][0]
                        typeChar = tiffTypes[type][1]
                        input.seek(entryOffset + 8)
                        value = input.read(typeSize)
                        value = int(struct.unpack(typeChar, value)[0])
                        if tag == 256:
                            width = value
                        else:
                            height = value
                    if width > -1 and height > -1:
                        break
            except Exception as e:
                raise UnknownImageFormat(str(e))
        elif size >= 2:
                # see http://en.wikipedia.org/wiki/ICO_(file_format)
            imgtype = 'ICO'
            input.seek(0)
            reserved = input.read(2)
            if 0 != struct.unpack("<H", reserved)[0]:
                raise UnknownImageFormat(FILE_UNKNOWN)
            format = input.read(2)
            assert 1 == struct.unpack("<H", format)[0]
            num = input.read(2)
            num = struct.unpack("<H", num)[0]
            if num > 1:
                import warnings
                warnings.warn("ICO File contains more than one image")
            # http://msdn.microsoft.com/en-us/library/ms997538.aspx
            w = input.read(1)
            h = input.read(1)
            width = ord(w)
            height = ord(h)
        else:
            raise UnknownImageFormat(FILE_UNKNOWN)

    return Image(path=file_path,
                 type=imgtype,
                 file_size=size,
                 width=width,
                 height=height)

import unittest

class Test_get_image_size(unittest.TestCase):
    data = [{
        'path': 'lookmanodeps.png',
        'width': 251,
        'height': 208,
        'file_size': 22228,
        'type': 'PNG'}]

    def setUp(self):
        pass

    def test_get_image_metadata(self):
        img = self.data[0]
        output = get_image_metadata(img['path'])
        self.assertTrue(output)
        self.assertEqual(output.path, img['path'])
        self.assertEqual(output.width, img['width'])
        self.assertEqual(output.height, img['height'])
        self.assertEqual(output.type, img['type'])
        self.assertEqual(output.file_size, img['file_size'])
        for field in image_fields:
            self.assertEqual(getattr(output, field), img[field])

    def test_get_image_metadata__ENOENT_OSError(self):
        with self.assertRaises(OSError):
            get_image_metadata('THIS_DOES_NOT_EXIST')

    def test_get_image_metadata__not_an_image_UnknownImageFormat(self):
        with self.assertRaises(UnknownImageFormat):
            get_image_metadata('README.rst')

    def test_get_image_size(self):
        img = self.data[0]
        output = get_image_size(img['path'])
        self.assertTrue(output)
        self.assertEqual(output,
                         (img['width'],
                          img['height']))

    def tearDown(self):
        pass

def main(argv=None):
    """
    Print image metadata fields for the given file path.

    Keyword Arguments:
        argv (list): commandline arguments (e.g. sys.argv[1:])
    Returns:
        int: zero for OK
    """
    import logging
    import optparse
    import sys

    prs = optparse.OptionParser(
        usage="%prog [-v|--verbose] [--json|--json-indent] <path0> [<pathN>]",
        description="Print metadata for the given image paths "
                    "(without image library bindings).")

    prs.add_option('--json',
                   dest='json',
                   action='store_true')
    prs.add_option('--json-indent',
                   dest='json_indent',
                   action='store_true')

    prs.add_option('-v', '--verbose',
                   dest='verbose',
                   action='store_true',)
    prs.add_option('-q', '--quiet',
                   dest='quiet',
                   action='store_true',)
    prs.add_option('-t', '--test',
                   dest='run_tests',
                   action='store_true',)

    argv = list(argv) if argv is not None else sys.argv[1:]
    (opts, args) = prs.parse_args(args=argv)
    loglevel = logging.INFO
    if opts.verbose:
        loglevel = logging.DEBUG
    elif opts.quiet:
        loglevel = logging.ERROR
    logging.basicConfig(level=loglevel)
    log = logging.getLogger()
    log.debug('argv: %r', argv)
    log.debug('opts: %r', opts)
    log.debug('args: %r', args)

    if opts.run_tests:
        import sys
        sys.argv = [sys.argv[0]] + args
        import unittest
        return unittest.main()

    output_func = Image.to_str_row
    if opts.json_indent:
        import functools
        output_func = functools.partial(Image.to_str_json, indent=2)
    elif opts.json:
        output_func = Image.to_str_json
    elif opts.verbose:
        output_func = Image.to_str_row_verbose

    EX_OK = 0
    EX_NOT_OK = 2

    if len(args) < 1:
        prs.print_help()
        print('')
        prs.error("You must specify one or more paths to image files")

    errors = []
    for path_arg in args:
        try:
            img = get_image_metadata(path_arg)
            print(output_func(img))
        except KeyboardInterrupt:
            raise
        except OSError as e:
            log.error((path_arg, e))
            errors.append((path_arg, e))
        except Exception as e:
            log.exception(e)
            errors.append((path_arg, e))
            pass
    if len(errors):
        import pprint
        print("ERRORS", file=sys.stderr)
        print("======", file=sys.stderr)
        print(pprint.pformat(errors, indent=2), file=sys.stderr)
        return EX_NOT_OK
    return EX_OK

if __name__ == "__main__":
    import sys
    sys.exit(main(argv=sys.argv[1:]))
<|endoftext|>


<commit_before>"""
We should have only celery tasks in this module.
To know how to call one of these tasks, see
http://docs.celeryproject.org/en/latest/userguide/calling.html#guide-calling
If you want to obtain some cool executions flows (e.g. mapreduce)
have a look at the canvas section in the docs
http://docs.celeryproject.org/en/latest/userguide/canvas.html#guide-canvas
"""
import simplejson as json
import re
import time

from celery import task
from django.conf import settings
from django.core.urlresolvers import reverse

from thclient import TreeherderArtifactCollection, TreeherderRequest

from treeherder.log_parser.artifactbuildercollection import \
    ArtifactBuilderCollection
from treeherder.log_parser.utils import (get_error_search_term,
                                         get_crash_signature,
                                         get_bugs_for_search_term)

from treeherder.etl.oauth_utils import OAuthCredentials

@task(name='parse-log')
def parse_log(project, job_log_url, job_guid, check_errors=False):
    """
    Call ArtifactBuilderCollection on the given job.
    """

    credentials = OAuthCredentials.get_credentials(project)
    req = TreeherderRequest(
        protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
        host=settings.TREEHERDER_REQUEST_HOST,
        project=project,
        oauth_key=credentials.get('consumer_key', None),
        oauth_secret=credentials.get('consumer_secret', None),
    )
    update_endpoint = 'job-log-url/{0}/update_parse_status'.format(job_log_url['id'])

    try:
        log_url = job_log_url['url']
        mozharness_pattern = re.compile(
            '^\d+:\d+:\d+[ ]+(?:DEBUG|INFO|WARNING|ERROR|CRITICAL|FATAL) - [ ]?'
        )

        bugs_cache = {'open': {}, 'closed': {}}
        bug_suggestions = {'open': {}, 'closed': {}}

        # return the resultset with the job id to identify if the UI wants
        # to fetch the whole thing.

        bugscache_uri = '{0}{1}'.format(
            settings.API_HOSTNAME,
            reverse("bugscache-list")
        )

        if log_url:
            # parse a log given its url
            artifact_bc = ArtifactBuilderCollection(
                log_url,
                check_errors=check_errors,
            )
            artifact_bc.parse()

            artifact_list = []
            for name, artifact in artifact_bc.artifacts.items():
                artifact_list.append((job_guid, name, 'json', json.dumps(artifact)))

            if check_errors:
                all_errors = artifact_bc.artifacts.get(
                    'Structured Log', {}
                    ).get(
                        'step_data', {}
                        ).get(
                            'all_errors', [] )

                for err in all_errors:
                    # remove the mozharness prefix
                    clean_line = mozharness_pattern.sub('', err['line']).strip()
                    # get a meaningful search term out of the error line
                    search_term = get_error_search_term(clean_line)
                    # collect open and closed bugs suggestions
                    for status in ('open', 'closed'):
                        if not search_term:
                            bug_suggestions[status][clean_line] = []
                            continue
                        if search_term not in bugs_cache[status]:
                            # retrieve the list of suggestions from the api
                            bugs_cache[status][search_term] = get_bugs_for_search_term(
                                search_term,
                                status,
                                bugscache_uri
                            )
                            # no suggestions, try to use the crash signature as search term
                            if not bugs_cache[status][search_term]:
                                crash_signature = get_crash_signature(search_term)
                                if crash_signature:
                                    bugs_cache[status][search_term] = get_bugs_for_search_term(
                                        search_term,
                                        status,
                                        bugscache_uri
                                    )
                        bug_suggestions[status][clean_line] = bugs_cache[status][search_term]

            # store the artifacts generated
            tac = TreeherderArtifactCollection()
            for artifact in artifact_list:
                ta = tac.get_artifact({
                    "job_guid": artifact[0],
                    "name": artifact[1],
                    "type": artifact[2],
                    "blob": artifact[3]
                })
                tac.add(ta)

            req.post(tac)

            # send an update to job_log_url
            # the job_log_url status changes
            # from pending to running
            current_timestamp = time.time()
            status = 'parsed'
            req.send(
                update_endpoint,
                method='POST',
                data={
                    'parse_status': status,
                    'parse_timestamp': current_timestamp
                }
            )

    except Exception, e:
        # send an update to job_log_url
        # the job_log_url status changes
        # from pending to running
        current_timestamp = time.time()
        status = 'failed'
        req.send(
            update_endpoint,
            method='POST',
            data={
                'parse_status': status,
                'parse_timestamp': current_timestamp
            }
        )
        # re raise the exception to leave a trace in the log
        raise
<commit_msg>enable automatic retries for the parse-log task<commit_after>"""
We should have only celery tasks in this module.
To know how to call one of these tasks, see
http://docs.celeryproject.org/en/latest/userguide/calling.html#guide-calling
If you want to obtain some cool executions flows (e.g. mapreduce)
have a look at the canvas section in the docs
http://docs.celeryproject.org/en/latest/userguide/canvas.html#guide-canvas
"""
import simplejson as json
import re
import time

from celery import task
from django.conf import settings
from django.core.urlresolvers import reverse

from thclient import TreeherderArtifactCollection, TreeherderRequest

from treeherder.log_parser.artifactbuildercollection import \
    ArtifactBuilderCollection
from treeherder.log_parser.utils import (get_error_search_term,
                                         get_crash_signature,
                                         get_bugs_for_search_term)

from treeherder.etl.oauth_utils import OAuthCredentials

@task(name='parse-log', max_retries=3)
def parse_log(project, job_log_url, job_guid, check_errors=False):
    """
    Call ArtifactBuilderCollection on the given job.
    """
    credentials = OAuthCredentials.get_credentials(project)
    req = TreeherderRequest(
        protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
        host=settings.TREEHERDER_REQUEST_HOST,
        project=project,
        oauth_key=credentials.get('consumer_key', None),
        oauth_secret=credentials.get('consumer_secret', None),
    )
    update_endpoint = 'job-log-url/{0}/update_parse_status'.format(job_log_url['id'])

    try:
        log_url = job_log_url['url']
        mozharness_pattern = re.compile(
            '^\d+:\d+:\d+[ ]+(?:DEBUG|INFO|WARNING|ERROR|CRITICAL|FATAL) - [ ]?'
        )

        bugs_cache = {'open': {}, 'closed': {}}
        bug_suggestions = {'open': {}, 'closed': {}}

        # return the resultset with the job id to identify if the UI wants
        # to fetch the whole thing.

        bugscache_uri = '{0}{1}'.format(
            settings.API_HOSTNAME,
            reverse("bugscache-list")
        )

        if log_url:
            # parse a log given its url
            artifact_bc = ArtifactBuilderCollection(
                log_url,
                check_errors=check_errors,
            )
            artifact_bc.parse()

            artifact_list = []
            for name, artifact in artifact_bc.artifacts.items():
                artifact_list.append((job_guid, name, 'json', json.dumps(artifact)))

            if check_errors:
                all_errors = artifact_bc.artifacts.get(
                    'Structured Log', {}
                    ).get(
                        'step_data', {}
                        ).get(
                            'all_errors', [] )

                for err in all_errors:
                    # remove the mozharness prefix
                    clean_line = mozharness_pattern.sub('', err['line']).strip()
                    # get a meaningful search term out of the error line
                    search_term = get_error_search_term(clean_line)
                    # collect open and closed bugs suggestions
                    for status in ('open', 'closed'):
                        if not search_term:
                            bug_suggestions[status][clean_line] = []
                            continue
                        if search_term not in bugs_cache[status]:
                            # retrieve the list of suggestions from the api
                            bugs_cache[status][search_term] = get_bugs_for_search_term(
                                search_term,
                                status,
                                bugscache_uri
                            )
                            # no suggestions, try to use the crash signature as search term
                            if not bugs_cache[status][search_term]:
                                crash_signature = get_crash_signature(search_term)
                                if crash_signature:
                                    bugs_cache[status][search_term] = get_bugs_for_search_term(
                                        search_term,
                                        status,
                                        bugscache_uri
                                    )
                        bug_suggestions[status][clean_line] = bugs_cache[status][search_term]

            # store the artifacts generated
            tac = TreeherderArtifactCollection()
            for artifact in artifact_list:
                ta = tac.get_artifact({
                    "job_guid": artifact[0],
                    "name": artifact[1],
                    "type": artifact[2],
                    "blob": artifact[3]
                })
                tac.add(ta)

            req.post(tac)

            # send an update to job_log_url
            # the job_log_url status changes
            # from pending to running
            current_timestamp = time.time()
            status = 'parsed'
            req.send(
                update_endpoint,
                method='POST',
                data={
                    'parse_status': status,
                    'parse_timestamp': current_timestamp
                }
            )

    except Exception, e:
        parse_log.retry(exc=e)
        # send an update to job_log_url
        # the job_log_url status changes
        # from pending to running
        current_timestamp = time.time()
        status = 'failed'
        req.send(
            update_endpoint,
            method='POST',
            data={
                'parse_status': status,
                'parse_timestamp': current_timestamp
            }
        )
        # re raise the exception to leave a trace in the log
        raise
<|endoftext|>


<commit_before>from flask import Flask, request, g
from flask.ext.cache import Cache
from flask.ext.compress import Compress
from flask.ext.restless import APIManager
from flask_jwt import JWT
from flask_mail import Mail
from gfsad.exceptions import FieldError
from flask.ext.celery import Celery
from flask_limiter import Limiter, HEADERS
import os
# APIManager.APINAME_FORMAT = 'api.{0}'
# APIManager.BLUEPRINTNAME_FORMAT = '{0}'

class JSONLimiter(Limiter):
    def inject_headers(self, response):
        current_limit = getattr(g, 'view_rate_limit', None)
        if self.enabled and self.headers_enabled and current_limit:
            window_stats = self.limiter.get_window_stats(*current_limit)
            response.headers.add(
                self.header_mapping[HEADERS.LIMIT],
                str(current_limit[0].amount)
            )
            response.headers.add(
                self.header_mapping[HEADERS.REMAINING],
                window_stats[1]
            )
            response.headers.add(
                self.header_mapping[HEADERS.RESET],
                window_stats[0]
            )
        return response

cache = Cache(config={'CACHE_TYPE': 'simple'})
compress = Compress()
limiter = JSONLimiter(headers_enabled=True, global_limits=["1000 per hour", "200 per minute", "5 per second"])
api = APIManager()
jwt = JWT()
celery = Celery()
mail = Mail()

from gfsad.models import db, User

def add_cors_headers(response):
    response.headers['Access-Control-Allow-Origin'] = '*'
    # Do nothing for post, patch, delete etc..
    try:
        method = [e for e in request.url_rule.methods][-1]
    except AttributeError:
        # print "add_cors_headers: Attribute Error - " + str(response)
        return response

    if method == 'PUT' or method == 'PATCH' or method == 'DELETE' or method == 'POST':
        return response

    # set cache max age
    if '/api' in request.url_rule.rule:
        response.headers['Cache-Control'] = 'max-age=120'

    elif '/gee' in request.url_rule.rule:
        response.headers['Cache-Control'] = 'max-age=4000000'

    else:
        response.headers['Cache-Control'] = 'max-age=0'

    return response

def create_app(config='Testing'):
    app = Flask(__name__)

    # Configure the flask app
    app.config.from_object("gfsad.config."+config)

    # initialize all of the extensions
    jwt.init_app(app)
    celery.init_app(app)
    db.init_app(app)
    limiter.init_app(app)
    cache.init_app(app)
    compress.init_app(app)
    mail.init_app(app)
    api.init_app(app, flask_sqlalchemy_db=db)

    # import and register all of the blueprints
    from gfsad.views.auth import auth
    from gfsad.views.gee import gee
    from gfsad.views.aws import aws

    app.register_blueprint(gee)
    app.register_blueprint(aws)
    app.register_blueprint(auth)

    from gfsad.views.api import init_api

    init_api(app)

    # import and init error handlers
    from gfsad.views.errors import init_error_handlers

    init_error_handlers(app)

    # cors headers and cache
    app.after_request(add_cors_headers)

    from gfsad.auth import load_user
    @limiter.request_filter
    def registered():
        """
        Removes limit if user is registered and using a token.
        :return:
        """
        return load_user() != "anonymous"

    return app

if __name__ == "__main__":
    app = create_app()
<commit_msg>add some headers<commit_after>from flask import Flask, request, g
from flask.ext.cache import Cache
from flask.ext.compress import Compress
from flask.ext.restless import APIManager
from flask_jwt import JWT
from flask_mail import Mail
from gfsad.exceptions import FieldError
from flask.ext.celery import Celery
from flask_limiter import Limiter, HEADERS
import os
# APIManager.APINAME_FORMAT = 'api.{0}'
# APIManager.BLUEPRINTNAME_FORMAT = '{0}'

class JSONLimiter(Limiter):
    def inject_headers(self, response):
        current_limit = getattr(g, 'view_rate_limit', None)
        if self.enabled and self.headers_enabled and current_limit:
            window_stats = self.limiter.get_window_stats(*current_limit)
            response.headers.add(
                self.header_mapping[HEADERS.LIMIT],
                str(current_limit[0].amount)
            )
            response.headers.add(
                self.header_mapping[HEADERS.REMAINING],
                window_stats[1]
            )
            response.headers.add(
                self.header_mapping[HEADERS.RESET],
                window_stats[0]
            )
        return response

cache = Cache(config={'CACHE_TYPE': 'simple'})
compress = Compress()
limiter = JSONLimiter(headers_enabled=True, global_limits=["1000 per hour", "200 per minute", "5 per second"])
api = APIManager()
jwt = JWT()
celery = Celery()
mail = Mail()

from gfsad.models import db, User

def add_cors_headers(response):
    response.headers['Access-Control-Allow-Origin'] = '*'
    response.headers['Access-Control-Allow-Headers'] = 'accept, content-type'
    response.headers['Access-Control-Allow-Method'] = 'POST, GET, PATCH, DELETE'
    # Do nothing for post, patch, delete etc..
    try:
        method = [e for e in request.url_rule.methods][-1]
    except AttributeError:
        # print "add_cors_headers: Attribute Error - " + str(response)
        return response

    if method == 'PUT' or method == 'PATCH' or method == 'DELETE' or method == 'POST':
        return response

    # set cache max age
    if '/api' in request.url_rule.rule:
        response.headers['Cache-Control'] = 'max-age=120'

    elif '/gee' in request.url_rule.rule:
        response.headers['Cache-Control'] = 'max-age=4000000'

    else:
        response.headers['Cache-Control'] = 'max-age=0'

    return response

def create_app(config='Testing'):
    app = Flask(__name__)

    # Configure the flask app
    app.config.from_object("gfsad.config."+config)

    # initialize all of the extensions
    jwt.init_app(app)
    celery.init_app(app)
    db.init_app(app)
    limiter.init_app(app)
    cache.init_app(app)
    compress.init_app(app)
    mail.init_app(app)
    api.init_app(app, flask_sqlalchemy_db=db)

    # import and register all of the blueprints
    from gfsad.views.auth import auth
    from gfsad.views.gee import gee
    from gfsad.views.aws import aws

    app.register_blueprint(gee)
    app.register_blueprint(aws)
    app.register_blueprint(auth)

    from gfsad.views.api import init_api

    init_api(app)

    # import and init error handlers
    from gfsad.views.errors import init_error_handlers

    init_error_handlers(app)

    # cors headers and cache
    app.after_request(add_cors_headers)

    from gfsad.auth import load_user
    @limiter.request_filter
    def registered():
        """
        Removes limit if user is registered and using a token.
        :return:
        """
        return load_user() != "anonymous"

    return app

if __name__ == "__main__":
    app = create_app()
<|endoftext|>


<commit_before>import os
import sys
import shutil
import tempfile
import traceback
from os.path import dirname, isdir, join, realpath
from aspen import resources, sockets

CWD = os.getcwd()
FSFIX = os.path.realpath(os.path.join(tempfile.gettempdir(), 'fsfix'))

def convert_path(path):
    """Given a Unix path, convert it for the current platform.
    """
    return os.sep.join(path.split('/'))

def convert_paths(paths):
    """Given a tuple of Unix paths, convert them for the current platform.
    """
    return tuple([convert_path(p) for p in paths])

def mk(*treedef):
    """Given a treedef, build a filesystem fixture in FSFIX.

    treedef is a sequence of strings and tuples. If a string, it is interpreted
    as a path to a directory that should be created. If a tuple, the first
    element is a path to a file, the second is the contents of the file. We do
    it this way to ease cross-platform testing.

    """
    root = FSFIX
    os.mkdir(root)
    for item in treedef:
        if isinstance(item, basestring):
            path = convert_path(item.lstrip('/'))
            path = os.sep.join([root, path])
            os.makedirs(path)
        elif isinstance(item, tuple):
            filepath, contents = item
            path = convert_path(filepath.lstrip('/'))
            path = os.sep.join([root, path])
            parent = dirname(path)
            if not isdir(parent):
                os.makedirs(parent)
            file(path, 'w').write(contents)

def fix(path=''):
    """Given a relative path, return an absolute path under FSFIX.

    The incoming path is in UNIX form (/foo/bar.html). The outgoing path is in
    native form, with symlinks removed.

    """
    path = os.sep.join([FSFIX] + path.split('/'))
    return realpath(path)

def rm():
    """Remove the filesystem fixture at FSFIX.
    """
    if isdir(FSFIX):
        shutil.rmtree(FSFIX)

def teardown_function(function):
    teardown()

def teardown():
    """Standard teardown function.

    - reset the current working directory
    - remove FSFIX = %{tempdir}/fsfix
    - reset Aspen's global state
    - remove '.aspen' from sys.path
    - remove 'foo' from sys.modules
    - clear out sys.path_importer_cache
    - clear out execution.extras

    """
    os.chdir(CWD)
    rm()
    # Reset some process-global caches. Hrm ...
    resources.__cache__ = {}
    sockets.__sockets__ = {}
    sockets.__channels__ = {}
    sys.path_importer_cache = {} # see test_weird.py
    if 'fsfix' in sys.path[0]:
        sys.path = sys.path[1:]
    if 'foo' in sys.modules:
        del sys.modules['foo']
    import aspen.execution
    aspen.execution.clear_changes()

teardown() # start clean

def torndown(func):
    func.teardown = teardown
    return func

def path(*parts):
    """Given relative path parts, convert to absolute path on the filesystem.
    """
    return realpath(join(*parts))
<commit_msg>Nuke unused torndown function<commit_after>import os
import sys
import shutil
import tempfile
import traceback
from os.path import dirname, isdir, join, realpath
from aspen import resources, sockets

CWD = os.getcwd()
FSFIX = os.path.realpath(os.path.join(tempfile.gettempdir(), 'fsfix'))

def convert_path(path):
    """Given a Unix path, convert it for the current platform.
    """
    return os.sep.join(path.split('/'))

def convert_paths(paths):
    """Given a tuple of Unix paths, convert them for the current platform.
    """
    return tuple([convert_path(p) for p in paths])

def mk(*treedef):
    """Given a treedef, build a filesystem fixture in FSFIX.

    treedef is a sequence of strings and tuples. If a string, it is interpreted
    as a path to a directory that should be created. If a tuple, the first
    element is a path to a file, the second is the contents of the file. We do
    it this way to ease cross-platform testing.

    """
    root = FSFIX
    os.mkdir(root)
    for item in treedef:
        if isinstance(item, basestring):
            path = convert_path(item.lstrip('/'))
            path = os.sep.join([root, path])
            os.makedirs(path)
        elif isinstance(item, tuple):
            filepath, contents = item
            path = convert_path(filepath.lstrip('/'))
            path = os.sep.join([root, path])
            parent = dirname(path)
            if not isdir(parent):
                os.makedirs(parent)
            file(path, 'w').write(contents)

def fix(path=''):
    """Given a relative path, return an absolute path under FSFIX.

    The incoming path is in UNIX form (/foo/bar.html). The outgoing path is in
    native form, with symlinks removed.

    """
    path = os.sep.join([FSFIX] + path.split('/'))
    return realpath(path)

def rm():
    """Remove the filesystem fixture at FSFIX.
    """
    if isdir(FSFIX):
        shutil.rmtree(FSFIX)

def teardown_function(function):
    teardown()

def teardown():
    """Standard teardown function.

    - reset the current working directory
    - remove FSFIX = %{tempdir}/fsfix
    - reset Aspen's global state
    - remove '.aspen' from sys.path
    - remove 'foo' from sys.modules
    - clear out sys.path_importer_cache
    - clear out execution.extras

    """
    os.chdir(CWD)
    rm()
    # Reset some process-global caches. Hrm ...
    resources.__cache__ = {}
    sockets.__sockets__ = {}
    sockets.__channels__ = {}
    sys.path_importer_cache = {} # see test_weird.py
    if 'fsfix' in sys.path[0]:
        sys.path = sys.path[1:]
    if 'foo' in sys.modules:
        del sys.modules['foo']
    import aspen.execution
    aspen.execution.clear_changes()

teardown() # start clean

def path(*parts):
    """Given relative path parts, convert to absolute path on the filesystem.
    """
    return realpath(join(*parts))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

import os
import yaml
from fabric.api import task, local, settings, warn_only
from cuisine import file_exists

@task
def up():
    """ Boot instances """

    # Read config file
    cfg = read_ymlfile('openstack.yml')

    # Get absolute path of public key
    key_file = os.path.abspath(os.path.expanduser(cfg['key_file']))

    # Check if file exist
    if not os.path.exists(key_file):
        print "{} doesn't exist"
        exit(1)

    # Get fingerprint
    fingerprint = local('ssh-keygen -l -f {}|awk \'{{print $2}}\''.format(key_file), capture=True)

    # Check if fingerprint exists on the list
    with settings(warn_only=True):
        test = local('nova keypair-list|grep {}'.format(fingerprint))
    print test.return_code

def read_ymlfile(file_name):
    """ Check the status """
    # Read cofiguration file to cfg
    cfg_dir = os.path.dirname(__file__).replace('fabfile','ymlfile')
    cfg_file = '{0}/{1}'.format(cfg_dir, file_name)
    f = open(cfg_file)
    cfg = yaml.safe_load(f)
    f.close()

    return cfg
<commit_msg>created class OpenStack<commit_after>#!/usr/bin/env python
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

import os
import yaml
from fabric.api import task, local, settings, warn_only
from cuisine import file_exists

@task
def up():
    """ Boot instances """

    # call class OpenStack
    op = OpenStack()

    # Check if fingerprint exists on the list
    op.check_key(cfg)

class OpenStack:

    def __init__(self):

        cfg_dir = os.path.dirname(__file__).replace('fabfile','ymlfile')
        cfg_file = '{0}/{1}'.format(cfg_dir, 'openstack.yml')
        f = open(cfg_file)
        cfg = yaml.safe_load(f)
        cfg['key_file'] = os.path.abspath(os.path.expanduser(cfg['key_file']))
        key_fingerprint = \
                local('ssh-keygen -l -f {}|awk \'{{print $2}}\''.format(key_file), capture=True)
        f.close()

    def check_key(self):

        if not os.path.exists(cfg['key_file']):
            print "{} doesn't exist".format(cfg['key_file'])
            exit(1)

        with settings(warn_only=True):
            output = local('nova keypair-list|grep {}'.format(key_fingerprint))
        if not output.return_code == 0:
            print "ERROR: your key is not registered yet."
            exit(1)
        if not output.split()[1] == cfg['key_name']:
            print "your key is already registered with a different name."
            exit(1)

    #def check_image(self):
    #    with settings(warn_only=True):
<|endoftext|>


<commit_before>import json

from django.contrib import admin
import requests

from .models import GistPage

class GistPageAdmin(admin.ModelAdmin):
    fields = (
        'gist_url',
        'site_url',
        'template',
        'style',
        'script',
        # 'sites',
    )

    def save_model(self, request, obj, form, change):
        def process(gist_files_dict):
            for filename, meta in gist_files_dict.items():
                yield meta['filename'], meta['content']

        # http://developer.github.com/v3/gists/#get-a-single-gist
        gist_id = obj.gist_url.strip('/').split('/')[-1]
        endpoint = 'https://api.github.com/gists/'
        response = requests.get('%s%s' % (endpoint, gist_id))
        if response.status_code != 200:
            # TODO validation error
            return

        # clear out old values
        obj.template = ''
        obj.style = ''
        obj.script = ''

        data = json.loads(response.content)
        files = data['files']

        for filename, content in process(files):
            if filename == 'index.html':
                obj.template = content
            extension = filename.rsplit('.', 1)[-1]
            if extension == 'css':
                obj.style += content
            if extension == 'js':
                obj.script += content

        obj.save()

admin.site.register(GistPage, GistPageAdmin)
<commit_msg>hide gist contents in a collapsed state<commit_after>import json

from django.contrib import admin
import requests

from .models import GistPage

class GistPageAdmin(admin.ModelAdmin):
    fieldsets = (
        (None, {
            'fields': (
                'gist_url',
                'site_url',
            )
        }),
        ('Advanced options', {
            'classes': ('collapse', ),
            'fields': (
                'template',
                'style',
                'script',
            )
        })
        # 'sites',
    )

    def save_model(self, request, obj, form, change):
        def process(gist_files_dict):
            for filename, meta in gist_files_dict.items():
                yield meta['filename'], meta['content']

        # http://developer.github.com/v3/gists/#get-a-single-gist
        gist_id = obj.gist_url.strip('/').split('/')[-1]
        endpoint = 'https://api.github.com/gists/'
        response = requests.get('%s%s' % (endpoint, gist_id))
        if response.status_code != 200:
            # TODO validation error
            return

        # clear out old values
        obj.template = ''
        obj.style = ''
        obj.script = ''

        data = json.loads(response.content)
        files = data['files']

        for filename, content in process(files):
            if filename == 'index.html':
                obj.template = content
            extension = filename.rsplit('.', 1)[-1]
            if extension == 'css':
                obj.style += content
            if extension == 'js':
                obj.script += content

        obj.save()

admin.site.register(GistPage, GistPageAdmin)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import os
import tempfile
import shutil

# Constants giving names of file formats
FMT_PLINK_ASSOC = 'plink_assoc' # Treated as a list of locations to use
FMT_GENEMANIA_INTER ='genemania_interaction'
   # 3 columns tab-separated no header:
   #
   # geneid [tab] gene_id [tab] weight

FMT_HOTNET2_EDGE = 'hotnet2_edge' # 3 columns space-separated no header:
                                  # geneid geneid {}
                                  # geneid must be integer
FMT_GENE_LIST='gene_list' # newline separated list of gene ids
FMT_2_COL_GENE_NETWORK='networkx_2_col' # Tab-separated 2 column with
                                        # gene ids from a gene list.
                                        # No header.
FMT_PLINK_4_FUNSEQ='plink_4_funseq' # Tab separated 5 column
                                    # chrom start stop ref alt
                                    #
                                    # chrom is in format "chr2"
FMT_LOCATION_2_GENE_NAME='location_2_gene_name' # Tab separated 4 col No header
                                                # chromosome
                                                # start pos
                                                # end pos
                                                # gene name

FMT_HOTNET2_GENE_INDEX='hotnet2_gene_index' # 2 col space separated
                                            # gene number (used in edge file)
                                            # gene name (from gene list file)

class InputFile(object):
    """Represents a data in a specified format"""
    def __init__(self, file_format, path):
        self.file_format = file_format
        self.path = path
    def __repr__(self):
        return "InputFile('{}','{}')".format(self.file_format, self.path)

def readable_dir(prospective_dir):
    """Function for verifying that something is a readable directory. If not raises an exception, if so, returns prospective_dir

    Taken from stackoverflow question: http://stackoverflow.com/q/11415570/309334
    and not tested
    """
    if not os.path.isdir(prospective_dir):
        raise Exception("readable_dir:{0} is not a valid path".format(prospective_dir))
    if os.access(prospective_dir, os.R_OK):
        return prospective_dir
    else:
        raise Exception("readable_dir:{0} is not a readable dir".format(prospective_dir))

def parsed_command_line():
    """Returns an object that results from parsing the command-line for this program argparse.ArgumentParser(...).parse_ags()
    """
    parser = argparse.ArgumentParser(
        description='Run multiple network snp analysis algorithms');
    parser.add_argument('--plink_assoc_in', type=argparse.FileType('r'),
                        help='Path to a plink association file https://www.cog-genomics.org/plink2/formats#assoc')
    parser.add_argument('--genemania_prot_prot_in', type=argparse.FileType('r'),
                        help='Path to a protein-protein-interaction network in 3-column genemania output format http://pages.genemania.org/data/')
    parser.add_argument('--location_2_gene_name', type=argparse.FileType('r'),
                        help='mapping of locations to gene names. Must be same names as used in network. 4 column tab separated: chromosme start end gene_name')
    parser.add_argument('--output_dir', type=readable_dir,
                        help='The output directory where everything will dump its output')
    return parser.parse_args()

def input_files(parsed_args):
    """Returns a list of input files that were passed on the command line

    parsed_args: the result of parsing the command-line arguments
    """
    files=[]
    if parsed_args.plink_assoc_in:
        files.append(InputFile(FMT_PLINK_ASSOC, parsed_args.plink_assoc_in.name))
    if parsed_args.genemania_prot_prot_in:
        files.append(InputFile(FMT_GENEMANIA_INTER, parsed_args.genemania_prot_prot_in.name))
    if parsed_args.location_2_gene_name:
        files.append(InputFile(FMT_LOCATION_2_GENE_NAME, parsed_args.location_2_gene_name.name))
    return files

def plink_assoc_to_2_col_gene_network(input_path_in_tuple, output_path):
    """Create a new networkx formatted interaction file at output_path"""
    input_path = input_path_in_tuple[0]
    print "Converting "+input_path+" to "+FMT_2_COL_GENE_NETWORK
    pass

def genemania_inter_to_hotnet2_edge(input_path_in_tuple, output_path):
    """Create a new hotnet2_edge formatted file at output_path

    Hotnet2 expects a list of edges in the form
    id [space] id [space] {}
    with no header
    """
    input_path = input_path_in_tuple[0]
    print "Converting "+input_path+" to "+FMT_HOTNET2_EDGE
    pass

def plink_assoc_to_plink_4_funseq(input_path_in_tuple, output_path):
    """Create a new plink_4_funseq formatted file at output_path"""
    input_path = input_path_in_tuple[0]
    print "Converting "+input_path+" to "+FMT_PLINK_4_FUNSEQ
    pass

def plink_assoc_to_gene_list(input_path_in_tuple, output_path):
    """Create a new gene_list formatted file at output_path"""
    input_path = input_path_in_tuple[0]
    print "Converting "+input_path+" to "+FMT_GENE_LIST
    pass

class Conversion(object):
    def __init__(self, input_formats, output_format, function):
        self.input_formats = input_formats
        self.output_format = output_format
        self.function = function

converters = (
    Conversion((FMT_PLINK_ASSOC,),
               FMT_2_COL_GENE_NETWORK,
               plink_assoc_to_2_col_gene_network),
    Conversion((FMT_PLINK_ASSOC,),
               FMT_GENE_LIST,
               plink_assoc_to_gene_list),
    Conversion((FMT_GENEMANIA_INTER,),
               FMT_HOTNET2_EDGE,genemania_inter_to_hotnet2_edge),
    Conversion((FMT_PLINK_ASSOC,),
               FMT_PLINK_4_FUNSEQ, plink_assoc_to_plink_4_funseq)
)

def possible_inputs(starting_input_formats, converters):
    possible = set(starting_input_formats)
    old_size = len(possible)
    while True:
        for c in converters:
            inputs = set(c.input_formats)
            if inputs.issubset(possible):
                possible.add(c.output_format)
        if len(possible) == old_size:
            break
        else:
            old_size = len(possible)
    return possible

def formats(files):
    return [f.file_format for f in files]

def all_inputs(starting_input_files, converters, path_for_created):
    files = starting_input_files
    old_num_formats = len(set(formats(files)))
    while True:
        for c in converters:
            if c.output_format not in formats(files):
               inputs = set(c.input_formats)
               if inputs.issubset(set(formats(files))):
                   new_path = os.path.join(path_for_created, c.output_format)
                   new_file = InputFile(c.output_format, new_path)
                   input_paths = [f.path for f in files if f.file_format
                                  in c.input_formats]
                   c.function(input_paths, new_path)
                   files.append(new_file)
        new_num_formats = len(set(formats(files)))
        if new_num_formats == old_num_formats:
            break
        else:
            old_num_formats = new_num_formats
    return files

def path_for_format(input_files, file_format):
    f = [fil for fil in input_files if fil.file_format == file_format]
    if len(f) > 1:
        raise RuntimeError("Multiple files for a given format not supported")
    elif len(f) == 0:
        return None
    else:
        return f[0].path

class Analyzer(object):
    def  __init__(self):
       pass
    def requires(self):
        """Return an iterable of the file formats required to do the
        analysis"""
        raise NotImplementedError()
    def run_with(self, input_files, output_dir):
        """Run the analysis with the given input files"""
        raise NotImplementedError()
    def can_run_with(self, available_formats):
        """Return true if self.requires() is a subset of available_formats"""
        return set(self.requires()).issubset(set(available_formats))
    def missing(self, available_formats):
        """Return list of missing items"""
        return set(self.requires()) - set(available_formats)

class Hotnet2(Analyzer):
    def requires(self):
        return (FMT_HOTNET2_EDGE,FMT_HOTNET2_GENE_INDEX, FMT_GENE_LIST) #TODO dummy list

class Networkx(Analyzer):
    def requires(self):
        return (FMT_GENE_LIST, FMT_2_COL_GENE_NETWORK, FMT_LOCATION_2_GENE_NAME)

class Funseq2(Analyzer):
    def requires(self):
        return (FMT_PLINK_4_FUNSEQ,)

analyzers = {
    Hotnet2(),
    Networkx(),
    Funseq2()
}

parsed = parsed_command_line()
print ",".join([str(i) for i in input_files(parsed)])
avail = input_files(parsed)
possible = possible_inputs([a.file_format for a in avail], converters)
temp_dir_path = tempfile.mkdtemp()
print "Possible: "+",".join(sorted(list(possible)));
print "Could not run:" + ",".join(a.__class__.__name__+" is missing:"+",".join(a.missing(possible)) for a in analyzers if not a.can_run_with(possible))
print "Could run:" + ",".join(a.__class__.__name__ for a in analyzers if a.can_run_with(possible))
print "Converting inputs"
inputs = all_inputs(avail, converters, temp_dir_path)
print "Files after conversions:",",".join(f.path for f in inputs)

shutil.rmtree(temp_dir_path)
<commit_msg>Improve directory error messages<commit_after>#!/usr/bin/env python

import argparse
import os
import tempfile
import shutil

# Constants giving names of file formats
FMT_PLINK_ASSOC = 'plink_assoc' # Treated as a list of locations to use
FMT_GENEMANIA_INTER ='genemania_interaction'
   # 3 columns tab-separated no header:
   #
   # geneid [tab] gene_id [tab] weight

FMT_HOTNET2_EDGE = 'hotnet2_edge' # 3 columns space-separated no header:
                                  # geneid geneid {}
                                  # geneid must be integer
FMT_GENE_LIST='gene_list' # newline separated list of gene ids
FMT_2_COL_GENE_NETWORK='networkx_2_col' # Tab-separated 2 column with
                                        # gene ids from a gene list.
                                        # No header.
FMT_PLINK_4_FUNSEQ='plink_4_funseq' # Tab separated 5 column
                                    # chrom start stop ref alt
                                    #
                                    # chrom is in format "chr2"
FMT_LOCATION_2_GENE_NAME='location_2_gene_name' # Tab separated 4 col No header
                                                # chromosome
                                                # start pos
                                                # end pos
                                                # gene name

FMT_HOTNET2_GENE_INDEX='hotnet2_gene_index' # 2 col space separated
                                            # gene number (used in edge file)
                                            # gene name (from gene list file)

class InputFile(object):
    """Represents a data in a specified format"""
    def __init__(self, file_format, path):
        self.file_format = file_format
        self.path = path
    def __repr__(self):
        return "InputFile('{}','{}')".format(self.file_format, self.path)

def readable_dir(prospective_dir):
    """Function for verifying that something is a readable directory. If not raises an exception, if so, returns prospective_dir

    Taken from stackoverflow question: http://stackoverflow.com/q/11415570/309334
    and not tested
    """
    if not os.path.isdir(prospective_dir):
        raise Exception("{0} is not a pre-existing readable directory path".format(prospective_dir))
    if os.access(prospective_dir, os.R_OK):
        return prospective_dir
    else:
        raise Exception("Can't read from {0}".format(prospective_dir))

def parsed_command_line():
    """Returns an object that results from parsing the command-line for this program argparse.ArgumentParser(...).parse_ags()
    """
    parser = argparse.ArgumentParser(
        description='Run multiple network snp analysis algorithms');
    parser.add_argument('--plink_assoc_in', type=argparse.FileType('r'),
                        help='Path to a plink association file https://www.cog-genomics.org/plink2/formats#assoc')
    parser.add_argument('--genemania_prot_prot_in', type=argparse.FileType('r'),
                        help='Path to a protein-protein-interaction network in 3-column genemania output format http://pages.genemania.org/data/')
    parser.add_argument('--location_2_gene_name', type=argparse.FileType('r'),
                        help='mapping of locations to gene names. Must be same names as used in network. 4 column tab separated: chromosme start end gene_name')
    parser.add_argument('--output_dir', type=readable_dir,
                        help='The output directory where everything will dump its output')
    return parser.parse_args()

def input_files(parsed_args):
    """Returns a list of input files that were passed on the command line

    parsed_args: the result of parsing the command-line arguments
    """
    files=[]
    if parsed_args.plink_assoc_in:
        files.append(InputFile(FMT_PLINK_ASSOC, parsed_args.plink_assoc_in.name))
    if parsed_args.genemania_prot_prot_in:
        files.append(InputFile(FMT_GENEMANIA_INTER, parsed_args.genemania_prot_prot_in.name))
    if parsed_args.location_2_gene_name:
        files.append(InputFile(FMT_LOCATION_2_GENE_NAME, parsed_args.location_2_gene_name.name))
    return files

def plink_assoc_to_2_col_gene_network(input_path_in_tuple, output_path):
    """Create a new networkx formatted interaction file at output_path"""
    input_path = input_path_in_tuple[0]
    print "Converting "+input_path+" to "+FMT_2_COL_GENE_NETWORK
    pass

def genemania_inter_to_hotnet2_edge(input_path_in_tuple, output_path):
    """Create a new hotnet2_edge formatted file at output_path

    Hotnet2 expects a list of edges in the form
    id [space] id [space] {}
    with no header
    """
    input_path = input_path_in_tuple[0]
    print "Converting "+input_path+" to "+FMT_HOTNET2_EDGE
    pass

def plink_assoc_to_plink_4_funseq(input_path_in_tuple, output_path):
    """Create a new plink_4_funseq formatted file at output_path"""
    input_path = input_path_in_tuple[0]
    print "Converting "+input_path+" to "+FMT_PLINK_4_FUNSEQ
    pass

def plink_assoc_to_gene_list(input_path_in_tuple, output_path):
    """Create a new gene_list formatted file at output_path"""
    input_path = input_path_in_tuple[0]
    print "Converting "+input_path+" to "+FMT_GENE_LIST
    pass

class Conversion(object):
    def __init__(self, input_formats, output_format, function):
        self.input_formats = input_formats
        self.output_format = output_format
        self.function = function

converters = (
    Conversion((FMT_PLINK_ASSOC,),
               FMT_2_COL_GENE_NETWORK,
               plink_assoc_to_2_col_gene_network),
    Conversion((FMT_PLINK_ASSOC,),
               FMT_GENE_LIST,
               plink_assoc_to_gene_list),
    Conversion((FMT_GENEMANIA_INTER,),
               FMT_HOTNET2_EDGE,genemania_inter_to_hotnet2_edge),
    Conversion((FMT_PLINK_ASSOC,),
               FMT_PLINK_4_FUNSEQ, plink_assoc_to_plink_4_funseq)
)

def possible_inputs(starting_input_formats, converters):
    possible = set(starting_input_formats)
    old_size = len(possible)
    while True:
        for c in converters:
            inputs = set(c.input_formats)
            if inputs.issubset(possible):
                possible.add(c.output_format)
        if len(possible) == old_size:
            break
        else:
            old_size = len(possible)
    return possible

def formats(files):
    return [f.file_format for f in files]

def all_inputs(starting_input_files, converters, path_for_created):
    files = starting_input_files
    old_num_formats = len(set(formats(files)))
    while True:
        for c in converters:
            if c.output_format not in formats(files):
               inputs = set(c.input_formats)
               if inputs.issubset(set(formats(files))):
                   new_path = os.path.join(path_for_created, c.output_format)
                   new_file = InputFile(c.output_format, new_path)
                   input_paths = [f.path for f in files if f.file_format
                                  in c.input_formats]
                   c.function(input_paths, new_path)
                   files.append(new_file)
        new_num_formats = len(set(formats(files)))
        if new_num_formats == old_num_formats:
            break
        else:
            old_num_formats = new_num_formats
    return files

def path_for_format(input_files, file_format):
    f = [fil for fil in input_files if fil.file_format == file_format]
    if len(f) > 1:
        raise RuntimeError("Multiple files for a given format not supported")
    elif len(f) == 0:
        return None
    else:
        return f[0].path

class Analyzer(object):
    def  __init__(self):
       pass
    def requires(self):
        """Return an iterable of the file formats required to do the
        analysis"""
        raise NotImplementedError()
    def run_with(self, input_files, output_dir):
        """Run the analysis with the given input files"""
        raise NotImplementedError()
    def can_run_with(self, available_formats):
        """Return true if self.requires() is a subset of available_formats"""
        return set(self.requires()).issubset(set(available_formats))
    def missing(self, available_formats):
        """Return list of missing items"""
        return set(self.requires()) - set(available_formats)

class Hotnet2(Analyzer):
    def requires(self):
        return (FMT_HOTNET2_EDGE,FMT_HOTNET2_GENE_INDEX, FMT_GENE_LIST) #TODO dummy list

class Networkx(Analyzer):
    def requires(self):
        return (FMT_GENE_LIST, FMT_2_COL_GENE_NETWORK, FMT_LOCATION_2_GENE_NAME)

class Funseq2(Analyzer):
    def requires(self):
        return (FMT_PLINK_4_FUNSEQ,)

analyzers = {
    Hotnet2(),
    Networkx(),
    Funseq2()
}

parsed = parsed_command_line()
print ",".join([str(i) for i in input_files(parsed)])
avail = input_files(parsed)
possible = possible_inputs([a.file_format for a in avail], converters)
temp_dir_path = tempfile.mkdtemp()
print "Possible: "+",".join(sorted(list(possible)));
print "Could not run:" + ",".join(a.__class__.__name__+" is missing:"+",".join(a.missing(possible)) for a in analyzers if not a.can_run_with(possible))
print "Could run:" + ",".join(a.__class__.__name__ for a in analyzers if a.can_run_with(possible))
print "Converting inputs"
inputs = all_inputs(avail, converters, temp_dir_path)
print "Files after conversions:",",".join(f.path for f in inputs)

shutil.rmtree(temp_dir_path)
<|endoftext|>


<commit_before># Copyright (c) 2001-2004 Twisted Matrix Laboratories.
# See LICENSE for details.
"""
A generic resource for publishing objects via JSON-RPC.

API Stability: semi-stable

Maintainer: U{Duncan McGreggor <mailto:oubiwann@adytum.us>}
"""
from twisted.internet import defer, protocol, reactor
from twisted.protocols import basic
from twisted.python import log

from txjsonrpc import jsonrpclib
from txjsonrpc.jsonrpc import (
    BaseProxy, BaseQueryFactory, BaseSubhandler, Introspection)

class JSONRPC(basic.NetstringReceiver, BaseSubhandler):
    """
    A protocol that implements JSON-RPC.

    Methods published can return JSON-RPC serializable results, Faults,
    Binary, Boolean, DateTime, Deferreds, or Handler instances.

    By default methods beginning with 'jsonrpc_' are published.
    """
    # Error codes for Twisted, if they conflict with yours then
    # modify them at runtime.
    NOT_FOUND = 8001
    FAILURE = 8002

    separator = '.'
    closed = 0

    def __call__(self):
        return self

    def connectionMade(self):
        self.MAX_LENGTH = self.factory.maxLength

    def stringReceived(self, line):
        parser, unmarshaller = jsonrpclib.getparser()
        deferred = defer.maybeDeferred(parser.feed, line)
        deferred.addCallback(lambda x: self._cbDispatch(
            parser, unmarshaller))
        deferred.addErrback(self._ebRender)
        deferred.addCallback(self._cbRender)
        return deferred

    def _cbDispatch(self, parser, unmarshaller):
        parser.close()
        args, functionPath = unmarshaller.close(), unmarshaller.getmethodname()
        function = self._getFunction(functionPath)
        return defer.maybeDeferred(function, *args)

    def _cbRender(self, result):
        if not isinstance(result, jsonrpclib.Fault):
            result = (result,)
        #s = None
        #e = None
        try:
            s = jsonrpclib.dumps(result)
        except:
            f = jsonrpclib.Fault(self.FAILURE, "can't serialize output")
            #e = jsonrpclib.dumps(f)
            s = jsonrpclib.dumps(f)
        #result = {'result': result, 'error': e}
        #return self.sendString(jsonrpclib.dumps(result))
        return self.sendString(s)

    def _ebRender(self, failure):
        if isinstance(failure.value, jsonrpclib.Fault):
            return failure.value
        log.err(failure)
        return jsonrpclib.Fault(self.FAILURE, "error")

class QueryProtocol(basic.NetstringReceiver):

    def connectionMade(self):
        self.data = ''
        msg = self.factory.payload
        packet = '%d:%s,' % (len(msg), msg)
        self.transport.write(packet)

    def stringReceived(self, string):
        self.factory.data = string
        self.transport.loseConnection()

class QueryFactory(BaseQueryFactory):

    protocol = QueryProtocol
    data = ''

    def clientConnectionLost(self, _, reason):
        self.parseResponse(self.data)

class Proxy(BaseProxy):
    """
    A Proxy for making remote JSON-RPC calls.

    Pass the URL of the remote JSON-RPC server to the constructor.

    Use proxy.callRemote('foobar', *args) to call remote method
    'foobar' with *args.
    """

    def __init__(self, host, port, version=jsonrpclib.VERSION_PRE1,
                 factoryClass=QueryFactory):
        """
        @type host: C{str}
        @param host: The host to which method calls are made.

        @type port: C{integer}
        @param port: The host's port to which method calls are made.

        @type version: C{int}
        @param version: The version indicates which JSON-RPC spec to support.
        The available choices are jsonrpclib.VERSION*. The default is to use
        the version of the spec that txJSON-RPC was originally released with,
        pre-Version 1.0.

        @type factoryClass: C{object}
        @param factoryClass: The factoryClass should be a subclass of
        QueryFactory (class, not instance) that will be used instead of
        QueryFactory.
        """
        BaseProxy.__init__(self, version, factoryClass)
        self.host = host
        self.port = port

    def callRemote(self, method, *args, **kwargs):
        version = self._getVersion(kwargs)
        factoryClass = self._getFactoryClass(kwargs)
        factory = factoryClass(method, version, *args)
        reactor.connectTCP(self.host, self.port, factory)
        return factory.deferred

class RPCFactory(protocol.ServerFactory):

    protocol = None

    def __init__(self, rpcClass, maxLength=1024):
        self.maxLength = maxLength
        self.protocol = rpcClass
        self.subHandlers = {}

    def buildProtocol(self, addr):
        p = protocol.ServerFactory.buildProtocol(self, addr)
        for key, val in self.subHandlers.items():
            klass, args, kws = val
            if args and args[0] == 'protocol':
                p.putSubHandler(key, klass(p))
            else:
                p.putSubHandler(key, klass(*args, **kws))
        return p

    def putSubHandler(self, name, klass, args=(), kws={}):
        self.subHandlers[name] = (klass, args, kws)

    def addIntrospection(self):
        self.putSubHandler('system', Introspection, ('protocol',))

__all__ = ["JSONRPC", "Proxy", "RPCFactory"]
<commit_msg>imported patch stefan.sayer-v2.patch<commit_after># Copyright (c) 2001-2004 Twisted Matrix Laboratories.
# See LICENSE for details.
"""
A generic resource for publishing objects via JSON-RPC.

API Stability: semi-stable

Maintainer: U{Duncan McGreggor <mailto:oubiwann@adytum.us>}
"""
from twisted.internet import defer, protocol, reactor
from twisted.protocols import basic
from twisted.python import log

from txjsonrpc import jsonrpclib
from txjsonrpc.jsonrpc import (
    BaseProxy, BaseQueryFactory, BaseSubhandler, Introspection)

class JSONRPC(basic.NetstringReceiver, BaseSubhandler):
    """
    A protocol that implements JSON-RPC.

    Methods published can return JSON-RPC serializable results, Faults,
    Binary, Boolean, DateTime, Deferreds, or Handler instances.

    By default methods beginning with 'jsonrpc_' are published.
    """
    # Error codes for Twisted, if they conflict with yours then
    # modify them at runtime.
    NOT_FOUND = 8001
    FAILURE = 8002

    separator = '.'
    closed = 0

    def __init__(self, version=jsonrpclib.VERSION_2):
        self.version = version

    def __call__(self):
        return self

    def connectionMade(self):
        self.MAX_LENGTH = self.factory.maxLength

    def stringReceived(self, line):
        parser, unmarshaller = jsonrpclib.getparser()
        deferred = defer.maybeDeferred(parser.feed, line)

        req, req_id = self._cbDispatch(parser, unmarshaller)
        deferred.addCallback(lambda x: req)
        deferred.addErrback(self._ebRender, req_id = req_id)
        deferred.addCallback(self._cbRender, req_id = req_id)
        return deferred

    def _cbDispatch(self, parser, unmarshaller):
        parser.close()
        args, functionPath, req_id  = unmarshaller.close(), unmarshaller.getmethodname(), unmarshaller.getid()
        function = self._getFunction(functionPath)
        return defer.maybeDeferred(function, *args), req_id

    def _cbRender(self, result, req_id):
        if not isinstance(result, jsonrpclib.Fault):
            result = (result,)
        #s = None
        #e = None
        try:
            s = jsonrpclib.dumps(result, id=req_id, version=self.version)
        except:
            f = jsonrpclib.Fault(self.FAILURE, "can't serialize output")
            #e = jsonrpclib.dumps(f)
            s = jsonrpclib.dumps(f, id=req_id, version=self.version)
        #result = {'result': result, 'error': e}
        #return self.sendString(jsonrpclib.dumps(result))
        return self.sendString(s)

    def _ebRender(self, failure, req_id):
        if isinstance(failure.value, jsonrpclib.Fault):
            return failure.value
        log.err(failure)
        return jsonrpclib.Fault(self.FAILURE, "error")

class QueryProtocol(basic.NetstringReceiver):

    def connectionMade(self):
        self.data = ''
        msg = self.factory.payload
        packet = '%d:%s,' % (len(msg), msg)
        self.transport.write(packet)

    def stringReceived(self, string):
        self.factory.data = string
        self.transport.loseConnection()

class QueryFactory(BaseQueryFactory):

    protocol = QueryProtocol
    data = ''

    def clientConnectionLost(self, _, reason):
        self.parseResponse(self.data)

class Proxy(BaseProxy):
    """
    A Proxy for making remote JSON-RPC calls.

    Pass the URL of the remote JSON-RPC server to the constructor.

    Use proxy.callRemote('foobar', *args) to call remote method
    'foobar' with *args.
    """

    def __init__(self, host, port, version=jsonrpclib.VERSION_PRE1,
                 factoryClass=QueryFactory):
        """
        @type host: C{str}
        @param host: The host to which method calls are made.

        @type port: C{integer}
        @param port: The host's port to which method calls are made.

        @type version: C{int}
        @param version: The version indicates which JSON-RPC spec to support.
        The available choices are jsonrpclib.VERSION*. The default is to use
        the version of the spec that txJSON-RPC was originally released with,
        pre-Version 1.0.

        @type factoryClass: C{object}
        @param factoryClass: The factoryClass should be a subclass of
        QueryFactory (class, not instance) that will be used instead of
        QueryFactory.
        """
        BaseProxy.__init__(self, version, factoryClass)
        self.host = host
        self.port = port

    def callRemote(self, method, *args, **kwargs):
        version = self._getVersion(kwargs)
        factoryClass = self._getFactoryClass(kwargs)
        factory = factoryClass(method, version, *args)
        reactor.connectTCP(self.host, self.port, factory)
        return factory.deferred

class RPCFactory(protocol.ServerFactory):

    protocol = None

    def __init__(self, rpcClass, maxLength=1024):
        self.maxLength = maxLength
        self.protocol = rpcClass
        self.subHandlers = {}

    def buildProtocol(self, addr):
        p = protocol.ServerFactory.buildProtocol(self, addr)
        for key, val in self.subHandlers.items():
            klass, args, kws = val
            if args and args[0] == 'protocol':
                p.putSubHandler(key, klass(p))
            else:
                p.putSubHandler(key, klass(*args, **kws))
        return p

    def putSubHandler(self, name, klass, args=(), kws={}):
        self.subHandlers[name] = (klass, args, kws)

    def addIntrospection(self):
        self.putSubHandler('system', Introspection, ('protocol',))

__all__ = ["JSONRPC", "Proxy", "RPCFactory"]
<|endoftext|>


<commit_before>import pycurl
import json
from StringIO import StringIO
import subprocess
import os
import gzip
import tempfile
import urllib
from random import random
from time import sleep
import shutil

#main_url = "http://localhost:8888/"
api_url = "https://api.github.com"
site_url = "https://github.com"
url_code = getattr(pycurl, "URL")

def wait_for_rate_limit(path, sleep_time=60):
    sleep(sleep_time)
    d = json.loads(get_url("/rate_limit")) # rate_limit is not rate_limited!
    if d['rate']['remaining'] == 0:
        new_sleep = sleep_time * 2
        if new_sleep > 2 * 60 * 60:
            raise Exception("We should never sleep for " + str(new_sleep))
        print "sleeping for " + str(new_sleep)
        wait_for_rate_limit(path, sleep_time * 2)
    else:
        return get_url(path)

def get_url(path):
    buffer = StringIO()
    handle = pycurl.Curl()
    handle.setopt(pycurl.URL, str(api_url + path))
    handle.setopt(pycurl.WRITEFUNCTION, buffer.write)
    handle.perform()
    if handle.getinfo(handle.RESPONSE_CODE) != 200:
        return wait_for_rate_limit(path)
    return buffer.getvalue()

def get_followers(user):
    body = get_url("/users/" + user + "/followers")
    return [user['login'] for user in json.loads(body)]

def get_user_repos(user):
    repos = get_url("/users/" + user + "/repos")
    return [(repo['name'], repo['default_branch'], repo['fork']) for repo in json.loads(repos)]

def get_org_repos(org):
    repos = get_url("/orgs/" + org + "/repos")
    return [(repo['name'], repo['default_branch'], repo['fork']) for repo in json.loads(repos)]

def get_parent_repo(owner, repo_name):
    body = get_url("/repos/" + owner + "/" + repo_name)
    parent = json.loads(body)['parent']
    return parent['owner']['login'], parent['name'], parent['default_branch']

def get_parent_repos(owner, func):
    repos = func(owner)
    parent_repos = []
    for repo, branch, fork in repos:
        if fork:
            parent_repos += [get_parent_repo(owner, repo)]
        else:
            parent_repos += [(owner, repo, branch)]
    return parent_repos

def get_user_parent_repos(user):
    return get_parent_repos(user, get_user_repos)

def get_org_parent_repos(user):
    return get_parent_repos(user, get_org_repos)

def actual_clone_repo(owner, repo, branch):
    clone_repo = site_url + "/" + owner + "/" + repo
    directory = tempfile.mkdtemp() + "/" + repo
    subprocess.check_output(["git", "clone", clone_repo, directory], stderr=subprocess.STDOUT)
    return directory

def clone_repo(owner, repo, branch):
    # this function saves a lot of bandwidth
    url = site_url + "/" + owner + "/" + repo + "/zipball/" + branch
    directory = tempfile.mkdtemp() + "/"
    zippath = directory + repo + ".zip"
    urllib.urlretrieve(url, zippath)
    import zipfile
    with zipfile.ZipFile(zippath, 'r') as myzip:
        myzip.extractall(path=directory)
    os.unlink(zippath)
    return directory

def get_gzip_size(fullfilename):
    with open(fullfilename, "rb") as fin:
        outname = tempfile.NamedTemporaryFile(dir="/tmp", prefix="gzipped_").name
        with gzip.open(outname, "wb") as fout:
            fout.writelines(fin)
    size = os.path.getsize(outname)
    os.remove(outname)
    return size

def analyze_file(root, filename):
    if '.git' in root:
        return
    if '.' in filename:
        extension = filename.split('.')[-1]
        original_size = os.path.getsize(root + "/" + filename)
        if original_size == 0:
            return
        gzip_size = get_gzip_size(root + "/" + filename)
        return (extension, gzip_size / float(original_size))

def analyze_repo(directory):
    for root, dirs, files in os.walk(directory):
        for filename in files:
            try:
                info = analyze_file(root, filename)
            except Exception as e:
                print e
            if info:
                yield info
    shutil.rmtree(directory)
    os.rmdir(os.path.dirname(directory))

def get_users_with_n_followers(n):
    url = '/search/users?' + urllib.urlencode({'q':'followers:' + str(n)})
    body = get_url(url)
    return [user['login'] for user in json.loads(body)['items']]

def get_random_users():
    rand = int(random() * 40)
    return get_users_with_n_followers(100 + rand)

if __name__ == "__main__":
    #print get_followers("pooya")
    #print get_user_repos("pooya")
    #print get_org_repos("discoproject")
    #print get_parent_repo("pooya", "disco")
    #print get_user_parent_repos("pooya")
    #print get_user_parent_repos("discoproject")
    #directory = clone_repo("2600hz", "kazoo")
    directory = clone_repo("talko", "plists", "master")
    #analyze_repo(directory)
    #print get_random_users()
    pass

<commit_msg>cosmetic changes<commit_after>import pycurl
import json
from StringIO import StringIO
import subprocess
import os
import gzip
import tempfile
import urllib
from random import random
from time import sleep
import shutil

#main_url = "http://localhost:8888/"
api_url = "https://api.github.com"
site_url = "https://github.com"
url_code = getattr(pycurl, "URL")

def wait_for_rate_limit(path, sleep_time=60):
    sleep(sleep_time)
    d = json.loads(get_url("/rate_limit")) # rate_limit is not rate_limited!
    if d['rate']['remaining'] == 0:
        new_sleep = sleep_time * 2
        if new_sleep > 2 * 60 * 60:
            raise Exception("We should never sleep for " + str(new_sleep))
        print "sleeping for " + str(new_sleep)
        wait_for_rate_limit(path, sleep_time * 2)
    else:
        return get_url(path)

def get_url(path):
    buffer = StringIO()
    handle = pycurl.Curl()
    handle.setopt(pycurl.URL, str(api_url + path))
    handle.setopt(pycurl.WRITEFUNCTION, buffer.write)
    handle.perform()
    if handle.getinfo(handle.RESPONSE_CODE) != 200:
        return wait_for_rate_limit(path)
    return buffer.getvalue()

def get_followers(user):
    body = get_url("/users/" + user + "/followers")
    return [user['login'] for user in json.loads(body)]

def get_user_repos(user):
    repos = get_url("/users/" + user + "/repos")
    return [(repo['name'], repo['default_branch'], repo['fork']) for repo in json.loads(repos)]

def get_org_repos(org):
    repos = get_url("/orgs/" + org + "/repos")
    return [(repo['name'], repo['default_branch'], repo['fork']) for repo in json.loads(repos)]

def get_parent_repo(owner, repo_name):
    body = get_url("/repos/" + owner + "/" + repo_name)
    parent = json.loads(body)['parent']
    return parent['owner']['login'], parent['name'], parent['default_branch']

def get_parent_repos(owner, func):
    repos = func(owner)
    parent_repos = []
    for repo, branch, fork in repos:
        if fork:
            parent_repos += [get_parent_repo(owner, repo)]
        else:
            parent_repos += [(owner, repo, branch)]
    return parent_repos

def get_user_parent_repos(user):
    return get_parent_repos(user, get_user_repos)

def get_org_parent_repos(user):
    return get_parent_repos(user, get_org_repos)

def actual_clone_repo(owner, repo, branch):
    clone_repo = site_url + "/" + owner + "/" + repo
    directory = tempfile.mkdtemp() + "/" + repo
    subprocess.check_output(["git", "clone", clone_repo, directory], stderr=subprocess.STDOUT)
    return directory

def clone_repo(owner, repo, branch):
    # this function saves a lot of bandwidth
    url = site_url + "/" + owner + "/" + repo + "/zipball/" + branch
    directory = tempfile.mkdtemp() + "/"
    zippath = directory + repo + ".zip"
    urllib.urlretrieve(url, zippath)
    import zipfile
    with zipfile.ZipFile(zippath, 'r') as myzip:
        myzip.extractall(path=directory)
    os.unlink(zippath)
    return directory

def get_gzip_size(fullfilename):
    with open(fullfilename, "rb") as fin:
        outname = tempfile.NamedTemporaryFile(dir="/tmp", prefix="gzipped_").name
        with gzip.open(outname, "wb") as fout:
            fout.writelines(fin)
    size = os.path.getsize(outname)
    os.remove(outname)
    return size

def analyze_file(root, filename):
    if '.git' in root:
        return
    if '.' in filename:
        extension = filename.split('.')[-1]
        original_size = os.path.getsize(root + "/" + filename)
        if original_size == 0:
            return
        gzip_size = get_gzip_size(root + "/" + filename)
        return (extension, gzip_size / float(original_size))

def analyze_repo(directory):
    for root, dirs, files in os.walk(directory):
        for filename in files:
            try:
                info = analyze_file(root, filename)
            except Exception as e:
                print e
            if info:
                yield info
    shutil.rmtree(directory)

def get_users_with_n_followers(n):
    url = '/search/users?' + urllib.urlencode({'q':'followers:' + str(n)})
    body = get_url(url)
    return [user['login'] for user in json.loads(body)['items']]

def get_random_users():
    rand = int(random() * 40)
    return get_users_with_n_followers(100 + rand)

if __name__ == "__main__":
    #print get_followers("pooya")
    #print get_user_repos("pooya")
    #print get_org_repos("discoproject")
    #print get_parent_repo("pooya", "disco")
    #print get_user_parent_repos("pooya")
    #print get_user_parent_repos("discoproject")
    #directory = clone_repo("2600hz", "kazoo")
    directory = clone_repo("talko", "plists", "master")
    #analyze_repo(directory)
    #print get_random_users()
    pass

<|endoftext|>