<commit_before>import getpass
import os
import stat
from optparse import make_option

from django.conf import settings
from django.core.management.base import BaseCommand

from ui.models import TwitterFilter

class Command(BaseCommand):
    help = "create/update filterstream process config files for supervisord"
    option_list = BaseCommand.option_list + (
        make_option('--tfilterid', action='store', default=None,
                    dest='tfilterid', help='specify the filter rule id'),
        )

    def handle(self, *args, **options):
        twitter_filters = TwitterFilter.objects.filter(is_active=True)
        if options.get('tfilterid', None):
            twitter_filters = twitter_filters.filter(
                id=options.get('tfilterid'))
        #projectroot = os.path.join(os.path.dirname(__file__), '..', '..', '..')
        projectroot = settings.SFM_ROOT
        if settings.SUPERVISOR_PROCESS_USER:
            processowner = settings.SUPERVISOR_PROCESS_USER
        else:
            processowner = getpass.getuser()
        for tfilter in twitter_filters:
            contents = "[program:filterstream-%s]" % tfilter.id + '\n' + \
                       "command=%s/ENV/bin/python " % projectroot + \
                       "%s/sfm/manage.py " % projectroot + \
                       "filterstream " + \
                       "--tfilterid=%s --save" % tfilter.id + '\n' \
                       "user=%s" % processowner + '\n' \
                       "autostart=true" + '\n' \
                       "autorestart=true" + '\n' \
                       "stderr_logfile=/var/log/" \
                       "sfm-filterstream-%s.err.log" % tfilter.id + '\n' \
                       "stdout_logfile=/var/log/" \
                       "sfm-filterstream-%s.out.log" % tfilter.id
            filename = "sfm-twitter-filter-%s.conf" % tfilter.id
            file_path = "%s/sfm/sfm/supervisor.d/%s" % (projectroot, filename)
            # Remove any existing config file
            # we don't assume that the contents are up-to-date
            # (PATH settings may have changed, etc.)
            if os.path.exists(file_path):
                os.remove(file_path)
                print "Removed old configuration file for TwitterFilter %d" % \
                    tfilter.id

            fp = open(file_path, "wb")
            fp.write(contents)
            filestatus = os.stat(file_path)
            # do a chmod +x
            os.chmod(file_path, filestatus.st_mode | stat.S_IXUSR |
                     stat.S_IXGRP | stat.S_IXOTH)
            fp.close()
            print "Created configuration file for TwitterFilter %d" % \
                  tfilter.id
<commit_msg>removed extraneous commented-out line<commit_after>import getpass
import os
import stat
from optparse import make_option

from django.conf import settings
from django.core.management.base import BaseCommand

from ui.models import TwitterFilter

class Command(BaseCommand):
    help = "create/update filterstream process config files for supervisord"
    option_list = BaseCommand.option_list + (
        make_option('--tfilterid', action='store', default=None,
                    dest='tfilterid', help='specify the filter rule id'),
        )

    def handle(self, *args, **options):
        twitter_filters = TwitterFilter.objects.filter(is_active=True)
        if options.get('tfilterid', None):
            twitter_filters = twitter_filters.filter(
                id=options.get('tfilterid'))
        projectroot = settings.SFM_ROOT
        if settings.SUPERVISOR_PROCESS_USER:
            processowner = settings.SUPERVISOR_PROCESS_USER
        else:
            processowner = getpass.getuser()
        for tfilter in twitter_filters:
            contents = "[program:filterstream-%s]" % tfilter.id + '\n' + \
                       "command=%s/ENV/bin/python " % projectroot + \
                       "%s/sfm/manage.py " % projectroot + \
                       "filterstream " + \
                       "--tfilterid=%s --save" % tfilter.id + '\n' \
                       "user=%s" % processowner + '\n' \
                       "autostart=true" + '\n' \
                       "autorestart=true" + '\n' \
                       "stderr_logfile=/var/log/" \
                       "sfm-filterstream-%s.err.log" % tfilter.id + '\n' \
                       "stdout_logfile=/var/log/" \
                       "sfm-filterstream-%s.out.log" % tfilter.id
            filename = "sfm-twitter-filter-%s.conf" % tfilter.id
            file_path = "%s/sfm/sfm/supervisor.d/%s" % (projectroot, filename)
            # Remove any existing config file
            # we don't assume that the contents are up-to-date
            # (PATH settings may have changed, etc.)
            if os.path.exists(file_path):
                os.remove(file_path)
                print "Removed old configuration file for TwitterFilter %d" % \
                    tfilter.id

            fp = open(file_path, "wb")
            fp.write(contents)
            filestatus = os.stat(file_path)
            # do a chmod +x
            os.chmod(file_path, filestatus.st_mode | stat.S_IXUSR |
                     stat.S_IXGRP | stat.S_IXOTH)
            fp.close()
            print "Created configuration file for TwitterFilter %d" % \
                  tfilter.id
<|endoftext|>


<commit_before>import codecs
from optparse import make_option
import sys
from django.core.management.base import BaseCommand, CommandError

from ui.models import TwitterUser, TwitterUserSet, TwitterUserItem
from ui.utils import make_date_aware, xls_tweets_workbook

class Command(BaseCommand):
    help = 'export data for a user or a set in csv or xls'

    option_list = BaseCommand.option_list + (
        make_option('--start-date', action='store', default=None,
                    type='string', dest='start_date',
                    help='earliest date (YYYY-MM-DD) for export'),
        make_option('--end-date', action='store', default=None,
                    type='string', dest='end_date',
                    help='latest date (YYYY-MM-DD) for export'),
        make_option('--twitter-user', action='store', default=None,
                    type='string', dest='twitter_user',
                    help='username to export'),
        make_option('--set-name', action='store', default=None,
                    type='string', dest='set_name',
                    help='set name to export'),
        make_option('--xls', action='store_true', default=False,
                    dest='xls', help='export in .xls format'),
        make_option('--filename', action='store', default=None,
                    type='string', dest='filename',
                    help='filename to export (required with --xls)'),
    )

    def handle(self, *args, **options):
        twitter_user = user_set = start_dt = end_dt = xls = filename = None
        if options['filename']:
            filename = options.get('filename')
        xls = options['xls']
        if xls and filename is None:
            raise CommandError("When --xls is specified, \
--filename=FILENAME is required")
        if not xls and filename is not None:
            raise CommandError("Writing CSV files currently not yet \
supported; recommend piping output to a file")
        if options['twitter_user']:
            try:
                twitter_user = TwitterUser.objects.get(
                    name=options.get('twitter_user'))
                qs = twitter_user.items.all()
            except TwitterUser.DoesNotExist:
                raise CommandError('TwitterUser %s does not exist' %
                                   options.get('twitter_user'))
        elif options['set_name']:
            try:
                user_set = TwitterUserSet.objects.get(
                    name=options.get('set_name'))
                qs = TwitterUserItem.objects.filter(
                    twitter_user__sets__in=[user_set])
            except TwitterUserSet.DoesNotExist:
                raise CommandError('TwitterUserSet %s does not exist' %
                                   options['set_name'])
        else:
            raise CommandError('please provide either twitteruser or setname')
        if options['start_date']:
            start_dt = make_date_aware(options.get('start_date'))
            if not start_dt:
                raise CommandError('dates must be in the format YYYY-MM-DD')
            qs = qs.filter(date_published__gte=start_dt)
        if options['end_date']:
            end_dt = make_date_aware(options.get('end_date'))
            if not end_dt:
                raise CommandError('dates must be in the format YYYY-MM-DD')
            qs = qs.filter(date_published__lte=end_dt)
        if start_dt and end_dt:
            if end_dt < start_dt:
                raise CommandError('start date must be earlier than end date')
        # tweak for python 2.7 to avoid having to set PYTHONIOENCODING=utf8
        # in environment, see Graham Fawcett's comment/suggestion at:
        #   nedbatchelder.com/blog/200401/printing_unicode_from_python.html
        writer_class = codecs.getwriter('utf-8')
        sys.stdout = writer_class(sys.stdout, 'replace')
        if xls:
            tworkbook = xls_tweets_workbook(qs, TwitterUserItem.csv_headers)
            tworkbook.save(filename)
        else:
            for tui in qs:
                print '\t'.join(tui.csv)
<commit_msg>Made twitter-user option case-insensitive (this was driving me crazy)<commit_after>import codecs
from optparse import make_option
import sys
from django.core.management.base import BaseCommand, CommandError

from ui.models import TwitterUser, TwitterUserSet, TwitterUserItem
from ui.utils import make_date_aware, xls_tweets_workbook

class Command(BaseCommand):
    help = 'export data for a user or a set in csv or xls'

    option_list = BaseCommand.option_list + (
        make_option('--start-date', action='store', default=None,
                    type='string', dest='start_date',
                    help='earliest date (YYYY-MM-DD) for export'),
        make_option('--end-date', action='store', default=None,
                    type='string', dest='end_date',
                    help='latest date (YYYY-MM-DD) for export'),
        make_option('--twitter-user', action='store', default=None,
                    type='string', dest='twitter_user',
                    help='username to export'),
        make_option('--set-name', action='store', default=None,
                    type='string', dest='set_name',
                    help='set name to export'),
        make_option('--xls', action='store_true', default=False,
                    dest='xls', help='export in .xls format'),
        make_option('--filename', action='store', default=None,
                    type='string', dest='filename',
                    help='filename to export (required with --xls)'),
    )

    def handle(self, *args, **options):
        twitter_user = user_set = start_dt = end_dt = xls = filename = None
        if options['filename']:
            filename = options.get('filename')
        xls = options['xls']
        if xls and filename is None:
            raise CommandError("When --xls is specified, \
--filename=FILENAME is required")
        if not xls and filename is not None:
            raise CommandError("Writing CSV files currently not yet \
supported; recommend piping output to a file")
        if options['twitter_user']:
            try:
                twitter_user = TwitterUser.objects.get(
                    name__iexact=options.get('twitter_user'))
                qs = twitter_user.items.all()
            except TwitterUser.DoesNotExist:
                raise CommandError('TwitterUser %s does not exist' %
                                   options.get('twitter_user'))
        elif options['set_name']:
            try:
                user_set = TwitterUserSet.objects.get(
                    name=options.get('set_name'))
                qs = TwitterUserItem.objects.filter(
                    twitter_user__sets__in=[user_set])
            except TwitterUserSet.DoesNotExist:
                raise CommandError('TwitterUserSet %s does not exist' %
                                   options['set_name'])
        else:
            raise CommandError('please provide either twitteruser or setname')
        if options['start_date']:
            start_dt = make_date_aware(options.get('start_date'))
            if not start_dt:
                raise CommandError('dates must be in the format YYYY-MM-DD')
            qs = qs.filter(date_published__gte=start_dt)
        if options['end_date']:
            end_dt = make_date_aware(options.get('end_date'))
            if not end_dt:
                raise CommandError('dates must be in the format YYYY-MM-DD')
            qs = qs.filter(date_published__lte=end_dt)
        if start_dt and end_dt:
            if end_dt < start_dt:
                raise CommandError('start date must be earlier than end date')
        # tweak for python 2.7 to avoid having to set PYTHONIOENCODING=utf8
        # in environment, see Graham Fawcett's comment/suggestion at:
        #   nedbatchelder.com/blog/200401/printing_unicode_from_python.html
        writer_class = codecs.getwriter('utf-8')
        sys.stdout = writer_class(sys.stdout, 'replace')
        if xls:
            tworkbook = xls_tweets_workbook(qs, TwitterUserItem.csv_headers)
            tworkbook.save(filename)
        else:
            for tui in qs:
                print '\t'.join(tui.csv)
<|endoftext|>


<commit_before>############################################################################
#
# Copyright (C) 2015 The Qt Company Ltd.
# Contact: http://www.qt.io/licensing
#
# This file is part of Qt Creator.
#
# Commercial License Usage
# Licensees holding valid commercial Qt licenses may use this file in
# accordance with the commercial license agreement provided with the
# Software or, alternatively, in accordance with the terms contained in
# a written agreement between you and The Qt Company.  For licensing terms and
# conditions see http://www.qt.io/terms-conditions.  For further information
# use the contact form at http://www.qt.io/contact-us.
#
# GNU Lesser General Public License Usage
# Alternatively, this file may be used under the terms of the GNU Lesser
# General Public License version 2.1 or version 3 as published by the Free
# Software Foundation and appearing in the file LICENSE.LGPLv21 and
# LICENSE.LGPLv3 included in the packaging of this file.  Please review the
# following information to ensure the GNU Lesser General Public License
# requirements will be met: https://www.gnu.org/licenses/lgpl.html and
# http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
#
# In addition, as a special exception, The Qt Company gives you certain additional
# rights.  These rights are described in The Qt Company LGPL Exception
# version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
#
#############################################################################

from dumper import *

def stripTypeName(value):
    type = value.type
    try:
        type = type.target()
    except:
        pass
    return str(type.unqualified())

def extractPointerType(d, value):
    postfix = ""
    while stripTypeName(value) == "CPlusPlus::PointerType":
        postfix += "*"
        value = d.downcast(value["_elementType"]["_type"])
    try:
        return readLiteral(d, value["_name"]) + postfix
    except:
        if str(value.type.unqualified().target()) == "CPlusPlus::IntegerType":
            return "int" + postfix
        return "<unsupported>"

def readTemplateName(d, value):
    name = readLiteral(d, value["_identifier"]) + "<"
    args = value["_templateArguments"]
    impl = args["_M_impl"]
    start = impl["_M_start"]
    size = impl["_M_finish"] - start
    try:
        d.check(0 <= size and size <= 100)
        d.checkPointer(start)
        for i in range(int(size)):
            if i > 0:
                name += ", "
            name += extractPointerType(d, d.downcast(start[i]["_type"]))
    except:
        return "<not accessible>"
    name += ">"
    return name

def readLiteral(d, value):
    if d.isNull(value):
        return "<null>"
    value = d.downcast(value)
    type = value.type.unqualified()
    try:
        type = type.target()
    except:
        pass
    typestr = str(type)
    if typestr == "CPlusPlus::TemplateNameId":
        return readTemplateName(d, value)
    elif typestr == "CPlusPlus::QualifiedNameId":
        return readLiteral(d, value["_base"]) + "::" + readLiteral(d, value["_name"])
    try:
        return d.extractBlob(value["_chars"], value["_size"]).toString()
    except:
        return "<unsupported>"

def dumpLiteral(d, value):
    d.putValue(d.hexencode(readLiteral(d, value)), Hex2EncodedLatin1)

def qdump__Core__Id(d, value):
    try:
        name = d.parseAndEvaluate("Core::nameForId(%d)" % value["m_id"])
        d.putSimpleCharArray(name)
    except:
        d.putValue(value["m_id"])
    d.putPlainChildren(value)

def qdump__Debugger__Internal__GdbMi(d, value):
    str = d.encodeByteArray(value["m_name"]) + "3a20" \
        + d.encodeByteArray(value["m_data"])
    d.putValue(str, Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__Debugger__Internal__DisassemblerLine(d, value):
    d.putByteArrayValue(value["m_data"])
    d.putPlainChildren(value)

def qdump__Debugger__Internal__WatchData(d, value):
    d.putByteArrayValue(value["iname"])
    d.putPlainChildren(value)

def qdump__Debugger__Internal__WatchItem(d, value):
    d.putByteArrayValue(value["iname"])
    d.putPlainChildren(value)

def qdump__Debugger__Internal__BreakpointModelId(d, value):
    d.putValue("%s.%s" % (int(value["m_majorPart"]), int(value["m_minorPart"])))
    d.putPlainChildren(value)

def qdump__Debugger__Internal__ThreadId(d, value):
    d.putValue("%s" % value["m_id"])
    d.putPlainChildren(value)

def qdump__CPlusPlus__ByteArrayRef(d, value):
    d.putSimpleCharArray(value["m_start"], value["m_length"])
    d.putPlainChildren(value)

def qdump__CPlusPlus__Identifier(d, value):
    d.putSimpleCharArray(value["_chars"], value["_size"])
    d.putPlainChildren(value)

def qdump__CPlusPlus__Symbol(d, value):
    dumpLiteral(d, value["_name"])
    d.putBetterType(value.type)
    d.putPlainChildren(value)

def qdump__CPlusPlus__Class(d, value):
    qdump__CPlusPlus__Symbol(d, value)

def qdump__CPlusPlus__IntegerType(d, value):
    d.putValue(value["_kind"])
    d.putPlainChildren(value)

def qdump__CPlusPlus__FullySpecifiedType(d, value):
    type = d.downcast(value["_type"])
    typeName = stripTypeName(type)
    if typeName == "CPlusPlus::NamedType":
        dumpLiteral(d, type["_name"])
    elif typeName == "CPlusPlus::PointerType":
        d.putValue(d.hexencode(extractPointerType(d, type)), Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__CPlusPlus__NamedType(d, value):
    dumpLiteral(d, value["_name"])
    d.putBetterType(value.type)
    d.putPlainChildren(value)

def qdump__CPlusPlus__PointerType(d, value):
    d.putValue(d.hexencode(extractPointerType(d, value)), Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__CPlusPlus__TemplateNameId(d, value):
    dumpLiteral(d, value)
    d.putBetterType(value.type)
    d.putPlainChildren(value)

def qdump__CPlusPlus__QualifiedNameId(d, value):
    dumpLiteral(d, value)
    d.putPlainChildren(value)

def qdump__CPlusPlus__Literal(d, value):
    dumpLiteral(d, value)
    d.putPlainChildren(value)

def qdump__CPlusPlus__StringLiteral(d, value):
    d.putSimpleCharArray(value["_chars"], value["_size"])
    d.putPlainChildren(value)

def qdump__CPlusPlus__Internal__Value(d, value):
    d.putValue(value["l"])
    d.putPlainChildren(value)

def qdump__Utils__FileName(d, value):
    d.putStringValue(value)
    d.putPlainChildren(value)

def qdump__Utils__ElfSection(d, value):
    d.putByteArrayValue(value["name"])
    d.putPlainChildren(value)

def qdump__CPlusPlus__Token(d, value):
    k = value["f"]["kind"]
    e = int(k)
    type = str(k.cast(d.lookupType("CPlusPlus::Kind")))[11:] # Strip "CPlusPlus::"
    try:
        if e == 6:
            type = readLiteral(d, value["identifier"]) + " (%s)" % type
        elif e >= 7 and e <= 23:
            type = readLiteral(d, value["literal"]) + " (%s)" % type
    except:
        pass
    d.putValue(type)
    d.putPlainChildren(value)

def qdump__CPlusPlus__Internal__PPToken(d, value):
    data, size, alloc = d.byteArrayData(value["m_src"])
    length = int(value["f"]["utf16chars"])
    offset = int(value["utf16charOffset"])
    #warn("size: %s, alloc: %s, offset: %s, length: %s, data: %s"
    #    % (size, alloc, offset, length, data))
    d.putValue(d.readMemory(data + offset, min(100, length)),
        Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__ProString(d, value):
    try:
        s = value["m_string"]
        data, size, alloc = d.stringData(s)
        data += 2 * int(value["m_offset"])
        size = int(value["m_length"])
        s = d.readMemory(data, 2 * size)
        d.putValue(s, Hex4EncodedLittleEndian)
    except:
        d.putEmptyValue()
    d.putPlainChildren(value)

<commit_msg>Dumper: Display "void" for VoidType<commit_after>############################################################################
#
# Copyright (C) 2015 The Qt Company Ltd.
# Contact: http://www.qt.io/licensing
#
# This file is part of Qt Creator.
#
# Commercial License Usage
# Licensees holding valid commercial Qt licenses may use this file in
# accordance with the commercial license agreement provided with the
# Software or, alternatively, in accordance with the terms contained in
# a written agreement between you and The Qt Company.  For licensing terms and
# conditions see http://www.qt.io/terms-conditions.  For further information
# use the contact form at http://www.qt.io/contact-us.
#
# GNU Lesser General Public License Usage
# Alternatively, this file may be used under the terms of the GNU Lesser
# General Public License version 2.1 or version 3 as published by the Free
# Software Foundation and appearing in the file LICENSE.LGPLv21 and
# LICENSE.LGPLv3 included in the packaging of this file.  Please review the
# following information to ensure the GNU Lesser General Public License
# requirements will be met: https://www.gnu.org/licenses/lgpl.html and
# http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
#
# In addition, as a special exception, The Qt Company gives you certain additional
# rights.  These rights are described in The Qt Company LGPL Exception
# version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
#
#############################################################################

from dumper import *

def stripTypeName(value):
    type = value.type
    try:
        type = type.target()
    except:
        pass
    return str(type.unqualified())

def extractPointerType(d, value):
    postfix = ""
    while stripTypeName(value) == "CPlusPlus::PointerType":
        postfix += "*"
        value = d.downcast(value["_elementType"]["_type"])
    try:
        return readLiteral(d, value["_name"]) + postfix
    except:
        typeName = str(value.type.unqualified().target())
        if typeName == "CPlusPlus::IntegerType":
            return "int" + postfix
        elif typeName == "CPlusPlus::VoidType":
            return "void" + postfix
        return "<unsupported>"

def readTemplateName(d, value):
    name = readLiteral(d, value["_identifier"]) + "<"
    args = value["_templateArguments"]
    impl = args["_M_impl"]
    start = impl["_M_start"]
    size = impl["_M_finish"] - start
    try:
        d.check(0 <= size and size <= 100)
        d.checkPointer(start)
        for i in range(int(size)):
            if i > 0:
                name += ", "
            name += extractPointerType(d, d.downcast(start[i]["_type"]))
    except:
        return "<not accessible>"
    name += ">"
    return name

def readLiteral(d, value):
    if d.isNull(value):
        return "<null>"
    value = d.downcast(value)
    type = value.type.unqualified()
    try:
        type = type.target()
    except:
        pass
    typestr = str(type)
    if typestr == "CPlusPlus::TemplateNameId":
        return readTemplateName(d, value)
    elif typestr == "CPlusPlus::QualifiedNameId":
        return readLiteral(d, value["_base"]) + "::" + readLiteral(d, value["_name"])
    try:
        return d.extractBlob(value["_chars"], value["_size"]).toString()
    except:
        return "<unsupported>"

def dumpLiteral(d, value):
    d.putValue(d.hexencode(readLiteral(d, value)), Hex2EncodedLatin1)

def qdump__Core__Id(d, value):
    try:
        name = d.parseAndEvaluate("Core::nameForId(%d)" % value["m_id"])
        d.putSimpleCharArray(name)
    except:
        d.putValue(value["m_id"])
    d.putPlainChildren(value)

def qdump__Debugger__Internal__GdbMi(d, value):
    str = d.encodeByteArray(value["m_name"]) + "3a20" \
        + d.encodeByteArray(value["m_data"])
    d.putValue(str, Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__Debugger__Internal__DisassemblerLine(d, value):
    d.putByteArrayValue(value["m_data"])
    d.putPlainChildren(value)

def qdump__Debugger__Internal__WatchData(d, value):
    d.putByteArrayValue(value["iname"])
    d.putPlainChildren(value)

def qdump__Debugger__Internal__WatchItem(d, value):
    d.putByteArrayValue(value["iname"])
    d.putPlainChildren(value)

def qdump__Debugger__Internal__BreakpointModelId(d, value):
    d.putValue("%s.%s" % (int(value["m_majorPart"]), int(value["m_minorPart"])))
    d.putPlainChildren(value)

def qdump__Debugger__Internal__ThreadId(d, value):
    d.putValue("%s" % value["m_id"])
    d.putPlainChildren(value)

def qdump__CPlusPlus__ByteArrayRef(d, value):
    d.putSimpleCharArray(value["m_start"], value["m_length"])
    d.putPlainChildren(value)

def qdump__CPlusPlus__Identifier(d, value):
    d.putSimpleCharArray(value["_chars"], value["_size"])
    d.putPlainChildren(value)

def qdump__CPlusPlus__Symbol(d, value):
    dumpLiteral(d, value["_name"])
    d.putBetterType(value.type)
    d.putPlainChildren(value)

def qdump__CPlusPlus__Class(d, value):
    qdump__CPlusPlus__Symbol(d, value)

def qdump__CPlusPlus__IntegerType(d, value):
    d.putValue(value["_kind"])
    d.putPlainChildren(value)

def qdump__CPlusPlus__FullySpecifiedType(d, value):
    type = d.downcast(value["_type"])
    typeName = stripTypeName(type)
    if typeName == "CPlusPlus::NamedType":
        dumpLiteral(d, type["_name"])
    elif typeName == "CPlusPlus::PointerType":
        d.putValue(d.hexencode(extractPointerType(d, type)), Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__CPlusPlus__NamedType(d, value):
    dumpLiteral(d, value["_name"])
    d.putBetterType(value.type)
    d.putPlainChildren(value)

def qdump__CPlusPlus__PointerType(d, value):
    d.putValue(d.hexencode(extractPointerType(d, value)), Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__CPlusPlus__TemplateNameId(d, value):
    dumpLiteral(d, value)
    d.putBetterType(value.type)
    d.putPlainChildren(value)

def qdump__CPlusPlus__QualifiedNameId(d, value):
    dumpLiteral(d, value)
    d.putPlainChildren(value)

def qdump__CPlusPlus__Literal(d, value):
    dumpLiteral(d, value)
    d.putPlainChildren(value)

def qdump__CPlusPlus__StringLiteral(d, value):
    d.putSimpleCharArray(value["_chars"], value["_size"])
    d.putPlainChildren(value)

def qdump__CPlusPlus__Internal__Value(d, value):
    d.putValue(value["l"])
    d.putPlainChildren(value)

def qdump__Utils__FileName(d, value):
    d.putStringValue(value)
    d.putPlainChildren(value)

def qdump__Utils__ElfSection(d, value):
    d.putByteArrayValue(value["name"])
    d.putPlainChildren(value)

def qdump__CPlusPlus__Token(d, value):
    k = value["f"]["kind"]
    e = int(k)
    type = str(k.cast(d.lookupType("CPlusPlus::Kind")))[11:] # Strip "CPlusPlus::"
    try:
        if e == 6:
            type = readLiteral(d, value["identifier"]) + " (%s)" % type
        elif e >= 7 and e <= 23:
            type = readLiteral(d, value["literal"]) + " (%s)" % type
    except:
        pass
    d.putValue(type)
    d.putPlainChildren(value)

def qdump__CPlusPlus__Internal__PPToken(d, value):
    data, size, alloc = d.byteArrayData(value["m_src"])
    length = int(value["f"]["utf16chars"])
    offset = int(value["utf16charOffset"])
    #warn("size: %s, alloc: %s, offset: %s, length: %s, data: %s"
    #    % (size, alloc, offset, length, data))
    d.putValue(d.readMemory(data + offset, min(100, length)),
        Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__ProString(d, value):
    try:
        s = value["m_string"]
        data, size, alloc = d.stringData(s)
        data += 2 * int(value["m_offset"])
        size = int(value["m_length"])
        s = d.readMemory(data, 2 * size)
        d.putValue(s, Hex4EncodedLittleEndian)
    except:
        d.putEmptyValue()
    d.putPlainChildren(value)

<|endoftext|>


<commit_before>import http.client
import json
import decimal

class BTERConnection:
    domain = 'data.bter.com'
    def __init__(self, timeout=30):
        self.conn = http.client.HTTPSConnection(self.domain, timeout=timeout)

    def close(self):
        self.conn.close()

    def makeRequest(self, url, method='POST', extra_headers=None, params=''):
        headers = {"Content-type": "application/x-www-form-urlencoded"}
        if extra_headers is not None:
            headers.update(extra_headers)

        self.conn.request(method, url, params, headers)
        response = self.conn.getresponse().read()

        return response

    def makeJSONRequest(self, url, method='POST', extra_headers=None, params=""):
        response = self.makeRequest(url, method, extra_headers, params)
        return parseJSONResponse(response)

def parseJSONResponse(response):
    def parse_decimal(var):
        return decimal.Decimal(var)

    try:
        r = json.loads(response, parse_float=parse_decimal, parse_int=parse_decimal)
    except Exception as e:
        msg = "Error while attempting to parse JSON response: %s\nResponse:\n%r" % (e, response)
        raise Exception(msg)

    return r
<commit_msg>adding decode to response to allow for direct parsing<commit_after>import http.client
import json
import decimal

class BTERConnection:
    domain = 'data.bter.com'
    def __init__(self, timeout=30):
        self.conn = http.client.HTTPSConnection(self.domain, timeout=timeout)

    def close(self):
        self.conn.close()

    def makeRequest(self, url, method='POST', extra_headers=None, params=''):
        headers = {"Content-type": "application/x-www-form-urlencoded"}
        if extra_headers is not None:
            headers.update(extra_headers)

        self.conn.request(method, url, params, headers)
        response = self.conn.getresponse().read().decode()

        return response

    def makeJSONRequest(self, url, method='POST', extra_headers=None, params=""):
        response = self.makeRequest(url, method, extra_headers, params)
        return parseJSONResponse(response)

def parseJSONResponse(response):
    def parse_decimal(var):
        return decimal.Decimal(var)

    try:
        r = json.loads(response, parse_float=parse_decimal, parse_int=parse_decimal)
    except Exception as e:
        msg = "Error while attempting to parse JSON response: %s\nResponse:\n%r" % (e, response)
        raise Exception(msg)

    return r
<|endoftext|>


<commit_before>############################################################################
#
# Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies).
# Contact: http://www.qt-project.org/legal
#
# This file is part of Qt Creator.
#
# Commercial License Usage
# Licensees holding valid commercial Qt licenses may use this file in
# accordance with the commercial license agreement provided with the
# Software or, alternatively, in accordance with the terms contained in
# a written agreement between you and Digia.  For licensing terms and
# conditions see http://qt.digia.com/licensing.  For further information
# use the contact form at http://qt.digia.com/contact-us.
#
# GNU Lesser General Public License Usage
# Alternatively, this file may be used under the terms of the GNU Lesser
# General Public License version 2.1 as published by the Free Software
# Foundation and appearing in the file LICENSE.LGPL included in the
# packaging of this file.  Please review the following information to
# ensure the GNU Lesser General Public License version 2.1 requirements
# will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
#
# In addition, as a special exception, Digia gives you certain additional
# rights.  These rights are described in the Digia Qt LGPL Exception
# version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
#
#############################################################################

from dumper import *

def qdump__Core__Id(d, value):
    try:
        name = d.parseAndEvaluate("Core::nameForId(%d)" % value["m_id"])
        d.putValue(d.encodeCharArray(name), Hex2EncodedLatin1)
        d.putPlainChildren(value)
    except:
        d.putValue(value["m_id"])
        d.putNumChild(0)

def qdump__Debugger__Internal__GdbMi(d, value):
    str = d.encodeByteArray(value["m_name"]) + "3a20" \
        + d.encodeByteArray(value["m_data"])
    d.putValue(str, Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__Debugger__Internal__WatchData(d, value):
    d.putByteArrayValue(value["iname"])
    d.putPlainChildren(value)

def qdump__Debugger__Internal__WatchItem(d, value):
    d.putByteArrayValue(value["iname"])
    d.putPlainChildren(value)

def qdump__Debugger__Internal__BreakpointModelId(d, value):
    d.putValue("%s.%s" % (int(value["m_majorPart"]), int(value["m_minorPart"])))
    d.putPlainChildren(value)

def qdump__Debugger__Internal__ThreadId(d, value):
    d.putValue("%s" % value["m_id"])
    d.putPlainChildren(value)

def qdump__CPlusPlus__ByteArrayRef(d, value):
    d.putValue(d.encodeCharArray(value["m_start"], 100, value["m_length"]),
        Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__CPlusPlus__Identifier(d, value):
    d.putValue(d.encodeCharArray(value["_chars"]), Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__CPlusPlus__IntegerType(d, value):
    d.putValue(value["_kind"])
    d.putPlainChildren(value)

def qdump__CPlusPlus__NamedType(d, value):
    literal = downcast(value["_name"])
    d.putValue(d.encodeCharArray(literal["_chars"]), Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__CPlusPlus__TemplateNameId(d, value):
    s = d.encodeCharArray(value["_identifier"]["_chars"])
    d.putValue(s + "3c2e2e2e3e", Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__CPlusPlus__Literal(d, value):
    d.putValue(d.encodeCharArray(value["_chars"]), Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__CPlusPlus__StringLiteral(d, value):
    d.putValue(d.encodeCharArray(value["_chars"]), Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__CPlusPlus__Internal__Value(d, value):
    d.putValue(value["l"])
    d.putPlainChildren(value)

def qdump__Utils__FileName(d, value):
    d.putStringValue(value)
    d.putPlainChildren(value)

def qdump__Utils__ElfSection(d, value):
    d.putByteArrayValue(value["name"])
    d.putPlainChildren(value)

def qdump__CPlusPlus__Token(d, value):
    k = value["f"]["kind"]
    if int(k) == 6:
        d.putValue("T_IDENTIFIER. offset: %d, len: %d"
            % (value["offset"], value["f"]["length"]))
    elif int(k) == 7:
        d.putValue("T_NUMERIC_LITERAL. offset: %d, value: %d"
            % (value["offset"], value["f"]["length"]))
    else:
        val = str(k.cast(d.lookupType("CPlusPlus::Kind")))
        d.putValue(val[11:]) # Strip "CPlusPlus::"
    d.putPlainChildren(value)

def qdump__CPlusPlus__Internal__PPToken(d, value):
    data, size, alloc = d.byteArrayData(value["m_src"])
    length = int(value["f"]["length"])
    offset = int(value["offset"])
    #warn("size: %s, alloc: %s, offset: %s, length: %s, data: %s"
    #    % (size, alloc, offset, length, data))
    d.putValue(d.readMemory(data + offset, min(100, length)),
        Hex2EncodedLatin1)
    d.putPlainChildren(value)
<commit_msg>Debugger: Fix internal type dumper<commit_after>############################################################################
#
# Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies).
# Contact: http://www.qt-project.org/legal
#
# This file is part of Qt Creator.
#
# Commercial License Usage
# Licensees holding valid commercial Qt licenses may use this file in
# accordance with the commercial license agreement provided with the
# Software or, alternatively, in accordance with the terms contained in
# a written agreement between you and Digia.  For licensing terms and
# conditions see http://qt.digia.com/licensing.  For further information
# use the contact form at http://qt.digia.com/contact-us.
#
# GNU Lesser General Public License Usage
# Alternatively, this file may be used under the terms of the GNU Lesser
# General Public License version 2.1 as published by the Free Software
# Foundation and appearing in the file LICENSE.LGPL included in the
# packaging of this file.  Please review the following information to
# ensure the GNU Lesser General Public License version 2.1 requirements
# will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
#
# In addition, as a special exception, Digia gives you certain additional
# rights.  These rights are described in the Digia Qt LGPL Exception
# version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
#
#############################################################################

from dumper import *

def qdump__Core__Id(d, value):
    try:
        name = d.parseAndEvaluate("Core::nameForId(%d)" % value["m_id"])
        d.putValue(d.encodeCharArray(name), Hex2EncodedLatin1)
        d.putPlainChildren(value)
    except:
        d.putValue(value["m_id"])
        d.putNumChild(0)

def qdump__Debugger__Internal__GdbMi(d, value):
    str = d.encodeByteArray(value["m_name"]) + "3a20" \
        + d.encodeByteArray(value["m_data"])
    d.putValue(str, Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__Debugger__Internal__WatchData(d, value):
    d.putByteArrayValue(value["iname"])
    d.putPlainChildren(value)

def qdump__Debugger__Internal__WatchItem(d, value):
    d.putByteArrayValue(value["iname"])
    d.putPlainChildren(value)

def qdump__Debugger__Internal__BreakpointModelId(d, value):
    d.putValue("%s.%s" % (int(value["m_majorPart"]), int(value["m_minorPart"])))
    d.putPlainChildren(value)

def qdump__Debugger__Internal__ThreadId(d, value):
    d.putValue("%s" % value["m_id"])
    d.putPlainChildren(value)

def qdump__CPlusPlus__ByteArrayRef(d, value):
    d.putValue(d.encodeCharArray(value["m_start"], 100, value["m_length"]),
        Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__CPlusPlus__Identifier(d, value):
    d.putValue(d.encodeCharArray(value["_chars"]), Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__CPlusPlus__IntegerType(d, value):
    d.putValue(value["_kind"])
    d.putPlainChildren(value)

def qdump__CPlusPlus__NamedType(d, value):
    literal = d.downcast(value["_name"])
    d.putValue(d.encodeCharArray(literal["_chars"]), Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__CPlusPlus__TemplateNameId(d, value):
    s = d.encodeCharArray(value["_identifier"]["_chars"])
    d.putValue(s + "3c2e2e2e3e", Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__CPlusPlus__Literal(d, value):
    d.putValue(d.encodeCharArray(value["_chars"]), Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__CPlusPlus__StringLiteral(d, value):
    d.putValue(d.encodeCharArray(value["_chars"]), Hex2EncodedLatin1)
    d.putPlainChildren(value)

def qdump__CPlusPlus__Internal__Value(d, value):
    d.putValue(value["l"])
    d.putPlainChildren(value)

def qdump__Utils__FileName(d, value):
    d.putStringValue(value)
    d.putPlainChildren(value)

def qdump__Utils__ElfSection(d, value):
    d.putByteArrayValue(value["name"])
    d.putPlainChildren(value)

def qdump__CPlusPlus__Token(d, value):
    k = value["f"]["kind"]
    if int(k) == 6:
        d.putValue("T_IDENTIFIER. offset: %d, len: %d"
            % (value["offset"], value["f"]["length"]))
    elif int(k) == 7:
        d.putValue("T_NUMERIC_LITERAL. offset: %d, value: %d"
            % (value["offset"], value["f"]["length"]))
    else:
        val = str(k.cast(d.lookupType("CPlusPlus::Kind")))
        d.putValue(val[11:]) # Strip "CPlusPlus::"
    d.putPlainChildren(value)

def qdump__CPlusPlus__Internal__PPToken(d, value):
    data, size, alloc = d.byteArrayData(value["m_src"])
    length = int(value["f"]["length"])
    offset = int(value["offset"])
    #warn("size: %s, alloc: %s, offset: %s, length: %s, data: %s"
    #    % (size, alloc, offset, length, data))
    d.putValue(d.readMemory(data + offset, min(100, length)),
        Hex2EncodedLatin1)
    d.putPlainChildren(value)
<|endoftext|>


<commit_before>import html

from data_types.issue import Issue
from data_types.repository import Repository
from data_types.user import User
from .base import EventBase

class EventIssues(EventBase):

    def __init__(self, sdk):
        self.issue = None
        self.repository = None
        self.sender = None
        self.sdk = sdk

    """
    IssuesEvent

    Triggered when an issue is assigned, unassigned, labeled, unlabeled,
    opened, edited, milestoned, demilestoned, closed, or reopened.

    https://developer.github.com/v3/activity/events/types/#issuesevent
    """

    async def process(self, payload, chat_id):
        """
        Processes Issues event
        :param payload: JSON object with payload
        :param chat_id: current user chat token
        :return:
        """

        self.sdk.log("Issues event payload taken")

        try:
            self.issue = Issue(payload['issue'])
            self.repository = Repository(payload['repository'])
            self.sender = User(payload['sender'])

        except Exception as e:
            self.sdk.log('Cannot process IssuesEvent payload because of {}'.format(e))

        action = payload['action']

        available_actions = {
            'opened': self.opened,
            'closed': self.closed,
            'assigned': self.assigned
        }

        if action not in available_actions:
            self.sdk.log('Unsupported Issues action: {}'.format(action))
            return

        # call action handler
        await available_actions[action](chat_id, payload)

    async def opened(self, chat_id, payload):
        """
        Issue opened action
        :param chat_id: Current user chat token
        :param payload: GitHub payload
        :return:
        """

        message = "✏ {} opened new issue «{}» [{}]".format(
                        self.sender.login,
                        html.escape(self.issue.title),
                        self.repository.html_url,
                        self.repository.name
                    ) + "\n\n"

        if len(self.issue.body):
            message += html.escape(self.issue.body) + "\n\n"

        message += self.issue.html_url

        await self.sdk.send_text_to_chat(
            chat_id,
            message,
            'HTML'
        )

    async def closed(self, chat_id, payload):
        """
        Close issue action
        :param chat_id: Current user chat token
        :param payload: GitHub payload
        :return:
        """
        message = "✅ {} closes issue «{}» [{}]".format(
            self.sender.login,
            html.escape(self.issue.title),
            self.repository.html_url,
            self.repository.name
        ) + "\n\n"

        if len(self.issue.body):
            message += html.escape(self.issue.body) + "\n\n"

        message += self.issue.html_url

        await self.sdk.send_text_to_chat(
            chat_id,
            message,
            'HTML'
        )

    async def assigned(self, chat_id, payload):
        """
        Issue assigned action
        :param chat_id: Current user chat token
        :param payload: GitHub payload
        :return:
        """

        assignee = User(payload['assignee'])

        message = "📌 {assignee} has been assigned to the issue «{issue_title}» " \
                  "by {author} [{repository_name}]".format(
                    assignee=assignee.login,
                    author=self.sender.login,
                    issue_title=html.escape(self.issue.title),
                    repository_name=self.repository.name
        ) + "\n\n"

        message += self.issue.html_url

        await self.sdk.send_text_to_chat(
            chat_id,
            message,
            'HTML'
        )
<commit_msg>Update event issue closed message<commit_after>import html

from data_types.issue import Issue
from data_types.repository import Repository
from data_types.user import User
from .base import EventBase

class EventIssues(EventBase):

    def __init__(self, sdk):
        self.issue = None
        self.repository = None
        self.sender = None
        self.sdk = sdk

    """
    IssuesEvent

    Triggered when an issue is assigned, unassigned, labeled, unlabeled,
    opened, edited, milestoned, demilestoned, closed, or reopened.

    https://developer.github.com/v3/activity/events/types/#issuesevent
    """

    async def process(self, payload, chat_id):
        """
        Processes Issues event
        :param payload: JSON object with payload
        :param chat_id: current user chat token
        :return:
        """

        self.sdk.log("Issues event payload taken")

        try:
            self.issue = Issue(payload['issue'])
            self.repository = Repository(payload['repository'])
            self.sender = User(payload['sender'])

        except Exception as e:
            self.sdk.log('Cannot process IssuesEvent payload because of {}'.format(e))

        action = payload['action']

        available_actions = {
            'opened': self.opened,
            'closed': self.closed,
            'assigned': self.assigned
        }

        if action not in available_actions:
            self.sdk.log('Unsupported Issues action: {}'.format(action))
            return

        # call action handler
        await available_actions[action](chat_id, payload)

    async def opened(self, chat_id, payload):
        """
        Issue opened action
        :param chat_id: Current user chat token
        :param payload: GitHub payload
        :return:
        """

        message = "✏ {} opened new issue «{}» [{}]".format(
                        self.sender.login,
                        html.escape(self.issue.title),
                        self.repository.html_url,
                        self.repository.name
                    ) + "\n\n"

        if len(self.issue.body):
            message += html.escape(self.issue.body) + "\n\n"

        message += self.issue.html_url

        await self.sdk.send_text_to_chat(
            chat_id,
            message,
            'HTML'
        )

    async def closed(self, chat_id, payload):
        """
        Close issue action
        :param chat_id: Current user chat token
        :param payload: GitHub payload
        :return:
        """
        message = "☑ {} closed issue «{}» [{}]".format(
            self.sender.login,
            html.escape(self.issue.title),
            self.repository.html_url,
            self.repository.name
        ) + "\n\n"

        if len(self.issue.body):
            message += html.escape(self.issue.body) + "\n\n"

        message += self.issue.html_url

        await self.sdk.send_text_to_chat(
            chat_id,
            message,
            'HTML'
        )

    async def assigned(self, chat_id, payload):
        """
        Issue assigned action
        :param chat_id: Current user chat token
        :param payload: GitHub payload
        :return:
        """

        assignee = User(payload['assignee'])

        message = "📌 {assignee} has been assigned to the issue «{issue_title}» " \
                  "by {author} [{repository_name}]".format(
                    assignee=assignee.login,
                    author=self.sender.login,
                    issue_title=html.escape(self.issue.title),
                    repository_name=self.repository.name
        ) + "\n\n"

        message += self.issue.html_url

        await self.sdk.send_text_to_chat(
            chat_id,
            message,
            'HTML'
        )
<|endoftext|>


<commit_before>from unittest.mock import MagicMock

from buffpy.models.link import Link

def test_links_shares():
    """ Test link"s shares retrieving from constructor. """

    mocked_api = MagicMock()
    mocked_api.get.return_value = {"shares": 123}

    link = Link(api=mocked_api, url="www.google.com")

    assert link["shares"] == 123
    assert link["url"] == "www.google.com"
    mocked_api.get.assert_called_once_with(url="links/shares.json?url=www.google.com")

def test_links_get_shares():
    """ Test link"s shares retrieving method. """

    mocked_api = MagicMock()
    mocked_api.get.return_value = {"shares": 123}

    link = Link(api=mocked_api, url="www.google.com")

    assert link["shares"] == 123
    assert link["url"] == "www.google.com"
    assert link.get_shares() == 123

    mocked_api.get.assert_any_call(url="links/shares.json?url=www.google.com")
    assert mocked_api.get.call_count == 2
<commit_msg>Fix docs typo<commit_after>from unittest.mock import MagicMock

from buffpy.models.link import Link

def test_links_shares():
    """ Should retrieve shares in contructor. """

    mocked_api = MagicMock()
    mocked_api.get.return_value = {"shares": 123}

    link = Link(api=mocked_api, url="www.google.com")

    assert link["shares"] == 123
    assert link["url"] == "www.google.com"
    mocked_api.get.assert_called_once_with(url="links/shares.json?url=www.google.com")

def test_links_get_shares():
    """ Should properly retrieve shares. """

    mocked_api = MagicMock()
    mocked_api.get.return_value = {"shares": 123}

    link = Link(api=mocked_api, url="www.google.com")

    assert link["shares"] == 123
    assert link["url"] == "www.google.com"
    assert link.get_shares() == 123

    mocked_api.get.assert_any_call(url="links/shares.json?url=www.google.com")
    assert mocked_api.get.call_count == 2
<|endoftext|>


<commit_before>import logging
import imp

from django.conf import settings
from django.utils.importlib import import_module
from django.utils import timezone
from an_evt import views as an_views

import cronjobs
from models import CronLock, CronJob
from django.db import transaction
import datetime

log = logging.getLogger('cron')

LOCK = getattr(settings, 'CRONJOB_LOCK_PREFIX', 'lock')
LOCK_TIMEOUT = getattr(settings, 'LOCK_TIMEOUT', 5 * 60 * 60)

def run_all_jobs():
    transaction.commit_unless_managed()
    for app in settings.INSTALLED_APPS:
        try:
            app_path = import_module(app).__path__
        except AttributeError:
            continue

        try:
            imp.find_module('user_stats', app_path)
        except ImportError as e:
            continue

        import_module('%s.user_stats' % app)

    registered = cronjobs.registered
    parameters = cronjobs.parameters
    log.debug(registered)
    jobs_run = 0
    job_created = False
    for script_num in xrange(0,len(registered.keys())):
        script = registered.keys()[script_num]
        param_name = parameters.keys()[script_num]
        lock_name = 'django_cron.%s.%s' % (LOCK, script)

        #Get metadata about job
        job_data = CronJob.objects.filter(name=lock_name).order_by("date_created")
        job_data_count = job_data.count()
        time_between_runs = parameters[param_name].get('time_between_jobs',2 * 60 * 60)
        if job_data_count >=1:
            remove_locks(job_data[1:])
            job_data = job_data[0]
        else:
            job_data = CronJob(
                name = lock_name,
                time_between_runs = time_between_runs,
            )
            job_data.save()
            job_created = True

        #Check if job needs to run
        current_time = timezone.now()
        if job_data.date_modified < (current_time - datetime.timedelta(seconds=job_data.time_between_runs)) or job_created:
            # Acquire lock if needed.
            log.debug("Job needs to be run.")
            has_lock = False
            if script in cronjobs.registered_lock:
                try:
                    cron_locks = get_all_locks(lock_name)
                    lock_count = cron_locks.count()

                    #Remove extraneous locks
                    if lock_count>=1:
                        remove_locks(cron_locks[1:])
                        lock = cron_locks[0]
                        #Remove lock if it has timed out
                        if lock.date_modified < (current_time - datetime.timedelta(seconds=LOCK_TIMEOUT)):
                            lock.delete()
                        else:
                            has_lock = True
                        transaction.commit_unless_managed()

                    if not has_lock:
                        create_lock(lock_name)
                except:
                    log.exception("Could not modify locks correctly.")

            if not has_lock:
                try:
                    run_single_job(registered[script], parameters[script])
                    jobs_run+=1
                    job.data.time_between_runs = time_between_runs
                    job_data.save()
                except:
                    log.error("Failed to run job {0} with params {1}".format(script, parameters[script]))
                remove_locks(get_all_locks(lock_name))

    return jobs_run

def create_lock(lock_name):
    lock = CronLock(
        name=lock_name
    )
    lock.save()

def get_all_locks(lock_name):
    cron_locks = CronLock.objects.filter(name=lock_name).order_by("-date_created")
    return cron_locks

def remove_locks(cron_locks):
    for lock in cron_locks:
        lock.delete()

def run_single_job(f, params):
    log.info("Beginning job: %s %s" % (f,params))
    db = an_views.get_database(f)
    fs = an_views.get_filesystem(f)
    f(fs, db, params)
    log.info("Ending job: %s %s" % (f, params))
<commit_msg>Fix how cron handles time between jobs<commit_after>import logging
import imp

from django.conf import settings
from django.utils.importlib import import_module
from django.utils import timezone
from an_evt import views as an_views

import cronjobs
from models import CronLock, CronJob
from django.db import transaction
import datetime

log = logging.getLogger('cron')

LOCK = getattr(settings, 'CRONJOB_LOCK_PREFIX', 'lock')
LOCK_TIMEOUT = getattr(settings, 'LOCK_TIMEOUT', 5 * 60 * 60)

def run_all_jobs():
    transaction.commit_unless_managed()
    for app in settings.INSTALLED_APPS:
        try:
            app_path = import_module(app).__path__
        except AttributeError:
            continue

        try:
            imp.find_module('user_stats', app_path)
        except ImportError as e:
            continue

        import_module('%s.user_stats' % app)

    registered = cronjobs.registered
    parameters = cronjobs.parameters
    log.debug(registered)
    jobs_run = 0
    job_created = False
    for script_num in xrange(0,len(registered.keys())):
        script = registered.keys()[script_num]
        param_name = parameters.keys()[script_num]
        lock_name = 'django_cron.%s.%s' % (LOCK, script)

        #Get metadata about job
        job_data = CronJob.objects.filter(name=lock_name).order_by("date_created")
        job_data_count = job_data.count()
        time_between_runs = parameters[param_name].get('time_between_jobs',2 * 60 * 60)
        if job_data_count >=1:
            remove_locks(job_data[1:])
            job_data = job_data[0]
        else:
            job_data = CronJob(
                name = lock_name
            )
            job_data.save()
            job_created = True

        #Check if job needs to run
        current_time = timezone.now()
        if job_data.date_modified < (current_time - datetime.timedelta(seconds=time_between_runs)) or job_created or job_data.date_modified>current_time :
            # Acquire lock if needed.
            log.debug("Job needs to be run.")
            has_lock = False
            if script in cronjobs.registered_lock:
                try:
                    cron_locks = get_all_locks(lock_name)
                    lock_count = cron_locks.count()

                    #Remove extraneous locks
                    if lock_count>=1:
                        remove_locks(cron_locks[1:])
                        lock = cron_locks[0]
                        #Remove lock if it has timed out
                        if lock.date_modified < (current_time - datetime.timedelta(seconds=LOCK_TIMEOUT)):
                            lock.delete()
                        else:
                            has_lock = True
                        transaction.commit_unless_managed()

                    if not has_lock:
                        create_lock(lock_name)
                except:
                    log.exception("Could not modify locks correctly.")

            if not has_lock:
                try:
                    run_single_job(registered[script], parameters[script])
                    jobs_run+=1
                    job_data.save()
                except:
                    log.exception("Failed to run job {0} with params {1}".format(script, parameters[script]))
                remove_locks(get_all_locks(lock_name))

    return jobs_run

def create_lock(lock_name):
    lock = CronLock(
        name=lock_name
    )
    lock.save()

def get_all_locks(lock_name):
    cron_locks = CronLock.objects.filter(name=lock_name).order_by("-date_created")
    return cron_locks

def remove_locks(cron_locks):
    for lock in cron_locks:
        lock.delete()

def run_single_job(f, params):
    log.info("Beginning job: %s %s" % (f,params))
    db = an_views.get_database(f)
    fs = an_views.get_filesystem(f)
    f(fs, db, params)
    log.info("Ending job: %s %s" % (f, params))
<|endoftext|>


<commit_before>import re

from .utils import _perform_request, markdown_table_row, format_datetime, format_utc

conf = None

class IssueThread(object):
    '''
    Everything related to an issue in GitLab, e.g. the issue itself and subsequent comments.
    '''
    def __init__(self, config, fields):
        global conf
        conf = config
        self.load_objects(fields)

    def load_objects(self, fields):
        '''
        Load the issue object and the comment objects.
        If conf.dry_run=False, then Attachments are created in GitLab in this step.
        '''
        self.issue = Issue(fields)
        self.comments = []
        '''
        fields["long_desc"] gets peared down in Issue creation (above). This is because bugzilla
        lacks the concept of an issue description, so the first comment is harvested for
        the issue description, as well as any subsequent comments that are simply attachments
        from the original reporter. What remains below should be a list of genuine comments.
        '''

        for comment_fields in fields["long_desc"]:
            if comment_fields.get("thetext"):
                self.comments.append(Comment(comment_fields))

    def save(self):
        '''
        Save the issue and all of the comments to GitLab.
        If conf.dry_run=True, then only the HTTP request that would be made is printed.
        '''
        self.issue.save()

        for comment in self.comments:
            comment.issue_id = self.issue.id
            comment.save()

        # close the issue in GitLab, if it is resolved in Bugzilla
        if self.issue.status in conf.bugzilla_closed_states:
            self.issue.close()

class Issue(object):
    '''
    The issue model
    '''
    required_fields = ["sudo", "title", "description"]
    data_fields = ["sudo", "created_at", "title", "description", "assignee_ids", "milestone",
                   "labels"]

    def __init__(self, bugzilla_fields):
        self.headers = conf.default_headers
        validate_user(bugzilla_fields["reporter"])
        validate_user(bugzilla_fields["assigned_to"])
        self.load_fields(bugzilla_fields)

    def load_fields(self, fields):
        self.title = fields["short_desc"]
        self.sudo = conf.gitlab_users[conf.bugzilla_users[fields["reporter"]]]
        self.assignee_ids = [conf.gitlab_users[conf.bugzilla_users[fields["assigned_to"]]]]
        self.created_at = format_utc(fields["creation_ts"])
        self.status = fields["bug_status"]
        self.create_labels(fields["component"], fields.get("op_sys"))
        self.create_description(fields)

    def create_labels(self, component, operating_system):
        '''
        Creates 3 types of labels: default labels listed in the configuration, component labels,
        and operating system labels.
        '''
        labels = []
        if conf.default_gitlab_labels:
            labels.extend(conf.default_gitlab_labels)

        component_label = conf.component_mappings.get(component)
        if component_label:
            labels.append(component_label)

        # Do not create a label if the OS is other. That is a meaningless label.
        if operating_system and operating_system != "Other":
            labels.append(operating_system)

        self.labels = ",".join(labels)

    def create_description(self, fields):
        '''
        An opinionated description body creator.
        '''
        ext_description = ""

        # markdown table header
        self.description = markdown_table_row("", "")
        self.description += markdown_table_row("---", "---")

        if conf.include_bugzilla_link:
            bug_id = fields["bug_id"]
            link = "{}/show_bug.cgi?id={}".format(conf.bugzilla_base_url, bug_id)
            self.description += markdown_table_row("Bugzilla Link",
                                                   "[{}]({})".format(bug_id, link))

        formatted_dt = format_datetime(fields["creation_ts"], conf.datetime_format_string)
        self.description += markdown_table_row("Created on", formatted_dt)

        if fields.get("resolution"):
            self.description += markdown_table_row("Resolution", fields["resolution"])
            self.description += markdown_table_row("Resolved on",
                                                   format_datetime(fields["delta_ts"],
                                                                   conf.datetime_format_string))

        self.description += markdown_table_row("Version", fields.get("version"))
        self.description += markdown_table_row("OS", fields.get("op_sys"))
        self.description += markdown_table_row("Architecture", fields.get("rep_platform"))

        # add first comment to the issue description
        attachments = []
        to_delete = []
        comment0 = fields["long_desc"][0]
        if (fields["reporter"] == comment0["who"] and comment0["thetext"]):
            ext_description += "\n## Extended Description \n"
            ext_description += "\n\n".join(re.split("\n*", comment0["thetext"]))
            self.update_attachments(fields["reporter"], comment0, attachments)
            del fields["long_desc"][0]

        for i in range(0, len(fields["long_desc"])):
            comment = fields["long_desc"][i]
            if self.update_attachments(fields["reporter"], comment, attachments):
                to_delete.append(i)

        # delete comments that have already added to the issue description
        for i in reversed(to_delete):
            del fields["long_desc"][i]

        if attachments:
            self.description += markdown_table_row("Attachments", ", ".join(attachments))

        if ext_description:
            # for situations where the reporter is a generic or old user, specify the original
            # reporter in the description body
            if fields["reporter"] == conf.bugzilla_auto_reporter:
                # try to get reporter email from the body
                description, part, user_data = ext_description.rpartition("Submitter was ")
                # partition found matching string
                if part:
                    regex = r"^(\S*)\s?.*$"
                    email = re.match(regex, user_data, flags=re.M).group(1)
                    self.description += markdown_table_row("Reporter", email)
            # Add original reporter to the markdown table
            elif conf.bugzilla_users[fields["reporter"]] == conf.gitlab_misc_user:
                self.description += markdown_table_row("Reporter", fields["reporter"])

            self.description += ext_description

    def update_attachments(self, reporter, comment, attachments):
        '''
        Fetches attachments from comment if authored by reporter.
        '''
        if comment.get("attachid") and comment.get("who") == reporter:
            filename = Attachment.parse_file_description(comment.get("thetext"))
            attachment_markdown = Attachment(comment.get("attachid"), filename).save()
            attachments.append(attachment_markdown)
            return True
        return False

    def validate(self):
        for field in self.required_fields:
            value = getattr(self, field)
            if not value:
                raise Exception("Missing value for required field: {}".format(field))
        return True

    def save(self):
        self.validate()
        url = "{}/projects/{}/issues".format(conf.gitlab_base_url, conf.gitlab_project_id)
        data = {k: v for k, v in self.__dict__.items() if k in self.data_fields}
        self.headers["sudo"] = self.sudo

        response = _perform_request(url, "post", headers=self.headers, data=data, json=True,
                                    dry_run=conf.dry_run)

        if conf.dry_run:
            # assign a random number so that program can continue
            self.id = 5
            return

        self.id = response["iid"]

    def close(self):
        url = "{}/projects/{}/issues/{}".format(conf.gitlab_base_url, conf.gitlab_project_id,
                                                self.id)
        data = {
            "state_event": "close",
        }
        self.headers["sudo"] = self.sudo

        _perform_request(url, "put", headers=self.headers, data=data, dry_run=conf.dry_run)

class Comment(object):
    '''
    The comment model
    '''

    required_fields = ["sudo", "body", "issue_id"]
    data_fields = ["created_at", "body"]

    def __init__(self, bugzilla_fields):
        self.headers = conf.default_headers
        validate_user(bugzilla_fields["who"])
        self.load_fields(bugzilla_fields)

    def load_fields(self, fields):
        self.sudo = conf.gitlab_users[conf.bugzilla_users[fields["who"]]]
        # if unable to comment as the original user, put username in comment body
        self.created_at = format_utc(fields["bug_when"])
        if conf.bugzilla_users[fields["who"]] == conf.gitlab_misc_user:
            self.body = "By {} on {}\n\n".format(fields["who"],
                                                 format_datetime(fields["bug_when"],
                                                 conf.datetime_format_string))
        else:
            self.body = format_datetime(fields["bug_when"], conf.datetime_format_string)
            self.body += "\n\n"

        # if this comment is actually an attachment, upload the attachment and add the
        # markdown to the comment body
        if fields.get("attachid"):
            filename = Attachment.parse_file_description(fields["thetext"])
            attachment_markdown = Attachment(fields["attachid"], filename).save()
            self.body += attachment_markdown
        else:
            self.body += fields["thetext"]

    def validate(self):
        for field in self.required_fields:
            value = getattr(self, field)
            if not value:
                raise Exception("Missing value for required field: {}".format(field))

    def save(self):
        self.validate()
        self.headers["sudo"] = self.sudo
        url = "{}/projects/{}/issues/{}/notes".format(conf.gitlab_base_url, conf.gitlab_project_id,
                                                      self.issue_id)
        data = {k: v for k, v in self.__dict__.items() if k in self.data_fields}

        _perform_request(url, "post", headers=self.headers, data=data, json=True,
                         dry_run=conf.dry_run)

class Attachment(object):
    '''
    The attachment model
    '''
    def __init__(self, bugzilla_attachment_id, file_description):
        self.id = bugzilla_attachment_id
        self.file_description = file_description
        self.headers = conf.default_headers

    @classmethod
    def parse_file_description(cls, comment):
        regex = "^Created attachment (\d*)\s?(.*)$"
        matches = re.match(regex, comment, flags=re.M)
        if not matches:
            raise Exception("Failed to match comment string: {}".format(comment))
        return matches.group(2)

    def save(self):
        url = "{}/attachment.cgi?id={}".format(conf.bugzilla_base_url, self.id)
        result = _perform_request(url, "get", json=False)
        filename = self.file_description
        # Use real filename to store attachment but descriptive name for issue text
        if 'Content-disposition' in result.headers:
            regex = "^.*; filename=\"(.*)\"$"
            matches = re.match(regex, result.headers['Content-disposition'], flags=re.M)
            if matches:
                filename = matches.group(1)

        url = "{}/projects/{}/uploads".format(conf.gitlab_base_url, conf.gitlab_project_id)
        f = {"file": (filename, result.content)}

        attachment = _perform_request(url, "post", headers=self.headers, files=f, json=True,
                                      dry_run=conf.dry_run)

        link = filename
        if attachment and attachment["markdown"]:
           # [mail_route.zip](/uploads/e943e69eb2478529f2f1c7c7ea00fb46/mail_route.zip)
            regex = "^\[.*\]\((.*)\)$"
            matches = re.match(regex, attachment["markdown"], flags=re.M)
            if matches:
                link = matches.group(1)

        return u"[{}]({})".format(self.file_description, link)

def validate_user(bugzilla_user):
    if bugzilla_user not in conf.bugzilla_users:
        raise Exception("Bugzilla user `{}` not found in user_mappings.yml. "
                        "Please add them before continuing.".format(bugzilla_user))
<commit_msg>corrected link parsing taking into account optional ! mark<commit_after>import re

from .utils import _perform_request, markdown_table_row, format_datetime, format_utc

conf = None

class IssueThread(object):
    '''
    Everything related to an issue in GitLab, e.g. the issue itself and subsequent comments.
    '''
    def __init__(self, config, fields):
        global conf
        conf = config
        self.load_objects(fields)

    def load_objects(self, fields):
        '''
        Load the issue object and the comment objects.
        If conf.dry_run=False, then Attachments are created in GitLab in this step.
        '''
        self.issue = Issue(fields)
        self.comments = []
        '''
        fields["long_desc"] gets peared down in Issue creation (above). This is because bugzilla
        lacks the concept of an issue description, so the first comment is harvested for
        the issue description, as well as any subsequent comments that are simply attachments
        from the original reporter. What remains below should be a list of genuine comments.
        '''

        for comment_fields in fields["long_desc"]:
            if comment_fields.get("thetext"):
                self.comments.append(Comment(comment_fields))

    def save(self):
        '''
        Save the issue and all of the comments to GitLab.
        If conf.dry_run=True, then only the HTTP request that would be made is printed.
        '''
        self.issue.save()

        for comment in self.comments:
            comment.issue_id = self.issue.id
            comment.save()

        # close the issue in GitLab, if it is resolved in Bugzilla
        if self.issue.status in conf.bugzilla_closed_states:
            self.issue.close()

class Issue(object):
    '''
    The issue model
    '''
    required_fields = ["sudo", "title", "description"]
    data_fields = ["sudo", "created_at", "title", "description", "assignee_ids", "milestone",
                   "labels"]

    def __init__(self, bugzilla_fields):
        self.headers = conf.default_headers
        validate_user(bugzilla_fields["reporter"])
        validate_user(bugzilla_fields["assigned_to"])
        self.load_fields(bugzilla_fields)

    def load_fields(self, fields):
        self.title = fields["short_desc"]
        self.sudo = conf.gitlab_users[conf.bugzilla_users[fields["reporter"]]]
        self.assignee_ids = [conf.gitlab_users[conf.bugzilla_users[fields["assigned_to"]]]]
        self.created_at = format_utc(fields["creation_ts"])
        self.status = fields["bug_status"]
        self.create_labels(fields["component"], fields.get("op_sys"))
        self.create_description(fields)

    def create_labels(self, component, operating_system):
        '''
        Creates 3 types of labels: default labels listed in the configuration, component labels,
        and operating system labels.
        '''
        labels = []
        if conf.default_gitlab_labels:
            labels.extend(conf.default_gitlab_labels)

        component_label = conf.component_mappings.get(component)
        if component_label:
            labels.append(component_label)

        # Do not create a label if the OS is other. That is a meaningless label.
        if operating_system and operating_system != "Other":
            labels.append(operating_system)

        self.labels = ",".join(labels)

    def create_description(self, fields):
        '''
        An opinionated description body creator.
        '''
        ext_description = ""

        # markdown table header
        self.description = markdown_table_row("", "")
        self.description += markdown_table_row("---", "---")

        if conf.include_bugzilla_link:
            bug_id = fields["bug_id"]
            link = "{}/show_bug.cgi?id={}".format(conf.bugzilla_base_url, bug_id)
            self.description += markdown_table_row("Bugzilla Link",
                                                   "[{}]({})".format(bug_id, link))

        formatted_dt = format_datetime(fields["creation_ts"], conf.datetime_format_string)
        self.description += markdown_table_row("Created on", formatted_dt)

        if fields.get("resolution"):
            self.description += markdown_table_row("Resolution", fields["resolution"])
            self.description += markdown_table_row("Resolved on",
                                                   format_datetime(fields["delta_ts"],
                                                                   conf.datetime_format_string))

        self.description += markdown_table_row("Version", fields.get("version"))
        self.description += markdown_table_row("OS", fields.get("op_sys"))
        self.description += markdown_table_row("Architecture", fields.get("rep_platform"))

        # add first comment to the issue description
        attachments = []
        to_delete = []
        comment0 = fields["long_desc"][0]
        if (fields["reporter"] == comment0["who"] and comment0["thetext"]):
            ext_description += "\n## Extended Description \n"
            ext_description += "\n\n".join(re.split("\n*", comment0["thetext"]))
            self.update_attachments(fields["reporter"], comment0, attachments)
            del fields["long_desc"][0]

        for i in range(0, len(fields["long_desc"])):
            comment = fields["long_desc"][i]
            if self.update_attachments(fields["reporter"], comment, attachments):
                to_delete.append(i)

        # delete comments that have already added to the issue description
        for i in reversed(to_delete):
            del fields["long_desc"][i]

        if attachments:
            self.description += markdown_table_row("Attachments", ", ".join(attachments))

        if ext_description:
            # for situations where the reporter is a generic or old user, specify the original
            # reporter in the description body
            if fields["reporter"] == conf.bugzilla_auto_reporter:
                # try to get reporter email from the body
                description, part, user_data = ext_description.rpartition("Submitter was ")
                # partition found matching string
                if part:
                    regex = r"^(\S*)\s?.*$"
                    email = re.match(regex, user_data, flags=re.M).group(1)
                    self.description += markdown_table_row("Reporter", email)
            # Add original reporter to the markdown table
            elif conf.bugzilla_users[fields["reporter"]] == conf.gitlab_misc_user:
                self.description += markdown_table_row("Reporter", fields["reporter"])

            self.description += ext_description

    def update_attachments(self, reporter, comment, attachments):
        '''
        Fetches attachments from comment if authored by reporter.
        '''
        if comment.get("attachid") and comment.get("who") == reporter:
            filename = Attachment.parse_file_description(comment.get("thetext"))
            attachment_markdown = Attachment(comment.get("attachid"), filename).save()
            attachments.append(attachment_markdown)
            return True
        return False

    def validate(self):
        for field in self.required_fields:
            value = getattr(self, field)
            if not value:
                raise Exception("Missing value for required field: {}".format(field))
        return True

    def save(self):
        self.validate()
        url = "{}/projects/{}/issues".format(conf.gitlab_base_url, conf.gitlab_project_id)
        data = {k: v for k, v in self.__dict__.items() if k in self.data_fields}
        self.headers["sudo"] = self.sudo

        response = _perform_request(url, "post", headers=self.headers, data=data, json=True,
                                    dry_run=conf.dry_run)

        if conf.dry_run:
            # assign a random number so that program can continue
            self.id = 5
            return

        self.id = response["iid"]

    def close(self):
        url = "{}/projects/{}/issues/{}".format(conf.gitlab_base_url, conf.gitlab_project_id,
                                                self.id)
        data = {
            "state_event": "close",
        }
        self.headers["sudo"] = self.sudo

        _perform_request(url, "put", headers=self.headers, data=data, dry_run=conf.dry_run)

class Comment(object):
    '''
    The comment model
    '''

    required_fields = ["sudo", "body", "issue_id"]
    data_fields = ["created_at", "body"]

    def __init__(self, bugzilla_fields):
        self.headers = conf.default_headers
        validate_user(bugzilla_fields["who"])
        self.load_fields(bugzilla_fields)

    def load_fields(self, fields):
        self.sudo = conf.gitlab_users[conf.bugzilla_users[fields["who"]]]
        # if unable to comment as the original user, put username in comment body
        self.created_at = format_utc(fields["bug_when"])
        if conf.bugzilla_users[fields["who"]] == conf.gitlab_misc_user:
            self.body = "By {} on {}\n\n".format(fields["who"],
                                                 format_datetime(fields["bug_when"],
                                                 conf.datetime_format_string))
        else:
            self.body = format_datetime(fields["bug_when"], conf.datetime_format_string)
            self.body += "\n\n"

        # if this comment is actually an attachment, upload the attachment and add the
        # markdown to the comment body
        if fields.get("attachid"):
            filename = Attachment.parse_file_description(fields["thetext"])
            attachment_markdown = Attachment(fields["attachid"], filename).save()
            self.body += attachment_markdown
        else:
            self.body += fields["thetext"]

    def validate(self):
        for field in self.required_fields:
            value = getattr(self, field)
            if not value:
                raise Exception("Missing value for required field: {}".format(field))

    def save(self):
        self.validate()
        self.headers["sudo"] = self.sudo
        url = "{}/projects/{}/issues/{}/notes".format(conf.gitlab_base_url, conf.gitlab_project_id,
                                                      self.issue_id)
        data = {k: v for k, v in self.__dict__.items() if k in self.data_fields}

        _perform_request(url, "post", headers=self.headers, data=data, json=True,
                         dry_run=conf.dry_run)

class Attachment(object):
    '''
    The attachment model
    '''
    def __init__(self, bugzilla_attachment_id, file_description):
        self.id = bugzilla_attachment_id
        self.file_description = file_description
        self.headers = conf.default_headers

    @classmethod
    def parse_file_description(cls, comment):
        regex = "^Created attachment (\d*)\s?(.*)$"
        matches = re.match(regex, comment, flags=re.M)
        if not matches:
            raise Exception("Failed to match comment string: {}".format(comment))
        return matches.group(2)

    def save(self):
        url = "{}/attachment.cgi?id={}".format(conf.bugzilla_base_url, self.id)
        result = _perform_request(url, "get", json=False)
        filename = self.file_description
        # Use real filename to store attachment but descriptive name for issue text
        if 'Content-disposition' in result.headers:
            regex = "^.*; filename=\"(.*)\"$"
            matches = re.match(regex, result.headers['Content-disposition'], flags=re.M)
            if matches:
                filename = matches.group(1)

        url = "{}/projects/{}/uploads".format(conf.gitlab_base_url, conf.gitlab_project_id)
        f = {"file": (filename, result.content)}

        attachment = _perform_request(url, "post", headers=self.headers, files=f, json=True,
                                      dry_run=conf.dry_run)

        link = filename
        if attachment and attachment["markdown"]:
           # [mail_route.zip](/uploads/e943e69eb2478529f2f1c7c7ea00fb46/mail_route.zip)
            regex = "^!?\[.*\]\((.*)\)$"
            matches = re.match(regex, attachment["markdown"], flags=re.M)
            if matches:
                link = matches.group(1)

        return u"[{}]({})".format(self.file_description, link)

def validate_user(bugzilla_user):
    if bugzilla_user not in conf.bugzilla_users:
        raise Exception("Bugzilla user `{}` not found in user_mappings.yml. "
                        "Please add them before continuing.".format(bugzilla_user))
<|endoftext|>


<commit_before>from github2.core import BaseData, GithubCommand, Attribute

class Repository(BaseData):
    name = Attribute("Name of repository.")
    description = Attribute("Repository description.")
    forks = Attribute("Number of forks of this repository.")
    watchers = Attribute("Number of people watching this repository.")
    private = Attribute("If True, the repository is private.")
    url = Attribute("Canonical URL to this repository")
    fork = Attribute("If True, this is a fork of another repository.")
    owner = Attribute("Username of the user owning this repository.")
    homepage = Attribute("Homepage for this project.")
    open_issues = Attribute("List of open issues for this repository.")

    def __repr__(self):
        return "<Repository: %s/%s>" % (self.owner, self.name)

class Repositories(GithubCommand):
    domain = "repos"

    def search(self, query):
        return self.make_request("search", query, filter="repositories")

    def show(self, project):
        return self.get_value("show", project, filter="repository",
                              datatype=Repository)

    def list(self, for_user=None):
        """Return a list of all repositories for a user.

        If no user is given, repositoris for the currently logged in user are
        returned.
        """
        for_user = for_user or self.request.username
        return self.get_values("show", for_user, filter="repositories",
                               datatype=Repository)

    def watch(self, project):
        return self.make_request("watch", project)

    def unwatch(self, project):
        return self.make_request("unwatch", project)

    def fork(self, project):
        return self.get_value("fork", project, filter="repository",
                              datatype=Repository)

    def create(self, name, description=None, homepage=None, public=True):
        repo_data = {"name": name, "description": description,
                     "homepage": homepage, "public": str(int(public))}
        return self.get_value("create", post_data=repo_data,
                              filter="repository", datatype=Repository)

    def delete(self, name):
        return self.make_request("delete", name)

    def set_private(self, repo_name):
        return self.make_request("set/private", repo_name)

    def set_public(self, repo_name):
        return self.make_request("set/public", repo_name)

    def list_collaborators(self, project):
        """Lists all the collaborators in a project (user/repro)."""
        return self.make_request("show", project, "collaborators",
                                 filter="collaborators")

    def add_collaborator(self, repo_name, username):
        """Adds an add_collaborator to a repro.

        Do not prefix repro_name with the user owning the repro like you
        do in list_collaborators()"""
        return self.make_request("collaborators", repo_name, "add", username)

    def remove_collaborator(self, repo_name, username):
        """Removes an add_collaborator from a repro.

        Do not prefix repro_name with the user owning the repro like you
        do in list_collaborators()"""
        return self.make_request("collaborators", repo_name, "remove",
                                 username, method="POST")

    def network(self, project):
        return self.make_request("show", project, "network", filter="network")

    def tags(self, project):
        return self.make_request("show", project, "tags", filter="tags")

    def branches(self, project):
        return self.make_request("show", project, "branches",
                                 filter="branches")

     def list_contributors(self, project):
         """Lists all the contributors in a project (user/repro)."""
         return self.make_request("show", project, "contributors",
                               filter="contributors")<commit_msg>Now supports contributors<commit_after>from github2.core import BaseData, GithubCommand, Attribute

class Repository(BaseData):
    name = Attribute("Name of repository.")
    description = Attribute("Repository description.")
    forks = Attribute("Number of forks of this repository.")
    watchers = Attribute("Number of people watching this repository.")
    private = Attribute("If True, the repository is private.")
    url = Attribute("Canonical URL to this repository")
    fork = Attribute("If True, this is a fork of another repository.")
    owner = Attribute("Username of the user owning this repository.")
    homepage = Attribute("Homepage for this project.")
    open_issues = Attribute("List of open issues for this repository.")

    def __repr__(self):
        return "<Repository: %s/%s>" % (self.owner, self.name)

class Repositories(GithubCommand):
    domain = "repos"

    def search(self, query):
        return self.make_request("search", query, filter="repositories")

    def show(self, project):
        return self.get_value("show", project, filter="repository",
                              datatype=Repository)

    def list(self, for_user=None):
        """Return a list of all repositories for a user.

        If no user is given, repositoris for the currently logged in user are
        returned.
        """
        for_user = for_user or self.request.username
        return self.get_values("show", for_user, filter="repositories",
                               datatype=Repository)

    def watch(self, project):
        return self.make_request("watch", project)

    def unwatch(self, project):
        return self.make_request("unwatch", project)

    def fork(self, project):
        return self.get_value("fork", project, filter="repository",
                              datatype=Repository)

    def create(self, name, description=None, homepage=None, public=True):
        repo_data = {"name": name, "description": description,
                     "homepage": homepage, "public": str(int(public))}
        return self.get_value("create", post_data=repo_data,
                              filter="repository", datatype=Repository)

    def delete(self, name):
        return self.make_request("delete", name)

    def set_private(self, repo_name):
        return self.make_request("set/private", repo_name)

    def set_public(self, repo_name):
        return self.make_request("set/public", repo_name)

    def list_collaborators(self, project):
        """Lists all the collaborators in a project (user/repro)."""
        return self.make_request("show", project, "collaborators",
                                 filter="collaborators")

    def add_collaborator(self, repo_name, username):
        """Adds an add_collaborator to a repro.

        Do not prefix repro_name with the user owning the repro like you
        do in list_collaborators()"""
        return self.make_request("collaborators", repo_name, "add", username)

    def remove_collaborator(self, repo_name, username):
        """Removes an add_collaborator from a repro.

        Do not prefix repro_name with the user owning the repro like you
        do in list_collaborators()"""
        return self.make_request("collaborators", repo_name, "remove",
                                 username, method="POST")

    def network(self, project):
        return self.make_request("show", project, "network", filter="network")

    def tags(self, project):
        return self.make_request("show", project, "tags", filter="tags")

    def branches(self, project):
        return self.make_request("show", project, "branches",
                                 filter="branches")

    def list_contributors(self, project):
        """Lists all the contributors in a project (user/repro)."""
        return self.make_request("show", project, "contributors",
                           filter="contributors")<|endoftext|>


<commit_before>from django import template

register = template.Library()

@register.inclusion_tag('show/inclusion_tags/radioshow_entryitem_listing.html', takes_context=True)
def radioshow_entryitem_listing(context, object_list):
    context.update({'object_list': object_list})
    return context
<commit_msg>added header and detail tags<commit_after>from django import template

register = template.Library()

@register.inclusion_tag('show/inclusion_tags/radioshow_entryitem_listing.html', takes_context=True)
def radioshow_entryitem_listing(context, object_list):
    context.update({'object_list': object_list})
    return context

@register.inclusion_tag('show/inclusion_tags/showcontributor_header.html')
def showcontributor_header(obj):
    return {'object': obj}

@register.inclusion_tag('show/inclusion_tags/showcontributor_detail.html')
def showcontributor_detail(obj):
    return {'object': obj}
<|endoftext|>