<commit_before>import pickle
import inspect
import simplejson
import urllib2
import sys

from django.shortcuts import render
from django.http import HttpResponse
from django.conf import settings
from django.views.decorators.csrf import csrf_exempt

from fabric.api import *

from tailor.decorators import tailored

def schema(request):
    '''
    Parses the project fabfile and returns API listing
    the available commands.  Commands are made available by
    adding the @tailored (name?) decorator to a fabric function.
    '''

    #The directory that hold the fabfile needs to be added to the python path
    from conf import fabfile

    fab_props = dir(fabfile)

    # Exclude certain properties in the fabfile
    # TODO: replace by opting in with decorator, instead of opting everything else out
    exclude_list = ['__builtins__', '__doc__', '__file__', '__name__', '__package__', 'abort', 'cd', 'execute', 'fastprint', 'get', 'glob', 'hide', 'hosts',  'lcd',  'local',  'open_shell', 'os', 'output', 'parallel', 'path',  'prefix', 'prompt', 'put', 'puts', 'reboot',  'requests', 'require', 'roles', 'run', 'runs_once', 'serial', 'settings', 'setup', 'show', 'simplejson', 'ssh', 'sudo', 'task', 'time', 'warn', 'with_settings', 'with_statement', 'paths',]

    fab_dict = {}
    fab_tasks = {}
    fab_dependencies = {}
    fab_dict['tasks'] = fab_tasks
    fab_dict['dependencies'] = fab_dependencies

    for prop in fab_props:
        if not prop in exclude_list:
            # If it's a callable, pickle it
            if hasattr( eval('fabfile.%s' % prop), '__call__' ):
                if hasattr( eval('fabfile.%s' % prop), 'tailored' ):
                    _callable = eval('fabfile.%s' % prop)
                    callable_source = inspect.getsource(_callable)
                    fab_tasks[prop] = (pickle.dumps(callable_source))
                elif hasattr( eval('fabfile.%s' % prop), 'dependency' ):
                    _callable = eval('fabfile.%s' % prop)
                    callable_source = inspect.getsource(_callable)
                    fab_dependencies[prop] = (pickle.dumps(callable_source))
            # Else just use the value
            else:
                fab_dict[prop] = eval('fabfile.%s' % prop)

    #response = {}
    response = simplejson.dumps(fab_dict)

    return HttpResponse(response, mimetype='application/json', status=200)

@csrf_exempt
def fab(request):
    '''
    Accepts JSON (and more later on?) data describing fabric commands
    and runs them if they exist and are allowed.

    #Test it locally
    curl --dump-header - -H "Content-Type: application/json" -X POST --data '{"hosts": ["server1.example.com"],"commands": ["alpha", "kick_apache"] }' http://localhost:8000/tailor/api/v1/fab/

    # NOTE: This is all PoC at this point.  Lots of hard-coded values
    # TODO: Seperate all this out to methods
    '''

    import fabric

    # Turn off output as to not write against stdout and stderr

    if request.method == 'POST':

        try:
            _input = request.raw_post_data
            _input = simplejson.loads(request.raw_post_data)
            #print _input['hosts']
        #except JSONDecodeError, e:
            #print "Couldn't parse JSON: %s" % e
        except Exception, e:
            print "Error: %s" % e

        try:
            client_url = "http://localhost:8001/tailor/api/v1/schema/"
            client_data = urllib2.urlopen(client_url)
            client_json = client_data.read()
            client_dict = simplejson.loads(client_json)

            #Need this?
            from fabric.api import execute

            # TODO: dynamically configure the 'env' dict from
            #env = client_dict['env']
            #env.apache_bin_dir = "/etc/init.d/apache2"
            #env.user = 'fabric'

            #Set Host via POST Data
            env.hosts = _input['hosts']

            import StringIO
            file = open("fabric_stuff.py", "w")
            new_string = ""
            new_string = new_string + "from fabric.api import * \n\n\nimport fabric\n\n\n"

            new_string = new_string + '''fabric.state.output["status"] = False\nfabric.state.output["running"] = False\nfabric.state.output["user"] = False\nfabric.state.output["warnings"] = False\nfabric.state.output["stderr"] = False\nfabric.state.output['stdout'] = False\nfabric.state.output['aborts'] = False\n\n'''

            for _varname, _var in client_dict['env'].iteritems():
                if isinstance(_var, str):
                    new_string = new_string + "env.%s = \"%s\"" % (_varname, _var) + "\n"
                else:
                    new_string = new_string + "env.%s = %s" % (_varname, _var) + "\n"

            new_string = new_string + "\n\n"
            for dep, dep_func in client_dict['dependencies'].iteritems():
                new_string = new_string + pickle.loads(str(dep_func))

            new_string = new_string + "\n\n"
            for task, task_func in client_dict['tasks'].iteritems():
                new_string = new_string + pickle.loads(str(task_func))

            file.write(new_string)
            file.close()
            import fabric_stuff

            for command in _input['commands']:
                execute(eval("fabric_stuff." + command))

            import os
            os.remove("fabric_stuff.py")

            #respond
            response_dict = {'success':True, 'message':"Commands Executed"}
            response = simplejson.dumps(response_dict)

            return HttpResponse(response, mimetype='application/json', status=200)
        except Exception, e:
            print "Error: %s" % e
            response_dict = {'success':False, 'message':"Coudn't not execute commands"}
            response = simplejson.dumps(response_dict)
            return HttpResponse(response, mimetype='application/json', status=400)
    else:
        response = "Method is not allow. Only POST is allowed"
        return HttpResponse(response, status=400)

<commit_msg>Asking for forgiveness issue #5<commit_after>import pickle
import inspect
import simplejson
import urllib2
import sys

from django.shortcuts import render
from django.http import HttpResponse
from django.conf import settings
from django.views.decorators.csrf import csrf_exempt

from fabric.api import *

from tailor.decorators import tailored

def schema(request):
    '''
    Parses the project fabfile and returns API listing
    the available commands.  Commands are made available by
    adding the @tailored (name?) decorator to a fabric function.
    '''

    #The directory that hold the fabfile needs to be added to the python path
    from conf import fabfile

    fab_props = dir(fabfile)

    # Exclude certain properties in the fabfile
    # TODO: replace by opting in with decorator, instead of opting everything else out
    exclude_list = ['__builtins__', '__doc__', '__file__', '__name__', '__package__', 'abort', 'cd', 'execute', 'fastprint', 'get', 'glob', 'hide', 'hosts',  'lcd',  'local',  'open_shell', 'os', 'output', 'parallel', 'path',  'prefix', 'prompt', 'put', 'puts', 'reboot',  'requests', 'require', 'roles', 'run', 'runs_once', 'serial', 'settings', 'setup', 'show', 'simplejson', 'ssh', 'sudo', 'task', 'time', 'warn', 'with_settings', 'with_statement', 'paths',]

    fab_dict = {}
    fab_tasks = {}
    fab_dependencies = {}
    fab_dict['tasks'] = fab_tasks
    fab_dict['dependencies'] = fab_dependencies

    for prop in fab_props:
        if not prop in exclude_list:
            # If it's a callable, pickle it
            if hasattr( eval('fabfile.%s' % prop), '__call__' ):
                if hasattr( eval('fabfile.%s' % prop), 'tailored' ):
                    _callable = eval('fabfile.%s' % prop)
                    callable_source = inspect.getsource(_callable)
                    fab_tasks[prop] = (pickle.dumps(callable_source))
                elif hasattr( eval('fabfile.%s' % prop), 'dependency' ):
                    _callable = eval('fabfile.%s' % prop)
                    callable_source = inspect.getsource(_callable)
                    fab_dependencies[prop] = (pickle.dumps(callable_source))
            # Else just use the value
            else:
                fab_dict[prop] = eval('fabfile.%s' % prop)

    #response = {}
    response = simplejson.dumps(fab_dict)

    return HttpResponse(response, mimetype='application/json', status=200)

@csrf_exempt
def fab(request):
    '''
    Accepts JSON (and more later on?) data describing fabric commands
    and runs them if they exist and are allowed.

    #Test it locally
    curl --dump-header - -H "Content-Type: application/json" -X POST --data '{"hosts": ["server1.example.com"],"commands": ["alpha", "kick_apache"] }' http://localhost:8000/tailor/api/v1/fab/

    # NOTE: This is all PoC at this point.  Lots of hard-coded values
    # TODO: Seperate all this out to methods
    '''

    import fabric

    # Turn off output as to not write against stdout and stderr

    if request.method == 'POST':

        try:
            _input = request.raw_post_data
            _input = simplejson.loads(request.raw_post_data)
            #print _input['hosts']
        #except JSONDecodeError, e:
            #print "Couldn't parse JSON: %s" % e
        except Exception, e:
            print "Error: %s" % e

        try:
            client_url = "http://localhost:8001/tailor/api/v1/schema/"
            client_data = urllib2.urlopen(client_url)
            client_json = client_data.read()
            client_dict = simplejson.loads(client_json)

            #Need this?
            from fabric.api import execute

            # TODO: dynamically configure the 'env' dict from
            #env = client_dict['env']
            #env.apache_bin_dir = "/etc/init.d/apache2"
            #env.user = 'fabric'

            #Set Host via POST Data
            env.hosts = _input['hosts']

            file = open("fabric_stuff.py", "w")
            new_string = ""
            new_string = new_string + "from fabric.api import *\nfrom tailor.decorators import *\n\n\nimport fabric\n\n\n"

            new_string = new_string + '''fabric.state.output["status"] = False\nfabric.state.output["running"] = False\nfabric.state.output["user"] = False\nfabric.state.output["warnings"] = False\nfabric.state.output["stderr"] = False\nfabric.state.output['stdout'] = False\nfabric.state.output['aborts'] = False\n\n'''

            for _varname, _var in client_dict['env'].iteritems():
                if isinstance(_var, str):
                    new_string = new_string + "env.%s = \"%s\"" % (_varname, _var) + "\n"
                else:
                    new_string = new_string + "env.%s = %s" % (_varname, _var) + "\n"

            new_string = new_string + "\n\n"
            for dep, dep_func in client_dict['dependencies'].iteritems():
                new_string = new_string + pickle.loads(str(dep_func))

            new_string = new_string + "\n\n"
            for task, task_func in client_dict['tasks'].iteritems():
                new_string = new_string + pickle.loads(str(task_func))

            file.write(new_string)
            file.close()
            import fabric_stuff

            for command in _input['commands']:
                try:
                    execute(eval("fabric_stuff." + command))
                except AttributeError:
                    response_dict = {'success': False, 'message': "Method does not exist."}
                    response = simplejson.dumps(response_dict)
                    return HttpResponse(response, mimetype='application/json', status=400)
            import os
            os.remove("fabric_stuff.py")

            #respond
            response_dict = {'success':True, 'message':"Commands Executed"}
            response = simplejson.dumps(response_dict)

            return HttpResponse(response, mimetype='application/json', status=200)
        except Exception, e:
            print "Error: %s" % e
            response_dict = {'success':False, 'message':"Coudn't not execute commands"}
            response = simplejson.dumps(response_dict)
            return HttpResponse(response, mimetype='application/json', status=400)
    else:
        response = "Method is not allow. Only POST is allowed"
        return HttpResponse(response, status=400)

<|endoftext|>


<commit_before>#!/usr/bin/python

import sys
import os
import requests
import urllib
import re

g_user = None
g_pass = None
g_sprint = None
g_csv = False

class Person:
    def __init__(self, name):
        self.name = name
        self.resolved_list = []
        self.resolved_story_points = 0
        self.unresolved_list = []
        self.unresolved_story_points = 0

    def add(self, issue):
        story_points = issue[u'fields'][u'customfield_10004']
        if story_points is None:
            story_points = 0
        else:
            story_points = float(story_points)
        resolution = issue[u'fields'][u'resolution']
        if resolution is None:
            self.unresolved_story_points += story_points
        else:
            self.resolved_story_points += story_points

    def _printbar(self, label, value, char):
        bars_per_day = 4
        sys.stdout.write(label + ":  ")
        num_bars = int(value * bars_per_day)
        i = 0
        while (num_bars > 0):
            if (i % bars_per_day == 0):
                sys.stdout.write(" ")
            sys.stdout.write(char)
            num_bars -= 1
            i += 1
        sys.stdout.write("\n")

    def emit_csv(self):
        print (self.name + "," + str(self.resolved_story_points) + "," + str(self.unresolved_story_points))

    def emit_barchart(self):
        print("")
        print("-----" + self.name + "-----")
        self._printbar("  resolved", self.resolved_story_points, "R")
        self._printbar("unresolved", self.unresolved_story_points, "U")

    def emit(self):
        global g_csv
        if g_csv:
            self.emit_csv()
        else:
            self.emit_barchart()

class PeopleManager:
    def __init__(self):
        self.people_map = {}

    def add(self, issue):
        assignee = issue[u'fields'][u'assignee']
        if (assignee is None):
            print("ERROR: assignee is none for issue: " + str(issue))
            sys.exit(1)
        assignee_name = issue[u'fields'][u'assignee'][u'name']
        person = self.find(assignee_name)
        person.add(issue)

    def find(self, name):
        if name not in self.people_map:
            person = Person(name)
            self.people_map[name] = person
        person = self.people_map[name]
        return person

    def emit(self):
        global g_csv
        if g_csv:
            print("name,resolved,unresolved")
        for key in sorted(self.people_map.keys()):
            person = self.people_map[key]
            person.emit()

def usage():
    print("")
    print("usage:  " + g_script_name + " --user username --pass password --sprint sprintname")
    print("")
    sys.exit(1)

def unknown_arg(s):
    print("")
    print("ERROR: Unknown argument: " + s)
    print("")
    usage()

def parse_config_file():
    global g_user
    global g_pass
    global g_sprint
    home = os.path.expanduser("~")
    config_file = os.path.join(home, ".h2o_jira")
    if (os.path.exists(config_file)):
        with open(config_file) as f:
            for line in f:
                match_groups = re.search(r"\s*(\S+)\s*=\s*([\S\s]+)\s*", line.rstrip())
                if (match_groups is not None):
                    key = match_groups.group(1)
                    value = match_groups.group(2)
                    if (key == "user"):
                        g_user = value
                    elif (key == "pass"):
                        g_pass = value
                    elif (key == "sprint"):
                        g_sprint = value

def parse_args(argv):
    global g_user
    global g_pass
    global g_sprint
    global g_csv

    i = 1
    while (i < len(argv)):
        s = argv[i]

        if (s == "--user"):
            i += 1
            if (i > len(argv)):
                usage()
            g_user = argv[i]
        elif (s == "--pass"):
            i += 1
            if (i > len(argv)):
                usage()
            g_pass = argv[i]
        elif (s == "--sprint"):
            i += 1
            if (i > len(argv)):
                usage()
            g_sprint = argv[i]
        elif (s == "-csv"):
            g_csv = True
        elif (s == "-h" or s == "--h" or s == "-help" or s == "--help"):
            usage()
        else:
            unknown_arg(s)

        i += 1

    if (g_user is None):
        print "ERROR: user is not specified"
        usage()

    if (g_pass is None):
        print "ERROR: pass is not specified"
        usage()

    if (g_sprint is None):
        print "ERROR: sprint is not specified"
        usage()

def main(argv):
    """
    Main program.

    @return: none
    """
    global g_script_name

    g_script_name = os.path.basename(argv[0])
    parse_config_file()
    parse_args(argv)

    url = 'https://0xdata.atlassian.net/rest/api/2/search?jql=sprint="' + urllib.quote(g_sprint) + '"&maxResults=1000'
    r = requests.get(url, auth=(g_user, g_pass))
    if (r.status_code != 200):
        print("ERROR: status code is " + str(r.status_code))
        sys.exit(1)
    j = r.json()
    issues = j[u'issues']
    pm = PeopleManager()
    for issue in issues:
        pm.add(issue)

    pm.emit()
    print("")

if __name__ == "__main__":
    main(sys.argv)
<commit_msg>Remove idea inspection warning.<commit_after>#!/usr/bin/python

import sys
import os
import requests
import urllib
import re

g_user = None
g_pass = None
g_sprint = None
g_csv = False

class Person:
    def __init__(self, name):
        self.name = name
        self.resolved_list = []
        self.resolved_story_points = 0
        self.unresolved_list = []
        self.unresolved_story_points = 0

    def add(self, issue):
        story_points = issue[u'fields'][u'customfield_10004']
        if story_points is None:
            story_points = 0
        else:
            story_points = float(story_points)
        resolution = issue[u'fields'][u'resolution']
        if resolution is None:
            self.unresolved_story_points += story_points
        else:
            self.resolved_story_points += story_points

    def emit(self):
        global g_csv
        if g_csv:
            self._emit_csv()
        else:
            self._emit_barchart()

    def _emit_csv(self):
        print (self.name + "," + str(self.resolved_story_points) + "," + str(self.unresolved_story_points))

    def _emit_barchart(self):
        print("")
        print("-----" + self.name + "-----")
        Person._printbar("  resolved", self.resolved_story_points, "R")
        Person._printbar("unresolved", self.unresolved_story_points, "U")

    @staticmethod
    def _printbar(label, value, char):
        bars_per_day = 4
        sys.stdout.write(label + ":  ")
        num_bars = int(value * bars_per_day)
        i = 0
        while (num_bars > 0):
            if (i % bars_per_day == 0):
                sys.stdout.write(" ")
            sys.stdout.write(char)
            num_bars -= 1
            i += 1
        sys.stdout.write("\n")

class PeopleManager:
    def __init__(self):
        self.people_map = {}

    def add(self, issue):
        assignee = issue[u'fields'][u'assignee']
        if (assignee is None):
            print("ERROR: assignee is none for issue: " + str(issue))
            sys.exit(1)
        assignee_name = issue[u'fields'][u'assignee'][u'name']
        person = self.find(assignee_name)
        person.add(issue)

    def find(self, name):
        if name not in self.people_map:
            person = Person(name)
            self.people_map[name] = person
        person = self.people_map[name]
        return person

    def emit(self):
        global g_csv
        if g_csv:
            print("name,resolved,unresolved")
        for key in sorted(self.people_map.keys()):
            person = self.people_map[key]
            person.emit()

def usage():
    print("")
    print("usage:  " + g_script_name + " --user username --pass password --sprint sprintname")
    print("")
    sys.exit(1)

def unknown_arg(s):
    print("")
    print("ERROR: Unknown argument: " + s)
    print("")
    usage()

def parse_config_file():
    global g_user
    global g_pass
    global g_sprint
    home = os.path.expanduser("~")
    config_file = os.path.join(home, ".h2o_jira")
    if (os.path.exists(config_file)):
        with open(config_file) as f:
            for line in f:
                match_groups = re.search(r"\s*(\S+)\s*=\s*([\S\s]+)\s*", line.rstrip())
                if (match_groups is not None):
                    key = match_groups.group(1)
                    value = match_groups.group(2)
                    if (key == "user"):
                        g_user = value
                    elif (key == "pass"):
                        g_pass = value
                    elif (key == "sprint"):
                        g_sprint = value

def parse_args(argv):
    global g_user
    global g_pass
    global g_sprint
    global g_csv

    i = 1
    while (i < len(argv)):
        s = argv[i]

        if (s == "--user"):
            i += 1
            if (i > len(argv)):
                usage()
            g_user = argv[i]
        elif (s == "--pass"):
            i += 1
            if (i > len(argv)):
                usage()
            g_pass = argv[i]
        elif (s == "--sprint"):
            i += 1
            if (i > len(argv)):
                usage()
            g_sprint = argv[i]
        elif (s == "-csv"):
            g_csv = True
        elif (s == "-h" or s == "--h" or s == "-help" or s == "--help"):
            usage()
        else:
            unknown_arg(s)

        i += 1

    if (g_user is None):
        print "ERROR: user is not specified"
        usage()

    if (g_pass is None):
        print "ERROR: pass is not specified"
        usage()

    if (g_sprint is None):
        print "ERROR: sprint is not specified"
        usage()

def main(argv):
    """
    Main program.

    @return: none
    """
    global g_script_name

    g_script_name = os.path.basename(argv[0])
    parse_config_file()
    parse_args(argv)

    url = 'https://0xdata.atlassian.net/rest/api/2/search?jql=sprint="' + urllib.quote(g_sprint) + '"&maxResults=1000'
    r = requests.get(url, auth=(g_user, g_pass))
    if (r.status_code != 200):
        print("ERROR: status code is " + str(r.status_code))
        sys.exit(1)
    j = r.json()
    issues = j[u'issues']
    pm = PeopleManager()
    for issue in issues:
        pm.add(issue)

    pm.emit()
    print("")

if __name__ == "__main__":
    main(sys.argv)
<|endoftext|>


<commit_before>from luigi import Parameter, WrapperTask

from lib.timespan import get_timespan

from tasks.base_tasks import (ColumnsTask, DownloadUnzipTask, Shp2TempTableTask, SimplifiedTempTableTask, TableTask,
                              TagsTask)
from tasks.util import shell
from tasks.meta import GEOM_REF, GEOM_NAME, OBSColumn, current_session, OBSTag
from tasks.tags import SectionTags, SubsectionTags, BoundaryTags

from collections import OrderedDict

GEO_STE = 'STE'
GEO_SA4 = 'SA4'
GEO_SA3 = 'SA3'
GEO_SA2 = 'SA2'
GEO_SA1 = 'SA1'
GEO_GCCSA = 'GCCSA'
GEO_LGA = 'LGA'
GEO_SLA = 'SLA'
GEO_SSC = 'SSC'
GEO_POA = 'POA'
GEO_CED = 'CED'
GEO_SED = 'SED'
GEO_SOS = 'SOS'
GEO_SOSR = 'SOSR'
GEO_UCL = 'UCL'
GEO_SUA = 'SUA'
GEO_RA = 'RA'

GEOGRAPHIES = (
    GEO_STE,
    GEO_SA4,
    GEO_SA3,
    GEO_SA2,
    GEO_SA1,
    GEO_GCCSA,
    GEO_LGA,
    GEO_SLA,
    GEO_SSC,
    GEO_POA,
    GEO_CED,
    GEO_SED,
    GEO_SOS,
    GEO_SOSR,
    GEO_UCL,
    GEO_SUA,
    GEO_RA
)

GEOGRAPHY = {
    GEO_STE: {'name': 'State/Territory', 'weight': 17, 'region_col': 'STATE_CODE', 'proper_name': 'STATE_NAME'},
    GEO_SA4: {'name': 'Statistical Area Level 4', 'weight': 16, 'region_col': 'SA4_CODE', 'proper_name': 'SA4_NAME'},
    GEO_SA3: {'name': 'Statistical Area Level 3', 'weight': 15, 'region_col': 'SA3_CODE', 'proper_name': 'SA3_NAME'},
    GEO_SA2: {'name': 'Statistical Area Level 2', 'weight': 14, 'region_col': 'SA2_MAIN', 'proper_name': 'SA2_NAME'},
    GEO_SA1: {'name': 'Statistical Area Level 1', 'weight': 13, 'region_col': 'SA1_7DIGIT', 'proper_name': 'STATE_NAME'},
    GEO_GCCSA: {'name': 'Greater Capital City Statistical Areas', 'weight': 12, 'region_col': 'GCCSA_CODE', 'proper_name': 'GCCSA_NAME'},
    GEO_LGA: {'name': 'Local Government Areas', 'weight': 11, 'region_col': 'LGA_CODE', 'proper_name': 'LGA_NAME'},
    GEO_SLA: {'name': 'Statistical Local Areas', 'weight': 10, 'region_col': 'SLA_MAIN', 'proper_name': 'SLA_NAME'},
    GEO_SSC: {'name': 'State Suburbs', 'weight': 9, 'region_col': 'SSC_CODE', 'proper_name': 'SSC_NAME'},
    GEO_POA: {'name': 'Postal Areas', 'weight': 8, 'region_col': 'POA_CODE', 'proper_name': 'POA_NAME'},
    GEO_CED: {'name': 'Commonwealth Electoral Divisions', 'weight': 7, 'region_col': 'CED_CODE', 'proper_name': 'CED_NAME'},
    GEO_SED: {'name': 'State Electoral Divisions', 'weight': 6, 'region_col': 'SED_CODE', 'proper_name': 'SED_NAME'},
    GEO_SOS: {'name': 'Section of State', 'weight': 5, 'region_col': 'SOS_CODE', 'proper_name': 'SOS_NAME'},
    GEO_SOSR: {'name': 'Section of State Ranges', 'weight': 4, 'region_col': 'SOSR_CODE', 'proper_name': 'SOSR_NAME'},
    GEO_UCL: {'name': 'Urban Centres and Localities', 'weight': 3, 'region_col': 'UCL_CODE', 'proper_name': 'UCL_NAME'},
    GEO_SUA: {'name': 'Significant Urban Areas', 'weight': 2, 'region_col': 'SUA_CODE', 'proper_name': 'SUA_NAME'},
    GEO_RA: {'name': 'Remoteness Areas', 'weight': 1, 'region_col': 'RA_CODE', 'proper_name': 'RA_NAME'},
}

class SourceTags(TagsTask):

    def tags(self):
        return [
            OBSTag(id='au-census',
                   name='Australian Bureau of Statistics (ABS)',
                   type='source',
                   description='The `Australian Bureau of Statistics <http://abs.gov.au/websitedbs/censushome.nsf/home/datapacks>`')
        ]

class LicenseTags(TagsTask):

    def tags(self):
        return [
            OBSTag(id='au-datapacks-license',
                   name='Creative Commons Attribution 2.5 Australia licence',
                   type='license',
                   description='DataPacks is licenced under a `Creative Commons Attribution 2.5 Australia licence <https://creativecommons.org/licenses/by/2.5/au/>`_')
        ]

class DownloadGeography(DownloadUnzipTask):

    year = Parameter()
    resolution = Parameter()

    URL = 'http://www.censusdata.abs.gov.au/CensusOutput/copsubdatapacks.nsf/All%20docs%20by%20catNo/Boundaries_{year}_{resolution}/\$File/{year}_{resolution}_shape.zip'

    def download(self):
        shell('wget -O {output}.zip {url}'.format(
            output=self.output().path, url=self.URL.format(resolution=self.resolution, year=self.year)
        ))

class ImportGeography(Shp2TempTableTask):
    '''
    Import geographies into postgres by geography level
    '''

    year = Parameter()
    resolution = Parameter()

    def requires(self):
        return DownloadGeography(resolution=self.resolution, year=self.year)

    def input_shp(self):
        cmd = 'ls {input}/*.shp'.format(
            input=self.input().path
        )
        for shp in shell(cmd).strip().split('\n'):
            yield shp

class SimplifiedRawGeometry(SimplifiedTempTableTask):
    year = Parameter()
    resolution = Parameter()

    def requires(self):
        return ImportGeography(year=self.year, resolution=self.resolution)

class GeographyColumns(ColumnsTask):

    resolution = Parameter()

    def version(self):
        return 2

    def requires(self):
        return {
            'sections': SectionTags(),
            'subsections': SubsectionTags(),
            'source': SourceTags(),
            'license': LicenseTags(),
            'boundary': BoundaryTags(),
        }

    def columns(self):
        input_ = self.input()
        sections = input_['sections']
        subsections = input_['subsections']
        source = input_['source']['au-census']
        license = input_['license']['au-datapacks-license']
        boundary_type = input_['boundary']

        geom = OBSColumn(
            id=self.resolution,
            type='Geometry',
            name=GEOGRAPHY[self.resolution]['name'],
            description='',
            weight=GEOGRAPHY[self.resolution]['weight'],
            tags=[source, license, sections['au'], subsections['boundary']],
        )
        geom_id = OBSColumn(
            type='Text',
            id=self.resolution + '_id',
            weight=0,
            targets={geom: GEOM_REF},
        )
        geom_name = OBSColumn(
            type='Text',
            name= 'Proper name of {}'.format(GEOGRAPHY[self.resolution]['name']),
            id=self.resolution + '_name',
            description='',
            weight=1,
            targets={geom: GEOM_NAME},
            tags=[source, license, sections['au'], subsections['names']],
        )

        cartographic_boundaries = [GEO_LGA, GEO_POA, GEO_CED, GEO_SED, GEO_SSC,
                                   GEO_SA1, GEO_SA2, GEO_SA3, GEO_SA4,
                                   GEO_STE, GEO_GCCSA, GEO_UCL,
                                   GEO_SOS, GEO_SOSR, GEO_SUA, GEO_RA]
        interpolated_boundaries = [GEO_SA1, GEO_SA2, GEO_SA3, GEO_SA4,
                                   GEO_STE, GEO_GCCSA, GEO_UCL,
                                   GEO_SOS, GEO_SOSR, GEO_SUA, GEO_RA]

        cols = OrderedDict([
            ('geom_name', geom_name),
            ('geom_id', geom_id),
            ('the_geom', geom),
        ])

        for colname, col in cols.items():
            if col.id in interpolated_boundaries:
                col.tags.append(boundary_type['interpolation_boundary'])
            if col.id in cartographic_boundaries:
                col.tags.append(boundary_type['cartographic_boundary'])
        return cols

class Geography(TableTask):

    year = Parameter()
    resolution = Parameter()

    def version(self):
        return 5

    def requires(self):
        return {
            'data': SimplifiedRawGeometry(resolution=self.resolution, year=self.year),
            'columns': GeographyColumns(resolution=self.resolution)
        }

    def table_timespan(self):
        return get_timespan(str(self.year))

    def columns(self):
        return self.input()['columns']

    def populate(self):
        session = current_session()
        session.execute('INSERT INTO {output} '
                        'SELECT {geom_name} as geom_name, '
                        '       {region_col} as geom_id, '
                        '       wkb_geometry as the_geom '
                        'FROM {input} '.format(
                            geom_name=GEOGRAPHY[self.resolution]['proper_name'],
                            region_col=GEOGRAPHY[self.resolution]['region_col'],
                            output=self.output().table,
                            input=self.input()['data'].table))

class AllGeographies(WrapperTask):

    year = Parameter()

    def requires(self):
        for resolution in GEOGRAPHIES:
            yield Geography(resolution=resolution, year=self.year)
<commit_msg>Added AU geo to repo<commit_after>from luigi import Parameter, WrapperTask

from lib.timespan import get_timespan

from tasks.base_tasks import (ColumnsTask, DownloadUnzipTask, Shp2TempTableTask, SimplifiedTempTableTask, TableTask,
                              TagsTask, RepoFile)
from tasks.util import shell
from tasks.meta import GEOM_REF, GEOM_NAME, OBSColumn, current_session, OBSTag
from tasks.tags import SectionTags, SubsectionTags, BoundaryTags

from collections import OrderedDict
from shutil import copyfile

GEO_STE = 'STE'
GEO_SA4 = 'SA4'
GEO_SA3 = 'SA3'
GEO_SA2 = 'SA2'
GEO_SA1 = 'SA1'
GEO_GCCSA = 'GCCSA'
GEO_LGA = 'LGA'
GEO_SLA = 'SLA'
GEO_SSC = 'SSC'
GEO_POA = 'POA'
GEO_CED = 'CED'
GEO_SED = 'SED'
GEO_SOS = 'SOS'
GEO_SOSR = 'SOSR'
GEO_UCL = 'UCL'
GEO_SUA = 'SUA'
GEO_RA = 'RA'

GEOGRAPHIES = (
    GEO_STE,
    GEO_SA4,
    GEO_SA3,
    GEO_SA2,
    GEO_SA1,
    GEO_GCCSA,
    GEO_LGA,
    GEO_SLA,
    GEO_SSC,
    GEO_POA,
    GEO_CED,
    GEO_SED,
    GEO_SOS,
    GEO_SOSR,
    GEO_UCL,
    GEO_SUA,
    GEO_RA
)

GEOGRAPHY = {
    GEO_STE: {'name': 'State/Territory', 'weight': 17, 'region_col': 'STATE_CODE', 'proper_name': 'STATE_NAME'},
    GEO_SA4: {'name': 'Statistical Area Level 4', 'weight': 16, 'region_col': 'SA4_CODE', 'proper_name': 'SA4_NAME'},
    GEO_SA3: {'name': 'Statistical Area Level 3', 'weight': 15, 'region_col': 'SA3_CODE', 'proper_name': 'SA3_NAME'},
    GEO_SA2: {'name': 'Statistical Area Level 2', 'weight': 14, 'region_col': 'SA2_MAIN', 'proper_name': 'SA2_NAME'},
    GEO_SA1: {'name': 'Statistical Area Level 1', 'weight': 13, 'region_col': 'SA1_7DIGIT', 'proper_name': 'STATE_NAME'},
    GEO_GCCSA: {'name': 'Greater Capital City Statistical Areas', 'weight': 12, 'region_col': 'GCCSA_CODE', 'proper_name': 'GCCSA_NAME'},
    GEO_LGA: {'name': 'Local Government Areas', 'weight': 11, 'region_col': 'LGA_CODE', 'proper_name': 'LGA_NAME'},
    GEO_SLA: {'name': 'Statistical Local Areas', 'weight': 10, 'region_col': 'SLA_MAIN', 'proper_name': 'SLA_NAME'},
    GEO_SSC: {'name': 'State Suburbs', 'weight': 9, 'region_col': 'SSC_CODE', 'proper_name': 'SSC_NAME'},
    GEO_POA: {'name': 'Postal Areas', 'weight': 8, 'region_col': 'POA_CODE', 'proper_name': 'POA_NAME'},
    GEO_CED: {'name': 'Commonwealth Electoral Divisions', 'weight': 7, 'region_col': 'CED_CODE', 'proper_name': 'CED_NAME'},
    GEO_SED: {'name': 'State Electoral Divisions', 'weight': 6, 'region_col': 'SED_CODE', 'proper_name': 'SED_NAME'},
    GEO_SOS: {'name': 'Section of State', 'weight': 5, 'region_col': 'SOS_CODE', 'proper_name': 'SOS_NAME'},
    GEO_SOSR: {'name': 'Section of State Ranges', 'weight': 4, 'region_col': 'SOSR_CODE', 'proper_name': 'SOSR_NAME'},
    GEO_UCL: {'name': 'Urban Centres and Localities', 'weight': 3, 'region_col': 'UCL_CODE', 'proper_name': 'UCL_NAME'},
    GEO_SUA: {'name': 'Significant Urban Areas', 'weight': 2, 'region_col': 'SUA_CODE', 'proper_name': 'SUA_NAME'},
    GEO_RA: {'name': 'Remoteness Areas', 'weight': 1, 'region_col': 'RA_CODE', 'proper_name': 'RA_NAME'},
}

class SourceTags(TagsTask):

    def tags(self):
        return [
            OBSTag(id='au-census',
                   name='Australian Bureau of Statistics (ABS)',
                   type='source',
                   description='The `Australian Bureau of Statistics <http://abs.gov.au/websitedbs/censushome.nsf/home/datapacks>`')
        ]

class LicenseTags(TagsTask):

    def tags(self):
        return [
            OBSTag(id='au-datapacks-license',
                   name='Creative Commons Attribution 2.5 Australia licence',
                   type='license',
                   description='DataPacks is licenced under a `Creative Commons Attribution 2.5 Australia licence <https://creativecommons.org/licenses/by/2.5/au/>`_')
        ]

class DownloadGeography(DownloadUnzipTask):

    year = Parameter()
    resolution = Parameter()

    URL = 'http://www.censusdata.abs.gov.au/CensusOutput/copsubdatapacks.nsf/All%20docs%20by%20catNo/Boundaries_{year}_{resolution}/\$File/{year}_{resolution}_shape.zip'

    def version(self):
        return 1

    def requires(self):
            return RepoFile(resource_id=self.task_id,
                            version=self.version(),
                            url=self.URL.format(resolution=self.resolution, year=self.year))

    def download(self):
        self.output().makedirs()
        copyfile(self.input().path, '{output}.zip'.format(output=self.output().path))

class ImportGeography(Shp2TempTableTask):
    '''
    Import geographies into postgres by geography level
    '''

    year = Parameter()
    resolution = Parameter()

    def requires(self):
        return DownloadGeography(resolution=self.resolution, year=self.year)

    def input_shp(self):
        cmd = 'ls {input}/*.shp'.format(
            input=self.input().path
        )
        for shp in shell(cmd).strip().split('\n'):
            yield shp

class SimplifiedRawGeometry(SimplifiedTempTableTask):
    year = Parameter()
    resolution = Parameter()

    def requires(self):
        return ImportGeography(year=self.year, resolution=self.resolution)

class GeographyColumns(ColumnsTask):

    resolution = Parameter()

    def version(self):
        return 2

    def requires(self):
        return {
            'sections': SectionTags(),
            'subsections': SubsectionTags(),
            'source': SourceTags(),
            'license': LicenseTags(),
            'boundary': BoundaryTags(),
        }

    def columns(self):
        input_ = self.input()
        sections = input_['sections']
        subsections = input_['subsections']
        source = input_['source']['au-census']
        license = input_['license']['au-datapacks-license']
        boundary_type = input_['boundary']

        geom = OBSColumn(
            id=self.resolution,
            type='Geometry',
            name=GEOGRAPHY[self.resolution]['name'],
            description='',
            weight=GEOGRAPHY[self.resolution]['weight'],
            tags=[source, license, sections['au'], subsections['boundary']],
        )
        geom_id = OBSColumn(
            type='Text',
            id=self.resolution + '_id',
            weight=0,
            targets={geom: GEOM_REF},
        )
        geom_name = OBSColumn(
            type='Text',
            name= 'Proper name of {}'.format(GEOGRAPHY[self.resolution]['name']),
            id=self.resolution + '_name',
            description='',
            weight=1,
            targets={geom: GEOM_NAME},
            tags=[source, license, sections['au'], subsections['names']],
        )

        cartographic_boundaries = [GEO_LGA, GEO_POA, GEO_CED, GEO_SED, GEO_SSC,
                                   GEO_SA1, GEO_SA2, GEO_SA3, GEO_SA4,
                                   GEO_STE, GEO_GCCSA, GEO_UCL,
                                   GEO_SOS, GEO_SOSR, GEO_SUA, GEO_RA]
        interpolated_boundaries = [GEO_SA1, GEO_SA2, GEO_SA3, GEO_SA4,
                                   GEO_STE, GEO_GCCSA, GEO_UCL,
                                   GEO_SOS, GEO_SOSR, GEO_SUA, GEO_RA]

        cols = OrderedDict([
            ('geom_name', geom_name),
            ('geom_id', geom_id),
            ('the_geom', geom),
        ])

        for colname, col in cols.items():
            if col.id in interpolated_boundaries:
                col.tags.append(boundary_type['interpolation_boundary'])
            if col.id in cartographic_boundaries:
                col.tags.append(boundary_type['cartographic_boundary'])
        return cols

class Geography(TableTask):

    year = Parameter()
    resolution = Parameter()

    def version(self):
        return 5

    def requires(self):
        return {
            'data': SimplifiedRawGeometry(resolution=self.resolution, year=self.year),
            'columns': GeographyColumns(resolution=self.resolution)
        }

    def table_timespan(self):
        return get_timespan(str(self.year))

    def columns(self):
        return self.input()['columns']

    def populate(self):
        session = current_session()
        session.execute('INSERT INTO {output} '
                        'SELECT {geom_name} as geom_name, '
                        '       {region_col} as geom_id, '
                        '       wkb_geometry as the_geom '
                        'FROM {input} '.format(
                            geom_name=GEOGRAPHY[self.resolution]['proper_name'],
                            region_col=GEOGRAPHY[self.resolution]['region_col'],
                            output=self.output().table,
                            input=self.input()['data'].table))

class AllGeographies(WrapperTask):

    year = Parameter()

    def requires(self):
        for resolution in GEOGRAPHIES:
            yield Geography(resolution=resolution, year=self.year)
<|endoftext|>


<commit_before>#!/usr/bin/python

import sys
import numpy as np
import pysam
from collections import Counter

csv=sys.argv[1].split(".csv")[0]
csv=csv.split("/")[-1]
bam=sys.argv[2].split(".bam")[0]
bam=bam.split("/")[-1]

#Ensure the bam and csv file match
if csv==bam:
    print("working with csv: "+csv + " and bam: " +bam)
else:
   print( "bam:"+bam +" does not match csv: " +csv)
   sys.exit(1)

#Put header on true false and exp csv files
with open(sys.argv[4],"w") as read_csv:
    read_csv.write("Sample,Mutation,Variable,Value,Count\n")

with pysam.AlignmentFile(sys.argv[2], "rb") as bamfile:
    with open(sys.argv[1],'r') as in_var:
        with open(sys.argv[3],"w") as outfile_csv:

            header=in_var.readline().strip()+",\"MapQ\"" +",\"Read_pos\""+",\"Phred\""+'\n' # add the mapq column to the header of the input csv
            outfile_csv.write(header) # write the header to the output file

            for line in in_var:
                mapq=[]# This will hold a list of the mapping qualtties that map to the variant
                phred=[] #This will hold a list of the phred that map to the variant
                Read_pos=[]  # This will hold a list of position relative to the read
                line=line.strip()
                data=line # save the line as it is to append the data to later
                line=line.replace('"','').split(",")
                chr=line[0]
                pos=int(line[1])
                py_pos=pos-1
                ref=line[2]
                var=line[3]
                sample= line[16]
                mutation= line[17] # mutation name
               # print(mutation)
                for pileupcolumn in bamfile.pileup(chr,py_pos,py_pos+1,truncate=True,stepper="all",max_depth=1E6):
                    if pileupcolumn.pos==py_pos:
                        for pileupread in pileupcolumn.pileups:
                            if  not pileupread.is_del and not pileupread.is_refskip:
                                called_base=pileupread.alignment.query_sequence[pileupread.query_position]
                                called_phred=pileupread.alignment.query_qualities[pileupread.query_position]
                                if called_phred>25 and called_base==var: # change this if you change the phred cut off in deepSNV
                                    mapq.append(pileupread.alignment.mapping_quality)
                                    phred.append(called_phred)
                                    Read_pos.append(pileupread.query_position)
                mean_map=np.mean(mapq)
                mean_phred=np.mean(phred)
                mean_Read_pos=np.mean(Read_pos)
                if mean_map==[]:
                    print( "OOPS didn't find the variant looks like you didn't fix the bug")
                    sys.exit(1)

                data=data+","+str(mean_map)+","+str(mean_Read_pos)+","+str(mean_phred)+'\n'
                outfile_csv.write(data) # write to the summary file

                mapq=Counter(mapq)
                phred=Counter(phred)
                Read_pos=Counter(Read_pos)
                with open(sys.argv[4],"a") as read_csv:

                    for qual in mapq:
                        read_csv.write(sample+","+mutation+","+"MapQ"+","+str(qual)+","+str(mapq[qual])+'\n')
                    for qual in Read_pos:
                        read_csv.write(sample+","+mutation+","+"Read_pos"+","+str(qual)+","+str(Read_pos[qual])+'\n')
                    for qual in phred:
                        read_csv.write(sample+","+mutation+","+"Phred"+","+str(qual)+","+str(phred[qual])+'\n')
<commit_msg>Update mapq.py<commit_after>#!/usr/bin/python

import sys
import numpy as np
import pysam
from collections import Counter

csv=sys.argv[1].split(".csv")[0]
csv=csv.split("/")[-1]
bam=sys.argv[2].split(".bam")[0]
bam=bam.split("/")[-1]

#Ensure the bam and csv file match
if csv==bam:
    print("working with csv: "+csv + " and bam: " +bam)
else:
   print( "bam:"+bam +" does not match csv: " +csv)
   sys.exit(1)

#Put header on true false and exp csv files
with open(sys.argv[4],"w") as read_csv:
    read_csv.write("Sample,Mutation,Variable,Value,Count\n")

with pysam.AlignmentFile(sys.argv[2], "rb") as bamfile:
    with open(sys.argv[1],'r') as in_var:
        with open(sys.argv[3],"w") as outfile_csv:

            header=in_var.readline().strip()+",\"MapQ\"" +",\"Read_pos\""+",\"Phred\""+'\n' # add the mapq column to the header of the input csv
            outfile_csv.write(header) # write the header to the output file

            for line in in_var:
                mapq=[]# This will hold a list of the mapping qualtties that map to the variant
                phred=[] #This will hold a list of the phred that map to the variant
                Read_pos=[]  # This will hold a list of position relative to the read
                line=line.strip()
                data=line # save the line as it is to append the data to later
                line=line.replace('"','').split(",")
                chr=line[0]
                pos=int(line[1])
                py_pos=pos-1
                ref=line[2]
                var=line[3]
                sample= line[16]
                mutation= line[17] # mutation name
               # print(mutation)
                for pileupcolumn in bamfile.pileup(chr,py_pos,py_pos+1,truncate=True,stepper="all",max_depth=1E6):
                    if pileupcolumn.pos==py_pos:
                        for pileupread in pileupcolumn.pileups:
                            if  not pileupread.is_del and not pileupread.is_refskip:
                                called_base=pileupread.alignment.query_sequence[pileupread.query_position]
                                called_phred=pileupread.alignment.query_qualities[pileupread.query_position]
                                if called_phred>29 and called_base==var: # change this if you change the phred cut off in deepSNV
                                    mapq.append(pileupread.alignment.mapping_quality)
                                    phred.append(called_phred)
                                    Read_pos.append(pileupread.query_position)
                mean_map=np.mean(mapq)
                mean_phred=np.mean(phred)
                mean_Read_pos=np.mean(Read_pos)
                if mean_map==[]:
                    print( "OOPS didn't find the variant looks like you didn't fix the bug")
                    sys.exit(1)

                data=data+","+str(mean_map)+","+str(mean_Read_pos)+","+str(mean_phred)+'\n'
                outfile_csv.write(data) # write to the summary file

                mapq=Counter(mapq)
                phred=Counter(phred)
                Read_pos=Counter(Read_pos)
                with open(sys.argv[4],"a") as read_csv:

                    for qual in mapq:
                        read_csv.write(sample+","+mutation+","+"MapQ"+","+str(qual)+","+str(mapq[qual])+'\n')
                    for qual in Read_pos:
                        read_csv.write(sample+","+mutation+","+"Read_pos"+","+str(qual)+","+str(Read_pos[qual])+'\n')
                    for qual in phred:
                        read_csv.write(sample+","+mutation+","+"Phred"+","+str(qual)+","+str(phred[qual])+'\n')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

STARTING_MESSAGE = "Sorry, a poll is already being set up in this channel. To cancel a poll that has not begun yet, say `!poll end`"
BUSY_MESSAGE = "Sorry, a poll is already in progress in this channel. Please have the person hosting the poll cancel it, or ask an admin to end it using `!poll end`"

def poll(response, options, sc, args):
    print "We are in the poll function"

    if args[0] == "begin":
        """
        set options[0] to the timestamp of the poll(that will be set by start), and options[1] to an array that contains the polling options, and options[2] to the status of the poll (none, starting, ongoing, ended).
        options = ["timestamp",[polling_options],"status"]
        """
        if options[2] == "none":
            args.pop(0)  # get rid of the "begin" element
            return begin(response, options, args)  # logic for starting

        elif options[2] == "starting":
            return [STARTING_MESSAGE]

        elif options[2] == "ongoing":
            return [BUSY_MESSAGE]

        elif options[2] == "ended":
            return [BUSY_MESSAGE]

    elif args[0] == "start":
        if options[2] == "none":
            return ["Sorry, a poll has not been started on this channel. To start a poll, type `!poll begin`"]

        elif options[2] == "starting":
            return start(response, options, args)

        elif options[2] == "ongoing":
            return [BUSY_MESSAGE]

        elif options[2] == "ended":
            return [BUSY_MESSAGE]

    elif args[0] == "end":
        return end(response, options, sc, args)
    elif args[0] == "results":
        return [results(response,options,sc,args)]
    return ["You seemed to have used this method incorrectly... see !help to see how to use it"]

def begin(response, options, args):
    op = []
    polling_options = []
    option_string = ""
    for s in range(0, len(args)):
        option_string = option_string + args[s] + " "
        if ";" in args[s]:
            polling_options.append(option_string[:-2])
            option_string = ""

    message = "Options are:\n"
    for option in polling_options:
        message = message + option + "\n "
    options[1] = polling_options
    options[2] = "starting"
    return ["Does this look correct?\n" + message, "If this is correct, type `!poll start` to begin your poll, or `!poll end` to cancel"]

def start(response, options, args):
    arr_of_emojis = [":one:",":two:",":three:",":four:",":five:"]
    ops = options[1]
    options[1] = dict()
    options[2] = "ongoing"
    message = ""
    for s in range(0,len(ops)):
        options[1][ops[s]] = arr_of_emojis[s]
    for key in options[1]:
        message = message + key + ": " + options[1][key] + "\n"
    return ["Poll starting", "POLL_BEGIN " + response["channel"] + ";\n" + message]

def end(response, options, sc, args):
    if options[2] in ["none", "starting"]:
        options[2] = "none"
        return ["Poll cancelled"]

    elif options[2] == "ongoing":
        options[2] = "ended"
        return ["Poll concluded", results(response,options,sc,args)]

    else:
        options[2] = "ended"
        return [results(response,options,sc,args)]

def results(response, options, sc, args):
    r = sc.api_call(
            "reactions.get",
            channel=response["channel"],
            timestamp=options[0],
            full="true"
        )

    reac_dict = dict()
    winner = ""
    count = 0
    for reaction in r["message"]["reactions"]:
        reac_dict[reaction["name"]] = reaction["count"]
        print "comparing"
        if int(reaction["count"]) > count:
            winner = reaction["name"]
            count = int(reaction["count"])

    message = "Results:\n"
    for key in reac_dict:
        message = message + key + ": " + str(reac_dict[key]) + "\n"

    message += "\nThe Winner is:\n:tada:" + winner + ":tada:"
    return message
<commit_msg>Got rid of some old print statements<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

STARTING_MESSAGE = "Sorry, a poll is already being set up in this channel. To cancel a poll that has not begun yet, say `!poll end`"
BUSY_MESSAGE = "Sorry, a poll is already in progress in this channel. Please have the person hosting the poll cancel it, or ask an admin to end it using `!poll end`"

def poll(response, options, sc, args):
    if args[0] == "begin":
        """
        set options[0] to the timestamp of the poll(that will be set by start's posted message (see bot.py)), and options[1] to an array that contains the polling options, and options[2] to the status of the poll (none, starting, ongoing, ended).
        options = ["timestamp",[polling_options],"status"]
        """
        if options[2] == "none":
            args.pop(0)  # get rid of the "begin" element
            return begin(response, options, args)  # logic for starting

        elif options[2] == "starting":
            return [STARTING_MESSAGE]

        elif options[2] == "ongoing":
            return [BUSY_MESSAGE]

        elif options[2] == "ended":
            return [BUSY_MESSAGE]

    elif args[0] == "start":
        if options[2] == "none":
            return ["Sorry, a poll has not been started on this channel. To start a poll, type `!poll begin`"]

        elif options[2] == "starting":
            return start(response, options, args)

        elif options[2] == "ongoing":
            return [BUSY_MESSAGE]

        elif options[2] == "ended":
            return [BUSY_MESSAGE]

    elif args[0] == "end":
        return end(response, options, sc, args)
    elif args[0] == "results":
        return [results(response,options,sc,args)]
    return ["You seemed to have used this method incorrectly... see !help to see how to use it"]

def begin(response, options, args):
    op = []
    polling_options = []
    option_string = ""
    for s in range(0, len(args)):
        option_string = option_string + args[s] + " "
        if ";" in args[s]:
            polling_options.append(option_string[:-2])
            option_string = ""

    message = "Options are:\n"
    for option in polling_options:
        message = message + option + "\n "
    options[1] = polling_options
    options[2] = "starting"
    return ["Does this look correct?\n" + message, "If this is correct, type `!poll start` to begin your poll, or `!poll end` to cancel"]

def start(response, options, args):
    arr_of_emojis = [":one:",":two:",":three:",":four:",":five:"]
    ops = options[1]
    options[1] = dict()
    options[2] = "ongoing"
    message = ""
    for s in range(0,len(ops)):
        options[1][ops[s]] = arr_of_emojis[s]
    for key in options[1]:
        message = message + key + ": " + options[1][key] + "\n"
    return ["Poll starting", "POLL_BEGIN " + response["channel"] + ";\n" + message]

def end(response, options, sc, args):
    if options[2] in ["none", "starting"]:
        options[2] = "none"
        return ["Poll cancelled"]

    elif options[2] == "ongoing":
        options[2] = "ended"
        return ["Poll concluded", results(response,options,sc,args)]

    else:
        options[2] = "ended"
        return [results(response,options,sc,args)]

def results(response, options, sc, args):
    r = sc.api_call(
            "reactions.get",
            channel=response["channel"],
            timestamp=options[0],
            full="true"
        )

    reac_dict = dict()
    winner = ""
    count = 0
    for reaction in r["message"]["reactions"]:
        reac_dict[reaction["name"]] = reaction["count"]
        if int(reaction["count"]) > count:
            winner = reaction["name"]
            count = int(reaction["count"])

    message = "Results:\n"
    for key in reac_dict:
        message = message + key + ": " + str(reac_dict[key]) + "\n"

    message += "\nThe Winner is:\n:tada:" + winner + ":tada:"
    return message
<|endoftext|>


<commit_before><commit_msg>Stub to build in function handling, some incompleated piece added<commit_after>import argparse
import ProjectsDAO
import ProjectAutoDetector

class BuildInFunctionRunner(object):
    def __init__(self):
        self.arg_parser = argparse.ArgumentParser()
        self.__configure_parser__()

    def __configure_parser__(self):
        self.arg_parser.add_argument('--autoscan', )
<|endoftext|>


<commit_before><commit_msg>[BugFix: ZSTAC-43519]<commit_after><|endoftext|>


<commit_before>from django.conf import settings
import requests

from . import BaseBackend

class JunebugMessageSendingError(Exception):
    '''Exception that is raised when errors occur when trying to send messages
    through Junebug.'''

class JunebugMessageSender(object):
    def __init__(self, base_url, channel_id, from_address):
        self.base_url = base_url
        self.channel_id = channel_id
        self.from_address = from_address
        self.session = requests.Session()

    @property
    def url(self):
        return '%s/channels/%s/messages/' % (
            self.base_url.rstrip('/'), self.channel_id)

    def split_urn(self, urn):
        try:
            type_, address = urn.split(':', 1)
        except ValueError:
            raise JunebugMessageSendingError('Invalid URN: %s' % urn)
        return type_, address

    def send_message(self, message):
        if not message.urn:
            raise JunebugMessageSendingError(
                'Cannot send message without URN: %r' % message)
        _, to_addr = self.split_urn(message.urn)
        data = {
            'to': to_addr,
            'from': self.from_address,
            'content': message.text,
        }
        self.session.post(self.url, json=data)

class JunebugBackend(BaseBackend):
    '''
    Junebug instance as a backend.
    '''

    def __init__(self):
        self.message_sender = JunebugMessageSender(
            settings.JUNEBUG_API_ROOT, settings.JUNEBUG_CHANNEL_ID,
            settings.JUNEBUG_FROM_ADDRESS)

    def pull_contacts(
            self, org, modified_after, modified_before,
            progress_callback=None):
        """
        Pulls contacts modified in the given time window

        :param org: the org
        :param datetime modified_after: pull contacts modified after this
        :param datetime modified_before: pull contacts modified before this
        :param progress_callback:
            callable that will be called from time to time with number of
            contacts pulled
        :return:
            tuple of the number of contacts created, updated, deleted and
            ignored
        """
        return (0, 0, 0, 0)

    def pull_fields(self, org):
        """
        Pulls all contact fields

        :param org: the org
        :return:
            tuple of the number of fields created, updated, deleted and ignored
        """
        return (0, 0, 0, 0)

    def pull_groups(self, org):
        """
        Pulls all contact groups

        :param org: the org
        :return:
            tuple of the number of groups created, updated, deleted and ignored
        """
        return (0, 0, 0, 0)

    def pull_labels(self, org):
        """
        Pulls all message labels

        :param org: the org
        :return:
            tuple of the number of labels created, updated, deleted and ignored
        """
        return (0, 0, 0, 0)

    def pull_messages(
            self, org, modified_after, modified_before, as_handled=False,
            progress_callback=None):
        """
        Pulls messages modified in the given time window

        :param org: the org
        :param datetime modified_after: pull messages modified after this
        :param datetime modified_before: pull messages modified before this
        :param bool as_handled:
            whether messages should be saved as already handled
        :param progress_callback:
            callable that will be called from time to time with number of
            messages pulled
        :return:
            tuple of the number of messages created, updated, deleted and
            ignored
        """
        return (0, 0, 0, 0)

    def push_label(self, org, label):
        """
        Pushes a new or updated label

        :param org: the org
        :param label: the label
        """

    def push_outgoing(self, org, outgoing, as_broadcast=False):
        """
        Pushes (i.e. sends) outgoing messages

        :param org: the org
        :param outgoing: the outgoing messages
        :param as_broadcast:
            whether outgoing messages differ only by recipient and so can be
            sent as single broadcast
        """
        for message in outgoing:
            self.message_sender.send_message(message)

    def add_to_group(self, org, contact, group):
        """
        Adds the given contact to a group

        :param org: the org
        :param contact: the contact
        :param group: the group
        """

    def remove_from_group(self, org, contact, group):
        """
        Removes the given contact from a group

        :param org: the org
        :param contact: the contact
        :param group: the group
        """

    def stop_runs(self, org, contact):
        """
        Stops any ongoing flow runs for the given contact

        :param org: the org
        :param contact: the contact
        """

    def label_messages(self, org, messages, label):
        """
        Adds a label to the given messages

        :param org: the org
        :param messages: the messages
        :param label: the label
        """

    def unlabel_messages(self, org, messages, label):
        """
        Removes a label from the given messages

        :param org: the org
        :param messages: the messages
        :param label: the label
        """

    def archive_messages(self, org, messages):
        """
        Archives the given messages

        :param org: the org
        :param messages: the messages
        """

    def archive_contact_messages(self, org, contact):
        """
        Archives all messages for the given contact

        :param org: the org
        :param contact: the contact
        """

    def restore_messages(self, org, messages):
        """
        Restores (un-archives) the given messages

        :param org: the org
        :param messages: the messages
        """

    def flag_messages(self, org, messages):
        """
        Flags the given messages

        :param org: the org
        :param messages: the messages
        """

    def unflag_messages(self, org, messages):
        """
        Un-flags the given messages

        :param org: the org
        :param messages: the messages
        """

    def fetch_contact_messages(
            self, org, contact, created_after, created_before):
        """
        Fetches additional messages to display on a case timeline

        :param org: the org
        :param contact: the contact
        :param created_after: include messages created after this time
        :param created_before: include messages created before this time
        :return:
            the messages as JSON objects in reverse chronological order. JSON
            format should match that returned by Message.as_json() for incoming
            messages and Outgoing.as_json() for outgoing messages.
        """
        return []

    def get_url_patterns(self):
        """
        Returns the list of URL patterns that should be registered for this
        backend.

        :return: a list of URL patterns.
        """
        # TODO: Implement views for receiving messages.
        return []
<commit_msg>Add comment detailing sending messages only with URN<commit_after>from django.conf import settings
import requests

from . import BaseBackend

class JunebugMessageSendingError(Exception):
    '''Exception that is raised when errors occur when trying to send messages
    through Junebug.'''

class JunebugMessageSender(object):
    def __init__(self, base_url, channel_id, from_address):
        self.base_url = base_url
        self.channel_id = channel_id
        self.from_address = from_address
        self.session = requests.Session()

    @property
    def url(self):
        return '%s/channels/%s/messages/' % (
            self.base_url.rstrip('/'), self.channel_id)

    def split_urn(self, urn):
        try:
            type_, address = urn.split(':', 1)
        except ValueError:
            raise JunebugMessageSendingError('Invalid URN: %s' % urn)
        return type_, address

    def send_message(self, message):
        if not message.urn:
            # If we don't have an URN for a message, we cannot send it, because
            # we don't have an address to send it to.
            # TODO: Add sending to contacts with Identity Store integration.
            raise JunebugMessageSendingError(
                'Cannot send message without URN: %r' % message)
        _, to_addr = self.split_urn(message.urn)
        data = {
            'to': to_addr,
            'from': self.from_address,
            'content': message.text,
        }
        self.session.post(self.url, json=data)

class JunebugBackend(BaseBackend):
    '''
    Junebug instance as a backend.
    '''

    def __init__(self):
        self.message_sender = JunebugMessageSender(
            settings.JUNEBUG_API_ROOT, settings.JUNEBUG_CHANNEL_ID,
            settings.JUNEBUG_FROM_ADDRESS)

    def pull_contacts(
            self, org, modified_after, modified_before,
            progress_callback=None):
        """
        Pulls contacts modified in the given time window

        :param org: the org
        :param datetime modified_after: pull contacts modified after this
        :param datetime modified_before: pull contacts modified before this
        :param progress_callback:
            callable that will be called from time to time with number of
            contacts pulled
        :return:
            tuple of the number of contacts created, updated, deleted and
            ignored
        """
        return (0, 0, 0, 0)

    def pull_fields(self, org):
        """
        Pulls all contact fields

        :param org: the org
        :return:
            tuple of the number of fields created, updated, deleted and ignored
        """
        return (0, 0, 0, 0)

    def pull_groups(self, org):
        """
        Pulls all contact groups

        :param org: the org
        :return:
            tuple of the number of groups created, updated, deleted and ignored
        """
        return (0, 0, 0, 0)

    def pull_labels(self, org):
        """
        Pulls all message labels

        :param org: the org
        :return:
            tuple of the number of labels created, updated, deleted and ignored
        """
        return (0, 0, 0, 0)

    def pull_messages(
            self, org, modified_after, modified_before, as_handled=False,
            progress_callback=None):
        """
        Pulls messages modified in the given time window

        :param org: the org
        :param datetime modified_after: pull messages modified after this
        :param datetime modified_before: pull messages modified before this
        :param bool as_handled:
            whether messages should be saved as already handled
        :param progress_callback:
            callable that will be called from time to time with number of
            messages pulled
        :return:
            tuple of the number of messages created, updated, deleted and
            ignored
        """
        return (0, 0, 0, 0)

    def push_label(self, org, label):
        """
        Pushes a new or updated label

        :param org: the org
        :param label: the label
        """

    def push_outgoing(self, org, outgoing, as_broadcast=False):
        """
        Pushes (i.e. sends) outgoing messages

        :param org: the org
        :param outgoing: the outgoing messages
        :param as_broadcast:
            whether outgoing messages differ only by recipient and so can be
            sent as single broadcast
        """
        for message in outgoing:
            self.message_sender.send_message(message)

    def add_to_group(self, org, contact, group):
        """
        Adds the given contact to a group

        :param org: the org
        :param contact: the contact
        :param group: the group
        """

    def remove_from_group(self, org, contact, group):
        """
        Removes the given contact from a group

        :param org: the org
        :param contact: the contact
        :param group: the group
        """

    def stop_runs(self, org, contact):
        """
        Stops any ongoing flow runs for the given contact

        :param org: the org
        :param contact: the contact
        """

    def label_messages(self, org, messages, label):
        """
        Adds a label to the given messages

        :param org: the org
        :param messages: the messages
        :param label: the label
        """

    def unlabel_messages(self, org, messages, label):
        """
        Removes a label from the given messages

        :param org: the org
        :param messages: the messages
        :param label: the label
        """

    def archive_messages(self, org, messages):
        """
        Archives the given messages

        :param org: the org
        :param messages: the messages
        """

    def archive_contact_messages(self, org, contact):
        """
        Archives all messages for the given contact

        :param org: the org
        :param contact: the contact
        """

    def restore_messages(self, org, messages):
        """
        Restores (un-archives) the given messages

        :param org: the org
        :param messages: the messages
        """

    def flag_messages(self, org, messages):
        """
        Flags the given messages

        :param org: the org
        :param messages: the messages
        """

    def unflag_messages(self, org, messages):
        """
        Un-flags the given messages

        :param org: the org
        :param messages: the messages
        """

    def fetch_contact_messages(
            self, org, contact, created_after, created_before):
        """
        Fetches additional messages to display on a case timeline

        :param org: the org
        :param contact: the contact
        :param created_after: include messages created after this time
        :param created_before: include messages created before this time
        :return:
            the messages as JSON objects in reverse chronological order. JSON
            format should match that returned by Message.as_json() for incoming
            messages and Outgoing.as_json() for outgoing messages.
        """
        return []

    def get_url_patterns(self):
        """
        Returns the list of URL patterns that should be registered for this
        backend.

        :return: a list of URL patterns.
        """
        # TODO: Implement views for receiving messages.
        return []
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import fnmatch
import html
import locale
import os.path
import re
import subprocess
import sys

regex = re.compile(
    r"^(?P<filename>[^:]*):(?P<line>\d*):(?:(?P<column>\d*):)? (?P<message_id>[^:]*): (?P<message>.*) \((?P<message_url>.*)\)$",
    re.MULTILINE,
)

BUILDIFIER_URL = "https://github.com/bazelbuild/buildtools/tree/master/buildifier"

# https://github.com/bazelbuild/buildtools/blob/master/buildifier/buildifier.go#L333
# Buildifier error code for "needs formatting". We should fail on all other error codes > 0
# since they indicate a problem in how Buildifier is used.
BUILDIFIER_FORMAT_ERROR_CODE = 4

def eprint(*args, **kwargs):
    """
    Print to stderr and flush (just in case).
    """
    print(*args, flush=True, file=sys.stderr, **kwargs)

def upload_output(output):
    # Generate output usable by Buildkite's annotations.
    eprint("--- :hammer_and_wrench: Printing raw output for debugging")
    eprint(output)

    eprint("+++ :buildkite: Uploading output via 'buildkite annotate'")
    result = subprocess.run(
        ["buildkite-agent", "annotate", "--style", "warning", "--context", "buildifier"],
        input=output.encode(locale.getpreferredencoding(False)),
    )
    if result.returncode != 0:
        eprint(
            ":rotating_light: 'buildkite-agent annotate' failed with exit code {}".format(
                result.returncode
            )
        )

def get_file_url(filename, line):
    commit = os.environ.get("BUILDKITE_COMMIT")
    repo = os.environ.get("BUILDKITE_PULL_REQUEST_REPO", os.environ.get("BUILDKITE_REPO", None))
    if not commit or not repo:
        return None

    # Example 1: https://github.com/bazelbuild/bazel.git
    # Example 2: git://github.com/philwo/bazel.git
    # Example 3: git@github.com:bazelbuild/bazel.git
    match = re.match(r"(?:(?:git|https?)://|git@)(github.com[:/].*)\.git", repo)
    if match:
        return "https://{}/blob/{}/{}#L{}".format(
            match[1].replace(":", "/"), commit, filename, line
        )

    return None

def run_buildifier(flag, files, what):
    eprint("+++ :bazel: Running buildifier ({})".format(what))
    return subprocess.run(
        ["buildifier", flag] + files, capture_output=True, universal_newlines=True
    )

def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]

    # Gather all files to process.
    eprint("+++ :female-detective: Looking for WORKSPACE, BUILD, BUILD.bazel and *.bzl files")
    files = []
    build_bazel_found = False
    for root, _, filenames in os.walk("."):
        for filename in filenames:
            if fnmatch.fnmatch(filename, "BUILD.bazel"):
                build_bazel_found = True
            for pattern in ("WORKSPACE", "BUILD", "BUILD.bazel", "*.bzl"):
                if fnmatch.fnmatch(filename, pattern):
                    files.append(os.path.relpath(os.path.join(root, filename)))
    if build_bazel_found:
        eprint(
            "Found BUILD.bazel files in the workspace, thus ignoring BUILD files without suffix."
        )
        files = [fname for fname in files if not fnmatch.fnmatch(os.path.basename(fname), "BUILD")]
    if not files:
        eprint("No files found, exiting.")
        return 0

    files = sorted(files)

    # Run formatter before linter since --lint=warn implies --mode=fix,
    # thus fixing any format issues.
    formatter_result = run_buildifier("--mode=check", files, "format check")
    if formatter_result.returncode and formatter_result.returncode != BUILDIFIER_FORMAT_ERROR_CODE:
        output = "##### :bazel: buildifier: error while checking format:\n"
        output += "
" + html.escape(formatter_result.stderr) + "
"
        if "BUILDKITE_JOB_ID" in os.environ:
            output += "\n\nSee [job {job}](#{job})\n".format(job=os.environ["BUILDKITE_JOB_ID"])

        upload_output(output)
        return formatter_result.returncode

    # Format: "<file name> # reformated"
    unformatted_files = [l.partition(" ")[0] for l in formatter_result.stdout.splitlines()]
    if unformatted_files:
        eprint(
            "+++ :construction: Found {} file(s) that must be formatted".format(
                len(unformatted_files)
            )
        )

    linter_result = run_buildifier("--lint=warn", files, "lint checks")
    if linter_result.returncode == 0 and not unformatted_files:
        # If buildifier was happy, there's nothing left to do for us.
        eprint("+++ :tada: Buildifier found nothing to complain about")
        return 0

    # Parse output.
    eprint("+++ :gear: Parsing buildifier output")
    findings = list(regex.finditer(linter_result.stderr))
    output = "##### :bazel: buildifier: found {} problems in your WORKSPACE, BUILD and *.bzl files\n".format(
        len(findings)
    )
    output += "
"
    for finding in findings:
        file_url = get_file_url(finding["filename"], finding["line"])
        if file_url:
            output += '{}:{}:'.format(
                file_url, finding["filename"], finding["line"]
            )
        else:
            output += "{}:{}:".format(finding["filename"], finding["line"])
        if finding["column"]:
            output += "{}:".format(finding["column"])
        output += ' {}: {}\n'.format(
            finding["message_url"], finding["message_id"], finding["message"]
        )
    output = output.strip() + "
"

    if unformatted_files:
        output += (
            "\n
There are also {} unformatted file(s) in the repository. "
            'Please download buildifier and run the following '
            "command in your workspace:"
            "
buildifier {}".format(
                len(unformatted_files), BUILDIFIER_URL, " ".join(unformatted_files)
            )
        )

    upload_output(output)

    # Preserve buildifier's exit code.
    return linter_result.returncode

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>Print Buildifier version, if available. (#533)<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import fnmatch
import html
import locale
import os.path
import re
import subprocess
import sys

regex = re.compile(
    r"^(?P<filename>[^:]*):(?P<line>\d*):(?:(?P<column>\d*):)? (?P<message_id>[^:]*): (?P<message>.*) \((?P<message_url>.*)\)$",
    re.MULTILINE,
)

BUILDIFIER_VERSION_PATTERN = re.compile(r"^buildifier version: ([\.\w]+)$", re.MULTILINE)

BUILDIFIER_URL = "https://github.com/bazelbuild/buildtools/tree/master/buildifier"

# https://github.com/bazelbuild/buildtools/blob/master/buildifier/buildifier.go#L333
# Buildifier error code for "needs formatting". We should fail on all other error codes > 0
# since they indicate a problem in how Buildifier is used.
BUILDIFIER_FORMAT_ERROR_CODE = 4

def eprint(*args, **kwargs):
    """
    Print to stderr and flush (just in case).
    """
    print(*args, flush=True, file=sys.stderr, **kwargs)

def upload_output(output):
    # Generate output usable by Buildkite's annotations.
    eprint("--- :hammer_and_wrench: Printing raw output for debugging")
    eprint(output)

    eprint("+++ :buildkite: Uploading output via 'buildkite annotate'")
    result = subprocess.run(
        ["buildkite-agent", "annotate", "--style", "warning", "--context", "buildifier"],
        input=output.encode(locale.getpreferredencoding(False)),
    )
    if result.returncode != 0:
        eprint(
            ":rotating_light: 'buildkite-agent annotate' failed with exit code {}".format(
                result.returncode
            )
        )

def get_file_url(filename, line):
    commit = os.environ.get("BUILDKITE_COMMIT")
    repo = os.environ.get("BUILDKITE_PULL_REQUEST_REPO", os.environ.get("BUILDKITE_REPO", None))
    if not commit or not repo:
        return None

    # Example 1: https://github.com/bazelbuild/bazel.git
    # Example 2: git://github.com/philwo/bazel.git
    # Example 3: git@github.com:bazelbuild/bazel.git
    match = re.match(r"(?:(?:git|https?)://|git@)(github.com[:/].*)\.git", repo)
    if match:
        return "https://{}/blob/{}/{}#L{}".format(
            match[1].replace(":", "/"), commit, filename, line
        )

    return None

def run_buildifier(flag, files=None, version=None, what=None):
    label = "+++ :bazel: Running "
    if version:
        label += "Buildifier " + version
    else:
        label += "unreleased Buildifier"
    if what:
        label += ": " + what

    eprint(label)

    args = ["buildifier", flag]
    if files:
        args += files

    return subprocess.run(args, capture_output=True, universal_newlines=True)

def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]

    # Gather all files to process.
    eprint("+++ :female-detective: Looking for WORKSPACE, BUILD, BUILD.bazel and *.bzl files")
    files = []
    build_bazel_found = False
    for root, _, filenames in os.walk("."):
        for filename in filenames:
            if fnmatch.fnmatch(filename, "BUILD.bazel"):
                build_bazel_found = True
            for pattern in ("WORKSPACE", "BUILD", "BUILD.bazel", "*.bzl"):
                if fnmatch.fnmatch(filename, pattern):
                    files.append(os.path.relpath(os.path.join(root, filename)))
    if build_bazel_found:
        eprint(
            "Found BUILD.bazel files in the workspace, thus ignoring BUILD files without suffix."
        )
        files = [fname for fname in files if not fnmatch.fnmatch(os.path.basename(fname), "BUILD")]
    if not files:
        eprint("No files found, exiting.")
        return 0

    files = sorted(files)

    eprint("+++ :female-detective: Detecting Buildifier version")
    version_result = run_buildifier("--version", what="Version info")
    match = BUILDIFIER_VERSION_PATTERN.search(version_result.stdout)
    version = match.group(1) if match and match.group(1) != "redacted" else None

    # Run formatter before linter since --lint=warn implies --mode=fix,
    # thus fixing any format issues.
    formatter_result = run_buildifier(
        "--mode=check", files=files, version=version, what="Format check"
    )
    if formatter_result.returncode and formatter_result.returncode != BUILDIFIER_FORMAT_ERROR_CODE:
        output = "##### :bazel: buildifier: error while checking format:\n"
        output += "
" + html.escape(formatter_result.stderr) + "
"
        if "BUILDKITE_JOB_ID" in os.environ:
            output += "\n\nSee [job {job}](#{job})\n".format(job=os.environ["BUILDKITE_JOB_ID"])

        upload_output(output)
        return formatter_result.returncode

    # Format: "<file name> # reformated"
    unformatted_files = [l.partition(" ")[0] for l in formatter_result.stdout.splitlines()]
    if unformatted_files:
        eprint(
            "+++ :construction: Found {} file(s) that must be formatted".format(
                len(unformatted_files)
            )
        )

    linter_result = run_buildifier("--lint=warn", files=files, version=version, what="Lint checks")
    if linter_result.returncode == 0 and not unformatted_files:
        # If buildifier was happy, there's nothing left to do for us.
        eprint("+++ :tada: Buildifier found nothing to complain about")
        return 0

    # Parse output.
    eprint("+++ :gear: Parsing buildifier output")
    findings = list(regex.finditer(linter_result.stderr))
    output = "##### :bazel: buildifier: found {} problems in your WORKSPACE, BUILD and *.bzl files\n".format(
        len(findings)
    )
    output += "
"
    for finding in findings:
        file_url = get_file_url(finding["filename"], finding["line"])
        if file_url:
            output += '{}:{}:'.format(
                file_url, finding["filename"], finding["line"]
            )
        else:
            output += "{}:{}:".format(finding["filename"], finding["line"])
        if finding["column"]:
            output += "{}:".format(finding["column"])
        output += ' {}: {}\n'.format(
            finding["message_url"], finding["message_id"], finding["message"]
        )
    output = output.strip() + "
"

    if unformatted_files:
        output += (
            "\n
There are also {} unformatted file(s) in the repository. "
            'Please download buildifier and run the following '
            "command in your workspace:"
            "
buildifier {}".format(
                len(unformatted_files), BUILDIFIER_URL, " ".join(unformatted_files)
            )
        )

    upload_output(output)

    # Preserve buildifier's exit code.
    return linter_result.returncode

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>