<commit_before>#!/usr/bin/env python

# Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Command-line application to run a query using user credentials.

You must supply a client secrets file, which would normally be bundled with
your application.
"""

import argparse

from google.cloud import bigquery
from google_auth_oauthlib import flow

def run_query(credentials, project, query):
    client = bigquery.Client(project=project, credentials=credentials)
    query_job = client.query(query)

    # Print the results.
    for row in query_job.result():  # Wait for the job to complete.
        print(row)

def authenticate_and_query(project, query, launch_browser=True):
    appflow = flow.InstalledAppFlow.from_client_secrets_file(
        'client_secrets.json',
        scopes=['https://www.googleapis.com/auth/bigquery'])

    if launch_browser:
        appflow.run_local_server()
    else:
        appflow.run_console()

    run_query(appflow.credentials, project, query)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument(
        '--launch-browser',
        help='Use a local server flow to authenticate. ',
        action='store_true')
    parser.add_argument('project', help='Project to use for BigQuery billing.')
    parser.add_argument('query', help='BigQuery SQL Query.')

    args = parser.parse_args()

    authenticate_and_query(
        args.project, args.query, launch_browser=args.launch_browser)
<commit_msg>Move imports into bigquery samples<commit_after>#!/usr/bin/env python

# Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Command-line application to run a query using user credentials.

You must supply a client secrets file, which would normally be bundled with
your application.
"""

import argparse

def run_query(credentials, project, query):
    from google.cloud import bigquery

    client = bigquery.Client(project=project, credentials=credentials)
    query_job = client.query(query)

    # Print the results.
    for row in query_job.result():  # Wait for the job to complete.
        print(row)

def authenticate_and_query(project, query, launch_browser=True):
    from google_auth_oauthlib import flow

    appflow = flow.InstalledAppFlow.from_client_secrets_file(
        'client_secrets.json',
        scopes=['https://www.googleapis.com/auth/bigquery'])

    if launch_browser:
        appflow.run_local_server()
    else:
        appflow.run_console()

    run_query(appflow.credentials, project, query)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument(
        '--launch-browser',
        help='Use a local server flow to authenticate. ',
        action='store_true')
    parser.add_argument('project', help='Project to use for BigQuery billing.')
    parser.add_argument('query', help='BigQuery SQL Query.')

    args = parser.parse_args()

    authenticate_and_query(
        args.project, args.query, launch_browser=args.launch_browser)
<|endoftext|>


<commit_before>import json

from django.core import serializers
from django.core.serializers.json import DjangoJSONEncoder

from ..generic.websockets import WebsocketMultiplexer
from ..sessions import enforce_ordering
from .base import Binding

class WebsocketBinding(Binding):
    """
    Websocket-specific outgoing binding subclass that uses JSON encoding
    and the built-in JSON/WebSocket multiplexer.

    To implement outbound, implement:
     - group_names, which returns a list of group names to send to

    To implement inbound, implement:
     - has_permission, which says if the user can do the action on an instance

    Optionally also implement:
     - serialize_data, which returns JSON-safe data from a model instance
     - create, which takes incoming data and makes a model instance
     - update, which takes incoming data and a model instance and applies one to the other
    """

    # Mark as abstract
    model = None

    # Stream multiplexing name
    stream = None

    # Decorators
    strict_ordering = False
    slight_ordering = False

    # Outbound
    @classmethod
    def encode(cls, stream, payload):
        return WebsocketMultiplexer.encode(stream, payload)

    def serialize(self, instance, action):
        payload = {
            "action": action,
            "pk": instance.pk,
            "data": self.serialize_data(instance),
            "model": self.model_label,
        }
        return payload

    def serialize_data(self, instance):
        """
        Serializes model data into JSON-compatible types.
        """
        if self.fields is not None:
            if self.fields == '__all__' or list(self.fields) == ['__all__']:
                fields = None
            else:
                fields = self.fields
        else:
            fields = [f.name for f in instance._meta.get_fields() if f.name not in self.exclude]
        data = serializers.serialize('json', [instance], fields=fields)
        return json.loads(data)[0]['fields']

    # Inbound
    @classmethod
    def get_handler(cls):
        """
        Adds decorators to trigger_inbound.
        """
        # Get super-handler
        handler = super(WebsocketBinding, cls).get_handler()
        # Ordering decorators
        if cls.strict_ordering:
            return enforce_ordering(handler, slight=False)
        elif cls.slight_ordering:
            return enforce_ordering(handler, slight=True)
        else:
            return handler

    @classmethod
    def trigger_inbound(cls, message, **kwargs):
        """
        Overrides base trigger_inbound to ignore connect/disconnect.
        """
        # Only allow received packets through further.
        if message.channel.name != "websocket.receive":
            return
        super(WebsocketBinding, cls).trigger_inbound(message, **kwargs)

    def deserialize(self, message):
        """
        You must hook this up behind a Deserializer, so we expect the JSON
        already dealt with.
        """
        body = json.loads(message['text'])
        action = body['action']
        pk = body.get('pk', None)
        data = body.get('data', None)
        return action, pk, data

    def _hydrate(self, pk, data):
        """
        Given a raw "data" section of an incoming message, returns a
        DeserializedObject.
        """
        s_data = [
            {
                "pk": pk,
                "model": self.model_label,
                "fields": data,
            }
        ]
        # TODO: Avoid the JSON roundtrip by using encoder directly?
        return list(serializers.deserialize("json", json.dumps(s_data)))[0]

    def create(self, data):
        self._hydrate(None, data).save()

    def update(self, pk, data):
        instance = self.model.objects.get(pk=pk)
        hydrated = self._hydrate(pk, data)

        if self.fields is not None:
            for name in data.keys():
                if name in self.fields or self.fields == ['__all__']:
                    setattr(instance, name, getattr(hydrated.object, name))
        else:
            for name in data.keys():
                if name not in self.exclude:
                    setattr(instance, name, getattr(hydrated.object, name))
        instance.save()

class WebsocketBindingWithMembers(WebsocketBinding):
    """
    Outgoing binding binding subclass based on WebsocketBinding.
    Additionally enables sending of member variables, properties and methods.
    Member methods can only have self as a required argument.
    Just add the name of the member to the send_members-list.
    Example:

    class MyModel(models.Model):
        my_field = models.IntegerField(default=0)
        my_var = 3

        @property
        def my_property(self):
            return self.my_var + self.my_field

        def my_function(self):
            return self.my_var - self.my_vield

    class MyBinding(BindingWithMembersMixin, WebsocketBinding):
        model = MyModel
        stream = 'mystream'

        send_members = ['my_var', 'my_property', 'my_function']
    """

    model = None
    send_members = []

    encoder = DjangoJSONEncoder()

    def serialize_data(self, instance, **kwargs):
        data = super(WebsocketBindingWithMembers, self).serialize_data(instance, **kwargs)
        member_data = {}
        for m in self.send_members:
            member = instance
            for s in m.split('.'):
                member = getattr(member, s)
            if callable(member):
                member_data[m.replace('.', '__')] = member()
            else:
                member_data[m.replace('.', '__')] = member
        member_data = json.loads(self.encoder.encode(member_data))
        # the update never overwrites any value from data,
        # because an object can't have two attributes with the same name
        data.update(member_data)
        return data
<commit_msg>Remove encode/decode overhead at binding (#611)<commit_after>import json

from django.core import serializers
from django.core.serializers.json import DjangoJSONEncoder

from ..generic.websockets import WebsocketMultiplexer
from ..sessions import enforce_ordering
from .base import Binding

class WebsocketBinding(Binding):
    """
    Websocket-specific outgoing binding subclass that uses JSON encoding
    and the built-in JSON/WebSocket multiplexer.

    To implement outbound, implement:
     - group_names, which returns a list of group names to send to

    To implement inbound, implement:
     - has_permission, which says if the user can do the action on an instance

    Optionally also implement:
     - serialize_data, which returns JSON-safe data from a model instance
     - create, which takes incoming data and makes a model instance
     - update, which takes incoming data and a model instance and applies one to the other
    """

    # Mark as abstract
    model = None

    # Stream multiplexing name
    stream = None

    # Decorators
    strict_ordering = False
    slight_ordering = False

    # Outbound
    @classmethod
    def encode(cls, stream, payload):
        return WebsocketMultiplexer.encode(stream, payload)

    def serialize(self, instance, action):
        payload = {
            "action": action,
            "pk": instance.pk,
            "data": self.serialize_data(instance),
            "model": self.model_label,
        }
        return payload

    def serialize_data(self, instance):
        """
        Serializes model data into JSON-compatible types.
        """
        if self.fields is not None:
            if self.fields == '__all__' or list(self.fields) == ['__all__']:
                fields = None
            else:
                fields = self.fields
        else:
            fields = [f.name for f in instance._meta.get_fields() if f.name not in self.exclude]
        data = serializers.serialize('json', [instance], fields=fields)
        return json.loads(data)[0]['fields']

    # Inbound
    @classmethod
    def get_handler(cls):
        """
        Adds decorators to trigger_inbound.
        """
        # Get super-handler
        handler = super(WebsocketBinding, cls).get_handler()
        # Ordering decorators
        if cls.strict_ordering:
            return enforce_ordering(handler, slight=False)
        elif cls.slight_ordering:
            return enforce_ordering(handler, slight=True)
        else:
            return handler

    @classmethod
    def trigger_inbound(cls, message, **kwargs):
        """
        Overrides base trigger_inbound to ignore connect/disconnect.
        """
        # Only allow received packets through further.
        if message.channel.name != "websocket.receive":
            return
        super(WebsocketBinding, cls).trigger_inbound(message, **kwargs)

    def deserialize(self, message):
        """
        You must hook this up behind a Deserializer, so we expect the JSON
        already dealt with.
        """
        body = json.loads(message['text'])
        action = body['action']
        pk = body.get('pk', None)
        data = body.get('data', None)
        return action, pk, data

    def _hydrate(self, pk, data):
        """
        Given a raw "data" section of an incoming message, returns a
        DeserializedObject.
        """
        s_data = [
            {
                "pk": pk,
                "model": self.model_label,
                "fields": data,
            }
        ]
        return list(serializers.deserialize("python", s_data))[0]

    def create(self, data):
        self._hydrate(None, data).save()

    def update(self, pk, data):
        instance = self.model.objects.get(pk=pk)
        hydrated = self._hydrate(pk, data)

        if self.fields is not None:
            for name in data.keys():
                if name in self.fields or self.fields == ['__all__']:
                    setattr(instance, name, getattr(hydrated.object, name))
        else:
            for name in data.keys():
                if name not in self.exclude:
                    setattr(instance, name, getattr(hydrated.object, name))
        instance.save()

class WebsocketBindingWithMembers(WebsocketBinding):
    """
    Outgoing binding binding subclass based on WebsocketBinding.
    Additionally enables sending of member variables, properties and methods.
    Member methods can only have self as a required argument.
    Just add the name of the member to the send_members-list.
    Example:

    class MyModel(models.Model):
        my_field = models.IntegerField(default=0)
        my_var = 3

        @property
        def my_property(self):
            return self.my_var + self.my_field

        def my_function(self):
            return self.my_var - self.my_vield

    class MyBinding(BindingWithMembersMixin, WebsocketBinding):
        model = MyModel
        stream = 'mystream'

        send_members = ['my_var', 'my_property', 'my_function']
    """

    model = None
    send_members = []

    encoder = DjangoJSONEncoder()

    def serialize_data(self, instance, **kwargs):
        data = super(WebsocketBindingWithMembers, self).serialize_data(instance, **kwargs)
        member_data = {}
        for m in self.send_members:
            member = instance
            for s in m.split('.'):
                member = getattr(member, s)
            if callable(member):
                member_data[m.replace('.', '__')] = member()
            else:
                member_data[m.replace('.', '__')] = member
        member_data = json.loads(self.encoder.encode(member_data))
        # the update never overwrites any value from data,
        # because an object can't have two attributes with the same name
        data.update(member_data)
        return data
<|endoftext|>


<commit_before>import json

from django.core.context_processors import csrf
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render, render_to_response
from django.template import Template, Context
from django.template.loader import get_template

from pathgenerator.models import Place, Path, UserForm

# Create your views here.
def map_page(request):
    t = get_template('map.html')
    html = t.render(Context())
    return HttpResponse(html)

def mapsearch_page(request):
    # Need to add csrf token to every page with a form or ajax
    c = {"tag_list":["restaurant", "cafe", "book_store", "zoo"],
        "login_form": UserForm()}
    c.update(csrf(request))
    return render_to_response('map_search.html', c)

def place(request):
    if request.method == 'POST':
        # this is a list of strings with one element
        place_string = request.POST.get('results')
        # print json_string
        process_place_json(place_string)
        return HttpResponse('success')
    else:
        return HttpResponse('Not implemented')

def process_place_json(place_string):
    '''
    Processes json into places.
    string_list - a list of strings which are json objects
    '''
    # this is now a list of dicts
    place_json = json.loads(place_string)
    for place in place_json:
        # Create a new Place only if not currently exists
        # This is a workaround because django doesn't like the string 'id'
        _place_id = str(place['id'])
        if Place.objects.filter(google_id=_place_id).exists():
            print 'Object already in database'
        else:
            # Not sure if there will be decimal problems
            lat = place['geometry']['location']['d']
            lng = place['geometry']['location']['e']
            Place.objects.create(
                latitude=lat,
                longitude=lng,
                address='',
                name=place['name'],
                source='google',
                google_reference=place['reference'],
                google_id=_place_id,
                raw_data=json.dumps(place, separators=(',',':')),
                )

# Json format contains
# start, end - which are google latlng
# waypointIDs - strings that should correspond to an existing Place
#

# start, end, name
# start: {lat:
# lng:
# }

# waypoints: {
#     id: asdf,
#     lat:
#     lng:
# }
# use decorator?
def path(request):
    # User must be logged in to get and post paths
    # if not request.user.is_authenticated():
    #     return HttpResponse("User not logged in")
    if request.method == "GET":
        # get user's path
        path_name = request.GET['path-name']
        # problem:raises error if the path doesn't exist?
        try:
            path = Path.objects.get(user=request.user, name=path_name)
        except:
            # Status code for "no content"
            return HttpResponse(content="Path with specified user and name doesn't exist")
        # return a Json object that can be reconstituted into a path
        # Have to load waypoints otherwise it doesn't get deserialized correctly
        path_dict = {
            "name": path.name,
            "start": {
                "lat": str(path.start_lat),
                "lng": str(path.start_lng)
            },
            "end": {
                "lat": str(path.end_lat),
                "lng": str(path.end_lng),
            },
            "waypoints": json.loads(path.json)
        }
        json_string = json.dumps(path_dict)
        return HttpResponse(json.dumps(path_dict), content_type="application/json")
    elif request.method == "POST":
        # waypoints is a json
        path_name = request.POST['name']
        # Prevent paths with duplicate names
        # If you want to edit a path, should use a PUT request
        if Path.objects.filter(name=path_name, user=request.user).exists():
            return HttpResponse('Path already exists')
        waypoints = request.POST['waypoints']
        start = json.loads(request.POST['start'])
        end = json.loads(request.POST['end'])
        # get user
        path = Path.objects.create(
            name=path_name,
            json=waypoints,
            user=request.user,
            start_lat=start['d'],
            start_lng=start['e'],
            end_lat=end['d'],
            end_lng=end['e']
            )
        # get a Place for each waypoint
        waypoints = json.loads(waypoints)
        for waypoint in waypoints:
            # We assume that a place exists. Problem: duplicate places?
            place = Place.objects.filter(google_id=waypoint['id'])
            if place.count() != 1:
                print place
                return HttpResponse('more than one error with same id')
            path.places.add(place)
        path.save() # is this necessary?
        return HttpResponse('success')
    else:
        return HttpResponse('not implemented yet')

def saved_path_page(request):
    # TODO implement display of user paths
    # Should not be able to view this page without being logged in
    if request.user.is_authenticated():
        # Get all paths that belong to user
        paths = Path.objects.filter(user=request.user)
        c = {"paths": paths}
        c.update(csrf(request))
        return render_to_response('saved_path_page.html', c)
    else:
        # if user not logged in, can't see saved paths
        return HttpResponseRedirect('/')

'''
TODO: actually put this in database
Implement getting paths from ID.
Query which paths a user has.
'''
<commit_msg>Fixes place saving code<commit_after>import json

from django.core.context_processors import csrf
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render, render_to_response
from django.template import Template, Context
from django.template.loader import get_template

from pathgenerator.models import Place, Path, UserForm

# Create your views here.
def map_page(request):
    t = get_template('map.html')
    html = t.render(Context())
    return HttpResponse(html)

def mapsearch_page(request):
    # Need to add csrf token to every page with a form or ajax
    c = {"tag_list":["restaurant", "cafe", "book_store", "zoo"],
        "login_form": UserForm()}
    c.update(csrf(request))
    return render_to_response('map_search.html', c)

def place(request):
    if request.method == 'POST':
        # this is a list of strings with one element
        place_string = request.POST.get('results')
        # print json_string
        process_place_json(place_string)
        return HttpResponse('success')
    else:
        return HttpResponse('Not implemented')

def process_place_json(place_string):
    '''
    Processes json into places.
    string_list - a list of strings which are json objects
    '''
    # this is now a list of dicts
    place_json = json.loads(place_string)
    for place in place_json:
        # Create a new Place only if not currently exists
        # This is a workaround because django doesn't like the string 'id'
        _place_id = str(place['id'])
        if Place.objects.filter(google_id=_place_id).exists():
            print 'Object already in database'
        else:
            # Not sure if there will be decimal problems
            lat = place['geometry']['location']['d']
            lng = place['geometry']['location']['e']
            Place.objects.create(
                latitude=lat,
                longitude=lng,
                address='',
                name=place['name'],
                source='google',
                google_reference=place['reference'],
                google_id=_place_id,
                raw_data=json.dumps(place, separators=(',',':')),
                )

# Json format contains
# start, end - which are google latlng
# waypointIDs - strings that should correspond to an existing Place
#

# start, end, name
# start: {lat:
# lng:
# }

# waypoints: {
#     id: asdf,
#     lat:
#     lng:
# }
# use decorator?
def path(request):
    # User must be logged in to get and post paths
    if not request.user.is_authenticated():
        return HttpResponse("User not logged in")
    if request.method == "GET":
        # get user's path
        path_name = request.GET['path-name']
        # problem:raises error if the path doesn't exist?
        try:
            path = Path.objects.get(user=request.user, name=path_name)
        except:
            # Status code for "no content"
            return HttpResponse(content="Path with specified user and name doesn't exist")
        # return a Json object that can be reconstituted into a path
        # Have to load waypoints otherwise it doesn't get deserialized correctly
        path_dict = {
            "name": path.name,
            "start": {
                "lat": str(path.start_lat),
                "lng": str(path.start_lng)
            },
            "end": {
                "lat": str(path.end_lat),
                "lng": str(path.end_lng),
            },
            "waypoints": json.loads(path.json)
        }
        json_string = json.dumps(path_dict)
        return HttpResponse(json.dumps(path_dict), content_type="application/json")
    elif request.method == "POST":
        # waypoints is a json
        path_name = request.POST['name']
        # Prevent paths with duplicate names
        # If you want to edit a path, should use a PUT request
        if Path.objects.filter(name=path_name, user=request.user).exists():
            return HttpResponse('Path already exists')
        waypoints = request.POST['waypoints']
        start = json.loads(request.POST['start'])
        end = json.loads(request.POST['end'])
        # get user
        path = Path.objects.create(
            name=path_name,
            json=waypoints,
            user=request.user,
            start_lat=start['d'],
            start_lng=start['e'],
            end_lat=end['d'],
            end_lng=end['e']
            )
        # get a Place for each waypoint
        waypoints = json.loads(waypoints)
        for waypoint in waypoints:
            # We assume that a place exists. Problem: duplicate places?
            try:
                place = Place.objects.get(google_id=waypoint['id'])
            except:
                return HttpResponse('more than one place with same id')
            path.places.add(place)
        path.save() # is this necessary?
        return HttpResponse('success')
    else:
        return HttpResponse('not implemented yet')

def saved_path_page(request):
    # TODO implement display of user paths
    # Should not be able to view this page without being logged in
    if request.user.is_authenticated():
        # Get all paths that belong to user
        paths = Path.objects.filter(user=request.user)
        c = {"paths": paths}
        c.update(csrf(request))
        return render_to_response('saved_path_page.html', c)
    else:
        # if user not logged in, can't see saved paths
        return HttpResponseRedirect('/')

'''
TODO: actually put this in database
Implement getting paths from ID.
Query which paths a user has.
'''
<|endoftext|>


<commit_before>from __future__ import unicode_literals
'''
Inspired by http://www.thescripts.com/forum/thread46361.html
'''

#===================================================================================================
# Reraise
#===================================================================================================
def Reraise(exception, message, separator='\n'):
    '''
    Raised the same exception given, with an additional message.

    :param Exception exception:
        Original exception being raised with additional messages

    :param unicode message:
        Message to be added to the given exception

    :param unicode separator:
        String separating `message` from the `exception`'s original message.

    e.g.
        try:
            raise RuntimeError('original message')
        except Exception, e:
            Reraise(e, 'message')

        >>> RuntimeError:
        >>> message
        >>> original message

        try:
            raise RuntimeError('original message')
        except Exception, e:
            Reraise(e, '[message]', separator=' ')

        >>> RuntimeError:
        >>> [message] original message
    '''
    import sys

    # IMPORTANT: This should be the first thing to be done in this method or else we might risk losing the frame and
    # corruping the Traceback
    frame = sys.exc_info()[-1]

    from ben10.foundation.exceptions import ExceptionToUnicode

    if hasattr(exception, 'reraised_message'):
        current_message = exception.reraised_message
    else:
        current_message = ExceptionToUnicode(exception)

    # Build the new message
    if not current_message.startswith(separator):
        current_message = separator + current_message
    message = '\n' + message + current_message

    if exception.__class__ in _SPECIAL_EXCEPTION_MAP:
        # Handling for special case, some exceptions have different behaviors.
        exception = _SPECIAL_EXCEPTION_MAP[exception.__class__](*exception.args)

    else:
        # In Python 2.5 overriding the exception "__str__" has no effect in "unicode()". Instead, we
        # must change the "args" attribute which is used to build the string representation.
        # Even though the documentation says "args" will be deprecated, it uses its first argument
        # in unicode() implementation and not "message".
        exception.args = (exception.message,)

    exception.message = message
    # keep the already decoded message in the object in case this exception is reraised again
    exception.reraised_message = message

    # Reraise the exception with the EXTRA message information
    raise exception, None, frame

#===================================================================================================
# SPECIAL_EXCEPTIONS
#===================================================================================================
# [[[cog
# SPECIAL_EXCEPTIONS = (
#     KeyError,
#     OSError,
#     SyntaxError,
#     UnicodeDecodeError,
#     UnicodeEncodeError,
# )
# from ben10.foundation.string import Dedent
# exception_map = []
# for exception_class in SPECIAL_EXCEPTIONS:
#     superclass_name = exception_class.__name__
#     exception_map.append('\n        ' + superclass_name + ' : Reraised' + superclass_name + ',')
#     cog.out(Dedent(
#         '''
#         class Reraised%(superclass_name)s(%(superclass_name)s):
#             def __init__(self, *args):
#                 %(superclass_name)s.__init__(self, *args)
#                 self.message = None
#
#             def __str__(self):
#                 return self.message
#
#
#         '''% locals()
#     ))
# cog.out(Dedent(
#     '''
#     _SPECIAL_EXCEPTION_MAP = {%s
#     }
#     ''' % ''.join(exception_map)
# ))
# ]]]
class ReraisedKeyError(KeyError):
    def __init__(self, *args):
        KeyError.__init__(self, *args)
        self.message = None

    def __str__(self):
        return self.message

class ReraisedOSError(OSError):
    def __init__(self, *args):
        OSError.__init__(self, *args)
        self.message = None

    def __str__(self):
        return self.message

class ReraisedSyntaxError(SyntaxError):
    def __init__(self, *args):
        SyntaxError.__init__(self, *args)
        self.message = None

    def __str__(self):
        return self.message

class ReraisedUnicodeDecodeError(UnicodeDecodeError):
    def __init__(self, *args):
        UnicodeDecodeError.__init__(self, *args)
        self.message = None

    def __str__(self):
        return self.message

class ReraisedUnicodeEncodeError(UnicodeEncodeError):
    def __init__(self, *args):
        UnicodeEncodeError.__init__(self, *args)
        self.message = None

    def __str__(self):
        return self.message

_SPECIAL_EXCEPTION_MAP = {
    KeyError : ReraisedKeyError,
    OSError : ReraisedOSError,
    SyntaxError : ReraisedSyntaxError,
    UnicodeDecodeError : ReraisedUnicodeDecodeError,
    UnicodeEncodeError : ReraisedUnicodeEncodeError,
}
# [[[end]]] (checksum: 457262f8ea7b2cb41fc010baa24fac41)
<commit_msg>Inlined 'frame' and added comment to avoid using try/except mechanism inside Reraise()<commit_after>from __future__ import unicode_literals
'''
Inspired by http://www.thescripts.com/forum/thread46361.html
'''

#===================================================================================================
# Reraise
#===================================================================================================
def Reraise(exception, message, separator='\n'):
    '''
    Raised the same exception given, with an additional message.

    :param Exception exception:
        Original exception being raised with additional messages

    :param unicode message:
        Message to be added to the given exception

    :param unicode separator:
        String separating `message` from the `exception`'s original message.

    e.g.
        try:
            raise RuntimeError('original message')
        except Exception, e:
            Reraise(e, 'message')

        >>> RuntimeError:
        >>> message
        >>> original message

        try:
            raise RuntimeError('original message')
        except Exception, e:
            Reraise(e, '[message]', separator=' ')

        >>> RuntimeError:
        >>> [message] original message
    '''
    from ben10.foundation.exceptions import ExceptionToUnicode
    import sys

    # IMPORTANT: Do NOT use try/except mechanisms in this method or the sys.exc_info()[-1] will be invalid

    if hasattr(exception, 'reraised_message'):
        current_message = exception.reraised_message
    else:
        current_message = ExceptionToUnicode(exception)

    # Build the new message
    if not current_message.startswith(separator):
        current_message = separator + current_message
    message = '\n' + message + current_message

    if exception.__class__ in _SPECIAL_EXCEPTION_MAP:
        # Handling for special case, some exceptions have different behaviors.
        exception = _SPECIAL_EXCEPTION_MAP[exception.__class__](*exception.args)

    else:
        # In Python 2.5 overriding the exception "__str__" has no effect in "unicode()". Instead, we
        # must change the "args" attribute which is used to build the string representation.
        # Even though the documentation says "args" will be deprecated, it uses its first argument
        # in unicode() implementation and not "message".
        exception.args = (exception.message,)

    exception.message = message
    # keep the already decoded message in the object in case this exception is reraised again
    exception.reraised_message = message

    # Reraise the exception with the EXTRA message information
    raise exception, None, sys.exc_info()[-1]

#===================================================================================================
# SPECIAL_EXCEPTIONS
#===================================================================================================
# [[[cog
# SPECIAL_EXCEPTIONS = (
#     KeyError,
#     OSError,
#     SyntaxError,
#     UnicodeDecodeError,
#     UnicodeEncodeError,
# )
# from ben10.foundation.string import Dedent
# exception_map = []
# for exception_class in SPECIAL_EXCEPTIONS:
#     superclass_name = exception_class.__name__
#     exception_map.append('\n        ' + superclass_name + ' : Reraised' + superclass_name + ',')
#     cog.out(Dedent(
#         '''
#         class Reraised%(superclass_name)s(%(superclass_name)s):
#             def __init__(self, *args):
#                 %(superclass_name)s.__init__(self, *args)
#                 self.message = None
#
#             def __str__(self):
#                 return self.message
#
#
#         '''% locals()
#     ))
# cog.out(Dedent(
#     '''
#     _SPECIAL_EXCEPTION_MAP = {%s
#     }
#     ''' % ''.join(exception_map)
# ))
# ]]]
class ReraisedKeyError(KeyError):
    def __init__(self, *args):
        KeyError.__init__(self, *args)
        self.message = None

    def __str__(self):
        return self.message

class ReraisedOSError(OSError):
    def __init__(self, *args):
        OSError.__init__(self, *args)
        self.message = None

    def __str__(self):
        return self.message

class ReraisedSyntaxError(SyntaxError):
    def __init__(self, *args):
        SyntaxError.__init__(self, *args)
        self.message = None

    def __str__(self):
        return self.message

class ReraisedUnicodeDecodeError(UnicodeDecodeError):
    def __init__(self, *args):
        UnicodeDecodeError.__init__(self, *args)
        self.message = None

    def __str__(self):
        return self.message

class ReraisedUnicodeEncodeError(UnicodeEncodeError):
    def __init__(self, *args):
        UnicodeEncodeError.__init__(self, *args)
        self.message = None

    def __str__(self):
        return self.message

_SPECIAL_EXCEPTION_MAP = {
    KeyError : ReraisedKeyError,
    OSError : ReraisedOSError,
    SyntaxError : ReraisedSyntaxError,
    UnicodeDecodeError : ReraisedUnicodeDecodeError,
    UnicodeEncodeError : ReraisedUnicodeEncodeError,
}
# [[[end]]] (checksum: 457262f8ea7b2cb41fc010baa24fac41)
<|endoftext|>


<commit_before># Copyright 2011 The greplin-nagios-utils Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""The Greplin monitoring package."""

import httplib
import json
import socket
import sys
import time

UNKNOWN = 3

CRITICAL = 2

WARNING = 1

OK = 0

STATUS_NAME = ['OK', 'WARN', 'CRIT', 'UNKNOWN']

def wgetWithTimeout(host, port, path, timeout, secure = False):
  """Gets an http page, but times out if it's too slow."""
  start = time.time()
  try:
    if secure:
      conn = httplib.HTTPSConnection(host, port, timeout=timeout)
    else:
      conn = httplib.HTTPConnection(host, port, timeout=timeout)
    conn.request('GET', path)
    body = conn.getresponse().read()
    return time.time() - start, body

  except (socket.gaierror, socket.error):
    print("CRIT: Could not connect to %s" % host)
    exit(CRITICAL)

  except socket.timeout:
    print("CRIT: Timed out after %s seconds" % timeout)
    exit(CRITICAL)

def parseJson(text):
  """Parses JSON, exiting with CRIT if the parse fails."""
  try:
    return json.loads(text)

  except ValueError, e:
    print('CRIT: %s' % e)
    exit(CRITICAL)

def parseJsonFile(filename):
  """Parses JSON from a file, exiting with UNKNOWN if the file does not exist."""
  try:
    with open(filename) as f:
      return parseJson(f.read())
  except IOError, e:
    print('UNKNOWN: %s' % e)
    exit(UNKNOWN)

def lookup(source, *keys, **kw):
  """Successively looks up each key, returning the default keyword arg if a dead end is reached."""
  fallback = kw.get('default')
  try:
    for key in keys:
      source = source[key]
    return source
  except KeyError:
    return fallback
  except AttributeError:
    return fallback
  except TypeError:
    return fallback

def statValue(data, *keys, **kw):
  """Returns the value of a stat."""
  return float(lookup(data, *keys, **kw))

def percent(value):
  """Formats the given float as a percentage."""
  return "%f%%" % (value * 100)

def parseArgs(scriptName, *args, **kw):
  """Parses arguments to the script."""
  argv = kw.get('argv', sys.argv)
  if len(argv) != len(args) + 1:
    print('USAGE: %s %s' % (scriptName, ' '.join([name for name, _ in args])))
    exit(UNKNOWN)

  result = {}
  idx = 0
  for name, fn in args:
    try:
      idx += 1
      result[name] = fn(argv[idx])
    except ValueError:
      print("Invalid value for %s: %r." % (name, argv[1]))
      exit(UNKNOWN)
  return result

class Rule(object):
  """A rule for when to warn or crit based on a stat value."""

  def check(self, value):
    """Checks if this rule should result in a WARN or CRIT."""
    raise NotImplementedError

class Minimum(Rule):
  """A rule that specifies minimum acceptable levels for a metric."""

  def __init__(self, warnLevel, critLevel, unit = ''):
    Rule.__init__(self)
    assert critLevel <= warnLevel
    self.warnLevel = warnLevel
    self.critLevel = critLevel
    self.unit = unit

  def check(self, value):
    """Checks if the given value is under the minimums."""
    if value < self.critLevel:
      return CRITICAL
    elif value < self.warnLevel:
      return WARNING
    else:
      return OK

  def format(self, name, value):
    """Formats as perf data."""
    return "'%s'=%d%s;%d;%d;;;" % (name, value, self.unit, self.warnLevel, self.critLevel)

  def message(self, name, value):
    """Create an error message."""
    if self.check(value) == CRITICAL:
      return '%s: %d%s < %d%s' % (name, value, self.unit, self.critLevel, self.unit)
    elif self.check(value) == WARNING:
      return '%s: %d%s < %d%s' % (name, value, self.unit, self.warnLevel, self.unit)

class Maximum(Rule):
  """A rule that specifies maximum acceptable levels for a metric."""

  def __init__(self, warnLevel, critLevel, unit = ''):
    Rule.__init__(self)
    assert critLevel >= warnLevel
    self.warnLevel = warnLevel
    self.critLevel = critLevel
    self.unit = unit

  def check(self, value):
    """Checks if the given value exceeds the maximums."""
    if value > self.critLevel:
      return CRITICAL
    elif value > self.warnLevel:
      return WARNING
    else:
      return OK

  def format(self, name, value):
    """Formats as perf data."""
    return "'%s'=%d%s;%d;%d;;;" % (name, value, self.unit, self.warnLevel, self.critLevel)

  def message(self, name, value):
    """Create an error message."""
    if self.check(value) == CRITICAL:
      return '%s: %d%s > %d%s' % (name, value, self.unit, self.critLevel, self.unit)
    elif self.check(value) == WARNING:
      return '%s: %d%s > %d%s' % (name, value, self.unit, self.warnLevel, self.unit)

class ResponseBuilder(object):
  """NRPE response builder."""

  def __init__(self):
    self._stats = []
    self._status = OK
    self._messages = []

  def addValue(self, name, value):
    """Adds a value to be tracked."""
    self._stats.append("'%s'=%s;;;;;" % (name, str(value)))
    return self

  def addStatLookup(self, name, data, *keys, **kw):
    """Adds a stat from a sequential key lookup."""
    value = lookup(data, *keys, **kw)
    return self.addValue(name, str(value) + kw.get('suffix', ''))

  def addStatChildren(self, name, data, *keys, **kw):
    """Adds a child for each child of the given dict."""
    values = lookup(data, *keys, **kw)
    if values:
      for childName, value in values.items():
        self.addValue(name % childName, str(value) + kw.get('suffix', ''))
    return self

  def addRule(self, name, rule, value):
    """Adds an alert rule and associated performance data."""
    status = rule.check(value)
    if status:
      self._status = max(self._status, rule.check(value))
      self._messages.append(rule.message(name, value))
    self._stats.append(rule.format(name, value))
    return self

  def warnIf(self, condition, message=None):
    """Warn on a given condition."""
    if condition:
      self.warn(message)

  def critIf(self, condition, message=None):
    """Mark state as critical on the given condition."""
    if condition:
      self.crit(message)

  def unknownIf(self, condition, message=None):
    """Mark state as unknown on the given condition."""
    if condition:
      self.unknown(message)

  def warn(self, message=None):
    """Mark state as warning."""
    self._status = max(self._status, WARNING)
    if message is not None:
      self.message(message)
    return self

  def crit(self, message=None):
    """Mark state as critical."""
    self._status = max(self._status, CRITICAL)
    if message is not None:
      self.message(message)
    return self

  def unknown(self, message=None):
    """Mark state as unknown."""
    self._status = max(self._status, UNKNOWN)
    if message is not None:
      self.message(message)
    return self

  def message(self, message):
    """Set the output message."""
    if message:
      self._messages.append(message)
    return self

  def build(self):
    """Builds the response."""
    return ' '.join(self._stats)

  def finish(self):
    """Builds the response, prints it, and exits."""
    output = STATUS_NAME[self._status]
    if self._messages:
      output += ': ' + (', '.join(self._messages))
    if self._stats:
      output += '|' + self.build()

    print(output)
    sys.exit(self._status)
<commit_msg>Builder pattern fix<commit_after># Copyright 2011 The greplin-nagios-utils Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""The Greplin monitoring package."""

import httplib
import json
import socket
import sys
import time

UNKNOWN = 3

CRITICAL = 2

WARNING = 1

OK = 0

STATUS_NAME = ['OK', 'WARN', 'CRIT', 'UNKNOWN']

def wgetWithTimeout(host, port, path, timeout, secure = False):
  """Gets an http page, but times out if it's too slow."""
  start = time.time()
  try:
    if secure:
      conn = httplib.HTTPSConnection(host, port, timeout=timeout)
    else:
      conn = httplib.HTTPConnection(host, port, timeout=timeout)
    conn.request('GET', path)
    body = conn.getresponse().read()
    return time.time() - start, body

  except (socket.gaierror, socket.error):
    print("CRIT: Could not connect to %s" % host)
    exit(CRITICAL)

  except socket.timeout:
    print("CRIT: Timed out after %s seconds" % timeout)
    exit(CRITICAL)

def parseJson(text):
  """Parses JSON, exiting with CRIT if the parse fails."""
  try:
    return json.loads(text)

  except ValueError, e:
    print('CRIT: %s' % e)
    exit(CRITICAL)

def parseJsonFile(filename):
  """Parses JSON from a file, exiting with UNKNOWN if the file does not exist."""
  try:
    with open(filename) as f:
      return parseJson(f.read())
  except IOError, e:
    print('UNKNOWN: %s' % e)
    exit(UNKNOWN)

def lookup(source, *keys, **kw):
  """Successively looks up each key, returning the default keyword arg if a dead end is reached."""
  fallback = kw.get('default')
  try:
    for key in keys:
      source = source[key]
    return source
  except KeyError:
    return fallback
  except AttributeError:
    return fallback
  except TypeError:
    return fallback

def statValue(data, *keys, **kw):
  """Returns the value of a stat."""
  return float(lookup(data, *keys, **kw))

def percent(value):
  """Formats the given float as a percentage."""
  return "%f%%" % (value * 100)

def parseArgs(scriptName, *args, **kw):
  """Parses arguments to the script."""
  argv = kw.get('argv', sys.argv)
  if len(argv) != len(args) + 1:
    print('USAGE: %s %s' % (scriptName, ' '.join([name for name, _ in args])))
    exit(UNKNOWN)

  result = {}
  idx = 0
  for name, fn in args:
    try:
      idx += 1
      result[name] = fn(argv[idx])
    except ValueError:
      print("Invalid value for %s: %r." % (name, argv[1]))
      exit(UNKNOWN)
  return result

class Rule(object):
  """A rule for when to warn or crit based on a stat value."""

  def check(self, value):
    """Checks if this rule should result in a WARN or CRIT."""
    raise NotImplementedError

class Minimum(Rule):
  """A rule that specifies minimum acceptable levels for a metric."""

  def __init__(self, warnLevel, critLevel, unit = ''):
    Rule.__init__(self)
    assert critLevel <= warnLevel
    self.warnLevel = warnLevel
    self.critLevel = critLevel
    self.unit = unit

  def check(self, value):
    """Checks if the given value is under the minimums."""
    if value < self.critLevel:
      return CRITICAL
    elif value < self.warnLevel:
      return WARNING
    else:
      return OK

  def format(self, name, value):
    """Formats as perf data."""
    return "'%s'=%d%s;%d;%d;;;" % (name, value, self.unit, self.warnLevel, self.critLevel)

  def message(self, name, value):
    """Create an error message."""
    if self.check(value) == CRITICAL:
      return '%s: %d%s < %d%s' % (name, value, self.unit, self.critLevel, self.unit)
    elif self.check(value) == WARNING:
      return '%s: %d%s < %d%s' % (name, value, self.unit, self.warnLevel, self.unit)

class Maximum(Rule):
  """A rule that specifies maximum acceptable levels for a metric."""

  def __init__(self, warnLevel, critLevel, unit = ''):
    Rule.__init__(self)
    assert critLevel >= warnLevel
    self.warnLevel = warnLevel
    self.critLevel = critLevel
    self.unit = unit

  def check(self, value):
    """Checks if the given value exceeds the maximums."""
    if value > self.critLevel:
      return CRITICAL
    elif value > self.warnLevel:
      return WARNING
    else:
      return OK

  def format(self, name, value):
    """Formats as perf data."""
    return "'%s'=%d%s;%d;%d;;;" % (name, value, self.unit, self.warnLevel, self.critLevel)

  def message(self, name, value):
    """Create an error message."""
    if self.check(value) == CRITICAL:
      return '%s: %d%s > %d%s' % (name, value, self.unit, self.critLevel, self.unit)
    elif self.check(value) == WARNING:
      return '%s: %d%s > %d%s' % (name, value, self.unit, self.warnLevel, self.unit)

class ResponseBuilder(object):
  """NRPE response builder."""

  def __init__(self):
    self._stats = []
    self._status = OK
    self._messages = []

  def addValue(self, name, value):
    """Adds a value to be tracked."""
    self._stats.append("'%s'=%s;;;;;" % (name, str(value)))
    return self

  def addStatLookup(self, name, data, *keys, **kw):
    """Adds a stat from a sequential key lookup."""
    value = lookup(data, *keys, **kw)
    return self.addValue(name, str(value) + kw.get('suffix', ''))

  def addStatChildren(self, name, data, *keys, **kw):
    """Adds a child for each child of the given dict."""
    values = lookup(data, *keys, **kw)
    if values:
      for childName, value in values.items():
        self.addValue(name % childName, str(value) + kw.get('suffix', ''))
    return self

  def addRule(self, name, rule, value):
    """Adds an alert rule and associated performance data."""
    status = rule.check(value)
    if status:
      self._status = max(self._status, rule.check(value))
      self._messages.append(rule.message(name, value))
    self._stats.append(rule.format(name, value))
    return self

  def warnIf(self, condition, message=None):
    """Warn on a given condition."""
    if condition:
      self.warn(message)
    return self

  def critIf(self, condition, message=None):
    """Mark state as critical on the given condition."""
    if condition:
      self.crit(message)
    return self

  def unknownIf(self, condition, message=None):
    """Mark state as unknown on the given condition."""
    if condition:
      self.unknown(message)
    return self

  def warn(self, message=None):
    """Mark state as warning."""
    self._status = max(self._status, WARNING)
    if message is not None:
      self.message(message)
    return self

  def crit(self, message=None):
    """Mark state as critical."""
    self._status = max(self._status, CRITICAL)
    if message is not None:
      self.message(message)
    return self

  def unknown(self, message=None):
    """Mark state as unknown."""
    self._status = max(self._status, UNKNOWN)
    if message is not None:
      self.message(message)
    return self

  def message(self, message):
    """Set the output message."""
    if message:
      self._messages.append(message)
    return self

  def build(self):
    """Builds the response."""
    return ' '.join(self._stats)

  def finish(self):
    """Builds the response, prints it, and exits."""
    output = STATUS_NAME[self._status]
    if self._messages:
      output += ': ' + (', '.join(self._messages))
    if self._stats:
      output += '|' + self.build()

    print(output)
    sys.exit(self._status)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Arne Neumann <discoursegraphs.programming@arne.cl>

import sys
import os
import re
from itertools import chain
from networkx import MultiDiGraph, write_gpickle

# The words 'das' and 'es were annotatated in the Potsdam Commentary
# Corpus (PCC). Annotation options: '/n' (nominal), '/a' (abstract),
# '/r' (relative pronoun) or '/p' (pleonastic). If the annotator was
# uncertain, the annotation is marked with a question mark.
#
# Examples: 'Das/a', 'es/p?'
ANNOTATED_ANAPHORA_REGEX = re.compile('(?P<token>([Dd]a|[Ee])s)/(?P<annotation>[anpr])(?P<uncertain>\??)')

ANNOTATION_TYPES = {'n': 'nominal',
                    'a': 'abstract',
                    'r': 'relative',
                    'p': 'pleonastic'}

class AnaphoraDocumentGraph(MultiDiGraph):
    """
    represents a text in which abstract anaphora were annotated
    as a graph.

    Attributes
    ----------
    tokens : list of int
        a list of node IDs (int) which represent the tokens in the
        order they occur in the text
    root_node_name : str
        name of the document root node ID
        (default: 'anaphoricity:root_node')
    """
    def __init__(self, anaphora_filepath, name=None):
        """
        Reads an abstract anaphora annotation file, creates a directed
        graph and adds a node for each token, as well as an edge from
        the root node to each token.
        If a token is annotated, it will have an attribute 'annotation',
        which maps to a dict with the keys 'anaphoricity' (str) and
        'certainty' (float).

        'anaphoricity' is one of the following: 'abstract', 'nominal',
        'pleonastic' or 'relative'.

        Parameters
        ----------
        anaphora_filepath : str
            relative or absolute path to an anaphora annotation file.
            The format of the file was created ad-hoc by one of our
            students for his diploma thesis. It consists of tokenized
            plain text (one sentence per line with spaces between
            tokens).
            A token is annotated by appending '/' and one of the letters
            'a' (abstract), 'n' (nominal), 'p' (pleonastic),
            'r' (relative pronoun) and optionally a question mark to
            signal uncertainty.
        name : str or None
            the name or ID of the graph to be generated. If no name is
            given, the basename of the input file is used.
        """
        # super calls __init__() of base class MultiDiGraph
        super(AnaphoraDocumentGraph, self).__init__()
        self.name = os.path.basename(anaphora_filepath)
        self.root_node_name = 'anaphoricity:root_node'
        self.add_node(self.root_node_name, layers={'anaphoricity'})
        self.tokens = []

        with open(anaphora_filepath, 'r') as anno_file:
            annotated_lines = anno_file.readlines()
            tokens = list(chain.from_iterable(line.split()
                                        for line in annotated_lines))
            for i, token in enumerate(tokens):
                self.__add_token_to_document(token, i)
                self.tokens.append(i)

    def __add_token_to_document(self, token, token_id):
        """
        adds a token to the document graph as a node with the given ID.

        Parameters
        ----------
        token : str
            the token to be added to the document graph
        token_id : int
            the node ID of the token to be added, which must not yet
            exist in the document graph
        """
        regex_match = ANNOTATED_ANAPHORA_REGEX.search(token)
        if regex_match: # token is annotated
            unannotated_token = regex_match.group('token')
            annotation = regex_match.group('annotation')
            certainty = 1.0 if not regex_match.group('uncertain') else 0.5
            self.add_node(token_id, token=unannotated_token,
                annotation={'anaphoricity': ANNOTATION_TYPES[annotation],
                    'certainty': certainty},
                    layers={'anaphoricity', 'anaphoricity:token'})
        else: # token is not annotated
            self.add_node(token_id, token=token,
                layers={'anaphoricity', 'anaphoricity:token'})
        self.add_edge(self.root_node_name, token_id,
            layers={'anaphoricity', 'anaphoricity:token'})

if __name__ == '__main__':
    if len(sys.argv) != 3:
        sys.stderr.write('Usage: {0} anaphoricity_input_file networkx_pickle_output_file\n'.format(sys.argv[0]))
        sys.exit(1)
    else:
        anaphora_filepath = sys.argv[1]
        pickle_filepath = sys.argv[2]
        assert os.path.isfile(anaphora_filepath)
        anaphora_docgraph = AnaphoraDocumentGraph(anaphora_filepath)
        write_gpickle(anaphora_docgraph, pickle_filepath)

<commit_msg>added layer string to anaphoricity annotation<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Arne Neumann <discoursegraphs.programming@arne.cl>

import sys
import os
import re
from itertools import chain
from networkx import MultiDiGraph, write_gpickle

# The words 'das' and 'es were annotatated in the Potsdam Commentary
# Corpus (PCC). Annotation options: '/n' (nominal), '/a' (abstract),
# '/r' (relative pronoun) or '/p' (pleonastic). If the annotator was
# uncertain, the annotation is marked with a question mark.
#
# Examples: 'Das/a', 'es/p?'
ANNOTATED_ANAPHORA_REGEX = re.compile('(?P<token>([Dd]a|[Ee])s)/(?P<annotation>[anpr])(?P<uncertain>\??)')

ANNOTATION_TYPES = {'n': 'nominal',
                    'a': 'abstract',
                    'r': 'relative',
                    'p': 'pleonastic'}

class AnaphoraDocumentGraph(MultiDiGraph):
    """
    represents a text in which abstract anaphora were annotated
    as a graph.

    Attributes
    ----------
    tokens : list of int
        a list of node IDs (int) which represent the tokens in the
        order they occur in the text
    root_node_name : str
        name of the document root node ID
        (default: 'anaphoricity:root_node')
    """
    def __init__(self, anaphora_filepath, name=None):
        """
        Reads an abstract anaphora annotation file, creates a directed
        graph and adds a node for each token, as well as an edge from
        the root node to each token.
        If a token is annotated, it will have an attribute 'annotation',
        which maps to a dict with the keys 'anaphoricity' (str) and
        'certainty' (float).

        'anaphoricity' is one of the following: 'abstract', 'nominal',
        'pleonastic' or 'relative'.

        Parameters
        ----------
        anaphora_filepath : str
            relative or absolute path to an anaphora annotation file.
            The format of the file was created ad-hoc by one of our
            students for his diploma thesis. It consists of tokenized
            plain text (one sentence per line with spaces between
            tokens).
            A token is annotated by appending '/' and one of the letters
            'a' (abstract), 'n' (nominal), 'p' (pleonastic),
            'r' (relative pronoun) and optionally a question mark to
            signal uncertainty.
        name : str or None
            the name or ID of the graph to be generated. If no name is
            given, the basename of the input file is used.
        """
        # super calls __init__() of base class MultiDiGraph
        super(AnaphoraDocumentGraph, self).__init__()
        self.name = os.path.basename(anaphora_filepath)
        self.root_node_name = 'anaphoricity:root_node'
        self.add_node(self.root_node_name, layers={'anaphoricity'})
        self.tokens = []

        with open(anaphora_filepath, 'r') as anno_file:
            annotated_lines = anno_file.readlines()
            tokens = list(chain.from_iterable(line.split()
                                        for line in annotated_lines))
            for i, token in enumerate(tokens):
                self.__add_token_to_document(token, i)
                self.tokens.append(i)

    def __add_token_to_document(self, token, token_id):
        """
        adds a token to the document graph as a node with the given ID.

        Parameters
        ----------
        token : str
            the token to be added to the document graph
        token_id : int
            the node ID of the token to be added, which must not yet
            exist in the document graph
        """
        regex_match = ANNOTATED_ANAPHORA_REGEX.search(token)
        if regex_match: # token is annotated
            unannotated_token = regex_match.group('token')
            annotation = regex_match.group('annotation')
            certainty = 1.0 if not regex_match.group('uncertain') else 0.5
            self.add_node(token_id, attr_dict={
                'anaphoricity:annotation': ANNOTATION_TYPES[annotation],
                'anaphoricity:certainty': certainty},
                token=unannotated_token,
                layers={'anaphoricity', 'anaphoricity:token'})
        else: # token is not annotated
            self.add_node(token_id, token=token,
                layers={'anaphoricity', 'anaphoricity:token'})
        self.add_edge(self.root_node_name, token_id,
            layers={'anaphoricity', 'anaphoricity:token'})

if __name__ == '__main__':
    if len(sys.argv) != 3:
        sys.stderr.write('Usage: {0} anaphoricity_input_file networkx_pickle_output_file\n'.format(sys.argv[0]))
        sys.exit(1)
    else:
        anaphora_filepath = sys.argv[1]
        pickle_filepath = sys.argv[2]
        assert os.path.isfile(anaphora_filepath)
        anaphora_docgraph = AnaphoraDocumentGraph(anaphora_filepath)
        write_gpickle(anaphora_docgraph, pickle_filepath)

<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import platform
import re
import shutil
import stat
import subprocess
import sys
import time
from HTMLParser import HTMLParser

from optparse import OptionParser
from subprocesses import captureStdout, normExpUserPath, shellify, vdump

devnull = open(os.devnull, "w")

# Use curl/wget rather than urllib because urllib can't check certs.
useCurl = False

def readFromURL(url):
    '''
    Reads in a URL and returns its contents as a list.
    '''
    inpCmdList = ['curl', '--silent', url] if useCurl else ['wget', '-q', '-O', '-', url]
    out, retVal = captureStdout(inpCmdList, combineStderr=True, ignoreExitCode=True)
    if not useCurl and retVal == 5:
        print 'Unable to read from URL. If you installed wget using MacPorts, you should put ' + \
              '"CA_CERTIFICATE=/opt/local/share/curl/curl-ca-bundle.crt" (without the quotes) ' + \
              'in ~/.wgetrc'
        raise Exception('Unable to read from URL. Please check your ~/.wgetrc file.')
    elif retVal != 0:
        print 'inpCmdList is: ' + shellify(inpCmdList)
        raise Exception('The following exit code was returned: ' + str(retVal))
    else:
        return out

def downloadURL(url, dest):
    '''
    Reads in a URL and downloads it to a destination.
    '''
    inpCmdList = ['curl', '--output', dest, url] if useCurl else ['wget', '-O', dest, url]
    out, retVal = captureStdout(inpCmdList, combineStderr=True, ignoreExitCode=True)
    if retVal != 0:
        print out
        raise Exception('Return code is not 0, but is: ' + str(retVal))
    return dest

def parseOptions():
    usage = 'Usage: %prog [options]'
    parser = OptionParser(usage)
    parser.disable_interspersed_args()

    parser.set_defaults(
        compileType = 'dbg',
        downloadFolder = os.getcwdu(),
        repoName = 'mozilla-central',
        enableJsShell = False,
    )

    parser.add_option('-c', '--compiletype', dest='compileType',
                      help='Sets the compile type to be downloaded. Must be "dbg" or "opt".' + \
                           'Defaults to "%default".')
    parser.add_option('-a', '--architecture',
                      dest='arch',
                      type='choice',
                      choices=['32', '64'],
                      help='Test architecture. Only accepts "32" or "64"')
    parser.add_option('-w', '--downloadfolder', dest='downloadFolder',
                      help='Sets the folder to download builds in. Defaults to the current ' + \
                           'working directory, which is "%default".')
    parser.add_option('-r', '--repoName', dest='repoName',
                      help='Sets the repository to be fuzzed. Defaults to "%default".')
    parser.add_option('-d', '--remotedir', dest='remoteDir',
                      help='Sets the remote directory from which the files are to be obtained ' + \
                           'from. The default location is at "%default".')
    parser.add_option('-s', '--enable-jsshell', dest='enableJsShell', action='store_true',
                      help='Sets the compile type to be fuzzed. Defaults to "%default".')

    options, args = parser.parse_args()
    assert options.compileType in ['dbg', 'opt']
    return options

class MyHTMLParser(HTMLParser):
    def getHrefLinks(self, html):
        '''
        Returns href links as a list.
        '''
        self.hrefLinksList = []
        self.feed(html)
        return self.hrefLinksList
    def handle_starttag(self, tag, attrs):
        aTagFound = False
        if tag == 'a':
            aTagFound = True
        for attr in attrs:
            if not aTagFound:
                break
            if aTagFound and attr[0] == 'href' and attr[1][:1] not in ('?', '/'):
                self.hrefLinksList.append(attr[1])

def httpDirList(directory):
    '''
    Reads in an input directory and returns the href links in the directory as a list.
    '''
    print "Looking in " + directory + "..."
    page = readFromURL(directory)
    vdump('Finished reading from: ' + directory)

    parser = MyHTMLParser()
    fileList = parser.getHrefLinks(page)
    return fileList

def unzip(fn, dest):
    '''
    Extracts .zip files to their destination.
    '''
    captureStdout(['unzip', fn, '-d', dest])

def untarbz2(fn, dest):
    '''
    Extracts .tar.bz2 files to their destination.
    '''
    if not os.path.exists(dest):
        os.mkdir(dest)
    captureStdout(['tar', '-C', dest, '-xjf', os.path.abspath(fn)])

def undmg(fn, dest, mountpoint):
    '''
    Extracts .dmg files to their destination via a mount point.
    '''
    if os.path.exists(mountpoint):
        # If the mount point already exists, detach it first.
        captureStdout(['hdiutil', 'detach', mountpoint, '-force'])
    captureStdout(['hdiutil', 'attach', '-quiet', '-mountpoint', mountpoint, fn])
    try:
        apps = filter(lambda s: s.endswith('.app'), os.listdir(mountpoint))
        assert len(apps) == 1
        shutil.copytree(mountpoint + '/' + apps[0], dest + '/' + apps[0])
    finally:
        captureStdout(['hdiutil', 'detach', mountpoint])

def downloadBuild(httpDir, targetDir, jsShell=False, wantSymbols=True, wantTests=True):
    '''
    Downloads the build specified, along with symbols and tests. Returns True when all are obtained.
    '''
    wantSymbols = wantSymbols and not jsShell  # Bug 715365, js shell currently lacks native symbols
    wantTests = wantTests and not jsShell
    gotApp = False
    gotTests = False
    gotTxtFile = False
    gotSyms = False
    # Create build folder and a download subfolder.
    buildDir = os.path.abspath(normExpUserPath(os.path.join(targetDir, 'build')))
    if os.path.exists(buildDir):
        shutil.rmtree(buildDir)
    os.mkdir(buildDir)
    downloadFolder = os.path.join(buildDir, 'download')
    os.mkdir(downloadFolder)

    # Hack #1 for making os.path.join(reftestScriptDir, automation.DEFAULT_APP) work is to:
    # Call this directory "dist".
    appDir = os.path.join(buildDir, 'dist') + os.sep
    testsDir = os.path.join(buildDir, 'tests') + os.sep
    symbolsDir = os.path.join(buildDir, 'symbols') + os.sep
    fileHttpRawList = httpDirList(httpDir)
    # We only want files, those with file extensions, not folders.
    fileHttpList = [httpDir + x for x in fileHttpRawList if '.' in x and 'mozilla.org' not in x]

    for remotefn in fileHttpList:
        localfn = os.path.join(downloadFolder, remotefn.split('/')[-1])
        if remotefn.endswith('.tests.zip') and wantTests:
            print 'Downloading tests...',
            dlAction = downloadURL(remotefn, localfn)
            print 'extracting...',
            unzip(dlAction, testsDir)
            print 'completed!'
            gotTests = True
        if remotefn.split('/')[-1].endswith('.txt'):
            print 'Downloading text file...',
            downloadURL(remotefn, localfn)
            print 'completed!'
            gotTxtFile = True
        if jsShell:
            if remotefn.split('/')[-1].startswith('jsshell-'):
                print 'Downloading js shell...',
                dlAction = downloadURL(remotefn, localfn)
                print 'extracting...',
                unzip(dlAction, appDir)
                print 'completed!'
                gotApp = True  # Bug 715365 - note that js shell currently lacks native symbols
        else:
            if remotefn.endswith('.linux-i686.tar.bz2') or \
                remotefn.endswith('.linux-x86_64.tar.bz2'):
                print 'Downloading application...',
                dlAction = downloadURL(remotefn, localfn)
                print 'extracting...',
                untarbz2(dlAction, appDir)
                print 'completed!'

                # Hack #2 to make os.path.join(reftestScriptDir, automation.DEFAULT_APP) work.
                os.rename(os.path.join(appDir, 'firefox'), os.path.join(appDir, 'bin'))
                stackwalk = os.path.join(buildDir, 'minidump_stackwalk')
                stackwalkUrl = \
            'http://hg.mozilla.org/build/tools/raw-file/default/breakpad/linux/minidump_stackwalk' \
            if remotefn.endswith('.linux-i686.tar.bz2') else \
            'http://hg.mozilla.org/build/tools/raw-file/default/breakpad/linux64/minidump_stackwalk'
                downloadURL(stackwalkUrl, stackwalk)
                os.chmod(stackwalk, stat.S_IRWXU)
                gotApp = True
            if remotefn.endswith('.win32.zip'):
                print 'Downloading application...',
                dlAction = downloadURL(remotefn, localfn)
                print 'extracting...',
                unzip(dlAction, appDir)
                print 'completed!'

                # Hack #2 for making os.path.join(reftestScriptDir, automation.DEFAULT_APP) work.
                os.rename(os.path.join(appDir, 'firefox'), os.path.join(appDir, 'bin'))
                for filename in ['minidump_stackwalk.exe', 'cyggcc_s-1.dll',
                                 'cygstdc++-6.dll', 'cygwin1.dll']:
                    remoteURL = \
            'http://hg.mozilla.org/build/tools/raw-file/default/breakpad/win32/%s' % filename
                    localfile = os.path.join(buildDir, filename)
                    downloadURL(remoteURL, localfile)
                gotApp = True
            if remotefn.endswith('.mac.dmg') or remotefn.endswith('.mac64.dmg'):
                print 'Downloading application...',
                dlAction = downloadURL(remotefn, localfn)
                print 'extracting...',
                undmg(dlAction, appDir, os.path.join(buildDir, 'MOUNTEDDMG'))
                print 'completed!'

                stackwalk = os.path.join(buildDir, 'minidump_stackwalk')
                stackwalkUrl = \
            'http://hg.mozilla.org/build/tools/raw-file/default/breakpad/osx/minidump_stackwalk'
                downloadURL(stackwalkUrl, stackwalk)
                os.chmod(stackwalk, stat.S_IRWXU)
                gotApp = True
            if remotefn.endswith('.crashreporter-symbols.zip') and wantSymbols:
                print 'Downloading crash reporter symbols...',
                dlAction = downloadURL(remotefn, localfn)
                print 'extracting...',
                unzip(dlAction, symbolsDir)
                print 'completed!'
                gotSyms = True
    return gotApp and gotTxtFile and (gotTests or not wantTests) and (gotSyms or not wantSymbols)

def isNumericSubDir(n):
    '''
    Returns True if input is a numeric directory, False if not. e.g. 1234/ returns True
    '''
    return re.match(r'^\d+$', n.split('/')[0])

def downloadLatestBuild(buildType, workingDir, getJsShell=False):
    '''
    Downloads the latest build based on machine type, e.g. mozilla-central-macosx-debug.
    '''
    buildsHttpDir = 'https://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/' + \
                    buildType + '/'
    buildsRawList = httpDirList(buildsHttpDir)
    buildsSubDirList = filter(isNumericSubDir, buildsRawList)
    builds = [c[:-1] for c in buildsSubDirList]
    for b in reversed(builds):  # Try downloading the latest build first.
        fullb = buildsHttpDir + b + '/'
        if downloadBuild(fullb, workingDir, jsShell=getJsShell):
            return fullb
    raise Exception('No builds in ' + buildsHttpDir + '!')

def mozPlatformDetails():
    '''
    Determines the platform of the system and returns the RelEng-specific build type.
    '''
    s = platform.system()
    if s == "Darwin":
        return ("macosx", "macosx64", platform.architecture()[0] == "64bit")
    elif s == "Linux":
        return ("linux", "linux64", platform.machine() == "x86_64")
    elif s in ("Microsoft", "Windows"):
        return ("win32", "win64", False)
    else:
        raise Exception("Unknown platform.system(): " + s)

def mozPlatform(arch):
    '''
    Returns the native build type of the current machine.
    '''
    (name32, name64, native64) = mozPlatformDetails()
    if arch == "64":
        return name64
    elif arch == "32":
        return name32
    elif arch == None:
        # FIXME: Eventually, we should set 64-bit as native for Win64. We should also aim to test
        # both 32-bit and 64-bit Firefox builds on any platform that supports both. Let us make
        # sure Python detects 32-bit Windows vs 64-bit Windows correctly before changing this.
        return (name64 if native64 else name32)
    else:
        raise Exception("The arch passed to mozPlatform must be '64', '32', or None")

def defaultBuildType(options):
    '''
    For Firefox, returns the default build type as per RelEng, e.g. mozilla-central-macosx-debug
    For the js shell, returns FIXME.
    '''
    return options.repoName + '-' + mozPlatform(options.arch) + '-debug' \
        if options.compileType == 'dbg' else ''

def main():
    options = parseOptions()
    if options.remoteDir is not None:
        print downloadBuild(options.remoteDir, options.downloadFolder, jsShell=options.enableJsShell)
    else:
        downloadLatestBuild(defaultBuildType(options), options.downloadFolder,
                            getJsShell=options.enableJsShell)

if __name__ == "__main__":
    main()
<commit_msg>Fix paren-o<commit_after>#!/usr/bin/env python

import os
import platform
import re
import shutil
import stat
import subprocess
import sys
import time
from HTMLParser import HTMLParser

from optparse import OptionParser
from subprocesses import captureStdout, normExpUserPath, shellify, vdump

devnull = open(os.devnull, "w")

# Use curl/wget rather than urllib because urllib can't check certs.
useCurl = False

def readFromURL(url):
    '''
    Reads in a URL and returns its contents as a list.
    '''
    inpCmdList = ['curl', '--silent', url] if useCurl else ['wget', '-q', '-O', '-', url]
    out, retVal = captureStdout(inpCmdList, combineStderr=True, ignoreExitCode=True)
    if not useCurl and retVal == 5:
        print 'Unable to read from URL. If you installed wget using MacPorts, you should put ' + \
              '"CA_CERTIFICATE=/opt/local/share/curl/curl-ca-bundle.crt" (without the quotes) ' + \
              'in ~/.wgetrc'
        raise Exception('Unable to read from URL. Please check your ~/.wgetrc file.')
    elif retVal != 0:
        print 'inpCmdList is: ' + shellify(inpCmdList)
        raise Exception('The following exit code was returned: ' + str(retVal))
    else:
        return out

def downloadURL(url, dest):
    '''
    Reads in a URL and downloads it to a destination.
    '''
    inpCmdList = ['curl', '--output', dest, url] if useCurl else ['wget', '-O', dest, url]
    out, retVal = captureStdout(inpCmdList, combineStderr=True, ignoreExitCode=True)
    if retVal != 0:
        print out
        raise Exception('Return code is not 0, but is: ' + str(retVal))
    return dest

def parseOptions():
    usage = 'Usage: %prog [options]'
    parser = OptionParser(usage)
    parser.disable_interspersed_args()

    parser.set_defaults(
        compileType = 'dbg',
        downloadFolder = os.getcwdu(),
        repoName = 'mozilla-central',
        enableJsShell = False,
    )

    parser.add_option('-c', '--compiletype', dest='compileType',
                      help='Sets the compile type to be downloaded. Must be "dbg" or "opt".' + \
                           'Defaults to "%default".')
    parser.add_option('-a', '--architecture',
                      dest='arch',
                      type='choice',
                      choices=['32', '64'],
                      help='Test architecture. Only accepts "32" or "64"')
    parser.add_option('-w', '--downloadfolder', dest='downloadFolder',
                      help='Sets the folder to download builds in. Defaults to the current ' + \
                           'working directory, which is "%default".')
    parser.add_option('-r', '--repoName', dest='repoName',
                      help='Sets the repository to be fuzzed. Defaults to "%default".')
    parser.add_option('-d', '--remotedir', dest='remoteDir',
                      help='Sets the remote directory from which the files are to be obtained ' + \
                           'from. The default location is at "%default".')
    parser.add_option('-s', '--enable-jsshell', dest='enableJsShell', action='store_true',
                      help='Sets the compile type to be fuzzed. Defaults to "%default".')

    options, args = parser.parse_args()
    assert options.compileType in ['dbg', 'opt']
    return options

class MyHTMLParser(HTMLParser):
    def getHrefLinks(self, html):
        '''
        Returns href links as a list.
        '''
        self.hrefLinksList = []
        self.feed(html)
        return self.hrefLinksList
    def handle_starttag(self, tag, attrs):
        aTagFound = False
        if tag == 'a':
            aTagFound = True
        for attr in attrs:
            if not aTagFound:
                break
            if aTagFound and attr[0] == 'href' and attr[1][:1] not in ('?', '/'):
                self.hrefLinksList.append(attr[1])

def httpDirList(directory):
    '''
    Reads in an input directory and returns the href links in the directory as a list.
    '''
    print "Looking in " + directory + "..."
    page = readFromURL(directory)
    vdump('Finished reading from: ' + directory)

    parser = MyHTMLParser()
    fileList = parser.getHrefLinks(page)
    return fileList

def unzip(fn, dest):
    '''
    Extracts .zip files to their destination.
    '''
    captureStdout(['unzip', fn, '-d', dest])

def untarbz2(fn, dest):
    '''
    Extracts .tar.bz2 files to their destination.
    '''
    if not os.path.exists(dest):
        os.mkdir(dest)
    captureStdout(['tar', '-C', dest, '-xjf', os.path.abspath(fn)])

def undmg(fn, dest, mountpoint):
    '''
    Extracts .dmg files to their destination via a mount point.
    '''
    if os.path.exists(mountpoint):
        # If the mount point already exists, detach it first.
        captureStdout(['hdiutil', 'detach', mountpoint, '-force'])
    captureStdout(['hdiutil', 'attach', '-quiet', '-mountpoint', mountpoint, fn])
    try:
        apps = filter(lambda s: s.endswith('.app'), os.listdir(mountpoint))
        assert len(apps) == 1
        shutil.copytree(mountpoint + '/' + apps[0], dest + '/' + apps[0])
    finally:
        captureStdout(['hdiutil', 'detach', mountpoint])

def downloadBuild(httpDir, targetDir, jsShell=False, wantSymbols=True, wantTests=True):
    '''
    Downloads the build specified, along with symbols and tests. Returns True when all are obtained.
    '''
    wantSymbols = wantSymbols and not jsShell  # Bug 715365, js shell currently lacks native symbols
    wantTests = wantTests and not jsShell
    gotApp = False
    gotTests = False
    gotTxtFile = False
    gotSyms = False
    # Create build folder and a download subfolder.
    buildDir = os.path.abspath(normExpUserPath(os.path.join(targetDir, 'build')))
    if os.path.exists(buildDir):
        shutil.rmtree(buildDir)
    os.mkdir(buildDir)
    downloadFolder = os.path.join(buildDir, 'download')
    os.mkdir(downloadFolder)

    # Hack #1 for making os.path.join(reftestScriptDir, automation.DEFAULT_APP) work is to:
    # Call this directory "dist".
    appDir = os.path.join(buildDir, 'dist') + os.sep
    testsDir = os.path.join(buildDir, 'tests') + os.sep
    symbolsDir = os.path.join(buildDir, 'symbols') + os.sep
    fileHttpRawList = httpDirList(httpDir)
    # We only want files, those with file extensions, not folders.
    fileHttpList = [httpDir + x for x in fileHttpRawList if '.' in x and 'mozilla.org' not in x]

    for remotefn in fileHttpList:
        localfn = os.path.join(downloadFolder, remotefn.split('/')[-1])
        if remotefn.endswith('.tests.zip') and wantTests:
            print 'Downloading tests...',
            dlAction = downloadURL(remotefn, localfn)
            print 'extracting...',
            unzip(dlAction, testsDir)
            print 'completed!'
            gotTests = True
        if remotefn.split('/')[-1].endswith('.txt'):
            print 'Downloading text file...',
            downloadURL(remotefn, localfn)
            print 'completed!'
            gotTxtFile = True
        if jsShell:
            if remotefn.split('/')[-1].startswith('jsshell-'):
                print 'Downloading js shell...',
                dlAction = downloadURL(remotefn, localfn)
                print 'extracting...',
                unzip(dlAction, appDir)
                print 'completed!'
                gotApp = True  # Bug 715365 - note that js shell currently lacks native symbols
        else:
            if remotefn.endswith('.linux-i686.tar.bz2') or \
                remotefn.endswith('.linux-x86_64.tar.bz2'):
                print 'Downloading application...',
                dlAction = downloadURL(remotefn, localfn)
                print 'extracting...',
                untarbz2(dlAction, appDir)
                print 'completed!'

                # Hack #2 to make os.path.join(reftestScriptDir, automation.DEFAULT_APP) work.
                os.rename(os.path.join(appDir, 'firefox'), os.path.join(appDir, 'bin'))
                stackwalk = os.path.join(buildDir, 'minidump_stackwalk')
                stackwalkUrl = \
            'http://hg.mozilla.org/build/tools/raw-file/default/breakpad/linux/minidump_stackwalk' \
            if remotefn.endswith('.linux-i686.tar.bz2') else \
            'http://hg.mozilla.org/build/tools/raw-file/default/breakpad/linux64/minidump_stackwalk'
                downloadURL(stackwalkUrl, stackwalk)
                os.chmod(stackwalk, stat.S_IRWXU)
                gotApp = True
            if remotefn.endswith('.win32.zip'):
                print 'Downloading application...',
                dlAction = downloadURL(remotefn, localfn)
                print 'extracting...',
                unzip(dlAction, appDir)
                print 'completed!'

                # Hack #2 for making os.path.join(reftestScriptDir, automation.DEFAULT_APP) work.
                os.rename(os.path.join(appDir, 'firefox'), os.path.join(appDir, 'bin'))
                for filename in ['minidump_stackwalk.exe', 'cyggcc_s-1.dll',
                                 'cygstdc++-6.dll', 'cygwin1.dll']:
                    remoteURL = \
            'http://hg.mozilla.org/build/tools/raw-file/default/breakpad/win32/%s' % filename
                    localfile = os.path.join(buildDir, filename)
                    downloadURL(remoteURL, localfile)
                gotApp = True
            if remotefn.endswith('.mac.dmg') or remotefn.endswith('.mac64.dmg'):
                print 'Downloading application...',
                dlAction = downloadURL(remotefn, localfn)
                print 'extracting...',
                undmg(dlAction, appDir, os.path.join(buildDir, 'MOUNTEDDMG'))
                print 'completed!'

                stackwalk = os.path.join(buildDir, 'minidump_stackwalk')
                stackwalkUrl = \
            'http://hg.mozilla.org/build/tools/raw-file/default/breakpad/osx/minidump_stackwalk'
                downloadURL(stackwalkUrl, stackwalk)
                os.chmod(stackwalk, stat.S_IRWXU)
                gotApp = True
            if remotefn.endswith('.crashreporter-symbols.zip') and wantSymbols:
                print 'Downloading crash reporter symbols...',
                dlAction = downloadURL(remotefn, localfn)
                print 'extracting...',
                unzip(dlAction, symbolsDir)
                print 'completed!'
                gotSyms = True
    return gotApp and gotTxtFile and (gotTests or not wantTests) and (gotSyms or not wantSymbols)

def isNumericSubDir(n):
    '''
    Returns True if input is a numeric directory, False if not. e.g. 1234/ returns True
    '''
    return re.match(r'^\d+$', n.split('/')[0])

def downloadLatestBuild(buildType, workingDir, getJsShell=False):
    '''
    Downloads the latest build based on machine type, e.g. mozilla-central-macosx-debug.
    '''
    buildsHttpDir = 'https://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/' + \
                    buildType + '/'
    buildsRawList = httpDirList(buildsHttpDir)
    buildsSubDirList = filter(isNumericSubDir, buildsRawList)
    builds = [c[:-1] for c in buildsSubDirList]
    for b in reversed(builds):  # Try downloading the latest build first.
        fullb = buildsHttpDir + b + '/'
        if downloadBuild(fullb, workingDir, jsShell=getJsShell):
            return fullb
    raise Exception('No builds in ' + buildsHttpDir + '!')

def mozPlatformDetails():
    '''
    Determines the platform of the system and returns the RelEng-specific build type.
    '''
    s = platform.system()
    if s == "Darwin":
        return ("macosx", "macosx64", platform.architecture()[0] == "64bit")
    elif s == "Linux":
        return ("linux", "linux64", platform.machine() == "x86_64")
    elif s in ("Microsoft", "Windows"):
        return ("win32", "win64", False)
    else:
        raise Exception("Unknown platform.system(): " + s)

def mozPlatform(arch):
    '''
    Returns the native build type of the current machine.
    '''
    (name32, name64, native64) = mozPlatformDetails()
    if arch == "64":
        return name64
    elif arch == "32":
        return name32
    elif arch == None:
        # FIXME: Eventually, we should set 64-bit as native for Win64. We should also aim to test
        # both 32-bit and 64-bit Firefox builds on any platform that supports both. Let us make
        # sure Python detects 32-bit Windows vs 64-bit Windows correctly before changing this.
        return (name64 if native64 else name32)
    else:
        raise Exception("The arch passed to mozPlatform must be '64', '32', or None")

def defaultBuildType(options):
    '''
    For Firefox, returns the default build type as per RelEng, e.g. mozilla-central-macosx-debug
    For the js shell, returns FIXME.
    '''
    return options.repoName + '-' + mozPlatform(options.arch) + ('-debug' if options.compileType == 'dbg' else '')

def main():
    options = parseOptions()
    if options.remoteDir is not None:
        print downloadBuild(options.remoteDir, options.downloadFolder, jsShell=options.enableJsShell)
    else:
        downloadLatestBuild(defaultBuildType(options), options.downloadFolder,
                            getJsShell=options.enableJsShell)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import math

class Settings():
    """Robot mapping. Values that are changed often go here."""

    #Values to be changed through drive station
    num_precision_one = 0.60
    num_precision_two = 0.50
    num_scaling = 1.25
    num_macro_timeout = 15
    str_macro_name = "macro.csv"

    #Tilt pot setpoints .158
#    kMaxDown = 0.70 #derp, was .790
 #   kMaxUp = kMaxDown - 0.580
  #  kTop = kMaxUp + 0.050
#    kTopShot = kMaxDown - 0.498
 #   kTopShotAtBase = kMaxDown - 0.509
  #  kBottom = kMaxDown - 0.050
 #   kShootLevel = kMaxDown - 0.144
 #   kShootAtBase = kMaxDown - 0.316
    #2nd set of wheels is touching batter
  #  kShootRamp = kMaxDown - 0.464
    #Backed into tower
   # kLongShot = kMaxDown - 0.158
    #14 feet from tower

    #Encoder shenanigans
    kMaxDown = 3600
    kMaxUp = 1180
    kTop = 2000
    kTopShot = 2300
    kTopShotAtBase = 2500
    kBottom = 2900
    kShootLevel = 2700
    kShootAtBase = 2600
    kShootRamp = 2500
    kLongShot = 2400
<commit_msg>Numbers dont work like that lol<commit_after>import math

class Settings():
    """Robot mapping. Values that are changed often go here."""

    #Values to be changed through drive station
    num_precision_one = 0.60
    num_precision_two = 0.50
    num_scaling = 1.25
    num_macro_timeout = 15
    str_macro_name = "macro.csv"

    #Tilt pot setpoints .158
#    kMaxDown = 0.70 #derp, was .790
 #   kMaxUp = kMaxDown - 0.580
  #  kTop = kMaxUp + 0.050
#    kTopShot = kMaxDown - 0.498
 #   kTopShotAtBase = kMaxDown - 0.509
  #  kBottom = kMaxDown - 0.050
 #   kShootLevel = kMaxDown - 0.144
 #   kShootAtBase = kMaxDown - 0.316
    #2nd set of wheels is touching batter
  #  kShootRamp = kMaxDown - 0.464
    #Backed into tower
   # kLongShot = kMaxDown - 0.158
    #14 feet from tower

    #Encoder shenanigans
    kMaxDown = 3600
    kMaxUp = 1300
    kTop = 2000
    kTopShot = 2300
    kTopShotAtBase = 2500
    kBottom = 2900
    kShootLevel = 2700
    kShootAtBase = 2600
    kShootRamp = 2500
    kLongShot = 2400
<|endoftext|>