<commit_before>from datetime import timedelta
from urlparse import parse_qs
import re

from django.http import QueryDict, HttpResponseRedirect
from django.core.exceptions import ImproperlyConfigured

from fandjango.views import authorize_application
from fandjango.models import Facebook, User, OAuthToken
from fandjango.settings import (
    FACEBOOK_APPLICATION_SECRET_KEY, FACEBOOK_APPLICATION_ID,
    FANDJANGO_CACHE_SIGNED_REQUEST, DISABLED_PATHS, ENABLED_PATHS
)
from fandjango.utils import (
    is_disabled_path, is_enabled_path,
    authorization_denied_view, get_post_authorization_redirect_url
)

from facepy import SignedRequest, GraphAPI

try:
    from django.utils.timezone import now
except ImportError:
    from datetime import datetime
    def now():
        return datetime.now()

from dateutil.tz import tzlocal

class BaseMiddleware():

    def is_valid_path(self, request):
        if ENABLED_PATHS and DISABLED_PATHS:
            raise ImproperlyConfigured(
                'You may configure either FANDJANGO_ENABLED_PATHS '
                'or FANDJANGO_DISABLED_PATHS, but not both.'
            )

        if DISABLED_PATHS and is_disabled_path(request.path):
            return

        if ENABLED_PATHS and not is_enabled_path(request.path):
            return

        return True

class FacebookMiddleware(BaseMiddleware):
    """Middleware for Facebook auth on websites."""

    def process_request(self, request):
        """Process the signed request."""

        if not self.is_valid_path(request):
            return

        if hasattr(request, "facebook") and request.facebook:
            return

        # An error occured during authorization...
        if 'error' in request.GET:
            # The user refused to authorize the application...
            if request.GET['error'] == 'access_denied':
                return authorization_denied_view(request)

        request.facebook = Facebook()
        oauth_token = False

        # Is there a token in the current session already present?
        if 'oauth_token' in request.session:
            try:
                # Check if the current token is already in DB
                oauth_token = OAuthToken.objects.get(token=request.session['oauth_token'])
            except OAuthToken.DoesNotExist:
                # It's in the session but it's not in the DB, weird, better delete it and get new one..
                del request.session['oauth_token']
                request.facebook = False
                return

        # Is there a code in the GET request?
        elif 'code' in request.GET:
            try:
                graph = GraphAPI()

                # Exchange code for an access_token
                response = graph.get('oauth/access_token',
                    client_id = FACEBOOK_APPLICATION_ID,
                    redirect_uri = get_post_authorization_redirect_url(request, canvas=False),
                    client_secret = FACEBOOK_APPLICATION_SECRET_KEY,
                    code = request.GET['code'],
                )

                components = parse_qs(response)

                # Save new OAuth-token in DB
                oauth_token, created = OAuthToken.objects.get_or_create(
                    token = components['access_token'][0],
                    issued_at = now(),
                    expires_at = now() + timedelta(seconds = int(components['expires'][0]))
                )
            except GraphAPI.OAuthError:
                pass

        # There is a valid access_token
        if oauth_token:
            # Redirect to Facebook authorization if the OAuth token has expired
            if oauth_token.expired:
                request.facebook = False
                return

            # Is there a user already connected to the current token?
            try:
                user = oauth_token.user
                # Update user's details
                user.last_seen_at = now()
                user.authorized = True
                user.save()
            except User.DoesNotExist:
                graph = GraphAPI(oauth_token.token)
                profile = graph.get('me')

                # Either the user already exists and its just a new token, or user and token both are new
                try:
                    user = User.objects.get(facebook_id = profile.get('id'))
                except User.DoesNotExist:
                    # Create a new user to go with token
                    user = User.objects.create(
                        facebook_id = profile.get('id'),
                        oauth_token = oauth_token
                    )

                user.synchronize(profile)

                # Delete old access token if there is any and  only if the new one is different
                old_oauth_token = None
                if user.oauth_token != oauth_token:
                    old_oauth_token = user.oauth_token
                    user.oauth_token = oauth_token

                user.save()

                if old_oauth_token:
                    old_oauth_token.delete()

            if not user.oauth_token.extended:
                # Attempt to extend the OAuth token, but ignore exceptions raised by
                # bug #102727766518358 in the Facebook Platform.
                #
                # http://developers.facebook.com/bugs/102727766518358/
                try:
                    user.oauth_token.extend()
                except:
                    pass

            request.facebook.user = user
            request.session['oauth_token'] = user.oauth_token.token

    def process_response(self, request, response):
        """
        Set compact P3P policies and save signed request to cookie.

        P3P is a WC3 standard (see http://www.w3.org/TR/P3P/), and although largely ignored by most
        browsers it is considered by IE before accepting third-party cookies (ie. cookies set by
        documents in iframes). If they are not set correctly, IE will not set these cookies.
        """
        # if request.facebook and request.facebook.user and "code" in request.REQUEST:
        #     """ Remove the code query param """
        #     path = request.get_full_path()
        #     path = re.sub(r'&?code=[a-zA-Z0-9_\-]+&?', '', path)
        #     path = re.sub(r'&?ref=web_canvas&?', '', path)
        #     return HttpResponseRedirect(path)

        response['P3P'] = 'CP="IDC CURa ADMa OUR IND PHY ONL COM STA"'
        return response

class FacebookCanvasMiddleware(BaseMiddleware):
    """Middleware for Facebook canvas applications."""

    def process_request(self, request):
        """Process the signed request."""

        if not self.is_valid_path(request):
            return

        if hasattr(request, "facebook") and request.facebook:
            return

        # An error occured during authorization...
        if 'error' in request.GET:
            # The user refused to authorize the application...
            if request.GET['error'] == 'access_denied':
                return authorization_denied_view(request)

        # Signed request found in either GET, POST or COOKIES...
        if 'signed_request' in request.REQUEST or 'signed_request' in request.COOKIES:
            request.facebook = Facebook()

            # If the request method is POST and its body only contains the signed request,
            # chances are it's a request from the Facebook platform and we'll override
            # the request method to HTTP GET to rectify their misinterpretation
            # of the HTTP standard.
            #
            # References:
            # "POST for Canvas" migration at http://developers.facebook.com/docs/canvas/post/
            # "Incorrect use of the HTTP protocol" discussion at http://forum.developers.facebook.net/viewtopic.php?id=93554
            if request.method == 'POST' and 'signed_request' in request.POST:
                request.POST = QueryDict('')
                request.method = 'GET'

            try:
                request.facebook.signed_request = SignedRequest(
                    signed_request = request.REQUEST.get('signed_request') or request.COOKIES.get('signed_request'),
                    application_secret_key = FACEBOOK_APPLICATION_SECRET_KEY
                )
            except SignedRequest.Error:
                request.facebook = False

            # Valid signed request and user has authorized the application
            if request.facebook and request.facebook.signed_request.user.has_authorized_application:

                # Redirect to Facebook Authorization if the OAuth token has expired
                if request.facebook.signed_request.user.oauth_token.has_expired:
                    return authorize_application(
                        request = request,
                        redirect_uri = get_post_authorization_redirect_url(request)
                    )

                # Initialize a User object and its corresponding OAuth token
                try:
                    user = User.objects.get(facebook_id=request.facebook.signed_request.user.id)
                except User.DoesNotExist:
                    oauth_token = OAuthToken.objects.create(
                        token = request.facebook.signed_request.user.oauth_token.token,
                        issued_at = request.facebook.signed_request.user.oauth_token.issued_at.replace(tzinfo=tzlocal()),
                        expires_at = request.facebook.signed_request.user.oauth_token.expires_at.replace(tzinfo=tzlocal())
                    )

                    user = User.objects.create(
                        facebook_id = request.facebook.signed_request.user.id,
                        oauth_token = oauth_token
                    )

                    user.synchronize()

                # Update the user's details and OAuth token
                else:
                    user.last_seen_at = now()

                    if 'signed_request' in request.REQUEST:
                        user.authorized = True

                        if request.facebook.signed_request.user.oauth_token:
                            user.oauth_token.token = request.facebook.signed_request.user.oauth_token.token
                            user.oauth_token.issued_at = request.facebook.signed_request.user.oauth_token.issued_at.replace(tzinfo=tzlocal())
                            user.oauth_token.expires_at = request.facebook.signed_request.user.oauth_token.expires_at.replace(tzinfo=tzlocal())
                            user.oauth_token.save()

                    user.save()

                if not user.oauth_token.extended:
                    # Attempt to extend the OAuth token, but ignore exceptions raised by
                    # bug #102727766518358 in the Facebook Platform.
                    #
                    # http://developers.facebook.com/bugs/102727766518358/
                    try:
                        user.oauth_token.extend()
                    except:
                        pass

                request.facebook.user = user

        # ... no signed request found.
        else:
            request.facebook = False

    def process_response(self, request, response):
        """
        Set compact P3P policies and save signed request to cookie.

        P3P is a WC3 standard (see http://www.w3.org/TR/P3P/), and although largely ignored by most
        browsers it is considered by IE before accepting third-party cookies (ie. cookies set by
        documents in iframes). If they are not set correctly, IE will not set these cookies.
        """
        if FANDJANGO_CACHE_SIGNED_REQUEST:
            if 'signed_request' in request.REQUEST:
                response.set_cookie('signed_request', request.REQUEST['signed_request'])
            response['P3P'] = 'CP="IDC CURa ADMa OUR IND PHY ONL COM STA"'
        return response

<commit_msg>Added redirect to remove fb query params<commit_after>from datetime import timedelta
from urlparse import parse_qs
import re

from django.http import QueryDict, HttpResponseRedirect
from django.core.exceptions import ImproperlyConfigured

from fandjango.views import authorize_application
from fandjango.models import Facebook, User, OAuthToken
from fandjango.settings import (
    FACEBOOK_APPLICATION_SECRET_KEY, FACEBOOK_APPLICATION_ID,
    FANDJANGO_CACHE_SIGNED_REQUEST, DISABLED_PATHS, ENABLED_PATHS
)
from fandjango.utils import (
    is_disabled_path, is_enabled_path,
    authorization_denied_view, get_post_authorization_redirect_url
)

from facepy import SignedRequest, GraphAPI

try:
    from django.utils.timezone import now
except ImportError:
    from datetime import datetime
    def now():
        return datetime.now()

from dateutil.tz import tzlocal

class BaseMiddleware():

    def is_valid_path(self, request):
        if ENABLED_PATHS and DISABLED_PATHS:
            raise ImproperlyConfigured(
                'You may configure either FANDJANGO_ENABLED_PATHS '
                'or FANDJANGO_DISABLED_PATHS, but not both.'
            )

        if DISABLED_PATHS and is_disabled_path(request.path):
            return

        if ENABLED_PATHS and not is_enabled_path(request.path):
            return

        return True

class FacebookMiddleware(BaseMiddleware):
    """Middleware for Facebook auth on websites."""

    def process_request(self, request):
        """Process the signed request."""

        if not self.is_valid_path(request):
            return

        if hasattr(request, "facebook") and request.facebook:
            return

        # An error occured during authorization...
        if 'error' in request.GET:
            # The user refused to authorize the application...
            if request.GET['error'] == 'access_denied':
                return authorization_denied_view(request)

        request.facebook = Facebook()
        oauth_token = False

        # Is there a token in the current session already present?
        if 'oauth_token' in request.session:
            try:
                # Check if the current token is already in DB
                oauth_token = OAuthToken.objects.get(token=request.session['oauth_token'])
            except OAuthToken.DoesNotExist:
                # It's in the session but it's not in the DB, weird, better delete it and get new one..
                del request.session['oauth_token']
                request.facebook = False
                return

        # Is there a code in the GET request?
        elif 'code' in request.GET:
            try:
                graph = GraphAPI()

                # Exchange code for an access_token
                response = graph.get('oauth/access_token',
                    client_id = FACEBOOK_APPLICATION_ID,
                    redirect_uri = get_post_authorization_redirect_url(request, canvas=False),
                    client_secret = FACEBOOK_APPLICATION_SECRET_KEY,
                    code = request.GET['code'],
                )

                components = parse_qs(response)

                # Save new OAuth-token in DB
                oauth_token, created = OAuthToken.objects.get_or_create(
                    token = components['access_token'][0],
                    issued_at = now(),
                    expires_at = now() + timedelta(seconds = int(components['expires'][0]))
                )
            except GraphAPI.OAuthError:
                pass

        # There is a valid access_token
        if oauth_token:
            # Redirect to Facebook authorization if the OAuth token has expired
            if oauth_token.expired:
                request.facebook = False
                return

            # Is there a user already connected to the current token?
            try:
                user = oauth_token.user
                # Update user's details
                user.last_seen_at = now()
                user.authorized = True
                user.save()
            except User.DoesNotExist:
                graph = GraphAPI(oauth_token.token)
                profile = graph.get('me')

                # Either the user already exists and its just a new token, or user and token both are new
                try:
                    user = User.objects.get(facebook_id = profile.get('id'))
                except User.DoesNotExist:
                    # Create a new user to go with token
                    user = User.objects.create(
                        facebook_id = profile.get('id'),
                        oauth_token = oauth_token
                    )

                user.synchronize(profile)

                # Delete old access token if there is any and  only if the new one is different
                old_oauth_token = None
                if user.oauth_token != oauth_token:
                    old_oauth_token = user.oauth_token
                    user.oauth_token = oauth_token

                user.save()

                if old_oauth_token:
                    old_oauth_token.delete()

            if not user.oauth_token.extended:
                # Attempt to extend the OAuth token, but ignore exceptions raised by
                # bug #102727766518358 in the Facebook Platform.
                #
                # http://developers.facebook.com/bugs/102727766518358/
                try:
                    user.oauth_token.extend()
                except:
                    pass

            request.facebook.user = user
            request.session['oauth_token'] = user.oauth_token.token

    def process_response(self, request, response):
        """
        Set compact P3P policies and save signed request to cookie.

        P3P is a WC3 standard (see http://www.w3.org/TR/P3P/), and although largely ignored by most
        browsers it is considered by IE before accepting third-party cookies (ie. cookies set by
        documents in iframes). If they are not set correctly, IE will not set these cookies.
        """
        if request.facebook and request.facebook.user and "code" in request.REQUEST:
            """ Remove the code query param """
            path = request.get_full_path()
            path = re.sub(r'&?code=[a-zA-Z0-9_\-]+&?', '', path)
            path = re.sub(r'&?ref=web_canvas&?', '', path)
            return HttpResponseRedirect(path)

        response['P3P'] = 'CP="IDC CURa ADMa OUR IND PHY ONL COM STA"'
        return response

class FacebookCanvasMiddleware(BaseMiddleware):
    """Middleware for Facebook canvas applications."""

    def process_request(self, request):
        """Process the signed request."""

        if not self.is_valid_path(request):
            return

        if hasattr(request, "facebook") and request.facebook:
            return

        # An error occured during authorization...
        if 'error' in request.GET:
            # The user refused to authorize the application...
            if request.GET['error'] == 'access_denied':
                return authorization_denied_view(request)

        # Signed request found in either GET, POST or COOKIES...
        if 'signed_request' in request.REQUEST or 'signed_request' in request.COOKIES:
            request.facebook = Facebook()

            # If the request method is POST and its body only contains the signed request,
            # chances are it's a request from the Facebook platform and we'll override
            # the request method to HTTP GET to rectify their misinterpretation
            # of the HTTP standard.
            #
            # References:
            # "POST for Canvas" migration at http://developers.facebook.com/docs/canvas/post/
            # "Incorrect use of the HTTP protocol" discussion at http://forum.developers.facebook.net/viewtopic.php?id=93554
            if request.method == 'POST' and 'signed_request' in request.POST:
                request.POST = QueryDict('')
                request.method = 'GET'

            try:
                request.facebook.signed_request = SignedRequest(
                    signed_request = request.REQUEST.get('signed_request') or request.COOKIES.get('signed_request'),
                    application_secret_key = FACEBOOK_APPLICATION_SECRET_KEY
                )
            except SignedRequest.Error:
                request.facebook = False

            # Valid signed request and user has authorized the application
            if request.facebook and request.facebook.signed_request.user.has_authorized_application:

                # Redirect to Facebook Authorization if the OAuth token has expired
                if request.facebook.signed_request.user.oauth_token.has_expired:
                    return authorize_application(
                        request = request,
                        redirect_uri = get_post_authorization_redirect_url(request)
                    )

                # Initialize a User object and its corresponding OAuth token
                try:
                    user = User.objects.get(facebook_id=request.facebook.signed_request.user.id)
                except User.DoesNotExist:
                    oauth_token = OAuthToken.objects.create(
                        token = request.facebook.signed_request.user.oauth_token.token,
                        issued_at = request.facebook.signed_request.user.oauth_token.issued_at.replace(tzinfo=tzlocal()),
                        expires_at = request.facebook.signed_request.user.oauth_token.expires_at.replace(tzinfo=tzlocal())
                    )

                    user = User.objects.create(
                        facebook_id = request.facebook.signed_request.user.id,
                        oauth_token = oauth_token
                    )

                    user.synchronize()

                # Update the user's details and OAuth token
                else:
                    user.last_seen_at = now()

                    if 'signed_request' in request.REQUEST:
                        user.authorized = True

                        if request.facebook.signed_request.user.oauth_token:
                            user.oauth_token.token = request.facebook.signed_request.user.oauth_token.token
                            user.oauth_token.issued_at = request.facebook.signed_request.user.oauth_token.issued_at.replace(tzinfo=tzlocal())
                            user.oauth_token.expires_at = request.facebook.signed_request.user.oauth_token.expires_at.replace(tzinfo=tzlocal())
                            user.oauth_token.save()

                    user.save()

                if not user.oauth_token.extended:
                    # Attempt to extend the OAuth token, but ignore exceptions raised by
                    # bug #102727766518358 in the Facebook Platform.
                    #
                    # http://developers.facebook.com/bugs/102727766518358/
                    try:
                        user.oauth_token.extend()
                    except:
                        pass

                request.facebook.user = user

        # ... no signed request found.
        else:
            request.facebook = False

    def process_response(self, request, response):
        """
        Set compact P3P policies and save signed request to cookie.

        P3P is a WC3 standard (see http://www.w3.org/TR/P3P/), and although largely ignored by most
        browsers it is considered by IE before accepting third-party cookies (ie. cookies set by
        documents in iframes). If they are not set correctly, IE will not set these cookies.
        """
        if FANDJANGO_CACHE_SIGNED_REQUEST:
            if 'signed_request' in request.REQUEST:
                response.set_cookie('signed_request', request.REQUEST['signed_request'])
            response['P3P'] = 'CP="IDC CURa ADMa OUR IND PHY ONL COM STA"'
        return response

<|endoftext|>


<commit_before>#!/usr/bin/env python
"""The google-java-format program (https://github.com/google/google-java-format)
reformats Java source code, but it creates poor formatting for annotations
in comments.
Run this script on files after running google-java-format, and it will perform
small changes in place to improve formatting of annotations in comments.
If called with no arguments, it reads from and writes to standard output.

You typically will not run this program directly; it is run by
run-google-java-format.py.
"""

from __future__ import print_function

import os
import os.path
import re
import sys

def eprint(*args, **kwargs):
    "Print to standard error"
    print(*args, file=sys.stderr, **kwargs)

# pylint: disable=line-too-long, multiple-statements

# These are type annotations, which should NOT go on their own line.
# A type annotation's @Target annotation contains "TYPE_USE".
# This includes private type annotations used for testing, to keep the Checker
# Framework test source code looking nice.
# To generate this list:
#   grep --recursive --files-with-matches -e '^@Target\b.*TYPE_USE' $CHECKERFRAMEWORK/checker/src/main/java $CHECKERFRAMEWORK/framework/src/main/java $CHECKERFRAMEWORK/docs/examples/units-extension $CHECKERFRAMEWORK/framework/src/test/java $t/object-construction-checker | grep -v '~' | sed 's/.*\///' | awk '{print $1} END {print "NotNull.java"; print "UbTop.java"; print "LbTop.java"; print "UB_TOP.java"; print "LB_TOP.java";}' | sed 's/\(.*\)\.java/    "\1",/' | sort | uniq > type-qualifiers.txt
typeAnnotations = set([
    "A",
    "ACCBottom",
    "Acceleration",
    "ACCTop",
    "AlwaysSafe",
    "Angle",
    "AnnoWithStringArg",
    "Area",
    "ArrayLen",
    "ArrayLenRange",
    "ArrayWithoutPackage",
    "AwtAlphaCompositingRule",
    "AwtColorSpace",
    "AwtCursorType",
    "AwtFlowLayout",
    "B",
    "BinaryName",
    "BinaryNameInUnnamedPackage",
    "BinaryNameOrPrimitiveType",
    "BinaryNameWithoutPackage",
    "BoolVal",
    "Bottom",
    "BottomQualifier",
    "BottomThis",
    "BottomVal",
    "C",
    "CalledMethods",
    "CalledMethodsBottom",
    "CalledMethodsPredicate",
    "CalledMethodsTop",
    "CanonicalName",
    "CanonicalNameOrEmpty",
    "CCBottom",
    "CCTop",
    "cd",
    "ClassBound",
    "ClassGetName",
    "ClassGetSimpleName",
    "ClassVal",
    "ClassValBottom",
    "CompilerMessageKey",
    "CompilerMessageKeyBottom",
    "Constant",
    "Critical",
    "Current",
    "D",
    "DefaultType",
    "degrees",
    "Det",
    "DotSeparatedIdentifiers",
    "DotSeparatedIdentifiersOrPrimitiveType",
    "DoubleVal",
    "E",
    "Encrypted",
    "Even",
    "F",
    "FBCBottom",
    "FEBot",
    "Fenum",
    "FenumBottom",
    "FenumTop",
    "FETop",
    "FieldDescriptor",
    "FieldDescriptorForPrimitive",
    "FieldDescriptorForPrimitiveOrArrayInUnnamedPackage",
    "FieldDescriptorWithoutPackage",
    "FlowExp",
    "Format",
    "FormatBottom",
    "FqBinaryName",
    "Frequency",
    "FullyQualifiedName",
    "g",
    "GTENegativeOne",
    "GuardedBy",
    "GuardedByBottom",
    "GuardedByUnknown",
    "GuardSatisfied",
    "h",
    "H1Bot",
    "H1Invalid",
    "H1Poly",
    "H1S1",
    "H1S2",
    "H1Top",
    "H2Bot",
    "H2Poly",
    "H2S1",
    "H2S2",
    "H2Top",
    "Hz",
    "I18nFormat",
    "I18nFormatBottom",
    "I18nFormatFor",
    "I18nInvalidFormat",
    "I18nUnknownFormat",
    "Identifier",
    "IdentifierOrArray",
    "IdentifierOrPrimitiveType",
    "ImplicitAnno",
    "IndexFor",
    "IndexOrHigh",
    "IndexOrLow",
    "Initialized",
    "InternalForm",
    "Interned",
    "InternedDistinct",
    "IntRange",
    "IntVal",
    "InvalidFormat",
    "K",
    "KeyFor",
    "KeyForBottom",
    "KeyForType",
    "kg",
    "kHz",
    "km",
    "km2",
    "kmPERh",
    "LbTop",
    "LB_TOP",
    "LeakedToResult",
    "Length",
    "LengthOf",
    "LessThan",
    "LessThanBottom",
    "LessThanUnknown",
    "LocalizableKey",
    "LocalizableKeyBottom",
    "Localized",
    "LowerBoundBottom",
    "LowerBoundUnknown",
    "LTEqLengthOf",
    "LTLengthOf",
    "LTOMLengthOf",
    "Luminance",
    "m",
    "m2",
    "Mass",
    "MaybeAliased",
    "MaybeDerivedFromConstant",
    "MaybePresent",
    "MaybeThis",
    "MethodDescriptor",
    "MethodVal",
    "MethodValBottom",
    "min",
    "MinLen",
    "mm",
    "mm2",
    "mol",
    "MonotonicNonNull",
    "MonotonicNonNullType",
    "MonotonicOdd",
    "mPERs",
    "mPERs2",
    "NegativeIndexFor",
    "NonConstant",
    "NonDet",
    "NonLeaked",
    "NonNegative",
    "NonNull",
    "NonNullType",
    "NonRaw",
    "NotCalledMethods",
    "NotNull",
    "NotQualifier",
    "NTDBottom",
    "NTDMiddle",
    "NTDSide",
    "NTDTop",
    "Nullable",
    "NullableType",
    "Odd",
    "OrderNonDet",
    "Parent",
    "PatternA",
    "PatternAB",
    "PatternAC",
    "PatternB",
    "PatternBC",
    "PatternBottomFull",
    "PatternBottomPartial",
    "PatternC",
    "PatternUnknown",
    "Poly",
    "PolyAll",
    "PolyConstant",
    "PolyDet",
    "PolyEncrypted",
    "PolyFenum",
    "PolyIndex",
    "PolyInterned",
    "PolyKeyFor",
    "PolyLength",
    "PolyLowerBound",
    "PolyNull",
    "PolyNullType",
    "PolyPresent",
    "PolyRaw",
    "PolyReflection",
    "PolyRegex",
    "PolySameLen",
    "PolySignature",
    "PolySigned",
    "PolyTainted",
    "PolyTypeDeclDefault",
    "PolyUI",
    "PolyUnit",
    "PolyUpperBound",
    "PolyValue",
    "Positive",
    "Present",
    "PrimitiveType",
    "PropertyKey",
    "PropertyKeyBottom",
    "PurityUnqualified",
    "Qualifier",
    "radians",
    "Raw",
    "ReflectBottom",
    "Regex",
    "RegexBottom",
    "ReportUnqualified",
    "s",
    "SameLen",
    "SameLenBottom",
    "SameLenUnknown",
    "SearchIndexBottom",
    "SearchIndexFor",
    "SearchIndexUnknown",
    "Sibling1",
    "Sibling2",
    "SiblingWithFields",
    "SignatureBottom",
    "Signed",
    "SignednessBottom",
    "SignednessGlb",
    "SignedPositive",
    "SignedPositiveFromUnsigned",
    "Speed",
    "StringVal",
    "SubQual",
    "Substance",
    "SubstringIndexBottom",
    "SubstringIndexFor",
    "SubstringIndexUnknown",
    "SuperQual",
    "SwingBoxOrientation",
    "SwingCompassDirection",
    "SwingElementOrientation",
    "SwingHorizontalOrientation",
    "SwingSplitPaneOrientation",
    "SwingTextOrientation",
    "SwingTitleJustification",
    "SwingTitlePosition",
    "SwingVerticalOrientation",
    "Tainted",
    "Temperature",
    "This",
    "Time",
    "Top",
    "TypeDeclDefaultBottom",
    "TypeDeclDefaultMiddle",
    "TypeDeclDefaultTop",
    "UbTop",
    "UB_TOP",
    "UI",
    "UnderInitialization",
    "Unique",
    "UnitsBottom",
    "UnknownClass",
    "UnknownCompilerMessageKey",
    "UnknownFormat",
    "UnknownInitialization",
    "UnknownInterned",
    "UnknownKeyFor",
    "UnknownLocalizableKey",
    "UnknownLocalized",
    "UnknownMethod",
    "UnknownPropertyKey",
    "UnknownRegex",
    "UnknownSignedness",
    "UnknownThis",
    "UnknownUnits",
    "UnknownVal",
    "Unsigned",
    "Untainted",
    "UpperBoundBottom",
    "UpperBoundUnknown",
    "Value",
    "WholeProgramInferenceBottom",
])

# File .type-annotations can add to the typeAnnotations variable.
if os.path.isfile(".type-annotations"):
    # pylint: disable=bad-whitespace
    exec (open(".type-annotations").read())

debug = False
# debug = True

def debug_print(*args, **kwargs):
    """Print, only if the debug variable is set."""
    if debug: print(*args, **kwargs)

# Two annotations in a row, or an annotation abutting array brackets "[]".
# Space is inserted between.
abuttingannoRegex = re.compile(r"(/\*@[A-Za-z0-9_]+\*/)(\[\]|/\*@[A-Za-z0-9_]+\*/)")
# Voodoo annotation with extra space after
voodootrailingspaceRegex = re.compile(r"(/\*>>> ?@.*\bthis\*/) (\))")

# Matches the argument to an annotation.
# 3 cases:
#   ()
#   (".*")
#   (.*)
# The regex tries to safely permit "()" within a string in an annotation, such as
#   @GuardedBy("c1.getFieldPure2()")
annoargRegex = r'(?: *(?:\( *\)|\( *"[^"]*" *\)|\([^")][^)]*\)))?'
# Matches an annotation
annoRegex = r"@[A-Za-z0-9_.]+" + annoargRegex

# Matches, at the end of its line (in capturing group 2):
#  * An annotation
#    This is a bit dangerous!  It might be within a comment.
#    The script tries to heuristically detect this.
#  * An annotation in comments
#  * a comment like /*offset = */ that should appear right
#    before the argument it documents.
#    (Supported in main google-java-format as of April 26, 2017, but not yet in a release:
#    https://github.com/google/google-java-format/commit/ca0c4d90cdbb46b3a2bf9c2b83d0bd558cccc41e )
# The annotation will be moved to the beginning of the following line,
# if it appears in typeAnnotations.
trailingannoRegex = re.compile(r"^(.*?)[ \t]*(" + annoRegex + r"|/\*" + annoRegex +
                               r"\*/|/\* *[A-Za-z0-9_]+ *= *\*/)$")

whitespaceRegex = re.compile(r"^([ \t]*).*$")

emptylineRegex = re.compile(r"^[ \t]*$")

# Heuristic: matches if the line might be within a //, /*, or Javadoc comment.
withinCommentRegex = re.compile(r"//|/\*(?!.*\/*/)|^[ \t]*\*[ \t]")

startsWithCommentRegex = re.compile(r"^[ \t]*(//|/\*$|/\*[^@]|\*|void\b)")

def insert_after_whitespace(insertion, s):
    """Return s, with insertion inserted after its leading whitespace."""
    m = re.match(whitespaceRegex, s)
    return s[0:m.end(1)] + insertion + s[m.end(1):]

def fixup_loop(infile, outfile):
    """Fix up formatting while reading from infile and writing to outfile."""
    prev = ""  # previous line, which might end with a type annotation.
    for line in infile:
        # Handle trailing space after a voodoo comment
        line = voodootrailingspaceRegex.sub(r"\1\2", line)
        # Handle abutting annotations in comments
        m = re.search(abuttingannoRegex, line)
        while m:
            debug_print("found abutting", line)
            line = line[0:m.end(1)] + " " + line[m.start(2):]
            m = re.search(abuttingannoRegex, line)
        # Don't move an annotation to the start of a comment line
        if re.search(startsWithCommentRegex, line):
            m = None
            debug_print("Don't prepend to comment", prev, line)
        else:
            # Handle annotations at end of line that should be at beginning of
            # next line.
            m = re.search(trailingannoRegex, prev)
            debug_print("trailing? (pre-loop)", m, prev, line)
        while m:
            debug_print("found trailing", prev, line)
            anno = m.group(2)
            if not base_annotation(anno) in typeAnnotations:
                break
            debug_print("prev was:", prev)
            candidate_prev = prev[0:m.end(1)] + prev[m.end(2):]
            debug_print("candidate_prev is :", candidate_prev)
            if re.search(withinCommentRegex, candidate_prev):
                debug_print("withinCommentRegex prohibits action")
                break
            prev = candidate_prev
            debug_print("prev is:", prev)
            if re.search(emptylineRegex, prev):
                prev = ""
                debug_print("prev is empty")
            debug_print("line was:", line)
            line = insert_after_whitespace(anno + " ", line)
            debug_print("line is :", line)
            m = re.search(trailingannoRegex, prev)
            debug_print("trailing? (post-loop-body)", m, prev, line)
        outfile.write(prev)
        prev = line
    outfile.write(prev)

def base_annotation(annotation):
    """Remove leading and trailing comment characters, spaces, arguments, and at sign.
Example: base_annotation('/*@RequiresNonNull("FileIO.data_trace_state")*/' => 'RequiresNonNull'"""
    debug_print("base_annotation <=", annotation)

    # Remove comments
    if annotation.startswith("/*"):
        annotation = annotation[2:]
    if annotation.endswith("*/"):
        annotation = annotation[:-2]

    # Remove arguments
    idx = annotation.find('(')
    if idx != -1:
        annotation = annotation[0:idx]

    # Remove package names
    idx = annotation.rfind('.')
    if idx != -1:
        annotation = annotation[idx + 1:]

    annotation = annotation.strip()
    if annotation.startswith("@"):
        annotation = annotation[1:]
    debug_print("base_annotation =>", annotation)
    return annotation

if len(sys.argv) == 1:
    fixup_loop(sys.stdin, sys.stdout)
else:
    for fname in sys.argv[1:]:
        outfname = fname + '.out'
        with open(fname, 'r') as infile:
            with open(outfname, 'w') as outfile:
                fixup_loop(infile, outfile)
        os.rename(outfname, fname)
<commit_msg>EnumVal is a type annotation<commit_after>#!/usr/bin/env python
"""The google-java-format program (https://github.com/google/google-java-format)
reformats Java source code, but it creates poor formatting for annotations
in comments.
Run this script on files after running google-java-format, and it will perform
small changes in place to improve formatting of annotations in comments.
If called with no arguments, it reads from and writes to standard output.

You typically will not run this program directly; it is run by
run-google-java-format.py.
"""

from __future__ import print_function

import os
import os.path
import re
import sys

def eprint(*args, **kwargs):
    "Print to standard error"
    print(*args, file=sys.stderr, **kwargs)

# pylint: disable=line-too-long, multiple-statements

# These are type annotations, which should NOT go on their own line.
# A type annotation's @Target annotation contains "TYPE_USE".
# This includes private type annotations used for testing, to keep the Checker
# Framework test source code looking nice.
# To generate this list:
#   grep --recursive --files-with-matches -e '^@Target\b.*TYPE_USE' $CHECKERFRAMEWORK/checker/src/main/java $CHECKERFRAMEWORK/framework/src/main/java $CHECKERFRAMEWORK/docs/examples/units-extension $CHECKERFRAMEWORK/framework/src/test/java $t/object-construction-checker | grep -v '~' | sed 's/.*\///' | awk '{print $1} END {print "NotNull.java"; print "UbTop.java"; print "LbTop.java"; print "UB_TOP.java"; print "LB_TOP.java";}' | sed 's/\(.*\)\.java/    "\1",/' | sort | uniq > type-qualifiers.txt
typeAnnotations = set([
    "A",
    "ACCBottom",
    "Acceleration",
    "ACCTop",
    "AlwaysSafe",
    "Angle",
    "AnnoWithStringArg",
    "Area",
    "ArrayLen",
    "ArrayLenRange",
    "ArrayWithoutPackage",
    "AwtAlphaCompositingRule",
    "AwtColorSpace",
    "AwtCursorType",
    "AwtFlowLayout",
    "B",
    "BinaryName",
    "BinaryNameInUnnamedPackage",
    "BinaryNameOrPrimitiveType",
    "BinaryNameWithoutPackage",
    "BoolVal",
    "Bottom",
    "BottomQualifier",
    "BottomThis",
    "BottomVal",
    "C",
    "CalledMethods",
    "CalledMethodsBottom",
    "CalledMethodsPredicate",
    "CalledMethodsTop",
    "CanonicalName",
    "CanonicalNameOrEmpty",
    "CCBottom",
    "CCTop",
    "cd",
    "ClassBound",
    "ClassGetName",
    "ClassGetSimpleName",
    "ClassVal",
    "ClassValBottom",
    "CompilerMessageKey",
    "CompilerMessageKeyBottom",
    "Constant",
    "Critical",
    "Current",
    "D",
    "DefaultType",
    "degrees",
    "Det",
    "DotSeparatedIdentifiers",
    "DotSeparatedIdentifiersOrPrimitiveType",
    "DoubleVal",
    "E",
    "Encrypted",
    "EnumVal",
    "Even",
    "F",
    "FBCBottom",
    "FEBot",
    "Fenum",
    "FenumBottom",
    "FenumTop",
    "FETop",
    "FieldDescriptor",
    "FieldDescriptorForPrimitive",
    "FieldDescriptorForPrimitiveOrArrayInUnnamedPackage",
    "FieldDescriptorWithoutPackage",
    "FlowExp",
    "Format",
    "FormatBottom",
    "FqBinaryName",
    "Frequency",
    "FullyQualifiedName",
    "g",
    "GTENegativeOne",
    "GuardedBy",
    "GuardedByBottom",
    "GuardedByUnknown",
    "GuardSatisfied",
    "h",
    "H1Bot",
    "H1Invalid",
    "H1Poly",
    "H1S1",
    "H1S2",
    "H1Top",
    "H2Bot",
    "H2Poly",
    "H2S1",
    "H2S2",
    "H2Top",
    "Hz",
    "I18nFormat",
    "I18nFormatBottom",
    "I18nFormatFor",
    "I18nInvalidFormat",
    "I18nUnknownFormat",
    "Identifier",
    "IdentifierOrArray",
    "IdentifierOrPrimitiveType",
    "ImplicitAnno",
    "IndexFor",
    "IndexOrHigh",
    "IndexOrLow",
    "Initialized",
    "InternalForm",
    "Interned",
    "InternedDistinct",
    "IntRange",
    "IntVal",
    "InvalidFormat",
    "K",
    "KeyFor",
    "KeyForBottom",
    "KeyForType",
    "kg",
    "kHz",
    "km",
    "km2",
    "kmPERh",
    "LbTop",
    "LB_TOP",
    "LeakedToResult",
    "Length",
    "LengthOf",
    "LessThan",
    "LessThanBottom",
    "LessThanUnknown",
    "LocalizableKey",
    "LocalizableKeyBottom",
    "Localized",
    "LowerBoundBottom",
    "LowerBoundUnknown",
    "LTEqLengthOf",
    "LTLengthOf",
    "LTOMLengthOf",
    "Luminance",
    "m",
    "m2",
    "Mass",
    "MaybeAliased",
    "MaybeDerivedFromConstant",
    "MaybePresent",
    "MaybeThis",
    "MethodDescriptor",
    "MethodVal",
    "MethodValBottom",
    "min",
    "MinLen",
    "mm",
    "mm2",
    "mol",
    "MonotonicNonNull",
    "MonotonicNonNullType",
    "MonotonicOdd",
    "mPERs",
    "mPERs2",
    "NegativeIndexFor",
    "NonConstant",
    "NonDet",
    "NonLeaked",
    "NonNegative",
    "NonNull",
    "NonNullType",
    "NonRaw",
    "NotCalledMethods",
    "NotNull",
    "NotQualifier",
    "NTDBottom",
    "NTDMiddle",
    "NTDSide",
    "NTDTop",
    "Nullable",
    "NullableType",
    "Odd",
    "OrderNonDet",
    "Parent",
    "PatternA",
    "PatternAB",
    "PatternAC",
    "PatternB",
    "PatternBC",
    "PatternBottomFull",
    "PatternBottomPartial",
    "PatternC",
    "PatternUnknown",
    "Poly",
    "PolyAll",
    "PolyConstant",
    "PolyDet",
    "PolyEncrypted",
    "PolyFenum",
    "PolyIndex",
    "PolyInterned",
    "PolyKeyFor",
    "PolyLength",
    "PolyLowerBound",
    "PolyNull",
    "PolyNullType",
    "PolyPresent",
    "PolyRaw",
    "PolyReflection",
    "PolyRegex",
    "PolySameLen",
    "PolySignature",
    "PolySigned",
    "PolyTainted",
    "PolyTypeDeclDefault",
    "PolyUI",
    "PolyUnit",
    "PolyUpperBound",
    "PolyValue",
    "Positive",
    "Present",
    "PrimitiveType",
    "PropertyKey",
    "PropertyKeyBottom",
    "PurityUnqualified",
    "Qualifier",
    "radians",
    "Raw",
    "ReflectBottom",
    "Regex",
    "RegexBottom",
    "ReportUnqualified",
    "s",
    "SameLen",
    "SameLenBottom",
    "SameLenUnknown",
    "SearchIndexBottom",
    "SearchIndexFor",
    "SearchIndexUnknown",
    "Sibling1",
    "Sibling2",
    "SiblingWithFields",
    "SignatureBottom",
    "Signed",
    "SignednessBottom",
    "SignednessGlb",
    "SignedPositive",
    "SignedPositiveFromUnsigned",
    "Speed",
    "StringVal",
    "SubQual",
    "Substance",
    "SubstringIndexBottom",
    "SubstringIndexFor",
    "SubstringIndexUnknown",
    "SuperQual",
    "SwingBoxOrientation",
    "SwingCompassDirection",
    "SwingElementOrientation",
    "SwingHorizontalOrientation",
    "SwingSplitPaneOrientation",
    "SwingTextOrientation",
    "SwingTitleJustification",
    "SwingTitlePosition",
    "SwingVerticalOrientation",
    "Tainted",
    "Temperature",
    "This",
    "Time",
    "Top",
    "TypeDeclDefaultBottom",
    "TypeDeclDefaultMiddle",
    "TypeDeclDefaultTop",
    "UbTop",
    "UB_TOP",
    "UI",
    "UnderInitialization",
    "Unique",
    "UnitsBottom",
    "UnknownClass",
    "UnknownCompilerMessageKey",
    "UnknownFormat",
    "UnknownInitialization",
    "UnknownInterned",
    "UnknownKeyFor",
    "UnknownLocalizableKey",
    "UnknownLocalized",
    "UnknownMethod",
    "UnknownPropertyKey",
    "UnknownRegex",
    "UnknownSignedness",
    "UnknownThis",
    "UnknownUnits",
    "UnknownVal",
    "Unsigned",
    "Untainted",
    "UpperBoundBottom",
    "UpperBoundUnknown",
    "Value",
    "WholeProgramInferenceBottom",
])

# File .type-annotations can add to the typeAnnotations variable.
if os.path.isfile(".type-annotations"):
    # pylint: disable=bad-whitespace
    exec (open(".type-annotations").read())

debug = False
# debug = True

def debug_print(*args, **kwargs):
    """Print, only if the debug variable is set."""
    if debug: print(*args, **kwargs)

# Two annotations in a row, or an annotation abutting array brackets "[]".
# Space is inserted between.
abuttingannoRegex = re.compile(r"(/\*@[A-Za-z0-9_]+\*/)(\[\]|/\*@[A-Za-z0-9_]+\*/)")
# Voodoo annotation with extra space after
voodootrailingspaceRegex = re.compile(r"(/\*>>> ?@.*\bthis\*/) (\))")

# Matches the argument to an annotation.
# 3 cases:
#   ()
#   (".*")
#   (.*)
# The regex tries to safely permit "()" within a string in an annotation, such as
#   @GuardedBy("c1.getFieldPure2()")
annoargRegex = r'(?: *(?:\( *\)|\( *"[^"]*" *\)|\([^")][^)]*\)))?'
# Matches an annotation
annoRegex = r"@[A-Za-z0-9_.]+" + annoargRegex

# Matches, at the end of its line (in capturing group 2):
#  * An annotation
#    This is a bit dangerous!  It might be within a comment.
#    The script tries to heuristically detect this.
#  * An annotation in comments
#  * a comment like /*offset = */ that should appear right
#    before the argument it documents.
#    (Supported in main google-java-format as of April 26, 2017, but not yet in a release:
#    https://github.com/google/google-java-format/commit/ca0c4d90cdbb46b3a2bf9c2b83d0bd558cccc41e )
# The annotation will be moved to the beginning of the following line,
# if it appears in typeAnnotations.
trailingannoRegex = re.compile(r"^(.*?)[ \t]*(" + annoRegex + r"|/\*" + annoRegex +
                               r"\*/|/\* *[A-Za-z0-9_]+ *= *\*/)$")

whitespaceRegex = re.compile(r"^([ \t]*).*$")

emptylineRegex = re.compile(r"^[ \t]*$")

# Heuristic: matches if the line might be within a //, /*, or Javadoc comment.
withinCommentRegex = re.compile(r"//|/\*(?!.*\/*/)|^[ \t]*\*[ \t]")

startsWithCommentRegex = re.compile(r"^[ \t]*(//|/\*$|/\*[^@]|\*|void\b)")

def insert_after_whitespace(insertion, s):
    """Return s, with insertion inserted after its leading whitespace."""
    m = re.match(whitespaceRegex, s)
    return s[0:m.end(1)] + insertion + s[m.end(1):]

def fixup_loop(infile, outfile):
    """Fix up formatting while reading from infile and writing to outfile."""
    prev = ""  # previous line, which might end with a type annotation.
    for line in infile:
        # Handle trailing space after a voodoo comment
        line = voodootrailingspaceRegex.sub(r"\1\2", line)
        # Handle abutting annotations in comments
        m = re.search(abuttingannoRegex, line)
        while m:
            debug_print("found abutting", line)
            line = line[0:m.end(1)] + " " + line[m.start(2):]
            m = re.search(abuttingannoRegex, line)
        # Don't move an annotation to the start of a comment line
        if re.search(startsWithCommentRegex, line):
            m = None
            debug_print("Don't prepend to comment", prev, line)
        else:
            # Handle annotations at end of line that should be at beginning of
            # next line.
            m = re.search(trailingannoRegex, prev)
            debug_print("trailing? (pre-loop)", m, prev, line)
        while m:
            debug_print("found trailing", prev, line)
            anno = m.group(2)
            if not base_annotation(anno) in typeAnnotations:
                break
            debug_print("prev was:", prev)
            candidate_prev = prev[0:m.end(1)] + prev[m.end(2):]
            debug_print("candidate_prev is :", candidate_prev)
            if re.search(withinCommentRegex, candidate_prev):
                debug_print("withinCommentRegex prohibits action")
                break
            prev = candidate_prev
            debug_print("prev is:", prev)
            if re.search(emptylineRegex, prev):
                prev = ""
                debug_print("prev is empty")
            debug_print("line was:", line)
            line = insert_after_whitespace(anno + " ", line)
            debug_print("line is :", line)
            m = re.search(trailingannoRegex, prev)
            debug_print("trailing? (post-loop-body)", m, prev, line)
        outfile.write(prev)
        prev = line
    outfile.write(prev)

def base_annotation(annotation):
    """Remove leading and trailing comment characters, spaces, arguments, and at sign.
Example: base_annotation('/*@RequiresNonNull("FileIO.data_trace_state")*/' => 'RequiresNonNull'"""
    debug_print("base_annotation <=", annotation)

    # Remove comments
    if annotation.startswith("/*"):
        annotation = annotation[2:]
    if annotation.endswith("*/"):
        annotation = annotation[:-2]

    # Remove arguments
    idx = annotation.find('(')
    if idx != -1:
        annotation = annotation[0:idx]

    # Remove package names
    idx = annotation.rfind('.')
    if idx != -1:
        annotation = annotation[idx + 1:]

    annotation = annotation.strip()
    if annotation.startswith("@"):
        annotation = annotation[1:]
    debug_print("base_annotation =>", annotation)
    return annotation

if len(sys.argv) == 1:
    fixup_loop(sys.stdin, sys.stdout)
else:
    for fname in sys.argv[1:]:
        outfname = fname + '.out'
        with open(fname, 'r') as infile:
            with open(outfname, 'w') as outfile:
                fixup_loop(infile, outfile)
        os.rename(outfname, fname)
<|endoftext|>


<commit_before>from attest import Tests, assert_hook, raises

from plastic.resourcedir import ResourceDirectory

tests = Tests()
rdir = ResourceDirectory(__name__, 'resources')

# $ tree plastictests/resources/
# plastictests/resources/
# ├── a
# ├── b
# ├── c
# │   ├── a
# │   ├── b
# │   ├── c
# │   │   └── a
# │   └── d
# └── d

@tests.test
def rdir_contains():
    assert 'a' in rdir
    assert 'b' in rdir
    assert 'c' in rdir
    assert 'c/a' in rdir
    assert 'c/b' in rdir
    assert 'c/c' in rdir
    assert 'c/c/a' in rdir
    assert 'c/c/b' not in rdir
    assert 'c/d' in rdir
    assert 'c/d/b' not in rdir
    assert 'c/e' not in rdir
    assert 'd' in rdir
    assert 'e' not in rdir

@tests.test
def rdir_getitem():
    f = rdir['a']
    read = f.read().strip()
    f.close()
    assert read == 'a'
    with rdir['a'] as f:
        read = f.read().strip()
        assert read == 'a'
    with rdir['b'] as f:
        read = f.read().strip()
        assert read == 'b'
    assert isinstance(rdir['c'], ResourceDirectory)
    with rdir['c']['a'] as f:
        read = f.read().strip()
        assert read == 'c-a'
    with rdir['c']['b'] as f:
        read = f.read().strip()
        assert read == 'c-b'
    assert isinstance(rdir['c']['c'], ResourceDirectory)
    with rdir['c']['c']['a'] as f:
        read = f.read().strip()
        assert read == 'c-c-a'
    with rdir['c/a'] as f:
        read = f.read().strip()
        assert read == 'c-a'
    with rdir['c/b'] as f:
        read = f.read().strip()
        assert read == 'c-b'
    assert isinstance(rdir['c/c'], ResourceDirectory)
    with rdir['c/c/a'] as f:
        read = f.read().strip()
        assert read == 'c-c-a'
    with raises(KeyError):
        rdir['c/c/b']
    assert isinstance(rdir['c/d'], ResourceDirectory)
    with raises(KeyError):
        rdir['c/e']
    assert isinstance(rdir['d'], ResourceDirectory)
    with raises(KeyError):
        rdir['e']

@tests.test
def rdir_iter():
    expected = set(['a', 'b', 'c', 'c/a', 'c/b', 'c/c', 'c/c/a', 'c/d',
                    'c/d/a', 'd', 'd/a'])
    assert set(rdir) == expected
    assert set(rdir['c']) == set(['a', 'b', 'c', 'c/a', 'd', 'd/a'])
    assert set(rdir['c']['c']) == set(['a'])
    assert set(rdir['c/c']) == set(['a'])

@tests.test
def rdir_len():
    assert len(rdir) == 11
    assert len(rdir['c']) == 6
    assert len(rdir['c']['c']) == 1
    assert len(rdir['c/c']) == 1
    assert len(rdir['d']) == 1

<commit_msg>Changed a comment: tree<commit_after>from attest import Tests, assert_hook, raises

from plastic.resourcedir import ResourceDirectory

tests = Tests()
rdir = ResourceDirectory(__name__, 'resources')

# $ tree plastictests/resources/
# plastictests/resources/
# ├── a
# ├── b
# ├── c
# │   ├── a
# │   ├── b
# │   ├── c
# │   │   └── a
# │   └── d
# │       └── a
# └── d
#     └── a
#
#     4 directories, 7 files

@tests.test
def rdir_contains():
    assert 'a' in rdir
    assert 'b' in rdir
    assert 'c' in rdir
    assert 'c/a' in rdir
    assert 'c/b' in rdir
    assert 'c/c' in rdir
    assert 'c/c/a' in rdir
    assert 'c/c/b' not in rdir
    assert 'c/d' in rdir
    assert 'c/d/b' not in rdir
    assert 'c/e' not in rdir
    assert 'd' in rdir
    assert 'e' not in rdir

@tests.test
def rdir_getitem():
    f = rdir['a']
    read = f.read().strip()
    f.close()
    assert read == 'a'
    with rdir['a'] as f:
        read = f.read().strip()
        assert read == 'a'
    with rdir['b'] as f:
        read = f.read().strip()
        assert read == 'b'
    assert isinstance(rdir['c'], ResourceDirectory)
    with rdir['c']['a'] as f:
        read = f.read().strip()
        assert read == 'c-a'
    with rdir['c']['b'] as f:
        read = f.read().strip()
        assert read == 'c-b'
    assert isinstance(rdir['c']['c'], ResourceDirectory)
    with rdir['c']['c']['a'] as f:
        read = f.read().strip()
        assert read == 'c-c-a'
    with rdir['c/a'] as f:
        read = f.read().strip()
        assert read == 'c-a'
    with rdir['c/b'] as f:
        read = f.read().strip()
        assert read == 'c-b'
    assert isinstance(rdir['c/c'], ResourceDirectory)
    with rdir['c/c/a'] as f:
        read = f.read().strip()
        assert read == 'c-c-a'
    with raises(KeyError):
        rdir['c/c/b']
    assert isinstance(rdir['c/d'], ResourceDirectory)
    with raises(KeyError):
        rdir['c/e']
    assert isinstance(rdir['d'], ResourceDirectory)
    with raises(KeyError):
        rdir['e']

@tests.test
def rdir_iter():
    expected = set(['a', 'b', 'c', 'c/a', 'c/b', 'c/c', 'c/c/a', 'c/d',
                    'c/d/a', 'd', 'd/a'])
    assert set(rdir) == expected
    assert set(rdir['c']) == set(['a', 'b', 'c', 'c/a', 'd', 'd/a'])
    assert set(rdir['c']['c']) == set(['a'])
    assert set(rdir['c/c']) == set(['a'])

@tests.test
def rdir_len():
    assert len(rdir) == 11
    assert len(rdir['c']) == 6
    assert len(rdir['c']['c']) == 1
    assert len(rdir['c/c']) == 1
    assert len(rdir['d']) == 1

<|endoftext|>


<commit_before>from six.moves import map as imap
from toolz import compose, identity

class ApplyAsyncResult(object):
    """An object that boxes results for calls to
    :meth:`~zipline.utils.pool.SequentialPool.apply_async`.

    Parameters
    ----------
    value : any
        The result of calling the function, or any exception that was raised.
    successful : bool
        If ``True``, ``value`` is the return value of the function.
        If ``False``, ``value`` is the exception that was raised when calling
        the functions.
    """
    def __init__(self, value, successful):
        self._value = value
        self._successful = successful

    def successful(self):
        """Did the function execute without raising an exception?
        """
        return self._successful

    def get(self):
        """Return the result of calling the function or reraise any exceptions
        that were raised.
        """
        if not self._successful:
            raise self._value
        return self._value

    def ready(self):
        """Has the function finished executing.

        Notes
        -----
        In the :class:`~zipline.utils.pool.SequentialPool` case, this is always
        ``True``.
        """
        return True

    def wait(self):
        """Wait until the function is finished executing.

        Notes
        -----
        In the :class:`~zipline.utils.pool.SequentialPool` case, this is a nop
        because the function is computed eagerly in the same thread as the
        call to :meth:`~zipline.utils.pool.SequentialPool.apply_async`.
        """
        pass

class SequentialPool(object):
    """A dummy pool object that iterates sequentially in a single thread.

    Methods
    -------
    map(f: callable[A, B], iterable: iterable[A]) -> list[B]
        Apply a function to each of the elements of ``iterable``.
    imap(f: callable[A, B], iterable: iterable[A]) -> iterable[B]
        Lazily apply a function to each of the elements of ``iterable``.
    imap_unordered(f: callable[A, B], iterable: iterable[A]) -> iterable[B]
        Lazily apply a function to each of the elements of ``iterable`` but
        yield values as they become available. The resulting iterable is
        unordered.

    Notes
    -----
    This object is useful for testing to mock out the ``Pool`` interface
    provided by gevent or multiprocessing.

    See Also
    --------
    :class:`multiprocessing.Pool`
    """
    map = staticmethod(compose(list, imap))
    imap = imap_unordered = staticmethod(imap)

    @staticmethod
    def apply_async(f, args=(), kwargs=None, callback=None):
        """Apply a function but emulate the API of an asynchronous call.

        Parameters
        ----------
        f : callable
            The function to call.
        args : tuple, optional
            The positional arguments.
        kwargs : dict, optional
            The keyword arguments.

        Returns
        -------
        future : ApplyAsyncResult
            The result of calling the function boxed in a future-like api.

        Notes
        -----
        This calls the function eagerly but wraps it so that ``SequentialPool``
        can be used where a :class:`multiprocessing.Pool` or
        :class:`gevent.pool.Pool` would be used.
        """
        try:
            value = (identity if callback is None else callback)(
                f(*args, **kwargs or {}),
            )
            successful = True
        except Exception as e:
            value = e
            successful = False

        return ApplyAsyncResult(value, successful)

    @staticmethod
    def apply(f, args=(), kwargs=None):
        """Apply a function.

        Parameters
        ----------
        f : callable
            The function to call.
        args : tuple, optional
            The positional arguments.
        kwargs : dict, optional
            The keyword arguments.

        Returns
        -------
        result : any
            f(*args, **kwargs)
        """
        return f(*args, **kwargs or {})
<commit_msg>MAINT: Mock out more pool methods.<commit_after>from six.moves import map as imap
from toolz import compose, identity

class ApplyAsyncResult(object):
    """An object that boxes results for calls to
    :meth:`~zipline.utils.pool.SequentialPool.apply_async`.

    Parameters
    ----------
    value : any
        The result of calling the function, or any exception that was raised.
    successful : bool
        If ``True``, ``value`` is the return value of the function.
        If ``False``, ``value`` is the exception that was raised when calling
        the functions.
    """
    def __init__(self, value, successful):
        self._value = value
        self._successful = successful

    def successful(self):
        """Did the function execute without raising an exception?
        """
        return self._successful

    def get(self):
        """Return the result of calling the function or reraise any exceptions
        that were raised.
        """
        if not self._successful:
            raise self._value
        return self._value

    def ready(self):
        """Has the function finished executing.

        Notes
        -----
        In the :class:`~zipline.utils.pool.SequentialPool` case, this is always
        ``True``.
        """
        return True

    def wait(self):
        """Wait until the function is finished executing.

        Notes
        -----
        In the :class:`~zipline.utils.pool.SequentialPool` case, this is a nop
        because the function is computed eagerly in the same thread as the
        call to :meth:`~zipline.utils.pool.SequentialPool.apply_async`.
        """
        pass

class SequentialPool(object):
    """A dummy pool object that iterates sequentially in a single thread.

    Methods
    -------
    map(f: callable[A, B], iterable: iterable[A]) -> list[B]
        Apply a function to each of the elements of ``iterable``.
    imap(f: callable[A, B], iterable: iterable[A]) -> iterable[B]
        Lazily apply a function to each of the elements of ``iterable``.
    imap_unordered(f: callable[A, B], iterable: iterable[A]) -> iterable[B]
        Lazily apply a function to each of the elements of ``iterable`` but
        yield values as they become available. The resulting iterable is
        unordered.

    Notes
    -----
    This object is useful for testing to mock out the ``Pool`` interface
    provided by gevent or multiprocessing.

    See Also
    --------
    :class:`multiprocessing.Pool`
    """
    map = staticmethod(compose(list, imap))
    imap = imap_unordered = staticmethod(imap)

    @staticmethod
    def apply_async(f, args=(), kwargs=None, callback=None):
        """Apply a function but emulate the API of an asynchronous call.

        Parameters
        ----------
        f : callable
            The function to call.
        args : tuple, optional
            The positional arguments.
        kwargs : dict, optional
            The keyword arguments.

        Returns
        -------
        future : ApplyAsyncResult
            The result of calling the function boxed in a future-like api.

        Notes
        -----
        This calls the function eagerly but wraps it so that ``SequentialPool``
        can be used where a :class:`multiprocessing.Pool` or
        :class:`gevent.pool.Pool` would be used.
        """
        try:
            value = (identity if callback is None else callback)(
                f(*args, **kwargs or {}),
            )
            successful = True
        except Exception as e:
            value = e
            successful = False

        return ApplyAsyncResult(value, successful)

    @staticmethod
    def apply(f, args=(), kwargs=None):
        """Apply a function.

        Parameters
        ----------
        f : callable
            The function to call.
        args : tuple, optional
            The positional arguments.
        kwargs : dict, optional
            The keyword arguments.

        Returns
        -------
        result : any
            f(*args, **kwargs)
        """
        return f(*args, **kwargs or {})

    @staticmethod
    def close():
        pass

    @staticmethod
    def join():
        pass
<|endoftext|>


<commit_before>import os
import socket

from pyramid.authentication import AuthTktAuthenticationPolicy
from pyramid.authorization import ACLAuthorizationPolicy
from pyramid.config import Configurator
from pyramid_beaker import session_factory_from_settings
from social_pyramid.models import init_social
from sqlalchemy import create_engine
from sqlalchemy import event
from sqlalchemy.exc import DisconnectionError

import local_settings as app_local_settings
import settings as app_settings
from fantasydota.scripts.init_tables import create_tables
from fantasydota.util.jsonhelpers import custom_json_renderer
from .models import (
    Base,
    DBSession)

# I was getting 2006 mysql gone away if website left for few hours. didnt know why
def checkout_listener(dbapi_con, con_record, con_proxy):
    try:
        try:
            dbapi_con.ping(False)
        except TypeError:
            dbapi_con.ping()
    except dbapi_con.OperationalError as exc:
        if exc.args[0] in (2006, 2013, 2014, 2045, 2055):
            raise DisconnectionError()
        else:
            raise

def get_settings(module):
    return { key: value for key, value in module.__dict__.items()
              if key not in module.__builtins__ and
                 key not in ['__builtins__', '__file__'] }

def main(global_config, **settings):
    """ This function returns a Pyramid WSGI application.
    """
    sqlalchemy_url = os.path.expandvars(settings.get('sqlalchemy.url'))
    engine = create_engine(sqlalchemy_url, echo=False, pool_size=100, pool_recycle=3600)
    event.listen(engine, 'checkout', checkout_listener)
    DBSession.configure(bind=engine)

    # Need https set up on local machine for secure True to work locally
    secure = socket.gethostname() == 'fantasyesport'
    authn_policy = AuthTktAuthenticationPolicy(settings.get('authn_policy_secr'), hashalg='sha512', http_only=True,
                                               secure=secure,
                                               max_age=10000000)
    authz_policy = ACLAuthorizationPolicy()

    my_session_factory = session_factory_from_settings(settings)

    config = Configurator(settings=settings, session_factory=my_session_factory, authentication_policy=authn_policy,
        authorization_policy=authz_policy,)
    config.add_route('index', '/')
    config.add_route('done', '/done')

    config.registry.settings.update(get_settings(app_settings))
    config.registry.settings.update(get_settings(app_local_settings))

    init_social(config, Base, DBSession)  # is this the right place?
    Base.metadata.bind = engine
    Base.metadata.create_all(engine)
    create_tables(DBSession)

    config.include('social_pyramid')
    config.scan('social_pyramid')

    # pyramid_social_auth.register_provider(settings, google.GoogleOAuth2)
    # psa.register_provider(settings, facebook.FacebookOAuth2)

    config.add_request_method('.auth.get_user', 'user', reify=True)  # user or username? should it start with .?

    config.add_renderer('json', custom_json_renderer())
    config.add_static_view('static', 'static', cache_max_age=3600)
    config.add_route('login', '/login')
    config.add_route('logout', '/logout')
    config.add_route('view_faq', '/faq')

    config.add_route('register', '/register')
    config.add_route('change_password', '/changePassword')
    config.add_route('forgot_password', '/forgotPassword')
    config.add_route('reset_password', '/resetPassword')
    config.add_route('reset_password_page', '/resetPasswordPage')
    config.add_route('update_email_settings', '/updateEmailSettings')
    config.add_route("add_friend", '/addFriend')
    config.add_route('clear_notifications', '/clearNotifications')

    config.add_route('view_rules', '/rules')
    config.add_route('view_team', '/team')
    config.add_route('transfer_proxy', '/transfer_proxy')
    config.add_route('leaderboard', '/leaderboard')
    config.add_route('account_settings', '/accountSettings')
    config.add_route('profile', '/profile')

    config.scan()
    return config.make_wsgi_app()
<commit_msg>readd hall of fame route<commit_after>import os
import socket

from pyramid.authentication import AuthTktAuthenticationPolicy
from pyramid.authorization import ACLAuthorizationPolicy
from pyramid.config import Configurator
from pyramid_beaker import session_factory_from_settings
from social_pyramid.models import init_social
from sqlalchemy import create_engine
from sqlalchemy import event
from sqlalchemy.exc import DisconnectionError

import local_settings as app_local_settings
import settings as app_settings
from fantasydota.scripts.init_tables import create_tables
from fantasydota.util.jsonhelpers import custom_json_renderer
from .models import (
    Base,
    DBSession)

# I was getting 2006 mysql gone away if website left for few hours. didnt know why
def checkout_listener(dbapi_con, con_record, con_proxy):
    try:
        try:
            dbapi_con.ping(False)
        except TypeError:
            dbapi_con.ping()
    except dbapi_con.OperationalError as exc:
        if exc.args[0] in (2006, 2013, 2014, 2045, 2055):
            raise DisconnectionError()
        else:
            raise

def get_settings(module):
    return { key: value for key, value in module.__dict__.items()
              if key not in module.__builtins__ and
                 key not in ['__builtins__', '__file__'] }

def main(global_config, **settings):
    """ This function returns a Pyramid WSGI application.
    """
    sqlalchemy_url = os.path.expandvars(settings.get('sqlalchemy.url'))
    engine = create_engine(sqlalchemy_url, echo=False, pool_size=100, pool_recycle=3600)
    event.listen(engine, 'checkout', checkout_listener)
    DBSession.configure(bind=engine)

    # Need https set up on local machine for secure True to work locally
    secure = socket.gethostname() == 'fantasyesport'
    authn_policy = AuthTktAuthenticationPolicy(settings.get('authn_policy_secr'), hashalg='sha512', http_only=True,
                                               secure=secure,
                                               max_age=10000000)
    authz_policy = ACLAuthorizationPolicy()

    my_session_factory = session_factory_from_settings(settings)

    config = Configurator(settings=settings, session_factory=my_session_factory, authentication_policy=authn_policy,
        authorization_policy=authz_policy,)
    config.add_route('index', '/')
    config.add_route('done', '/done')

    config.registry.settings.update(get_settings(app_settings))
    config.registry.settings.update(get_settings(app_local_settings))

    init_social(config, Base, DBSession)  # is this the right place?
    Base.metadata.bind = engine
    Base.metadata.create_all(engine)
    create_tables(DBSession)

    config.include('social_pyramid')
    config.scan('social_pyramid')

    # pyramid_social_auth.register_provider(settings, google.GoogleOAuth2)
    # psa.register_provider(settings, facebook.FacebookOAuth2)

    config.add_request_method('.auth.get_user', 'user', reify=True)  # user or username? should it start with .?

    config.add_renderer('json', custom_json_renderer())
    config.add_static_view('static', 'static', cache_max_age=3600)
    config.add_route('login', '/login')
    config.add_route('logout', '/logout')
    config.add_route('view_faq', '/faq')
    config.add_route('hall_of_fame', '/hallOfFame')

    config.add_route('register', '/register')
    config.add_route('change_password', '/changePassword')
    config.add_route('forgot_password', '/forgotPassword')
    config.add_route('reset_password', '/resetPassword')
    config.add_route('reset_password_page', '/resetPasswordPage')
    config.add_route('update_email_settings', '/updateEmailSettings')
    config.add_route("add_friend", '/addFriend')
    config.add_route('clear_notifications', '/clearNotifications')

    config.add_route('view_rules', '/rules')
    config.add_route('view_team', '/team')
    config.add_route('transfer_proxy', '/transfer_proxy')
    config.add_route('leaderboard', '/leaderboard')
    config.add_route('account_settings', '/accountSettings')
    config.add_route('profile', '/profile')

    config.scan()
    return config.make_wsgi_app()
<|endoftext|>


<commit_before>import warnings

import numpy as np
from scipy.spatial.distance import cosine

from .utils import compute_fragments, average_agg_tanimoto, \
    compute_scaffolds, fingerprints, \
    get_mol, canonic_smiles, mol_passes_filters, \
    logP, QED, SA, NP, weight
from moses.utils import mapper
from .utils_fcd import get_predictions, calculate_frechet_distance
from multiprocessing import Pool
from moses.utils import disable_rdkit_log, enable_rdkit_log

def get_all_metrics(test, gen, k=[1000, 10000], n_jobs=1, gpu=-1,
                    batch_size=512, test_scaffolds=None,
                    ptest=None, ptest_scaffolds=None):
    '''
    Computes all available metrics between test (scaffold test) and generated sets of SMILES.
    Parameters:
        test: list of test SMILES
        gen: list of generated SMILES
        k: list with values for unique@k.
            Will calculate number of unique molecules in the first k molecules.
        n_jobs: number of workers for parallel processing
        gpu: index of GPU for FCD metric
        batch_size: batch size for FCD metric
        test_scaffolds: list of scaffold test SMILES
            Will compute only on the general test set if not specified
        ptest: dict with precalculated statistics of the test set
        ptest_scaffolds: dict with precalculated statistics of the scaffold test set

    Available metrics:
        * %valid
        * %unique@k
        * Frechet ChemNet Distance (FCD)
        * Fragment similarity (Frag)
        * Scaffold similarity (Scaf)
        * Similarity to nearest neighbour (SNN)
        * Internal diversity (IntDiv)
        * %passes filters (Filters)
        * Distribution difference for logP, SA, QED, NP, weight
    '''
    disable_rdkit_log()
    metrics = {}
    if n_jobs != 1:
        pool = Pool(n_jobs)
    else:
        pool = 1
    metrics['valid'] = fraction_valid(gen, n_jobs=n_jobs)
    gen = remove_invalid(gen, canonize=True)
    if not isinstance(k, (list, tuple)):
        k = [k]
    for _k in k:
        metrics['unique@{}'.format(_k)] = fraction_unique(gen, _k, pool)

    if ptest is None:
        ptest = compute_intermediate_statistics(test, n_jobs=n_jobs, gpu=gpu, batch_size=batch_size)
    if test_scaffolds is not None and ptest_scaffolds is None:
        ptest_scaffolds = compute_intermediate_statistics(test_scaffolds, n_jobs=n_jobs,
                                                                 gpu=gpu, batch_size=batch_size)
    mols = mapper(pool)(get_mol, gen)
    kwargs = {'n_jobs': pool, 'gpu': gpu, 'batch_size': batch_size}
    metrics['FCD/Test'] = FCDMetric(**kwargs)(gen=gen, ptest=ptest['FCD'])
    metrics['SNN/Test'] = SNNMetric(**kwargs)(gen=mols, ptest=ptest['SNN'])
    metrics['Frag/Test'] = FragMetric(**kwargs)(gen=mols, ptest=ptest['Frag'])
    metrics['Scaf/Test'] = ScafMetric(**kwargs)(gen=mols, ptest=ptest['Scaf'])
    if ptest_scaffolds is not None:
        metrics['FCD/TestSF'] = FCDMetric(**kwargs)(gen=gen, ptest=ptest_scaffolds['FCD'])
        metrics['SNN/TestSF'] = SNNMetric(**kwargs)(gen=mols, ptest=ptest_scaffolds['SNN'])
        metrics['Frag/TestSF'] = FragMetric(**kwargs)(gen=mols, ptest=ptest_scaffolds['Frag'])
        metrics['Scaf/TestSF'] = ScafMetric(**kwargs)(gen=mols, ptest=ptest_scaffolds['Scaf'])

    metrics['IntDiv'] = internal_diversity(mols, pool)
    metrics['IntDiv2'] = internal_diversity(mols, pool, p=2)
    metrics['Filters'] = fraction_passes_filters(mols, pool)

    # Properties
    for name, func in [('logP', logP), ('SA', SA),
                       ('QED', QED), ('NP', NP),
                       ('weight', weight)]:
        metrics[name] = FrechetMetric(func, **kwargs)(gen=mols, ptest=ptest[name])
    enable_rdkit_log()
    if n_jobs != 1:
        pool.terminate()
    return metrics

def compute_intermediate_statistics(smiles, n_jobs=1, gpu=-1, batch_size=512):
    '''
    The function precomputes statistics such as mean and variance for FCD, etc.
    It is useful to compute the statistics for test and scaffold test sets to
        speedup metrics calculation.
    '''

    if n_jobs != 1:
        pool = Pool(n_jobs)
    else:
        pool = 1
    statistics = {}
    mols = mapper(pool)(get_mol, smiles)
    kwargs = {'n_jobs': n_jobs, 'gpu': gpu, 'batch_size': batch_size}
    statistics['FCD'] = FCDMetric(**kwargs).precalc(smiles)
    statistics['SNN'] = SNNMetric(**kwargs).precalc(mols)
    statistics['Frag'] = FragMetric(**kwargs).precalc(mols)
    statistics['Scaf'] = ScafMetric(**kwargs).precalc(mols)
    for name, func in [('logP', logP), ('SA', SA),
                       ('QED', QED), ('NP', NP),
                       ('weight', weight)]:
        statistics[name] = FrechetMetric(func, **kwargs).precalc(mols)
    if n_jobs != 1:
        pool.terminate()
    return statistics

def fraction_passes_filters(gen, n_jobs=1):
    '''
    Computes the fraction of molecules that pass filters:
    * MCF
    * PAINS
    * Only allowed atoms ('C','N','S','O','F','Cl','Br','H')
    * No charges
    '''
    passes = mapper(n_jobs)(mol_passes_filters, gen)
    return np.mean(passes)

def internal_diversity(gen, n_jobs=1, gpu=-1, fp_type='morgan', gen_fps=None, p=1):
    '''
    Computes internal diversity as:
    1/|A|^2 sum_{x, y in AxA} (1-tanimoto(x, y))
    '''
    if gen_fps is None:
        gen_fps = fingerprints(gen, fp_type=fp_type, n_jobs=n_jobs)
    return 1 - (average_agg_tanimoto(gen_fps, gen_fps,
                                     agg='mean', gpu=gpu, p=p)).mean()

def fraction_unique(gen, k=None, n_jobs=1, check_validity=True):
    '''
    Computes a number of unique molecules
    :param gen: list of SMILES
    :param k: compute unique@k
    :param check_validity: raises ValueError if invalid molecules are present
    '''
    if k is not None:
        if len(gen) < k:
            warnings.warn(
                "Can't compute unique@{}. gen contains only {} molecules".format(
                    k, len(gen)))
        gen = gen[:k]
    canonic = set(mapper(n_jobs)(canonic_smiles, gen))
    if None in canonic and check_validity:
        raise ValueError("Invalid molecule passed to unique@k")
    return len(canonic) / len(gen)

def fraction_valid(gen, n_jobs=1):
    '''
    Computes a number of valid molecules
    :param gen: list of SMILES
    '''
    gen = mapper(n_jobs)(get_mol, gen)
    return 1 - gen.count(None) / len(gen)

def remove_invalid(gen, canonize=True, n_jobs=1):
    '''
    Removes invalid molecules from the dataset
    '''
    if canonize:
        mols = mapper(n_jobs)(get_mol, gen)
        return [gen_ for gen_, mol in zip(gen, mols) if mol is not None]
    else:
        return [x for x in mapper(n_jobs)(canonic_smiles, gen) if
                x is not None]

class Metric:
    def __init__(self, n_jobs=1, gpu=-1, batch_size=512, **kwargs):
        self.n_jobs = n_jobs
        self.gpu = gpu
        self.batch_size = batch_size
        for k, v in kwargs.values():
            setattr(self, k, v)

    def __call__(self, test=None, gen=None, ptest=None, pgen=None):
        assert (test is None) != (ptest is None), "specify test xor ptest"
        assert (gen is None) != (pgen is None), "specify gen xor pgen"
        if ptest is None:
            ptest = self.precalc(test)
        if pgen is None:
            pgen = self.precalc(gen)
        return self.metric(ptest, pgen)

    def precalc(self, moleclues):
        raise NotImplementedError

    def metric(self, ptest, pgen):
        raise NotImplementedError

class FCDMetric(Metric):
    '''
    Computes Frechet ChemNet Distance
    '''
    def precalc(self, smiles):
        if len(smiles) < 2:
            warnings.warn("Can't compute FCD for less than 2 molecules")
            return np.nan

        chemnet_activations = get_predictions(smiles, gpu=self.gpu,
                                              batch_size=self.batch_size)
        mu = chemnet_activations.mean(0)
        sigma = np.cov(chemnet_activations.T)
        return {'mu': mu, 'sigma': sigma}

    def metric(self, ptest, pgen):
        return calculate_frechet_distance(ptest['mu'], ptest['sigma'],
                                          pgen['mu'], pgen['sigma'])

class SNNMetric(Metric):
    '''
    Computes average max similarities of gen SMILES to test SMILES
    '''
    def __init__(self, fp_type='morgan', **kwargs):
        self.fp_type = fp_type
        super().__init__(**kwargs)

    def precalc(self, mols):
        return {'fps': fingerprints(mols, n_jobs=self.n_jobs, fp_type=self.fp_type)}

    def metric(self, ptest, pgen):
        return average_agg_tanimoto(ptest['fps'], pgen['fps'], gpu=self.gpu)

def cos_distance(test_counts, gen_counts):
    '''
    Computes 1 - cosine similarity between
     dictionaries of form {name: count}. Non-present
     elements are considered zero
    '''
    if len(test_counts) == 0 or len(gen_counts) == 0:
        return np.nan
    keys = np.unique(list(test_counts.keys()) + list(gen_counts.keys()))
    test_vec = np.array([test_counts.get(k, 0) for k in keys])
    gen_vec = np.array([gen_counts.get(k, 0) for k in keys])
    return 1 - cosine(test_vec, gen_vec)

class FragMetric(Metric):
    def precalc(self, mols):
        return {'frag': compute_fragments(mols, n_jobs=self.n_jobs)}

    def metric(self, ptest, pgen):
        return cos_distance(ptest['frag'], pgen['frag'])

class ScafMetric(Metric):
    def precalc(self, mols):
        return {'scaf': compute_scaffolds(mols, n_jobs=self.n_jobs)}

    def metric(self, ptest, pgen):
        return cos_distance(ptest['scaf'], pgen['scaf'])

class FrechetMetric(Metric):
    def __init__(self, func=None, **kwargs):
        self.func = func
        super().__init__(**kwargs)

    def precalc(self, mols):
        if self.func is not None:
            values = mapper(self.n_jobs)(self.func, mols)
        else:
            values = mols
        return {'mu': np.mean(values), 'var': np.var(values)}

    def metric(self, ptest, pgen):
        return calculate_frechet_distance(ptest['mu'], ptest['var'],
                                          pgen['mu'], pgen['var'])
<commit_msg>Metrics minor fixes (#17)<commit_after>import warnings

import numpy as np
from scipy.spatial.distance import cosine

from .utils import compute_fragments, average_agg_tanimoto, \
    compute_scaffolds, fingerprints, \
    get_mol, canonic_smiles, mol_passes_filters, \
    logP, QED, SA, NP, weight
from moses.utils import mapper
from .utils_fcd import get_predictions, calculate_frechet_distance
from multiprocessing import Pool
from moses.utils import disable_rdkit_log, enable_rdkit_log

def get_all_metrics(test, gen, k=[1000, 10000], n_jobs=1, gpu=-1,
                    batch_size=512, test_scaffolds=None,
                    ptest=None, ptest_scaffolds=None):
    '''
    Computes all available metrics between test (scaffold test) and generated sets of SMILES.
    Parameters:
        test: list of test SMILES
        gen: list of generated SMILES
        k: list with values for unique@k.
            Will calculate number of unique molecules in the first k molecules.
        n_jobs: number of workers for parallel processing
        gpu: index of GPU for FCD metric and internal diversity, -1 means use CPU
        batch_size: batch size for FCD metric
        test_scaffolds: list of scaffold test SMILES
            Will compute only on the general test set if not specified
        ptest: dict with precalculated statistics of the test set
        ptest_scaffolds: dict with precalculated statistics of the scaffold test set

    Available metrics:
        * %valid
        * %unique@k
        * Frechet ChemNet Distance (FCD)
        * Fragment similarity (Frag)
        * Scaffold similarity (Scaf)
        * Similarity to nearest neighbour (SNN)
        * Internal diversity (IntDiv)
        * Internal diversity 2: using square root of mean squared Tanimoto similarity (IntDiv2)
        * %passes filters (Filters)
        * Distribution difference for logP, SA, QED, NP, weight
    '''
    disable_rdkit_log()
    metrics = {}
    if n_jobs != 1:
        pool = Pool(n_jobs)
    else:
        pool = 1
    metrics['valid'] = fraction_valid(gen, n_jobs=n_jobs)
    gen = remove_invalid(gen, canonize=True)
    if not isinstance(k, (list, tuple)):
        k = [k]
    for _k in k:
        metrics['unique@{}'.format(_k)] = fraction_unique(gen, _k, pool)

    if ptest is None:
        ptest = compute_intermediate_statistics(test, n_jobs=n_jobs, gpu=gpu, batch_size=batch_size)
    if test_scaffolds is not None and ptest_scaffolds is None:
        ptest_scaffolds = compute_intermediate_statistics(test_scaffolds, n_jobs=n_jobs,
                                                                 gpu=gpu, batch_size=batch_size)
    mols = mapper(pool)(get_mol, gen)
    kwargs = {'n_jobs': pool, 'gpu': gpu, 'batch_size': batch_size}
    metrics['FCD/Test'] = FCDMetric(**kwargs)(gen=gen, ptest=ptest['FCD'])
    metrics['SNN/Test'] = SNNMetric(**kwargs)(gen=mols, ptest=ptest['SNN'])
    metrics['Frag/Test'] = FragMetric(**kwargs)(gen=mols, ptest=ptest['Frag'])
    metrics['Scaf/Test'] = ScafMetric(**kwargs)(gen=mols, ptest=ptest['Scaf'])
    if ptest_scaffolds is not None:
        metrics['FCD/TestSF'] = FCDMetric(**kwargs)(gen=gen, ptest=ptest_scaffolds['FCD'])
        metrics['SNN/TestSF'] = SNNMetric(**kwargs)(gen=mols, ptest=ptest_scaffolds['SNN'])
        metrics['Frag/TestSF'] = FragMetric(**kwargs)(gen=mols, ptest=ptest_scaffolds['Frag'])
        metrics['Scaf/TestSF'] = ScafMetric(**kwargs)(gen=mols, ptest=ptest_scaffolds['Scaf'])

    metrics['IntDiv'] = internal_diversity(mols, pool, gpu=gpu)
    metrics['IntDiv2'] = internal_diversity(mols, pool, gpu=gpu, p=2)
    metrics['Filters'] = fraction_passes_filters(mols, pool)

    # Properties
    for name, func in [('logP', logP), ('SA', SA),
                       ('QED', QED), ('NP', NP),
                       ('weight', weight)]:
        metrics[name] = FrechetMetric(func, **kwargs)(gen=mols, ptest=ptest[name])
    enable_rdkit_log()
    if n_jobs != 1:
        pool.terminate()
    return metrics

def compute_intermediate_statistics(smiles, n_jobs=1, gpu=-1, batch_size=512):
    '''
    The function precomputes statistics such as mean and variance for FCD, etc.
    It is useful to compute the statistics for test and scaffold test sets to
        speedup metrics calculation.
    '''

    if n_jobs != 1:
        pool = Pool(n_jobs)
    else:
        pool = 1
    statistics = {}
    mols = mapper(pool)(get_mol, smiles)
    kwargs = {'n_jobs': n_jobs, 'gpu': gpu, 'batch_size': batch_size}
    statistics['FCD'] = FCDMetric(**kwargs).precalc(smiles)
    statistics['SNN'] = SNNMetric(**kwargs).precalc(mols)
    statistics['Frag'] = FragMetric(**kwargs).precalc(mols)
    statistics['Scaf'] = ScafMetric(**kwargs).precalc(mols)
    for name, func in [('logP', logP), ('SA', SA),
                       ('QED', QED), ('NP', NP),
                       ('weight', weight)]:
        statistics[name] = FrechetMetric(func, **kwargs).precalc(mols)
    if n_jobs != 1:
        pool.terminate()
    return statistics

def fraction_passes_filters(gen, n_jobs=1):
    '''
    Computes the fraction of molecules that pass filters:
    * MCF
    * PAINS
    * Only allowed atoms ('C','N','S','O','F','Cl','Br','H')
    * No charges
    '''
    passes = mapper(n_jobs)(mol_passes_filters, gen)
    return np.mean(passes)

def internal_diversity(gen, n_jobs=1, gpu=-1, fp_type='morgan', gen_fps=None, p=1):
    '''
    Computes internal diversity as:
    1/|A|^2 sum_{x, y in AxA} (1-tanimoto(x, y))
    '''
    if gen_fps is None:
        gen_fps = fingerprints(gen, fp_type=fp_type, n_jobs=n_jobs)
    return 1 - (average_agg_tanimoto(gen_fps, gen_fps,
                                     agg='mean', gpu=gpu, p=p)).mean()

def fraction_unique(gen, k=None, n_jobs=1, check_validity=True):
    '''
    Computes a number of unique molecules
    :param gen: list of SMILES
    :param k: compute unique@k
    :param check_validity: raises ValueError if invalid molecules are present
    '''
    if k is not None:
        if len(gen) < k:
            warnings.warn(
                "Can't compute unique@{}. gen contains only {} molecules".format(
                    k, len(gen)))
        gen = gen[:k]
    canonic = set(mapper(n_jobs)(canonic_smiles, gen))
    if None in canonic and check_validity:
        raise ValueError("Invalid molecule passed to unique@k")
    return len(canonic) / len(gen)

def fraction_valid(gen, n_jobs=1):
    '''
    Computes a number of valid molecules
    :param gen: list of SMILES
    '''
    gen = mapper(n_jobs)(get_mol, gen)
    return 1 - gen.count(None) / len(gen)

def remove_invalid(gen, canonize=True, n_jobs=1):
    '''
    Removes invalid molecules from the dataset
    '''
    if canonize:
        mols = mapper(n_jobs)(get_mol, gen)
        return [gen_ for gen_, mol in zip(gen, mols) if mol is not None]
    else:
        return [x for x in mapper(n_jobs)(canonic_smiles, gen) if
                x is not None]

class Metric:
    def __init__(self, n_jobs=1, gpu=-1, batch_size=512, **kwargs):
        self.n_jobs = n_jobs
        self.gpu = gpu
        self.batch_size = batch_size
        for k, v in kwargs.values():
            setattr(self, k, v)

    def __call__(self, test=None, gen=None, ptest=None, pgen=None):
        assert (test is None) != (ptest is None), "specify test xor ptest"
        assert (gen is None) != (pgen is None), "specify gen xor pgen"
        if ptest is None:
            ptest = self.precalc(test)
        if pgen is None:
            pgen = self.precalc(gen)
        return self.metric(ptest, pgen)

    def precalc(self, moleclues):
        raise NotImplementedError

    def metric(self, ptest, pgen):
        raise NotImplementedError

class FCDMetric(Metric):
    '''
    Computes Frechet ChemNet Distance
    '''
    def precalc(self, smiles):
        if len(smiles) < 2:
            warnings.warn("Can't compute FCD for less than 2 molecules")
            return np.nan

        chemnet_activations = get_predictions(smiles, gpu=self.gpu,
                                              batch_size=self.batch_size)
        mu = chemnet_activations.mean(0)
        sigma = np.cov(chemnet_activations.T)
        return {'mu': mu, 'sigma': sigma}

    def metric(self, ptest, pgen):
        return calculate_frechet_distance(ptest['mu'], ptest['sigma'],
                                          pgen['mu'], pgen['sigma'])

class SNNMetric(Metric):
    '''
    Computes average max similarities of gen SMILES to test SMILES
    '''
    def __init__(self, fp_type='morgan', **kwargs):
        self.fp_type = fp_type
        super().__init__(**kwargs)

    def precalc(self, mols):
        return {'fps': fingerprints(mols, n_jobs=self.n_jobs, fp_type=self.fp_type)}

    def metric(self, ptest, pgen):
        return average_agg_tanimoto(ptest['fps'], pgen['fps'], gpu=self.gpu)

def cos_distance(test_counts, gen_counts):
    '''
    Computes 1 - cosine similarity between
     dictionaries of form {name: count}. Non-present
     elements are considered zero
    '''
    if len(test_counts) == 0 or len(gen_counts) == 0:
        return np.nan
    keys = np.unique(list(test_counts.keys()) + list(gen_counts.keys()))
    test_vec = np.array([test_counts.get(k, 0) for k in keys])
    gen_vec = np.array([gen_counts.get(k, 0) for k in keys])
    return 1 - cosine(test_vec, gen_vec)

class FragMetric(Metric):
    def precalc(self, mols):
        return {'frag': compute_fragments(mols, n_jobs=self.n_jobs)}

    def metric(self, ptest, pgen):
        return cos_distance(ptest['frag'], pgen['frag'])

class ScafMetric(Metric):
    def precalc(self, mols):
        return {'scaf': compute_scaffolds(mols, n_jobs=self.n_jobs)}

    def metric(self, ptest, pgen):
        return cos_distance(ptest['scaf'], pgen['scaf'])

class FrechetMetric(Metric):
    def __init__(self, func=None, **kwargs):
        self.func = func
        super().__init__(**kwargs)

    def precalc(self, mols):
        if self.func is not None:
            values = mapper(self.n_jobs)(self.func, mols)
        else:
            values = mols
        return {'mu': np.mean(values), 'var': np.var(values)}

    def metric(self, ptest, pgen):
        return calculate_frechet_distance(ptest['mu'], ptest['var'],
                                          pgen['mu'], pgen['var'])
<|endoftext|>


<commit_before>from flask import Flask
from .vasttrafik import getVasttrafikJSON
app = Flask(__name__)

@app.route('/')
def hello_world():
    return 'Hello, World!'

@app.route('/vasttrafik/<id>')
def vasttrafik(id=None):
    return getVasttrafikJSON(id)
<commit_msg>add Access-Control-Allow-Origin response header<commit_after>from flask import Flask, Response
from .vasttrafik import getVasttrafikJSON
app = Flask(__name__)

@app.route('/')
def hello_world():
    return 'Hello, World!'

@app.route('/vasttrafik/<id>')
def vasttrafik(id=None):
    resp = Response(getVasttrafikJSON(id))
    resp.headers['Access-Control-Allow-Origin'] = 'http://localhost:3000'
    return resp
<|endoftext|>


<commit_before>from django.db.models import Sum, Min, Max

from ....pricing import Price
from . import models

def get_variant_price(variant, currency, quantity=1, **kwargs):
    try:
        base_price = models.ProductPrice.objects.get(product=variant.product)
    except models.ProductPrice.DoesNotExist:
        return kwargs.pop('price', None)
    cart = kwargs.get('cart', None)
    if base_price.qty_mode == 'product' and cart:
        cart_quantity = cart.items.filter(variant__in=variant.product.variants.filter())\
                                                .aggregate(Sum('quantity'))['quantity__sum'] or 0
        if 'cartitem' in kwargs:
            quantity = cart_quantity
        else:
            quantity += cart_quantity
    try:
        price = base_price.qty_overrides.filter(min_qty__lte=quantity)\
                    .order_by('-min_qty')[0].price
    except IndexError:
        price = base_price.price
    try:
        offset = variant.variantpriceoffset.price_offset
    except models.VariantPriceOffset.DoesNotExist:
        offset = 0
    price = price + offset
    return Price(net=price, gross=price, currency=currency)

def get_product_price_range(product, currency, **kwargs):
    try:
        base_price = models.ProductPrice.objects.get(product=product)
    except models.ProductPrice.DoesNotExist:
        return kwargs.pop('price', (None, None))
    try:
        price = base_price.qty_overrides.filter(min_qty__lte=1).order_by('-min_qty')[0].price
    except IndexError:
        price = base_price.price
    max_price = min_price = price
    min_offset = base_price.offsets.aggregate(Min('price_offset'))['price_offset__min']
    max_offset = base_price.offsets.aggregate(Max('price_offset'))['price_offset__max']
    if max_offset is not None:
        max_price = max(price, price + max_offset)
    if min_offset is not None:
        min_price = min(price, price + min_offset)
    return ( Price(net=min_price, gross=min_price, currency=currency),
             Price(net=max_price, gross=max_price, currency=currency) )

<commit_msg>Cosmetics.<commit_after>from django.db.models import Sum, Min, Max

from ....pricing import Price
from . import models

def get_variant_price(variant, currency, quantity=1, **kwargs):
    try:
        base_price = models.ProductPrice.objects.get(product=variant.product)
    except models.ProductPrice.DoesNotExist:
        return kwargs.pop('price', None)
    cart = kwargs.get('cart', None)
    if base_price.qty_mode == 'product' and cart:
        cart_quantity = (cart.items.filter(variant__in=variant.product.variants.all())
                                   .aggregate(Sum('quantity'))['quantity__sum'] or 0)
        if 'cartitem' in kwargs:
            quantity = cart_quantity
        else:
            quantity += cart_quantity
    try:
        price = base_price.qty_overrides.filter(min_qty__lte=quantity) \
                                        .order_by('-min_qty')[0].price
    except IndexError:
        price = base_price.price
    try:
        offset = variant.variantpriceoffset.price_offset
    except models.VariantPriceOffset.DoesNotExist:
        offset = 0
    price = price + offset
    return Price(net=price, gross=price, currency=currency)

def get_product_price_range(product, currency, **kwargs):
    try:
        base_price = models.ProductPrice.objects.get(product=product)
    except models.ProductPrice.DoesNotExist:
        return kwargs.pop('price', (None, None))
    try:
        price = base_price.qty_overrides.filter(min_qty__lte=1).order_by('-min_qty')[0].price
    except IndexError:
        price = base_price.price
    max_price = min_price = price
    min_offset = base_price.offsets.aggregate(Min('price_offset'))['price_offset__min']
    max_offset = base_price.offsets.aggregate(Max('price_offset'))['price_offset__max']
    if max_offset is not None:
        max_price = max(price, price + max_offset)
    if min_offset is not None:
        min_price = min(price, price + min_offset)
    return ( Price(net=min_price, gross=min_price, currency=currency),
             Price(net=max_price, gross=max_price, currency=currency) )

<|endoftext|>


<commit_before>import vim
import re
import json
from os import path

curr_file = vim.eval("expand('%:p')")
curr_file_noext = path.splitext(curr_file)[0]

ccd = vim.eval("l:ccd")
opts = []

with open(ccd) as database:
    # Search for the right entry in the database matching file names
    for d in json.load(database):
        # This is an entry without a file attribute
        if 'file' not in d:
            continue

        # This entry is about a different file. We consider file names
        # without extension to handle header files which do not have
        # an entry in the database.
        d_file_noext = path.splitext(d['file'])[0]
        if d_file_noext != curr_file_noext:
            continue

        for result in re.finditer(r'-D\s*[^\s]+', d['command']):
            opts.append(result.group(0))
        for result in re.finditer(r'-isystem\s*[^\s]+', d['command']):
            opts.append(result.group(0))
        for result in re.finditer(r'-I\s*([^\s]+)', d['command']):
            opts.append('-I' + path.join(d['directory'], result.group(1)))
        break

vim.command("let l:clang_options = '" + ' '.join(opts) + "'")
<commit_msg>compdb: Try harder to find entry<commit_after>import vim
import re
import json
from os import path

def get_entry_by_filename(entries, fn, key=None):
    """Search for an entry in entries where key(entry['file']) matches
    key(fn) and returns it. If key is None than file names are compared
    directly. Returns None if none is found."""

    if key is None:
        key = lambda x: x

    for e in entries:
        if key(fn) == key(e['file']):
            return e

    return None

def find_entry(entries, fn):
    """Find an entry that matches the given filename fn more or less."""

    entry = get_entry_by_filename(entries, curr_file)
    if entry is not None:
        return entry

    key = lambda fn: path.splitext(fn)[0]
    entry = get_entry_by_filename(entries, curr_file, key)
    if entry is not None:
        return entry

    key = lambda fn: path.basename(path.splitext(fn)[0])
    entry = get_entry_by_filename(entries, curr_file, key)
    if entry is not None:
        return entry

    return None

ccd = vim.eval("l:ccd")
with open(ccd) as database:
    entries = json.load(database)

    curr_file = vim.eval("expand('%:p')")
    entry = find_entry(entries, curr_file)

    if entry is not None:
        opts = []

        for result in re.finditer(r'-D\s*[^\s]+', entry['command']):
            opts.append(result.group(0))
        for result in re.finditer(r'-isystem\s*[^\s]+', entry['command']):
            opts.append(result.group(0))
        for result in re.finditer(r'-I\s*([^\s]+)', entry['command']):
            opts.append('-I' + path.join(entry['directory'], result.group(1)))

        vim.command("let l:clang_options = '" + ' '.join(opts) + "'")
<|endoftext|>