<commit_before># Copyright (c) 2012-2013 Mitch Garnaat http://garnaat.org/
# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish, dis-
# tribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the fol-
# lowing conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#
import base64
import datetime
from hashlib import sha256
from hashlib import sha1
import hmac
import logging
from email.utils import formatdate
from botocore.exceptions import UnknownSignatureVersionError

logger = logging.getLogger(__name__)

try:
    from urllib.parse import quote
    from urllib.parse import unquote
    from urllib.parse import urlsplit
    from urllib.parse import parse_qsl
except ImportError:
    from urllib import quote
    from urllib import unquote
    from urlparse import urlsplit
    from urlparse import parse_qsl

PostContentType = 'application/x-www-form-urlencoded; charset=UTF-8'

class SigV2Auth(object):
    """
    Sign a request with Signature V2.
    """
    def __init__(self, credentials, service_name=None, region_name=None):
        self.credentials = credentials
        self.service_name = service_name
        self.region_name = region_name

    def calc_signature(self, request, params):
        split = urlsplit(request.url)
        path = split.path
        if len(path) == 0:
            path = '/'
        string_to_sign = '%s\n%s\n%s\n' % (request.method,
                                           split.netloc,
                                           path)
        lhmac = hmac.new(self.credentials.secret_key.encode('utf-8'),
                         digestmod=sha256)
        pairs = []
        for key in sorted(params):
            value = params[key]
            pairs.append(quote(key, safe='') + '=' +
                         quote(value, safe='-_~'))
        qs = '&'.join(pairs)
        string_to_sign += qs
        logger.debug('string_to_sign')
        logger.debug(string_to_sign)
        lhmac.update(string_to_sign.encode('utf-8'))
        b64 = base64.b64encode(lhmac.digest()).strip().decode('utf-8')
        return (qs, b64)

    def add_auth(self, request):
        # The auth handler is the last thing called in the
        # preparation phase of a prepared request.
        # Because of this we have to parse the query params
        # from the request body so we can update them with
        # the sigv2 auth params.
        if request.data:
            # POST
            params = request.data
        else:
            # GET
            params = request.param
        params['AWSAccessKeyId'] = self.credentials.access_key
        params['SignatureVersion'] = '2'
        params['SignatureMethod'] = 'HmacSHA256'
        params['Timestamp'] = datetime.datetime.utcnow().isoformat()
        if self.credentials.token:
            params['SecurityToken'] = self.credentials.token
        qs, signature = self.calc_signature(request, params)
        params['Signature'] = signature
        return request

class SigV3Auth(object):
    def __init__(self, credentials, service_name=None, region_name=None):
        self.credentials = credentials
        self.service_name = service_name
        self.region_name = region_name

    def add_auth(self, request):
        if 'Date' not in request.headers:
            request.headers['Date'] = formatdate(usegmt=True)
        if self.credentials.token:
            request.headers['X-Amz-Security-Token'] = self.credentials.token
        new_hmac = hmac.new(self.credentials.secret_key.encode('utf-8'),
                            digestmod=sha256)
        new_hmac.update(request.headers['Date'].encode('utf-8'))
        encoded_signature = base64.encodestring(new_hmac.digest()).strip()
        signature = ('AWS3-HTTPS AWSAccessKeyId=%s,Algorithm=%s,Signature=%s' %
                     (self.credentials.access_key, 'HmacSHA256',
                      encoded_signature.decode('utf-8')))
        request.headers['X-Amzn-Authorization'] = signature

class SigV4Auth(object):
    """
    Sign a request with Signature V4.
    """

    def __init__(self, credentials, service_name=None, region_name=None):
        self.credentials = credentials
        self.now = datetime.datetime.utcnow()
        self.timestamp = self.now.strftime('%Y%m%dT%H%M%SZ')
        self.region_name = region_name
        self.service_name = service_name

    def _sign(self, key, msg, hex=False):
        if hex:
            sig = hmac.new(key, msg.encode('utf-8'), sha256).hexdigest()
        else:
            sig = hmac.new(key, msg.encode('utf-8'), sha256).digest()
        return sig

    def headers_to_sign(self, request):
        """
        Select the headers from the request that need to be included
        in the StringToSign.
        """
        headers_to_sign = {}
        split = urlsplit(request.url)
        headers_to_sign = {'Host': split.netloc}
        for name, value in request.headers.items():
            lname = name.lower()
            if lname.startswith('x-amz'):
                headers_to_sign[name] = value
            elif lname == 'content-type':
                headers_to_sign[name] = value
        return headers_to_sign

    def canonical_query_string(self, request):
        cqs = ''
        if request.method == 'GET':
            params = dict(parse_qsl(urlsplit(request.url).query))
            l = []
            for param in params:
                value = str(params[param])
                l.append('%s=%s' % (quote(param, safe='-_.~'),
                                    quote(value, safe='-_.~')))
            l = sorted(l)
            cqs = '&'.join(l)
        return cqs

    def canonical_headers(self, headers_to_sign):
        """
        Return the headers that need to be included in the StringToSign
        in their canonical form by converting all header keys to lower
        case, sorting them in alphabetical order and then joining
        them into a string, separated by newlines.
        """
        l = ['%s:%s' % (n.lower().strip(),
                      headers_to_sign[n].strip()) for n in headers_to_sign]
        l = sorted(l)
        return '\n'.join(l)

    def signed_headers(self, headers_to_sign):
        l = ['%s' % n.lower().strip() for n in headers_to_sign]
        l = sorted(l)
        return ';'.join(l)

    def payload(self, request):
        if request.body:
            return sha256(request.body).hexdigest()
        else:
            return sha256('').hexdigest()

    def canonical_request(self, request):
        cr = [request.method.upper()]
        path = urlsplit(request.url).path
        cr.append(path)
        cr.append(self.canonical_query_string(request))
        headers_to_sign = self.headers_to_sign(request)
        cr.append(self.canonical_headers(headers_to_sign) + '\n')
        cr.append(self.signed_headers(headers_to_sign))
        cr.append(self.payload(request))
        return '\n'.join(cr)

    def scope(self, args):
        scope = [self.credentials.access_key]
        scope.append(self.timestamp[0:8])
        scope.append(self.region_name)
        scope.append(self.service_name)
        scope.append('aws4_request')
        return '/'.join(scope)

    def credential_scope(self, args):
        scope = []
        scope.append(self.timestamp[0:8])
        scope.append(self.region_name)
        scope.append(self.service_name)
        scope.append('aws4_request')
        return '/'.join(scope)

    def string_to_sign(self, request, canonical_request):
        """
        Return the canonical StringToSign as well as a dict
        containing the original version of all headers that
        were included in the StringToSign.
        """
        sts = ['AWS4-HMAC-SHA256']
        sts.append(request.headers['X-Amz-Date'])
        sts.append(self.credential_scope(request))
        sts.append(sha256(canonical_request).hexdigest())
        return '\n'.join(sts)

    def signature(self, string_to_sign):
        key = self.credentials.secret_key
        k_date = self._sign(('AWS4' + key).encode('utf-8'),
                            self.timestamp[0:8])
        k_region = self._sign(k_date, self.region_name)
        k_service = self._sign(k_region, self.service_name)
        k_signing = self._sign(k_service, 'aws4_request')
        return self._sign(k_signing, string_to_sign, hex=True)

    def add_auth(self, request):
        # This could be a retry.  Make sure the previous
        # authorization header is removed first.
        split = urlsplit(request.url)
        if 'Authorization' in request.headers:
            del request.headers['Authorization']
        request.headers['X-Amz-Date'] = self.timestamp
        if self.credentials.token:
            request.headers['X-Amz-Security-Token'] = self.credentials.token
        canonical_request = self.canonical_request(request)
        logger.debug('CanonicalRequest:\n%s' % canonical_request)
        string_to_sign = self.string_to_sign(request, canonical_request)
        logger.debug('StringToSign:\n%s' % string_to_sign)
        signature = self.signature(string_to_sign)
        logger.debug('Signature:\n%s' % signature)
        l = ['AWS4-HMAC-SHA256 Credential=%s' % self.scope(request)]
        headers_to_sign = self.headers_to_sign(request)
        l.append('SignedHeaders=%s' % self.signed_headers(headers_to_sign))
        l.append('Signature=%s' % signature)
        request.headers['Authorization'] = ','.join(l)
        return request

class HmacV1Auth(object):

    # List of Query String Arguments of Interest
    QSAOfInterest = ['acl', 'cors', 'defaultObjectAcl', 'location', 'logging',
                     'partNumber', 'policy', 'requestPayment', 'torrent',
                     'versioning', 'versionId', 'versions', 'website',
                     'uploads', 'uploadId', 'response-content-type',
                     'response-content-language', 'response-expires',
                     'response-cache-control', 'response-content-disposition',
                     'response-content-encoding', 'delete', 'lifecycle',
                     'tagging', 'restore', 'storageClass']

    def __init__(self, credentials, service_name=None, region_name=None):
        self.credentials = credentials
        self.service_name = service_name
        self.region_name = region_name

    def sign_string(self, string_to_sign):
        new_hmac = hmac.new(self.credentials.secret_key.encode('utf-8'),
                            digestmod=sha1)
        new_hmac.update(string_to_sign.encode('utf-8'))
        return base64.encodestring(new_hmac.digest()).strip()

    def canonical_standard_headers(self, headers):
        interesting_headers = ['content-md5', 'content-type', 'date']
        hoi = []
        if 'Date' not in headers:
            headers['Date'] = formatdate(usegmt=True)
        for ih in interesting_headers:
            found = False
            for key in headers:
                lk = key.lower()
                if headers[key] != None and lk == ih:
                    hoi.append(headers[key].strip())
                    found = True
            if not found:
                hoi.append('')
        return '\n'.join(hoi)

    def canonical_custom_headers(self, headers):
        custom_headers = {}
        for key in headers:
            lk = key.lower()
            if headers[key] != None:
                # TODO: move hardcoded prefix to provider
                if lk.startswith('x-amz-'):
                    custom_headers[lk] = headers[key].strip()
        sorted_header_keys = sorted(custom_headers.keys())
        hoi = []
        for key in sorted_header_keys:
            hoi.append("%s:%s" % (key, custom_headers[key]))
        return '\n'.join(hoi)

    def unquote_v(self, nv):
        """
        TODO: Do we need this?
        """
        if len(nv) == 1:
            return nv
        else:
            return (nv[0], unquote(nv[1]))

    def canonical_resource(self, split):
        # don't include anything after the first ? in the resource...
        # unless it is one of the QSA of interest, defined above
        buf = split.path

        if split.query:
            qsa = split.query.split('&')
            qsa = [a.split('=', 1) for a in qsa]
            qsa = [self.unquote_v(a) for a in qsa if a[0] in self.QSAOfInterest]
            if len(qsa) > 0:
                qsa.sort(cmp=lambda x, y:cmp(x[0], y[0]))
                qsa = ['='.join(a) for a in qsa]
                buf += '?'
                buf += '&'.join(qsa)

        return buf

    def canonical_string(self, method, split, headers, expires=None):
        cs = method.upper() + '\n'
        cs += self.canonical_standard_headers(headers) + '\n'
        custom_headers = self.canonical_custom_headers(headers)
        if custom_headers:
            cs += custom_headers + '\n'
        cs += self.canonical_resource(split)
        return cs

    def get_signature(self, method, split, headers, expires=None):
        if self.credentials.token:
            #TODO: remove hardcoded header name
            headers['security_token'] = self.credentials.token
        string_to_sign = self.canonical_string(method,
                                               split,
                                               headers)
        logger.debug('StringToSign:\n%s' % string_to_sign)
        return self.sign_string(string_to_sign)

    def add_auth(self, request):
        split = urlsplit(request.url)
        signature = self.get_signature(request.method, split,
                                       request.headers)
        request.headers['Authorization'] = ("AWS %s:%s" % (self.credentials.access_key,
                                                           signature))

def get_auth(signature_version, *args, **kw):
    if signature_version == 'v2':
        return SigV2Auth(*args, **kw)
    elif signature_version == 'v4':
        return SigV4Auth(*args, **kw)
    elif signature_version == 's3':
        return HmacV1Auth(*args, **kw)
    elif signature_version == 'v3https':
        return SigV3Auth(*args, **kw)
    raise UnknownSignatureVersionError(signature_version=signature_version)
<commit_msg>py3 bug, encode strings before hashing<commit_after># Copyright (c) 2012-2013 Mitch Garnaat http://garnaat.org/
# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish, dis-
# tribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the fol-
# lowing conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#
import base64
import datetime
from hashlib import sha256
from hashlib import sha1
import hmac
import logging
from email.utils import formatdate
from botocore.exceptions import UnknownSignatureVersionError

logger = logging.getLogger(__name__)

try:
    from urllib.parse import quote
    from urllib.parse import unquote
    from urllib.parse import urlsplit
    from urllib.parse import parse_qsl
except ImportError:
    from urllib import quote
    from urllib import unquote
    from urlparse import urlsplit
    from urlparse import parse_qsl

PostContentType = 'application/x-www-form-urlencoded; charset=UTF-8'

class SigV2Auth(object):
    """
    Sign a request with Signature V2.
    """
    def __init__(self, credentials, service_name=None, region_name=None):
        self.credentials = credentials
        self.service_name = service_name
        self.region_name = region_name

    def calc_signature(self, request, params):
        split = urlsplit(request.url)
        path = split.path
        if len(path) == 0:
            path = '/'
        string_to_sign = '%s\n%s\n%s\n' % (request.method,
                                           split.netloc,
                                           path)
        lhmac = hmac.new(self.credentials.secret_key.encode('utf-8'),
                         digestmod=sha256)
        pairs = []
        for key in sorted(params):
            value = params[key]
            pairs.append(quote(key, safe='') + '=' +
                         quote(value, safe='-_~'))
        qs = '&'.join(pairs)
        string_to_sign += qs
        logger.debug('string_to_sign')
        logger.debug(string_to_sign)
        lhmac.update(string_to_sign.encode('utf-8'))
        b64 = base64.b64encode(lhmac.digest()).strip().decode('utf-8')
        return (qs, b64)

    def add_auth(self, request):
        # The auth handler is the last thing called in the
        # preparation phase of a prepared request.
        # Because of this we have to parse the query params
        # from the request body so we can update them with
        # the sigv2 auth params.
        if request.data:
            # POST
            params = request.data
        else:
            # GET
            params = request.param
        params['AWSAccessKeyId'] = self.credentials.access_key
        params['SignatureVersion'] = '2'
        params['SignatureMethod'] = 'HmacSHA256'
        params['Timestamp'] = datetime.datetime.utcnow().isoformat()
        if self.credentials.token:
            params['SecurityToken'] = self.credentials.token
        qs, signature = self.calc_signature(request, params)
        params['Signature'] = signature
        return request

class SigV3Auth(object):
    def __init__(self, credentials, service_name=None, region_name=None):
        self.credentials = credentials
        self.service_name = service_name
        self.region_name = region_name

    def add_auth(self, request):
        if 'Date' not in request.headers:
            request.headers['Date'] = formatdate(usegmt=True)
        if self.credentials.token:
            request.headers['X-Amz-Security-Token'] = self.credentials.token
        new_hmac = hmac.new(self.credentials.secret_key.encode('utf-8'),
                            digestmod=sha256)
        new_hmac.update(request.headers['Date'].encode('utf-8'))
        encoded_signature = base64.encodestring(new_hmac.digest()).strip()
        signature = ('AWS3-HTTPS AWSAccessKeyId=%s,Algorithm=%s,Signature=%s' %
                     (self.credentials.access_key, 'HmacSHA256',
                      encoded_signature.decode('utf-8')))
        request.headers['X-Amzn-Authorization'] = signature

class SigV4Auth(object):
    """
    Sign a request with Signature V4.
    """

    def __init__(self, credentials, service_name=None, region_name=None):
        self.credentials = credentials
        self.now = datetime.datetime.utcnow()
        self.timestamp = self.now.strftime('%Y%m%dT%H%M%SZ')
        self.region_name = region_name
        self.service_name = service_name

    def _sign(self, key, msg, hex=False):
        if hex:
            sig = hmac.new(key, msg.encode('utf-8'), sha256).hexdigest()
        else:
            sig = hmac.new(key, msg.encode('utf-8'), sha256).digest()
        return sig

    def headers_to_sign(self, request):
        """
        Select the headers from the request that need to be included
        in the StringToSign.
        """
        headers_to_sign = {}
        split = urlsplit(request.url)
        headers_to_sign = {'Host': split.netloc}
        for name, value in request.headers.items():
            lname = name.lower()
            if lname.startswith('x-amz'):
                headers_to_sign[name] = value
            elif lname == 'content-type':
                headers_to_sign[name] = value
        return headers_to_sign

    def canonical_query_string(self, request):
        cqs = ''
        if request.method == 'GET':
            params = dict(parse_qsl(urlsplit(request.url).query))
            l = []
            for param in params:
                value = str(params[param])
                l.append('%s=%s' % (quote(param, safe='-_.~'),
                                    quote(value, safe='-_.~')))
            l = sorted(l)
            cqs = '&'.join(l)
        return cqs

    def canonical_headers(self, headers_to_sign):
        """
        Return the headers that need to be included in the StringToSign
        in their canonical form by converting all header keys to lower
        case, sorting them in alphabetical order and then joining
        them into a string, separated by newlines.
        """
        l = ['%s:%s' % (n.lower().strip(),
                      headers_to_sign[n].strip()) for n in headers_to_sign]
        l = sorted(l)
        return '\n'.join(l)

    def signed_headers(self, headers_to_sign):
        l = ['%s' % n.lower().strip() for n in headers_to_sign]
        l = sorted(l)
        return ';'.join(l)

    def payload(self, request):
        if request.body:
            return sha256(request.body.encode('utf-8')).hexdigest()
        else:
            return sha256('').hexdigest()

    def canonical_request(self, request):
        cr = [request.method.upper()]
        path = urlsplit(request.url).path
        cr.append(path)
        cr.append(self.canonical_query_string(request))
        headers_to_sign = self.headers_to_sign(request)
        cr.append(self.canonical_headers(headers_to_sign) + '\n')
        cr.append(self.signed_headers(headers_to_sign))
        cr.append(self.payload(request))
        return '\n'.join(cr)

    def scope(self, args):
        scope = [self.credentials.access_key]
        scope.append(self.timestamp[0:8])
        scope.append(self.region_name)
        scope.append(self.service_name)
        scope.append('aws4_request')
        return '/'.join(scope)

    def credential_scope(self, args):
        scope = []
        scope.append(self.timestamp[0:8])
        scope.append(self.region_name)
        scope.append(self.service_name)
        scope.append('aws4_request')
        return '/'.join(scope)

    def string_to_sign(self, request, canonical_request):
        """
        Return the canonical StringToSign as well as a dict
        containing the original version of all headers that
        were included in the StringToSign.
        """
        sts = ['AWS4-HMAC-SHA256']
        sts.append(request.headers['X-Amz-Date'])
        sts.append(self.credential_scope(request))
        sts.append(sha256(canonical_request.encode('utf-8')).hexdigest())
        return '\n'.join(sts)

    def signature(self, string_to_sign):
        key = self.credentials.secret_key
        k_date = self._sign(('AWS4' + key).encode('utf-8'),
                            self.timestamp[0:8])
        k_region = self._sign(k_date, self.region_name)
        k_service = self._sign(k_region, self.service_name)
        k_signing = self._sign(k_service, 'aws4_request')
        return self._sign(k_signing, string_to_sign, hex=True)

    def add_auth(self, request):
        # This could be a retry.  Make sure the previous
        # authorization header is removed first.
        split = urlsplit(request.url)
        if 'Authorization' in request.headers:
            del request.headers['Authorization']
        request.headers['X-Amz-Date'] = self.timestamp
        if self.credentials.token:
            request.headers['X-Amz-Security-Token'] = self.credentials.token
        canonical_request = self.canonical_request(request)
        logger.debug('CanonicalRequest:\n%s' % canonical_request)
        string_to_sign = self.string_to_sign(request, canonical_request)
        logger.debug('StringToSign:\n%s' % string_to_sign)
        signature = self.signature(string_to_sign)
        logger.debug('Signature:\n%s' % signature)
        l = ['AWS4-HMAC-SHA256 Credential=%s' % self.scope(request)]
        headers_to_sign = self.headers_to_sign(request)
        l.append('SignedHeaders=%s' % self.signed_headers(headers_to_sign))
        l.append('Signature=%s' % signature)
        request.headers['Authorization'] = ','.join(l)
        return request

class HmacV1Auth(object):

    # List of Query String Arguments of Interest
    QSAOfInterest = ['acl', 'cors', 'defaultObjectAcl', 'location', 'logging',
                     'partNumber', 'policy', 'requestPayment', 'torrent',
                     'versioning', 'versionId', 'versions', 'website',
                     'uploads', 'uploadId', 'response-content-type',
                     'response-content-language', 'response-expires',
                     'response-cache-control', 'response-content-disposition',
                     'response-content-encoding', 'delete', 'lifecycle',
                     'tagging', 'restore', 'storageClass']

    def __init__(self, credentials, service_name=None, region_name=None):
        self.credentials = credentials
        self.service_name = service_name
        self.region_name = region_name

    def sign_string(self, string_to_sign):
        new_hmac = hmac.new(self.credentials.secret_key.encode('utf-8'),
                            digestmod=sha1)
        new_hmac.update(string_to_sign.encode('utf-8'))
        return base64.encodestring(new_hmac.digest()).strip()

    def canonical_standard_headers(self, headers):
        interesting_headers = ['content-md5', 'content-type', 'date']
        hoi = []
        if 'Date' not in headers:
            headers['Date'] = formatdate(usegmt=True)
        for ih in interesting_headers:
            found = False
            for key in headers:
                lk = key.lower()
                if headers[key] != None and lk == ih:
                    hoi.append(headers[key].strip())
                    found = True
            if not found:
                hoi.append('')
        return '\n'.join(hoi)

    def canonical_custom_headers(self, headers):
        custom_headers = {}
        for key in headers:
            lk = key.lower()
            if headers[key] != None:
                # TODO: move hardcoded prefix to provider
                if lk.startswith('x-amz-'):
                    custom_headers[lk] = headers[key].strip()
        sorted_header_keys = sorted(custom_headers.keys())
        hoi = []
        for key in sorted_header_keys:
            hoi.append("%s:%s" % (key, custom_headers[key]))
        return '\n'.join(hoi)

    def unquote_v(self, nv):
        """
        TODO: Do we need this?
        """
        if len(nv) == 1:
            return nv
        else:
            return (nv[0], unquote(nv[1]))

    def canonical_resource(self, split):
        # don't include anything after the first ? in the resource...
        # unless it is one of the QSA of interest, defined above
        buf = split.path

        if split.query:
            qsa = split.query.split('&')
            qsa = [a.split('=', 1) for a in qsa]
            qsa = [self.unquote_v(a) for a in qsa if a[0] in self.QSAOfInterest]
            if len(qsa) > 0:
                qsa.sort(cmp=lambda x, y:cmp(x[0], y[0]))
                qsa = ['='.join(a) for a in qsa]
                buf += '?'
                buf += '&'.join(qsa)

        return buf

    def canonical_string(self, method, split, headers, expires=None):
        cs = method.upper() + '\n'
        cs += self.canonical_standard_headers(headers) + '\n'
        custom_headers = self.canonical_custom_headers(headers)
        if custom_headers:
            cs += custom_headers + '\n'
        cs += self.canonical_resource(split)
        return cs

    def get_signature(self, method, split, headers, expires=None):
        if self.credentials.token:
            #TODO: remove hardcoded header name
            headers['security_token'] = self.credentials.token
        string_to_sign = self.canonical_string(method,
                                               split,
                                               headers)
        logger.debug('StringToSign:\n%s' % string_to_sign)
        return self.sign_string(string_to_sign)

    def add_auth(self, request):
        split = urlsplit(request.url)
        signature = self.get_signature(request.method, split,
                                       request.headers)
        request.headers['Authorization'] = ("AWS %s:%s" % (self.credentials.access_key,
                                                           signature))

def get_auth(signature_version, *args, **kw):
    if signature_version == 'v2':
        return SigV2Auth(*args, **kw)
    elif signature_version == 'v4':
        return SigV4Auth(*args, **kw)
    elif signature_version == 's3':
        return HmacV1Auth(*args, **kw)
    elif signature_version == 'v3https':
        return SigV3Auth(*args, **kw)
    raise UnknownSignatureVersionError(signature_version=signature_version)
<|endoftext|>


<commit_before>import functools

__version__ = "0.1"
__all__ = "Plugin Injector inject".split()

import inspect
import sys

py32 = sys.version_info >= (3, 2, 0)

class InjectError(RuntimeError):
    pass

class _InjectionPoint(object):
    """ The object returned by :func:`inject`. """

    def __init__(self, name, parameters=None, config=None):
        self.name = name
        self.parameters = parameters or ()
        self.config = config or {}

    def __eq__(self, other):
        if isinstance(other, _InjectionPoint):
            return self.__dict__ == other.__dict__
        return False

class _ProviderCache(dict):
    """ A self-filling cache for :meth:`Injector.resolve` results. """

    def __init__(self, injector):
        super(_ProviderCache, self).__init__()
        self.injector = injector

    def __missing__(self, func):
        self[func] = value = list(self.injector._resolve(func).items())
        return value

def _unwrap(func):
    if inspect.isclass(func):
        func = func.__init__
    while hasattr(func, '__wrapped__'):
        func = func.__wrapped__
    return func

def _make_null_resolver(name, provider):
    msg = "The dependency provider for %r does not accept configuration (it is not a resolver)." % name
    def null_resolver(*a, **ka):
        if a or ka:
            raise InjectError(msg)
        return provider
    return null_resolver

class Injector(object):
    def __init__(self):
        self.__cache = _ProviderCache(self)
        self._resolvers = {}
        self._never_inject = set('self')

    def add_value(self, name, value, alias=()):
        """ Register a dependency value.

        The dependency value is re-used for every injection and treated as a singleton.

        :param name: Name of the injection point.
        :param value: The singleton to provide.
        :param alias: A list of alternative injection points.
        :return: None
        """
        self.add_provider(name, lambda: value, alias=alias)

    def add_provider(self, name, func, alias=()):
        """ Register a dependency provider.

        A *provider* returns the requested dependency when called. The provider is called with no arguments
        every time the dependency is needed. It is possible to inject other dependencies into the call signature of a
        provider.

        :param name: Name of the injection point.
        :param func: The provider callable.
        :param alias: A list of alternative injection points.
        :return: None
        """
        self.add_resolver(name, _make_null_resolver(name, func), alias=alias)

    def add_resolver(self, name, func, alias=()):
        """ Register a dependency provider resolver.

        A *resolver* returns a cache-able *provider* and may accept injection-point specific configuration. The resolver
        is usually called only once per injection point and the return value is cached. It must return a (callable)
        provider. It is possible to inject other dependencies into the call signature of a resolver.

        :param name: Name of the injection point.
        :param func: The resolver callable.
        :param alias: A list of alternative injection points.
        :return: None
        """
        self._resolvers[name] = func
        for name in alias:
            assert isinstance(name, str)
            self._resolvers[name] = func
        self.__cache.clear()

    def remove(self, name):
        """ Remove any dependency, provider or resolver bound to the named injection point.
        :param name: Name of the injection point to clear.
        :return: None
        """
        if self._resolvers.pop(name):
            self.__cache.clear()

    def provider(self, name, alias=()):
        """ Decorator to register a dependency provider. See :func:`add_provider` for a description.
        :param name: Name of the injection point.
        :param alias: A list of alias names for this injection point.
        :return: Decorator that registers the provider function to the injector.
        """
        assert isinstance(name, str)

        def decorator(func):
            self.add_provider(name, func, alias=alias)
            return func

        return decorator

    def resolver(self, name, alias=()):
        """ Decorator to register a dependency provider resolver. See :func:`add_resolver` for a description.
        :param name: Name of the injection point.
        :param alias: A list of alias names for this injection point.
        :return: Decorator that registers the resolver to the injector.
        """
        assert isinstance(name, str)

        def decorator(func):
            self.add_resolver(name, func, alias=alias)
            return func

        return decorator

    def inspect(self, func):
        """ Return a dict that maps parameter names to injection points for the provided callable. """
        func = _unwrap(func)

        if py32:
            args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations = inspect.getfullargspec(func)
        else:
            args, varargs, keywords, defaults = inspect.getargspec(func)
            kwonlyargs, kwonlydefaults, annotations = [], {}, {}

        defaults = defaults or ()

        injection_points = {}

        for arg in args[:len(args) - len(defaults or [])]:
            if arg not in self._never_inject:
                injection_points[arg] = _InjectionPoint(arg)

        for arg, value in zip(args[::-1], defaults[::-1]):
            if isinstance(value, _InjectionPoint):
                injection_points[arg] = value

        for arg, value in kwonlydefaults.items():
            if isinstance(value, _InjectionPoint):
                injection_points[arg] = value

        for arg, value in annotations.items():
            if isinstance(value, _InjectionPoint):
                injection_points[arg] = value

        return injection_points

    def _resolve(self, func):
        """ Given a callable, return a dict that maps argument names to provider callables. The providers are
            resolved and wrapped already and should be called with no arguments to receive the injectable.

            This is called by __ProviderCache.__missing__ and should not be used in other situations.
        """
        results = {}
        for arg, ip in self.inspect(func).items():
            results[arg] = self._prime(ip.name, ip.parameters, ip.config)
        return results

    def _prime(self, name, parameters=None, config=None):
        """ Prepare a named resolver with the specified parameters and configuration.

            Internal use only. See _resolve()
        """
        try:
            provider_resolver = self._resolvers[name]
        except KeyError:
            raise InjectError("Could not resolve provider for injection point %r" % name)

        provider = self.call_inject(provider_resolver, *parameters, **config)
        return self.wrap(provider)

    def call_inject(self, func, *a, **ka):
        """ Call a function and inject missing dependencies. If you want to call the same function multiple times,
            consider :method:`wrap`ing it.
        """
        for key, producer in self.__cache[func]:
            if key not in ka:
                ka[key] = producer()
        return func(*a, **ka)

    def wrap(self, func):
        """ Turn a function into a dependency managed callable.

        Usage::
            @injector.wrap
            def my_func(db: inject('database')):
                pass

        or::
            managed_callable = injector.wrap(my_callable)

        :param func: A callable with at least one injectable parameter.
        :return: A wrapped function that calls :method:`call_inject` internally.

        If the provided function does not accept any injectable parameters, it is returned unchanged.
        """
        cache = self.__cache  # Avoid dot lookup in hot path

        # Skip wrapping for functions with no injection points
        if not self.inspect(func):
            return func

        @functools.wraps(func)
        def wrapper(*a, **ka):
            # PERF: Inlined call_inject call. Keep in sync with the implementation above.
            for key, producer in cache[func]:
                if key not in ka:
                    ka[key] = producer()
            return func(*a, **ka)

        return wrapper

class Plugin(Injector):
    api = 2

    def __init__(self):
        super(Plugin, self).__init__()
        self.app = None

    def setup(self, app):
        from bottle import request, response

        self.app = app
        self.add_value('injector', self)
        self.add_value('config', app.config)
        self.add_value('app', app)
        self.add_value('request', request, alias=['req', 'rq'])
        self.add_value('response', response, alias=['res', 'rs'])

    def apply(self, callback, route):
        if self.inspect(callback):
            return self.wrap(callback)
        return callback

def inject(name, *args, **kwargs):
    """ Mark an argument in a function signature as an injection point.

        The return value can be used as an annotation (Python 3) or default value (Python 2) for parameters that should
        be replaced by dependency injection.

    Usage::
        def my_func(a: inject('name'),
                    b: inject('name', 'opt', conf="val") # Resolvers only.
            ):
            pass

    :param name: Name of the dependency to inject.
    :param args: Additional arguments passed to the dependency provider resolver.
    :param kwargs: Additional keyword arguments passed to the dependency provider resolver.
    :return:
    """
    return _InjectionPoint(name, args, kwargs)

<commit_msg>Store injector reference to wrapped callables.<commit_after>import functools

__version__ = "0.1"
__all__ = "Plugin Injector inject".split()

import inspect
import sys

py32 = sys.version_info >= (3, 2, 0)

class InjectError(RuntimeError):
    pass

class _InjectionPoint(object):
    """ The object returned by :func:`inject`. """

    def __init__(self, name, parameters=None, config=None):
        self.name = name
        self.parameters = parameters or ()
        self.config = config or {}

    def __eq__(self, other):
        if isinstance(other, _InjectionPoint):
            return self.__dict__ == other.__dict__
        return False

class _ProviderCache(dict):
    """ A self-filling cache for :meth:`Injector.resolve` results. """

    def __init__(self, injector):
        super(_ProviderCache, self).__init__()
        self.injector = injector

    def __missing__(self, func):
        self[func] = value = list(self.injector._resolve(func).items())
        return value

def _unwrap(func):
    if inspect.isclass(func):
        func = func.__init__
    while hasattr(func, '__wrapped__'):
        func = func.__wrapped__
    return func

def _make_null_resolver(name, provider):
    msg = "The dependency provider for %r does not accept configuration (it is not a resolver)." % name
    def null_resolver(*a, **ka):
        if a or ka:
            raise InjectError(msg)
        return provider
    return null_resolver

class Injector(object):
    def __init__(self):
        self.__cache = _ProviderCache(self)
        self._resolvers = {}
        self._never_inject = set('self')

    def add_value(self, name, value, alias=()):
        """ Register a dependency value.

        The dependency value is re-used for every injection and treated as a singleton.

        :param name: Name of the injection point.
        :param value: The singleton to provide.
        :param alias: A list of alternative injection points.
        :return: None
        """
        self.add_provider(name, lambda: value, alias=alias)

    def add_provider(self, name, func, alias=()):
        """ Register a dependency provider.

        A *provider* returns the requested dependency when called. The provider is called with no arguments
        every time the dependency is needed. It is possible to inject other dependencies into the call signature of a
        provider.

        :param name: Name of the injection point.
        :param func: The provider callable.
        :param alias: A list of alternative injection points.
        :return: None
        """
        self.add_resolver(name, _make_null_resolver(name, func), alias=alias)

    def add_resolver(self, name, func, alias=()):
        """ Register a dependency provider resolver.

        A *resolver* returns a cache-able *provider* and may accept injection-point specific configuration. The resolver
        is usually called only once per injection point and the return value is cached. It must return a (callable)
        provider. It is possible to inject other dependencies into the call signature of a resolver.

        :param name: Name of the injection point.
        :param func: The resolver callable.
        :param alias: A list of alternative injection points.
        :return: None
        """
        self._resolvers[name] = func
        for name in alias:
            assert isinstance(name, str)
            self._resolvers[name] = func
        self.__cache.clear()

    def remove(self, name):
        """ Remove any dependency, provider or resolver bound to the named injection point.
        :param name: Name of the injection point to clear.
        :return: None
        """
        if self._resolvers.pop(name):
            self.__cache.clear()

    def provider(self, name, alias=()):
        """ Decorator to register a dependency provider. See :func:`add_provider` for a description.
        :param name: Name of the injection point.
        :param alias: A list of alias names for this injection point.
        :return: Decorator that registers the provider function to the injector.
        """
        assert isinstance(name, str)

        def decorator(func):
            self.add_provider(name, func, alias=alias)
            return func

        return decorator

    def resolver(self, name, alias=()):
        """ Decorator to register a dependency provider resolver. See :func:`add_resolver` for a description.
        :param name: Name of the injection point.
        :param alias: A list of alias names for this injection point.
        :return: Decorator that registers the resolver to the injector.
        """
        assert isinstance(name, str)

        def decorator(func):
            self.add_resolver(name, func, alias=alias)
            return func

        return decorator

    def inspect(self, func):
        """ Return a dict that maps parameter names to injection points for the provided callable. """
        func = _unwrap(func)

        if py32:
            args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations = inspect.getfullargspec(func)
        else:
            args, varargs, keywords, defaults = inspect.getargspec(func)
            kwonlyargs, kwonlydefaults, annotations = [], {}, {}

        defaults = defaults or ()

        injection_points = {}

        for arg in args[:len(args) - len(defaults or [])]:
            if arg not in self._never_inject:
                injection_points[arg] = _InjectionPoint(arg)

        for arg, value in zip(args[::-1], defaults[::-1]):
            if isinstance(value, _InjectionPoint):
                injection_points[arg] = value

        for arg, value in kwonlydefaults.items():
            if isinstance(value, _InjectionPoint):
                injection_points[arg] = value

        for arg, value in annotations.items():
            if isinstance(value, _InjectionPoint):
                injection_points[arg] = value

        return injection_points

    def _resolve(self, func):
        """ Given a callable, return a dict that maps argument names to provider callables. The providers are
            resolved and wrapped already and should be called with no arguments to receive the injectable.

            This is called by __ProviderCache.__missing__ and should not be used in other situations.
        """
        results = {}
        for arg, ip in self.inspect(func).items():
            results[arg] = self._prime(ip.name, ip.parameters, ip.config)
        return results

    def _prime(self, name, parameters=None, config=None):
        """ Prepare a named resolver with the specified parameters and configuration.

            Internal use only. See _resolve()
        """
        try:
            provider_resolver = self._resolvers[name]
        except KeyError:
            raise InjectError("Could not resolve provider for injection point %r" % name)

        provider = self.call_inject(provider_resolver, *parameters, **config)
        return self.wrap(provider)

    def call_inject(self, func, *a, **ka):
        """ Call a function and inject missing dependencies. If you want to call the same function multiple times,
            consider :method:`wrap`ing it.
        """
        for key, producer in self.__cache[func]:
            if key not in ka:
                ka[key] = producer()
        return func(*a, **ka)

    def wrap(self, func):
        """ Turn a function into a dependency managed callable.

        Usage::
            @injector.wrap
            def my_func(db: inject('database')):
                pass

        or::
            managed_callable = injector.wrap(my_callable)

        :param func: A callable with at least one injectable parameter.
        :return: A wrapped function that calls :method:`call_inject` internally.

        If the provided function does not accept any injectable parameters, it is returned unchanged.
        """
        cache = self.__cache  # Avoid dot lookup in hot path

        # Skip wrapping for functions with no injection points
        if not self.inspect(func):
            return func

        @functools.wraps(func)
        def wrapper(*a, **ka):
            # PERF: Inlined call_inject call. Keep in sync with the implementation above.
            for key, producer in cache[func]:
                if key not in ka:
                    ka[key] = producer()
            return func(*a, **ka)

        wrapper.__injector__ = self
        return wrapper

class Plugin(Injector):
    api = 2

    def __init__(self):
        super(Plugin, self).__init__()
        self.app = None

    def setup(self, app):
        from bottle import request, response

        self.app = app
        self.add_value('injector', self)
        self.add_value('config', app.config)
        self.add_value('app', app)
        self.add_value('request', request, alias=['req', 'rq'])
        self.add_value('response', response, alias=['res', 'rs'])

    def apply(self, callback, route):
        if self.inspect(callback):
            return self.wrap(callback)
        return callback

def inject(name, *args, **kwargs):
    """ Mark an argument in a function signature as an injection point.

        The return value can be used as an annotation (Python 3) or default value (Python 2) for parameters that should
        be replaced by dependency injection.

    Usage::
        def my_func(a: inject('name'),
                    b: inject('name', 'opt', conf="val") # Resolvers only.
            ):
            pass

    :param name: Name of the dependency to inject.
    :param args: Additional arguments passed to the dependency provider resolver.
    :param kwargs: Additional keyword arguments passed to the dependency provider resolver.
    :return:
    """
    return _InjectionPoint(name, args, kwargs)

<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf8 -*-
################################################################################
##
## Copyright (C) 2012 Typhos
##
## This Source Code Form is subject to the terms of the Mozilla Public
## License, v. 2.0. If a copy of the MPL was not distributed with this
## file, You can obtain one at http://mozilla.org/MPL/2.0/.
##
################################################################################

import bplib
import bplib.json

class Context(object):
    def __init__(self):
        self.config = None
        self.tag_config = None
        self.sources = {}
        self.drops = {}
        self.next_source_id = 0

    def load_config(self):
        self.config = bplib.load_yaml_file(open("data/rules.yaml"))
        self.tag_config = bplib.load_yaml_file(open("data/tags.yaml"))

    def load_sources(self):
        for sr_key in self.config["Subreddits"]:
            self.load_subreddit(sr_key)

    def load_subreddit(self, sr_key):
        emotes_filename = "emotes/" + sr_key + ".json"
        tag_filename = "tags/" + sr_key + ".json"

        try:
            emote_data = bplib.json.load(open(emotes_filename))
        except IOError:
            return None # Doesn't exist or has no emotes

        try:
            tag_data = bplib.json.load(open(tag_filename))
        except IOError:
            tag_data = {} # No tags is fine

        source_name = "r/" + sr_key
        sr = Source.load(source_name, emote_data, tag_data, context=self)
        sr.source_id = self.next_source_id
        self.next_source_id += 1
        self.sources[source_name] = sr
        return sr

class Source(object):
    def __init__(self, name, emotes, context=None):
        self.name = name
        self.emotes = emotes
        self.context = context

        self.variant_matches = None
        self.emote_groups = None
        self._tag_data = {} # For checktags

    def merge(self, emote_data, tag_data):
        for (name, data) in emote_data.items():
            tags = set(tag_data.get(name, []))
            self.emotes[name] = Emote.load(name, data, tags, source=self, context=self.context)

            if self.context:
                if "+drop" in tags:
                    if name in self.context.drops:
                        print("Source.load_from_data(): ERROR: %s in %s is marked as +drop, but %s has already claimed the name" %
                                (name, self.name, self.context.drops[name].name))
                    self.context.drops[name] = self
        self._tag_data.update(tag_data)

    @classmethod
    def load(cls, name, emote_data, tag_data, context=None):
        source = cls(name, {}, context=context)
        source.merge(emote_data, tag_data)
        return source

    def dump(self):
        return {name: emote.dump() for (name, emote) in self.emotes.items()}

    def dump_tags(self):
        data = {}
        for (name, emote) in self.emotes.items():
            if emote.tags:
                data[name] = sorted(emote.tags)
        return data

    def dropped_emotes(self):
        # Returns emotes with +drop
        for emote in self.emotes.values():
            if emote.ignore:
                continue
            owner = self.context.drops.get(emote.name, None)
            if owner is not None and owner is not self:
                yield emote

    def undropped_emotes(self):
        # Returns emotes that don't have a +drop applied to them
        for emote in self.emotes.values():
            if emote.ignore:
                continue
            owner = self.context.drops.get(emote.name, None)
            if owner is None or owner is self:
                yield emote

    def is_ignored(self, emote):
        if emote.ignore:
            return True
        if "+remove" in emote.all_tags(self.context):
            return True
        owner = self.context.drops.get(emote.name, None)
        if owner is not None and owner is not self:
            return True
        return False

    def unignored_emotes(self):
        # Returns emotes that actually matter (not removed and not dropped)
        for emote in self.emotes.values():
            if self.is_ignored(emote):
                continue
            yield emote

    def group_emotes(self):
        groups = {}
        for emote in self.unignored_emotes():
            base = emote.base_variant()
            if emote.name[:2].lower() == "/r":
                # Try to follow reversed emotes back to the unreversed name
                r_emote = self.emotes.get("/" + emote.name[2:])
                if r_emote is not None and not self.is_ignored(r_emote):
                    base = r_emote.base_variant()
            if hasattr(base, "info_set"):
                info = base.info_set()
            else:
                info = None # Custom emote
            groups.setdefault(info, []).append(emote)
        self.emote_groups = groups

    def match_variants(self):
        core_emotes, variants = {}, []
        matches, errors = {}, []

        # Locate core emotes. Ignore all variants but the base one.
        for emote in self.unignored_emotes():
            is_core = "+v" not in emote.tags
            if is_core:
                # An emote always matches itself.
                matches[emote] = emote
            base_variant = emote.base_variant()
            if hasattr(base_variant, "info_set"):
                # Uniquely identifying sprite data
                info = base_variant.info_set()
                if is_core:
                    if info in core_emotes:
                        # Only one core emote per sprite, please. Common
                        # exception: hovermotes.
                        if len(emote.variants) == len(core_emotes[info].variants):
                            errors.append("Multiple non-variant emotes (prev %s, new %s)" % (core_emotes[info].name, emote.name))
                    core_emotes[info] = emote
                else:
                    variants.append((emote, info))
            elif not is_core:
                # Custom +v emotes are not permitted
                errors.append("Variant custom emote %s" % (emote.name))
                matches[emote] = emote

        # Match everything up.
        for (emote, info) in variants:
            base_variant = emote.base_variant()
            base = None
            # Easy match- another identical emote exists.
            if info in core_emotes:
                base = core_emotes[info]
            elif emote.name[:2].lower() == "/r":
                # Try non-reversed name?
                unreversed_emote = self.emotes.get("/" + emote.name[2:])
                if unreversed_emote is not None and not self.is_ignored(unreversed_emote):
                    unreversed_base = unreversed_emote.base_variant()
                    if hasattr(unreversed_base, "info_set"):
                        info = unreversed_base.info_set()
                        if info in core_emotes:
                            # Perfect match
                            base = core_emotes[info]
            # Can't find it. Is there an explicit rule?
            match_config = self.context.config["RootVariantEmotes"].get(self.name, {})
            if emote.name in match_config:
                if base is None:
                    base = self.emotes[match_config[emote.name]]
                else:
                    # Would be nice to know
                    errors.append("Extraneous match config for %s" % (emote.name))
            elif base is None:
                errors.append("Cannot locate root emote for %s" % (emote.name))
            matches[emote] = base # Even if None

        self.variant_matches = matches
        return errors

    def __repr__(self):
        return "<Source %s>" % (self.name)

class Emote(object):
    def __init__(self, name, variants, tags, ignore, source=None):
        self.name = name
        self.variants = variants
        self.tags = tags
        self.ignore = ignore
        self.source = source

        # Caches
        self._implied_tags = None
        self._all_tags = None

    @classmethod
    def load(cls, name, data, tags, source=None, context=None):
        variants = {}
        for (suffix, vdata) in data["Emotes"].items():
            type = vdata.pop("Type", "Normal")
            class_ = {"Normal": NormalVariant, "Custom": CustomVariant}[type]
            variant = class_.load(name, suffix, vdata)
            variants[suffix] = variant
        ignore = data.pop("Ignore", False)
        return cls(name, variants, tags, ignore, source=source)

    def dump(self):
        data = {"Emotes": {suffix: variant.dump() for (suffix, variant) in self.variants.items()}}
        if self.ignore:
            data["Ignore"] = self.ignore
        return data

    def base_variant(self):
        if "" in self.variants:
            return self.variants[""]
        elif len(self.variants) == 1:
            key = next(iter(self.variants.keys()))
            ## Should probably suffice
            #if key not in (":after", ":before"):
            #    print("WARNING: Unknown primary base suffix %r" % (key))
            return self.variants[key]
        raise ValueError("Cannot locate base emote for %r" % (self.name))

    def implied_tags(self, context):
        if self._implied_tags is None:
            self._implied_tags = set()
            for tag in self.tags:
                self._implied_tags |= set(context.tag_config["TagImplications"].get(tag, []))
        return self._implied_tags

    def all_tags(self, context):
        if self._all_tags is None:
            self._all_tags = self.implied_tags(context) | self.tags
        return self._all_tags

    def __repr__(self):
        return "<Emote %s>" % (self.name)

class EmoteCSSBlock(object):
    def __init__(self, name, suffix, css, ignore):
        self.name = name
        self.suffix = suffix
        self.css = css
        self.ignore = ignore

class _Variant(object):
    def selector(self):
        name = self.name[1:].replace("!", "_excl_").replace(":", "_colon_").replace("#", "_hash_").replace("/", "_slash_")
        return ".bpmote-" + bplib.combine_name_pair(name, self.suffix).lower()

class CustomVariant(_Variant):
    def __init__(self, name, suffix, css):
        self.name = name
        self.suffix = suffix
        self.css = css

    @classmethod
    def load(cls, name, suffix, data, emote=None):
        return cls(name, suffix, data["CSS"])

    def dump(self):
        return {"Type": "Custom", "CSS": self.css}

    def to_css(self):
        return self.css.copy()

    def __repr__(self):
        if self.suffix:
            return "<CustomVariant %s %s>" % (self.name, self.suffix)
        else:
            return "<CustomVariant %s>" % (self.name)

class NormalVariant(_Variant):
    def __init__(self, name, suffix, image_url, size, offset, css):
        self.name = name
        self.suffix = suffix
        self.image_url = image_url
        self.size = size
        self.offset = offset
        self.css = css

    @classmethod
    def load(cls, name, suffix, data):
        return cls(name, suffix, data["Image"], tuple(data["Size"]), tuple(data.get("Offset", (0, 0))), data.get("CSS", {}))

    def dump(self):
        data = {"Image": self.image_url, "Size": self.size}
        if self.offset != (0, 0):
            data["Offset"] = self.offset
        if self.css:
            data["CSS"] = self.css
        return data

    def info_set(self):
        return (self.image_url, self.size[0], self.size[1], self.offset[0], self.offset[1])

    def to_css(self):
        css = {
            "display": "block",
            "float": "left",
            "background-image": "url(%s)" % (self.image_url),
            "width": "%spx" % (self.size[0]),
            "height": "%spx" % (self.size[1]),
            }
        if self.offset != (0, 0) or self.suffix:
            css["background-position"] = "%spx %spx" % (self.offset[0], self.offset[1])
        css.update(self.css)
        return css

    def __repr__(self):
        if self.suffix:
            return "<NormalEmote %s %s>" % (self.name, self.suffix)
        else:
            return "<NormalEmote %s>" % (self.name)
<commit_msg>Fix group_emotes() to include +remove'ed emotes, which matters to tagapp<commit_after>#!/usr/bin/env python3
# -*- coding: utf8 -*-
################################################################################
##
## Copyright (C) 2012 Typhos
##
## This Source Code Form is subject to the terms of the Mozilla Public
## License, v. 2.0. If a copy of the MPL was not distributed with this
## file, You can obtain one at http://mozilla.org/MPL/2.0/.
##
################################################################################

import bplib
import bplib.json

class Context(object):
    def __init__(self):
        self.config = None
        self.tag_config = None
        self.sources = {}
        self.drops = {}
        self.next_source_id = 0

    def load_config(self):
        self.config = bplib.load_yaml_file(open("data/rules.yaml"))
        self.tag_config = bplib.load_yaml_file(open("data/tags.yaml"))

    def load_sources(self):
        for sr_key in self.config["Subreddits"]:
            self.load_subreddit(sr_key)

    def load_subreddit(self, sr_key):
        emotes_filename = "emotes/" + sr_key + ".json"
        tag_filename = "tags/" + sr_key + ".json"

        try:
            emote_data = bplib.json.load(open(emotes_filename))
        except IOError:
            return None # Doesn't exist or has no emotes

        try:
            tag_data = bplib.json.load(open(tag_filename))
        except IOError:
            tag_data = {} # No tags is fine

        source_name = "r/" + sr_key
        sr = Source.load(source_name, emote_data, tag_data, context=self)
        sr.source_id = self.next_source_id
        self.next_source_id += 1
        self.sources[source_name] = sr
        return sr

class Source(object):
    def __init__(self, name, emotes, context=None):
        self.name = name
        self.emotes = emotes
        self.context = context

        self.variant_matches = None
        self.emote_groups = None
        self._tag_data = {} # For checktags

    def merge(self, emote_data, tag_data):
        for (name, data) in emote_data.items():
            tags = set(tag_data.get(name, []))
            self.emotes[name] = Emote.load(name, data, tags, source=self, context=self.context)

            if self.context:
                if "+drop" in tags:
                    if name in self.context.drops:
                        print("Source.load_from_data(): ERROR: %s in %s is marked as +drop, but %s has already claimed the name" %
                                (name, self.name, self.context.drops[name].name))
                    self.context.drops[name] = self
        self._tag_data.update(tag_data)

    @classmethod
    def load(cls, name, emote_data, tag_data, context=None):
        source = cls(name, {}, context=context)
        source.merge(emote_data, tag_data)
        return source

    def dump(self):
        return {name: emote.dump() for (name, emote) in self.emotes.items()}

    def dump_tags(self):
        data = {}
        for (name, emote) in self.emotes.items():
            if emote.tags:
                data[name] = sorted(emote.tags)
        return data

    def dropped_emotes(self):
        # Returns emotes with +drop
        for emote in self.emotes.values():
            if emote.ignore:
                continue
            owner = self.context.drops.get(emote.name, None)
            if owner is not None and owner is not self:
                yield emote

    def undropped_emotes(self):
        # Returns emotes that don't have a +drop applied to them
        for emote in self.emotes.values():
            if emote.ignore:
                continue
            owner = self.context.drops.get(emote.name, None)
            if owner is None or owner is self:
                yield emote

    def is_ignored(self, emote):
        if emote.ignore:
            return True
        if "+remove" in emote.all_tags(self.context):
            return True
        owner = self.context.drops.get(emote.name, None)
        if owner is not None and owner is not self:
            return True
        return False

    def unignored_emotes(self):
        # Returns emotes that actually matter (not removed and not dropped)
        for emote in self.emotes.values():
            if self.is_ignored(emote):
                continue
            yield emote

    def group_emotes(self):
        groups = {}
        for emote in self.undropped_emotes():
            base = emote.base_variant()
            if emote.name[:2].lower() == "/r":
                # Try to follow reversed emotes back to the unreversed name
                r_emote = self.emotes.get("/" + emote.name[2:])
                if r_emote is not None and not self.is_ignored(r_emote):
                    base = r_emote.base_variant()
            if hasattr(base, "info_set"):
                info = base.info_set()
            else:
                info = None # Custom emote
            groups.setdefault(info, []).append(emote)
        self.emote_groups = groups

    def match_variants(self):
        core_emotes, variants = {}, []
        matches, errors = {}, []

        # Locate core emotes. Ignore all variants but the base one.
        for emote in self.unignored_emotes():
            is_core = "+v" not in emote.tags
            if is_core:
                # An emote always matches itself.
                matches[emote] = emote
            base_variant = emote.base_variant()
            if hasattr(base_variant, "info_set"):
                # Uniquely identifying sprite data
                info = base_variant.info_set()
                if is_core:
                    if info in core_emotes:
                        # Only one core emote per sprite, please. Common
                        # exception: hovermotes.
                        if len(emote.variants) == len(core_emotes[info].variants):
                            errors.append("Multiple non-variant emotes (prev %s, new %s)" % (core_emotes[info].name, emote.name))
                    core_emotes[info] = emote
                else:
                    variants.append((emote, info))
            elif not is_core:
                # Custom +v emotes are not permitted
                errors.append("Variant custom emote %s" % (emote.name))
                matches[emote] = emote

        # Match everything up.
        for (emote, info) in variants:
            base_variant = emote.base_variant()
            base = None
            # Easy match- another identical emote exists.
            if info in core_emotes:
                base = core_emotes[info]
            elif emote.name[:2].lower() == "/r":
                # Try non-reversed name?
                unreversed_emote = self.emotes.get("/" + emote.name[2:])
                if unreversed_emote is not None and not self.is_ignored(unreversed_emote):
                    unreversed_base = unreversed_emote.base_variant()
                    if hasattr(unreversed_base, "info_set"):
                        info = unreversed_base.info_set()
                        if info in core_emotes:
                            # Perfect match
                            base = core_emotes[info]
            # Can't find it. Is there an explicit rule?
            match_config = self.context.config["RootVariantEmotes"].get(self.name, {})
            if emote.name in match_config:
                if base is None:
                    base = self.emotes[match_config[emote.name]]
                else:
                    # Would be nice to know
                    errors.append("Extraneous match config for %s" % (emote.name))
            elif base is None:
                errors.append("Cannot locate root emote for %s" % (emote.name))
            matches[emote] = base # Even if None

        self.variant_matches = matches
        return errors

    def __repr__(self):
        return "<Source %s>" % (self.name)

class Emote(object):
    def __init__(self, name, variants, tags, ignore, source=None):
        self.name = name
        self.variants = variants
        self.tags = tags
        self.ignore = ignore
        self.source = source

        # Caches
        self._implied_tags = None
        self._all_tags = None

    @classmethod
    def load(cls, name, data, tags, source=None, context=None):
        variants = {}
        for (suffix, vdata) in data["Emotes"].items():
            type = vdata.pop("Type", "Normal")
            class_ = {"Normal": NormalVariant, "Custom": CustomVariant}[type]
            variant = class_.load(name, suffix, vdata)
            variants[suffix] = variant
        ignore = data.pop("Ignore", False)
        return cls(name, variants, tags, ignore, source=source)

    def dump(self):
        data = {"Emotes": {suffix: variant.dump() for (suffix, variant) in self.variants.items()}}
        if self.ignore:
            data["Ignore"] = self.ignore
        return data

    def base_variant(self):
        if "" in self.variants:
            return self.variants[""]
        elif len(self.variants) == 1:
            key = next(iter(self.variants.keys()))
            ## Should probably suffice
            #if key not in (":after", ":before"):
            #    print("WARNING: Unknown primary base suffix %r" % (key))
            return self.variants[key]
        raise ValueError("Cannot locate base emote for %r" % (self.name))

    def implied_tags(self, context):
        if self._implied_tags is None:
            self._implied_tags = set()
            for tag in self.tags:
                self._implied_tags |= set(context.tag_config["TagImplications"].get(tag, []))
        return self._implied_tags

    def all_tags(self, context):
        if self._all_tags is None:
            self._all_tags = self.implied_tags(context) | self.tags
        return self._all_tags

    def __repr__(self):
        return "<Emote %s>" % (self.name)

class EmoteCSSBlock(object):
    def __init__(self, name, suffix, css, ignore):
        self.name = name
        self.suffix = suffix
        self.css = css
        self.ignore = ignore

class _Variant(object):
    def selector(self):
        name = self.name[1:].replace("!", "_excl_").replace(":", "_colon_").replace("#", "_hash_").replace("/", "_slash_")
        return ".bpmote-" + bplib.combine_name_pair(name, self.suffix).lower()

class CustomVariant(_Variant):
    def __init__(self, name, suffix, css):
        self.name = name
        self.suffix = suffix
        self.css = css

    @classmethod
    def load(cls, name, suffix, data, emote=None):
        return cls(name, suffix, data["CSS"])

    def dump(self):
        return {"Type": "Custom", "CSS": self.css}

    def to_css(self):
        return self.css.copy()

    def __repr__(self):
        if self.suffix:
            return "<CustomVariant %s %s>" % (self.name, self.suffix)
        else:
            return "<CustomVariant %s>" % (self.name)

class NormalVariant(_Variant):
    def __init__(self, name, suffix, image_url, size, offset, css):
        self.name = name
        self.suffix = suffix
        self.image_url = image_url
        self.size = size
        self.offset = offset
        self.css = css

    @classmethod
    def load(cls, name, suffix, data):
        return cls(name, suffix, data["Image"], tuple(data["Size"]), tuple(data.get("Offset", (0, 0))), data.get("CSS", {}))

    def dump(self):
        data = {"Image": self.image_url, "Size": self.size}
        if self.offset != (0, 0):
            data["Offset"] = self.offset
        if self.css:
            data["CSS"] = self.css
        return data

    def info_set(self):
        return (self.image_url, self.size[0], self.size[1], self.offset[0], self.offset[1])

    def to_css(self):
        css = {
            "display": "block",
            "float": "left",
            "background-image": "url(%s)" % (self.image_url),
            "width": "%spx" % (self.size[0]),
            "height": "%spx" % (self.size[1]),
            }
        if self.offset != (0, 0) or self.suffix:
            css["background-position"] = "%spx %spx" % (self.offset[0], self.offset[1])
        css.update(self.css)
        return css

    def __repr__(self):
        if self.suffix:
            return "<NormalEmote %s %s>" % (self.name, self.suffix)
        else:
            return "<NormalEmote %s>" % (self.name)
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from uv.reporter.state import (active_reporter, get_reporter, report,
                               report_all, set_reporter, start_run)

from ._version import get_versions

__version__ = get_versions()['version']
del get_versions

__all__ = [
    "active_reporter",
    "get_reporter",
    "set_reporter",
    "report",
    "report_all",
    "start_run",
    "AbstractReporter",
    "LoggingReporter",
    "MemoryReporter",
]
<commit_msg>missing imports<commit_after>#!/usr/bin/python
#
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from uv.reporter.base import AbstractReporter
from uv.reporter.state import (active_reporter, get_reporter, report,
                               report_all, set_reporter, start_run)
from uv.reporter.store import LoggingReporter, MemoryReporter

from ._version import get_versions

__version__ = get_versions()['version']
del get_versions

__all__ = [
    "active_reporter",
    "get_reporter",
    "set_reporter",
    "report",
    "report_all",
    "start_run",
    "AbstractReporter",
    "LoggingReporter",
    "MemoryReporter",
]
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# Copyright (C) 2017-2020 The btclib developers
#
# This file is part of btclib. It is subject to the license terms in the
# LICENSE file found in the top-level directory of this distribution.
#
# No part of btclib including this file, may be copied, modified, propagated,
# or distributed except according to the terms contained in the LICENSE file.

from typing import TypedDict, List
from hashlib import sha256

from btclib.transactions import (
    Transaction,
    transaction_deserialize,
    transaction_serialize,
)
from . import varint

class BlockHeader(TypedDict):
    version: int = 0
    previousblockhash: str = ""
    merkleroot: str = ""
    time: int = 1
    bits: int = 0
    nonce: int = 0

def deserialize_block_header(data: bytes):
    header = BlockHeader()
    header["version"] = int.from_bytes(data[0:4], "little")
    header["previousblockhash"] = data[4:36][::-1].hex()
    header["merkleroot"] = data[36:68][::-1].hex()
    header["timestamp"] = int.from_bytes(data[68:72], "little")
    header["bits"] = int.from_bytes(data[72:76], "little")
    header["nonce"] = int.from_bytes(data[76:80], "little")
    header["hash"] = (
        sha256(sha256(serialize_block_header(header)).digest()).digest()[::-1].hex()
    )
    return header

def serialize_block_header(header: BlockHeader):
    out = header["version"].to_bytes(4, "little")
    out += bytes.fromhex(header["previousblockhash"])[::-1]
    out += bytes.fromhex(header["merkleroot"])[::-1]
    out += header["timestamp"].to_bytes(4, "little")
    out += header["bits"].to_bytes(4, "little")
    out += header["nonce"].to_bytes(4, "little")
    return out

class Block(TypedDict):
    header: BlockHeader
    transactions: List[Transaction] = []

def generate_merkle_root(transactions: List[Transaction]):
    hashes = [bytes.fromhex(tx["txid"])[::-1] for tx in transactions]
    hashes_buffer = []
    while len(hashes) != 1:
        if len(hashes) % 2 != 0:
            hashes.append(hashes[-1])
        for x in range(len(hashes) // 2):
            hashes_buffer.append(
                sha256(sha256(hashes[2 * x] + hashes[2 * x + 1]).digest()).digest()
            )
        hashes = hashes_buffer[:]
        hashes_buffer = []
    return hashes[0][::-1].hex()

def deserialize_block(data: bytes):
    block = Block()
    block["header"] = deserialize_block_header(data[:80])
    data = data[80:]
    transaction_count = varint.decode(data)
    data = data[len(varint.encode(transaction_count)) :]
    block["transactions"] = []
    for x in range(transaction_count):
        transaction = transaction_deserialize(data)
        block["transactions"].append(transaction)
        data = data[len(transaction_serialize(transaction)) :]

    return block

def serialize_block(block: Block):
    out = serialize_block_header(block["header"])
    out += varint.encode(len(block["transactions"]))
    for tx in block["transactions"]:
        out += transaction_serialize(tx)
    return out
<commit_msg>Clean code after merge<commit_after>#!/usr/bin/env python3

# Copyright (C) 2017-2020 The btclib developers
#
# This file is part of btclib. It is subject to the license terms in the
# LICENSE file found in the top-level directory of this distribution.
#
# No part of btclib including this file, may be copied, modified, propagated,
# or distributed except according to the terms contained in the LICENSE file.

from typing import TypedDict, List
from hashlib import sha256

from . import tx, tx_in, tx_out
from . import varint
from .utils import hash256

class BlockHeader(TypedDict):
    version: int = 0
    previousblockhash: str = ""
    merkleroot: str = ""
    time: int = 1
    bits: int = 0
    nonce: int = 0

def deserialize_block_header(data: bytes):
    header = BlockHeader()
    header["version"] = int.from_bytes(data[0:4], "little")
    header["previousblockhash"] = data[4:36][::-1].hex()
    header["merkleroot"] = data[36:68][::-1].hex()
    header["timestamp"] = int.from_bytes(data[68:72], "little")
    header["bits"] = int.from_bytes(data[72:76], "little")
    header["nonce"] = int.from_bytes(data[76:80], "little")
    header["hash"] = hash256(serialize_block_header(header))[::-1].hex()
    return header

def serialize_block_header(header: BlockHeader):
    out = header["version"].to_bytes(4, "little")
    out += bytes.fromhex(header["previousblockhash"])[::-1]
    out += bytes.fromhex(header["merkleroot"])[::-1]
    out += header["timestamp"].to_bytes(4, "little")
    out += header["bits"].to_bytes(4, "little")
    out += header["nonce"].to_bytes(4, "little")
    return out

class Block(TypedDict):
    header: BlockHeader
    transactions: List[tx.Tx] = []

def generate_merkle_root(transactions: List[tx.Tx]):
    hashes = [bytes.fromhex(tx.txid(transaction))[::-1] for transaction in transactions]
    hashes_buffer = []
    while len(hashes) != 1:
        if len(hashes) % 2 != 0:
            hashes.append(hashes[-1])
        for x in range(len(hashes) // 2):
            hashes_buffer.append(
                sha256(sha256(hashes[2 * x] + hashes[2 * x + 1]).digest()).digest()
            )
        hashes = hashes_buffer[:]
        hashes_buffer = []
    return hashes[0][::-1].hex()

def deserialize_block(data: bytes):
    block = Block()
    block["header"] = deserialize_block_header(data[:80])
    data = data[80:]
    transaction_count = varint.decode(data)
    data = data[len(varint.encode(transaction_count)) :]
    block["transactions"] = []
    for x in range(transaction_count):
        transaction = tx.deserialize(data)
        block["transactions"].append(transaction)
        data = data[len(tx.serialize(transaction)) :]

    return block

def serialize_block(block: Block):
    out = serialize_block_header(block["header"])
    out += varint.encode(len(block["transactions"]))
    for transaction in block["transactions"]:
        out += tx.serialize(transaction)
    return out
<|endoftext|>