<commit_before>"""
Incorporated from django-storages, copyright all of those listed in:
http://code.welldev.org/django-storages/src/tip/AUTHORS
"""
import urllib
import os
import mimetypes
import re

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from django.conf import settings
from django.core.files.base import File
from django.core.files.storage import Storage
from django.core.exceptions import ImproperlyConfigured

try:
    from boto.s3.connection import S3Connection
    from boto.exception import S3ResponseError
    from boto.s3.key import Key
except ImportError:
    raise ImproperlyConfigured(
        "Could not load boto's S3 bindings. Please install boto."
    )

AWS_REGIONS = [
    'eu-west-1',
    'us-east-1',
    'us-west-1',
    'us-west-2',
    'sa-east-1',
    'ap-northeast-1',
    'ap-southeast-1',
]

REGION_RE = re.compile(r's3-(.+).amazonaws.com')

ACCESS_KEY_NAME = getattr(settings, 'AWS_ACCESS_KEY_ID', None)
SECRET_KEY_NAME = getattr(settings, 'AWS_SECRET_ACCESS_KEY', None)
HEADERS = getattr(settings, 'AWS_HEADERS', {})
STORAGE_BUCKET_NAME = getattr(settings, 'AWS_STORAGE_BUCKET_NAME', None)
STORAGE_BUCKET_CNAME = getattr(settings, 'AWS_STORAGE_BUCKET_CNAME', None)
AWS_REGION = getattr(settings, 'AWS_REGION', 'us-east-1')
AUTO_CREATE_BUCKET = getattr(settings, 'AWS_AUTO_CREATE_BUCKET', True)
DEFAULT_ACL = getattr(settings, 'AWS_DEFAULT_ACL', 'public-read')
QUERYSTRING_AUTH = getattr(settings, 'AWS_QUERYSTRING_AUTH', True)
QUERYSTRING_EXPIRE = getattr(settings, 'AWS_QUERYSTRING_EXPIRE', 3600)
IS_GZIPPED = getattr(settings, 'AWS_IS_GZIPPED', False)
GZIP_CONTENT_TYPES = getattr(settings, 'GZIP_CONTENT_TYPES', (
    'text/css',
    'application/javascript',
    'application/x-javascript'
))

if IS_GZIPPED:
    from gzip import GzipFile

class S3BotoStorage(Storage):
    """Amazon Simple Storage Service using Boto"""

    def __init__(self, bucket=STORAGE_BUCKET_NAME,
                       bucket_cname=STORAGE_BUCKET_CNAME,
                       region=AWS_REGION, access_key=None,
                       secret_key=None, acl=DEFAULT_ACL,
                       headers=HEADERS, gzip=IS_GZIPPED,
                       gzip_content_types=GZIP_CONTENT_TYPES,
                       querystring_auth=QUERYSTRING_AUTH,
                       force_no_ssl=False):
        self.bucket_name = bucket
        self.bucket_cname = bucket_cname
        self.host = self._get_host(region)
        self.acl = acl
        self.headers = headers
        self.gzip = gzip
        self.gzip_content_types = gzip_content_types
        self.querystring_auth = querystring_auth
        self.force_no_ssl = force_no_ssl
        # This is called as chunks are uploaded to S3. Useful for getting
        # around limitations in eventlet for things like gunicorn.
        self.s3_callback_during_upload = None

        if not access_key and not secret_key:
            access_key, secret_key = self._get_access_keys()

        self.connection = S3Connection(
            access_key, secret_key, host=self.host,
        )

    @property
    def bucket(self):
        if not hasattr(self, '_bucket'):
            self._bucket = self._get_or_create_bucket(self.bucket_name)
        return self._bucket

    def _get_access_keys(self):
        access_key = ACCESS_KEY_NAME
        secret_key = SECRET_KEY_NAME
        if (access_key or secret_key) and (not access_key or not secret_key):
            access_key = os.environ.get(ACCESS_KEY_NAME)
            secret_key = os.environ.get(SECRET_KEY_NAME)

        if access_key and secret_key:
            # Both were provided, so use them
            return access_key, secret_key

        return None, None

    def _get_host(self, region):
        """
        Returns correctly formatted host. Accepted formats:

            * simple region name, eg 'us-west-1' (see list in AWS_REGIONS)
            * full host name, eg 's3-us-west-1.amazonaws.com'.
        """
        if 'us-east-1' in region:
            return 's3.amazonaws.com'
        elif region in AWS_REGIONS:
            return 's3-%s.amazonaws.com' % region
        elif region and not REGION_RE.findall(region):
            raise ImproperlyConfigured('AWS_REGION improperly configured!')
        # can be full host or empty string, default region
        return  region

    def _get_or_create_bucket(self, name):
        """Retrieves a bucket if it exists, otherwise creates it."""
        try:
            return self.connection.get_bucket(name)
        except S3ResponseError, e:
            if AUTO_CREATE_BUCKET:
                return self.connection.create_bucket(name)
            raise ImproperlyConfigured, ("Bucket specified by "
            "AWS_STORAGE_BUCKET_NAME does not exist. Buckets can be "
            "automatically created by setting AWS_AUTO_CREATE_BUCKET=True")

    def _clean_name(self, name):
        # Useful for windows' paths
        return os.path.normpath(name).replace('\\', '/')

    def _compress_content(self, content):
        """Gzip a given string."""
        zbuf = StringIO()
        zfile = GzipFile(mode='wb', compresslevel=6, fileobj=zbuf)
        zfile.write(content.read())
        zfile.close()
        content.file = zbuf
        return content

    def _open(self, name, mode='rb'):
        name = self._clean_name(name)
        return S3BotoStorageFile(name, mode, self)

    def _save(self, name, content):
        name = self._clean_name(name)

        if callable(self.headers):
            headers = self.headers(name, content)
        else:
            headers = self.headers

        if hasattr(content.file, 'content_type'):
            content_type = content.file.content_type
        else:
            content_type = mimetypes.guess_type(name)[0] or "application/x-octet-stream"

        if self.gzip and content_type in self.gzip_content_types:
            content = self._compress_content(content)
            headers.update({'Content-Encoding': 'gzip'})

        headers.update({
            'Content-Type': content_type,
            'Content-Length' : len(content),
        })

        content.name = name
        k = self.bucket.get_key(name)
        if not k:
            k = self.bucket.new_key(name)
        # The callback seen here is particularly important for async WSGI
        # servers. This allows us to call back to eventlet or whatever
        # async support library we're using periodically to prevent timeouts.
        k.set_contents_from_file(content, headers=headers, policy=self.acl,
                                 cb=self.s3_callback_during_upload,
                                 num_cb=-1)
        return name

    def delete(self, name):
        name = self._clean_name(name)
        self.bucket.delete_key(name)

    def exists(self, name):
        name = self._clean_name(name)
        k = Key(self.bucket, name)
        return k.exists()

    def listdir(self, name):
        name = self._clean_name(name)
        return [l.name for l in self.bucket.list() if not len(name) or l.name[:len(name)] == name]

    def size(self, name):
        name = self._clean_name(name)
        return self.bucket.get_key(name).size

    def url(self, name):
        name = self._clean_name(name)
        if self.bucket.get_key(name) is None:
            return ''
        return self.bucket.get_key(name).generate_url(QUERYSTRING_EXPIRE,
                                                      method='GET',
                                                      query_auth=self.querystring_auth,
                                                      force_http=self.force_no_ssl)

    def url_as_attachment(self, name, filename=None):
        name = self._clean_name(name)

        if filename:
            disposition = 'attachment; filename="%s"' % filename
        else:
            disposition = 'attachment;'

        response_headers = {
            'response-content-disposition': disposition,
        }

        return self.connection.generate_url(QUERYSTRING_EXPIRE, 'GET',
                                            bucket=self.bucket.name, key=name,
                                            query_auth=True,
                                            force_http=self.force_no_ssl,
                                            response_headers=response_headers)

    def get_available_name(self, name):
        """ Overwrite existing file with the same name. """
        name = self._clean_name(name)
        return name

class S3BotoStorage_AllPublic(S3BotoStorage):
    """
    Same as S3BotoStorage, but defaults to uploading everything with a
    public acl. This has two primary beenfits:

    1) Non-encrypted requests just make a lot better sense for certain things
       like profile images. Much faster, no need to generate S3 auth keys.
    2) Since we don't have to hit S3 for auth keys, this backend is much
       faster than S3BotoStorage, as it makes no attempt to validate whether
       keys exist.

    WARNING: This backend makes absolutely no attempt to verify whether the
    given key exists on self.url(). This is much faster, but be aware.
    """
    def __init__(self, *args, **kwargs):
        super(S3BotoStorage_AllPublic, self).__init__(acl='public-read',
                                                      querystring_auth=False,
                                                      force_no_ssl=True,
                                                      *args, **kwargs)

    def url(self, name):
        """
        Since we assume all public storage with no authorization keys, we can
        just simply dump out a URL rather than having to query S3 for new keys.
        """
        name = urllib.urlencode(self._clean_name(name))

        if self.bucket_cname:
            return "http://%s/%s" % (self.bucket_cname, name)
        elif self.host:
            return "http://%s/%s/%s" % (self.host, self.bucket_name, name)
        # No host ? Then it's the default region
        return "http://s3.amazonaws.com/%s/%s" % (self.bucket_name, name)

class S3BotoStorageFile(File):
    def __init__(self, name, mode, storage):
        self._storage = storage
        self.name = name
        self._mode = mode
        self.key = storage.bucket.get_key(name)
        self._is_dirty = False
        self.file = StringIO()

    @property
    def size(self):
        if not self.key:
            raise IOError('No such S3 key: %s' % self.name)
        return self.key.size

    def read(self, *args, **kwargs):
        self.file = StringIO()
        self._is_dirty = False
        if not self.key:
            raise IOError('No such S3 key: %s' % self.name)
        self.key.get_contents_to_file(self.file)
        return self.file.getvalue()

    def write(self, content):
        if 'w' not in self._mode:
            raise AttributeError("File was opened for read-only access.")
        self.file = StringIO(content)
        self._is_dirty = True

    def close(self):
        if self._is_dirty:
            if not self.key:
                self.key = self._storage.bucket.new_key(key_name=self.name)
            self.key.set_contents_from_string(self.file.getvalue(), headers=self._storage.headers, policy=self.storage.acl)
        self.key.close()
<commit_msg>Try that with quote_plus this time.<commit_after>"""
Incorporated from django-storages, copyright all of those listed in:
http://code.welldev.org/django-storages/src/tip/AUTHORS
"""
import urllib
import os
import mimetypes
import re

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from django.conf import settings
from django.core.files.base import File
from django.core.files.storage import Storage
from django.core.exceptions import ImproperlyConfigured

try:
    from boto.s3.connection import S3Connection
    from boto.exception import S3ResponseError
    from boto.s3.key import Key
except ImportError:
    raise ImproperlyConfigured(
        "Could not load boto's S3 bindings. Please install boto."
    )

AWS_REGIONS = [
    'eu-west-1',
    'us-east-1',
    'us-west-1',
    'us-west-2',
    'sa-east-1',
    'ap-northeast-1',
    'ap-southeast-1',
]

REGION_RE = re.compile(r's3-(.+).amazonaws.com')

ACCESS_KEY_NAME = getattr(settings, 'AWS_ACCESS_KEY_ID', None)
SECRET_KEY_NAME = getattr(settings, 'AWS_SECRET_ACCESS_KEY', None)
HEADERS = getattr(settings, 'AWS_HEADERS', {})
STORAGE_BUCKET_NAME = getattr(settings, 'AWS_STORAGE_BUCKET_NAME', None)
STORAGE_BUCKET_CNAME = getattr(settings, 'AWS_STORAGE_BUCKET_CNAME', None)
AWS_REGION = getattr(settings, 'AWS_REGION', 'us-east-1')
AUTO_CREATE_BUCKET = getattr(settings, 'AWS_AUTO_CREATE_BUCKET', True)
DEFAULT_ACL = getattr(settings, 'AWS_DEFAULT_ACL', 'public-read')
QUERYSTRING_AUTH = getattr(settings, 'AWS_QUERYSTRING_AUTH', True)
QUERYSTRING_EXPIRE = getattr(settings, 'AWS_QUERYSTRING_EXPIRE', 3600)
IS_GZIPPED = getattr(settings, 'AWS_IS_GZIPPED', False)
GZIP_CONTENT_TYPES = getattr(settings, 'GZIP_CONTENT_TYPES', (
    'text/css',
    'application/javascript',
    'application/x-javascript'
))

if IS_GZIPPED:
    from gzip import GzipFile

class S3BotoStorage(Storage):
    """Amazon Simple Storage Service using Boto"""

    def __init__(self, bucket=STORAGE_BUCKET_NAME,
                       bucket_cname=STORAGE_BUCKET_CNAME,
                       region=AWS_REGION, access_key=None,
                       secret_key=None, acl=DEFAULT_ACL,
                       headers=HEADERS, gzip=IS_GZIPPED,
                       gzip_content_types=GZIP_CONTENT_TYPES,
                       querystring_auth=QUERYSTRING_AUTH,
                       force_no_ssl=False):
        self.bucket_name = bucket
        self.bucket_cname = bucket_cname
        self.host = self._get_host(region)
        self.acl = acl
        self.headers = headers
        self.gzip = gzip
        self.gzip_content_types = gzip_content_types
        self.querystring_auth = querystring_auth
        self.force_no_ssl = force_no_ssl
        # This is called as chunks are uploaded to S3. Useful for getting
        # around limitations in eventlet for things like gunicorn.
        self.s3_callback_during_upload = None

        if not access_key and not secret_key:
            access_key, secret_key = self._get_access_keys()

        self.connection = S3Connection(
            access_key, secret_key, host=self.host,
        )

    @property
    def bucket(self):
        if not hasattr(self, '_bucket'):
            self._bucket = self._get_or_create_bucket(self.bucket_name)
        return self._bucket

    def _get_access_keys(self):
        access_key = ACCESS_KEY_NAME
        secret_key = SECRET_KEY_NAME
        if (access_key or secret_key) and (not access_key or not secret_key):
            access_key = os.environ.get(ACCESS_KEY_NAME)
            secret_key = os.environ.get(SECRET_KEY_NAME)

        if access_key and secret_key:
            # Both were provided, so use them
            return access_key, secret_key

        return None, None

    def _get_host(self, region):
        """
        Returns correctly formatted host. Accepted formats:

            * simple region name, eg 'us-west-1' (see list in AWS_REGIONS)
            * full host name, eg 's3-us-west-1.amazonaws.com'.
        """
        if 'us-east-1' in region:
            return 's3.amazonaws.com'
        elif region in AWS_REGIONS:
            return 's3-%s.amazonaws.com' % region
        elif region and not REGION_RE.findall(region):
            raise ImproperlyConfigured('AWS_REGION improperly configured!')
        # can be full host or empty string, default region
        return  region

    def _get_or_create_bucket(self, name):
        """Retrieves a bucket if it exists, otherwise creates it."""
        try:
            return self.connection.get_bucket(name)
        except S3ResponseError, e:
            if AUTO_CREATE_BUCKET:
                return self.connection.create_bucket(name)
            raise ImproperlyConfigured, ("Bucket specified by "
            "AWS_STORAGE_BUCKET_NAME does not exist. Buckets can be "
            "automatically created by setting AWS_AUTO_CREATE_BUCKET=True")

    def _clean_name(self, name):
        # Useful for windows' paths
        return os.path.normpath(name).replace('\\', '/')

    def _compress_content(self, content):
        """Gzip a given string."""
        zbuf = StringIO()
        zfile = GzipFile(mode='wb', compresslevel=6, fileobj=zbuf)
        zfile.write(content.read())
        zfile.close()
        content.file = zbuf
        return content

    def _open(self, name, mode='rb'):
        name = self._clean_name(name)
        return S3BotoStorageFile(name, mode, self)

    def _save(self, name, content):
        name = self._clean_name(name)

        if callable(self.headers):
            headers = self.headers(name, content)
        else:
            headers = self.headers

        if hasattr(content.file, 'content_type'):
            content_type = content.file.content_type
        else:
            content_type = mimetypes.guess_type(name)[0] or "application/x-octet-stream"

        if self.gzip and content_type in self.gzip_content_types:
            content = self._compress_content(content)
            headers.update({'Content-Encoding': 'gzip'})

        headers.update({
            'Content-Type': content_type,
            'Content-Length' : len(content),
        })

        content.name = name
        k = self.bucket.get_key(name)
        if not k:
            k = self.bucket.new_key(name)
        # The callback seen here is particularly important for async WSGI
        # servers. This allows us to call back to eventlet or whatever
        # async support library we're using periodically to prevent timeouts.
        k.set_contents_from_file(content, headers=headers, policy=self.acl,
                                 cb=self.s3_callback_during_upload,
                                 num_cb=-1)
        return name

    def delete(self, name):
        name = self._clean_name(name)
        self.bucket.delete_key(name)

    def exists(self, name):
        name = self._clean_name(name)
        k = Key(self.bucket, name)
        return k.exists()

    def listdir(self, name):
        name = self._clean_name(name)
        return [l.name for l in self.bucket.list() if not len(name) or l.name[:len(name)] == name]

    def size(self, name):
        name = self._clean_name(name)
        return self.bucket.get_key(name).size

    def url(self, name):
        name = self._clean_name(name)
        if self.bucket.get_key(name) is None:
            return ''
        return self.bucket.get_key(name).generate_url(QUERYSTRING_EXPIRE,
                                                      method='GET',
                                                      query_auth=self.querystring_auth,
                                                      force_http=self.force_no_ssl)

    def url_as_attachment(self, name, filename=None):
        name = self._clean_name(name)

        if filename:
            disposition = 'attachment; filename="%s"' % filename
        else:
            disposition = 'attachment;'

        response_headers = {
            'response-content-disposition': disposition,
        }

        return self.connection.generate_url(QUERYSTRING_EXPIRE, 'GET',
                                            bucket=self.bucket.name, key=name,
                                            query_auth=True,
                                            force_http=self.force_no_ssl,
                                            response_headers=response_headers)

    def get_available_name(self, name):
        """ Overwrite existing file with the same name. """
        name = self._clean_name(name)
        return name

class S3BotoStorage_AllPublic(S3BotoStorage):
    """
    Same as S3BotoStorage, but defaults to uploading everything with a
    public acl. This has two primary beenfits:

    1) Non-encrypted requests just make a lot better sense for certain things
       like profile images. Much faster, no need to generate S3 auth keys.
    2) Since we don't have to hit S3 for auth keys, this backend is much
       faster than S3BotoStorage, as it makes no attempt to validate whether
       keys exist.

    WARNING: This backend makes absolutely no attempt to verify whether the
    given key exists on self.url(). This is much faster, but be aware.
    """
    def __init__(self, *args, **kwargs):
        super(S3BotoStorage_AllPublic, self).__init__(acl='public-read',
                                                      querystring_auth=False,
                                                      force_no_ssl=True,
                                                      *args, **kwargs)

    def url(self, name):
        """
        Since we assume all public storage with no authorization keys, we can
        just simply dump out a URL rather than having to query S3 for new keys.
        """
        name = urllib.quote_plus(self._clean_name(name))

        if self.bucket_cname:
            return "http://%s/%s" % (self.bucket_cname, name)
        elif self.host:
            return "http://%s/%s/%s" % (self.host, self.bucket_name, name)
        # No host ? Then it's the default region
        return "http://s3.amazonaws.com/%s/%s" % (self.bucket_name, name)

class S3BotoStorageFile(File):
    def __init__(self, name, mode, storage):
        self._storage = storage
        self.name = name
        self._mode = mode
        self.key = storage.bucket.get_key(name)
        self._is_dirty = False
        self.file = StringIO()

    @property
    def size(self):
        if not self.key:
            raise IOError('No such S3 key: %s' % self.name)
        return self.key.size

    def read(self, *args, **kwargs):
        self.file = StringIO()
        self._is_dirty = False
        if not self.key:
            raise IOError('No such S3 key: %s' % self.name)
        self.key.get_contents_to_file(self.file)
        return self.file.getvalue()

    def write(self, content):
        if 'w' not in self._mode:
            raise AttributeError("File was opened for read-only access.")
        self.file = StringIO(content)
        self._is_dirty = True

    def close(self):
        if self._is_dirty:
            if not self.key:
                self.key = self._storage.bucket.new_key(key_name=self.name)
            self.key.set_contents_from_string(self.file.getvalue(), headers=self._storage.headers, policy=self.storage.acl)
        self.key.close()
<|endoftext|>


<commit_before># Copyright (c) 2011, Yeiniel Suarez Sosa.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright notice,
#      this list of conditions and the following disclaimer.
#
#    * Redistributions in binary form must reproduce the above copyright
#      notice, this list of conditions and the following disclaimer in the
#      documentation and/or other materials provided with the distribution.
#
#    * Neither the name of Yeiniel Suarez Sosa. nor the names of its
#      contributors may be used to endorse or promote products derived from
#      this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from .import foundation, infrastructure, mapping

__all__ = ['Application']

class Application(infrastructure.Application):
    """ Web application.

    This class built on top of the :mod:`.infrastructure` add a default
    service invoked for not mapped Web requests.
    """

<commit_msg>-some housekeeping to remove some unused imports<commit_after># Copyright (c) 2011, Yeiniel Suarez Sosa.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright notice,
#      this list of conditions and the following disclaimer.
#
#    * Redistributions in binary form must reproduce the above copyright
#      notice, this list of conditions and the following disclaimer in the
#      documentation and/or other materials provided with the distribution.
#
#    * Neither the name of Yeiniel Suarez Sosa. nor the names of its
#      contributors may be used to endorse or promote products derived from
#      this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from . import infrastructure

__all__ = ['Application']

class Application(infrastructure.Application):
    """ Web application.

    This class built on top of the :mod:`.infrastructure` add a default
    service invoked for not mapped Web requests.
    """

<|endoftext|>


<commit_before>#!/usr/bin/python
import os
import re
import math
import sys

bw_files = {}
nodes = {}

# XXX: Alpha smoothing, here or in tor?
#      - Tor, because we want to be able to opine about stuff
#        we did not measure this round
#      - Or maybe that's an argument for reading the whole consensus
#        and doing it here.. but that will always be one behind..

def base10_round(bw_val):
  # This keeps the first 3 decimal digits of the bw value only
  # to minimize changes for consensus diffs.
  # Resulting error is +/-0.5%
  return int(round(bw_val,-(int(math.log10(bw_val))-2)))

def closest_to_one(ratio_list):
  min_dist = 0x7fffffff
  min_item = -1
  for i in xrange(len(ratio_list)):
    if abs(1.0-ratio_list[i]) < min_dist:
      min_dist = abs(1.0-ratio_list[i])
      min_item = i
  return min_item

class Node:
  def __init__(self):
    self.idhex = None
    self.nick = None
    self.strm_bw = []
    self.filt_bw = []
    self.ns_bw = []
    self.chosen_sbw = None
    self.chosen_fbw = None
    self.sbw_ratio = None
    self.fbw_ratio = None
    self.ratio = None
    self.new_bw = None

  def add_line(self, line):
    if self.idhex and self.idhex != line.idhex:
      raise Exception("Line mismatch")
    self.idhex = line.idhex
    self.nick = line.nick
    self.strm_bw.append(line.strm_bw)
    self.filt_bw.append(line.filt_bw)
    self.ns_bw.append(line.ns_bw)

  def avg_strm_bw(self):
    return sum(self.strm_bw)/float(len(self.strm_bw))

  def avg_filt_bw(self):
    return sum(self.filt_bw)/float(len(self.filt_bw))

  def avg_ns_bw(self):
    return sum(self.ns_bw)/float(len(self.ns_bw))

  def choose_strm_bw(self, net_avg):
    i = closest_to_one(map(lambda f: f/net_avg, self.strm_bw))
    self.chosen_sbw = i
    return self.chosen_sbw

  def choose_filt_bw(self, net_avg):
    i = closest_to_one(map(lambda f: f/net_avg, self.filt_bw))
    self.chosen_fbw = i
    return self.chosen_fbw

class Line:
  def __init__(self, line):
    self.idhex = re.search("[\s]*node_id=([\S]+)[\s]*", line).group(1)
    self.nick = re.search("[\s]*nick=([\S]+)[\s]*", line).group(1)
    self.strm_bw = int(re.search("[\s]*strm_bw=([\S]+)[\s]*", line).group(1))
    self.filt_bw = int(re.search("[\s]*filt_bw=([\S]+)[\s]*", line).group(1))
    self.ns_bw = int(re.search("[\s]*ns_bw=([\S]+)[\s]*", line).group(1))

def main(argv):
  for da in argv[1:-1]:
    # First, create a list of the most recent files in the
    # scan dirs that are recent enough
    for root, dirs, f in os.walk(da):
      for ds in dirs:
        print ds
        if re.match("^scanner.[\d+]$", ds):
          print ds
          for sr, sd, files in os.walk(da+"/"+ds+"/scan-data"):
            for f in files:
              if re.search("^bws-[\S]+-done-", f):
                print sr+"/"+f
                found_done = True
                fp = file(sr+"/"+f, "r")
                slicenum = sr+"/"+fp.readline()
                timestamp = float(fp.readline())
                fp.close()
                if slicenum not in bw_files \
                       or bw_files[slicenum][0] < timestamp:
                  bw_files[slicenum] = (timestamp, sr+"/"+f)

  for (t,f) in bw_files.itervalues():
    fp = file(f, "r")
    fp.readline() # slicenum
    fp.readline() # timestamp
    for l in fp.readlines():
      try:
        line = Line(l)
        if line.idhex not in nodes:
          n = Node()
          nodes[line.idhex] = n
        else:
          n = nodes[line.idhex]
        n.add_line(line)
      except ValueError,e:
        print "Conversion error "+str(e)+" at "+l
    fp.close()

  pre_strm_avg = sum(map(lambda n: n.avg_strm_bw(), nodes.itervalues()))/ \
                  float(len(nodes))
  pre_filt_avg = sum(map(lambda n: n.avg_filt_bw(), nodes.itervalues()))/ \
                  float(len(nodes))

  for n in nodes.itervalues():
    n.choose_strm_bw(pre_strm_avg)
    n.choose_filt_bw(pre_filt_avg)

  true_strm_avg = sum(map(lambda n: n.strm_bw[n.chosen_sbw],
                       nodes.itervalues()))/float(len(nodes))
  true_filt_avg = sum(map(lambda n: n.filt_bw[n.chosen_fbw],
                       nodes.itervalues()))/float(len(nodes))

  for n in nodes.itervalues():
    n.fbw_ratio = n.filt_bw[n.chosen_fbw]/true_filt_avg
    n.sbw_ratio = n.strm_bw[n.chosen_sbw]/true_strm_avg
    if closest_to_one((n.sbw_ratio, n.fbw_ratio)) == 0:
      n.ratio = n.sbw_ratio
      n.new_bw = n.ns_bw[n.chosen_sbw]*n.ratio
    else:
      n.ratio = n.fbw_ratio
      n.new_bw = n.ns_bw[n.chosen_fbw]*n.ratio

  n_print = nodes.values()
  n_print.sort(lambda x,y: int(x.new_bw) - int(y.new_bw))

  oldest_timestamp = min(map(lambda (t,f): t, bw_files.itervalues()))
  out = file(argv[-1], "w")
  out.write(str(int(round(oldest_timestamp,0)))+"\n")
  for n in n_print:
    out.write("node_id="+n.idhex+" bw="+str(base10_round(n.new_bw))+" nick="+n.nick+"\n")
  out.close()

if __name__ == "__main__":
  main(sys.argv)
<commit_msg><commit_after>#!/usr/bin/python
import os
import re
import math
import sys
import socket

sys.path.append("../../")
from TorCtl.TorUtil import plog
from TorCtl import TorCtl,TorUtil

bw_files = {}
nodes = {}
prev_consensus = {}
ALPHA = 0.3333 # Prev consensus values count for 1/3 of the avg

def base10_round(bw_val):
  # This keeps the first 3 decimal digits of the bw value only
  # to minimize changes for consensus diffs.
  # Resulting error is +/-0.5%
  return int(round(bw_val,-(int(math.log10(bw_val))-2)))

def closest_to_one(ratio_list):
  min_dist = 0x7fffffff
  min_item = -1
  for i in xrange(len(ratio_list)):
    if abs(1.0-ratio_list[i]) < min_dist:
      min_dist = abs(1.0-ratio_list[i])
      min_item = i
  return min_item

class Node:
  def __init__(self):
    self.idhex = None
    self.nick = None
    self.strm_bw = []
    self.filt_bw = []
    self.ns_bw = []
    self.chosen_sbw = None
    self.chosen_fbw = None
    self.sbw_ratio = None
    self.fbw_ratio = None
    self.ratio = None
    self.new_bw = None

  def add_line(self, line):
    if self.idhex and self.idhex != line.idhex:
      raise Exception("Line mismatch")
    self.idhex = line.idhex
    self.nick = line.nick
    self.strm_bw.append(line.strm_bw)
    self.filt_bw.append(line.filt_bw)
    self.ns_bw.append(line.ns_bw)

  def avg_strm_bw(self):
    return sum(self.strm_bw)/float(len(self.strm_bw))

  def avg_filt_bw(self):
    return sum(self.filt_bw)/float(len(self.filt_bw))

  def avg_ns_bw(self):
    return sum(self.ns_bw)/float(len(self.ns_bw))

  def choose_strm_bw(self, net_avg):
    i = closest_to_one(map(lambda f: f/net_avg, self.strm_bw))
    self.chosen_sbw = i
    return self.chosen_sbw

  def choose_filt_bw(self, net_avg):
    i = closest_to_one(map(lambda f: f/net_avg, self.filt_bw))
    self.chosen_fbw = i
    return self.chosen_fbw

class Line:
  def __init__(self, line):
    self.idhex = re.search("[\s]*node_id=([\S]+)[\s]*", line).group(1)
    self.nick = re.search("[\s]*nick=([\S]+)[\s]*", line).group(1)
    self.strm_bw = int(re.search("[\s]*strm_bw=([\S]+)[\s]*", line).group(1))
    self.filt_bw = int(re.search("[\s]*filt_bw=([\S]+)[\s]*", line).group(1))
    self.ns_bw = int(re.search("[\s]*ns_bw=([\S]+)[\s]*", line).group(1))

def main(argv):
  TorUtil.read_config(argv[1]+"/scanner.1/bwauthority.cfg")

  s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  s.connect((TorUtil.control_host,TorUtil.control_port))
  c = TorCtl.Connection(s)
  c.authenticate_cookie(file(argv[1]+"/scanner.1/tor-data/control_auth_cookie",
                              "r"))
  ns_list = c.get_network_status()
  for n in ns_list:
    if n.bandwidth == None:
      plog("ERROR", "Your Tor is not providing NS w bandwidths!")
      sys.exit(0)
    prev_consensus["$"+n.idhex] = n

  for da in argv[1:-1]:
    # First, create a list of the most recent files in the
    # scan dirs that are recent enough
    for root, dirs, f in os.walk(da):
      for ds in dirs:
        print ds
        if re.match("^scanner.[\d+]$", ds):
          print ds
          for sr, sd, files in os.walk(da+"/"+ds+"/scan-data"):
            for f in files:
              if re.search("^bws-[\S]+-done-", f):
                print sr+"/"+f
                found_done = True
                fp = file(sr+"/"+f, "r")
                slicenum = sr+"/"+fp.readline()
                timestamp = float(fp.readline())
                fp.close()
                if slicenum not in bw_files \
                       or bw_files[slicenum][0] < timestamp:
                  bw_files[slicenum] = (timestamp, sr+"/"+f)

  for (t,f) in bw_files.itervalues():
    fp = file(f, "r")
    fp.readline() # slicenum
    fp.readline() # timestamp
    for l in fp.readlines():
      try:
        line = Line(l)
        if line.idhex not in nodes:
          n = Node()
          nodes[line.idhex] = n
        else:
          n = nodes[line.idhex]
        n.add_line(line)
      except ValueError,e:
        print "Conversion error "+str(e)+" at "+l
    fp.close()

  pre_strm_avg = sum(map(lambda n: n.avg_strm_bw(), nodes.itervalues()))/ \
                  float(len(nodes))
  pre_filt_avg = sum(map(lambda n: n.avg_filt_bw(), nodes.itervalues()))/ \
                  float(len(nodes))

  for n in nodes.itervalues():
    n.choose_strm_bw(pre_strm_avg)
    n.choose_filt_bw(pre_filt_avg)

  true_strm_avg = sum(map(lambda n: n.strm_bw[n.chosen_sbw],
                       nodes.itervalues()))/float(len(nodes))
  true_filt_avg = sum(map(lambda n: n.filt_bw[n.chosen_fbw],
                       nodes.itervalues()))/float(len(nodes))

  for n in nodes.itervalues():
    n.fbw_ratio = n.filt_bw[n.chosen_fbw]/true_filt_avg
    n.sbw_ratio = n.strm_bw[n.chosen_sbw]/true_strm_avg
    if closest_to_one((n.sbw_ratio, n.fbw_ratio)) == 0:
      n.ratio = n.sbw_ratio
      n.new_bw = n.ns_bw[n.chosen_sbw]*n.ratio
    else:
      n.ratio = n.fbw_ratio
      n.new_bw = n.ns_bw[n.chosen_fbw]*n.ratio
    if n.idhex in prev_consensus:
      n.new_bw = ((prev_consensus[n.idhex].bandwidth*ALPHA + n.new_bw)/(ALPHA + 1))/1024.0

  n_print = nodes.values()
  n_print.sort(lambda x,y: int(x.new_bw) - int(y.new_bw))

  oldest_timestamp = min(map(lambda (t,f): t, bw_files.itervalues()))
  out = file(argv[-1], "w")
  out.write(str(int(round(oldest_timestamp,0)))+"\n")
  for n in n_print:
    out.write("node_id="+n.idhex+" bw="+str(base10_round(n.new_bw))+" nick="+n.nick+"\n")
  out.close()

if __name__ == "__main__":
  main(sys.argv)
<|endoftext|>


<commit_before>#!/usr/bin/python
import os
import re
import math
import sys
import socket
import time
import traceback

sys.path.append("../../")
from TorCtl.TorUtil import plog
from TorCtl import TorCtl,TorUtil

bw_files = []
timestamps = {}
nodes = {}
prev_consensus = {}
ALPHA = 0.3333 # Prev consensus values count for 1/3 of the avg
MIN_REPORT = 60 # Percent of the network we must measure before reporting
# Keep most measurements in consideration. The code below chooses
# the most recent one. 15 days is just to stop us from choking up
# all the CPU once these things run for a year or so.
MAX_AGE = 60*60*24*15

def base10_round(bw_val):
  # This keeps the first 3 decimal digits of the bw value only
  # to minimize changes for consensus diffs.
  # Resulting error is +/-0.5%
  if bw_val == 0:
    plog("NOTICE", "Zero bandwidth!")
    return 0
  return int(max((1000,
                   round(round(bw_val,-(int(math.log10(bw_val))-2)),
                                                       -3)))/1000)

def closest_to_one(ratio_list):
  min_dist = 0x7fffffff
  min_item = -1
  for i in xrange(len(ratio_list)):
    if abs(1.0-ratio_list[i]) < min_dist:
      min_dist = abs(1.0-ratio_list[i])
      min_item = i
  return min_item

class NodeData:
  def __init__(self, timestamp):
    self.strm_bw = []
    self.filt_bw = []
    self.ns_bw = []
    self.timestamp = timestamp

class Node:
  def __init__(self):
    self.node_data = {}
    self.idhex = None
    self.nick = None
    self.chosen_time = None
    self.chosen_sbw = None
    self.chosen_fbw = None
    self.sbw_ratio = None
    self.fbw_ratio = None
    self.ratio = None
    self.new_bw = None
    self.change = None
    self.strm_bw = []
    self.filt_bw = []
    self.ns_bw = []
    self.timestamps = []

  def add_line(self, line):
    if self.idhex and self.idhex != line.idhex:
      raise Exception("Line mismatch")
    self.idhex = line.idhex
    self.nick = line.nick
    if line.slice_file not in self.node_data \
      or self.node_data[line.slice_file].timestamp < line.timestamp:
      self.node_data[line.slice_file] = NodeData(line.timestamp)

    # FIXME: This is kinda nutty. Can we simplify? For instance,
    # do these really need to be lists inside the nd?
    nd = self.node_data[line.slice_file]
    nd.strm_bw.append(line.strm_bw)
    nd.filt_bw.append(line.filt_bw)
    nd.ns_bw.append(line.ns_bw)

    self.strm_bw = []
    self.filt_bw = []
    self.ns_bw = []
    self.timestamps = []

    for nd in self.node_data.itervalues():
      self.strm_bw.extend(nd.strm_bw)
      self.filt_bw.extend(nd.filt_bw)
      self.ns_bw.extend(nd.ns_bw)
      for i in xrange(len(nd.ns_bw)):
        self.timestamps.append(nd.timestamp)

  def avg_strm_bw(self):
    return sum(self.strm_bw)/float(len(self.strm_bw))

  def avg_filt_bw(self):
    return sum(self.filt_bw)/float(len(self.filt_bw))

  def avg_ns_bw(self):
    return sum(self.ns_bw)/float(len(self.ns_bw))

  def choose_strm_bw(self, net_avg):
    i = closest_to_one(map(lambda f: f/net_avg, self.strm_bw))
    self.chosen_sbw = i
    return self.chosen_sbw

  def choose_filt_bw(self, net_avg):
    i = closest_to_one(map(lambda f: f/net_avg, self.filt_bw))
    self.chosen_fbw = i
    return self.chosen_fbw

class Line:
  def __init__(self, line, slice_file, timestamp):
    self.idhex = re.search("[\s]*node_id=([\S]+)[\s]*", line).group(1)
    self.nick = re.search("[\s]*nick=([\S]+)[\s]*", line).group(1)
    self.strm_bw = int(re.search("[\s]*strm_bw=([\S]+)[\s]*", line).group(1))
    self.filt_bw = int(re.search("[\s]*filt_bw=([\S]+)[\s]*", line).group(1))
    self.ns_bw = int(re.search("[\s]*ns_bw=([\S]+)[\s]*", line).group(1))
    self.slice_file = slice_file
    self.timestamp = timestamp

def main(argv):
  TorUtil.read_config(argv[1]+"/scanner.1/bwauthority.cfg")
  TorUtil.loglevel = "NOTICE"

  s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  s.connect((TorUtil.control_host,TorUtil.control_port))
  c = TorCtl.Connection(s)
  c.debug(file(argv[1]+"/aggregate-control.log", "w", buffering=0))
  c.authenticate_cookie(file(argv[1]+"/scanner.1/tor-data/control_auth_cookie",
                         "r"))

  ns_list = c.get_network_status()
  for n in ns_list:
    if n.bandwidth == None: n.bandwidth = -1
  ns_list.sort(lambda x, y: y.bandwidth - x.bandwidth)
  for n in ns_list:
    if n.bandwidth == -1: n.bandwidth = None
  got_ns_bw = False
  max_rank = len(ns_list)

  # FIXME: This is poor form.. We should subclass the Networkstatus class
  # instead of just adding members
  for i in xrange(max_rank):
    n = ns_list[i]
    n.list_rank = i
    if n.bandwidth == None:
      plog("NOTICE", "Your Tor is not providing NS w bandwidths for "+n.idhex)
    else:
      got_ns_bw = True
    n.measured = False
    prev_consensus["$"+n.idhex] = n

  if not got_ns_bw:
    # Sometimes the consensus lacks a descriptor. In that case,
    # it will skip outputting
    plog("ERROR", "Your Tor is not providing NS w bandwidths!")
    sys.exit(0)

  # Take the most recent timestamp from each scanner
  # and use the oldest for the timestamp of the result.
  # That way we can ensure all the scanners continue running.
  scanner_timestamps = []
  for da in argv[1:-1]:
    # First, create a list of the most recent files in the
    # scan dirs that are recent enough
    for root, dirs, f in os.walk(da):
      for ds in dirs:
        if re.match("^scanner.[\d+]$", ds):
          newest_timestamp = 0
          for sr, sd, files in os.walk(da+"/"+ds+"/scan-data"):
            for f in files:
              if re.search("^bws-[\S]+-done-", f):
                found_done = True
                fp = file(sr+"/"+f, "r")
                slicenum = sr+"/"+fp.readline()
                timestamp = float(fp.readline())
                fp.close()
                # old measurements are probably
                # better than no measurements. We may not
                # measure hibernating routers for days.
                # This filter is just to remove REALLY old files
                if time.time() - timestamp > MAX_AGE:
                  plog("INFO", "Skipping old file "+f)
                  continue
                if timestamp > newest_timestamp:
                  newest_timestamp = timestamp
                bw_files.append((slicenum, timestamp, sr+"/"+f))
                # FIXME: Can we kill this?
                if slicenum not in timestamps or \
                     timestamps[slicenum] < timestamp:
                  timestamps[slicenum] = timestamp
          scanner_timestamps.append(newest_timestamp)

  # Need to only use most recent slice-file for each node..
  for (s,t,f) in bw_files:
    fp = file(f, "r")
    fp.readline() # slicenum
    fp.readline() # timestamp
    for l in fp.readlines():
      try:
        line = Line(l,s,t)
        if line.idhex not in nodes:
          n = Node()
          nodes[line.idhex] = n
        else:
          n = nodes[line.idhex]
        n.add_line(line)
      except ValueError,e:
        plog("NOTICE", "Conversion error "+str(e)+" at "+l)
    fp.close()

  if len(nodes) == 0:
    plog("NOTICE", "No scan results yet.")
    sys.exit(1)

  pre_strm_avg = sum(map(lambda n: n.avg_strm_bw(), nodes.itervalues()))/ \
                  float(len(nodes))
  pre_filt_avg = sum(map(lambda n: n.avg_filt_bw(), nodes.itervalues()))/ \
                  float(len(nodes))

  plog("INFO", "Network pre_strm_avg: "+str(pre_strm_avg))
  plog("INFO", "Network pre_filt_avg: "+str(pre_filt_avg))

  for n in nodes.itervalues():
    n.choose_strm_bw(pre_strm_avg)
    n.choose_filt_bw(pre_filt_avg)
    plog("DEBUG", "Node "+n.nick+" chose sbw: "+\
                str(n.strm_bw[n.chosen_sbw])+" fbw: "+\
                str(n.filt_bw[n.chosen_fbw]))

  true_strm_avg = sum(map(lambda n: n.strm_bw[n.chosen_sbw],
                       nodes.itervalues()))/float(len(nodes))
  true_filt_avg = sum(map(lambda n: n.filt_bw[n.chosen_fbw],
                       nodes.itervalues()))/float(len(nodes))

  plog("INFO", "Network true_strm_avg: "+str(true_strm_avg))
  plog("INFO", "Network true_filt_avg: "+str(true_filt_avg))

  for n in nodes.itervalues():
    n.fbw_ratio = n.filt_bw[n.chosen_fbw]/true_filt_avg
    n.sbw_ratio = n.strm_bw[n.chosen_sbw]/true_strm_avg
    if n.sbw_ratio > n.fbw_ratio:
      n.ratio = n.sbw_ratio
      n.new_bw = n.ns_bw[n.chosen_sbw]*n.ratio
      n.chosen_time = n.timestamps[n.chosen_sbw]
      n.change = n.new_bw - n.ns_bw[n.chosen_sbw]
    else:
      n.ratio = n.fbw_ratio
      n.new_bw = n.ns_bw[n.chosen_fbw]*n.ratio
      n.chosen_time = n.timestamps[n.chosen_fbw]
      n.change = n.new_bw - n.ns_bw[n.chosen_fbw]
    if n.idhex in prev_consensus and prev_consensus[n.idhex].bandwidth != None:
      prev_consensus[n.idhex].measured = True
      # XXX: Maybe we should base this on the consensus value
      # at the time of measurement from the Node class.
      n.new_bw = ((prev_consensus[n.idhex].bandwidth*ALPHA + n.new_bw)/(ALPHA + 1))

  oldest_timestamp = min(map(lambda n: n.chosen_time,
             filter(lambda n: n.idhex in prev_consensus,
                       nodes.itervalues())))
  plog("INFO", "Oldest measured node: "+time.ctime(oldest_timestamp))

  missed_nodes = 0.0
  for n in prev_consensus.itervalues():
    if not n.measured:
      if "Fast" in n.flags and "Running" in n.flags:
        try:
          r = c.get_router(n)
        except TorCtl.ErrorReply:
          r = None
        if r and not r.down and r.bw > 0:
          if time.mktime(r.published.utctimetuple()) - r.uptime \
                 < oldest_timestamp:
            missed_nodes += 1.0
            # We still tend to miss about 80 nodes even with these
            # checks.. Possibly going in and out of hibernation?
            plog("INFO", "Didn't measure "+n.idhex+"="+n.nickname+" at "+str(round((100.0*n.list_rank)/max_rank,1))+" "+str(n.bandwidth))

  measured_pct = round(100.0*len(nodes)/(len(nodes)+missed_nodes),1)
  if measured_pct < MIN_REPORT:
    plog("NOTICE", "Did not measure "+str(MIN_REPORT)+"% of nodes yet ("+str(measured_pct)+"%)")
    sys.exit(1)

  plog("NOTICE", "Measured "+str(measured_pct)+"% of all tor nodes.")

  n_print = nodes.values()
  n_print.sort(lambda x,y: int(y.change) - int(x.change))

  out = file(argv[-1], "w")
  out.write(str(int(round(min(scanner_timestamps),0)))+"\n")
  for n in n_print:
    out.write("node_id="+n.idhex+" bw="+str(base10_round(n.new_bw))+" diff="+str(int(round(n.change/1000.0,0)))+ " nick="+n.nick+ " measured_at="+str(int(n.chosen_time))+"\n")
  out.close()

if __name__ == "__main__":
  try:
    main(sys.argv)
  except socket.error, e:
    traceback.print_exc()
    plog("WARN", "Socket error. Are the scanning Tors running?")
    sys.exit(1)
  except Exception, e:
    plog("ERROR", "Exception during aggregate: "+str(e))
    traceback.print_exc()
    sys.exit(1)
  sys.exit(0)
<commit_msg><commit_after>#!/usr/bin/python
import os
import re
import math
import sys
import socket
import time
import traceback

sys.path.append("../../")
from TorCtl.TorUtil import plog
from TorCtl import TorCtl,TorUtil

bw_files = []
timestamps = {}
nodes = {}
prev_consensus = {}
ALPHA = 0.3333 # Prev consensus values count for 1/3 of the avg
MIN_REPORT = 60 # Percent of the network we must measure before reporting
# Keep most measurements in consideration. The code below chooses
# the most recent one. 15 days is just to stop us from choking up
# all the CPU once these things run for a year or so.
MAX_AGE = 60*60*24*15

def base10_round(bw_val):
  # This keeps the first 3 decimal digits of the bw value only
  # to minimize changes for consensus diffs.
  # Resulting error is +/-0.5%
  if bw_val == 0:
    plog("NOTICE", "Zero bandwidth!")
    return 0
  return int(max((1000,
                   round(round(bw_val,-(int(math.log10(bw_val))-2)),
                                                       -3)))/1000)

def closest_to_one(ratio_list):
  min_dist = 0x7fffffff
  min_item = -1
  for i in xrange(len(ratio_list)):
    if abs(1.0-ratio_list[i]) < min_dist:
      min_dist = abs(1.0-ratio_list[i])
      min_item = i
  return min_item

class NodeData:
  def __init__(self, timestamp):
    self.strm_bw = []
    self.filt_bw = []
    self.ns_bw = []
    self.timestamp = timestamp

class Node:
  def __init__(self):
    self.node_data = {}
    self.idhex = None
    self.nick = None
    self.chosen_time = None
    self.chosen_sbw = None
    self.chosen_fbw = None
    self.sbw_ratio = None
    self.fbw_ratio = None
    self.ratio = None
    self.new_bw = None
    self.change = None
    self.strm_bw = []
    self.filt_bw = []
    self.ns_bw = []
    self.timestamps = []

  def add_line(self, line):
    if self.idhex and self.idhex != line.idhex:
      raise Exception("Line mismatch")
    self.idhex = line.idhex
    self.nick = line.nick
    if line.slice_file not in self.node_data \
      or self.node_data[line.slice_file].timestamp < line.timestamp:
      self.node_data[line.slice_file] = NodeData(line.timestamp)

    # FIXME: This is kinda nutty. Can we simplify? For instance,
    # do these really need to be lists inside the nd?
    nd = self.node_data[line.slice_file]
    nd.strm_bw.append(line.strm_bw)
    nd.filt_bw.append(line.filt_bw)
    nd.ns_bw.append(line.ns_bw)

    self.strm_bw = []
    self.filt_bw = []
    self.ns_bw = []
    self.timestamps = []

    for nd in self.node_data.itervalues():
      self.strm_bw.extend(nd.strm_bw)
      self.filt_bw.extend(nd.filt_bw)
      self.ns_bw.extend(nd.ns_bw)
      for i in xrange(len(nd.ns_bw)):
        self.timestamps.append(nd.timestamp)

  def avg_strm_bw(self):
    return sum(self.strm_bw)/float(len(self.strm_bw))

  def avg_filt_bw(self):
    return sum(self.filt_bw)/float(len(self.filt_bw))

  def avg_ns_bw(self):
    return sum(self.ns_bw)/float(len(self.ns_bw))

  def choose_strm_bw(self, net_avg):
    i = closest_to_one(map(lambda f: f/net_avg, self.strm_bw))
    self.chosen_sbw = i
    return self.chosen_sbw

  def choose_filt_bw(self, net_avg):
    i = closest_to_one(map(lambda f: f/net_avg, self.filt_bw))
    self.chosen_fbw = i
    return self.chosen_fbw

class Line:
  def __init__(self, line, slice_file, timestamp):
    self.idhex = re.search("[\s]*node_id=([\S]+)[\s]*", line).group(1)
    self.nick = re.search("[\s]*nick=([\S]+)[\s]*", line).group(1)
    self.strm_bw = int(re.search("[\s]*strm_bw=([\S]+)[\s]*", line).group(1))
    self.filt_bw = int(re.search("[\s]*filt_bw=([\S]+)[\s]*", line).group(1))
    self.ns_bw = int(re.search("[\s]*ns_bw=([\S]+)[\s]*", line).group(1))
    self.slice_file = slice_file
    self.timestamp = timestamp

def main(argv):
  TorUtil.read_config(argv[1]+"/scanner.1/bwauthority.cfg")
  TorUtil.loglevel = "NOTICE"

  s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  s.connect((TorUtil.control_host,TorUtil.control_port))
  c = TorCtl.Connection(s)
  c.debug(file(argv[1]+"/aggregate-control.log", "w", buffering=0))
  c.authenticate_cookie(file(argv[1]+"/scanner.1/tor-data/control_auth_cookie",
                         "r"))

  ns_list = c.get_network_status()
  for n in ns_list:
    if n.bandwidth == None: n.bandwidth = -1
  ns_list.sort(lambda x, y: y.bandwidth - x.bandwidth)
  for n in ns_list:
    if n.bandwidth == -1: n.bandwidth = None
  got_ns_bw = False
  max_rank = len(ns_list)

  # FIXME: This is poor form.. We should subclass the Networkstatus class
  # instead of just adding members
  for i in xrange(max_rank):
    n = ns_list[i]
    n.list_rank = i
    if n.bandwidth == None:
      plog("NOTICE", "Your Tor is not providing NS w bandwidths for "+n.idhex)
    else:
      got_ns_bw = True
    n.measured = False
    prev_consensus["$"+n.idhex] = n

  if not got_ns_bw:
    # Sometimes the consensus lacks a descriptor. In that case,
    # it will skip outputting
    plog("ERROR", "Your Tor is not providing NS w bandwidths!")
    sys.exit(0)

  # Take the most recent timestamp from each scanner
  # and use the oldest for the timestamp of the result.
  # That way we can ensure all the scanners continue running.
  scanner_timestamps = []
  for da in argv[1:-1]:
    # First, create a list of the most recent files in the
    # scan dirs that are recent enough
    for root, dirs, f in os.walk(da):
      for ds in dirs:
        if re.match("^scanner.[\d+]$", ds):
          newest_timestamp = 0
          for sr, sd, files in os.walk(da+"/"+ds+"/scan-data"):
            for f in files:
              if re.search("^bws-[\S]+-done-", f):
                found_done = True
                fp = file(sr+"/"+f, "r")
                slicenum = sr+"/"+fp.readline()
                timestamp = float(fp.readline())
                fp.close()
                # old measurements are probably
                # better than no measurements. We may not
                # measure hibernating routers for days.
                # This filter is just to remove REALLY old files
                if time.time() - timestamp > MAX_AGE:
                  plog("INFO", "Skipping old file "+f)
                  continue
                if timestamp > newest_timestamp:
                  newest_timestamp = timestamp
                bw_files.append((slicenum, timestamp, sr+"/"+f))
                # FIXME: Can we kill this?
                if slicenum not in timestamps or \
                     timestamps[slicenum] < timestamp:
                  timestamps[slicenum] = timestamp
          scanner_timestamps.append(newest_timestamp)

  # Need to only use most recent slice-file for each node..
  for (s,t,f) in bw_files:
    fp = file(f, "r")
    fp.readline() # slicenum
    fp.readline() # timestamp
    for l in fp.readlines():
      try:
        line = Line(l,s,t)
        if line.idhex not in nodes:
          n = Node()
          nodes[line.idhex] = n
        else:
          n = nodes[line.idhex]
        n.add_line(line)
      except ValueError,e:
        plog("NOTICE", "Conversion error "+str(e)+" at "+l)
    fp.close()

  if len(nodes) == 0:
    plog("NOTICE", "No scan results yet.")
    sys.exit(1)

  pre_strm_avg = sum(map(lambda n: n.avg_strm_bw(), nodes.itervalues()))/ \
                  float(len(nodes))
  pre_filt_avg = sum(map(lambda n: n.avg_filt_bw(), nodes.itervalues()))/ \
                  float(len(nodes))

  plog("INFO", "Network pre_strm_avg: "+str(pre_strm_avg))
  plog("INFO", "Network pre_filt_avg: "+str(pre_filt_avg))

  for n in nodes.itervalues():
    n.choose_strm_bw(pre_strm_avg)
    n.choose_filt_bw(pre_filt_avg)
    plog("DEBUG", "Node "+n.nick+" chose sbw: "+\
                str(n.strm_bw[n.chosen_sbw])+" fbw: "+\
                str(n.filt_bw[n.chosen_fbw]))

  true_strm_avg = sum(map(lambda n: n.strm_bw[n.chosen_sbw],
                       nodes.itervalues()))/float(len(nodes))
  true_filt_avg = sum(map(lambda n: n.filt_bw[n.chosen_fbw],
                       nodes.itervalues()))/float(len(nodes))

  plog("INFO", "Network true_strm_avg: "+str(true_strm_avg))
  plog("INFO", "Network true_filt_avg: "+str(true_filt_avg))

  for n in nodes.itervalues():
    n.fbw_ratio = n.filt_bw[n.chosen_fbw]/true_filt_avg
    n.sbw_ratio = n.strm_bw[n.chosen_sbw]/true_strm_avg
    if n.sbw_ratio > n.fbw_ratio:
      n.ratio = n.sbw_ratio
      n.new_bw = n.ns_bw[n.chosen_sbw]*n.ratio
      n.chosen_time = n.timestamps[n.chosen_sbw]
      n.change = 0 - n.ns_bw[n.chosen_sbw]
    else:
      n.ratio = n.fbw_ratio
      n.new_bw = n.ns_bw[n.chosen_fbw]*n.ratio
      n.chosen_time = n.timestamps[n.chosen_fbw]
      n.change = 0 - n.ns_bw[n.chosen_fbw]
    if n.idhex in prev_consensus and prev_consensus[n.idhex].bandwidth != None:
      prev_consensus[n.idhex].measured = True
      # XXX: Maybe we should base this on the consensus value
      # at the time of measurement from the Node class.
      n.new_bw = ((prev_consensus[n.idhex].bandwidth*ALPHA + n.new_bw)/(ALPHA + 1))
    n.change += n.new_bw

  oldest_timestamp = min(map(lambda n: n.chosen_time,
             filter(lambda n: n.idhex in prev_consensus,
                       nodes.itervalues())))
  plog("INFO", "Oldest measured node: "+time.ctime(oldest_timestamp))

  missed_nodes = 0.0
  for n in prev_consensus.itervalues():
    if not n.measured:
      if "Fast" in n.flags and "Running" in n.flags:
        try:
          r = c.get_router(n)
        except TorCtl.ErrorReply:
          r = None
        if r and not r.down and r.bw > 0:
          if time.mktime(r.published.utctimetuple()) - r.uptime \
                 < oldest_timestamp:
            missed_nodes += 1.0
            # We still tend to miss about 80 nodes even with these
            # checks.. Possibly going in and out of hibernation?
            plog("INFO", "Didn't measure "+n.idhex+"="+n.nickname+" at "+str(round((100.0*n.list_rank)/max_rank,1))+" "+str(n.bandwidth))

  measured_pct = round(100.0*len(nodes)/(len(nodes)+missed_nodes),1)
  if measured_pct < MIN_REPORT:
    plog("NOTICE", "Did not measure "+str(MIN_REPORT)+"% of nodes yet ("+str(measured_pct)+"%)")
    sys.exit(1)

  plog("NOTICE", "Measured "+str(measured_pct)+"% of all tor nodes.")

  n_print = nodes.values()
  n_print.sort(lambda x,y: int(y.change) - int(x.change))

  out = file(argv[-1], "w")
  out.write(str(int(round(min(scanner_timestamps),0)))+"\n")
  for n in n_print:
    out.write("node_id="+n.idhex+" bw="+str(base10_round(n.new_bw))+" diff="+str(int(round(n.change/1000.0,0)))+ " nick="+n.nick+ " measured_at="+str(int(n.chosen_time))+"\n")
  out.close()

if __name__ == "__main__":
  try:
    main(sys.argv)
  except socket.error, e:
    traceback.print_exc()
    plog("WARN", "Socket error. Are the scanning Tors running?")
    sys.exit(1)
  except Exception, e:
    plog("ERROR", "Exception during aggregate: "+str(e))
    traceback.print_exc()
    sys.exit(1)
  sys.exit(0)
<|endoftext|>


<commit_before>import scrapy
import logging
import datetime

from NewsCrawler.items import ProthomAloItem
from newspaper import Article

from NewsCrawler.Helpers.CustomNERTagger import Tagger
from NewsCrawler.credentials_and_configs.stanford_ner_path import STANFORD_CLASSIFIER_PATH, STANFORD_NER_PATH

from NewsCrawler.Helpers.date_helper import dateobject_to_split_date, DATETIME_FORMAT

from scrapy.exceptions import CloseSpider

from elasticsearch import Elasticsearch

es = Elasticsearch()

class ProthomAloSpider(scrapy.Spider):
    name = 'prothomalo'

    def __init__(self, start_date="01-07-2014", end_date="02-07-2014", delimiter='-'):
        self.start_day, self.start_month, self.start_year = dateobject_to_split_date(start_date)
        self.end_day, self.start_month, self.end_year = dateobject_to_split_date(end_date)

    def start_requests(self):
        self.begin_date = datetime.date(self.start_year, self.start_month, self.start_day)

        self.start_date = datetime.date(self.start_year, self.start_month, self.start_day)
        self.end_date = datetime.date(self.start_year, self.start_month, self.start_day)

        self.main_url = 'http://en.prothom-alo.com'
        self.url = 'http://en.prothom-alo.com/archive/' + self.start_date.__str__()

        # Creating the tagger object
        self.tagger = Tagger(classifier_path=STANFORD_CLASSIFIER_PATH, ner_path=STANFORD_NER_PATH)

        self.id = 0

        yield scrapy.Request(self.url, self.parse)

    def parse(self, response):
        # Selecting the list of news
        self.main_section = response.xpath("//div[@class='list list_square mb20']/ul/li")

        for sel in self.main_section:
            news_item = ProthomAloItem()
            news_item['newspaper_name'] = 'Prothom Alo'
            news_item['category'] = sel.xpath("h3/a/@href").extract_first().split('/')[0]
            news_item['url'] = self.main_url + sel.xpath("h3/a/@href").extract_first()
            news_item['title'] = sel.xpath("h3/a/text()").extract_first().strip()
            news_item['news_location'] = self.get_location(sel.xpath("div/span[1]/text()").extract_first())

            # Sometimes reporter - newslocation span is missing
            if (len(sel.xpath("div/span")) < 2):
                news_item['published_date'] = sel.xpath("div/span[1]/text()").extract_first().strip()
                news_item['reporter'] = None
            else:
                news_item['reporter'] = sel.xpath("div/span[1]/text()").extract_first().split('.')[0]
                news_item['published_date'] = sel.xpath("div/span[2]/text()").extract_first().strip()

            news_item['last_update'] = news_item['published_date']

            request = scrapy.Request(news_item['url'], callback=self.parseNews)
            request.meta['news_item'] = news_item
            yield request

    def parseNews(self, response):
        self.logger.info("TRYING")
        self.id += 1

        # Retreiving news item
        news_item = response.meta['news_item']

        # Currently detects one image
        news_item['images'] = self.get_image_url(response.xpath("//img/@src").extract())

        # Currently gets one caption
        news_item['image_captions'] = response.xpath("//div[@itemprop='articleBody']//span/text()").extract_first()

        #Getting the article
        paragraphs = response.xpath("//div[@itemprop='articleBody']//p/text()").extract()
        news_item['article'] = "".join([para.strip() for para in paragraphs])

        # Getting the breadcrumb
        news_item['breadcrumb'] = response.xpath("//div[@class='breadcrumb']/ul/li/a/strong/text()").extract()

        # Applying NLP from newspaper package [WARNING: slows the overall process]
        article = Article(url=news_item['url'])
        article.download()
        article.parse()
        article.nlp()
        news_item['generated_summary'] = article.summary
        news_item['generated_keywords'] = article.keywords

        # Tagging the Article
        try:
            self.tagger.entity_group(news_item['article'])
        except:
            print "NER Tagger Exception"
            self.logger.info("NER Crashed")

        news_item['ner_person'] = self.tagger.PERSON
        news_item['ner_organization'] = self.tagger.ORGANIZATION
        news_item['ner_time'] = self.tagger.TIME
        news_item['ner_percent'] = self.tagger.PERCENT
        news_item['ner_money'] = self.tagger.MONEY
        news_item['ner_location'] = self.tagger.LOCATION

        # Contains all occurances
        news_item['ner_list_person'] = self.tagger.LIST_PERSON
        news_item['ner_list_organization'] = self.tagger.LIST_ORGANIZATION
        news_item['ner_list_time'] = self.tagger.LIST_TIME
        news_item['ner_list_percent'] = self.tagger.LIST_PERCENT
        news_item['ner_list_money'] = self.tagger.LIST_MONEY
        news_item['ner_list_location'] = self.tagger.LIST_LOCATION

        #ML Tags
        news_item['ml_tags'] = None

        news_item['sentiment'] = self.tagger.get_indico_sentiment(news_item['article'])

        doc = {
            "news_url" : news_item['url'],
            "reporter" : news_item['reporter'],
            "published" : news_item['published_date'],
            "last_update" : news_item['published_date'],
            "title" : news_item['title'],
            "content" : news_item['article'],

            "images" : news_item['images'],
            "image_captions" : news_item['image_captions'],
            "breadcrumb" : news_item['breadcrumb'],
            "sentiment" : news_item['sentiment'],
            "ml_tags" : None,
            "section" : news_item['newspaper_section'],

            "ner_person" : news_item['ner_person'],
            "ner_organization" : news_item['ner_organization'],
            "ner_money" : news_item['ner_money'],
            "ner_time" : news_item['ner_time'],
            "ner_location" : news_item['ner_location'],
            "ner_percent" : news_item['ner_percent'],

            "ner_list_person" : news_item['ner_list_person'],
            "ner_list_organization" : news_item['ner_list_organization'],
            "ner_list_money" : news_item['ner_list_money'],
            "ner_list_time" : news_item['ner_list_time'],
            "ner_list_location" : news_item['ner_list_location'],
            "ner_list_percent" : news_item['ner_list_percent'],

            "generated_keywords" : news_item['generated_keywords'],
            "generated_summary" : news_item['generated_summary'],
            "timestamp" : datetime.datetime.now().strftime(DATETIME_FORMAT),
        }

        # Inserting data to Elasticsearch
        res = es.index(index="newspaper_index", doc_type="news", id=self.id, body=doc)

        # Output can also be saved as csv/json
        yield doc

    def get_location(self, inp):
        inp_list = inp.split('.')
        if (len(inp_list) < 2):
            return None
        return inp_list[-1].strip()

    def get_image_url(self, inp):
        for image_url in inp:
            if 'contents' and 'uploads' in image_url.split('/'):
                return self.main_url + image_url<commit_msg>fixed all major bugs<commit_after>import scrapy
import logging
import datetime

from NewsCrawler.items import ProthomAloItem
from newspaper import Article

from NewsCrawler.Helpers.CustomNERTagger import Tagger
from NewsCrawler.credentials_and_configs.stanford_ner_path import STANFORD_CLASSIFIER_PATH, STANFORD_NER_PATH

from NewsCrawler.Helpers.date_helper import dateobject_to_split_date, DATETIME_FORMAT, increase_day_by_one

from scrapy.exceptions import CloseSpider

from elasticsearch import Elasticsearch

es = Elasticsearch()

class ProthomAloSpider(scrapy.Spider):
    name = 'prothomalo'

    def __init__(self, start_date="01-07-2014", end_date="02-07-2014", delimiter='-'):
        self.start_day, self.start_month, self.start_year = dateobject_to_split_date(start_date)
        self.end_day, self.end_month, self.end_year = dateobject_to_split_date(end_date)

    def start_requests(self):
        self.begin_date = datetime.date(self.start_year, self.start_month, self.start_day)

        self.start_date = datetime.date(self.start_year, self.start_month, self.start_day)
        self.end_date = datetime.date(self.end_year, self.end_month, self.end_day)

        self.main_url = 'http://en.prothom-alo.com'
        self.baseurl = 'http://en.prothom-alo.com/archive/'
        self.url = 'http://en.prothom-alo.com/archive/' + self.start_date.__str__()

        # Creating the tagger object
        self.tagger = Tagger(classifier_path=STANFORD_CLASSIFIER_PATH, ner_path=STANFORD_NER_PATH)

        self.id = 0

        yield scrapy.Request(self.url, self.parse)

    def parse(self, response):
        # Selecting the list of news
        self.main_section = response.xpath("//div[@class='list list_square mb20']/ul/li")

        for sel in self.main_section:
            news_item = ProthomAloItem()
            news_item['newspaper_name'] = 'Prothom Alo'
            news_item['category'] = sel.xpath("h3/a/@href").extract_first().split('/')[0]
            news_item['url'] = self.main_url + sel.xpath("h3/a/@href").extract_first()
            news_item['title'] = sel.xpath("h3/a/text()").extract_first().strip()
            news_item['news_location'] = self.get_location(sel.xpath("div/span[1]/text()").extract_first())

            # Sometimes reporter - newslocation span is missing
            if (len(sel.xpath("div/span")) < 2):
                news_item['published_date'] = sel.xpath("div/span[1]/text()").extract_first().strip()
                news_item['reporter'] = None
            else:
                news_item['reporter'] = sel.xpath("div/span[1]/text()").extract_first().split('.')[0]
                news_item['published_date'] = sel.xpath("div/span[2]/text()").extract_first().strip()

            news_item['last_update'] = news_item['published_date']

            request = scrapy.Request(news_item['url'], callback=self.parseNews)
            request.meta['news_item'] = news_item
            yield request

        self.start_date = increase_day_by_one(self.start_date)

        self.logger.info("INCREASED: " + self.start_date.__str__())

        self.next_page = self.baseurl + self.start_date.__str__()

        # Crawling termination condition
        if self.start_date > self.end_date:
            raise CloseSpider('Done scraping from '+ self.begin_date.__str__() + ' upto ' +  self.end_date.__str__())

        try:
            self.logger.info("TRYING")
            yield scrapy.Request(self.next_page, callback=self.parse)
        except:
            self.logger.info("PROBLEM")
            self.start_date = increase_day_by_one(self.start_date)
            self.next_page = self.baseurl + self.start_date.__str__()
            yield scrapy.Request(self.next_page, callback=self.parse)

    def parseNews(self, response):
        self.logger.info("TRYING")
        self.id += 1

        # Retreiving news item
        news_item = response.meta['news_item']

        # Currently detects one image
        news_item['images'] = self.get_image_url(response.xpath("//img/@src").extract())

        # Currently gets one caption
        news_item['image_captions'] = response.xpath("//div[@itemprop='articleBody']//span/text()").extract_first()

        #Getting the article
        paragraphs = response.xpath("//div[@itemprop='articleBody']//p/text()").extract()
        news_item['article'] = "".join([para.strip() for para in paragraphs])

        # Getting the breadcrumb
        news_item['breadcrumb'] = response.xpath("//div[@class='breadcrumb']/ul/li/a/strong/text()").extract()

        # Applying NLP from newspaper package [WARNING: slows the overall process]
        article = Article(url=news_item['url'])
        article.download()
        article.parse()
        article.nlp()
        news_item['generated_summary'] = article.summary
        news_item['generated_keywords'] = article.keywords

        # Tagging the Article
        try:
            self.tagger.entity_group(news_item['article'])
        except:
            print "NER Tagger Exception"
            self.logger.info("NER Crashed")

        news_item['ner_person'] = self.tagger.PERSON
        news_item['ner_organization'] = self.tagger.ORGANIZATION
        news_item['ner_time'] = self.tagger.TIME
        news_item['ner_percent'] = self.tagger.PERCENT
        news_item['ner_money'] = self.tagger.MONEY
        news_item['ner_location'] = self.tagger.LOCATION

        # Contains all occurances
        news_item['ner_list_person'] = self.tagger.LIST_PERSON
        news_item['ner_list_organization'] = self.tagger.LIST_ORGANIZATION
        news_item['ner_list_time'] = self.tagger.LIST_TIME
        news_item['ner_list_percent'] = self.tagger.LIST_PERCENT
        news_item['ner_list_money'] = self.tagger.LIST_MONEY
        news_item['ner_list_location'] = self.tagger.LIST_LOCATION

        #ML Tags
        news_item['ml_tags'] = None

        news_item['sentiment'] = self.tagger.get_indico_sentiment(news_item['article'])

        doc = {
            "news_url" : news_item['url'],
            "reporter" : news_item['reporter'],
            "published" : news_item['published_date'],
            "last_update" : news_item['published_date'],
            "title" : news_item['title'],
            "content" : news_item['article'],

            "images" : news_item['images'],
            "image_captions" : news_item['image_captions'],
            "breadcrumb" : news_item['breadcrumb'],
            "sentiment" : news_item['sentiment'],
            "ml_tags" : None,
            "category" : news_item['category'],

            "ner_person" : news_item['ner_person'],
            "ner_organization" : news_item['ner_organization'],
            "ner_money" : news_item['ner_money'],
            "ner_time" : news_item['ner_time'],
            "ner_location" : news_item['ner_location'],
            "ner_percent" : news_item['ner_percent'],

            "ner_list_person" : news_item['ner_list_person'],
            "ner_list_organization" : news_item['ner_list_organization'],
            "ner_list_money" : news_item['ner_list_money'],
            "ner_list_time" : news_item['ner_list_time'],
            "ner_list_location" : news_item['ner_list_location'],
            "ner_list_percent" : news_item['ner_list_percent'],

            "generated_keywords" : news_item['generated_keywords'],
            "generated_summary" : news_item['generated_summary'],
            "timestamp" : datetime.datetime.now().strftime(DATETIME_FORMAT),
        }

        # Inserting data to Elasticsearch
        # res = es.index(index="newspaper_index", doc_type="news", id=self.id, body=doc)

        # Output can also be saved as csv/json
        yield doc

    def get_location(self, inp):
        inp_list = inp.split('.')
        if (len(inp_list) < 2):
            return None
        return inp_list[-1].strip()

    def get_image_url(self, inp):
        for image_url in inp:
            if 'contents' and 'uploads' in image_url.split('/'):
                return self.main_url + image_url<|endoftext|>


<commit_before>from logging import getLogger

from django.db.models import Sum, Value, Q
from django.db.models.functions import Concat
from django.utils import timezone

from axes.attempts import (
    clean_expired_user_attempts,
    get_user_attempts,
    reset_user_attempts,
)
from axes.conf import settings
from axes.handlers.base import AxesBaseHandler, AbstractAxesHandler
from axes.helpers import (
    get_client_str,
    get_client_username,
    get_credentials,
    get_failure_limit,
    get_query_str,
)
from axes.models import AccessLog, AccessAttempt
from axes.signals import user_locked_out

log = getLogger(__name__)

class AxesDatabaseHandler(AbstractAxesHandler, AxesBaseHandler):
    """
    Signal handler implementation that records user login attempts to database and locks users out if necessary.

    .. note:: The get_user_attempts function is called several time during the authentication and lockout
              process, caching its output can be dangerous.
    """

    def reset_attempts(
        self,
        *,
        ip_address: str = None,
        username: str = None,
        ip_or_username: bool = False,
    ) -> int:
        attempts = AccessAttempt.objects.all()

        if ip_or_username:
            attempts = attempts.filter(Q(ip_address=ip_address) | Q(username=username))
        else:
            if ip_address:
                attempts = attempts.filter(ip_address=ip_address)
            if username:
                attempts = attempts.filter(username=username)

        count, _ = attempts.delete()
        log.info("AXES: Reset %d access attempts from database.", count)

        return count

    def reset_logs(self, *, age_days: int = None) -> int:
        if age_days is None:
            count, _ = AccessLog.objects.all().delete()
            log.info("AXES: Reset all %d access logs from database.", count)
        else:
            limit = timezone.now() - timezone.timedelta(days=age_days)
            count, _ = AccessLog.objects.filter(attempt_time__lte=limit).delete()
            log.info(
                "AXES: Reset %d access logs older than %d days from database.",
                count,
                age_days,
            )

        return count

    def get_failures(self, request, credentials: dict = None) -> int:
        attempts_list = get_user_attempts(request, credentials)
        attempt_count = max(
            (
                attempts.aggregate(Sum("failures_since_start"))[
                    "failures_since_start__sum"
                ]
                or 0
            )
            for attempts in attempts_list
        )
        return attempt_count

    def user_login_failed(
        self, sender, credentials: dict, request=None, **kwargs
    ):  # pylint: disable=too-many-locals
        """
        When user login fails, save AccessAttempt record in database, mark request with lockout attribute and emit lockout signal.
        """

        log.info("AXES: User login failed, running database handler for failure.")

        if request is None:
            log.error(
                "AXES: AxesDatabaseHandler.user_login_failed does not function without a request."
            )
            return

        # 1. database query: Clean up expired user attempts from the database before logging new attempts
        clean_expired_user_attempts(request.axes_attempt_time)

        username = get_client_username(request, credentials)
        client_str = get_client_str(
            username,
            request.axes_ip_address,
            request.axes_user_agent,
            request.axes_path_info,
        )

        # This replaces null byte chars that crash saving failures.
        get_data = get_query_str(request.GET).replace("\0", "0x00")
        post_data = get_query_str(request.POST).replace("\0", "0x00")

        if self.is_whitelisted(request, credentials):
            log.info("AXES: Login failed from whitelisted client %s.", client_str)
            return

        # 2. database query: Get or create access record with the new failure data
        if settings.AXES_ONLY_USER_FAILURES and username is None:
            log.warning(
                "AXES: Username is None and AXES_ONLY_USER_FAILURES is enabled, new record will NOT be created."
            )
        else:
            attempt, created = AccessAttempt.objects.get_or_create(
                username=username,
                ip_address=request.axes_ip_address,
                user_agent=request.axes_user_agent,
                defaults={
                    "get_data": get_data,
                    "post_data": post_data,
                    "http_accept": request.axes_http_accept,
                    "path_info": request.axes_path_info,
                    "failures_since_start": 1,
                    "attempt_time": request.axes_attempt_time,
                },
            )

            # Record failed attempt with all the relevant information.
            # Filtering based on username, IP address and user agent handled elsewhere,
            # and this handler just records the available information for further use.
            if created:
                log.warning(
                    "AXES: New login failure by %s. Created new record in the database.",
                    client_str,
                )

            # 3. database query if there were previous attempts in the database
            # Update failed attempt information but do not touch the username, IP address, or user agent fields,
            # because attackers can request the site with multiple different configurations
            # in order to bypass the defense mechanisms that are used by the site.
            else:
                separator = "\n---------\n"

                attempt.get_data = Concat("get_data", Value(separator + get_data))
                attempt.post_data = Concat("post_data", Value(separator + post_data))
                attempt.http_accept = request.axes_http_accept
                attempt.path_info = request.axes_path_info
                attempt.failures_since_start += 1
                attempt.attempt_time = request.axes_attempt_time
                attempt.save()

                log.warning(
                    "AXES: Repeated login failure by %s. Count = %d of %d. Updated existing record in the database.",
                    client_str,
                    attempt.failures_since_start,
                    get_failure_limit(request, credentials),
                )

        # 3. or 4. database query: Calculate the current maximum failure number from the existing attempts
        failures_since_start = self.get_failures(request, credentials)

        if (
            settings.AXES_LOCK_OUT_AT_FAILURE
            and failures_since_start >= get_failure_limit(request, credentials)
        ):
            log.warning(
                "AXES: Locking out %s after repeated login failures.", client_str
            )

            request.axes_locked_out = True
            user_locked_out.send(
                "axes",
                request=request,
                username=username,
                ip_address=request.axes_ip_address,
            )

    def user_logged_in(
        self, sender, request, user, **kwargs
    ):  # pylint: disable=unused-argument
        """
        When user logs in, update the AccessLog related to the user.
        """

        # 1. database query: Clean up expired user attempts from the database
        clean_expired_user_attempts(request.axes_attempt_time)

        username = user.get_username()
        credentials = get_credentials(username)
        client_str = get_client_str(
            username,
            request.axes_ip_address,
            request.axes_user_agent,
            request.axes_path_info,
        )

        log.info("AXES: Successful login by %s.", client_str)

        if not settings.AXES_DISABLE_ACCESS_LOG:
            # 2. database query: Insert new access logs with login time
            AccessLog.objects.create(
                username=username,
                ip_address=request.axes_ip_address,
                user_agent=request.axes_user_agent,
                http_accept=request.axes_http_accept,
                path_info=request.axes_path_info,
                attempt_time=request.axes_attempt_time,
            )

        if settings.AXES_RESET_ON_SUCCESS:
            # 3. database query: Reset failed attempts for the logging in user
            count = reset_user_attempts(request, credentials)
            log.info(
                "AXES: Deleted %d failed login attempts by %s from database.",
                count,
                client_str,
            )

    def user_logged_out(
        self, sender, request, user, **kwargs
    ):  # pylint: disable=unused-argument
        """
        When user logs out, update the AccessLog related to the user.
        """

        # 1. database query: Clean up expired user attempts from the database
        clean_expired_user_attempts(request.axes_attempt_time)

        username = user.get_username() if user else None
        client_str = get_client_str(
            username,
            request.axes_ip_address,
            request.axes_user_agent,
            request.axes_path_info,
        )

        log.info("AXES: Successful logout by %s.", client_str)

        if username and not settings.AXES_DISABLE_ACCESS_LOG:
            # 2. database query: Update existing attempt logs with logout time
            AccessLog.objects.filter(
                username=username, logout_time__isnull=True
            ).update(logout_time=request.axes_attempt_time)

    def post_save_access_attempt(self, instance, **kwargs):
        """
        Handles the ``axes.models.AccessAttempt`` object post save signal.

        When needed, all post_save actions for this backend should be located
        here.
        """

    def post_delete_access_attempt(self, instance, **kwargs):
        """
        Handles the ``axes.models.AccessAttempt`` object post delete signal.

        When needed, all post_delete actions for this backend should be located
        here.
        """
<commit_msg>increment failures_since_start in an atomic manner<commit_after>from logging import getLogger

from django.db.models import F, Sum, Value, Q
from django.db.models.functions import Concat
from django.utils import timezone

from axes.attempts import (
    clean_expired_user_attempts,
    get_user_attempts,
    reset_user_attempts,
)
from axes.conf import settings
from axes.handlers.base import AxesBaseHandler, AbstractAxesHandler
from axes.helpers import (
    get_client_str,
    get_client_username,
    get_credentials,
    get_failure_limit,
    get_query_str,
)
from axes.models import AccessLog, AccessAttempt
from axes.signals import user_locked_out

log = getLogger(__name__)

class AxesDatabaseHandler(AbstractAxesHandler, AxesBaseHandler):
    """
    Signal handler implementation that records user login attempts to database and locks users out if necessary.

    .. note:: The get_user_attempts function is called several time during the authentication and lockout
              process, caching its output can be dangerous.
    """

    def reset_attempts(
        self,
        *,
        ip_address: str = None,
        username: str = None,
        ip_or_username: bool = False,
    ) -> int:
        attempts = AccessAttempt.objects.all()

        if ip_or_username:
            attempts = attempts.filter(Q(ip_address=ip_address) | Q(username=username))
        else:
            if ip_address:
                attempts = attempts.filter(ip_address=ip_address)
            if username:
                attempts = attempts.filter(username=username)

        count, _ = attempts.delete()
        log.info("AXES: Reset %d access attempts from database.", count)

        return count

    def reset_logs(self, *, age_days: int = None) -> int:
        if age_days is None:
            count, _ = AccessLog.objects.all().delete()
            log.info("AXES: Reset all %d access logs from database.", count)
        else:
            limit = timezone.now() - timezone.timedelta(days=age_days)
            count, _ = AccessLog.objects.filter(attempt_time__lte=limit).delete()
            log.info(
                "AXES: Reset %d access logs older than %d days from database.",
                count,
                age_days,
            )

        return count

    def get_failures(self, request, credentials: dict = None) -> int:
        attempts_list = get_user_attempts(request, credentials)
        attempt_count = max(
            (
                attempts.aggregate(Sum("failures_since_start"))[
                    "failures_since_start__sum"
                ]
                or 0
            )
            for attempts in attempts_list
        )
        return attempt_count

    def user_login_failed(
        self, sender, credentials: dict, request=None, **kwargs
    ):  # pylint: disable=too-many-locals
        """
        When user login fails, save AccessAttempt record in database, mark request with lockout attribute and emit lockout signal.
        """

        log.info("AXES: User login failed, running database handler for failure.")

        if request is None:
            log.error(
                "AXES: AxesDatabaseHandler.user_login_failed does not function without a request."
            )
            return

        # 1. database query: Clean up expired user attempts from the database before logging new attempts
        clean_expired_user_attempts(request.axes_attempt_time)

        username = get_client_username(request, credentials)
        client_str = get_client_str(
            username,
            request.axes_ip_address,
            request.axes_user_agent,
            request.axes_path_info,
        )

        # This replaces null byte chars that crash saving failures.
        get_data = get_query_str(request.GET).replace("\0", "0x00")
        post_data = get_query_str(request.POST).replace("\0", "0x00")

        if self.is_whitelisted(request, credentials):
            log.info("AXES: Login failed from whitelisted client %s.", client_str)
            return

        # 2. database query: Get or create access record with the new failure data
        if settings.AXES_ONLY_USER_FAILURES and username is None:
            log.warning(
                "AXES: Username is None and AXES_ONLY_USER_FAILURES is enabled, new record will NOT be created."
            )
        else:
            attempt, created = AccessAttempt.objects.get_or_create(
                username=username,
                ip_address=request.axes_ip_address,
                user_agent=request.axes_user_agent,
                defaults={
                    "get_data": get_data,
                    "post_data": post_data,
                    "http_accept": request.axes_http_accept,
                    "path_info": request.axes_path_info,
                    "failures_since_start": 1,
                    "attempt_time": request.axes_attempt_time,
                },
            )

            # Record failed attempt with all the relevant information.
            # Filtering based on username, IP address and user agent handled elsewhere,
            # and this handler just records the available information for further use.
            if created:
                log.warning(
                    "AXES: New login failure by %s. Created new record in the database.",
                    client_str,
                )

            # 3. database query if there were previous attempts in the database
            # Update failed attempt information but do not touch the username, IP address, or user agent fields,
            # because attackers can request the site with multiple different configurations
            # in order to bypass the defense mechanisms that are used by the site.
            else:
                separator = "\n---------\n"

                failures_since_start_prev = attempt.failures_since_start

                attempt.get_data = Concat("get_data", Value(separator + get_data))
                attempt.post_data = Concat("post_data", Value(separator + post_data))
                attempt.http_accept = request.axes_http_accept
                attempt.path_info = request.axes_path_info
                attempt.failures_since_start = F("failures_since_start") + 1
                attempt.attempt_time = request.axes_attempt_time
                attempt.save()

                log.warning(
                    "AXES: Repeated login failure by %s. Count = %d of %d. Updated existing record in the database.",
                    client_str,
                    # can be different from the actual value in the DB
                    failures_since_start_prev + 1,
                    get_failure_limit(request, credentials),
                )

        # 3. or 4. database query: Calculate the current maximum failure number from the existing attempts
        failures_since_start = self.get_failures(request, credentials)

        if (
            settings.AXES_LOCK_OUT_AT_FAILURE
            and failures_since_start >= get_failure_limit(request, credentials)
        ):
            log.warning(
                "AXES: Locking out %s after repeated login failures.", client_str
            )

            request.axes_locked_out = True
            user_locked_out.send(
                "axes",
                request=request,
                username=username,
                ip_address=request.axes_ip_address,
            )

    def user_logged_in(
        self, sender, request, user, **kwargs
    ):  # pylint: disable=unused-argument
        """
        When user logs in, update the AccessLog related to the user.
        """

        # 1. database query: Clean up expired user attempts from the database
        clean_expired_user_attempts(request.axes_attempt_time)

        username = user.get_username()
        credentials = get_credentials(username)
        client_str = get_client_str(
            username,
            request.axes_ip_address,
            request.axes_user_agent,
            request.axes_path_info,
        )

        log.info("AXES: Successful login by %s.", client_str)

        if not settings.AXES_DISABLE_ACCESS_LOG:
            # 2. database query: Insert new access logs with login time
            AccessLog.objects.create(
                username=username,
                ip_address=request.axes_ip_address,
                user_agent=request.axes_user_agent,
                http_accept=request.axes_http_accept,
                path_info=request.axes_path_info,
                attempt_time=request.axes_attempt_time,
            )

        if settings.AXES_RESET_ON_SUCCESS:
            # 3. database query: Reset failed attempts for the logging in user
            count = reset_user_attempts(request, credentials)
            log.info(
                "AXES: Deleted %d failed login attempts by %s from database.",
                count,
                client_str,
            )

    def user_logged_out(
        self, sender, request, user, **kwargs
    ):  # pylint: disable=unused-argument
        """
        When user logs out, update the AccessLog related to the user.
        """

        # 1. database query: Clean up expired user attempts from the database
        clean_expired_user_attempts(request.axes_attempt_time)

        username = user.get_username() if user else None
        client_str = get_client_str(
            username,
            request.axes_ip_address,
            request.axes_user_agent,
            request.axes_path_info,
        )

        log.info("AXES: Successful logout by %s.", client_str)

        if username and not settings.AXES_DISABLE_ACCESS_LOG:
            # 2. database query: Update existing attempt logs with logout time
            AccessLog.objects.filter(
                username=username, logout_time__isnull=True
            ).update(logout_time=request.axes_attempt_time)

    def post_save_access_attempt(self, instance, **kwargs):
        """
        Handles the ``axes.models.AccessAttempt`` object post save signal.

        When needed, all post_save actions for this backend should be located
        here.
        """

    def post_delete_access_attempt(self, instance, **kwargs):
        """
        Handles the ``axes.models.AccessAttempt`` object post delete signal.

        When needed, all post_delete actions for this backend should be located
        here.
        """
<|endoftext|>