<commit_before>#! /usr/bin/env python

import sys
import os
import argparse
import codecs
import csv
import hashlib
from operator import itemgetter
from itertools import groupby

"""
NOTES
-----

## Checksumming

Check out this:

<http://stackoverflow.com/questions/3431825/generating-an-md5-checksum-of-a-file>

## Batch validation

Before doing ANYTHING, we'll also need to do some basic validation at
the batch level, e.g.:

* Check for duplicate identifier - volumeNumber combinations (not permitted)
* Check for presence of different carrierTypes within one identifier (not permitted)
* Check for missing checksums
* Checksum verification for all items in batch
* Check if all imagePath fields in CSV correspond to actual dir in batch
* Check if all dirs in batch are represented as an imagePath field

This validation could either be done within this SIP creator, or as a separate script.

## Code reuse

* Metamorfoze Batch converter (CSV, validation, progress and error logging)
* KB-python-API (importing of bibliographical metadata from GGC)
* For metadata generation in e.g. METS format some libs probably exist already

"""

# Script name
scriptPath, scriptName = os.path.split(sys.argv[0])

# scriptName is empty when called from Java/Jython, so this needs a fix
if len(scriptName) == 0:
    scriptName = 'omsipcreator'

__version__ = "0.1.0"

# Create parser
parser = argparse.ArgumentParser(
    description="SIP creation tool for optical media images")

def main_is_frozen():
    return (hasattr(sys, "frozen") or  # new py2exe
            hasattr(sys, "importers")  # old py2exe
            or imp.is_frozen("__main__"))  # tools/freeze

def get_main_dir():
    if main_is_frozen():
        return os.path.dirname(sys.executable)
    return os.path.dirname(sys.argv[0])

def printWarning(msg):
    msgString=("User warning: " + msg +"\n")
    sys.stderr.write(msgString)

def printInfo(msg):
    msgString=(msg + "\n")
    sys.stderr.write(msgString)

def errorExit(msg):
    msgString=("Error: " + msg + "\n")
    sys.stderr.write(msgString)
    sys.exit()

def parseCommandLine():
    # Add arguments

    parser.add_argument('batchIn',
                        action="store",
                        type=str,
                        help="input batch")
    parser.add_argument('dirOut',
                        action="store",
                        type=str,
                        help="output directory")

    # Parse arguments
    args = parser.parse_args()

    return(args)

def main():

    # Constants (put in config file later)
    fileMetaCarriers = "metacarriers.csv"

    # Get input from command line
    args = parseCommandLine()
    batchIn = os.path.normpath(args.batchIn)
    dirOut = os.path.normpath(args.dirOut)

    # TODO: perhaps the checks below (which now all result in an errorexit)
    # could be formalised a bit, so that they can be reworked into a validation
    # report.

    # Check if batch dir exists
    if os.path.isdir(batchIn) == False:
        msg = "input batch directory does not exist"
        errorExit(msg)

    # Check if batch-level metadata file exists
    metaCarriers = os.path.normpath(batchIn + "/" + fileMetaCarriers)
    if os.path.isfile(metaCarriers) == False:
        msg = "File " + metaCarriers + " does not exist"
        errorExit(msg)

    # Read carrier-level metadata file as CSV and import header and
    # row data to 2 separate lists
    try:
        fMetaCarriers = open(metaCarriers,"rb")
        metaCarriersCSV = csv.reader(fMetaCarriers)
        headerMetaCarriers = next(metaCarriersCSV)
        rowsMetaCarriers = [row for row in metaCarriersCSV]
        fMetaCarriers.close()
    except IOError:
        msg = "cannot read " + metaCarriers
        errorExit(msg)
    except csv.Error:
        msg = "error parsing carrier metadata CSV"
        errorExit(msg)

    # Remove any empty list elements (e.g. due to EOL chars)
    # to avoid trouble with itemgetter
    for item in rowsMetaCarriers:
        if item == []:
            rowsMetaCarriers.remove(item)

    # ********
    # ** Verification of carrier-level metadata file **
    # ********

    # Set up lists for storing errors and warnings
    errors = []
    warnings = []

    # Header values of mandatory columns
    requiredColsMetaCarriers = ['IPIdentifier',
                                'IPIdentifierParent',
                                'imagePath',
                                'volumeNumber',
                                'carrierType']

    # Check that there is exactly one occurrence of each mandatory column
    for requiredCol in requiredColsMetaCarriers:
        occurs = headerMetaCarriers.count(requiredCol)
        if occurs != 1:
            error = "found " + str(occurs) + " occurrences of column " + requiredCol + " in " + fileMetaCarriers + \
            " (expected 1)"
            errors.append(error)

    # Set up dictionary to store header fields and corresponding col numbers
    colsMetaCarriers = {}

    col = 0
    for header in headerMetaCarriers:
        colsMetaCarriers[header] = col
        col += 1

    # Sort rows by IPIdentifier field
    rowsMetaCarriers.sort(key=itemgetter(0))

    print(rowsMetaCarriers)

    # Iterate over all carrier entries

    row = 0
    for carrier in rowsMetaCarriers:
        # Ignore header row
        if row != 0:
            IPIdentifier = carrier[colsMetaCarriers["IPIdentifier"]]
            IPIdentifierParent = carrier[colsMetaCarriers["IPIdentifierParent"]]
            imagePath = carrier[colsMetaCarriers["imagePath"]]
            volumeNumber = carrier[colsMetaCarriers["volumeNumber"]]
            carrierType = carrier[colsMetaCarriers["carrierType"]]

            print(IPIdentifier)

        row += 1

    #print(colsMetaCarriers)
    #print(errors)

    """
    # Create output dir if it doesn't exist already
    if os.path.isdir(dirOut) == False:
        try:
            os.makedirs(dirOut)
        except IOError:
            msg = "cannot create output directory"
            errorExit(msg)
    """

if __name__ == "__main__":
    main()
<commit_msg>Each IP represented as separate grouper object<commit_after>#! /usr/bin/env python

import sys
import os
import argparse
import codecs
import csv
import hashlib
from operator import itemgetter
from itertools import groupby

"""
NOTES
-----

## Checksumming

Check out this:

<http://stackoverflow.com/questions/3431825/generating-an-md5-checksum-of-a-file>

## Batch validation

Before doing ANYTHING, we'll also need to do some basic validation at
the batch level, e.g.:

* Check for duplicate identifier - volumeNumber combinations (not permitted)
* Check for presence of different carrierTypes within one identifier (not permitted)
* Check for missing checksums
* Checksum verification for all items in batch
* Check if all imagePath fields in CSV correspond to actual dir in batch
* Check if all dirs in batch are represented as an imagePath field

This validation could either be done within this SIP creator, or as a separate script.

## Code reuse

* Metamorfoze Batch converter (CSV, validation, progress and error logging)
* KB-python-API (importing of bibliographical metadata from GGC)
* For metadata generation in e.g. METS format some libs probably exist already

"""

# Script name
scriptPath, scriptName = os.path.split(sys.argv[0])

# scriptName is empty when called from Java/Jython, so this needs a fix
if len(scriptName) == 0:
    scriptName = 'omsipcreator'

__version__ = "0.1.0"

# Create parser
parser = argparse.ArgumentParser(
    description="SIP creation tool for optical media images")

def main_is_frozen():
    return (hasattr(sys, "frozen") or  # new py2exe
            hasattr(sys, "importers")  # old py2exe
            or imp.is_frozen("__main__"))  # tools/freeze

def get_main_dir():
    if main_is_frozen():
        return os.path.dirname(sys.executable)
    return os.path.dirname(sys.argv[0])

def printWarning(msg):
    msgString=("User warning: " + msg +"\n")
    sys.stderr.write(msgString)

def printInfo(msg):
    msgString=(msg + "\n")
    sys.stderr.write(msgString)

def errorExit(msg):
    msgString=("Error: " + msg + "\n")
    sys.stderr.write(msgString)
    sys.exit()

def parseCommandLine():
    # Add arguments

    parser.add_argument('batchIn',
                        action="store",
                        type=str,
                        help="input batch")
    parser.add_argument('dirOut',
                        action="store",
                        type=str,
                        help="output directory")

    # Parse arguments
    args = parser.parse_args()

    return(args)

def main():

    # Constants (put in config file later)
    fileMetaCarriers = "metacarriers.csv"

    # Get input from command line
    args = parseCommandLine()
    batchIn = os.path.normpath(args.batchIn)
    dirOut = os.path.normpath(args.dirOut)

    # TODO: perhaps the checks below (which now all result in an errorexit)
    # could be formalised a bit, so that they can be reworked into a validation
    # report.

    # Check if batch dir exists
    if os.path.isdir(batchIn) == False:
        msg = "input batch directory does not exist"
        errorExit(msg)

    # Check if batch-level metadata file exists
    metaCarriers = os.path.normpath(batchIn + "/" + fileMetaCarriers)
    if os.path.isfile(metaCarriers) == False:
        msg = "File " + metaCarriers + " does not exist"
        errorExit(msg)

    # Read carrier-level metadata file as CSV and import header and
    # row data to 2 separate lists
    try:
        fMetaCarriers = open(metaCarriers,"rb")
        metaCarriersCSV = csv.reader(fMetaCarriers)
        headerMetaCarriers = next(metaCarriersCSV)
        rowsMetaCarriers = [row for row in metaCarriersCSV]
        fMetaCarriers.close()
    except IOError:
        msg = "cannot read " + metaCarriers
        errorExit(msg)
    except csv.Error:
        msg = "error parsing carrier metadata CSV"
        errorExit(msg)

    # Remove any empty list elements (e.g. due to EOL chars)
    # to avoid trouble with itemgetter
    for item in rowsMetaCarriers:
        if item == []:
            rowsMetaCarriers.remove(item)

    # ********
    # ** Verification of carrier-level metadata file **
    # ********

    # Set up lists for storing errors and warnings
    errors = []
    warnings = []

    # Header values of mandatory columns
    requiredColsMetaCarriers = ['IPIdentifier',
                                'IPIdentifierParent',
                                'imagePath',
                                'volumeNumber',
                                'carrierType']

    # Check that there is exactly one occurrence of each mandatory column
    # TODO: bad things will happen in case of missing cols, so maybe re-introduce errorExit
    for requiredCol in requiredColsMetaCarriers:
        occurs = headerMetaCarriers.count(requiredCol)
        if occurs != 1:
            error = "found " + str(occurs) + " occurrences of column " + requiredCol + " in " + fileMetaCarriers + \
            " (expected 1)"
            errors.append(error)

    # Set up dictionary to store header fields and corresponding column numbers
    colsMetaCarriers = {}

    col = 0
    for header in headerMetaCarriers:
        colsMetaCarriers[header] = col
        col += 1

    # Sort rows by IPIdentifier field
    rowsMetaCarriers.sort(key=itemgetter(0))

    # Group by IPIdentifier field - creates a grouper object for each IP
    metaCarriersByIP = groupby(rowsMetaCarriers, itemgetter(0))

    # Iterate over IPs
    for key, group in metaCarriersByIP:
        # key is IPIdentifier (by which we grouped data)
        # group is another iterator (!)
        print(key, group)
        for record in group:
            print(record)

    # Iterate over all carrier entries

    row = 0
    for carrier in rowsMetaCarriers:
        # Ignore header row
        if row != 0:
            IPIdentifier = carrier[colsMetaCarriers["IPIdentifier"]]
            IPIdentifierParent = carrier[colsMetaCarriers["IPIdentifierParent"]]
            imagePath = carrier[colsMetaCarriers["imagePath"]]
            volumeNumber = carrier[colsMetaCarriers["volumeNumber"]]
            carrierType = carrier[colsMetaCarriers["carrierType"]]

            #print(IPIdentifier)

        row += 1

    #print(colsMetaCarriers)
    #print(errors)

    """
    # Create output dir if it doesn't exist already
    if os.path.isdir(dirOut) == False:
        try:
            os.makedirs(dirOut)
        except IOError:
            msg = "cannot create output directory"
            errorExit(msg)
    """

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from __future__ import absolute_import

# need to get system mendeley library
from mendeley.exception import MendeleyException
import mendeley as mendeley_lib

from util import remove_punctuation
import os
import requests

mendeley_session = None

def get_mendeley_session():
    a = requests.adapters.HTTPAdapter(pool_connections = 150, pool_maxsize = 150)

    mendeley_client = mendeley_lib.Mendeley(
        client_id=os.getenv("MENDELEY_OAUTH2_CLIENT_ID"),
        client_secret=os.getenv("MENDELEY_OAUTH2_SECRET"))
    auth = mendeley_client.start_client_credentials_flow()
    session = auth.authenticate()
    return session

def set_mendeley_data(product):
    global mendeley_session

    if not mendeley_session:
        mendeley_session = get_mendeley_session()

    resp = None
    try:
        biblio_title = remove_punctuation(product.title).lower()
        biblio_year = product.year
        if biblio_title and biblio_year:
            try:
                doc = mendeley_session.catalog.advanced_search(
                        title=biblio_title,
                        min_year=biblio_year,
                        max_year=biblio_year,
                        view='stats').list(page_size=1).items[0]
            except (UnicodeEncodeError, IndexError):
                biblio_title = remove_punctuation(product.title.encode('ascii','ignore'))
                try:
                    doc = mendeley_session.catalog.advanced_search(
                            title=biblio_title,
                            min_year=biblio_year,
                            max_year=biblio_year,
                            view='stats').list(page_size=1).items[0]
                except (IndexError):
                    return None

            mendeley_title = remove_punctuation(doc.title).lower()
            if biblio_title == mendeley_title:
                # print u"\nMatch! got the mendeley paper! for title {}".format(biblio_title)
                print "got mendeley for {}".format(self.id)
                resp = {}
                resp["reader_count"] = doc.reader_count
                resp["reader_count_by_academic_status"] = doc.reader_count_by_academic_status
                resp["reader_count_by_subdiscipline"] = doc.reader_count_by_subdiscipline
                resp["reader_count_by_country"] = doc.reader_count_by_country
                resp["abstract"] = doc.abstract
            else:
                # print u"didn't find {}".format(biblio_title)
                # print u"Mendeley: titles don't match so not using this match \n%s and \n%s" %(
                #     biblio_title, mendeley_title)
                resp = None
    except (KeyError, MendeleyException):
        # logger.info(u"No biblio found in _get_doc_by_title")
        pass

    # if doc:
        # try:
        #     drilldown_url = doc.link
        #     metrics_and_drilldown["mendeley:readers"] = (doc.reader_count, drilldown_url)
        #     metrics_and_drilldown["mendeley:career_stage"] = (doc.reader_count_by_academic_status, drilldown_url)
        #
        #     by_discipline = {}
        #     by_subdiscipline = doc.reader_count_by_subdiscipline
        #     if by_subdiscipline:
        #         for discipline, subdiscipline_breakdown in by_subdiscipline.iteritems():
        #             by_discipline[discipline] = sum(subdiscipline_breakdown.values())
        #         metrics_and_drilldown["mendeley:discipline"] = (by_discipline, drilldown_url)
        #
        #     by_country_iso = {}
        #     by_country_names = doc.reader_count_by_country
        #     if by_country_names:
        #         for country_name, country_breakdown in by_country_names.iteritems():
        #             iso = iso_code_from_name(country_name)
        #             if iso:
        #                 by_country_iso[iso] = country_breakdown
        #             else:
        #                 logger.error(u"Can't find country {country} in lookup".format(
        #                     country=country_name))
        #         if by_country_iso:
        #             metrics_and_drilldown["mendeley:countries"] = (by_country_iso, drilldown_url)
        #
        # except KeyError:
        #     pass

    return resp

<commit_msg>typo!<commit_after>from __future__ import absolute_import

# need to get system mendeley library
from mendeley.exception import MendeleyException
import mendeley as mendeley_lib

from util import remove_punctuation
import os
import requests

mendeley_session = None

def get_mendeley_session():
    a = requests.adapters.HTTPAdapter(pool_connections = 150, pool_maxsize = 150)

    mendeley_client = mendeley_lib.Mendeley(
        client_id=os.getenv("MENDELEY_OAUTH2_CLIENT_ID"),
        client_secret=os.getenv("MENDELEY_OAUTH2_SECRET"))
    auth = mendeley_client.start_client_credentials_flow()
    session = auth.authenticate()
    return session

def set_mendeley_data(product):
    global mendeley_session

    if not mendeley_session:
        mendeley_session = get_mendeley_session()

    resp = None
    try:
        biblio_title = remove_punctuation(product.title).lower()
        biblio_year = product.year
        if biblio_title and biblio_year:
            try:
                doc = mendeley_session.catalog.advanced_search(
                        title=biblio_title,
                        min_year=biblio_year,
                        max_year=biblio_year,
                        view='stats').list(page_size=1).items[0]
            except (UnicodeEncodeError, IndexError):
                biblio_title = remove_punctuation(product.title.encode('ascii','ignore'))
                try:
                    doc = mendeley_session.catalog.advanced_search(
                            title=biblio_title,
                            min_year=biblio_year,
                            max_year=biblio_year,
                            view='stats').list(page_size=1).items[0]
                except (IndexError):
                    return None

            mendeley_title = remove_punctuation(doc.title).lower()
            if biblio_title == mendeley_title:
                # print u"\nMatch! got the mendeley paper! for title {}".format(biblio_title)
                print "got mendeley for {}".format(product.id)
                resp = {}
                resp["reader_count"] = doc.reader_count
                resp["reader_count_by_academic_status"] = doc.reader_count_by_academic_status
                resp["reader_count_by_subdiscipline"] = doc.reader_count_by_subdiscipline
                resp["reader_count_by_country"] = doc.reader_count_by_country
                resp["abstract"] = doc.abstract
            else:
                # print u"didn't find {}".format(biblio_title)
                # print u"Mendeley: titles don't match so not using this match \n%s and \n%s" %(
                #     biblio_title, mendeley_title)
                resp = None
    except (KeyError, MendeleyException):
        # logger.info(u"No biblio found in _get_doc_by_title")
        pass

    # if doc:
        # try:
        #     drilldown_url = doc.link
        #     metrics_and_drilldown["mendeley:readers"] = (doc.reader_count, drilldown_url)
        #     metrics_and_drilldown["mendeley:career_stage"] = (doc.reader_count_by_academic_status, drilldown_url)
        #
        #     by_discipline = {}
        #     by_subdiscipline = doc.reader_count_by_subdiscipline
        #     if by_subdiscipline:
        #         for discipline, subdiscipline_breakdown in by_subdiscipline.iteritems():
        #             by_discipline[discipline] = sum(subdiscipline_breakdown.values())
        #         metrics_and_drilldown["mendeley:discipline"] = (by_discipline, drilldown_url)
        #
        #     by_country_iso = {}
        #     by_country_names = doc.reader_count_by_country
        #     if by_country_names:
        #         for country_name, country_breakdown in by_country_names.iteritems():
        #             iso = iso_code_from_name(country_name)
        #             if iso:
        #                 by_country_iso[iso] = country_breakdown
        #             else:
        #                 logger.error(u"Can't find country {country} in lookup".format(
        #                     country=country_name))
        #         if by_country_iso:
        #             metrics_and_drilldown["mendeley:countries"] = (by_country_iso, drilldown_url)
        #
        # except KeyError:
        #     pass

    return resp

<|endoftext|>


<commit_before>import json, urllib

from django.conf import settings
from django.utils import crypto

from mongoengine import queryset
from mongoengine.django import auth

import tweepy

from piplmesh.account import models

LAZYUSER_USERNAME_TEMPLATE = 'guest-%s'

class MongoEngineBackend(auth.MongoEngineBackend):
    # TODO: Implement object permission support
    supports_object_permissions = False
    # TODO: Implement anonymous user backend
    supports_anonymous_user = False
    # TODO: Implement inactive user backend
    supports_inactive_user = False

    def authenticate(self, username, password):
        user = self.user_class.objects(username__iexact=username).first()
        if user:
            if password and user.check_password(password):
                return user
        return None

    def get_user(self, user_id):
        try:
            return self.user_class.objects.with_id(user_id)
        except self.user_class.DoesNotExist:
            return None

    @property
    def user_class(self):
        return models.User

class FacebookBackend(MongoEngineBackend):
    """
    Facebook authentication.
    """

    def authenticate(self, facebook_access_token, request):
        # Retrieve user's profile information
        # TODO: Handle error, what if request was denied?
        facebook_profile_data = json.load(urllib.urlopen('https://graph.facebook.com/me?%s' % urllib.urlencode({'access_token': facebook_access_token})))

        try:
            user = self.user_class.objects.get(facebook_profile_data__id=facebook_profile_data.get('id'))
        except self.user_class.DoesNotExist:
            # We reload to make sure user object is recent
            user = request.user.reload()
            # TODO: Is it OK to override Facebook link if it already exist with some other Facebook user?

        user.facebook_access_token = facebook_access_token
        user.facebook_profile_data = facebook_profile_data

        if user.lazyuser_username and facebook_profile_data.get('username'):
            # TODO: Does Facebook have same restrictions on username content as we do?
            user.username = facebook_profile_data.get('username')
            user.lazyuser_username = False
        if user.first_name is None:
            user.first_name = facebook_profile_data.get('first_name') or None
        if user.last_name is None:
            user.last_name = facebook_profile_data.get('last_name') or None
        if user.email is None:
            # TODO: Do we know if all e-mail addresses given by Facebook are verified?
            # TODO: Does not Facebook support multiple e-mail addresses? Which one is given here?
            user.email = facebook_profile_data.get('email') or None
        if user.gender is None:
            # TODO: Does it really map so cleanly?
            user.gender = facebook_profile_data.get('gender') or None

        user.save()

        return user

class TwitterBackend(MongoEngineBackend):
    """
    Twitter authentication.

    Twitter profile data fields are:
        name: user's full name
        screen_name: user's username
        profile_image_url
        profile_background_image_url
        id: id of Twitter user
        id_str: string id of Twitter user
        created_at: full date of user's registration on Twitter
        location: user's location
        time_zone
        lang: user's preferred language
        url: URL of user's website
        description: user's description of themselves
        profile_sidebar_fill_color
        profile_text_color
        profile_background_color
        profile_link_color
        profile_sidebar_border_color
        friends_count
        followers_count
        statuses_count
        favourites_count
        listed_count
        notifications: boolean value
        geo_enabled: boolean value
        following: boolean value
        follow_request_sent: boolean value
        profile_use_background_image: boolean value
        verified: boolean value; tells whether user's email is verified
        protected: boolean value
        show_all_inline_media: boolean value
        is_translator: boolean value
        profile_background_tile: boolean value
        contributors_enabled: boolean value
        utc_offset: integer
    """

    def authenticate(self, twitter_access_token, request):
        twitter_auth = tweepy.OAuthHandler(settings.TWITTER_CONSUMER_KEY, settings.TWITTER_CONSUMER_SECRET)
        twitter_auth.set_access_token(twitter_access_token.key, twitter_access_token.secret)
        twitter_api = tweepy.API(twitter_auth)

        twitter_profile_data = twitter_api.me()

        try:
            user = self.user_class.objects.get(twitter_profile_data__id=twitter_profile_data.get('id'))
        except self.user_class.DoesNotExist:
            # We reload to make sure user object is recent
            user = request.user.reload()
            # TODO: Is it OK to override Twitter link if it already exist with some other Twitter user?

        user.twitter_access_token = models.TwitterAccessToken(key=twitter_access_token.key, secret=twitter_access_token.secret)
        user.twitter_profile_data = twitter_profile_data

        if user.lazyuser_username and twitter_profile_data.get('username'):
            # TODO: Does Twitter have same restrictions on username content as we do?
            user.username = twitter_profile_data.get('screen_name')
            user.lazyuser_username = False
        if user.first_name is None:
            user.first_name = twitter_profile_data.get('name') or None

        user.save()

        return user

class GoogleBackend(MongoEngineBackend):
    """
    Google authentication.

    Google profile data fields are:
        family_name: last name
        given_name: first name
        name: full name
        link: URL of Google user profile page
        picture: URL of profile picture
        locale: the language Google user is using
        gender: sex of Google user
        email: Google email of user
        id: id of Google user; should be a string
        verified_email: True, if email is verified by Google API
    """

    def authenticate(self, google_access_token, request):
        # Retrieve user's profile information
        # TODO: Handle error, what if request was denied?
        google_profile_data = json.load(urllib.urlopen('https://www.googleapis.com/oauth2/v1/userinfo?access_token=%s' % google_access_token))

        try:
            user = self.user_class.objects.get(google_profile_data__id=google_profile_data.get('id'))
        except self.user_class.DoesNotExist:
            # We reload to make sure user object is recent
            user = request.user.reload()
            # TODO: Is it OK to override Google link if it already exist with some other Google user?

        user.google_access_token = google_access_token
        user.google_profile_data = google_profile_data

        if user.first_name is None:
            user.first_name = google_profile_data.get('given_name') or None
        if user.last_name is None:
            user.last_name = google_profile_data.get('family_name') or None
        if user.email is None:
            user.email = google_profile_data.get('email') or None
        if user.gender is None:
            # TODO: Does it really map so cleanly?
            user.gender = google_profile_data.get('gender') or None

        user.save()

        return user

class FoursquareBackend(MongoEngineBackend):
    """
    Foursquare authentication.

    Foursquare user fields are:
    id - A unique identifier for this user.
    firstName - A user's first name.
    lastName - A user's last name.
    homeCity - User's home city.
    photo - URL of a profile picture for this user.
    gender - A user's gender: male, female, or none.
    relationship - (Optional) The relationship of the acting user (me) to this user (them).

    Fields present in user details:
    type, contact, pings, badges, checkins, mayorships, tips, todos, photos, friends, followers,
    requests, pageInfo.
    """

    def authenticate(self, foursquare_access_token, request):
        # Retrieve user's profile information
        # TODO: Handle error, what if request was denied?
        foursquare_user_data = json.load(urllib.urlopen('https://api.foursquare.com/v2/users/self?%s' % urllib.urlencode({'oauth_token': foursquare_access_token})))
        foursquare_profile_data = foursquare_user_data['response']['user']

        try:
            user = self.user_class.objects.get(foursquare_profile_data__id=foursquare_profile_data.get('id'))
        except self.user_class.DoesNotExist:
            # We reload to make sure user object is recent
            user = request.user.reload()
            # TODO: Is it OK to override Foursquare link if it already exist with some other Foursquare user?

        user.foursquare_access_token = foursquare_access_token
        user.foursquare_profile_data = foursquare_profile_data

        if user.first_name is None:
            user.first_name = foursquare_profile_data.get('firstName') or None
        if user.last_name is None:
            user.last_name = foursquare_profile_data.get('lastName') or None
        if user.email is None:
            user.email = foursquare_profile_data.get('contact', {}).get('email') or None
        if user.gender is None:
            # TODO: Does it really map so cleanly?
            user.gender = foursquare_profile_data.get('gender') or None

        user.save()

        return user

class LazyUserBackend(MongoEngineBackend):
    def authenticate(self):
        while True:
            try:
                username = LAZYUSER_USERNAME_TEMPLATE % crypto.get_random_string(6)
                user = self.user_class.objects.create(
                    username=username,
                )
                break
            except queryset.OperationError, e:
                msg = str(e)
                if 'E11000' in msg and 'duplicate key error' in msg and 'username' in msg:
                    continue
                raise

        return user
<commit_msg>Some changes.<commit_after>import json, urllib

from django.conf import settings
from django.utils import crypto

from mongoengine import queryset
from mongoengine.django import auth

import tweepy

from piplmesh.account import models

LAZYUSER_USERNAME_TEMPLATE = 'guest-%s'

class MongoEngineBackend(auth.MongoEngineBackend):
    # TODO: Implement object permission support
    supports_object_permissions = False
    # TODO: Implement anonymous user backend
    supports_anonymous_user = False
    # TODO: Implement inactive user backend
    supports_inactive_user = False

    def authenticate(self, username, password):
        user = self.user_class.objects(username__iexact=username).first()
        if user:
            if password and user.check_password(password):
                return user
        return None

    def get_user(self, user_id):
        try:
            return self.user_class.objects.with_id(user_id)
        except self.user_class.DoesNotExist:
            return None

    @property
    def user_class(self):
        return models.User

class FacebookBackend(MongoEngineBackend):
    """
    Facebook authentication.
    """

    def authenticate(self, facebook_access_token, request):
        # Retrieve user's profile information
        # TODO: Handle error, what if request was denied?
        facebook_profile_data = json.load(urllib.urlopen('https://graph.facebook.com/me?%s' % urllib.urlencode({'access_token': facebook_access_token})))

        try:
            user = self.user_class.objects.get(facebook_profile_data__id=facebook_profile_data.get('id'))
        except self.user_class.DoesNotExist:
            # We reload to make sure user object is recent
            user = request.user.reload()
            # TODO: Is it OK to override Facebook link if it already exist with some other Facebook user?

        user.facebook_access_token = facebook_access_token
        user.facebook_profile_data = facebook_profile_data

        if user.lazyuser_username and facebook_profile_data.get('username'):
            # TODO: Does Facebook have same restrictions on username content as we do?
            user.username = facebook_profile_data.get('username')
            user.lazyuser_username = False
        if user.first_name is None:
            user.first_name = facebook_profile_data.get('first_name') or None
        if user.last_name is None:
            user.last_name = facebook_profile_data.get('last_name') or None
        if user.email is None:
            # TODO: Do we know if all e-mail addresses given by Facebook are verified?
            # TODO: Does not Facebook support multiple e-mail addresses? Which one is given here?
            user.email = facebook_profile_data.get('email') or None
        if user.gender is None:
            # TODO: Does it really map so cleanly?
            user.gender = facebook_profile_data.get('gender') or None

        user.save()

        return user

class TwitterBackend(MongoEngineBackend):
    """
    Twitter authentication.

    Twitter profile data fields are:
        name: user's full name
        screen_name: user's username
        profile_image_url
        profile_background_image_url
        id: id of Twitter user
        id_str: string id of Twitter user
        created_at: full date of user's registration on Twitter
        location: user's location
        time_zone
        lang: user's preferred language
        url: URL of user's website
        description: user's description of themselves
        profile_sidebar_fill_color
        profile_text_color
        profile_background_color
        profile_link_color
        profile_sidebar_border_color
        friends_count
        followers_count
        statuses_count
        favourites_count
        listed_count
        notifications: boolean value
        geo_enabled: boolean value
        following: boolean value
        follow_request_sent: boolean value
        profile_use_background_image: boolean value
        verified: boolean value; tells whether user's email is verified
        protected: boolean value
        show_all_inline_media: boolean value
        is_translator: boolean value
        profile_background_tile: boolean value
        contributors_enabled: boolean value
        utc_offset: integer
    """

    def authenticate(self, twitter_access_token, request):
        twitter_auth = tweepy.OAuthHandler(settings.TWITTER_CONSUMER_KEY, settings.TWITTER_CONSUMER_SECRET)
        twitter_auth.set_access_token(twitter_access_token.key, twitter_access_token.secret)
        twitter_api = tweepy.API(twitter_auth)

        twitter_profile_data = twitter_api.me()

        try:
            user = self.user_class.objects.get(twitter_profile_data__id=twitter_profile_data.get('id'))
        except self.user_class.DoesNotExist:
            # We reload to make sure user object is recent
            user = request.user.reload()
            # TODO: Is it OK to override Twitter link if it already exist with some other Twitter user?

        user.twitter_access_token = models.TwitterAccessToken(key=twitter_access_token.key, secret=twitter_access_token.secret)
        user.twitter_profile_data = twitter_profile_data

        if user.lazyuser_username and twitter_profile_data.get('username'):
            # TODO: Does Twitter have same restrictions on username content as we do?
            user.username = twitter_profile_data.get('screen_name')
            user.lazyuser_username = False
        if user.first_name is None:
            user.first_name = twitter_profile_data.get('name') or None

        user.save()

        return user

class GoogleBackend(MongoEngineBackend):
    """
    Google authentication.

    Google profile data fields are:
        family_name: last name
        given_name: first name
        name: full name
        link: URL of Google user profile page
        picture: URL of profile picture
        locale: the language Google user is using
        gender: sex of Google user
        email: Google email of user
        id: id of Google user; should be a string
        verified_email: True, if email is verified by Google API
    """

    def authenticate(self, google_access_token, request):
        # Retrieve user's profile information
        # TODO: Handle error, what if request was denied?
        google_profile_data = json.load(urllib.urlopen('https://www.googleapis.com/oauth2/v1/userinfo?access_token=%s' % google_access_token))

        try:
            user = self.user_class.objects.get(google_profile_data__id=google_profile_data.get('id'))
        except self.user_class.DoesNotExist:
            # We reload to make sure user object is recent
            user = request.user.reload()
            # TODO: Is it OK to override Google link if it already exist with some other Google user?

        user.google_access_token = google_access_token
        user.google_profile_data = google_profile_data

        if user.first_name is None:
            user.first_name = google_profile_data.get('given_name') or None
        if user.last_name is None:
            user.last_name = google_profile_data.get('family_name') or None
        if user.email is None:
            user.email = google_profile_data.get('email') or None
        if user.gender is None:
            # TODO: Does it really map so cleanly?
            user.gender = google_profile_data.get('gender') or None

        user.save()

        return user

class FoursquareBackend(MongoEngineBackend):
    """
    Foursquare authentication.

    Foursquare user fields are:
        id - A unique identifier for this user.
        firstName - A user's first name.
        lastName - A user's last name.
        homeCity - User's home city.
        photo - URL of a profile picture for this user.
        gender - A user's gender: male, female, or none.
        relationship - (Optional) The relationship of the acting user (me) to this user (them).

        Fields present in user details:
        type, contact, pings, badges, checkins, mayorships, tips, todos, photos, friends, followers,
        requests, pageInfo.
    """

    def authenticate(self, foursquare_access_token, request):
        # Retrieve user's profile information
        # TODO: Handle error, what if request was denied?
        foursquare_user_data = json.load(urllib.urlopen('https://api.foursquare.com/v2/users/self?%s' % urllib.urlencode({'oauth_token': foursquare_access_token})))
        foursquare_profile_data = foursquare_user_data['response']['user']

        try:
            user = self.user_class.objects.get(foursquare_profile_data__id=foursquare_profile_data.get('id'))
        except self.user_class.DoesNotExist:
            # We reload to make sure user object is recent
            user = request.user.reload()
            # TODO: Is it OK to override Foursquare link if it already exist with some other Foursquare user?

        user.foursquare_access_token = foursquare_access_token
        user.foursquare_profile_data = foursquare_profile_data

        if user.first_name is None:
            user.first_name = foursquare_profile_data.get('firstName') or None
        if user.last_name is None:
            user.last_name = foursquare_profile_data.get('lastName') or None
        if user.email is None:
            user.email = foursquare_profile_data.get('contact', {}).get('email') or None
        if user.gender is None:
            # TODO: Does it really map so cleanly?
            user.gender = foursquare_profile_data.get('gender') or None

        user.save()

        return user

class LazyUserBackend(MongoEngineBackend):
    def authenticate(self):
        while True:
            try:
                username = LAZYUSER_USERNAME_TEMPLATE % crypto.get_random_string(6)
                user = self.user_class.objects.create(
                    username=username,
                )
                break
            except queryset.OperationError, e:
                msg = str(e)
                if 'E11000' in msg and 'duplicate key error' in msg and 'username' in msg:
                    continue
                raise

        return user
<|endoftext|>


<commit_before>import json
import lib.persist
import lib.upstream
import lib.sysinfo
import lib.pkg

def update_cache():
    lib.pkg.updateCache()

def send_register(_config, _logger):
    sys = lib.sysinfo.get_register_system()
    _logger.debug(
        "Sending to server (register " + lib.sysinfo.get_hostname() + ")...")
    response = lib.upstream.pushRegister(_config, sys)
    _logger.debug("Response:\n" + response)
    _config.set_registered()

def send_system_updateinstalled(_config, _logger):
    packages = lib.pkg.getPackageList()
    _logger.debug(
        "Sending to server (updateInstalled " + lib.sysinfo.get_hostname() + ")...")
    response = lib.upstream.pushSystemUpdateInstalled(
        _config, lib.sysinfo.get_urn(), packages)
    _logger.debug("Response:\n" + response)

def send_system_notify(_config, _logger):
    sys = lib.sysinfo.get_notify_system()
    sys = lib.pkg.addUpdates(sys)
    _logger.debug(
        "Sending to server (notify " + lib.sysinfo.get_hostname() + ")...")
    response = lib.upstream.pushSystemNotify(
        _config, lib.sysinfo.get_urn(), sys)
    _logger.debug("Response:\n" + response)

def do_update(_config, _logger):
    tasks = lib.persist.Persist("/opt/upd89/agent/tasks.data")
    for key in tasks.get_keys():
        json_data = tasks.get_key(key)
        _logger.debug("key: " + key +" - json: " + json_data)
        t = json.loads(json_data)
        p_list = list()
        for p in t.get("packages"):
            pkg_name = p.get("pkg_name")
            pkg_version = p.get("pdk_version")
            p_list.append(pkg_name)
        tasknotify = lib.pkg.do_update(p_list)
        response = lib.upstream.pushTaskNotify(_config, key, tasknotify)
        _logger.debug("Response:\n" + response)
        tasks.delete_key(key)
        tasks.close()

<commit_msg>quickfix<commit_after>import json
import lib.persist
import lib.upstream
import lib.sysinfo
import lib.pkg

def update_cache():
    lib.pkg.updateCache()

def send_register(_config, _logger):
    sys = lib.sysinfo.get_register_system()
    _logger.debug(
        "Sending to server (register " + lib.sysinfo.get_hostname() + ")...")
    response = lib.upstream.pushRegister(_config, sys)
    _logger.debug("Response:\n" + response)
    _config.set_registered()

def send_system_updateinstalled(_config, _logger):
    packages = lib.pkg.getPackageList()
    _logger.debug(
        "Sending to server (updateInstalled " + lib.sysinfo.get_hostname() + ")...")
    response = lib.upstream.pushSystemUpdateInstalled(
        _config, lib.sysinfo.get_urn(), packages)
    _logger.debug("Response:\n" + response)

def send_system_notify(_config, _logger):
    sys = lib.sysinfo.get_notify_system()
    sys = lib.pkg.addUpdates(sys)
    _logger.debug(
        "Sending to server (notify " + lib.sysinfo.get_hostname() + ")...")
    response = lib.upstream.pushSystemNotify(
        _config, lib.sysinfo.get_urn(), sys)
    _logger.debug("Response:\n" + response)

def do_update(_config, _logger):
    tasks = lib.persist.Persist("/opt/upd89/agent/tasks.data")
    for key in tasks.get_keys():
        json_data = tasks.get_key(key)
        _logger.debug("key: " + key +" - json: " + json_data)
        t = json.loads(json_data)
        p_list = list()
        for p in t.get("packages"):
            pkg_name = p.get("pkg_name")
            pkg_version = p.get("pdk_version")
            p_list.append(pkg_name)
        tasknotify = lib.pkg.do_update(p_list)
        response = lib.upstream.pushTaskNotify(_config, key, tasknotify)
        _logger.debug("Response:\n" + response)
        tasks.delete_key(key)
    tasks.close()

<|endoftext|>


<commit_before>import os
import mimetypes

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.files.base import File
from django.core.files.storage import Storage
from django.utils.functional import curry

ACCESS_KEY_NAME = 'AWS_ACCESS_KEY_ID'
SECRET_KEY_NAME = 'AWS_SECRET_ACCESS_KEY'
HEADERS = 'AWS_HEADERS'

DEFAULT_ACL= getattr(settings, 'AWS_DEFAULT_ACL', 'public-read')
QUERYSTRING_ACTIVE= getattr(settings, 'AWS_QUERYSTRING_ACTIVE', False)
QUERYSTRING_EXPIRE= getattr(settings, 'AWS_QUERYSTRING_EXPIRE', 60)

try:
    from S3 import AWSAuthConnection, QueryStringAuthGenerator
except ImportError:
    raise ImproperlyConfigured, "Could not load amazon's S3 bindings.\
    \nSee http://developer.amazonwebservices.com/connect/entry.jspa?externalID=134"

class S3Storage(Storage):
    """Amazon Simple Storage Service"""

    def __init__(self, bucket=settings.AWS_STORAGE_BUCKET_NAME,
            access_key=None, secret_key=None, acl=DEFAULT_ACL,
            calling_format=settings.AWS_CALLING_FORMAT):
        self.bucket = bucket
        self.acl = acl

        if not access_key and not secret_key:
             access_key, secret_key = self._get_access_keys()

        self.connection = AWSAuthConnection(access_key, secret_key,
                            calling_format=calling_format)
        self.generator = QueryStringAuthGenerator(access_key, secret_key,
                            calling_format=calling_format, is_secure=False)
        self.generator.set_expires_in(QUERYSTRING_EXPIRE)

        self.headers = getattr(settings, HEADERS, {})

    def _get_access_keys(self):
        access_key = getattr(settings, ACCESS_KEY_NAME, None)
        secret_key = getattr(settings, SECRET_KEY_NAME, None)
        if (access_key or secret_key) and (not access_key or not secret_key):
            access_key = os.environ.get(ACCESS_KEY_NAME)
            secret_key = os.environ.get(SECRET_KEY_NAME)

        if access_key and secret_key:
            # Both were provided, so use them
            return access_key, secret_key

        return None, None

    def _get_connection(self):
        return AWSAuthConnection(*self._get_access_keys())

    def _put_file(self, name, content):
        content_type = mimetypes.guess_type(name)[0] or "application/x-octet-stream"
        self.headers.update({'x-amz-acl': self.acl, 'Content-Type': content_type})
        response = self.connection.put(self.bucket, name, content, self.headers)
        if response.http_response.status != 200:
            raise IOError("S3StorageError: %s" % response.message)

    def _open(self, name, mode='rb'):
        remote_file = S3StorageFile(name, self, mode=mode)
        return remote_file

    def _read(self, name, start_range=None, end_range=None):
        if start_range is None:
            headers = {}
        else:
            headers = {'Range': 'bytes=%s-%s' % (start_range, end_range)}
        response = self.connection.get(self.bucket, name, headers)
        if response.http_response.status != 200:
            raise IOError("S3StorageError: %s" % response.message)
        headers = response.http_response.msg
        return response.object.data, headers.get('etag', None), headers.get('content-range', None)

    def _save(self, name, content):
        content.open()
        if hasattr(content, 'chunks'):
            content_str = ''.join(chunk for chunk in content.chunks())
        else:
            content_str = content.read()
        self._put_file(name, content_str)
        return name

    def delete(self, name):
        response = self.connection.delete(self.bucket, name)
        if response.http_response.status != 204:
            raise IOError("S3StorageError: %s" % response.message)

    def exists(self, name):
        response = self.connection._make_request('HEAD', self.bucket, name)
        return response.status == 200

    def size(self, name):
        response = self.connection._make_request('HEAD', self.bucket, name)
        content_length = response.getheader('Content-Length')
        return content_length and int(content_length) or 0

    def url(self, name):
        if QUERYSTRING_ACTIVE:
            return self.generator.generate_url('GET', self.bucket, name)
        else:
            return self.generator.make_bare_url(self.bucket, name)

    ## UNCOMMENT BELOW IF NECESSARY
    #def get_available_name(self, name):
    #    """ Overwrite existing file with the same name. """
    #    return name

class S3StorageFile(File):
    def __init__(self, name, storage, mode):
        self._name = name
        self._storage = storage
        self._mode = mode
        self._is_dirty = False
        self.file = StringIO()
        self.start_range = 0

    @property
    def size(self):
        if not hasattr(self, '_size'):
            self._size = self._storage.size(self._name)
        return self._size

    def read(self, num_bytes=None):
        if num_bytes is None:
            args = []
            self.start_range = 0
        else:
            args = [self.start_range, self.start_range+num_bytes-1]
        data, etags, content_range = self._storage._read(self._name, *args)
        if content_range is not None:
            current_range, size = content_range.split(' ', 1)[1].split('/', 1)
            start_range, end_range = current_range.split('-', 1)
            self._size, self.start_range = int(size), int(end_range)+1
        self.file = StringIO(data)
        return self.file.getvalue()

    def write(self, content):
        if 'w' not in self._mode:
            raise AttributeError("File was opened for read-only access.")
        self.file = StringIO(content)
        self._is_dirty = True

    def close(self):
        if self._is_dirty:
            self._storage._put_file(self._name, self.file.getvalue())
        self.file.close()
<commit_msg>S3: Added the AWS_S3_SECURE_URLS setting to provide the option of accessing S3 objects through https urls instead of only http. Thanks weswinham.<commit_after>import os
import mimetypes

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.files.base import File
from django.core.files.storage import Storage
from django.utils.functional import curry

try:
    from S3 import AWSAuthConnection, QueryStringAuthGenerator
except ImportError:
    raise ImproperlyConfigured, "Could not load amazon's S3 bindings.\
    \nSee http://developer.amazonwebservices.com/connect/entry.jspa?externalID=134"

ACCESS_KEY_NAME = 'AWS_ACCESS_KEY_ID'
SECRET_KEY_NAME = 'AWS_SECRET_ACCESS_KEY'
HEADERS = 'AWS_HEADERS'

DEFAULT_ACL= getattr(settings, 'AWS_DEFAULT_ACL', 'public-read')
QUERYSTRING_ACTIVE= getattr(settings, 'AWS_QUERYSTRING_ACTIVE', False)
QUERYSTRING_EXPIRE= getattr(settings, 'AWS_QUERYSTRING_EXPIRE', 60)
SECURE_URLS= getattr(settings, 'AWS_S3_SECURE_URLS', False)

class S3Storage(Storage):
    """Amazon Simple Storage Service"""

    def __init__(self, bucket=settings.AWS_STORAGE_BUCKET_NAME,
            access_key=None, secret_key=None, acl=DEFAULT_ACL,
            calling_format=settings.AWS_CALLING_FORMAT):
        self.bucket = bucket
        self.acl = acl

        if not access_key and not secret_key:
            access_key, secret_key = self._get_access_keys()

        self.connection = AWSAuthConnection(access_key, secret_key,
                            calling_format=calling_format)
        self.generator = QueryStringAuthGenerator(access_key, secret_key,
                            calling_format=calling_format,
                            is_secure=SECURE_URLS)
        self.generator.set_expires_in(QUERYSTRING_EXPIRE)

        self.headers = getattr(settings, HEADERS, {})

    def _get_access_keys(self):
        access_key = getattr(settings, ACCESS_KEY_NAME, None)
        secret_key = getattr(settings, SECRET_KEY_NAME, None)
        if (access_key or secret_key) and (not access_key or not secret_key):
            access_key = os.environ.get(ACCESS_KEY_NAME)
            secret_key = os.environ.get(SECRET_KEY_NAME)

        if access_key and secret_key:
            # Both were provided, so use them
            return access_key, secret_key

        return None, None

    def _get_connection(self):
        return AWSAuthConnection(*self._get_access_keys())

    def _put_file(self, name, content):
        content_type = mimetypes.guess_type(name)[0] or "application/x-octet-stream"
        self.headers.update({'x-amz-acl': self.acl, 'Content-Type': content_type})
        response = self.connection.put(self.bucket, name, content, self.headers)
        if response.http_response.status != 200:
            raise IOError("S3StorageError: %s" % response.message)

    def _open(self, name, mode='rb'):
        remote_file = S3StorageFile(name, self, mode=mode)
        return remote_file

    def _read(self, name, start_range=None, end_range=None):
        if start_range is None:
            headers = {}
        else:
            headers = {'Range': 'bytes=%s-%s' % (start_range, end_range)}
        response = self.connection.get(self.bucket, name, headers)
        if response.http_response.status != 200:
            raise IOError("S3StorageError: %s" % response.message)
        headers = response.http_response.msg
        return response.object.data, headers.get('etag', None), headers.get('content-range', None)

    def _save(self, name, content):
        content.open()
        if hasattr(content, 'chunks'):
            content_str = ''.join(chunk for chunk in content.chunks())
        else:
            content_str = content.read()
        self._put_file(name, content_str)
        return name

    def delete(self, name):
        response = self.connection.delete(self.bucket, name)
        if response.http_response.status != 204:
            raise IOError("S3StorageError: %s" % response.message)

    def exists(self, name):
        response = self.connection._make_request('HEAD', self.bucket, name)
        return response.status == 200

    def size(self, name):
        response = self.connection._make_request('HEAD', self.bucket, name)
        content_length = response.getheader('Content-Length')
        return content_length and int(content_length) or 0

    def url(self, name):
        if QUERYSTRING_ACTIVE:
            return self.generator.generate_url('GET', self.bucket, name)
        else:
            return self.generator.make_bare_url(self.bucket, name)

    ## UNCOMMENT BELOW IF NECESSARY
    #def get_available_name(self, name):
    #    """ Overwrite existing file with the same name. """
    #    return name

class S3StorageFile(File):
    def __init__(self, name, storage, mode):
        self._name = name
        self._storage = storage
        self._mode = mode
        self._is_dirty = False
        self.file = StringIO()
        self.start_range = 0

    @property
    def size(self):
        if not hasattr(self, '_size'):
            self._size = self._storage.size(self._name)
        return self._size

    def read(self, num_bytes=None):
        if num_bytes is None:
            args = []
            self.start_range = 0
        else:
            args = [self.start_range, self.start_range+num_bytes-1]
        data, etags, content_range = self._storage._read(self._name, *args)
        if content_range is not None:
            current_range, size = content_range.split(' ', 1)[1].split('/', 1)
            start_range, end_range = current_range.split('-', 1)
            self._size, self.start_range = int(size), int(end_range)+1
        self.file = StringIO(data)
        return self.file.getvalue()

    def write(self, content):
        if 'w' not in self._mode:
            raise AttributeError("File was opened for read-only access.")
        self.file = StringIO(content)
        self._is_dirty = True

    def close(self):
        if self._is_dirty:
            self._storage._put_file(self._name, self.file.getvalue())
        self.file.close()
<|endoftext|>


<commit_before>import app
from app import db
from datetime import datetime, date

class FriendRequest(db.Model):
    __tablename__ = "friend_requests"
    id = db.Column(db.Integer, primary_key=True)
    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
    friend_id = db.Column(db.Integer, db.ForeignKey('users.id'))
    created = db.Column(db.DateTime, default=datetime.utcnow)

class Friend(db.Model):
    __tablename__ = "friends"
    id = db.Column(db.Integer, primary_key=True)
    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
    friend_id = db.Column(db.Integer, db.ForeignKey('users.id'))
    created = db.Column(db.DateTime, default=datetime.utcnow)

class Message(db.Model):
    __tablename__ = "messages"
    id = db.Column(db.Integer, primary_key=True)
    sent_timestamp = db.Column(db.DateTime, default=datetime.utcnow, index=True)
    to_user_id = db.Column(db.Integer, nullable=False)
    from_user_id = db.Column(db.Integer, nullable=False)
    title = db.Column(db.String, index=True)
    body = db.Column(db.String, nullable=False, index=True)
    is_read = db.Column(db.Boolean, default=False)
    __table_args__ = (
        db.CheckConstraint("to_user_id != from_user_id"),
    )

    def from_user(self):
        user = User.query.filter_by(id=self.from_user_id).first()
        if user is not None:
            return user
        else:
            return None

    def to_user(self):
        user = User.query.filter_by(id=self.to_user_id).first()
        if user is not None:
            return user
        else:
            return None

    def __repr__(self):
        return "<Message: From: %d, to: %d, title: %r>" % (self.from_user_id, self.to_user_id, self.title)

class Task(db.Model):
    __tablename__ = "task"
    id = db.Column(db.Integer, primary_key=True)
    created = db.Column(db.DateTime, default=datetime.utcnow, index=True)
    end_timestamp = db.Column(db.DateTime, default=datetime.utcnow, index=True)
    description = db.Column(db.String, nullable=False, index=True)
    requester_id = db.Column(db.Integer, nullable=False)
    doer_id = db.Column(db.Integer, nullable=False)
    is_active = db.Column(db.Boolean, default=True)
    __table_args__ = (
        db.CheckConstraint("requester_id != doer_id"),
    )

    def requester(self):
        return db.session.query(User, Task).filter(self.requester_id == User.id).first()

    def doer(self):
        return db.session.query(User, Task).filter(self.doer_id == User.id).first()

    def __repr__(self):
        return "<Task: Requester: %d, doer: %d, '%r'>" % (self.requester_id, self.doer_id, self.description)

class User(db.Model):
    __tablename__ = "users"
    id = db.Column(db.Integer, primary_key=True)
    username = db.Column(db.String, index=True, unique=True)
    pw_hash = db.Column(db.String, index=True)
    age = db.Column(db.Integer)
    gender = db.Column(db.String(24))
    role = db.Column(db.String(24))
    orientation = db.Column(db.String(24))
    bio = db.Column(db.String)
    email = db.Column(db.String, nullable=False, unique=True)
    created = db.Column(db.DateTime, default=datetime.utcnow)
    user_role = db.Column(db.Integer, default=0)
    is_validated = db.Column(db.Boolean, default=False)
    password_reset_token = db.Column(db.String)
    password_reset_token_expiration = db.Column(db.DateTime)
    __table_args__ = (
        db.UniqueConstraint('username'),
        db.UniqueConstraint('email'),
        db.CheckConstraint('age is null or (age >= 18 and age <= 100)')
    )

    def get_all_friend_requests(self):
        return db.session.query(friend_requests).filter_by(user_id=self.id)

    def get_all_friends(self):
        return db.session.query(friends).filter_by(user_id=self.id)

    def get_all_inbox_messages(self):
        return Message.query.filter_by(to_user_id=self.id).order_by(Message.sent_timestamp.desc()).all()

    def get_all_outbox_messages(self):
        return Message.query.filter_by(from_user_id=self.id).order_by(Message.sent_timestamp.desc()).all()

    def num_unread_messages(self):
        return Message.query.filter_by(to_user_id=self.id).filter(~Message.is_read).count()

    def age_gender_role(self):
        return " ".join([str(x) if x is not None else "" for x in [self.age, self.gender, self.role]])

    def tasks_todo(self, active=True):
        return [x[1] for x in
                db.session.query(User, Task).filter(User.id == Task.doer_id and Task.is_active == bool(active)).
                    filter(User.id == self.id).all()
                ]

    def tasks_assigned(self, active=True):
        return [x[1] for x in
                    db.session.query(User, Task).filter(User.id == Task.requester_id and Task.is_active == bool(active)).
                        filter(User.id == self.id).all()
                ]

    def is_authenticated(self):
        return True

    def is_active(self):
        return True

    def is_anonymous(self):
        return False

    def get_id(self):
        return unicode(self.id)

    def __repr__(self):
        return "<User '%r'>" % (self.name)
<commit_msg>Huzzah! I'm finally starting to understand SQLAlchemy relationships.<commit_after>import app
from app import db
from datetime import datetime, date

class FriendRequest(db.Model):
    __tablename__ = "friend_requests"
    id = db.Column(db.Integer, primary_key=True)
    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
    friend_id = db.Column(db.Integer, db.ForeignKey('users.id'))
    created = db.Column(db.DateTime, default=datetime.utcnow)
    __table_args__ = (
        db.CheckConstraint("user_id != friend_id"),
    )

class Friend(db.Model):
    __tablename__ = "friends"
    id = db.Column(db.Integer, primary_key=True)
    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
    friend_id = db.Column(db.Integer, db.ForeignKey('users.id'))
    created = db.Column(db.DateTime, default=datetime.utcnow)
    __table_args__ = (
        db.CheckConstraint("user_id != friend_id"),
    )

class Message(db.Model):
    __tablename__ = "messages"
    id = db.Column(db.Integer, primary_key=True)
    sent_timestamp = db.Column(db.DateTime, default=datetime.utcnow, index=True)
    to_user_id = db.Column(db.Integer, nullable=False)
    from_user_id = db.Column(db.Integer, nullable=False)
    title = db.Column(db.String, index=True)
    body = db.Column(db.String, nullable=False, index=True)
    is_read = db.Column(db.Boolean, default=False)
    __table_args__ = (
        db.CheckConstraint("to_user_id != from_user_id"),
    )

    def from_user(self):
        user = User.query.filter_by(id=self.from_user_id).first()
        if user is not None:
            return user
        else:
            return None

    def to_user(self):
        user = User.query.filter_by(id=self.to_user_id).first()
        if user is not None:
            return user
        else:
            return None

    def __repr__(self):
        return "<Message: From: %d, to: %d, title: %r>" % (self.from_user_id, self.to_user_id, self.title)

class Task(db.Model):
    __tablename__ = "task"
    id = db.Column(db.Integer, primary_key=True)
    created = db.Column(db.DateTime, default=datetime.utcnow, index=True)
    end_timestamp = db.Column(db.DateTime, default=datetime.utcnow, index=True)
    description = db.Column(db.String, nullable=False, index=True)
    requester_id = db.Column(db.Integer, nullable=False)
    doer_id = db.Column(db.Integer, nullable=False)
    is_active = db.Column(db.Boolean, default=True)
    __table_args__ = (
        db.CheckConstraint("requester_id != doer_id"),
    )

    def requester(self):
        return db.session.query(User, Task).filter(self.requester_id == User.id).first()

    def doer(self):
        return db.session.query(User, Task).filter(self.doer_id == User.id).first()

    def __repr__(self):
        return "<Task: Requester: %d, doer: %d, '%r'>" % (self.requester_id, self.doer_id, self.description)

class User(db.Model):
    __tablename__ = "users"
    id = db.Column(db.Integer, primary_key=True)
    username = db.Column(db.String, index=True, unique=True)
    pw_hash = db.Column(db.String, index=True)
    age = db.Column(db.Integer)
    gender = db.Column(db.String(24))
    role = db.Column(db.String(24))
    orientation = db.Column(db.String(24))
    bio = db.Column(db.String)
    email = db.Column(db.String, nullable=False, unique=True)
    created = db.Column(db.DateTime, default=datetime.utcnow)
    user_role = db.Column(db.Integer, default=0)
    is_validated = db.Column(db.Boolean, default=False)
    password_reset_token = db.Column(db.String)
    password_reset_token_expiration = db.Column(db.DateTime)
    users_sending_friend_requests = db.relationship('User',
        secondary="friend_requests",
        primaryjoin=(FriendRequest.friend_id == id),
        secondaryjoin=(FriendRequest.user_id == id),
        backref=db.backref("users_with_pending_friend_requests", lazy="dynamic"),
        lazy="dynamic"
    )
    __table_args__ = (
        db.UniqueConstraint('username'),
        db.UniqueConstraint('email'),
        db.CheckConstraint('age is null or (age >= 18 and age <= 100)')
    )

    def get_users_sending_friend_requests(self):
        # return db.session.query(User, FriendRequest).join(FriendRequest, User.id == FriendRequest.user_id).filter(FriendRequest.friend_id == self.id).all()
        pass

    def get_all_friends(self):
        return db.session.query(User).join(Friend, User.id == Friend.friend_id).filter_by(user_id=self.id).all()

    def get_all_inbox_messages(self):
        return Message.query.filter_by(to_user_id=self.id).order_by(Message.sent_timestamp.desc()).all()

    def get_all_outbox_messages(self):
        return Message.query.filter_by(from_user_id=self.id).order_by(Message.sent_timestamp.desc()).all()

    def num_unread_messages(self):
        return Message.query.filter_by(to_user_id=self.id).filter(~Message.is_read).count()

    def age_gender_role(self):
        return " ".join([str(x) if x is not None else "" for x in [self.age, self.gender, self.role]])

    def tasks_todo(self, active=True):
        return [x[1] for x in
                db.session.query(User, Task).filter(User.id == Task.doer_id and Task.is_active == bool(active)).
                    filter(User.id == self.id).all()
                ]

    def tasks_assigned(self, active=True):
        return [x[1] for x in
                    db.session.query(User, Task).filter(User.id == Task.requester_id and Task.is_active == bool(active)).
                        filter(User.id == self.id).all()
                ]

    def is_authenticated(self):
        return True

    def is_active(self):
        return True

    def is_anonymous(self):
        return False

    def get_id(self):
        return unicode(self.id)

    def __repr__(self):
        return "<User '%r'>" % (self.name)
<|endoftext|>


<commit_before># BridgeDB by Nick Mathewson.
# Copyright (c) 2007, The Tor Project, Inc.
# See LICENSE for licensing information

"""
This module implements the web and email interfaces to the bridge database.
"""

from cStringIO import StringIO
import MimeWriter
import rfc822
import time
import logging

from zope.interface import implements

from twisted.internet import reactor
from twisted.internet.defer import Deferred
import twisted.web.resource
import twisted.web.server
import twisted.mail.smtp

import bridgedb.Dist

HTML_MESSAGE_TEMPLATE = """

Here are your bridge relays:

%s

Bridge relays (or "bridges" for short) are Tor relays that aren't listed
in the main directory. Since there is no complete public list of them,
even if your ISP is filtering connections to all the known Tor relays,
they probably won't be able to block all the bridges.

To use the above lines, go to Vidalia's Network settings page, and click
"My ISP blocks connections to the Tor network". Then add each bridge
address one at a time.

Configuring more than one bridge address will make your Tor connection
more stable, in case some of the bridges become unreachable.

Another way to find public bridge addresses is to send mail to
bridges@torproject.org with the line "get bridges" by itself in the body
of the mail. However, so we can make it harder for an attacker to learn
lots of bridge addresses, you must send this request from a gmail or
yahoo account.

""".strip()

EMAIL_MESSAGE_TEMPLATE = """\
[This is an automated message; please do not reply.]

Here are your bridge relays:

%s

Bridge relays (or "bridges" for short) are Tor relays that aren't listed
in the main directory. Since there is no complete public list of them,
even if your ISP is filtering connections to all the known Tor relays,
they probably won't be able to block all the bridges.

To use the above lines, go to Vidalia's Network settings page, and click
"My ISP blocks connections to the Tor network". Then add each bridge
address one at a time.

Configuring more than one bridge address will make your Tor connection
more stable, in case some of the bridges become unreachable.

Another way to find public bridge addresses is to visit
https://bridges.torproject.org/. The answers you get from that page
will change every few days, so check back periodically if you need more
bridge addresses.
"""

class WebResource(twisted.web.resource.Resource):
    """This resource is used by Twisted Web to give a web page with some
       bridges in response to a request."""
    isLeaf = True

    def __init__(self, distributor, schedule, N=1, useForwardedHeader=False):
        """Create a new WebResource.
             distributor -- an IPBasedDistributor object
             schedule -- an IntervalSchedule object
             N -- the number of bridges to hand out per query.
        """
        twisted.web.resource.Resource.__init__(self)
        self.distributor = distributor
        self.schedule = schedule
        self.nBridgesToGive = N
        self.useForwardedHeader = useForwardedHeader

    def render_GET(self, request):
        interval = self.schedule.getInterval(time.time())
        bridges = ( )
        ip = None
        if self.useForwardedHeader:
            h = request.getHeader("X-Forwarded-For")
            if h:
                ip = h.split(",")[-1].strip()
                if not bridgedb.Bridges.is_valid_ip(ip):
                    logging.warn("Got weird forwarded-for value %r",h)
                    ip = None
        else:
            ip = request.getClientIP()

        if ip:
            bridges = self.distributor.getBridgesForIP(ip, interval,
                                                       self.nBridgesToGive)
        if bridges:
            answer = "".join("%s\n" % b.getConfigLine() for b in bridges)
        else:
            answer = "No bridges available."

        logging.info("Replying to web request from %s", ip)
        return HTML_MESSAGE_TEMPLATE % answer

def addWebServer(cfg, dist, sched):
    """Set up a web server.
         cfg -- a configuration object from Main.  We use these options:
                HTTPS_N_BRIDGES_PER_ANSWER
                HTTP_UNENCRYPTED_PORT
                HTTP_UNENCRYPTED_BIND_IP
                HTTP_USE_IP_FROM_FORWARDED_HEADER
                HTTPS_PORT
                HTTPS_BIND_IP
                HTTPS_USE_IP_FROM_FORWARDED_HEADER
         dist -- an IPBasedDistributor object.
         sched -- an IntervalSchedule object.
    """
    Site = twisted.web.server.Site
    site = None
    if cfg.HTTP_UNENCRYPTED_PORT:
        ip = cfg.HTTP_UNENCRYPTED_BIND_IP or ""
        resource = WebResource(dist, sched, cfg.HTTPS_N_BRIDGES_PER_ANSWER,
                               cfg.HTTP_USE_IP_FROM_FORWARDED_HEADER)
        site = Site(resource)
        reactor.listenTCP(cfg.HTTP_UNENCRYPTED_PORT, site, interface=ip)
    if cfg.HTTPS_PORT:
        from twisted.internet.ssl import DefaultOpenSSLContextFactory
        #from OpenSSL.SSL import SSLv3_METHOD
        ip = cfg.HTTPS_BIND_IP or ""
        factory = DefaultOpenSSLContextFactory(cfg.HTTPS_KEY_FILE,
                                               cfg.HTTPS_CERT_FILE)
        resource = WebResource(dist, sched, cfg.HTTPS_N_BRIDGES_PER_ANSWER,
                               cfg.HTTPS_USE_IP_FROM_FORWARDED_HEADER)
        site = Site(resource)
        reactor.listenSSL(cfg.HTTPS_PORT, site, factory, interface=ip)
    return site

class MailFile:
    """A file-like object used to hand rfc822.Message a list of lines
       as though it were reading them from a file."""
    def __init__(self, lines):
        self.lines = lines
        self.idx = 0
    def readline(self):
        try :
            line = self.lines[self.idx]
            self.idx += 1
            return line
        except IndexError:
            return ""

def getMailResponse(lines, ctx):
    """Given a list of lines from an incoming email message, and a
       MailContext object, parse the email and decide what to do in response.
       If we want to answer, return a 2-tuple containing the address that
       will receive the response, and a readable filelike object containing
       the response.  Return None,None if we shouldn't answer.
    """
    # Extract data from the headers.
    msg = rfc822.Message(MailFile(lines))
    subject = msg.getheader("Subject", None)
    if not subject: subject = "[no subject]"
    clientFromAddr = msg.getaddr("From")
    clientSenderAddr = msg.getaddr("Sender")
    msgID = msg.getheader("Message-ID")
    if clientSenderAddr and clientSenderAddr[1]:
        clientAddr = clientSenderAddr[1]
    elif clientFromAddr and clientFromAddr[1]:
        clientAddr = clientFromAddr[1]
    else:
        logging.info("No From or Sender header on incoming mail.")
        return None,None

    try:
        _, addrdomain = bridgedb.Dist.extractAddrSpec(clientAddr.lower())
    except bridgedb.Dist.BadEmail:
	logging.info("Ignoring bad address on incoming email.")
        return None,None
    if not addrdomain:
        logging.info("Couldn't parse domain from %r", clientAddr)
    if addrdomain and ctx.cfg.EMAIL_DOMAIN_MAP:
        addrdomain = ctx.cfg.EMAIL_DOMAIN_MAP.get(addrdomain, addrdomain)
    rules = ctx.cfg.EMAIL_DOMAIN_RULES.get(addrdomain, [])
    if 'dkim' in rules:
        # getheader() returns the last of a given kind of header; we want
        # to get the first, so we use getheaders() instead.
        dkimHeaders = msg.getheaders("X-DKIM-Authentication-Results")
        dkimHeader = "<no header>"
        if dkimHeaders: dkimHeader = dkimHeaders[0]
        if not dkimHeader.startswith("pass"):
            logging.info("Got a bad dkim header (%r) on an incoming mail; "
                         "rejecting it.", dkimHeader)
            return None, None

    # Was the magic string included
    for ln in lines:
        if ln.strip().lower() in ("get bridges", "subject: get bridges"):
            break
    else:
        logging.info("Got a mail from %r with no bridge request; dropping",
                     clientAddr)
        return None,None

    # Figure out which bridges to send
    try:
        interval = ctx.schedule.getInterval(time.time())
        bridges = ctx.distributor.getBridgesForEmail(clientAddr,
                                                     interval, ctx.N)
    except bridgedb.Dist.BadEmail, e:
        logging.info("Got a mail from a bad email address %r: %s.",
                     clientAddr, e)
        return None, None

    # Generate the message.
    f = StringIO()
    w = MimeWriter.MimeWriter(f)
    w.addheader("From", ctx.fromAddr)
    w.addheader("To", clientAddr)
    w.addheader("Message-ID", twisted.mail.smtp.messageid())
    if not subject.startswith("Re:"): subject = "Re: %s"%subject
    w.addheader("Subject", subject)
    if msgID:
        w.addheader("In-Reply-To", msgID)
    w.addheader("Date", twisted.mail.smtp.rfc822date())
    body = w.startbody("text/plain")

    if bridges:
        answer = "".join("  %s\n" % b.getConfigLine() for b in bridges)
    else:
        answer = "(no bridges currently available)"
    body.write(EMAIL_MESSAGE_TEMPLATE % answer)

    f.seek(0)
    return clientAddr, f

def replyToMail(lines, ctx):
    """Given a list of lines from an incoming email message, and a
       MailContext object, possibly send a reply.
    """
    logging.info("Got a completed email; attempting to reply.")
    sendToUser, response = getMailResponse(lines, ctx)
    if response is None:
        return
    response.seek(0)
    d = Deferred()
    factory = twisted.mail.smtp.SMTPSenderFactory(
        ctx.smtpFromAddr,
        sendToUser,
        response,
        d)
    reactor.connectTCP(ctx.smtpServer, ctx.smtpPort, factory)
    logging.info("Sending reply to %r", sendToUser)
    return d

class MailContext:
    """Helper object that holds information used by email subsystem."""
    def __init__(self, cfg, dist, sched):
        # Reject any RCPT TO lines that aren't to this user.
        self.username = (cfg.EMAIL_USERNAME or
                         "bridges")
        # Reject any mail longer than this.
        self.maximumSize = 32*1024
        # Use this server for outgoing mail.
        self.smtpServer = "127.0.0.1"
        self.smtpPort = 25
        # Use this address in the MAIL FROM line for outgoing mail.
        self.smtpFromAddr = (cfg.EMAIL_SMTP_FROM_ADDR or
                             "bridges@torproject.org")
        # Use this address in the "From:" header for outgoing mail.
        self.fromAddr = (cfg.EMAIL_FROM_ADDR or
                         "bridges@torproject.org")
        # An EmailBasedDistributor object
        self.distributor = dist
        # An IntervalSchedule object
        self.schedule = sched
        # The number of bridges to send for each email.
        self.N = cfg.EMAIL_N_BRIDGES_PER_ANSWER

        self.cfg = cfg

class MailMessage:
    """Plugs into the Twisted Mail and receives an incoming message.
       Once the message is in, we reply or we don't. """
    implements(twisted.mail.smtp.IMessage)

    def __init__(self, ctx):
        """Create a new MailMessage from a MailContext."""
        self.ctx = ctx
        self.lines = []
        self.nBytes = 0
        self.ignoring = False

    def lineReceived(self, line):
        """Called when we get another line of an incoming message."""
        self.nBytes += len(line)
        logging.debug("> %s", line.rstrip("\r\n"))
        if self.nBytes > self.ctx.maximumSize:
            self.ignoring = True
        else:
            self.lines.append(line)

    def eomReceived(self):
        """Called when we receive the end of a message."""
        if not self.ignoring:
            replyToMail(self.lines, self.ctx)
        return twisted.internet.defer.succeed(None)

    def connectionLost(self):
        """Called if we die partway through reading a message."""
        pass

class MailDelivery:
    """Plugs into Twisted Mail and handles SMTP commands."""
    implements(twisted.mail.smtp.IMessageDelivery)
    def setBridgeDBContext(self, ctx):
        self.ctx = ctx
    def receivedHeader(self, helo, origin, recipients):
        #XXXX what is this for? what should it be?
        return "Received: BridgeDB"
    def validateFrom(self, helo, origin):
        return origin
    def validateTo(self, user):
        if user.dest.local != self.ctx.username:
            raise twisted.mail.smtp.SMTPBadRcpt(user)
        return lambda: MailMessage(self.ctx)

class MailFactory(twisted.mail.smtp.SMTPFactory):
    """Plugs into Twisted Mail; creates a new MailDelivery whenever we get
       a connection on the SMTP port."""
    def __init__(self, *a, **kw):
        twisted.mail.smtp.SMTPFactory.__init__(self, *a, **kw)
        self.delivery = MailDelivery()

    def setBridgeDBContext(self, ctx):
        self.ctx = ctx
        self.delivery.setBridgeDBContext(ctx)

    def buildProtocol(self, addr):
        p = twisted.mail.smtp.SMTPFactory.buildProtocol(self, addr)
        p.delivery = self.delivery
        return p

def addSMTPServer(cfg, dist, sched):
    """Set up a smtp server.
         cfg -- a configuration object from Main.  We use these options:
                EMAIL_BIND_IP
                EMAIL_PORT
                EMAIL_N_BRIDGES_PER_ANSWER
                EMAIL_DOMAIN_RULES
         dist -- an EmailBasedDistributor object.
         sched -- an IntervalSchedule object.
    """
    ctx = MailContext(cfg, dist, sched)
    factory = MailFactory()
    factory.setBridgeDBContext(ctx)
    ip = cfg.EMAIL_BIND_IP or ""
    reactor.listenTCP(cfg.EMAIL_PORT, factory, interface=ip)
    return factory

def runServers():
    """Start all the servers that we've configured. Exits when they do."""
    reactor.run()
<commit_msg>Actually enforce EMAIL_DOMAINS list.<commit_after># BridgeDB by Nick Mathewson.
# Copyright (c) 2007, The Tor Project, Inc.
# See LICENSE for licensing information

"""
This module implements the web and email interfaces to the bridge database.
"""

from cStringIO import StringIO
import MimeWriter
import rfc822
import time
import logging

from zope.interface import implements

from twisted.internet import reactor
from twisted.internet.defer import Deferred
import twisted.web.resource
import twisted.web.server
import twisted.mail.smtp

import bridgedb.Dist

HTML_MESSAGE_TEMPLATE = """

Here are your bridge relays:

%s

Bridge relays (or "bridges" for short) are Tor relays that aren't listed
in the main directory. Since there is no complete public list of them,
even if your ISP is filtering connections to all the known Tor relays,
they probably won't be able to block all the bridges.

To use the above lines, go to Vidalia's Network settings page, and click
"My ISP blocks connections to the Tor network". Then add each bridge
address one at a time.

Configuring more than one bridge address will make your Tor connection
more stable, in case some of the bridges become unreachable.

Another way to find public bridge addresses is to send mail to
bridges@torproject.org with the line "get bridges" by itself in the body
of the mail. However, so we can make it harder for an attacker to learn
lots of bridge addresses, you must send this request from a gmail or
yahoo account.

""".strip()

EMAIL_MESSAGE_TEMPLATE = """\
[This is an automated message; please do not reply.]

Here are your bridge relays:

%s

Bridge relays (or "bridges" for short) are Tor relays that aren't listed
in the main directory. Since there is no complete public list of them,
even if your ISP is filtering connections to all the known Tor relays,
they probably won't be able to block all the bridges.

To use the above lines, go to Vidalia's Network settings page, and click
"My ISP blocks connections to the Tor network". Then add each bridge
address one at a time.

Configuring more than one bridge address will make your Tor connection
more stable, in case some of the bridges become unreachable.

Another way to find public bridge addresses is to visit
https://bridges.torproject.org/. The answers you get from that page
will change every few days, so check back periodically if you need more
bridge addresses.
"""

class WebResource(twisted.web.resource.Resource):
    """This resource is used by Twisted Web to give a web page with some
       bridges in response to a request."""
    isLeaf = True

    def __init__(self, distributor, schedule, N=1, useForwardedHeader=False):
        """Create a new WebResource.
             distributor -- an IPBasedDistributor object
             schedule -- an IntervalSchedule object
             N -- the number of bridges to hand out per query.
        """
        twisted.web.resource.Resource.__init__(self)
        self.distributor = distributor
        self.schedule = schedule
        self.nBridgesToGive = N
        self.useForwardedHeader = useForwardedHeader

    def render_GET(self, request):
        interval = self.schedule.getInterval(time.time())
        bridges = ( )
        ip = None
        if self.useForwardedHeader:
            h = request.getHeader("X-Forwarded-For")
            if h:
                ip = h.split(",")[-1].strip()
                if not bridgedb.Bridges.is_valid_ip(ip):
                    logging.warn("Got weird forwarded-for value %r",h)
                    ip = None
        else:
            ip = request.getClientIP()

        if ip:
            bridges = self.distributor.getBridgesForIP(ip, interval,
                                                       self.nBridgesToGive)
        if bridges:
            answer = "".join("%s\n" % b.getConfigLine() for b in bridges)
        else:
            answer = "No bridges available."

        logging.info("Replying to web request from %s", ip)
        return HTML_MESSAGE_TEMPLATE % answer

def addWebServer(cfg, dist, sched):
    """Set up a web server.
         cfg -- a configuration object from Main.  We use these options:
                HTTPS_N_BRIDGES_PER_ANSWER
                HTTP_UNENCRYPTED_PORT
                HTTP_UNENCRYPTED_BIND_IP
                HTTP_USE_IP_FROM_FORWARDED_HEADER
                HTTPS_PORT
                HTTPS_BIND_IP
                HTTPS_USE_IP_FROM_FORWARDED_HEADER
         dist -- an IPBasedDistributor object.
         sched -- an IntervalSchedule object.
    """
    Site = twisted.web.server.Site
    site = None
    if cfg.HTTP_UNENCRYPTED_PORT:
        ip = cfg.HTTP_UNENCRYPTED_BIND_IP or ""
        resource = WebResource(dist, sched, cfg.HTTPS_N_BRIDGES_PER_ANSWER,
                               cfg.HTTP_USE_IP_FROM_FORWARDED_HEADER)
        site = Site(resource)
        reactor.listenTCP(cfg.HTTP_UNENCRYPTED_PORT, site, interface=ip)
    if cfg.HTTPS_PORT:
        from twisted.internet.ssl import DefaultOpenSSLContextFactory
        #from OpenSSL.SSL import SSLv3_METHOD
        ip = cfg.HTTPS_BIND_IP or ""
        factory = DefaultOpenSSLContextFactory(cfg.HTTPS_KEY_FILE,
                                               cfg.HTTPS_CERT_FILE)
        resource = WebResource(dist, sched, cfg.HTTPS_N_BRIDGES_PER_ANSWER,
                               cfg.HTTPS_USE_IP_FROM_FORWARDED_HEADER)
        site = Site(resource)
        reactor.listenSSL(cfg.HTTPS_PORT, site, factory, interface=ip)
    return site

class MailFile:
    """A file-like object used to hand rfc822.Message a list of lines
       as though it were reading them from a file."""
    def __init__(self, lines):
        self.lines = lines
        self.idx = 0
    def readline(self):
        try :
            line = self.lines[self.idx]
            self.idx += 1
            return line
        except IndexError:
            return ""

def getMailResponse(lines, ctx):
    """Given a list of lines from an incoming email message, and a
       MailContext object, parse the email and decide what to do in response.
       If we want to answer, return a 2-tuple containing the address that
       will receive the response, and a readable filelike object containing
       the response.  Return None,None if we shouldn't answer.
    """
    # Extract data from the headers.
    msg = rfc822.Message(MailFile(lines))
    subject = msg.getheader("Subject", None)
    if not subject: subject = "[no subject]"
    clientFromAddr = msg.getaddr("From")
    clientSenderAddr = msg.getaddr("Sender")
    msgID = msg.getheader("Message-ID")
    if clientSenderAddr and clientSenderAddr[1]:
        clientAddr = clientSenderAddr[1]
    elif clientFromAddr and clientFromAddr[1]:
        clientAddr = clientFromAddr[1]
    else:
        logging.info("No From or Sender header on incoming mail.")
        return None,None

    try:
        _, addrdomain = bridgedb.Dist.extractAddrSpec(clientAddr.lower())
    except bridgedb.Dist.BadEmail:
	logging.info("Ignoring bad address on incoming email.")
        return None,None
    if not addrdomain:
        logging.info("Couldn't parse domain from %r", clientAddr)
    if addrdomain and ctx.cfg.EMAIL_DOMAIN_MAP:
        addrdomain = ctx.cfg.EMAIL_DOMAIN_MAP.get(addrdomain, addrdomain)
    if addrdomain not in ctx.cfg.EMAIL_DOMAINS:
        logging.info("Unrecognized email domain %r", addrdomain)
        return None,None
    rules = ctx.cfg.EMAIL_DOMAIN_RULES.get(addrdomain, [])
    if 'dkim' in rules:
        # getheader() returns the last of a given kind of header; we want
        # to get the first, so we use getheaders() instead.
        dkimHeaders = msg.getheaders("X-DKIM-Authentication-Results")
        dkimHeader = "<no header>"
        if dkimHeaders: dkimHeader = dkimHeaders[0]
        if not dkimHeader.startswith("pass"):
            logging.info("Got a bad dkim header (%r) on an incoming mail; "
                         "rejecting it.", dkimHeader)
            return None, None

    # Was the magic string included
    for ln in lines:
        if ln.strip().lower() in ("get bridges", "subject: get bridges"):
            break
    else:
        logging.info("Got a mail from %r with no bridge request; dropping",
                     clientAddr)
        return None,None

    # Figure out which bridges to send
    try:
        interval = ctx.schedule.getInterval(time.time())
        bridges = ctx.distributor.getBridgesForEmail(clientAddr,
                                                     interval, ctx.N)
    except bridgedb.Dist.BadEmail, e:
        logging.info("Got a mail from a bad email address %r: %s.",
                     clientAddr, e)
        return None, None

    # Generate the message.
    f = StringIO()
    w = MimeWriter.MimeWriter(f)
    w.addheader("From", ctx.fromAddr)
    w.addheader("To", clientAddr)
    w.addheader("Message-ID", twisted.mail.smtp.messageid())
    if not subject.startswith("Re:"): subject = "Re: %s"%subject
    w.addheader("Subject", subject)
    if msgID:
        w.addheader("In-Reply-To", msgID)
    w.addheader("Date", twisted.mail.smtp.rfc822date())
    body = w.startbody("text/plain")

    if bridges:
        answer = "".join("  %s\n" % b.getConfigLine() for b in bridges)
    else:
        answer = "(no bridges currently available)"
    body.write(EMAIL_MESSAGE_TEMPLATE % answer)

    f.seek(0)
    return clientAddr, f

def replyToMail(lines, ctx):
    """Given a list of lines from an incoming email message, and a
       MailContext object, possibly send a reply.
    """
    logging.info("Got a completed email; attempting to reply.")
    sendToUser, response = getMailResponse(lines, ctx)
    if response is None:
        return
    response.seek(0)
    d = Deferred()
    factory = twisted.mail.smtp.SMTPSenderFactory(
        ctx.smtpFromAddr,
        sendToUser,
        response,
        d)
    reactor.connectTCP(ctx.smtpServer, ctx.smtpPort, factory)
    logging.info("Sending reply to %r", sendToUser)
    return d

class MailContext:
    """Helper object that holds information used by email subsystem."""
    def __init__(self, cfg, dist, sched):
        # Reject any RCPT TO lines that aren't to this user.
        self.username = (cfg.EMAIL_USERNAME or
                         "bridges")
        # Reject any mail longer than this.
        self.maximumSize = 32*1024
        # Use this server for outgoing mail.
        self.smtpServer = "127.0.0.1"
        self.smtpPort = 25
        # Use this address in the MAIL FROM line for outgoing mail.
        self.smtpFromAddr = (cfg.EMAIL_SMTP_FROM_ADDR or
                             "bridges@torproject.org")
        # Use this address in the "From:" header for outgoing mail.
        self.fromAddr = (cfg.EMAIL_FROM_ADDR or
                         "bridges@torproject.org")
        # An EmailBasedDistributor object
        self.distributor = dist
        # An IntervalSchedule object
        self.schedule = sched
        # The number of bridges to send for each email.
        self.N = cfg.EMAIL_N_BRIDGES_PER_ANSWER

        self.cfg = cfg

class MailMessage:
    """Plugs into the Twisted Mail and receives an incoming message.
       Once the message is in, we reply or we don't. """
    implements(twisted.mail.smtp.IMessage)

    def __init__(self, ctx):
        """Create a new MailMessage from a MailContext."""
        self.ctx = ctx
        self.lines = []
        self.nBytes = 0
        self.ignoring = False

    def lineReceived(self, line):
        """Called when we get another line of an incoming message."""
        self.nBytes += len(line)
        logging.debug("> %s", line.rstrip("\r\n"))
        if self.nBytes > self.ctx.maximumSize:
            self.ignoring = True
        else:
            self.lines.append(line)

    def eomReceived(self):
        """Called when we receive the end of a message."""
        if not self.ignoring:
            replyToMail(self.lines, self.ctx)
        return twisted.internet.defer.succeed(None)

    def connectionLost(self):
        """Called if we die partway through reading a message."""
        pass

class MailDelivery:
    """Plugs into Twisted Mail and handles SMTP commands."""
    implements(twisted.mail.smtp.IMessageDelivery)
    def setBridgeDBContext(self, ctx):
        self.ctx = ctx
    def receivedHeader(self, helo, origin, recipients):
        #XXXX what is this for? what should it be?
        return "Received: BridgeDB"
    def validateFrom(self, helo, origin):
        return origin
    def validateTo(self, user):
        if user.dest.local != self.ctx.username:
            raise twisted.mail.smtp.SMTPBadRcpt(user)
        return lambda: MailMessage(self.ctx)

class MailFactory(twisted.mail.smtp.SMTPFactory):
    """Plugs into Twisted Mail; creates a new MailDelivery whenever we get
       a connection on the SMTP port."""
    def __init__(self, *a, **kw):
        twisted.mail.smtp.SMTPFactory.__init__(self, *a, **kw)
        self.delivery = MailDelivery()

    def setBridgeDBContext(self, ctx):
        self.ctx = ctx
        self.delivery.setBridgeDBContext(ctx)

    def buildProtocol(self, addr):
        p = twisted.mail.smtp.SMTPFactory.buildProtocol(self, addr)
        p.delivery = self.delivery
        return p

def addSMTPServer(cfg, dist, sched):
    """Set up a smtp server.
         cfg -- a configuration object from Main.  We use these options:
                EMAIL_BIND_IP
                EMAIL_PORT
                EMAIL_N_BRIDGES_PER_ANSWER
                EMAIL_DOMAIN_RULES
         dist -- an EmailBasedDistributor object.
         sched -- an IntervalSchedule object.
    """
    ctx = MailContext(cfg, dist, sched)
    factory = MailFactory()
    factory.setBridgeDBContext(ctx)
    ip = cfg.EMAIL_BIND_IP or ""
    reactor.listenTCP(cfg.EMAIL_PORT, factory, interface=ip)
    return factory

def runServers():
    """Start all the servers that we've configured. Exits when they do."""
    reactor.run()
<|endoftext|>