<commit_before>s# coding=utf-8

from flask import Blueprint, Response, current_app, request, redirect

from slideatlas import models
from slideatlas.common_utils import jsonify

import base64
from bson import ObjectId

################################################################################
mod = Blueprint('tile', __name__)

################################################################################
@mod.route('/tile/<ObjectId:image_store_id>/<ObjectId:image_id>/<string:tile_name>')
# @security.login_required
def tile(image_store_id, image_id, tile_name):
    """
    Return a tile image.

    Note that the 'image_store' and 'image_id' URL parameters are accepted as
    an ObjectIds, to prevent unnecessary database queries if HTTP caching
    causes a 304 (Not Modified) response to be returned.
    """
    response = Response(content_type='image/jpeg')
    # tiles could change, especially if PTIFFs are rebuilt, so mark the ETag as
    #   weak
    # TODO: could we ensure that tiles are immutable?
    response.set_etag('%s-%s-v2' % (image_id, tile_name), weak=True)
    response.cache_control.public = True
    # since Last Modified is not used, and the ETag is weak, setting a max-age
    #   is important, so clients don't cache tiles too long
    response.cache_control.max_age = current_app.get_send_file_max_age(None)
    # Last Modified is not strictly required, per RFC 2616 14.29, and setting
    #   it accurately would require a database query; by leaving it unset,
    #   user agents will use only If-None-Match with ETags for cache validation

    response.make_conditional(request)
    if response.status_code != 304:  # Not Modified
        image_store = models.ImageStore.objects.get_or_404(id=image_store_id)
        try:
            tile_data = image_store.get_tile(image_id, tile_name)
        except models.DoesNotExist as e:
            # TODO: more specific exception
            current_app.logger.warning('Tile not loaded: %s' % e.message)
            return Response('{"error": "Tile loading error: %s"}' % e.message, status=404)
        response.set_data(tile_data)

    return response

@mod.route('/tile')
# @security.login_required
def tile_query():
    image_store_id = request.args.get('db')
    image_id = request.args.get('img')
    tile_name = request.args.get('name')

    return tile(image_store_id, image_id, tile_name)

################################################################################
@mod.route('/thumb/<ImageStore:image_store>/<Image:image>')
# @security.login_required
def thumb(image_store, image, imageobj):
    """
    Return a thumbnail image
    """
    if imageobj and 'girder' in imageobj:
        server = imageobj['girder']['server']
        girderItemId = imageobj['girder']['itemId']
        girder_url = server + '/api/v1/item/' + str(girderItemId) + '/tiles/thumbnail?height=100'
        resp = Response(girder_url, status=301)
        resp.headers['Location'] = girder_url
        return resp

    # TODO: support Not Modified) responses, but only after thumbnails are
    #   moved out of ImageStores; thumbnails are mutable, so 2 database lookups
    #   will currently always have to be made
    try:
        tile_data = image_store.get_thumb(image)
        return Response(tile_data, mimetype='image/jpeg')
    except models.DoesNotExist as e:
        current_app.logger.warning('Thumb not available: %s' % e.message)
        return Response('{"error": "Thumb loading error: %s"}' % e.message, status=404)

################################################################################
@mod.route('/thumb')
# @security.login_required
def thumb_query():
    image_id = request.args.get('img')
    image_store_id = request.args.get('db')

    image_store = models.ImageStore.objects.get_or_404(id=image_store_id)
    with image_store:
        imagecol = models.Image._get_collection()
        try:
            imageobj = imagecol.find_one({"_id": ObjectId(image_id)})
        except:
            # TODO: more specific exception
            current_app.logger.warning('Bad image id: %s' % image_id)
            return Response('{"error": "Thumb loading error: %s"}' % image_id, status=404)
        image = models.Image.objects.get_or_404(id=image_id)

    return thumb(image_store, image, imageobj)

################################################################################
@mod.route('/viewthumb')
def thumb_from_view():
    """
    Gets a thumbnail from view directly,
    Chains the request to view objects helper method
    """

    # Get parameters
    viewid = ObjectId(request.args.get("viewid", None))
    # which = ObjectId(request.args.get("which","macro"))
    which = "macro"
    force = bool(request.args.get("force", False))

    # Implementation without View
    viewcol = models.View._get_collection()
    viewobj = viewcol.find_one({"_id": viewid})
    if "thumbs" not in viewobj:
        viewobj["thumbs"] = {}

    # Make thumbnail
    if force or which not in viewobj["thumbs"]:
        # Refresh the thumbnail
        if which not in ["macro"]:
            # Only know how to make macro image
            # Todo: add support for label supported
            raise Exception("%s thumbnail creation not supported" % which)

        # Make the thumb
        # Get the image store and image id and off load the request
        istore = models.ImageStore.objects.get(id=viewobj["ViewerRecords"][0]["Database"])
        with istore:
            thumbimgdata = istore.make_thumb(
                models.Image.objects.get(id=viewobj["ViewerRecords"][0]["Image"]))

            viewcol.update({"_id": viewid},
                           {"$set": {"thumbs." + which: base64.b64encode(thumbimgdata)}})

    viewobj = viewcol.find_one({"_id": viewid})
    imagestr = viewobj["thumbs"][which]
    # current_app.logger.warning("Imagestr: " + imagestr)

    if int(request.args.get("binary", 0)):
        return Response(base64.b64decode(imagestr), mimetype="image/jpeg")
    else:
        return jsonify({which: imagestr})

@mod.route('/view')
def view_from_viewid():
    """
    redirects to the glviewer endpoint
    """

    # Get parameters
    viewid = ObjectId(request.args.get("viewid", None))

    # Implementation without View
    viewcol = models.View._get_collection()
    viewobj = viewcol.find_one({"_id": viewid})

    return redirect("/webgl-viewer?db=%s&view=%s" % (viewobj["ViewerRecords"][0]["Database"], viewid), code=302)
<commit_msg>Trial and error syntax 1<commit_after>s# coding=utf-8

from flask import Blueprint, Response, current_app, request, redirect

from slideatlas import models
from slideatlas.common_utils import jsonify

import base64
from bson import ObjectId

################################################################################
mod = Blueprint('tile', __name__)

################################################################################
@mod.route('/tile/<ObjectId:image_store_id>/<ObjectId:image_id>/<string:tile_name>')
# @security.login_required
def tile(image_store_id, image_id, tile_name):
    """
    Return a tile image.

    Note that the 'image_store' and 'image_id' URL parameters are accepted as
    an ObjectIds, to prevent unnecessary database queries if HTTP caching
    causes a 304 (Not Modified) response to be returned.
    """
    response = Response(content_type='image/jpeg')
    # tiles could change, especially if PTIFFs are rebuilt, so mark the ETag as
    #   weak
    # TODO: could we ensure that tiles are immutable?
    response.set_etag('%s-%s-v2' % (image_id, tile_name), weak=True)
    response.cache_control.public = True
    # since Last Modified is not used, and the ETag is weak, setting a max-age
    #   is important, so clients don't cache tiles too long
    response.cache_control.max_age = current_app.get_send_file_max_age(None)
    # Last Modified is not strictly required, per RFC 2616 14.29, and setting
    #   it accurately would require a database query; by leaving it unset,
    #   user agents will use only If-None-Match with ETags for cache validation

    response.make_conditional(request)
    if response.status_code != 304:  # Not Modified
        image_store = models.ImageStore.objects.get_or_404(id=image_store_id)
        try:
            tile_data = image_store.get_tile(image_id, tile_name)
        except models.DoesNotExist as e:
            # TODO: more specific exception
            current_app.logger.warning('Tile not loaded: %s' % e.message)
            return Response('{"error": "Tile loading error: %s"}' % e.message, status=404)
        response.set_data(tile_data)

    return response

@mod.route('/tile')
# @security.login_required
def tile_query():
    image_store_id = request.args.get('db')
    image_id = request.args.get('img')
    tile_name = request.args.get('name')

    return tile(image_store_id, image_id, tile_name)

################################################################################
@mod.route('/thumb/<ImageStore:image_store>/<Image:image>')
# @security.login_required
def thumb(image_store, image, imageobj):
    """
    Return a thumbnail image
    """
    #if imageobj and 'girder' in imageobj:
    #    server = imageobj['girder']['server']
    #    girderItemId = imageobj['girder']['itemId']
    #    girder_url = server + '/api/v1/item/' + str(girderItemId) + '/tiles/thumbnail?height=100'
    #    resp = Response(girder_url, status=301)
    #    resp.headers['Location'] = girder_url
    #    return resp

    # TODO: support Not Modified) responses, but only after thumbnails are
    #   moved out of ImageStores; thumbnails are mutable, so 2 database lookups
    #   will currently always have to be made
    try:
        tile_data = image_store.get_thumb(image)
        return Response(tile_data, mimetype='image/jpeg')
    except models.DoesNotExist as e:
        current_app.logger.warning('Thumb not available: %s' % e.message)
        return Response('{"error": "Thumb loading error: %s"}' % e.message, status=404)

################################################################################
@mod.route('/thumb')
# @security.login_required
def thumb_query():
    image_id = request.args.get('img')
    image_store_id = request.args.get('db')

    image_store = models.ImageStore.objects.get_or_404(id=image_store_id)
    with image_store:
        imagecol = models.Image._get_collection()
        try:
            imageobj = imagecol.find_one({"_id": ObjectId(image_id)})
        except:
            # TODO: more specific exception
            current_app.logger.warning('Bad image id: %s' % image_id)
            return Response('{"error": "Thumb loading error: %s"}' % image_id, status=404)
        image = models.Image.objects.get_or_404(id=image_id)

    return thumb(image_store, image, imageobj)

################################################################################
@mod.route('/viewthumb')
def thumb_from_view():
    """
    Gets a thumbnail from view directly,
    Chains the request to view objects helper method
    """

    # Get parameters
    viewid = ObjectId(request.args.get("viewid", None))
    # which = ObjectId(request.args.get("which","macro"))
    which = "macro"
    force = bool(request.args.get("force", False))

    # Implementation without View
    viewcol = models.View._get_collection()
    viewobj = viewcol.find_one({"_id": viewid})
    if "thumbs" not in viewobj:
        viewobj["thumbs"] = {}

    # Make thumbnail
    if force or which not in viewobj["thumbs"]:
        # Refresh the thumbnail
        if which not in ["macro"]:
            # Only know how to make macro image
            # Todo: add support for label supported
            raise Exception("%s thumbnail creation not supported" % which)

        # Make the thumb
        # Get the image store and image id and off load the request
        istore = models.ImageStore.objects.get(id=viewobj["ViewerRecords"][0]["Database"])
        with istore:
            thumbimgdata = istore.make_thumb(
                models.Image.objects.get(id=viewobj["ViewerRecords"][0]["Image"]))

            viewcol.update({"_id": viewid},
                           {"$set": {"thumbs." + which: base64.b64encode(thumbimgdata)}})

    viewobj = viewcol.find_one({"_id": viewid})
    imagestr = viewobj["thumbs"][which]
    # current_app.logger.warning("Imagestr: " + imagestr)

    if int(request.args.get("binary", 0)):
        return Response(base64.b64decode(imagestr), mimetype="image/jpeg")
    else:
        return jsonify({which: imagestr})

@mod.route('/view')
def view_from_viewid():
    """
    redirects to the glviewer endpoint
    """

    # Get parameters
    viewid = ObjectId(request.args.get("viewid", None))

    # Implementation without View
    viewcol = models.View._get_collection()
    viewobj = viewcol.find_one({"_id": viewid})

    return redirect("/webgl-viewer?db=%s&view=%s" % (viewobj["ViewerRecords"][0]["Database"], viewid), code=302)
<|endoftext|>


<commit_before>from mutagen.mp3 import MP3
from mutagen.id3 import ID3, TIT2
from mutagen.easyid3 import EasyID3
from shutil import make_archive
from bs4 import BeautifulSoup

import requests
import os
import sys
import urllib

def parse_file(url):

	print "Starting the parsing for: " + url
	r = requests.get(url)
	soup = BeautifulSoup(r.text)

	if "album" in url:
		songType = "album"
	else:
		songType = "track"

	albumTitle = soup.head.title.text

	embedBlock = r.text.split("var EmbedData = ")

	embedStringBlock = embedBlock[1]
	embedStringBlock = embedStringBlock.split("};")[0] + "}"
	embedStringBlock = embedStringBlock.strip().replace("   ", "")

	tralbum_param = "tralbum_param"
	name = "name"
	value = "value"
	swf_base_url = "swf_base_url"
	album_title = "album_title"
	art_url = "art_url"
	lg_art_url = "lg_art_url"
	numtracks = "numtracks"
	title = "title"
	artist = "artist"
	linkback = "linkback"

	embedData = eval(embedStringBlock)

	artistName = embedData['artist']

	if "name" in embedData.keys():
		fileType = "track"
		trackName = embedData['name']
	else:
		fileType = "album"

	albumTitle = embedData['album_title']

	block = r.text.split("var TralbumData = ")
	#print block[0]

	stringBlock = block[1]

	stringBlock = stringBlock.split("};")[0] + "}"
	#print stringBlock

	stringArray = stringBlock.split("\n")
	del stringArray[1:4]
	#print stringArray

	stringBlock = "".join(stringArray).strip().replace("    ", "")

	null = None
	current = "current"
	is_preorder = "is_preorder"
	album_is_preorder = "album_is_preorder"
	album_release_date = "album_release_date"
	album_url = "album_url"
	preorder_count = "preorder_count"
	hasAudio = "hasAudio"
	artThumbURL = "artThumbURL"
	artFullsizeUrl = "artFullsizeUrl"
	trackinfo = "trackinfo"
	playing_from = "playing_from"
	featured_track_id = "featured_track_id"
	initial_track_num = "initial_track_num"
	defaultPrice = "defaultPrice"
	freeDownloadPage = "freeDownloadPage"
	packages = "packages"
	maxPrice = "maxPrice"
	minPrice = "minPrice"
	FREE = "FREE"
	PAID = "PAID"
	artist = "artist"
	item_type = "item_type"
	id = "id"
	true = True
	false = False
	#print unicode(stringBlock.strip())
	data = eval(stringBlock.strip())

	artistName = data['artist']

	firstLetter = artistName[0]

	if not firstLetter.isalpha:
		firstLetter = "0"
	else:
		firstLetter = firstLetter.capitalize()

	if not os.path.exists("files"):
		os.makedirs("files")

	letterDirectory = "files/" + firstLetter

	if not os.path.exists("files/" + firstLetter):
		if(firstLetter.isalpha):
			os.makedirs("files/" + firstLetter)

	if not os.path.exists("files/" + firstLetter + "/" + artistName):
		os.makedirs("files/" + firstLetter + "/" + artistName)

	tracks = data['trackinfo']

	albumPath = albumTitle.replace(" ", "").replace("/","").replace(".", "")

	albumPath = "files/" + firstLetter + "/" + artistName + "/" + albumPath
	if not os.path.exists("files/zips"):
			os.makedirs("files/zips")

	if not os.path.exists(albumPath):
		os.makedirs(albumPath)

	for each in tracks:
		songTitle = each['title'].replace(" ", "").replace(".", "")
		songURL = each['file']['mp3-128']
		track_num = each['track_num']

		print "Now Downloading: " +  each['title'], each['file']['mp3-128']
		urllib.urlretrieve(songURL, albumPath + "/" + songTitle + ".mp3")

		print "Encoding . . . "
		audio = MP3(albumPath + "/" + songTitle + ".mp3")
		audio["TIT2"]=TIT2(encoding=3, text=["title"])
		audio.save()
		audio = EasyID3(albumPath + "/" + songTitle + ".mp3")
		audio["title"] = each['title']
		audio["artist"] = artistName
		audio["album"] = albumTitle
		#audio["tracknumber"] = track_num
		audio.save()

		#audiofile.tag.save()
		print "Done downloading " + songTitle

	if(len(tracks) > 1):
		if not os.path.isfile("files/zips/" + albumTitle.replace(" ", "") + ".zip"):
			make_archive("files/zips/" + albumTitle.replace(" ", ""), 'zip', albumPath)
		else:
			print "Already have a zipfile of this junts, serve that up!"
		#zip = zipfile.ZipFile("files/zips/" + albumTitle.replace(" ", "") + ".zip", 'w')
		#zipdir(albumPath + "/", zip)
		#zip.close

def parse_results(text):
	soup = BeautifulSoup(text)

	items = soup.findAll("li", "searchresult")

	for item in items:
		typeText = item.find(class_="itemtype").text.strip()
		albumTitle = item.find(class_="heading").text.strip()

		artistName = item.find(class_="subhead").text.strip()

		artistName = artistName.replace("by ", "")

		itemURL = item.find(class_="itemurl").text.strip()

		if "track" in itemURL:
			itemType = "track"
		else:
			itemType = "album"

		result = Result(itemURL, artistName, albumTitle, itemType)
		results.append(result)
		#print itemType, albumTitle, artistName, itemURL

def printTags():
	for r in results:
		print r.artist, r.title, r.type, r.url, r.type

def getTags(path):

	myfile = mpeg.Mpeg('inthecity.mp3')
	print myfile.artist
	print myfile.album
	print myfile.original

url = sys.argv[1]
if(len(sys.argv) != 2):
	print "usage: bandcamp-dl.py <url to download>"
	sys.exit

parse_file(url)<commit_msg>Some code fixes.<commit_after>from mutagen.mp3 import MP3
from mutagen.id3 import ID3, TIT2
from mutagen.easyid3 import EasyID3
from shutil import make_archive
from bs4 import BeautifulSoup

import requests
import os
import sys
import urllib

def parse_file(url):

	print "Starting the parsing for: " + url
	r = requests.get(url)
	soup = BeautifulSoup(r.text)

	if "album" in url:
		songType = "album"
	else:
		songType = "track"

	albumTitle = soup.head.title.text

	embedBlock = r.text.split("var EmbedData = ")

	embedStringBlock = embedBlock[1]
	embedStringBlock = embedStringBlock.split("};")[0] + "}"

	sys.exit()
	embedStringBlock = embedStringBlock.strip().replace("   ", "")

	tralbum_param = "tralbum_param"
	name = "name"
	value = "value"
	swf_base_url = "swf_base_url"
	album_title = "album_title"
	art_url = "art_url"
	lg_art_url = "lg_art_url"
	numtracks = "numtracks"
	title = "title"
	artist = "artist"
	linkback = "linkback"

	embedData = eval(embedStringBlock)

	artistName = embedData['artist']

	if "name" in embedData:
		fileType = "track"
		trackName = embedData['name']
	else:
		fileType = "album"

	albumTitle = embedData['album_title']

	block = r.text.split("var TralbumData = ")
	#print block[0]

	stringBlock = block[1]

	stringBlock = stringBlock.split("};")[0] + "}"
	#print stringBlock

	stringArray = stringBlock.split("\n")
	del stringArray[1:4]
	#print stringArray

	stringBlock = "".join(stringArray).strip().replace("    ", "")

	null = None
	current = "current"
	is_preorder = "is_preorder"
	album_is_preorder = "album_is_preorder"
	album_release_date = "album_release_date"
	album_url = "album_url"
	preorder_count = "preorder_count"
	hasAudio = "hasAudio"
	artThumbURL = "artThumbURL"
	artFullsizeUrl = "artFullsizeUrl"
	trackinfo = "trackinfo"
	playing_from = "playing_from"
	featured_track_id = "featured_track_id"
	initial_track_num = "initial_track_num"
	defaultPrice = "defaultPrice"
	freeDownloadPage = "freeDownloadPage"
	packages = "packages"
	maxPrice = "maxPrice"
	minPrice = "minPrice"
	FREE = "FREE"
	PAID = "PAID"
	artist = "artist"
	item_type = "item_type"
	id = "id"
	true = True
	false = False
	#print unicode(stringBlock.strip())
	data = eval(stringBlock.strip())

	artistName = data['artist']

	firstLetter = artistName[0]

	if not firstLetter.isalpha:
		firstLetter = "0"
	else:
		firstLetter = firstLetter.capitalize()

	if not os.path.exists("files"):
		os.makedirs("files")

	letterDirectory = "files/" + firstLetter

	if not os.path.exists("files/" + firstLetter):
		if(firstLetter.isalpha):
			os.makedirs("files/" + firstLetter)

	if not os.path.exists("files/" + firstLetter + "/" + artistName):
		os.makedirs("files/" + firstLetter + "/" + artistName)

	tracks = data['trackinfo']

	albumPath = albumTitle.replace(" ", "").replace("/","").replace(".", "")

	albumPath = "files/" + firstLetter + "/" + artistName + "/" + albumPath
	if not os.path.exists("files/zips"):
			os.makedirs("files/zips")

	if not os.path.exists(albumPath):
		os.makedirs(albumPath)

	for each in tracks:
		songTitle = each['title'].replace(" ", "").replace(".", "")
		songURL = each['file']['mp3-128']
		track_num = each['track_num']

		print "Now Downloading: " +  each['title'], each['file']['mp3-128']
		urllib.urlretrieve(songURL, albumPath + "/" + songTitle + ".mp3")

		print "Encoding . . . "
		audio = MP3(albumPath + "/" + songTitle + ".mp3")
		audio["TIT2"]=TIT2(encoding=3, text=["title"])
		audio.save()
		audio = EasyID3(albumPath + "/" + songTitle + ".mp3")
		audio["title"] = each['title']
		audio["artist"] = artistName
		audio["album"] = albumTitle
		#audio["tracknumber"] = track_num
		audio.save()

		#audiofile.tag.save()
		print "Done downloading " + songTitle

	if(len(tracks) > 1):
		if not os.path.isfile("files/zips/" + albumTitle.replace(" ", "") + ".zip"):
			make_archive("files/zips/" + albumTitle.replace(" ", ""), 'zip', albumPath)
		else:
			print "Already have a zipfile of this junts, serve that up!"
		#zip = zipfile.ZipFile("files/zips/" + albumTitle.replace(" ", "") + ".zip", 'w')
		#zipdir(albumPath + "/", zip)
		#zip.close

def parse_results(text):
	soup = BeautifulSoup(text)

	items = soup.findAll("li", "searchresult")

	for item in items:
		typeText = item.find(class_="itemtype").text.strip()
		albumTitle = item.find(class_="heading").text.strip()

		artistName = item.find(class_="subhead").text.strip()

		artistName = artistName.replace("by ", "")

		itemURL = item.find(class_="itemurl").text.strip()

		if "track" in itemURL:
			itemType = "track"
		else:
			itemType = "album"

		result = Result(itemURL, artistName, albumTitle, itemType)
		results.append(result)
		#print itemType, albumTitle, artistName, itemURL

def printTags():
	for r in results:
		print r.artist, r.title, r.type, r.url, r.type

def getTags(path):

	myfile = mpeg.Mpeg('inthecity.mp3')
	print myfile.artist
	print myfile.album
	print myfile.original

url = sys.argv[1]
if(len(sys.argv) != 2):
	print "usage: bandcamp-dl.py <url to download>"
	sys.exit()

parse_file(url)<|endoftext|>


<commit_before>from urllib import urlencode
from lxml import html
from urlparse import urljoin

categories = ['images']

base_url = 'https://www.deviantart.com/'
search_url = base_url+'search?'

def request(query, params):
    global search_url
    params['url'] = search_url + urlencode({'q': query})
    return params

def response(resp):
    global base_url
    results = []
    if resp.status_code == 302:
        return results
    dom = html.fromstring(resp.text)
    for result in dom.xpath('//div[contains(@class, "tt-a tt-fh")]'):
        link = result.xpath('.//a[contains(@class, "thumb")]')[0]
        url = urljoin(base_url, link.attrib.get('href'))
        title_links = result.xpath('.//span[@class="details"]//a[contains(@class, "t")]')
        title = ''.join(title_links[0].xpath('.//text()'))
        img_src = link.xpath('.//img')[0].attrib['src']
        print img_src
        results.append({'url': url, 'title': title, 'img_src': img_src, 'template': 'images.html'})
    return results
<commit_msg>[fix] debug print removed<commit_after>from urllib import urlencode
from lxml import html
from urlparse import urljoin

categories = ['images']

base_url = 'https://www.deviantart.com/'
search_url = base_url+'search?'

def request(query, params):
    global search_url
    params['url'] = search_url + urlencode({'q': query})
    return params

def response(resp):
    global base_url
    results = []
    if resp.status_code == 302:
        return results
    dom = html.fromstring(resp.text)
    for result in dom.xpath('//div[contains(@class, "tt-a tt-fh")]'):
        link = result.xpath('.//a[contains(@class, "thumb")]')[0]
        url = urljoin(base_url, link.attrib.get('href'))
        title_links = result.xpath('.//span[@class="details"]//a[contains(@class, "t")]')
        title = ''.join(title_links[0].xpath('.//text()'))
        img_src = link.xpath('.//img')[0].attrib['src']
        results.append({'url': url, 'title': title, 'img_src': img_src, 'template': 'images.html'})
    return results
<|endoftext|>


<commit_before># Copyright 2014 0xc0170
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from optparse import OptionParser
import yaml
import logging
import os
from yaml_parser import get_project_files, YAML_parser
from os.path import join
import sys
from os.path import basename
from ide import export

def run_generator(dic, project, ide):
    project_list = []
    yaml_files = get_project_files(dic, project) # TODO fix list inside list
    if yaml_files:
        for yaml_file in yaml_files:
            try:
                file = open(yaml_file)
            except IOError:
                raise RuntimeError("Cannot open a file: %s" % yaml_file)
            else:
                loaded_yaml = yaml.load(file)
                yaml_parser = YAML_parser()
                project_list.append(yaml_parser.parse_yaml(loaded_yaml, ide))
                file.close()
        yaml_parser_final = YAML_parser()
        process_data = yaml_parser_final.parse_list_yaml(project_list)
    else:
        raise RuntimeError("Project record is empty")

    logging.info("Generating project: %s" % project)
    #ide = get_ide(process_data)
    export(process_data, ide)

def process_all_projects(dic, ide):
    projects = []
    yaml_files = []
    for k,v in dic['projects'].items():
        projects.append(k);

    for project in projects:
        run_generator(dic, project, ide)

def scrape_dir():
    exts = ['s', 'c', 'cpp', 'h', 'inc', 'sct', 'ld']
    found = {x: [] for x in exts} # lists of found files
    ignore = '.'
    for dirpath, dirnames, files in os.walk(os.getcwd()):
        # Remove directories in ignore
        # directory names must match exactly!
        for idir in ignore:
            if idir in dirnames:
                dirnames.remove(idir)
        # Loop through the file names for the current step
        for name in files:
            # Split the name by '.' & get the last element
            ext = name.lower().rsplit('.', 1)[-1]
            # Save the full name if ext matches
            if ext in exts:
                relpath = dirpath.replace(os.getcwd(), "")
                found[ext].append(os.path.join(relpath, name))
    # The body of our log file
    logbody = ''
    # loop thru results
    for search in found:
        # Concatenate the result from the found dict
        logbody += "<< Results with the extension '%s' >>" % search
        logbody += '\n\n%s\n\n' % '\n'.join(found[search])
    # Write results to the logfile
    source_list_path = os.getcwd() + '\\tools\\exporters\\records\\'
    if not os.path.exists(source_list_path):
        os.makedirs(source_list_path)
    target_path = join(source_list_path, 'scrape.log')
    logging.debug("Generating: %s" % target_path)
    with open(target_path, 'w') as logfile:
        logfile.write('%s' % logbody)

def list_projects(dic):
    for k,v in dic['projects'].items():
        print k

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    # Should be launched from root/tools but all scripts are referenced to root
    root = os.path.normpath(os.getcwd() + os.sep + os.pardir)
    os.chdir(root)
    logging.debug('This should be the project root: %s', os.getcwd())

    # Parse Options
    parser = OptionParser()
    parser.add_option("-f", "--file", help="YAML projects file")
    parser.add_option("-p", "--project", help="Project to be generated")
    parser.add_option("-i", "--ide", help="Create project files for toolchain (uvision by default)")
    parser.add_option("-l", "--list", action="store_true", help="List projects defined in the project file.")

    (options, args) = parser.parse_args()

    if not options.file:
        # create a list of all files in the dir that we're interested in
        scrape_dir()
        # print help menu
        parser.print_help()
        sys.exit()

    if not options.ide:
        options.ide = "uvision"

    # always run from the root directory
    script_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
    # os.chdir(script_dir)
    # os.chdir('../')

    print "Processing projects file."
    project_file = open('tools//' + options.file)
    config = yaml.load(project_file)

    if options.list:
        print "Projects defined in the %s" % options.file
        list_projects(config)
        sys.exit()

    if options.project:
        run_generator(config, options.project, options.ide) # one project
    else:
        process_all_projects(config, options.ide) # all projects within project.yaml

    project_file.close()
<commit_msg>Update location of scrape.log to match new dir structure when used as submodule<commit_after># Copyright 2014 0xc0170
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from optparse import OptionParser
import yaml
import logging
import os
from yaml_parser import get_project_files, YAML_parser
from os.path import join
import sys
from os.path import basename
from ide import export

def run_generator(dic, project, ide):
    project_list = []
    yaml_files = get_project_files(dic, project) # TODO fix list inside list
    if yaml_files:
        for yaml_file in yaml_files:
            try:
                file = open(yaml_file)
            except IOError:
                raise RuntimeError("Cannot open a file: %s" % yaml_file)
            else:
                loaded_yaml = yaml.load(file)
                yaml_parser = YAML_parser()
                project_list.append(yaml_parser.parse_yaml(loaded_yaml, ide))
                file.close()
        yaml_parser_final = YAML_parser()
        process_data = yaml_parser_final.parse_list_yaml(project_list)
    else:
        raise RuntimeError("Project record is empty")

    logging.info("Generating project: %s" % project)
    #ide = get_ide(process_data)
    export(process_data, ide)

def process_all_projects(dic, ide):
    projects = []
    yaml_files = []
    for k,v in dic['projects'].items():
        projects.append(k);

    for project in projects:
        run_generator(dic, project, ide)

def scrape_dir():
    exts = ['s', 'c', 'cpp', 'h', 'inc', 'sct', 'ld']
    found = {x: [] for x in exts} # lists of found files
    ignore = '.'
    for dirpath, dirnames, files in os.walk(os.getcwd()):
        # Remove directories in ignore
        # directory names must match exactly!
        for idir in ignore:
            if idir in dirnames:
                dirnames.remove(idir)
        # Loop through the file names for the current step
        for name in files:
            # Split the name by '.' & get the last element
            ext = name.lower().rsplit('.', 1)[-1]
            # Save the full name if ext matches
            if ext in exts:
                relpath = dirpath.replace(os.getcwd(), "")
                found[ext].append(os.path.join(relpath, name))
    # The body of our log file
    logbody = ''
    # loop thru results
    for search in found:
        # Concatenate the result from the found dict
        logbody += "<< Results with the extension '%s' >>" % search
        logbody += '\n\n%s\n\n' % '\n'.join(found[search])
    # Write results to the logfile
    source_list_path = os.getcwd() + '\\tools\\records\\'
    if not os.path.exists(source_list_path):
        os.makedirs(source_list_path)
    target_path = join(source_list_path, 'scrape.log')
    logging.debug("Generating: %s" % target_path)
    with open(target_path, 'w') as logfile:
        logfile.write('%s' % logbody)

def list_projects(dic):
    for k,v in dic['projects'].items():
        print k

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    # Should be launched from root/tools but all scripts are referenced to root
    root = os.path.normpath(os.getcwd() + os.sep + os.pardir)
    os.chdir(root)
    logging.debug('This should be the project root: %s', os.getcwd())

    # Parse Options
    parser = OptionParser()
    parser.add_option("-f", "--file", help="YAML projects file")
    parser.add_option("-p", "--project", help="Project to be generated")
    parser.add_option("-i", "--ide", help="Create project files for toolchain (uvision by default)")
    parser.add_option("-l", "--list", action="store_true", help="List projects defined in the project file.")

    (options, args) = parser.parse_args()

    if not options.file:
        # create a list of all files in the dir that we're interested in
        scrape_dir()
        # print help menu
        parser.print_help()
        sys.exit()

    if not options.ide:
        options.ide = "uvision"

    # always run from the root directory
    script_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
    # os.chdir(script_dir)
    # os.chdir('../')

    print "Processing projects file."
    project_file = open('tools//' + options.file)
    config = yaml.load(project_file)

    if options.list:
        print "Projects defined in the %s" % options.file
        list_projects(config)
        sys.exit()

    if options.project:
        run_generator(config, options.project, options.ide) # one project
    else:
        process_all_projects(config, options.ide) # all projects within project.yaml

    project_file.close()
<|endoftext|>


<commit_before># A Django model field for use with Python 3 enums.
#
# Works with any enum whose values are integers. Subclasses the IntegerField
# to store the enum as integers in the database. Serializes to dotted names
# (e.g. AnimalType.Cat in the example below).
#
# A decorator is needed on Python enums in order to make them work with
# Django migrations, which require a deconstruct() method on the enum
# members.
#
# Example:
#
# import enum
# from enum3field import EnumField, django_enum
#
# @django_enum
# class AnimalType(enum.Enum):
#   Cat = 1
#   Dog = 2
#   Turtle = 3
#
# class Animal(models.Model):
#   animalType = EnumField(AnimalType)
#
#####################################################

import enum

from django.core import exceptions
from django.db import models

def django_enum(enum):
  # In order to make the enumeration serializable for migrations, instance members
  # must have a deconstruct method. But the enum class should not have a deconstruct
  # method or else the serialization of the enum itself as the first argument to
  # EnumField will fail. To achieve this, we added a deconstruct() method to the
  # members after the enum members have been created.
  def make_deconstructor(member):
    def deconstruct():
      return (enum.__module__ + '.' + enum.__name__, [member.value], {})
    return deconstruct
  for member in enum:
    member.deconstruct = make_deconstructor(member)
  return enum

class EnumField(models.IntegerField, metaclass=models.SubfieldBase):
  """A Django model field for use with Python 3 enums. Usage: fieldname = EnumField(enum_class, ....)"""

  def __init__(self, enum_class, *args, **kwargs):
    if not issubclass(enum_class, enum.Enum):
      raise ValueError("enum_class argument must be a Python 3 enum.")
    self.enum_class = enum.unique(enum_class) # ensure unique members to prevent accidental database corruption
    kwargs['choices'] = [(item, item.name) for item in self.enum_class]
    super(EnumField, self).__init__(*args, **kwargs)

  description = "A value of the %(enum_class) enumeration."

  def get_prep_value(self, value):
    # Normally value is an enumeration value. But when running `manage.py migrate`
    # we may get a serialized value. Use to_python to coerce to an enumeration
    # member as best as possible.
    value = self.to_python(value)

    if value is not None:
      # Validate
      if not isinstance(value, self.enum_class):
        raise exceptions.ValidationError(
          "'%s' must be a member of %s." % (value, self.enum_class),
          code='invalid',
          params={'value': value},
          )

      # enum member => member.value (should be an integer)
      value = value.value

    # integer => database
    return super(EnumField, self).get_prep_value(value)

  def to_python(self, value):
    # handle None and values of the correct type already
    if value is None or isinstance(value, self.enum_class):
      return value

    # When serializing to create a fixture, the default serialization
    # is to "EnumName.MemberName". Handle that.
    prefix = self.enum_class.__name__ + "."
    if isinstance(value, str) and value.startswith(prefix):
      try:
        return self.enum_class[value[len(prefix):]]
      except KeyError:
        raise exceptions.ValidationError(
          "'%s' does not refer to a member of %s." % (value, self.enum_class),
          code='invalid',
          params={'value': value},
          )

    # We may also get string versions of the integer form from forms,
    # and integers when querying a database.
    try:
      return self.enum_class(int(value))
    except ValueError:
      raise exceptions.ValidationError(
        "'%s' must be an integer value of %s." % (value, self.enum_class),
        code='invalid',
        params={'value': value},
        )

  def deconstruct(self):
    # Override the positional arguments info to include the enumeration class.
    tup = super(EnumField, self).deconstruct()
    return (tup[0], tup[1], [self.enum_class], tup[3])
<commit_msg>must disable IntegerField validators or saving forms (e.g. in the admin) wont work<commit_after># A Django model field for use with Python 3 enums.
#
# Works with any enum whose values are integers. Subclasses the IntegerField
# to store the enum as integers in the database. Serializes to dotted names
# (e.g. AnimalType.Cat in the example below).
#
# A decorator is needed on Python enums in order to make them work with
# Django migrations, which require a deconstruct() method on the enum
# members.
#
# Example:
#
# import enum
# from enum3field import EnumField, django_enum
#
# @django_enum
# class AnimalType(enum.Enum):
#   Cat = 1
#   Dog = 2
#   Turtle = 3
#
# class Animal(models.Model):
#   animalType = EnumField(AnimalType)
#
#####################################################

import enum

from django.core import exceptions
from django.db import models
from django.utils.functional import cached_property

def django_enum(enum):
  # In order to make the enumeration serializable for migrations, instance members
  # must have a deconstruct method. But the enum class should not have a deconstruct
  # method or else the serialization of the enum itself as the first argument to
  # EnumField will fail. To achieve this, we added a deconstruct() method to the
  # members after the enum members have been created.
  def make_deconstructor(member):
    def deconstruct():
      return (enum.__module__ + '.' + enum.__name__, [member.value], {})
    return deconstruct
  for member in enum:
    member.deconstruct = make_deconstructor(member)
  return enum

class EnumField(models.IntegerField, metaclass=models.SubfieldBase):
  """A Django model field for use with Python 3 enums. Usage: fieldname = EnumField(enum_class, ....)"""

  def __init__(self, enum_class, *args, **kwargs):
    if not issubclass(enum_class, enum.Enum):
      raise ValueError("enum_class argument must be a Python 3 enum.")
    self.enum_class = enum.unique(enum_class) # ensure unique members to prevent accidental database corruption
    kwargs['choices'] = [(item, item.name) for item in self.enum_class]
    super(EnumField, self).__init__(*args, **kwargs)

  description = "A value of the %(enum_class) enumeration."

  def get_prep_value(self, value):
    # Normally value is an enumeration value. But when running `manage.py migrate`
    # we may get a serialized value. Use to_python to coerce to an enumeration
    # member as best as possible.
    value = self.to_python(value)

    if value is not None:
      # Validate
      if not isinstance(value, self.enum_class):
        raise exceptions.ValidationError(
          "'%s' must be a member of %s." % (value, self.enum_class),
          code='invalid',
          params={'value': value},
          )

      # enum member => member.value (should be an integer)
      value = value.value

    # integer => database
    return super(EnumField, self).get_prep_value(value)

  def to_python(self, value):
    # handle None and values of the correct type already
    if value is None or isinstance(value, self.enum_class):
      return value

    # When serializing to create a fixture, the default serialization
    # is to "EnumName.MemberName". Handle that.
    prefix = self.enum_class.__name__ + "."
    if isinstance(value, str) and value.startswith(prefix):
      try:
        return self.enum_class[value[len(prefix):]]
      except KeyError:
        raise exceptions.ValidationError(
          "'%s' does not refer to a member of %s." % (value, self.enum_class),
          code='invalid',
          params={'value': value},
          )

    # We may also get string versions of the integer form from forms,
    # and integers when querying a database.
    try:
      return self.enum_class(int(value))
    except ValueError:
      raise exceptions.ValidationError(
        "'%s' must be an integer value of %s." % (value, self.enum_class),
        code='invalid',
        params={'value': value},
        )

  def deconstruct(self):
    # Override the positional arguments info to include the enumeration class.
    tup = super(EnumField, self).deconstruct()
    return (tup[0], tup[1], [self.enum_class], tup[3])

  @cached_property
  def validators(self):
    # IntegerField validators will not work on enum instances, and we don't need
    # any validation beyond conversion to an enum instance (which is performed
    # elsewhere), so we don't need to do any validation.
    return []

<|endoftext|>


<commit_before>#  Copyright 2008-2011 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from robot.result.visitor import ResultVisitor
from robot import utils

class XUnitWriter(ResultVisitor):
    """Provides an xUnit-compatible result file.

    Attempts to adhere to the de facto schema guessed by Peter Reilly, see:
    http://marc.info/?l=ant-dev&m=123551933508682
    """

    def __init__(self, output):
        self._writer = utils.XmlWriter(output)
        self._root_suite = None
        self._detail_serializer = _NopSerializer()

    def start_suite(self, suite):
        if self._root_suite:
            return
        self._root_suite = suite
        attrs = {'name': suite.name,
                 'tests': suite.statistics.all.total,
                 'errors': 0,
                 'failures': suite.statistics.all.failed,
                 'skip': 0}
        self._writer.start('testsuite', attrs)

    def end_suite(self, suite):
        if suite is self._root_suite:
            self._writer.end('testsuite')

    def start_test(self, test):
        attrs = {'classname': test.parent.longname,
                 'name': test.name,
                 'time': self._time_as_seconds(test.elapsedtime)}
        self._writer.start('testcase', attrs)
        if test.status == 'FAIL':
            self._detail_serializer = _FailedTestSerializer(self._writer, test)

    def _time_as_seconds(self, millis):
        return int(round(millis, -3) / 1000)

    def end_test(self, test):
        self._detail_serializer.end_test()
        self._detail_serializer = _NopSerializer()
        self._writer.end('testcase')

    def start_keyword(self, kw):
        pass

    def end_keyword(self, kw):
        pass

    def visit_message(self, msg):
        self._detail_serializer.message(msg)

    def end_result(self, result):
        self._writer.close()

class _FailedTestSerializer:
    """Specific policy to serialize a failed test case details"""

    def __init__(self, writer, test):
        self._writer = writer
        self._writer.start('failure',
                           {'message': test.message, 'type': 'AssertionError'})

    def end_test(self):
        self._writer.end('failure')

    def message(self, msg):
        """Populates the <failure> section, normally only with a 'Stacktrace'.

        There is a weakness here because filtering is based on message level:
        - DEBUG level is used by RF for 'Tracebacks' (what is expected here)
        - INFO and TRACE are used for keywords and arguments (not errors)
        - first FAIL message is already reported as <failure> attribute
        """
        if msg.level == 'DEBUG':
            self._writer.content(msg.message)

class _NopSerializer:
    """Default policy when there's no detail to serialize"""

    def end_test(self):
        pass

    def message(self, msg):
        pass
<commit_msg>XUnitWriter cleanup<commit_after>#  Copyright 2008-2011 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import os

from robot.result.visitor import ResultVisitor
from robot import utils

class XUnitWriter(ResultVisitor):
    """Provides an xUnit-compatible result file.

    Attempts to adhere to the de facto schema guessed by Peter Reilly, see:
    http://marc.info/?l=ant-dev&m=123551933508682
    """

    def __init__(self, output):
        self._writer = utils.XmlWriter(output)
        self._root_suite = None

    def start_suite(self, suite):
        if self._root_suite:
            return
        self._root_suite = suite
        attrs = {'name': suite.name,
                 'tests': suite.statistics.all.total,
                 'errors': 0,
                 'failures': suite.statistics.all.failed,
                 'skip': 0}
        self._writer.start('testsuite', attrs)

    def end_suite(self, suite):
        if suite is self._root_suite:
            self._writer.end('testsuite')

    def start_test(self, test):
        attrs = {'classname': test.parent.longname,
                 'name': test.name,
                 'time': self._time_as_seconds(test.elapsedtime)}
        self._writer.start('testcase', attrs)
        if test.status == 'FAIL':
            test.visit(TestFailureWriter(self._writer))

    def _time_as_seconds(self, millis):
        return int(round(millis, -3) / 1000)

    def end_test(self, test):
        self._writer.end('testcase')

    def visit_keyword(self, kw):
        pass

    def end_result(self, result):
        self._writer.close()

class TestFailureWriter(ResultVisitor):

    def __init__(self, writer):
        self._writer = writer

    def start_test(self, test):
        self._writer.start('failure', {'message': test.message,
                                       'type': 'AssertionError'})

    def end_test(self, test):
        self._writer.end('failure')

    def visit_message(self, msg):
        """Populates the <failure> section, normally only with a 'Stacktrace'.

        There is a weakness here because filtering is based on message level:
        - DEBUG level is used by RF for 'Tracebacks' (what is expected here)
        - INFO and TRACE are used for keywords and arguments (not errors)
        - first FAIL message is already reported as <failure> attribute
        """
        if msg.level == 'DEBUG':
            self._writer.content(msg.message + os.linesep)
<|endoftext|>


<commit_before><commit_msg>[ci] small fix to test_integration_wandb.py (#29895)<commit_after><|endoftext|>


<commit_before># Copyright (c) 2012 Ian C. Good
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#

"""Package defining common functionality for SMTP queuing. According to the
SMTP RFCs, a server must persistently store a message before telling the client
it accepts the message. Rather than being a traditional queue data structure,
SMTP queues will retry messages after certain periods of time.

"""

from __future__ import absolute_import

import time
import bisect
import collections
from itertools import imap, izip, repeat, chain

import gevent
from gevent import Greenlet
from gevent.event import Event
from gevent.lock import Semaphore
from gevent.pool import Pool

from slimta.logging import log_exception
from slimta.core import SlimtaError
from slimta.relay import PermanentRelayError, TransientRelayError
from slimta.smtp.reply import Reply
from slimta.bounce import Bounce
from slimta.policy import QueuePolicy

__all__ = ['QueueError', 'Queue', 'QueueStorage']

class QueueError(SlimtaError):
    """Base exception for errors in the queue package."""
    pass

class QueueStorage(object):
    """Base class to show the interface that classes must implement to be a
    storage mechanism for :class:`Queue`.

    """
    def __init__(self):
        pass

    def _remove_delivered_rcpts(self, envelope, rcpt_indexes):
        for index in sorted(rcpt_indexes, reverse=True):
            del envelope.recipients[index]

    def write(self, envelope, timestamp):
        """Writes the given envelope to storage, along with the timestamp of
        its next delivery attempt. The number of delivery attempts asociated
        with the message should start at zero.

        :param envelope: |Envelope| object to write.
        :param timestamp: Timestamp of message's next delivery attempt.
        :returns: Unique identifier string for the message in queue.
        :raises: :class:`QueueError`

        """
        raise NotImplementedError()

    def set_timestamp(self, id, timestamp):
        """Sets a new timestamp for the message's next delivery attempt.

        :param id: The unique identifier string for the message.
        :param timestamp: The new delivery attempt timestamp.
        :raises: :class:`QueueError`

        """
        raise NotImplementedError()

    def increment_attempts(self, id):
        """Increments the number of delivery attempts associated with the
        message.

        :param id: The unique identifier string for the message.
        :returns: The new number of message delivery attempts.
        :raises: :class:`QueueError`

        """
        raise NotImplementedError()

    def set_recipients_delivered(self, id, rcpt_indexes):
        """.. versionadded:: 1.1.0

        Marks the given recipients from the original envelope as
        already-delivered, meaning they will be skipped by future relay
        attempts.

        :param id: The unique identifier string for the message.
        :param rcpt_indexes: List of indexes in the original envelope's
                             :attr:`~slimta.envelope.Envelope.recipients` list
                             to mark as delivered.
        :raises: :class:`QueueError`

        """
        raise NotImplementedError()

    def load(self):
        """Loads all queued messages from the storage engine, such that the
        :class:`Queue` can be aware of all upcoming delivery attempts.

        :returns: Iterable that yields tuples of the form ``(timestamp, id)``
                  for each message in storage.
        :raises: :class:`QueueError`

        """
        raise NotImplementedError()

    def get(self, id):
        """Returns the |Envelope| object associated with the given unique
        identifier sring.

        The envelope's :attr:`~slimta.envelope.Envelope.recipients` should not
        include those marked as delivered by :meth:`.set_recipients_delivered`.

        :param id: The unique identifier string of the requested |Envelope|.
        :returns: Tuple with the |Envelope| object and number of attempts.
        :raises: :class:`KeyError`, :class:`QueueError`

        """
        raise NotImplementedError()

    def remove(self, id):
        """Removes the |Envelope| associated with the given unique identifier
        string from queue storage. This is typically used when the message has
        been successfully delivered or delivery has permanently failed.

        :param id: The unique identifier string of the |Envelope| to remove.
        :raises: :class:`QueueError`

        """
        raise NotImplementedError()

    def wait(self):
        """If messages are not being delivered from the same process in which
        they were received, the storage mechanism needs a way to wait until it
        is notified that a new message has been stored.

        :returns: An iterable or generator producing tuples with the timestamp
                  and unique identifier string of a new message in storage.
                  When the iterable or generator is exhausted, :meth:`.wait` is
                  simply called again.

        """
        raise NotImplementedError()

    def get_info(self):
        """.. versionadded:: 0.3.20

        Queries the storage backend for relevant information about the
        contents of the queue. The result is a :func:`dict` containing required
        keys along with any other custom keys dependent on the particular
        backend.

        Only one key is required in the result:

        * ``size``: The number of messages currently in the queue.

        :rtype: :func:`dict`

        """
        raise NotImplementedError()

class Queue(Greenlet):
    """Manages the queue of |Envelope| objects waiting for delivery. This is
    not a standard FIFO queue, a message's place in the queue depends entirely
    on the timestamp of its next delivery attempt.

    :param store: Object implementing :class:`QueueStorage`.
    :param relay: |Relay| object used to attempt message deliveries. If this
                  is not given, no deliveries will be attempted on received
                  messages.
    :param backoff: Function that, given an |Envelope| and number of delivery
                    attempts, will return the number of seconds before the next
                    attempt. If it returns ``None``, the message will be
                    permanently failed. The default backoff function simply
                    returns ``None`` and messages are never retried.
    :param bounce_factory: Function that produces a |Bounce| object given the
                           same parameters as the |Bounce| constructor. If the
                           function returns ``None``, no bounce is delivered.
                           By default, a new |Bounce| is created in every case.
    :param store_pool: Number of simultaneous operations performable against
                       the ``store`` object. Default is unlimited.
    :param relay_pool: Number of simultaneous operations performable against
                       the ``relay`` object. Default is unlimited.

    """

    def __init__(self, store, relay=None, backoff=None, bounce_factory=None,
                 store_pool=None, relay_pool=None):
        super(Queue, self).__init__()
        self.store = store
        self.relay = relay
        self.backoff = backoff or self._default_backoff
        self.bounce_factory = bounce_factory or Bounce
        self.wake = Event()
        self.queued = []
        self.active_ids = set()
        self.queued_ids = set()
        self.queued_lock = Semaphore(1)
        self.queue_policies = []
        self._use_pool('store_pool', store_pool)
        self._use_pool('relay_pool', relay_pool)

    def add_policy(self, policy):
        """Adds a |QueuePolicy| to be executed before messages are persisted
        to storage.

        :param policy: |QueuePolicy| object to execute.

        """
        if isinstance(policy, QueuePolicy):
            self.queue_policies.append(policy)
        else:
            raise TypeError('Argument not a QueuePolicy.')

    @staticmethod
    def _default_backoff(envelope, attempts):
        pass

    def _run_policies(self, envelope):
        results = [envelope]

        def recurse(current, i):
            try:
                policy = self.queue_policies[i]
            except IndexError:
                return
            ret = policy.apply(current)
            if ret:
                results.remove(current)
                results.extend(ret)
                for env in ret:
                    recurse(env, i+1)
            else:
                recurse(current, i+1)
        recurse(envelope, 0)
        return results

    def _use_pool(self, attr, pool):
        if pool is None:
            pass
        elif isinstance(pool, Pool):
            setattr(self, attr, pool)
        else:
            setattr(self, attr, Pool(pool))

    def _pool_run(self, which, func, *args, **kwargs):
        pool = getattr(self, which+'_pool', None)
        if pool:
            ret = pool.spawn(func, *args, **kwargs)
            return ret.get()
        else:
            return func(*args, **kwargs)

    def _pool_imap(self, which, func, *iterables):
        pool = getattr(self, which+'_pool', gevent)
        threads = imap(pool.spawn, repeat(func), *iterables)
        ret = []
        for thread in threads:
            thread.join()
            ret.append(thread.exception or thread.value)
        return ret

    def _pool_spawn(self, which, func, *args, **kwargs):
        pool = getattr(self, which+'_pool', gevent)
        return pool.spawn(func, *args, **kwargs)

    def _add_queued(self, entry):
        timestamp, id = entry
        if id not in self.queued_ids | self.active_ids:
            bisect.insort(self.queued, entry)
            self.queued_ids.add(id)
            self.wake.set()

    def enqueue(self, envelope):
        """Drops a new message in the queue for delivery. The first delivery
        attempt is made immediately (depending on relay pool availability).
        This method is not typically called directly, |Edge| objects use it
        when they receive new messages.

        :param envelope: |Envelope| object to enqueue.
        :returns: Zipped list of envelopes and their respective queue IDs (or
                  thrown :exc:`QueueError` objects).

        """
        now = time.time()
        envelopes = self._run_policies(envelope)
        ids = self._pool_imap('store', self.store.write, envelopes,
                              repeat(now))
        results = zip(envelopes, ids)
        for env, id in results:
            if not isinstance(id, BaseException):
                if self.relay:
                    self.active_ids.add(id)
                    self._pool_spawn('relay', self._attempt, id, env, 0)
            elif not isinstance(id, QueueError):
                raise id  # Re-raise exceptions that are not QueueError.
        return results

    def _load_all(self):
        for entry in self.store.load():
            self._add_queued(entry)

    def _remove(self, id):
        self._pool_spawn('store', self.store.remove, id)
        self.queued_ids.discard(id)
        self.active_ids.discard(id)

    def _bounce(self, envelope, reply):
        bounce = self.bounce_factory(envelope, reply)
        if bounce:
            return self.enqueue(bounce)

    def _perm_fail(self, id, envelope, reply):
        if id is not None:
            self._remove(id)
        if envelope.sender:  # Can't bounce to null-sender.
            self._pool_spawn('bounce', self._bounce, envelope, reply)

    def _split_by_reply(self, envelope, replies):
        if isinstance(replies, Reply):
            return [(replies, envelope)]
        groups = []
        for i, rcpt in enumerate(envelope.recipients):
            for reply, group_env in groups:
                if replies[i] == reply:
                    group_env.recipients.append(rcpt)
                    break
            else:
                group_env = envelope.copy([rcpt])
                groups.append((replies[i], group_env))
        return groups

    def _retry_later(self, id, envelope, replies):
        attempts = self.store.increment_attempts(id)
        wait = self.backoff(envelope, attempts)
        if wait is None:
            for reply, group_env in self._split_by_reply(envelope, replies):
                reply.message += ' (Too many retries)'
                self._perm_fail(None, group_env, reply)
            self._remove(id)
            return False
        else:
            when = time.time() + wait
            self.store.set_timestamp(id, when)
            self.active_ids.discard(id)
            self._add_queued((when, id))
            return True

    def _attempt(self, id, envelope, attempts):
        try:
            results = self.relay._attempt(envelope, attempts)
        except TransientRelayError as e:
            self._pool_spawn('store', self._retry_later, id, envelope, e.reply)
        except PermanentRelayError as e:
            self._perm_fail(id, envelope, e.reply)
        except Exception as e:
            log_exception(__name__)
            reply = Reply('450', '4.0.0 Unhandled delivery error: '+str(e))
            self._pool_spawn('store', self._retry_later, id, envelope, reply)
            raise
        else:
            if isinstance(results, collections.Sequence):
                self._handle_partial_relay(id, envelope, attempts, results)
            else:
                self._remove(id)

    def _handle_partial_relay(self, id, envelope, attempts, results):
        delivered = []
        tempfails = []
        permfails = []
        for i, rcpt in enumerate(envelope.recipients):
            rcpt_res = results[i]
            if not rcpt_res:
                delivered.append(i)
            elif isinstance(rcpt_res, PermanentRelayError):
                delivered.append(i)
                permfails.append((rcpt, rcpt_res.reply))
            elif isinstance(rcpt_res, TransientRelayError):
                tempfails.append((rcpt, rcpt_res.reply))
        if permfails:
            rcpts, replies = zip(*permfails)
            fail_env = envelope.copy(rcpts)
            for reply, group_env in self._split_by_reply(fail_env, replies):
                self._perm_fail(None, group_env, reply)
        if tempfails:
            rcpts, replies = zip(*tempfails)
            fail_env = envelope.copy(rcpts)
            if not self._retry_later(id, fail_env, replies):
                return
        else:
            self.store.remove(id)
            return
        self.store.set_recipients_delivered(id, delivered)

    def _dequeue(self, id):
        try:
            envelope, attempts = self.store.get(id)
        except KeyError:
            return
        self.active_ids.add(id)
        self._pool_spawn('relay', self._attempt, id, envelope, attempts)

    def _check_ready(self, now):
        last_i = 0
        for i, entry in enumerate(self.queued):
            timestamp, id = entry
            if now >= timestamp:
                self._pool_spawn('store', self._dequeue, id)
                last_i = i+1
            else:
                break
        if last_i > 0:
            self.queued = self.queued[last_i:]
            self.queued_ids = set([id for timestamp, id in self.queued])

    def _wait_store(self):
        while True:
            try:
                for entry in self.store.wait():
                    self._add_queued(entry)
            except NotImplementedError:
                return

    def _wait_ready(self, now):
        try:
            first = self.queued[0]
        except IndexError:
            self.wake.wait()
            self.wake.clear()
            return
        first_timestamp = first[0]
        if first_timestamp > now:
            self.wake.wait(first_timestamp-now)
            self.wake.clear()

    def flush(self):
        """Attempts to immediately flush all messages waiting in the queue,
        regardless of their retry timers.

        .. warning::

           This can be a very expensive operation, use with care.

        """
        self.wake.set()
        self.wake.clear()
        self.queued_lock.acquire()
        try:
            for entry in self.queued:
                self._pool_spawn('store', self._dequeue, entry[1])
            self.queued = []
        finally:
            self.queued_lock.release()

    def kill(self):
        """This method is used by |Queue| and |Queue|-like objects to properly
        end any associated services (such as running :class:`~gevent.Greenlet`
        threads) and close resources.

        """
        super(Queue, self).kill()

    def _run(self):
        if not self.relay:
            return
        self._pool_spawn('store', self._load_all)
        self._pool_spawn('store', self._wait_store)
        while True:
            self.queued_lock.acquire()
            try:
                now = time.time()
                self._check_ready(now)
                self._wait_ready(now)
            finally:
                self.queued_lock.release()

# vim:et:fdm=marker:sts=4:sw=4:ts=4
<commit_msg>allow bounces to be queued separately<commit_after># Copyright (c) 2012 Ian C. Good
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#

"""Package defining common functionality for SMTP queuing. According to the
SMTP RFCs, a server must persistently store a message before telling the client
it accepts the message. Rather than being a traditional queue data structure,
SMTP queues will retry messages after certain periods of time.

"""

from __future__ import absolute_import

import time
import bisect
import collections
from itertools import imap, izip, repeat, chain

import gevent
from gevent import Greenlet
from gevent.event import Event
from gevent.lock import Semaphore
from gevent.pool import Pool

from slimta.logging import log_exception
from slimta.core import SlimtaError
from slimta.relay import PermanentRelayError, TransientRelayError
from slimta.smtp.reply import Reply
from slimta.bounce import Bounce
from slimta.policy import QueuePolicy

__all__ = ['QueueError', 'Queue', 'QueueStorage']

class QueueError(SlimtaError):
    """Base exception for errors in the queue package."""
    pass

class QueueStorage(object):
    """Base class to show the interface that classes must implement to be a
    storage mechanism for :class:`Queue`.

    """
    def __init__(self):
        pass

    def _remove_delivered_rcpts(self, envelope, rcpt_indexes):
        for index in sorted(rcpt_indexes, reverse=True):
            del envelope.recipients[index]

    def write(self, envelope, timestamp):
        """Writes the given envelope to storage, along with the timestamp of
        its next delivery attempt. The number of delivery attempts asociated
        with the message should start at zero.

        :param envelope: |Envelope| object to write.
        :param timestamp: Timestamp of message's next delivery attempt.
        :returns: Unique identifier string for the message in queue.
        :raises: :class:`QueueError`

        """
        raise NotImplementedError()

    def set_timestamp(self, id, timestamp):
        """Sets a new timestamp for the message's next delivery attempt.

        :param id: The unique identifier string for the message.
        :param timestamp: The new delivery attempt timestamp.
        :raises: :class:`QueueError`

        """
        raise NotImplementedError()

    def increment_attempts(self, id):
        """Increments the number of delivery attempts associated with the
        message.

        :param id: The unique identifier string for the message.
        :returns: The new number of message delivery attempts.
        :raises: :class:`QueueError`

        """
        raise NotImplementedError()

    def set_recipients_delivered(self, id, rcpt_indexes):
        """.. versionadded:: 1.1.0

        Marks the given recipients from the original envelope as
        already-delivered, meaning they will be skipped by future relay
        attempts.

        :param id: The unique identifier string for the message.
        :param rcpt_indexes: List of indexes in the original envelope's
                             :attr:`~slimta.envelope.Envelope.recipients` list
                             to mark as delivered.
        :raises: :class:`QueueError`

        """
        raise NotImplementedError()

    def load(self):
        """Loads all queued messages from the storage engine, such that the
        :class:`Queue` can be aware of all upcoming delivery attempts.

        :returns: Iterable that yields tuples of the form ``(timestamp, id)``
                  for each message in storage.
        :raises: :class:`QueueError`

        """
        raise NotImplementedError()

    def get(self, id):
        """Returns the |Envelope| object associated with the given unique
        identifier sring.

        The envelope's :attr:`~slimta.envelope.Envelope.recipients` should not
        include those marked as delivered by :meth:`.set_recipients_delivered`.

        :param id: The unique identifier string of the requested |Envelope|.
        :returns: Tuple with the |Envelope| object and number of attempts.
        :raises: :class:`KeyError`, :class:`QueueError`

        """
        raise NotImplementedError()

    def remove(self, id):
        """Removes the |Envelope| associated with the given unique identifier
        string from queue storage. This is typically used when the message has
        been successfully delivered or delivery has permanently failed.

        :param id: The unique identifier string of the |Envelope| to remove.
        :raises: :class:`QueueError`

        """
        raise NotImplementedError()

    def wait(self):
        """If messages are not being delivered from the same process in which
        they were received, the storage mechanism needs a way to wait until it
        is notified that a new message has been stored.

        :returns: An iterable or generator producing tuples with the timestamp
                  and unique identifier string of a new message in storage.
                  When the iterable or generator is exhausted, :meth:`.wait` is
                  simply called again.

        """
        raise NotImplementedError()

    def get_info(self):
        """.. versionadded:: 0.3.20

        Queries the storage backend for relevant information about the
        contents of the queue. The result is a :func:`dict` containing required
        keys along with any other custom keys dependent on the particular
        backend.

        Only one key is required in the result:

        * ``size``: The number of messages currently in the queue.

        :rtype: :func:`dict`

        """
        raise NotImplementedError()

class Queue(Greenlet):
    """Manages the queue of |Envelope| objects waiting for delivery. This is
    not a standard FIFO queue, a message's place in the queue depends entirely
    on the timestamp of its next delivery attempt.

    :param store: Object implementing :class:`QueueStorage`.
    :param relay: |Relay| object used to attempt message deliveries. If this
                  is not given, no deliveries will be attempted on received
                  messages.
    :param backoff: Function that, given an |Envelope| and number of delivery
                    attempts, will return the number of seconds before the next
                    attempt. If it returns ``None``, the message will be
                    permanently failed. The default backoff function simply
                    returns ``None`` and messages are never retried.
    :param bounce_factory: Function that produces a |Bounce| object given the
                           same parameters as the |Bounce| constructor. If the
                           function returns ``None``, no bounce is delivered.
                           By default, a new |Bounce| is created in every case.
    :param bounce_queue: |Queue| object that will be used for delivering bounce
                         messages. The default is ``self``.
    :param store_pool: Number of simultaneous operations performable against
                       the ``store`` object. Default is unlimited.
    :param relay_pool: Number of simultaneous operations performable against
                       the ``relay`` object. Default is unlimited.

    """

    def __init__(self, store, relay=None, backoff=None, bounce_factory=None,
                 bounce_queue=None, store_pool=None, relay_pool=None):
        super(Queue, self).__init__()
        self.store = store
        self.relay = relay
        self.backoff = backoff or self._default_backoff
        self.bounce_factory = bounce_factory or Bounce
        self.bounce_queue = bounce_queue or self
        self.wake = Event()
        self.queued = []
        self.active_ids = set()
        self.queued_ids = set()
        self.queued_lock = Semaphore(1)
        self.queue_policies = []
        self._use_pool('store_pool', store_pool)
        self._use_pool('relay_pool', relay_pool)

    def add_policy(self, policy):
        """Adds a |QueuePolicy| to be executed before messages are persisted
        to storage.

        :param policy: |QueuePolicy| object to execute.

        """
        if isinstance(policy, QueuePolicy):
            self.queue_policies.append(policy)
        else:
            raise TypeError('Argument not a QueuePolicy.')

    @staticmethod
    def _default_backoff(envelope, attempts):
        pass

    def _run_policies(self, envelope):
        results = [envelope]

        def recurse(current, i):
            try:
                policy = self.queue_policies[i]
            except IndexError:
                return
            ret = policy.apply(current)
            if ret:
                results.remove(current)
                results.extend(ret)
                for env in ret:
                    recurse(env, i+1)
            else:
                recurse(current, i+1)
        recurse(envelope, 0)
        return results

    def _use_pool(self, attr, pool):
        if pool is None:
            pass
        elif isinstance(pool, Pool):
            setattr(self, attr, pool)
        else:
            setattr(self, attr, Pool(pool))

    def _pool_run(self, which, func, *args, **kwargs):
        pool = getattr(self, which+'_pool', None)
        if pool:
            ret = pool.spawn(func, *args, **kwargs)
            return ret.get()
        else:
            return func(*args, **kwargs)

    def _pool_imap(self, which, func, *iterables):
        pool = getattr(self, which+'_pool', gevent)
        threads = imap(pool.spawn, repeat(func), *iterables)
        ret = []
        for thread in threads:
            thread.join()
            ret.append(thread.exception or thread.value)
        return ret

    def _pool_spawn(self, which, func, *args, **kwargs):
        pool = getattr(self, which+'_pool', gevent)
        return pool.spawn(func, *args, **kwargs)

    def _add_queued(self, entry):
        timestamp, id = entry
        if id not in self.queued_ids | self.active_ids:
            bisect.insort(self.queued, entry)
            self.queued_ids.add(id)
            self.wake.set()

    def enqueue(self, envelope):
        """Drops a new message in the queue for delivery. The first delivery
        attempt is made immediately (depending on relay pool availability).
        This method is not typically called directly, |Edge| objects use it
        when they receive new messages.

        :param envelope: |Envelope| object to enqueue.
        :returns: Zipped list of envelopes and their respective queue IDs (or
                  thrown :exc:`QueueError` objects).

        """
        now = time.time()
        envelopes = self._run_policies(envelope)
        ids = self._pool_imap('store', self.store.write, envelopes,
                              repeat(now))
        results = zip(envelopes, ids)
        for env, id in results:
            if not isinstance(id, BaseException):
                if self.relay:
                    self.active_ids.add(id)
                    self._pool_spawn('relay', self._attempt, id, env, 0)
            elif not isinstance(id, QueueError):
                raise id  # Re-raise exceptions that are not QueueError.
        return results

    def _load_all(self):
        for entry in self.store.load():
            self._add_queued(entry)

    def _remove(self, id):
        self._pool_spawn('store', self.store.remove, id)
        self.queued_ids.discard(id)
        self.active_ids.discard(id)

    def _bounce(self, envelope, reply):
        bounce = self.bounce_factory(envelope, reply)
        if bounce:
            return self.bounce_queue.enqueue(bounce)

    def _perm_fail(self, id, envelope, reply):
        if id is not None:
            self._remove(id)
        if envelope.sender:  # Can't bounce to null-sender.
            self._pool_spawn('bounce', self._bounce, envelope, reply)

    def _split_by_reply(self, envelope, replies):
        if isinstance(replies, Reply):
            return [(replies, envelope)]
        groups = []
        for i, rcpt in enumerate(envelope.recipients):
            for reply, group_env in groups:
                if replies[i] == reply:
                    group_env.recipients.append(rcpt)
                    break
            else:
                group_env = envelope.copy([rcpt])
                groups.append((replies[i], group_env))
        return groups

    def _retry_later(self, id, envelope, replies):
        attempts = self.store.increment_attempts(id)
        wait = self.backoff(envelope, attempts)
        if wait is None:
            for reply, group_env in self._split_by_reply(envelope, replies):
                reply.message += ' (Too many retries)'
                self._perm_fail(None, group_env, reply)
            self._remove(id)
            return False
        else:
            when = time.time() + wait
            self.store.set_timestamp(id, when)
            self.active_ids.discard(id)
            self._add_queued((when, id))
            return True

    def _attempt(self, id, envelope, attempts):
        try:
            results = self.relay._attempt(envelope, attempts)
        except TransientRelayError as e:
            self._pool_spawn('store', self._retry_later, id, envelope, e.reply)
        except PermanentRelayError as e:
            self._perm_fail(id, envelope, e.reply)
        except Exception as e:
            log_exception(__name__)
            reply = Reply('450', '4.0.0 Unhandled delivery error: '+str(e))
            self._pool_spawn('store', self._retry_later, id, envelope, reply)
            raise
        else:
            if isinstance(results, collections.Sequence):
                self._handle_partial_relay(id, envelope, attempts, results)
            else:
                self._remove(id)

    def _handle_partial_relay(self, id, envelope, attempts, results):
        delivered = []
        tempfails = []
        permfails = []
        for i, rcpt in enumerate(envelope.recipients):
            rcpt_res = results[i]
            if not rcpt_res:
                delivered.append(i)
            elif isinstance(rcpt_res, PermanentRelayError):
                delivered.append(i)
                permfails.append((rcpt, rcpt_res.reply))
            elif isinstance(rcpt_res, TransientRelayError):
                tempfails.append((rcpt, rcpt_res.reply))
        if permfails:
            rcpts, replies = zip(*permfails)
            fail_env = envelope.copy(rcpts)
            for reply, group_env in self._split_by_reply(fail_env, replies):
                self._perm_fail(None, group_env, reply)
        if tempfails:
            rcpts, replies = zip(*tempfails)
            fail_env = envelope.copy(rcpts)
            if not self._retry_later(id, fail_env, replies):
                return
        else:
            self.store.remove(id)
            return
        self.store.set_recipients_delivered(id, delivered)

    def _dequeue(self, id):
        try:
            envelope, attempts = self.store.get(id)
        except KeyError:
            return
        self.active_ids.add(id)
        self._pool_spawn('relay', self._attempt, id, envelope, attempts)

    def _check_ready(self, now):
        last_i = 0
        for i, entry in enumerate(self.queued):
            timestamp, id = entry
            if now >= timestamp:
                self._pool_spawn('store', self._dequeue, id)
                last_i = i+1
            else:
                break
        if last_i > 0:
            self.queued = self.queued[last_i:]
            self.queued_ids = set([id for timestamp, id in self.queued])

    def _wait_store(self):
        while True:
            try:
                for entry in self.store.wait():
                    self._add_queued(entry)
            except NotImplementedError:
                return

    def _wait_ready(self, now):
        try:
            first = self.queued[0]
        except IndexError:
            self.wake.wait()
            self.wake.clear()
            return
        first_timestamp = first[0]
        if first_timestamp > now:
            self.wake.wait(first_timestamp-now)
            self.wake.clear()

    def flush(self):
        """Attempts to immediately flush all messages waiting in the queue,
        regardless of their retry timers.

        .. warning::

           This can be a very expensive operation, use with care.

        """
        self.wake.set()
        self.wake.clear()
        self.queued_lock.acquire()
        try:
            for entry in self.queued:
                self._pool_spawn('store', self._dequeue, entry[1])
            self.queued = []
        finally:
            self.queued_lock.release()

    def kill(self):
        """This method is used by |Queue| and |Queue|-like objects to properly
        end any associated services (such as running :class:`~gevent.Greenlet`
        threads) and close resources.

        """
        super(Queue, self).kill()

    def _run(self):
        if not self.relay:
            return
        self._pool_spawn('store', self._load_all)
        self._pool_spawn('store', self._wait_store)
        while True:
            self.queued_lock.acquire()
            try:
                now = time.time()
                self._check_ready(now)
                self._wait_ready(now)
            finally:
                self.queued_lock.release()

# vim:et:fdm=marker:sts=4:sw=4:ts=4
<|endoftext|>


<commit_before>from json import loads
from urllib import urlencode
from searx.utils import html_to_text

url = 'https://duckduckgo.com/'
search_url = url + 'd.js?{query}&p=1&s={offset}'
locale = 'us-en'

paging = True

def request(query, params):
    offset = (params['pageno'] - 1) * 30
    q = urlencode({'q': query,
                   'l': locale})
    params['url'] = search_url.format(query=q, offset=offset)
    return params

def response(resp):
    results = []
    search_res = loads(resp.text[resp.text.find('[{'):-2])[:-1]
    for r in search_res:
        if not r.get('t'):
            continue
        results.append({'title': r['t'],
                       'content': html_to_text(r['a']),
                       'url': r['u']})
    return results
<commit_msg>[mod] ddg engine mods<commit_after>from urllib import urlencode
from lxml.html import fromstring
from searx.utils import html_to_text

url = 'https://duckduckgo.com/html?{query}&s={offset}'
locale = 'us-en'

def request(query, params):
    offset = (params['pageno'] - 1) * 30
    q = urlencode({'q': query,
                   'l': locale})
    params['url'] = url.format(query=q, offset=offset)
    return params

def response(resp):
    result_xpath = '//div[@class="results_links results_links_deep web-result"]'
    url_xpath = './/a[@class="large"]/@href'
    title_xpath = './/a[@class="large"]//text()'
    content_xpath = './/div[@class="snippet"]//text()'
    results = []

    doc = fromstring(resp.text)

    for r in doc.xpath(result_xpath):
        res_url = r.xpath(url_xpath)[-1]
        if not res_url:
            continue
        title = html_to_text(''.join(r.xpath(title_xpath)))
        content = html_to_text(''.join(r.xpath(content_xpath)))
        results.append({'title': title,
                        'content': content,
                        'url': res_url})

    return results

#from json import loads
#search_url = url + 'd.js?{query}&p=1&s={offset}'
#
#paging = True
#
#
#def request(query, params):
#    offset = (params['pageno'] - 1) * 30
#    q = urlencode({'q': query,
#                   'l': locale})
#    params['url'] = search_url.format(query=q, offset=offset)
#    return params
#
#
#def response(resp):
#    results = []
#    search_res = loads(resp.text[resp.text.find('[{'):-2])[:-1]
#    for r in search_res:
#        if not r.get('t'):
#            continue
#        results.append({'title': r['t'],
#                       'content': html_to_text(r['a']),
#                       'url': r['u']})
#    return results
<|endoftext|>