<commit_before>import datetime
import re
from django.conf import settings
from django.http import Http404, HttpResponse, JsonResponse
from django.db.models import OuterRef, Subquery
from django.shortcuts import get_object_or_404, render, redirect
from django.utils.cache import add_never_cache_headers, patch_cache_control
from django.views.decorators.cache import cache_control, never_cache
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_POST
from django.views.generic import ListView, TemplateView
from .models import Paper, Render, PaperIsNotRenderableError
from ..scraper.arxiv_ids import remove_version_from_arxiv_id
from ..scraper.query import PaperNotFoundError

def add_paper_cache_control(response):
    patch_cache_control(response, public=True, max_age=settings.PAPER_CACHE_SECONDS)
    return response

class HomeView(TemplateView):
    template_name = "papers/home.html"

    def dispatch(self, *args, **kwargs):
        res = super(HomeView, self).dispatch(*args, **kwargs)
        return add_paper_cache_control(res)

class PaperListView(ListView):
    model = Paper
    paginate_by = 25

    def get_queryset(self):
        qs = super(PaperListView, self).get_queryset()
        return qs.machine_learning().has_successful_render()

    def dispatch(self, *args, **kwargs):
        res = super(PaperListView, self).dispatch(*args, **kwargs)
        return add_paper_cache_control(res)

def paper_detail(request, arxiv_id):
    arxiv_id, version = remove_version_from_arxiv_id(arxiv_id)
    if version is not None:
        return redirect("paper_detail", arxiv_id=arxiv_id)

    # Get the requested paper
    try:
        paper = Paper.objects.get(arxiv_id=arxiv_id)
    # If it doesn't exist, render it!
    except Paper.DoesNotExist:
        # update_or_create to avoid the race condition where several people
        # hit a new paper at the same time
        try:
            paper, created = Paper.objects.update_or_create_from_arxiv_id(arxiv_id)
        except PaperNotFoundError:
            raise Http404(f"Paper '{arxiv_id}' not found on arXiv")
        if created:
            try:
                paper.render()
            except PaperIsNotRenderableError:
                res = render(request, "papers/paper_detail_not_renderable.html",
                             {"paper": paper}, status=404)
                return add_paper_cache_control(res)

    # First, try to get the latest succeeded paper -- this is always what
    # we'll want to render.
    try:
        r = paper.renders.succeeded().not_expired().latest()
    except Render.DoesNotExist:
        # See if there is a render running
        try:
            r = paper.renders.not_expired().latest()
        except Render.DoesNotExist:
            try:
                # Either rendering has not started or it has expired.
                r = paper.render()
            except PaperIsNotRenderableError:
                res = render(request, "papers/paper_detail_not_renderable.html",
                             {"paper": paper}, status=404)
                return add_paper_cache_control(res)

        # Stuck for some reason
        if r.state == Render.STATE_UNSTARTED:
            r = paper.render()

        if r.state in Render.STATE_RUNNING:
            res = render(request, "papers/paper_detail_rendering.html", {
                'paper': paper,
                'render': r,
            })
            add_never_cache_headers(res)
            return res

        # Fall back to error if there is no successful or running render
        res = render(request, "papers/paper_detail_error.html",
                     {"paper": paper}, status=500)
        return add_paper_cache_control(res)

    processed_render = r.get_processed_render()

    res = render(request, "papers/paper_detail.html", {
        'paper': paper,
        'render': r,
        'body': processed_render['body'],
        'scripts': processed_render['scripts'],
        'styles': processed_render['styles'],
    })
    return add_paper_cache_control(res)

@never_cache
def paper_render_state(request, arxiv_id):
    paper = get_object_or_404(Paper, arxiv_id=arxiv_id)
    try:
        r = paper.renders.latest()
    except Render.DoesNotExist:
        raise Http404()
    return JsonResponse({'state': r.state})

@csrf_exempt
@require_POST
def render_update_state(request, pk):
    r = get_object_or_404(Render, pk=pk, container_is_removed=False)
    r.update_state(exit_code=request.POST.get('exit_code'))
    return HttpResponse()

ARXIV_URL_RE = re.compile(r'arxiv.org/[^\/]+/([\w\.]+?)(\.pdf)?$', re.I)
ARXIV_ID_RE = re.compile(r'^(?:arxiv:)?(\d+\.\d+(?:v\d+)?)$', re.I)
ARXIV_VANITY_RE = re.compile(r'(?:localhost\:\d+|arxiv-vanity\.com)/[^\/]+/([\w\.]+?)\/?$', re.I)

def convert_query_to_arxiv_id(query):
    query = query.strip()
    for regex in [ARXIV_URL_RE, ARXIV_ID_RE, ARXIV_VANITY_RE]:
        match = regex.search(query)
        if match:
            return match.group(1)

@never_cache
def paper_convert(request):
    if not request.GET.get('query'):
        return render(request, "papers/paper_convert_error.html", {
            "message": "No paper was given. Please enter something into the search box."
        })
    arxiv_id = convert_query_to_arxiv_id(request.GET['query'])
    if not arxiv_id:
        return render(request, "papers/paper_convert_error.html", {
            "message": "Could not find arXiv ID in that URL. Are you sure it's an arxiv.org URL?"
        })
    return redirect("paper_detail", arxiv_id=arxiv_id)

@cache_control(public=True, max_age=30)
def stats(request):
    past_30_days = Render.objects.filter(created_at__gt=datetime.datetime.today() - datetime.timedelta(days=30))

    newest_renders = Render.objects.filter(paper=OuterRef('pk')).order_by('-created_at')
    papers = Paper.objects.annotate(last_render_state=Subquery(newest_renders.values('state')[:1])).exclude(last_render_state=None)

    return render(request, "papers/stats.html", {
        "total_renders": Render.objects.count(),
        "successful_renders": Render.objects.filter(state=Render.STATE_SUCCESS).count(),
        "failed_renders": Render.objects.filter(state=Render.STATE_FAILURE).count(),
        "total_renders_30_days": past_30_days.count(),
        "successful_renders_30_days": past_30_days.filter(state=Render.STATE_SUCCESS).count(),
        "failed_renders_30_days": past_30_days.filter(state=Render.STATE_FAILURE).count(),
        "total_papers": papers.count(),
        "successful_papers": papers.filter(last_render_state=Render.STATE_SUCCESS).count(),
        "failed_papers": papers.filter(last_render_state=Render.STATE_FAILURE).count(),
    })
<commit_msg>Execute stats queries once<commit_after>import datetime
import re
from django.conf import settings
from django.http import Http404, HttpResponse, JsonResponse
from django.db.models import OuterRef, Subquery
from django.shortcuts import get_object_or_404, render, redirect
from django.utils.cache import add_never_cache_headers, patch_cache_control
from django.views.decorators.cache import cache_control, never_cache
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_POST
from django.views.generic import ListView, TemplateView
from .models import Paper, Render, PaperIsNotRenderableError
from ..scraper.arxiv_ids import remove_version_from_arxiv_id
from ..scraper.query import PaperNotFoundError

def add_paper_cache_control(response):
    patch_cache_control(response, public=True, max_age=settings.PAPER_CACHE_SECONDS)
    return response

class HomeView(TemplateView):
    template_name = "papers/home.html"

    def dispatch(self, *args, **kwargs):
        res = super(HomeView, self).dispatch(*args, **kwargs)
        return add_paper_cache_control(res)

class PaperListView(ListView):
    model = Paper
    paginate_by = 25

    def get_queryset(self):
        qs = super(PaperListView, self).get_queryset()
        return qs.machine_learning().has_successful_render()

    def dispatch(self, *args, **kwargs):
        res = super(PaperListView, self).dispatch(*args, **kwargs)
        return add_paper_cache_control(res)

def paper_detail(request, arxiv_id):
    arxiv_id, version = remove_version_from_arxiv_id(arxiv_id)
    if version is not None:
        return redirect("paper_detail", arxiv_id=arxiv_id)

    # Get the requested paper
    try:
        paper = Paper.objects.get(arxiv_id=arxiv_id)
    # If it doesn't exist, render it!
    except Paper.DoesNotExist:
        # update_or_create to avoid the race condition where several people
        # hit a new paper at the same time
        try:
            paper, created = Paper.objects.update_or_create_from_arxiv_id(arxiv_id)
        except PaperNotFoundError:
            raise Http404(f"Paper '{arxiv_id}' not found on arXiv")
        if created:
            try:
                paper.render()
            except PaperIsNotRenderableError:
                res = render(request, "papers/paper_detail_not_renderable.html",
                             {"paper": paper}, status=404)
                return add_paper_cache_control(res)

    # First, try to get the latest succeeded paper -- this is always what
    # we'll want to render.
    try:
        r = paper.renders.succeeded().not_expired().latest()
    except Render.DoesNotExist:
        # See if there is a render running
        try:
            r = paper.renders.not_expired().latest()
        except Render.DoesNotExist:
            try:
                # Either rendering has not started or it has expired.
                r = paper.render()
            except PaperIsNotRenderableError:
                res = render(request, "papers/paper_detail_not_renderable.html",
                             {"paper": paper}, status=404)
                return add_paper_cache_control(res)

        # Stuck for some reason
        if r.state == Render.STATE_UNSTARTED:
            r = paper.render()

        if r.state in Render.STATE_RUNNING:
            res = render(request, "papers/paper_detail_rendering.html", {
                'paper': paper,
                'render': r,
            })
            add_never_cache_headers(res)
            return res

        # Fall back to error if there is no successful or running render
        res = render(request, "papers/paper_detail_error.html",
                     {"paper": paper}, status=500)
        return add_paper_cache_control(res)

    processed_render = r.get_processed_render()

    res = render(request, "papers/paper_detail.html", {
        'paper': paper,
        'render': r,
        'body': processed_render['body'],
        'scripts': processed_render['scripts'],
        'styles': processed_render['styles'],
    })
    return add_paper_cache_control(res)

@never_cache
def paper_render_state(request, arxiv_id):
    paper = get_object_or_404(Paper, arxiv_id=arxiv_id)
    try:
        r = paper.renders.latest()
    except Render.DoesNotExist:
        raise Http404()
    return JsonResponse({'state': r.state})

@csrf_exempt
@require_POST
def render_update_state(request, pk):
    r = get_object_or_404(Render, pk=pk, container_is_removed=False)
    r.update_state(exit_code=request.POST.get('exit_code'))
    return HttpResponse()

ARXIV_URL_RE = re.compile(r'arxiv.org/[^\/]+/([\w\.]+?)(\.pdf)?$', re.I)
ARXIV_ID_RE = re.compile(r'^(?:arxiv:)?(\d+\.\d+(?:v\d+)?)$', re.I)
ARXIV_VANITY_RE = re.compile(r'(?:localhost\:\d+|arxiv-vanity\.com)/[^\/]+/([\w\.]+?)\/?$', re.I)

def convert_query_to_arxiv_id(query):
    query = query.strip()
    for regex in [ARXIV_URL_RE, ARXIV_ID_RE, ARXIV_VANITY_RE]:
        match = regex.search(query)
        if match:
            return match.group(1)

@never_cache
def paper_convert(request):
    if not request.GET.get('query'):
        return render(request, "papers/paper_convert_error.html", {
            "message": "No paper was given. Please enter something into the search box."
        })
    arxiv_id = convert_query_to_arxiv_id(request.GET['query'])
    if not arxiv_id:
        return render(request, "papers/paper_convert_error.html", {
            "message": "Could not find arXiv ID in that URL. Are you sure it's an arxiv.org URL?"
        })
    return redirect("paper_detail", arxiv_id=arxiv_id)

@cache_control(public=True, max_age=30)
def stats(request):
    past_30_days = Render.objects.filter(created_at__gt=datetime.datetime.today() - datetime.timedelta(days=30))

    newest_renders = Render.objects.filter(paper=OuterRef('pk')).order_by('-created_at')
    papers = Paper.objects.annotate(last_render_state=Subquery(newest_renders.values('state')[:1])).exclude(last_render_state=None)

    return render(request, "papers/stats.html", {
        "total_renders": int(Render.objects.count()),
        "successful_renders": int(Render.objects.filter(state=Render.STATE_SUCCESS).count()),
        "failed_renders": int(Render.objects.filter(state=Render.STATE_FAILURE).count()),
        "total_renders_30_days": int(past_30_days.count()),
        "successful_renders_30_days": int(past_30_days.filter(state=Render.STATE_SUCCESS).count()),
        "failed_renders_30_days": int(past_30_days.filter(state=Render.STATE_FAILURE).count()),
        "total_papers": int(papers.count()),
        "successful_papers": int(papers.filter(last_render_state=Render.STATE_SUCCESS).count()),
        "failed_papers": int(papers.filter(last_render_state=Render.STATE_FAILURE).count()),
    })
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import print_function

from keras.datasets import mnist
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.optimizers import SGD
from keras.utils import np_utils

from elephas.spark_model import SparkModel
from elephas.utils.rdd_utils import to_simple_rdd
from elephas import optimizers as elephas_optimizers

from pyspark import SparkContext, SparkConf

# Define basic parameters
batch_size = 64
nb_classes = 10
nb_epoch = 3

# Load data
(X_train, y_train), (X_test, y_test) = mnist.load_data()

X_train = X_train.reshape(60000, 784)
X_test = X_test.reshape(10000, 784)
X_train = X_train.astype("float32")
X_test = X_test.astype("float32")
X_train /= 255
X_test /= 255
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')

# Convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)

model = Sequential()
model.add(Dense(128, input_dim=784))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(10))
model.add(Activation('softmax'))

# Compile model
sgd = SGD(lr=0.1)
model.compile(loss='categorical_crossentropy', optimizer=sgd)

# Create Spark context
conf = SparkConf().setAppName('Mnist_Spark_MLP')  # .setMaster('local[8]')
sc = SparkContext(conf=conf)

# Build RDD from numpy features and labels
rdd = to_simple_rdd(sc, X_train, Y_train)

# Initialize SparkModel from Keras model and Spark context
adagrad = elephas_optimizers.Adagrad()
spark_model = SparkModel(sc,
                         model,
                         optimizer=adagrad,
                         frequency='epoch',
                         mode='asynchronous',
                         num_workers=2)

# Train Spark model
spark_model.train(rdd,
                  nb_epoch=nb_epoch,
                  batch_size=batch_size,
                  verbose=2,
                  validation_split=0.1)

# Evaluate Spark model by evaluating the underlying model
score = spark_model.get_network().evaluate(X_test,
                                           Y_test,
                                           show_accuracy=True,
                                           verbose=2)
print('Test accuracy:', score[1])
<commit_msg>Initialize SparkContext first to avoid timeout problem on YARN.<commit_after>from __future__ import absolute_import
from __future__ import print_function

from keras.datasets import mnist
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.optimizers import SGD
from keras.utils import np_utils

from elephas.spark_model import SparkModel
from elephas.utils.rdd_utils import to_simple_rdd
from elephas import optimizers as elephas_optimizers

from pyspark import SparkContext, SparkConf

# Define basic parameters
batch_size = 64
nb_classes = 10
nb_epoch = 3

# Create Spark context
conf = SparkConf().setAppName('Mnist_Spark_MLP')  # .setMaster('local[8]')
sc = SparkContext(conf=conf)

# Load data
(X_train, y_train), (X_test, y_test) = mnist.load_data()

X_train = X_train.reshape(60000, 784)
X_test = X_test.reshape(10000, 784)
X_train = X_train.astype("float32")
X_test = X_test.astype("float32")
X_train /= 255
X_test /= 255
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')

# Convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)

model = Sequential()
model.add(Dense(128, input_dim=784))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(10))
model.add(Activation('softmax'))

# Compile model
sgd = SGD(lr=0.1)
model.compile(loss='categorical_crossentropy', optimizer=sgd)

# Build RDD from numpy features and labels
rdd = to_simple_rdd(sc, X_train, Y_train)

# Initialize SparkModel from Keras model and Spark context
adagrad = elephas_optimizers.Adagrad()
spark_model = SparkModel(sc,
                         model,
                         optimizer=adagrad,
                         frequency='epoch',
                         mode='asynchronous',
                         num_workers=2)

# Train Spark model
spark_model.train(rdd,
                  nb_epoch=nb_epoch,
                  batch_size=batch_size,
                  verbose=2,
                  validation_split=0.1)

# Evaluate Spark model by evaluating the underlying model
score = spark_model.get_network().evaluate(X_test,
                                           Y_test,
                                           show_accuracy=True,
                                           verbose=2)
print('Test accuracy:', score[1])
<|endoftext|>


<commit_before>import sys
import os
import urllib2
import shutil

import tempfile as tmp
from zipfile import ZipFile
from cStringIO import StringIO

# import re
# import logging

# from buildmanager import xformvalidator
# from buildmanager.exceptions import BuildError

# from xformmanager.models import MetaDataValidationError
# from xformmanager.xformdef import FormDef, ElementDef

def add_to_jar(jar_file, path_to_add):
    '''adds files under /path_to_add to jar_file, return path to the new JAR'''
    if not os.path.isdir(path_to_add):
        raise "Trying to add non-existant directory '%s' to JAR" % path_to_add

    if not jar_file.endswith('.jar') or not os.path.isfile(jar_file):
        raise "'%s' isn't a JAR file" % jar_file

    tmpjar = os.path.join(tmp.mkdtemp(), os.path.basename(jar_file))
    shutil.copy2(jar_file, tmpjar)

    zf = ZipFile(tmpjar, 'a')

    for f in os.listdir(path_to_add):
        zf.write(os.path.join(path_to_add, f))

    zf.close

    return tmpjar

def modify_jad(jad_file, jar_file):
    tmpjad = os.path.join(tmp.mkdtemp(), os.path.basename(jad_file))
    shutil.copy2(jad_file, tmpjad)

    # modify tmpjad here...

    return tmpjad

def create_zip(target, files):
    ''' create zip from files list, returns created zip file'''
    print target, files
    zf = ZipFile(target, 'w')
    for f in files:
        zf.write(f)

    zf.close()
    return target

# http://bitbucket.org/ctsims/resourcetest/get/tip.zip
def grab_from(url):
    '''copy a file from a URL to a local tmp dir, returns path to local copy'''
    u = urllib2.urlopen(url)
    u = u.read()

    x, tmpfile = tmp.mkstemp()
    f = open(tmpfile, 'w')
    f.write(u)
    f.close()

    return tmpfile

def unzip_to_tmp(zip_file):
    '''
    extracts a resources zip.
    assumes that all files are on one root dir
    returns path of extracted files
    '''
    zf = ZipFile(zip_file)
    target_dir = tmp.mkdtemp()

    namelist = zf.namelist()
    filelist = filter( lambda x: not x.endswith( '/' ), namelist )

    for filename in filelist:
        basename = os.path.basename(filename)
        if basename.startswith('.'): continue #filename.endswith(".xml") or filename.endswith(".xhtml"):

        target_file = os.path.join(target_dir, basename)

        out = open(target_file, 'wb')

        buffer = StringIO( zf.read(filename))
        buflen = 2 ** 20
        datum = buffer.read(buflen)

        while datum:
            out.write(datum)
            datum = buffer.read(buflen)
        out.close()

    return target_dir
<commit_msg>remove leftover 'print'<commit_after>import sys
import os
import urllib2
import shutil

import tempfile as tmp
from zipfile import ZipFile
from cStringIO import StringIO

# import re
# import logging

# from buildmanager import xformvalidator
# from buildmanager.exceptions import BuildError

# from xformmanager.models import MetaDataValidationError
# from xformmanager.xformdef import FormDef, ElementDef

def add_to_jar(jar_file, path_to_add):
    '''adds files under /path_to_add to jar_file, return path to the new JAR'''
    if not os.path.isdir(path_to_add):
        raise "Trying to add non-existant directory '%s' to JAR" % path_to_add

    if not jar_file.endswith('.jar') or not os.path.isfile(jar_file):
        raise "'%s' isn't a JAR file" % jar_file

    tmpjar = os.path.join(tmp.mkdtemp(), os.path.basename(jar_file))
    shutil.copy2(jar_file, tmpjar)

    zf = ZipFile(tmpjar, 'a')

    for f in os.listdir(path_to_add):
        zf.write(os.path.join(path_to_add, f))

    zf.close

    return tmpjar

def modify_jad(jad_file, jar_file):
    tmpjad = os.path.join(tmp.mkdtemp(), os.path.basename(jad_file))
    shutil.copy2(jad_file, tmpjad)

    # modify tmpjad here...

    return tmpjad

def create_zip(target, files):
    ''' create zip from files list, returns created zip file'''
    zf = ZipFile(target, 'w')
    for f in files:
        zf.write(f)

    zf.close()
    return target

# http://bitbucket.org/ctsims/resourcetest/get/tip.zip
def grab_from(url):
    '''copy a file from a URL to a local tmp dir, returns path to local copy'''
    u = urllib2.urlopen(url)
    u = u.read()

    x, tmpfile = tmp.mkstemp()
    f = open(tmpfile, 'w')
    f.write(u)
    f.close()

    return tmpfile

def unzip_to_tmp(zip_file):
    '''
    extracts a resources zip.
    assumes that all files are on one root dir
    returns path of extracted files
    '''
    zf = ZipFile(zip_file)
    target_dir = tmp.mkdtemp()

    namelist = zf.namelist()
    filelist = filter( lambda x: not x.endswith( '/' ), namelist )

    for filename in filelist:
        basename = os.path.basename(filename)
        if basename.startswith('.'): continue #filename.endswith(".xml") or filename.endswith(".xhtml"):

        target_file = os.path.join(target_dir, basename)

        out = open(target_file, 'wb')

        buffer = StringIO( zf.read(filename))
        buflen = 2 ** 20
        datum = buffer.read(buflen)

        while datum:
            out.write(datum)
            datum = buffer.read(buflen)
        out.close()

    return target_dir
<|endoftext|>


<commit_before># Copyright (c) 2013, Web Notes Technologies Pvt. Ltd.
# MIT License. See license.txt

from __future__ import unicode_literals
import webnotes

from webnotes.utils import getdate, cint, add_months, date_diff, add_days, nowdate

weekdays = ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"]

class DocType:
	def __init__(self, d, dl):
		self.doc, self.doclist = d, dl

	def validate(self):
		if self.doc.starts_on and self.doc.ends_on and self.doc.starts_on > self.doc.ends_on:
			webnotes.msgprint(webnotes._("Event End must be after Start"), raise_exception=True)

def send_event_digest():
	today = nowdate()
	for user in webnotes.conn.sql("""select name, email, language
		from tabProfile where ifnull(enabled,0)=1
		and user_type='System User' and name not in ('Guest', 'Administrator')""", as_dict=1):
		events = get_events(today, today, user.name, for_reminder=True)
		if events:
			text = ""
			webnotes.set_user_lang(user.name, user.language)
			webnotes.load_translations("core", "doctype", "event")

			text = "
" + webnotes._("Events In Today's Calendar") + "
"
			for e in events:
				if e.all_day:
					e.starts_on = "All Day"
				text += "
%(starts_on)s: %(subject)s

%(description)s
" % e

			text += '
'\
				+ webnotes._("Daily Event Digest is sent for Calendar Events where reminders are set.")+'
'

			from webnotes.utils.email_lib import sendmail
			sendmail(recipients=user.email, subject=webnotes._("Upcoming Events for Today"),
				msg = text)

@webnotes.whitelist()
def get_events(start, end, user=None, for_reminder=False):
	if not user:
		user = webnotes.session.user
	roles = webnotes.get_roles(user)
	events = webnotes.conn.sql("""select name, subject, description,
		starts_on, ends_on, owner, all_day, event_type, repeat_this_event, repeat_on,
		monday, tuesday, wednesday, thursday, friday, saturday, sunday
		from tabEvent where ((
			(date(starts_on) between date('%(start)s') and date('%(end)s'))
			or (date(ends_on) between date('%(start)s') and date('%(end)s'))
			or (date(starts_on) <= date('%(start)s') and date(ends_on) >= date('%(end)s'))
		) or (
			date(starts_on) <= date('%(start)s') and ifnull(repeat_this_event,0)=1 and
			ifnull(repeat_till, "3000-01-01") > date('%(start)s')
		))
		%(reminder_condition)s
		and (event_type='Public' or owner='%(user)s'
		or exists(select * from `tabEvent User` where
			`tabEvent User`.parent=tabEvent.name and person='%(user)s')
		or exists(select * from `tabEvent Role` where
			`tabEvent Role`.parent=tabEvent.name
			and `tabEvent Role`.role in ('%(roles)s')))
		order by starts_on""" % {
			"start": start,
			"end": end,
			"reminder_condition": "and ifnull(send_reminder,0)=1" if for_reminder else "",
			"user": user,
			"roles": "', '".join(roles)
		}, as_dict=1)

	# process recurring events
	start = start.split(" ")[0]
	end = end.split(" ")[0]
	add_events = []

	def add_event(e, date):
		new_event = e.copy()
		new_event.starts_on = date + " " + e.starts_on.split(" ")[1]
		new_event.ends_on = date + " " + e.ends_on.split(" ")[1]
		add_events.append(new_event)

	for e in events:
		if e.repeat_this_event:
			event_start, time_str = e.starts_on.split(" ")
			if e.repeat_on=="Every Year":
				start_year = cint(start.split("-")[0])
				end_year = cint(end.split("-")[0])
				event_start = "-".join(event_start.split("-")[1:])

				# repeat for all years in period
				for year in range(start_year, end_year+1):
					date = str(year) + "-" + event_start
					if date >= start and date <= end:
						add_event(e, date)
				events.remove(e)

			if e.repeat_on=="Every Month":
				date = start.split("-")[0] + "-" + start.split("-")[1] + "-" + event_start.split("-")[2]

				# last day of month issue, start from prev month!
				try:
					getdate(date)
				except ValueError:
					date = date.split("-")
					date = date[0] + "-" + str(cint(date[1]) - 1) + "-" + date[2]

				start_from = date
				for i in xrange(int(date_diff(end, start) / 30) + 3):
					if date >= start and date <= end and date >= event_start:
						add_event(e, date)
					date = add_months(start_from, i+1)
				events.remove(e)

			if e.repeat_on=="Every Week":
				weekday = getdate(event_start).weekday()
				# monday is 0
				start_weekday = getdate(start).weekday()

				# start from nearest weeday after last monday
				date = add_days(start, weekday - start_weekday)

				for cnt in xrange(int(date_diff(end, start) / 7) + 3):
					if date >= start and date <= end and date >= event_start:
						add_event(e, date)

					date = add_days(date, 7)
				events.remove(e)

			if e.repeat_on=="Every Day":
				for cnt in xrange(date_diff(end, start) + 1):
					date = add_days(start, cnt)
					if date >= event_start and date <= end \
						and e[weekdays[getdate(date).weekday()]]:
						add_event(e, date)
				events.remove(e)

	events = events + add_events

	for e in events:
		# remove weekday properties (to reduce message size)
		for w in weekdays:
			del e[w]

	return events<commit_msg>[fix] [minor] recurring events<commit_after># Copyright (c) 2013, Web Notes Technologies Pvt. Ltd.
# MIT License. See license.txt

from __future__ import unicode_literals
import webnotes

from webnotes.utils import getdate, cint, add_months, date_diff, add_days, nowdate

weekdays = ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"]

class DocType:
	def __init__(self, d, dl):
		self.doc, self.doclist = d, dl

	def validate(self):
		if self.doc.starts_on and self.doc.ends_on and self.doc.starts_on > self.doc.ends_on:
			webnotes.msgprint(webnotes._("Event End must be after Start"), raise_exception=True)

def send_event_digest():
	today = nowdate()
	for user in webnotes.conn.sql("""select name, email, language
		from tabProfile where ifnull(enabled,0)=1
		and user_type='System User' and name not in ('Guest', 'Administrator')""", as_dict=1):
		events = get_events(today, today, user.name, for_reminder=True)
		if events:
			text = ""
			webnotes.set_user_lang(user.name, user.language)
			webnotes.load_translations("core", "doctype", "event")

			text = "
" + webnotes._("Events In Today's Calendar") + "
"
			for e in events:
				if e.all_day:
					e.starts_on = "All Day"
				text += "
%(starts_on)s: %(subject)s

%(description)s
" % e

			text += '
'\
				+ webnotes._("Daily Event Digest is sent for Calendar Events where reminders are set.")+'
'

			from webnotes.utils.email_lib import sendmail
			sendmail(recipients=user.email, subject=webnotes._("Upcoming Events for Today"),
				msg = text)

@webnotes.whitelist()
def get_events(start, end, user=None, for_reminder=False):
	if not user:
		user = webnotes.session.user
	roles = webnotes.get_roles(user)
	events = webnotes.conn.sql("""select name, subject, description,
		starts_on, ends_on, owner, all_day, event_type, repeat_this_event, repeat_on,
		monday, tuesday, wednesday, thursday, friday, saturday, sunday
		from tabEvent where ((
			(date(starts_on) between date('%(start)s') and date('%(end)s'))
			or (date(ends_on) between date('%(start)s') and date('%(end)s'))
			or (date(starts_on) <= date('%(start)s') and date(ends_on) >= date('%(end)s'))
		) or (
			date(starts_on) <= date('%(start)s') and ifnull(repeat_this_event,0)=1 and
			ifnull(repeat_till, "3000-01-01") > date('%(start)s')
		))
		%(reminder_condition)s
		and (event_type='Public' or owner='%(user)s'
		or exists(select * from `tabEvent User` where
			`tabEvent User`.parent=tabEvent.name and person='%(user)s')
		or exists(select * from `tabEvent Role` where
			`tabEvent Role`.parent=tabEvent.name
			and `tabEvent Role`.role in ('%(roles)s')))
		order by starts_on""" % {
			"start": start,
			"end": end,
			"reminder_condition": "and ifnull(send_reminder,0)=1" if for_reminder else "",
			"user": user,
			"roles": "', '".join(roles)
		}, as_dict=1)

	# process recurring events
	start = start.split(" ")[0]
	end = end.split(" ")[0]
	add_events = []
	remove_events = []

	def add_event(e, date):
		new_event = e.copy()
		new_event.starts_on = date + " " + e.starts_on.split(" ")[1]
		new_event.ends_on = date + " " + e.ends_on.split(" ")[1]
		add_events.append(new_event)

	for e in events:
		if e.repeat_this_event:
			event_start, time_str = e.starts_on.split(" ")
			if e.repeat_on=="Every Year":
				start_year = cint(start.split("-")[0])
				end_year = cint(end.split("-")[0])
				event_start = "-".join(event_start.split("-")[1:])

				# repeat for all years in period
				for year in range(start_year, end_year+1):
					date = str(year) + "-" + event_start
					if date >= start and date <= end:
						add_event(e, date)

				remove_events.append(e)

			if e.repeat_on=="Every Month":
				date = start.split("-")[0] + "-" + start.split("-")[1] + "-" + event_start.split("-")[2]

				# last day of month issue, start from prev month!
				try:
					getdate(date)
				except ValueError:
					date = date.split("-")
					date = date[0] + "-" + str(cint(date[1]) - 1) + "-" + date[2]

				start_from = date
				for i in xrange(int(date_diff(end, start) / 30) + 3):
					if date >= start and date <= end and date >= event_start:
						add_event(e, date)
					date = add_months(start_from, i+1)

				remove_events.append(e)

			if e.repeat_on=="Every Week":
				weekday = getdate(event_start).weekday()
				# monday is 0
				start_weekday = getdate(start).weekday()

				# start from nearest weeday after last monday
				date = add_days(start, weekday - start_weekday)

				for cnt in xrange(int(date_diff(end, start) / 7) + 3):
					if date >= start and date <= end and date >= event_start:
						add_event(e, date)

					date = add_days(date, 7)

				remove_events.append(e)

			if e.repeat_on=="Every Day":
				for cnt in xrange(date_diff(end, start) + 1):
					date = add_days(start, cnt)
					if date >= event_start and date <= end \
						and e[weekdays[getdate(date).weekday()]]:
						add_event(e, date)
				remove_events.append(e)

	for e in remove_events:
		events.remove(e)

	events = events + add_events

	for e in events:
		# remove weekday properties (to reduce message size)
		for w in weekdays:
			del e[w]

	return events<|endoftext|>


<commit_before>import numpy as np
import pickle

import tslearn.utils

__author__ = 'Romain Tavenard romain.tavenard[at]univ-rennes2.fr'

def test_arraylike_copy():
    X_npy = np.array([1, 2, 3])
    np.testing.assert_allclose(tslearn.utils._arraylike_copy(X_npy), X_npy)
    np.testing.assert_allclose(tslearn.utils._arraylike_copy(X_npy) is X_npy,
                               False)

def test_save_load():
    n, sz, d = 15, 10, 3
    rng = np.random.RandomState(0)
    dataset = rng.randn(n, sz, d)
    tslearn.utils.save_timeseries_txt("tmp-tslearn-test.txt", dataset)
    reloaded_dataset = tslearn.utils.load_timeseries_txt(
        "tmp-tslearn-test.txt")
    np.testing.assert_allclose(dataset, reloaded_dataset)

    dataset = tslearn.utils.to_time_series_dataset([[1, 2, 3, 4], [1, 2, 3]])
    tslearn.utils.save_timeseries_txt("tmp-tslearn-test.txt", dataset)
    reloaded_dataset = tslearn.utils.load_timeseries_txt(
        "tmp-tslearn-test.txt")
    for ts0, ts1 in zip(dataset, reloaded_dataset):
        np.testing.assert_allclose(ts0[:tslearn.utils.ts_size(ts0)],
                                   ts1[:tslearn.utils.ts_size(ts1)])

def test_label_categorizer():
    y = np.array([-1, 2, 1, 1, 2])
    lc = tslearn.utils.LabelCategorizer()
    lc.fit(y)

    s = pickle.dumps(lc)
    lc2 = pickle.loads(s)
    y_tr = lc2.inverse_transform([[0, 1, 0], [0, 0, 1], [1, 0, 0]])
    ref = np.array([1., 2., -1.])

    np.testing.assert_allclose(ref, y_tr)
<commit_msg>avoid creating temporary files in the project directory (that do not get cleaned up)<commit_after>from tempfile import NamedTemporaryFile

import numpy as np
import pickle

import tslearn.utils

__author__ = 'Romain Tavenard romain.tavenard[at]univ-rennes2.fr'

def test_arraylike_copy():
    X_npy = np.array([1, 2, 3])
    np.testing.assert_allclose(tslearn.utils._arraylike_copy(X_npy), X_npy)
    np.testing.assert_allclose(tslearn.utils._arraylike_copy(X_npy) is X_npy,
                               False)

def test_save_load_random():
    n, sz, d = 15, 10, 3
    rng = np.random.RandomState(0)
    dataset = rng.randn(n, sz, d)
    with NamedTemporaryFile(suffix=".txt") as text_file:
        tslearn.utils.save_timeseries_txt(text_file.name, dataset)
        reloaded_dataset = tslearn.utils.load_timeseries_txt(text_file.name)
    np.testing.assert_allclose(dataset, reloaded_dataset)

def test_save_load_known():
    dataset = tslearn.utils.to_time_series_dataset([[1, 2, 3, 4], [1, 2, 3]])
    with NamedTemporaryFile(suffix=".txt") as text_file:
        tslearn.utils.save_timeseries_txt(text_file.name, dataset)
        reloaded_dataset = tslearn.utils.load_timeseries_txt(text_file.name)
    for ts0, ts1 in zip(dataset, reloaded_dataset):
        np.testing.assert_allclose(ts0[:tslearn.utils.ts_size(ts0)],
                                   ts1[:tslearn.utils.ts_size(ts1)])

def test_label_categorizer():
    y = np.array([-1, 2, 1, 1, 2])
    lc = tslearn.utils.LabelCategorizer()
    lc.fit(y)

    s = pickle.dumps(lc)
    lc2 = pickle.loads(s)
    y_tr = lc2.inverse_transform([[0, 1, 0], [0, 0, 1], [1, 0, 0]])
    ref = np.array([1., 2., -1.])

    np.testing.assert_allclose(ref, y_tr)
<|endoftext|>


<commit_before>import logging
import apt
import apt.progress
import click
from ips_vagrant.cli import pass_context, Context

@click.command('setup', short_help='Run setup after a fresh Vagrant installation.')
@pass_context
def cli(ctx):
    """Run setup after a fresh Vagrant installation."""
    log = logging.getLogger('ipsv.setup')
    assert isinstance(ctx, Context)

    # Update the system
    click.echo('Updating package cache..')
    cache = apt.Cache()
    cache.update()
    cache.open(None)
    click.echo('Upgrading system packages..')
    cache.upgrade()
    cache.commit(apt.progress.TextFetchProgress(), apt.progress.InstallProgress())

    # Install our required packages
    requirements = ['nginx', 'php5-fpm', 'php5-curl', 'php5-gd', 'php5-imagick', 'php5-json', 'php5-mysql',
                    'php5-readline', 'php5-apcu']

    for requirement in requirements:
        # Make sure the package is available
        if requirement not in cache:
            log.warn('Required package {pkg} not available'.format(pkg=requirement))
            continue

        # Mark the package for installation
        click.echo('Marking package {pkg} for installation'.format(pkg=requirement))
        cache[requirement].mark_install()

    log.info('Committing package cache')
    cache.commit()
<commit_msg>apt.progress bugfix<commit_after>import apt
import click
import logging
from ips_vagrant.cli import pass_context, Context

@click.command('setup', short_help='Run setup after a fresh Vagrant installation.')
@pass_context
def cli(ctx):
    """Run setup after a fresh Vagrant installation."""
    log = logging.getLogger('ipsv.setup')
    assert isinstance(ctx, Context)

    # Update the system
    click.echo('Updating package cache..')
    cache = apt.Cache()
    cache.update()
    cache.open(None)
    click.echo('Upgrading system packages..')
    cache.upgrade()
    cache.commit()

    # Install our required packages
    requirements = ['nginx', 'php5-fpm', 'php5-curl', 'php5-gd', 'php5-imagick', 'php5-json', 'php5-mysql',
                    'php5-readline', 'php5-apcu']

    for requirement in requirements:
        # Make sure the package is available
        if requirement not in cache:
            log.warn('Required package {pkg} not available'.format(pkg=requirement))
            continue

        # Mark the package for installation
        click.echo('Marking package {pkg} for installation'.format(pkg=requirement))
        cache[requirement].mark_install()

    log.info('Committing package cache')
    cache.commit()
<|endoftext|>


<commit_before>import pymongo

from motor.motor_tornado import MotorClient
from pymongo import MongoClient

from .conf import config
from .exceptions import DbError

def connect_to_db(async=True):
    db_conf = config['db']

    with_auth = db_conf.get('user') and db_conf.get('password')
    if with_auth:
        auth = '{user}:{password}@'.format(**db_conf)
        if db_conf.get('authDbName'):
            auth_db = '/' + db_conf['authDbName']
    else:
        auth = ''
        auth_db = ''

    dest = '{host}:{port}'.format(**db_conf)

    conn_str = 'mongodb://' + auth + dest + auth_db
    client_cls = MotorClient if async else MongoClient
    db_client = client_cls(conn_str)

    db = getattr(db_client, db_conf['dbName'])

    return db_client, db

class ModelMeta(type):
    def __new__(cls, name, bases, attrs):
        attrs['collection_name'] = name.lower()
        return super().__new__(cls, name, bases, attrs)

class Model(metaclass=ModelMeta):
    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k, v)

    @classmethod
    def find(cls, **kwargs):
        return Collection(cls).find(**kwargs)

    @classmethod
    def find_one(cls, **kwargs):
        return Collection(cls).find_one(**kwargs)

# db = connect_to_db(False)[1]

class Collection:
    def __init__(self, model_cls, db):
        if not issubclass(model_cls, Model):
            raise ValueError(
                'Class "%s" is not a subclass of "logster.db.Model" class',
                model_cls.__name__)

        self.model_cls = model_cls
        self.collection = db[model_cls.collection_name]
        self.result = None
        self.cursor = None

    def find(self, **kwargs):
        self.result = None
        self.cursor = self.collection.find(kwargs)
        return self

    def sort(self, *args):
        if self.cursor is None:
            raise DbError('You must invoke `.find()` before sorting')

        sorters = [
            (arg, pymongo.ASCENDING) if arg[0] == '-'
            else (arg[1:], pymongo.DESCENDING) for arg in args]

        self.cursor = self.cursor.sort(sorters)
        return self

    def find_one(self, **kwargs):
        data = self.collection.find_one(kwargs)
        if data is not None:
            return self.model_cls(**data)
        return None

    def __iter__(self):
        if not self.result and not self.cursor:
            raise DbError()

        if not self.result:
            self.result = list(self.cursor)

        return iter(self.result)
<commit_msg>Do not redefine the collection name<commit_after>import pymongo

from motor.motor_tornado import MotorClient
from pymongo import MongoClient

from .conf import config
from .exceptions import DbError

def connect_to_db(async=True):
    db_conf = config['db']

    with_auth = db_conf.get('user') and db_conf.get('password')
    if with_auth:
        auth = '{user}:{password}@'.format(**db_conf)
        if db_conf.get('authDbName'):
            auth_db = '/' + db_conf['authDbName']
    else:
        auth = ''
        auth_db = ''

    dest = '{host}:{port}'.format(**db_conf)

    conn_str = 'mongodb://' + auth + dest + auth_db
    client_cls = MotorClient if async else MongoClient
    db_client = client_cls(conn_str)

    db = getattr(db_client, db_conf['dbName'])

    return db_client, db

class ModelMeta(type):
    def __new__(cls, name, bases, attrs):
        if 'collection_name' not in attrs:
            attrs['collection_name'] = name.lower()

        return super().__new__(cls, name, bases, attrs)

class Model(metaclass=ModelMeta):
    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k, v)

    @classmethod
    def find(cls, **kwargs):
        return Collection(cls).find(**kwargs)

    @classmethod
    def find_one(cls, **kwargs):
        return Collection(cls).find_one(**kwargs)

# db = connect_to_db(False)[1]

class Collection:
    def __init__(self, model_cls, db):
        if not issubclass(model_cls, Model):
            raise ValueError(
                'Class "%s" is not a subclass of "logster.db.Model" class',
                model_cls.__name__)

        self.model_cls = model_cls
        self.collection = db[model_cls.collection_name]
        self.result = None
        self.cursor = None

    def find(self, **kwargs):
        self.result = None
        self.cursor = self.collection.find(kwargs)
        return self

    def sort(self, *args):
        if self.cursor is None:
            raise DbError('You must invoke `.find()` before sorting')

        sorters = [
            (arg, pymongo.ASCENDING) if arg[0] == '-'
            else (arg[1:], pymongo.DESCENDING) for arg in args]

        self.cursor = self.cursor.sort(sorters)
        return self

    def find_one(self, **kwargs):
        data = self.collection.find_one(kwargs)
        if data is not None:
            return self.model_cls(**data)
        return None

    def __iter__(self):
        if not self.result and not self.cursor:
            raise DbError()

        if not self.result:
            self.result = list(self.cursor)

        return iter(self.result)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright (c) 2017-2018 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test resendwallettransactions RPC."""

from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import assert_equal, assert_raises_rpc_error

class ResendWalletTransactionsTest(BitcoinTestFramework):
    def set_test_params(self):
        self.num_nodes = 1
        self.extra_args = [['--walletbroadcast=false']]

    def skip_test_if_missing_module(self):
        self.skip_if_no_wallet()

    def run_test(self):
        # Should raise RPC_WALLET_ERROR (-4) if walletbroadcast is disabled.
        assert_raises_rpc_error(-4, "Error: Wallet transaction broadcasting is disabled with -walletbroadcast", self.nodes[0].resendwallettransactions)

        # Should return an empty array if there aren't unconfirmed wallet transactions.
        self.stop_node(0)
        self.start_node(0, extra_args=[])
        assert_equal(self.nodes[0].resendwallettransactions(), [])

        # Should return an array with the unconfirmed wallet transaction.
        txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1)
        assert_equal(self.nodes[0].resendwallettransactions(), [txid])

if __name__ == '__main__':
    ResendWalletTransactionsTest().main()
<commit_msg>[tests] Add test for wallet rebroadcasts<commit_after>#!/usr/bin/env python3
# Copyright (c) 2017-2018 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test that the wallet resends transactions periodically."""
from collections import defaultdict
import time

from test_framework.blocktools import create_block, create_coinbase
from test_framework.messages import ToHex
from test_framework.mininode import P2PInterface, mininode_lock
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import assert_equal, wait_until

class P2PStoreTxInvs(P2PInterface):
    def __init__(self):
        super().__init__()
        self.tx_invs_received = defaultdict(int)

    def on_inv(self, message):
        # Store how many times invs have been received for each tx.
        for i in message.inv:
            if i.type == 1:
                # save txid
                self.tx_invs_received[i.hash] += 1

class ResendWalletTransactionsTest(BitcoinTestFramework):
    def set_test_params(self):
        self.num_nodes = 1

    def skip_test_if_missing_module(self):
        self.skip_if_no_wallet()

    def run_test(self):
        node = self.nodes[0]  # alias

        node.add_p2p_connection(P2PStoreTxInvs())

        self.log.info("Create a new transaction and wait until it's broadcast")
        txid = int(node.sendtoaddress(node.getnewaddress(), 1), 16)

        # Can take a few seconds due to transaction trickling
        wait_until(lambda: node.p2p.tx_invs_received[txid] >= 1, lock=mininode_lock)

        # Add a second peer since txs aren't rebroadcast to the same peer (see filterInventoryKnown)
        node.add_p2p_connection(P2PStoreTxInvs())

        self.log.info("Create a block")
        # Create and submit a block without the transaction.
        # Transactions are only rebroadcast if there has been a block at least five minutes
        # after the last time we tried to broadcast. Use mocktime and give an extra minute to be sure.
        block_time = int(time.time()) + 6 * 60
        node.setmocktime(block_time)
        block = create_block(int(node.getbestblockhash(), 16), create_coinbase(node.getblockchaininfo()['blocks']), block_time)
        block.nVersion = 3
        block.rehash()
        block.solve()
        node.submitblock(ToHex(block))

        # Transaction should not be rebroadcast
        node.p2ps[1].sync_with_ping()
        assert_equal(node.p2ps[1].tx_invs_received[txid], 0)

        self.log.info("Transaction should be rebroadcast after 30 minutes")
        # Use mocktime and give an extra 5 minutes to be sure.
        rebroadcast_time = int(time.time()) + 41 * 60
        node.setmocktime(rebroadcast_time)
        wait_until(lambda: node.p2ps[1].tx_invs_received[txid] >= 1, lock=mininode_lock)

if __name__ == '__main__':
    ResendWalletTransactionsTest().main()
<|endoftext|>


<commit_before>import argparse
import sqlalchemy
import wlmhelpers

def run(stmt):
    return stmt.execute()

def create_monuments_table(db):
    metadata = sqlalchemy.MetaData(db)
    return sqlalchemy.Table("monuments_all", metadata, autoload=True)

def get_monuments_in_country(countryname, languagename, all_monuments):
    return run(all_monuments.select(sqlalchemy.and_(
        all_monuments.c.country == countryname,
        all_monuments.c.lang == languagename)
    ))

def main(arguments):
    db = sqlalchemy.create_engine('mysql://{}:{}@{}/{}?charset=utf8'.format(
        arguments.user,
        arguments.password,
        arguments.host,
        arguments.db), encoding='utf8')
    db.echo = False
    try:
        all_monuments = create_monuments_table(db)
        monuments_country = get_monuments_in_country(
            arguments.country, arguments.language, all_monuments)
        for x in monuments_country:
            print(x.name)
    except sqlalchemy.exc.SQLAlchemyError as exc:
        print("{} does not exist.".format(arguments.table))

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--host", default="localhost")
    parser.add_argument("--user", default="root")
    parser.add_argument("--password", default="")
    parser.add_argument("--db", default="wlm")
    parser.add_argument("--table")
    parser.add_argument("--language")
    parser.add_argument("--country")
    args = parser.parse_args()
    main(args)
<commit_msg>reflect monuments_all<commit_after>import argparse
import sqlalchemy
from sqlalchemy.ext.declarative import declarative_base
import wlmhelpers

Base = declarative_base()

def run(stmt):
    return stmt.execute()

def get_monuments_in_country(countryname, languagename, all_monuments):
    return run(all_monuments.select(sqlalchemy.and_(
        all_monuments.c.country == countryname,
        all_monuments.c.lang == languagename)
    ))

def get_specific_table_name(countryname, languagename):
    return "monuments_{}_({})".format(countryname, languagename)

def main(arguments):
    db = sqlalchemy.create_engine('mysql://{}:{}@{}/{}?charset=utf8'.format(
        arguments.user,
        arguments.password,
        arguments.host,
        arguments.db), encoding='utf8')
    db.echo = False
    try:
        metadata = sqlalchemy.MetaData(db)
        all_monuments = sqlalchemy.Table("monuments_all", metadata, autoload=True)
        associated_table = get_specific_table_name(arguments.country, arguments.language)
        monuments_country = get_monuments_in_country(
            arguments.country, arguments.language, all_monuments)
        for x in monuments_country:
            print(x.id, x.name)

    except sqlalchemy.exc.SQLAlchemyError as exc:
        print("{} does not exist.".format(arguments.table))

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--host", default="localhost")
    parser.add_argument("--user", default="root")
    parser.add_argument("--password", default="")
    parser.add_argument("--db", default="wlm")
    parser.add_argument("--table")
    parser.add_argument("--language", default="sv")
    parser.add_argument("--country", default="se-ship")
    args = parser.parse_args()
    main(args)
<|endoftext|>


<commit_before>from cloudmesh_base.Shell import Shell

from cloudmesh_base.locations import config_file
from cloudmesh_base.ConfigDict import ConfigDict
from cmd3.console import Console

class setup_management(object):

    def __init__(self, quiet=False):
        self.quiet = quiet
        try:
            self.filename = config_file("/cmd3.yaml")
            if not self.quiet:
                Console.ok("Reading " + self.filename + ". ok.")
        except:
            Console.error("problem loading cmd3.yaml file")
        try:
            self.config = ConfigDict(filename=self.filename)
            # print(self.config)
        except Exception, e:
            Console.error("problem with ConfigDict")
            print(e)

    def add(self, plugin, dryrun=False):

        if not plugin.endswith(".plugins"):
            plugin = plugin + ".plugins"

        modules = self.config["cmd3"]["modules"]
        if plugin in modules:
            if not self.quiet:
                Console.error("plugin {:} is already activated.".format(plugin))
        else:
            self.config["cmd3"]["modules"].append(plugin)
            if not self.quiet:
                Console.ok("Adding plugin " + plugin)

        if not dryrun:
            self.config.write(self.filename, output="yaml")
        else:
            print(self.config.yaml())

    def delete(self, plugin, dryrun=False):

        if not plugin.endswith(".plugins"):
            plugin = plugin + ".plugins"

        modules = self.config["cmd3"]["modules"]
        if plugin not in modules:
            Console.error("plugin {:} is already deactivated.".format(plugin))
        else:
            self.config["cmd3"]["modules"].remove(plugin)
            if not self.quiet:
                Console.ok("Deleting plugin " + plugin)

        if not dryrun:
            self.config.write(self.filename, output="yaml")
        else:
            print(self.config.yaml())

    def __str__(self):
        with open(self.filename, 'r') as f:
            r = f.read()
        return r

<commit_msg>setup management improvements<commit_after>from cloudmesh_base.Shell import Shell

from cloudmesh_base.locations import config_file
from cloudmesh_base.ConfigDict import ConfigDict
from cmd3.console import Console

class setup_management(object):
    def __init__(self, quiet=False):
        self.quiet = quiet
        try:
            self.filename = config_file("/cmd3.yaml")
            if not self.quiet:
                Console.ok("Reading " + self.filename + ". ok.")
        except:
            Console.error("problem loading cmd3.yaml file")
        try:
            self.config = ConfigDict(filename=self.filename)
            # print(self.config)
        except Exception, e:
            Console.error("problem with ConfigDict")
            print(e)

    def add(self, plugin, dryrun=False):

        if not plugin.endswith(".plugins"):
            plugin = plugin + ".plugins"

        modules = self.config["cmd3"]["modules"]
        if plugin in modules:
            if not self.quiet:
                Console.error("plugin {:} is already activated.".format(plugin))
        else:
            self.config["cmd3"]["modules"].append(plugin)
            if not self.quiet:
                Console.ok("Adding plugin " + plugin)

        if not dryrun:
            self.config.write(self.filename, output="yaml")
        else:
            print(self.config.yaml())

    def delete(self, plugin, dryrun=False):

        if not plugin.endswith(".plugins"):
            plugin = plugin + ".plugins"

        modules = self.config["cmd3"]["modules"]
        if plugin not in modules:
            Console.error("plugin {:} is already deactivated.".format(plugin))
        else:
            self.config["cmd3"]["modules"].remove(plugin)
            if not self.quiet:
                Console.ok("Deleting plugin " + plugin)

        if not dryrun:
            self.config.write(self.filename, output="yaml")
        else:
            print(self.config.yaml())

    def __str__(self):
        with open(self.filename, 'r') as f:
            r = f.read()
        return r

<|endoftext|>


<commit_before># Copyright 2014 Sardar Yumatov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import time
import select
import socket
import OpenSSL

try:
    import threading as _threading
except ImportError:
    import dummy_threading as _threading

# python 3 support
import six

from . import BaseBackend, BaseConnection
from ..certificate import BaseCertificate

# module level logger
LOG = logging.getLogger(__name__)

class Certificate(BaseCertificate):
    """ pyOpenSSL certificate implementation. """

    def load_context(self, cert_string=None, cert_file=None, key_string=None, key_file=None, passphrase=None):
        """ Initialize and load certificate context. """
        context = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv3_METHOD)
        if passphrase is not None and not isinstance(passphrase, six.binary_type):
            passphrase = six.b(passphrase)

        if cert_file:
            # we have to load certificate for equality check. there is no
            # other way to obtain certificate from context.
            if LOG.isEnabledFor(logging.DEBUG):
                LOG.debug("Certificate provided as file: %s", cert_file)

            with open(cert_file, 'rb') as fp:
                cert_string = fp.read()
        else:
            if LOG.isEnabledFor(logging.DEBUG):
                LOG.debug("Certificate provided as string")

        cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert_string)
        context.use_certificate(cert)

        if not key_string and not key_file:
            if LOG.isEnabledFor(logging.DEBUG):
                LOG.debug("Private key provided with certificate %s %s passphrase",
                            'file' if cert_file else 'string',
                            'with' if passphrase is not None else 'without')

            # OpenSSL is smart enought to locate private key in the certificate
            args = [OpenSSL.crypto.FILETYPE_PEM, cert_string]
            if passphrase is not None:
                args.append(passphrase)

            pk = OpenSSL.crypto.load_privatekey(*args)
            context.use_privatekey(pk)
        elif key_file and passphrase is None:
            if LOG.isEnabledFor(logging.DEBUG):
                LOG.debug("Private key provided as file without passphrase: %s", key_file)

            context.use_privatekey_file(key_file, OpenSSL.crypto.FILETYPE_PEM)
        else:
            if key_file:
                if LOG.isEnabledFor(logging.DEBUG):
                    LOG.debug("Private key provided as file withpassphrase: %s", key_file)

                # key file is provided with passphrase. context.use_privatekey_file
                # does not use passphrase, so we have to load the key file manually.
                with open(key_file, 'rb') as fp:
                    key_string = fp.read()
            else:
                if LOG.isEnabledFor(logging.DEBUG):
                    LOG.debug("Private key provided as string %s passphrase",
                                'with' if passphrase is not None else 'without')

            args = [OpenSSL.crypto.FILETYPE_PEM, key_string]
            if passphrase is not None:
                args.append(passphrase)

            pk = OpenSSL.crypto.load_privatekey(*args)
            context.use_privatekey(pk)

        # check if we are not passed some garbage
        context.check_privatekey()
        return context, cert

    def dump_certificate(self, raw_certificate):
        """ Dump certificate as PEM string.

            :Arguments:
                - context (object): certificate context as returned by :func:`load_context`

            :Returns:
                Certificate string as sequence of bytes.
        """
        return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, raw_certificate)

    def dump_digest(self, raw_certificate, digest):
        """ Dump certificate digest.

            :Arguments:
                - context (object): certificate context as returned by :func:`load_context`
                - digest (str): digest name, such as "sha1"

            :Returns:
                Digest as sequence of bytes.
        """
        return raw_certificate.digest(digest)

class Connection(BaseConnection):
    """ Connection to APNs. """

    def __init__(self, address, certificate, timeout=None):
        """ Open new connection to APNs using POSIX sockets and pyOpenSSL.

            :Arguments:
                - address (tuple): address as (host, port) tuple.
                - certificate (:class:`Certificate`): provider's certificate.
                - timeout (float): connection timeout in seconds.
        """
        super(Connection, self).__init__(address, certificate)
        self._socket = None
        self._connection = None
        self._timeout = timeout
        self._open_connection(timeout)

    def _open_connection(self, timeout):
        """ Open remote connection. """
        try:
            if LOG.isEnabledFor(logging.DEBUG):
                LOG.debug("Opening POSIX socket/pyOpenSSL connection to %r", self.address)
            self._socket = self._create_socket(timeout)
            self._configure_socket()
            self._connection = self._create_openssl_connection()
            self._configure_connection()
            self._connect_and_handshake()
            if LOG.isEnabledFor(logging.DEBUG):
                LOG.debug("SSL handshaking to %r completed", self.address)
        except:
            LOG.warning("Failed to establish socket/SSL connection to %r", self.address, exc_info=True)
            self.close()
            raise

    def _create_socket(self, timeout):
        """ Create new plain TCP socket. """
        return socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    def _configure_socket(self):
        """ Hook to configure socket parameters. """
        pass

    def _create_openssl_connection(self):
        """ Create new OpenSSL connection. """
        ctx = self.certificate.get_context()
        if self._timeout is not None:
            ctx.set_timeout(int(self._timeout))
        return OpenSSL.SSL.Connection(ctx, self._socket)

    def _configure_connection(self):
        """ Hookt to configure SSL connection. """
        pass

    def _connect_and_handshake(self):
        """ Connect to APNs and SSL handshake. """
        self._connection.connect(self.address)
        # NOTE: if we set any timeout on the underlying socket, then OpenSSL wrapper
        # will always immediately fail with WantReadError without even attempting
        # to block for that timeout. So we use an infinitelly blocking socket here,
        # but we have set timeout on context object and we hope it works.
        self._connection.setblocking(1)
        self._connection.do_handshake()

    def closed(self):
        """ Returns True if this connection is explicitly closed with :func:`close` call. """
        return self._socket is None

    def close(self):
        """ Close this connection. """
        if self._socket is not None:
            if self._connection is not None:
                try:
                    # tell SSL socket we are done
                    self._connection.shutdown()
                except:
                    pass
                try:
                    # free SSL related resources
                    self._connection.close()
                except:
                    pass

            try:
                # shutdown IO
                self._socket.shutdown(socket.SHUT_RDWR)
            except:
                pass
            try:
                # close socket
                self._socket.close()
            except:
                pass

            self._socket = None
            self._connection = None

    def reset(self):
        """ Flushes read buffer. """
        to_skip = self._connection.pending()
        if to_skip > 0:
            self._connection.recv(to_skip)

    def write(self, data, timeout):
        """ Write chunk of data. """
        if self.closed():
            raise IOError("Connection is closed")

        # FIXME: pyOpenSSL can't block with the socket if timeout is set to
        # any value except None. So it is either block forewer or immediate
        # WantReadError without waiting at all. we have to use select() for
        # blocking.
        self._connection.setblocking(1)
        self._socket.settimeout(timeout)  # None will set blocking mode
        waited = 0
        while True:
            try:
                # usually socket accepts any data regardless of network state.
                # it simply stays in the output buffer, which we can't flush
                # unless we disable Nagle algorithm entirely.
                if timeout is not None:
                    before = time.time()
                    sent = self._connection.send(data)
                    waited += time.time() - before
                else: # save the time syscalls
                    sent = self._connection.send(data)

                if sent == len(data):
                    return
                elif sent > 0:
                    data = data[sent:]
                    if timeout is not None:
                        # in case pyOpenSSL starts to actually wait timeouts
                        if timeout - waited >= 0:
                            self._socket.settimeout(timeout - waited)
                            # updated timeout, try again
                            continue
                    else:
                        # infinite timeout, keep trying
                        continue
            except OpenSSL.SSL.WantWriteError:
                # looks like we are in blocking mode, but pyOpenSSL doesn't want
                # to block or we are in a really short timeout.
                if timeout is not None:
                    waited += time.time() - before
                    if timeout - waited > 0:
                        before = time.time()
                        _, canwrite, _ = select.select((), (self._socket, ), (), timeout - waited)
                        waited += time.time() - before
                        if canwrite and timeout - waited >= 0:
                            # buffer ready for writing, try again
                            self._socket.settimeout(timeout - waited)
                            continue

            # we come here if send() blocking works and we waited more than timeout
            # or send() blocking doesn't work and we waited more than timeout
            # or if timeout is None and we still got WantWriteError
            LOG.warning("Write timeout %r it too short for buffer %s", timeout, len(data))
            raise IOError("Timeout exceeded")

    def peek(self, size):
        """ Peek chunk of data from the read buffer. """
        if self.closed():
            return None

        self._connection.setblocking(1)
        pending = self._connection.pending()
        if pending > 0:
            return self._connection.recv(min(pending, size))
        else:
            # do not trust pyOpenSSL, that piece of crap never sets pending buffer.
            canread, _, _ = select.select((self._socket, ), (), (), 0) # poll
            if canread:
                self._connection.setblocking(0)
                self._socket.settimeout(0)
                try:
                    ret = self._connection.recv(size)
                    if not ret:
                        ret = None

                    return ret
                except OpenSSL.SSL.ZeroReturnError:
                    # nice end of stream
                    return None
                except OpenSSL.SSL.WantReadError:
                    # no data is available yet
                    return None

        return None

    def read(self, size, timeout):
        """ Read chunk of data. """
        if self.closed():
            return None

        self._connection.setblocking(1)
        self._socket.settimeout(timeout)  # None will set blocking mode
        waited = 0
        while True:
            try:
                if timeout is not None:
                    before = time.time()
                    ret = self._connection.recv(size)
                    waited += time.time() - before
                else: # save the time syscalls
                    ret = self._connection.recv(size)

                if not ret:
                    # in case recv() responds with empty string
                    ret = None

                return ret
            except OpenSSL.SSL.ZeroReturnError:
                # nice end of stream
                return None
            except OpenSSL.SSL.WantReadError:
                # either pyOpenSSL is failing without even trying to block
                # or we really exceeded the timeout
                if timeout is not None:
                    waited += time.time() - before
                    if timeout - waited > 0:
                        before = time.time()
                        canread, _, _ = select.select((self._socket, ), (), (), timeout - waited)
                        waited += time.time() - before
                        if canread and timeout - waited >= 0:
                            # buffer ready for writing, try again
                            self._socket.settimeout(timeout - waited)
                            continue

            # we come here if read raises WantReadError, which means normal blocking
            # doesn't work or we really exceeded the timeout; timeout is None
            # and we still got WantReadError, which is strange; we waited all
            # time with select()
            #LOG.warning("Read timeout %r it too short for buffer %s", timeout, size)
            raise IOError("Timeout exceeded")

class Backend(BaseBackend):
    """ IO backend based on POSIX sockets. """
    # no, we can't detect if socket is closed without IO
    can_detect_close = False
    # raw connection implementation
    connection_class = Connection

    def get_new_connection(self, address, certificate, timeout=None):
        """ Open a new connection.

            :Arguments:
                - address (tuple): target (host, port).
                - certificate (:class:`Certificate`): certificate instance.
                - timeout (float): connection timeout in seconds
        """
        return self.connection_class(address, certificate, timeout=timeout)

    def get_certificate(self, cert_params):
        """ Create/load certificate from parameters. """
        return Certificate(**cert_params)

    def create_lock(self):
        """ Provides semaphore with ``threading.Lock`` interface. """
        return _threading.Lock()
<commit_msg>bump up tls version and disale sslv3 completely<commit_after># Copyright 2014 Sardar Yumatov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import time
import select
import socket
import OpenSSL

try:
    import threading as _threading
except ImportError:
    import dummy_threading as _threading

# python 3 support
import six

from . import BaseBackend, BaseConnection
from ..certificate import BaseCertificate

# module level logger
LOG = logging.getLogger(__name__)

class Certificate(BaseCertificate):
    """ pyOpenSSL certificate implementation. """

    def load_context(self, cert_string=None, cert_file=None, key_string=None,
                     key_file=None, passphrase=None,
                     context_method=OpenSSL.SSL.TLSv1_2_METHOD,
                     context_options=OpenSSL.SSL.OP_NO_SSLv3):
        """ Initialize and load certificate context. """
        context = OpenSSL.SSL.Context(context_method)
        context.set_options(context_options)
        if passphrase is not None and not isinstance(passphrase, six.binary_type):
            passphrase = six.b(passphrase)

        if cert_file:
            # we have to load certificate for equality check. there is no
            # other way to obtain certificate from context.
            if LOG.isEnabledFor(logging.DEBUG):
                LOG.debug("Certificate provided as file: %s", cert_file)

            with open(cert_file, 'rb') as fp:
                cert_string = fp.read()
        else:
            if LOG.isEnabledFor(logging.DEBUG):
                LOG.debug("Certificate provided as string")

        cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert_string)
        context.use_certificate(cert)

        if not key_string and not key_file:
            if LOG.isEnabledFor(logging.DEBUG):
                LOG.debug("Private key provided with certificate %s %s passphrase",
                            'file' if cert_file else 'string',
                            'with' if passphrase is not None else 'without')

            # OpenSSL is smart enought to locate private key in the certificate
            args = [OpenSSL.crypto.FILETYPE_PEM, cert_string]
            if passphrase is not None:
                args.append(passphrase)

            pk = OpenSSL.crypto.load_privatekey(*args)
            context.use_privatekey(pk)
        elif key_file and passphrase is None:
            if LOG.isEnabledFor(logging.DEBUG):
                LOG.debug("Private key provided as file without passphrase: %s", key_file)

            context.use_privatekey_file(key_file, OpenSSL.crypto.FILETYPE_PEM)
        else:
            if key_file:
                if LOG.isEnabledFor(logging.DEBUG):
                    LOG.debug("Private key provided as file withpassphrase: %s", key_file)

                # key file is provided with passphrase. context.use_privatekey_file
                # does not use passphrase, so we have to load the key file manually.
                with open(key_file, 'rb') as fp:
                    key_string = fp.read()
            else:
                if LOG.isEnabledFor(logging.DEBUG):
                    LOG.debug("Private key provided as string %s passphrase",
                                'with' if passphrase is not None else 'without')

            args = [OpenSSL.crypto.FILETYPE_PEM, key_string]
            if passphrase is not None:
                args.append(passphrase)

            pk = OpenSSL.crypto.load_privatekey(*args)
            context.use_privatekey(pk)

        # check if we are not passed some garbage
        context.check_privatekey()
        return context, cert

    def dump_certificate(self, raw_certificate):
        """ Dump certificate as PEM string.

            :Arguments:
                - context (object): certificate context as returned by :func:`load_context`

            :Returns:
                Certificate string as sequence of bytes.
        """
        return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, raw_certificate)

    def dump_digest(self, raw_certificate, digest):
        """ Dump certificate digest.

            :Arguments:
                - context (object): certificate context as returned by :func:`load_context`
                - digest (str): digest name, such as "sha1"

            :Returns:
                Digest as sequence of bytes.
        """
        return raw_certificate.digest(digest)

class Connection(BaseConnection):
    """ Connection to APNs. """

    def __init__(self, address, certificate, timeout=None):
        """ Open new connection to APNs using POSIX sockets and pyOpenSSL.

            :Arguments:
                - address (tuple): address as (host, port) tuple.
                - certificate (:class:`Certificate`): provider's certificate.
                - timeout (float): connection timeout in seconds.
        """
        super(Connection, self).__init__(address, certificate)
        self._socket = None
        self._connection = None
        self._timeout = timeout
        self._open_connection(timeout)

    def _open_connection(self, timeout):
        """ Open remote connection. """
        try:
            if LOG.isEnabledFor(logging.DEBUG):
                LOG.debug("Opening POSIX socket/pyOpenSSL connection to %r", self.address)
            self._socket = self._create_socket(timeout)
            self._configure_socket()
            self._connection = self._create_openssl_connection()
            self._configure_connection()
            self._connect_and_handshake()
            if LOG.isEnabledFor(logging.DEBUG):
                LOG.debug("SSL handshaking to %r completed", self.address)
        except:
            LOG.warning("Failed to establish socket/SSL connection to %r", self.address, exc_info=True)
            self.close()
            raise

    def _create_socket(self, timeout):
        """ Create new plain TCP socket. """
        return socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    def _configure_socket(self):
        """ Hook to configure socket parameters. """
        pass

    def _create_openssl_connection(self):
        """ Create new OpenSSL connection. """
        ctx = self.certificate.get_context()
        if self._timeout is not None:
            ctx.set_timeout(int(self._timeout))
        return OpenSSL.SSL.Connection(ctx, self._socket)

    def _configure_connection(self):
        """ Hookt to configure SSL connection. """
        pass

    def _connect_and_handshake(self):
        """ Connect to APNs and SSL handshake. """
        self._connection.connect(self.address)
        # NOTE: if we set any timeout on the underlying socket, then OpenSSL wrapper
        # will always immediately fail with WantReadError without even attempting
        # to block for that timeout. So we use an infinitelly blocking socket here,
        # but we have set timeout on context object and we hope it works.
        self._connection.setblocking(1)
        self._connection.do_handshake()

    def closed(self):
        """ Returns True if this connection is explicitly closed with :func:`close` call. """
        return self._socket is None

    def close(self):
        """ Close this connection. """
        if self._socket is not None:
            if self._connection is not None:
                try:
                    # tell SSL socket we are done
                    self._connection.shutdown()
                except:
                    pass
                try:
                    # free SSL related resources
                    self._connection.close()
                except:
                    pass

            try:
                # shutdown IO
                self._socket.shutdown(socket.SHUT_RDWR)
            except:
                pass
            try:
                # close socket
                self._socket.close()
            except:
                pass

            self._socket = None
            self._connection = None

    def reset(self):
        """ Flushes read buffer. """
        to_skip = self._connection.pending()
        if to_skip > 0:
            self._connection.recv(to_skip)

    def write(self, data, timeout):
        """ Write chunk of data. """
        if self.closed():
            raise IOError("Connection is closed")

        # FIXME: pyOpenSSL can't block with the socket if timeout is set to
        # any value except None. So it is either block forewer or immediate
        # WantReadError without waiting at all. we have to use select() for
        # blocking.
        self._connection.setblocking(1)
        self._socket.settimeout(timeout)  # None will set blocking mode
        waited = 0
        while True:
            try:
                # usually socket accepts any data regardless of network state.
                # it simply stays in the output buffer, which we can't flush
                # unless we disable Nagle algorithm entirely.
                if timeout is not None:
                    before = time.time()
                    sent = self._connection.send(data)
                    waited += time.time() - before
                else: # save the time syscalls
                    sent = self._connection.send(data)

                if sent == len(data):
                    return
                elif sent > 0:
                    data = data[sent:]
                    if timeout is not None:
                        # in case pyOpenSSL starts to actually wait timeouts
                        if timeout - waited >= 0:
                            self._socket.settimeout(timeout - waited)
                            # updated timeout, try again
                            continue
                    else:
                        # infinite timeout, keep trying
                        continue
            except OpenSSL.SSL.WantWriteError:
                # looks like we are in blocking mode, but pyOpenSSL doesn't want
                # to block or we are in a really short timeout.
                if timeout is not None:
                    waited += time.time() - before
                    if timeout - waited > 0:
                        before = time.time()
                        _, canwrite, _ = select.select((), (self._socket, ), (), timeout - waited)
                        waited += time.time() - before
                        if canwrite and timeout - waited >= 0:
                            # buffer ready for writing, try again
                            self._socket.settimeout(timeout - waited)
                            continue

            # we come here if send() blocking works and we waited more than timeout
            # or send() blocking doesn't work and we waited more than timeout
            # or if timeout is None and we still got WantWriteError
            LOG.warning("Write timeout %r it too short for buffer %s", timeout, len(data))
            raise IOError("Timeout exceeded")

    def peek(self, size):
        """ Peek chunk of data from the read buffer. """
        if self.closed():
            return None

        self._connection.setblocking(1)
        pending = self._connection.pending()
        if pending > 0:
            return self._connection.recv(min(pending, size))
        else:
            # do not trust pyOpenSSL, that piece of crap never sets pending buffer.
            canread, _, _ = select.select((self._socket, ), (), (), 0) # poll
            if canread:
                self._connection.setblocking(0)
                self._socket.settimeout(0)
                try:
                    ret = self._connection.recv(size)
                    if not ret:
                        ret = None

                    return ret
                except OpenSSL.SSL.ZeroReturnError:
                    # nice end of stream
                    return None
                except OpenSSL.SSL.WantReadError:
                    # no data is available yet
                    return None

        return None

    def read(self, size, timeout):
        """ Read chunk of data. """
        if self.closed():
            return None

        self._connection.setblocking(1)
        self._socket.settimeout(timeout)  # None will set blocking mode
        waited = 0
        while True:
            try:
                if timeout is not None:
                    before = time.time()
                    ret = self._connection.recv(size)
                    waited += time.time() - before
                else: # save the time syscalls
                    ret = self._connection.recv(size)

                if not ret:
                    # in case recv() responds with empty string
                    ret = None

                return ret
            except OpenSSL.SSL.ZeroReturnError:
                # nice end of stream
                return None
            except OpenSSL.SSL.WantReadError:
                # either pyOpenSSL is failing without even trying to block
                # or we really exceeded the timeout
                if timeout is not None:
                    waited += time.time() - before
                    if timeout - waited > 0:
                        before = time.time()
                        canread, _, _ = select.select((self._socket, ), (), (), timeout - waited)
                        waited += time.time() - before
                        if canread and timeout - waited >= 0:
                            # buffer ready for writing, try again
                            self._socket.settimeout(timeout - waited)
                            continue

            # we come here if read raises WantReadError, which means normal blocking
            # doesn't work or we really exceeded the timeout; timeout is None
            # and we still got WantReadError, which is strange; we waited all
            # time with select()
            #LOG.warning("Read timeout %r it too short for buffer %s", timeout, size)
            raise IOError("Timeout exceeded")

class Backend(BaseBackend):
    """ IO backend based on POSIX sockets. """
    # no, we can't detect if socket is closed without IO
    can_detect_close = False
    # raw connection implementation
    connection_class = Connection

    def get_new_connection(self, address, certificate, timeout=None):
        """ Open a new connection.

            :Arguments:
                - address (tuple): target (host, port).
                - certificate (:class:`Certificate`): certificate instance.
                - timeout (float): connection timeout in seconds
        """
        return self.connection_class(address, certificate, timeout=timeout)

    def get_certificate(self, cert_params):
        """ Create/load certificate from parameters. """
        return Certificate(**cert_params)

    def create_lock(self):
        """ Provides semaphore with ``threading.Lock`` interface. """
        return _threading.Lock()
<|endoftext|>


<commit_before>import editdistance
import requests
import logging

from util import send_email

logger = logging.getLogger('checker')

tags_to_check = [
    'name',
    'alt_name',
    'loc_name',
    'official_name',
    'short_name',
]
radius_meters = 500.0

def foursquare_checkin_has_matches(checkin, user):
    venue = checkin.get('venue')
    venue_name = venue.get('name')

    categories = venue.get('categories')
    for category in categories:
        if category.get('name').endswith('(private)'):
            logger.info("Skipping checkin at private venue")
            return

    user_email = user.get('contact', {}).get('email')
    if not user_email:
        logger.warn("This checkin didn't have a user email, so I didn't do anything")
        return

    query_parts = []
    for t in tags_to_check:
        for i in ('node', 'way', 'relation'):
            query_part = '{prim_type}["{tag}"](around:{radius},{lat},{lng});'.format(
                prim_type=i,
                tag=t,
                radius=radius_meters,
                lat=venue.get('location').get('lat'),
                lng=venue.get('location').get('lng'),
            )
            query_parts.append(query_part)

    query = '[out:json][timeout:5];({});out body;'.format(
            ''.join(query_parts),
        )

    response = requests.post('https://overpass-api.de/api/interpreter', data=query)

    response.raise_for_status()

    osm = response.json()
    elements = osm.get('elements')

    def match_amount(osm_obj):
        osm_name = None
        tags = osm_obj.get('tags')
        for t in tags_to_check:
            osm_name = tags.get(t)
            if osm_name:
                break

        if not osm_name:
            logger.warn("OSM object %s/%s matched but no name tags matched", osm_obj['type'], osm_obj['id'])
            return

        distance = editdistance.eval(venue_name, osm_name)
        edit_pct = float(distance) / max(len(venue_name), len(osm_name))

        return edit_pct

    # Attach match score to each element with a tuple
    potential_matches = [(match_amount(elem), elem) for elem in elements]
    # Sort the tuples based on their match score
    potential_matches = sorted(potential_matches, key=lambda e: e[0])
    # Only pay attention to the tuples that are decent matches
    potential_matches = filter(lambda p: p[0] > 50, potential_matches)

    if not potential_matches:
        logger.info("No matches! Send an e-mail.")

        message = u"""Hi {name},

You checked in at {venue_name} on Foursquare but that location doesn't seem to exist in OpenStreetMap. You should consider adding it!

In fact, here's a direct link to the area in your favorite editor:
https://www.openstreetmap.org/edit?zoom=17&lat={mlat}&lon={mlon}

To remind you where you went, here's a link to your checkin. Remember that you should not copy from external sources (like Foursquare) when editing.
https://foursquare.com/user/{user_id}/checkin/{checkin_id}

-Checkin Checker
(Reply to this e-mail for feedback/questions. Uninstall at https://foursquare.com/settings/connections to stop these e-mails.)""".format(
            name=user.get('firstName', 'Friend'),
            venue_name=venue_name,
            user_id=user['id'],
            checkin_id=checkin['id'],
            mlat=round(venue.get('location').get('lat'), 6),
            mlon=round(venue.get('location').get('lng'), 6),
            email=user_email,
        )
        send_email(user_email, "Your Recent Foursquare Checkin Isn't On OpenStreetMap", message)
    else:
        logger.info(u"Matches: {}".format(u', '.join(map(lambda i: '{}/{} ({:0.2f})'.format(i[1]['type'], i[1]['id'], i[0]), potential_matches))))
        best_match_score, best_match = potential_matches[0]

        if best_match_score > 70:
            logger.info(u"A really great match found: %s/%s (%0.2f)", best_match['type'], best_match['id'], best_match_score)

            tags = best_match['tags']
            questions = []
            if 'addr:housenumber' in tags:
                questions.append(" - Is the housenumber still '{}'?".format(tags['addr:housenumber']))
            else:
                questions.append(" - What is the housenumber?")
            if 'addr:street' in tags:
                questions.append(" - Is the venue still on '{}'?".format(tags['addr:street']))
            else:
                questions.append(" - What is the street name?")
            if 'phone' in tags:
                questions.append(" - Is the phone number still '{}'?".format(tags['phone']))
            else:
                questions.append(" - What is the phone number?")

            message = u"""Hi {name},

Your recent checkin to {venue_name} seems to match something in OpenStreetMap. While you're visiting this place, try collecting these missing attributes for OpenStreetMap:

{questions}

If you want, you can reply to this email and Ian will make these changes, or you can save your email as a draft/note for yourself later.

If you'd like to edit the OSM object directly, use this edit link:
https://www.openstreetmap.org/edit?{osm_type}={osm_id}

To remind you where you went, here's a link to your checkin. Remember that you should not copy from external sources (like Foursquare) when editing.
https://foursquare.com/user/{user_id}/checkin/{checkin_id}

-Checkin Checker
(Reply to this e-mail for feedback/questions. Uninstall at https://foursquare.com/settings/connections to stop these e-mails.)""".format(
                name=user.get('firstName', 'Friend'),
                venue_name=venue_name,
                user_id=user['id'],
                checkin_id=checkin['id'],
                mlat=round(venue.get('location').get('lat'), 6),
                mlon=round(venue.get('location').get('lng'), 6),
                osm_type=best_match['type'],
                osm_id=best_match['id'],
                questions='\n'.join(questions),
                email=user_email,
            )

            send_email(user_email, "Your Recent Foursquare Checkin Is On OpenStreetMap!", message)
<commit_msg>Edit at z19, round overpass lat/lon, log overpass query<commit_after>import editdistance
import requests
import logging

from util import send_email

logger = logging.getLogger('checker')

tags_to_check = [
    'name',
    'alt_name',
    'loc_name',
    'official_name',
    'short_name',
]
radius_meters = 500.0

def foursquare_checkin_has_matches(checkin, user):
    venue = checkin.get('venue')
    venue_name = venue.get('name')

    categories = venue.get('categories')
    for category in categories:
        if category.get('name').endswith('(private)'):
            logger.info("Skipping checkin at private venue")
            return

    user_email = user.get('contact', {}).get('email')
    if not user_email:
        logger.warn("This checkin didn't have a user email, so I didn't do anything")
        return

    query_parts = []
    for t in tags_to_check:
        for i in ('node', 'way', 'relation'):
            query_part = '{prim_type}["{tag}"](around:{radius},{lat},{lng});'.format(
                prim_type=i,
                tag=t,
                radius=radius_meters,
                lat=round(venue.get('location').get('lat'), 6),
                lng=round(venue.get('location').get('lng'), 6),
            )
            query_parts.append(query_part)

    query = '[out:json][timeout:5];({});out body;'.format(
            ''.join(query_parts),
        )

    logger.info("Querying Overpass with: %s", query)

    response = requests.post('https://overpass-api.de/api/interpreter', data=query)

    response.raise_for_status()

    osm = response.json()
    elements = osm.get('elements')

    def match_amount(osm_obj):
        osm_name = None
        tags = osm_obj.get('tags')
        for t in tags_to_check:
            osm_name = tags.get(t)
            if osm_name:
                break

        if not osm_name:
            logger.warn("OSM object %s/%s matched but no name tags matched", osm_obj['type'], osm_obj['id'])
            return

        distance = editdistance.eval(venue_name, osm_name)
        edit_pct = float(distance) / max(len(venue_name), len(osm_name))

        return edit_pct

    # Attach match score to each element with a tuple
    potential_matches = [(match_amount(elem), elem) for elem in elements]
    # Sort the tuples based on their match score
    potential_matches = sorted(potential_matches, key=lambda e: e[0])
    # Only pay attention to the tuples that are decent matches
    potential_matches = filter(lambda p: p[0] > 50, potential_matches)

    if not potential_matches:
        logger.info("No matches! Send an e-mail.")

        message = u"""Hi {name},

You checked in at {venue_name} on Foursquare but that location doesn't seem to exist in OpenStreetMap. You should consider adding it!

In fact, here's a direct link to the area in your favorite editor:
https://www.openstreetmap.org/edit?zoom=19&lat={mlat}&lon={mlon}

To remind you where you went, here's a link to your checkin. Remember that you should not copy from external sources (like Foursquare) when editing.
https://foursquare.com/user/{user_id}/checkin/{checkin_id}

-Checkin Checker
(Reply to this e-mail for feedback/questions. Uninstall at https://foursquare.com/settings/connections to stop these e-mails.)""".format(
            name=user.get('firstName', 'Friend'),
            venue_name=venue_name,
            user_id=user['id'],
            checkin_id=checkin['id'],
            mlat=round(venue.get('location').get('lat'), 6),
            mlon=round(venue.get('location').get('lng'), 6),
            email=user_email,
        )
        send_email(user_email, "Your Recent Foursquare Checkin Isn't On OpenStreetMap", message)
    else:
        logger.info(u"Matches: {}".format(u', '.join(map(lambda i: '{}/{} ({:0.2f})'.format(i[1]['type'], i[1]['id'], i[0]), potential_matches))))
        best_match_score, best_match = potential_matches[0]

        if best_match_score > 70:
            logger.info(u"A really great match found: %s/%s (%0.2f)", best_match['type'], best_match['id'], best_match_score)

            tags = best_match['tags']
            questions = []
            if 'addr:housenumber' in tags:
                questions.append(" - Is the housenumber still '{}'?".format(tags['addr:housenumber']))
            else:
                questions.append(" - What is the housenumber?")
            if 'addr:street' in tags:
                questions.append(" - Is the venue still on '{}'?".format(tags['addr:street']))
            else:
                questions.append(" - What is the street name?")
            if 'phone' in tags:
                questions.append(" - Is the phone number still '{}'?".format(tags['phone']))
            else:
                questions.append(" - What is the phone number?")

            message = u"""Hi {name},

Your recent checkin to {venue_name} seems to match something in OpenStreetMap. While you're visiting this place, try collecting these missing attributes for OpenStreetMap:

{questions}

If you want, you can reply to this email and Ian will make these changes, or you can save your email as a draft/note for yourself later.

If you'd like to edit the OSM object directly, use this edit link:
https://www.openstreetmap.org/edit?{osm_type}={osm_id}

To remind you where you went, here's a link to your checkin. Remember that you should not copy from external sources (like Foursquare) when editing.
https://foursquare.com/user/{user_id}/checkin/{checkin_id}

-Checkin Checker
(Reply to this e-mail for feedback/questions. Uninstall at https://foursquare.com/settings/connections to stop these e-mails.)""".format(
                name=user.get('firstName', 'Friend'),
                venue_name=venue_name,
                user_id=user['id'],
                checkin_id=checkin['id'],
                mlat=round(venue.get('location').get('lat'), 6),
                mlon=round(venue.get('location').get('lng'), 6),
                osm_type=best_match['type'],
                osm_id=best_match['id'],
                questions='\n'.join(questions),
                email=user_email,
            )

            send_email(user_email, "Your Recent Foursquare Checkin Is On OpenStreetMap!", message)
<|endoftext|>