<commit_before>import json
import logging

from django.http import HttpResponse, Http404
from django.shortcuts import get_object_or_404
from django.views.decorators.csrf import csrf_exempt
from django.core.exceptions import ObjectDoesNotExist as DoesNotExist
from django.db.utils import IntegrityError
from cbagent.collectors import Collector

from cbmonitor import models
from cbmonitor import forms
from cbmonitor.lib import Plotter

logger = logging.getLogger(__name__)

@csrf_exempt
def dispatcher(request, path):
    handler = {
        "add_cluster": add_cluster,
        "add_server": add_server,
        "add_bucket": add_bucket,
        "delete_cluster": delete_cluster,
        "delete_server": delete_server,
        "delete_bucket": delete_bucket,
        "get_tree_data": get_tree_data,
        "get_clusters": get_clusters,
        "get_servers": get_servers,
        "get_buckets": get_buckets,
        "get_metrics_and_events": get_metrics_and_events,
        "add_metric_or_event": add_metric_or_event,
        "add_snapshot": add_shapshot,
        "get_snapshots": get_snapshots,
        "get_collectors": get_collectors,
        "update_collectors": update_collectors,
        "plot": plot,
        "pdf": pdf,
    }.get(path)
    if handler:
        return handler(request)
    else:
        return HttpResponse(content='Wrong path: {0}'.format(path), status=404)

class ValidationError(Exception):

    def __init__(self, form):
        self.error = dict((item[0], item[1][0]) for item in form.errors.items())

    def __str__(self):
        return json.dumps(self.error)

def form_validation(method):
    def wrapper(*args, **kargs):
        try:
            response = method(*args, **kargs)
        except Http404, error:
            logger.warn(error)
            return HttpResponse(content=error, status=404)
        except ValidationError, error:
            logger.warn(error)
            return HttpResponse(content=error, status=400)
        except IntegrityError, error:
            logger.warn(error)
            return HttpResponse(content=error, status=400)
        else:
            return response or HttpResponse(content="Success")
    return wrapper

@form_validation
def add_cluster(request):
    form = forms.AddClusterForm(request.POST)
    if form.is_valid():
        if form.cleaned_data.get("master_node"):
            collector = Collector(form.settings)
            for bucket in collector._get_buckets():
                collector.mc.add_bucket(bucket)
            for node in collector._get_nodes():
                collector.mc.add_server(node)
        cluster = form.save()
        for collector_name in models.CollectorName.objects.all():
            models.Collector(name=collector_name, cluster=cluster).save()
    else:
        raise ValidationError(form)

@form_validation
def add_server(request):
    form = forms.AddServerForm(request.POST)
    if form.is_valid():
        form.save()
    else:
        raise ValidationError(form)

@form_validation
def add_bucket(request):
    form = forms.AddBucketForm(request.POST)
    if form.is_valid():
        form.save()
    else:
        raise ValidationError(form)

@form_validation
def delete_cluster(request):
    cluster = get_object_or_404(models.Cluster, name=request.POST["name"])

    form = forms.DeleteClusterForm(request.POST, instance=cluster)
    if form.is_valid():
        cluster.delete()
    else:
        raise ValidationError(form)

@form_validation
def delete_server(request):
    cluster = get_object_or_404(models.Cluster, name=request.POST["cluster"])
    server = get_object_or_404(models.Server, address=request.POST["address"],
                               cluster=cluster)

    form = forms.DeleteServerForm(request.POST, instance=server)
    if form.is_valid():
        server.delete()
    else:
        raise ValidationError(form)

@form_validation
def delete_bucket(request):
    cluster = get_object_or_404(models.Cluster, name=request.POST["cluster"])
    bucket = get_object_or_404(models.Bucket, name=request.POST["name"],
                               cluster=cluster)

    form = forms.DeleteBucketForm(request.POST, instance=bucket)
    if form.is_valid():
        bucket.delete()
    else:
        raise ValidationError(form)

def get_tree_data(request):
    """"Generate json data for jstree"""
    response = []

    for cluster in models.Cluster.objects.all():
        cluster_obj = {
            "data": cluster.name,
            "attr": {"class": "cluster", "id": cluster.name},
            "children": [
                {"data": "Servers",
                 "attr": {"class": "servers"},
                 "children": []},
                {"data": "Buckets",
                 "attr": {"class": "buckets"},
                 "children": []}]
        }
        for server in models.Server.objects.filter(cluster=cluster):
            server_obj = {
                "data": server.address,
                "attr": {"class": "server", "id": server.address}
            }
            cluster_obj["children"][0]["children"].append(server_obj)
        for bucket in models.Bucket.objects.filter(cluster=cluster):
            bucket_obj = {
                "data": bucket.name,
                "attr": {"class": "bucket", "id": bucket.name},
            }
            cluster_obj["children"][1]["children"].append(bucket_obj)

        response.append(cluster_obj)

    return HttpResponse(content=json.dumps(response))

@form_validation
def get_clusters(request):
    """Get list of active clusters"""
    clusters = [c.name for c in models.Cluster.objects.all()]
    content = json.dumps(sorted(clusters))
    return HttpResponse(content)

@form_validation
def get_servers(request):
    """Get list of active servers for given cluster"""
    form = forms.GetServersForm(request.GET)
    if form.is_valid():
        try:
            cluster = models.Cluster.objects.get(name=request.GET["cluster"])
            servers = models.Server.objects.filter(cluster=cluster).values()
            servers = [s["address"] for s in servers]
        except DoesNotExist:
            servers = []
    else:
        servers = []
    content = json.dumps(sorted(servers))
    return HttpResponse(content)

@form_validation
def get_buckets(request):
    """Get list of active buckets for given server"""
    form = forms.GetBucketsForm(request.GET)
    if form.is_valid():
        try:
            cluster = models.Cluster.objects.get(name=request.GET["cluster"])
            buckets = models.Bucket.objects.filter(cluster=cluster).values()
            buckets = [b["name"] for b in buckets]
        except DoesNotExist:
            buckets = []
    else:
        buckets = []
    content = json.dumps(sorted(buckets))
    return HttpResponse(content)

@form_validation
def get_metrics_and_events(request):
    """Get list of metrics or events for given cluster, server and bucket"""
    form = forms.GetMetricsAndEvents(request.GET)

    if form.is_valid():
        try:
            observables = models.Observable.objects.filter(**form.params).values()
            observables = [{"name": o["name"], "collector": o["collector"]}
                           for o in observables]
        except DoesNotExist:
            observables = []
    else:
        observables = []
    content = json.dumps(sorted(observables))
    return HttpResponse(content)

@form_validation
def add_metric_or_event(request):
    form = forms.AddMetricsAndEvents(request.POST)
    if form.is_valid():
        observable = form.save(commit=False)
        observable.bucket = form.cleaned_data["bucket"]
        observable.server = form.cleaned_data["server"]
        observable.save()
    else:
        raise ValidationError(form)

@form_validation
def add_shapshot(request):
    form = forms.AddSnapshot(request.POST)
    if form.is_valid():
        form.save()
    else:
        raise ValidationError(form)

@form_validation
def get_snapshots(request):
    snapshots = [s.name for s in models.Snapshot.objects.all()]
    content = json.dumps(snapshots)
    return HttpResponse(content)

@form_validation
def update_collectors(request):
    instance = get_object_or_404(models.Collector, name=request.POST["name"],
                                 cluster=request.POST["cluster"])
    form = forms.UpdateCollectors(request.POST, instance=instance)
    if form.is_valid():
        form.save()
    else:
        raise ValidationError(form)

@form_validation
def get_collectors(request):
    collectors = []

    form = forms.GetCollectors(request.GET)
    if form.is_valid():
        objects = models.Collector.objects.filter(**form.cleaned_data).values()
        for collector in objects:
            collectors.append({"name": collector["name_id"],
                               "interval": collector["interval"],
                               "enabled": collector["enabled"]})

    content = json.dumps(collectors)
    return HttpResponse(content)

def plot(request):
    snapshot = request.POST["snapshot"]
    plotter = Plotter()
    response = plotter.plot(snapshot)
    return HttpResponse(content=json.dumps(response))

def pdf(request):
    snapshot = request.POST["snapshot"]
    plotter = Plotter()
    response = plotter.pdf(snapshot)
    return HttpResponse(response)
<commit_msg>save cluster object before server/bucket metadata update<commit_after>import json
import logging

from django.http import HttpResponse, Http404
from django.shortcuts import get_object_or_404
from django.views.decorators.csrf import csrf_exempt
from django.core.exceptions import ObjectDoesNotExist as DoesNotExist
from django.db.utils import IntegrityError
from cbagent.collectors import Collector

from cbmonitor import models
from cbmonitor import forms
from cbmonitor.lib import Plotter

logger = logging.getLogger(__name__)

@csrf_exempt
def dispatcher(request, path):
    handler = {
        "add_cluster": add_cluster,
        "add_server": add_server,
        "add_bucket": add_bucket,
        "delete_cluster": delete_cluster,
        "delete_server": delete_server,
        "delete_bucket": delete_bucket,
        "get_tree_data": get_tree_data,
        "get_clusters": get_clusters,
        "get_servers": get_servers,
        "get_buckets": get_buckets,
        "get_metrics_and_events": get_metrics_and_events,
        "add_metric_or_event": add_metric_or_event,
        "add_snapshot": add_shapshot,
        "get_snapshots": get_snapshots,
        "get_collectors": get_collectors,
        "update_collectors": update_collectors,
        "plot": plot,
        "pdf": pdf,
    }.get(path)
    if handler:
        return handler(request)
    else:
        return HttpResponse(content='Wrong path: {0}'.format(path), status=404)

class ValidationError(Exception):

    def __init__(self, form):
        self.error = dict((item[0], item[1][0]) for item in form.errors.items())

    def __str__(self):
        return json.dumps(self.error)

def form_validation(method):
    def wrapper(*args, **kargs):
        try:
            response = method(*args, **kargs)
        except Http404, error:
            logger.warn(error)
            return HttpResponse(content=error, status=404)
        except ValidationError, error:
            logger.warn(error)
            return HttpResponse(content=error, status=400)
        except IntegrityError, error:
            logger.warn(error)
            return HttpResponse(content=error, status=400)
        else:
            return response or HttpResponse(content="Success")
    return wrapper

@form_validation
def add_cluster(request):
    form = forms.AddClusterForm(request.POST)
    if form.is_valid():
        cluster = form.save()
        if form.cleaned_data.get("master_node"):
            collector = Collector(form.settings)
            for bucket in collector._get_buckets():
                collector.mc.add_bucket(bucket)
            for node in collector._get_nodes():
                collector.mc.add_server(node)
        for collector_name in models.CollectorName.objects.all():
            models.Collector(name=collector_name, cluster=cluster).save()
    else:
        raise ValidationError(form)

@form_validation
def add_server(request):
    form = forms.AddServerForm(request.POST)
    if form.is_valid():
        form.save()
    else:
        raise ValidationError(form)

@form_validation
def add_bucket(request):
    form = forms.AddBucketForm(request.POST)
    if form.is_valid():
        form.save()
    else:
        raise ValidationError(form)

@form_validation
def delete_cluster(request):
    cluster = get_object_or_404(models.Cluster, name=request.POST["name"])

    form = forms.DeleteClusterForm(request.POST, instance=cluster)
    if form.is_valid():
        cluster.delete()
    else:
        raise ValidationError(form)

@form_validation
def delete_server(request):
    cluster = get_object_or_404(models.Cluster, name=request.POST["cluster"])
    server = get_object_or_404(models.Server, address=request.POST["address"],
                               cluster=cluster)

    form = forms.DeleteServerForm(request.POST, instance=server)
    if form.is_valid():
        server.delete()
    else:
        raise ValidationError(form)

@form_validation
def delete_bucket(request):
    cluster = get_object_or_404(models.Cluster, name=request.POST["cluster"])
    bucket = get_object_or_404(models.Bucket, name=request.POST["name"],
                               cluster=cluster)

    form = forms.DeleteBucketForm(request.POST, instance=bucket)
    if form.is_valid():
        bucket.delete()
    else:
        raise ValidationError(form)

def get_tree_data(request):
    """"Generate json data for jstree"""
    response = []

    for cluster in models.Cluster.objects.all():
        cluster_obj = {
            "data": cluster.name,
            "attr": {"class": "cluster", "id": cluster.name},
            "children": [
                {"data": "Servers",
                 "attr": {"class": "servers"},
                 "children": []},
                {"data": "Buckets",
                 "attr": {"class": "buckets"},
                 "children": []}]
        }
        for server in models.Server.objects.filter(cluster=cluster):
            server_obj = {
                "data": server.address,
                "attr": {"class": "server", "id": server.address}
            }
            cluster_obj["children"][0]["children"].append(server_obj)
        for bucket in models.Bucket.objects.filter(cluster=cluster):
            bucket_obj = {
                "data": bucket.name,
                "attr": {"class": "bucket", "id": bucket.name},
            }
            cluster_obj["children"][1]["children"].append(bucket_obj)

        response.append(cluster_obj)

    return HttpResponse(content=json.dumps(response))

@form_validation
def get_clusters(request):
    """Get list of active clusters"""
    clusters = [c.name for c in models.Cluster.objects.all()]
    content = json.dumps(sorted(clusters))
    return HttpResponse(content)

@form_validation
def get_servers(request):
    """Get list of active servers for given cluster"""
    form = forms.GetServersForm(request.GET)
    if form.is_valid():
        try:
            cluster = models.Cluster.objects.get(name=request.GET["cluster"])
            servers = models.Server.objects.filter(cluster=cluster).values()
            servers = [s["address"] for s in servers]
        except DoesNotExist:
            servers = []
    else:
        servers = []
    content = json.dumps(sorted(servers))
    return HttpResponse(content)

@form_validation
def get_buckets(request):
    """Get list of active buckets for given server"""
    form = forms.GetBucketsForm(request.GET)
    if form.is_valid():
        try:
            cluster = models.Cluster.objects.get(name=request.GET["cluster"])
            buckets = models.Bucket.objects.filter(cluster=cluster).values()
            buckets = [b["name"] for b in buckets]
        except DoesNotExist:
            buckets = []
    else:
        buckets = []
    content = json.dumps(sorted(buckets))
    return HttpResponse(content)

@form_validation
def get_metrics_and_events(request):
    """Get list of metrics or events for given cluster, server and bucket"""
    form = forms.GetMetricsAndEvents(request.GET)

    if form.is_valid():
        try:
            observables = models.Observable.objects.filter(**form.params).values()
            observables = [{"name": o["name"], "collector": o["collector"]}
                           for o in observables]
        except DoesNotExist:
            observables = []
    else:
        observables = []
    content = json.dumps(sorted(observables))
    return HttpResponse(content)

@form_validation
def add_metric_or_event(request):
    form = forms.AddMetricsAndEvents(request.POST)
    if form.is_valid():
        observable = form.save(commit=False)
        observable.bucket = form.cleaned_data["bucket"]
        observable.server = form.cleaned_data["server"]
        observable.save()
    else:
        raise ValidationError(form)

@form_validation
def add_shapshot(request):
    form = forms.AddSnapshot(request.POST)
    if form.is_valid():
        form.save()
    else:
        raise ValidationError(form)

@form_validation
def get_snapshots(request):
    snapshots = [s.name for s in models.Snapshot.objects.all()]
    content = json.dumps(snapshots)
    return HttpResponse(content)

@form_validation
def update_collectors(request):
    instance = get_object_or_404(models.Collector, name=request.POST["name"],
                                 cluster=request.POST["cluster"])
    form = forms.UpdateCollectors(request.POST, instance=instance)
    if form.is_valid():
        form.save()
    else:
        raise ValidationError(form)

@form_validation
def get_collectors(request):
    collectors = []

    form = forms.GetCollectors(request.GET)
    if form.is_valid():
        objects = models.Collector.objects.filter(**form.cleaned_data).values()
        for collector in objects:
            collectors.append({"name": collector["name_id"],
                               "interval": collector["interval"],
                               "enabled": collector["enabled"]})

    content = json.dumps(collectors)
    return HttpResponse(content)

def plot(request):
    snapshot = request.POST["snapshot"]
    plotter = Plotter()
    response = plotter.plot(snapshot)
    return HttpResponse(content=json.dumps(response))

def pdf(request):
    snapshot = request.POST["snapshot"]
    plotter = Plotter()
    response = plotter.pdf(snapshot)
    return HttpResponse(response)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import speech_recognition as sr
from secrets import *

# obtain path to "english.wav" in the same folder as this script
from os import path
AUDIO_FILE = path.join(path.dirname(path.realpath(__file__)), sample_wav)

# use the audio file as the audio source
r = sr.Recognizer()
with sr.AudioFile(AUDIO_FILE) as source:
    audio = r.record(source) # read the entire audio file

# recognize speech using Microsoft Bing Voice Recognition
BING_KEY = bing_speech_api_key # Microsoft Bing Voice Recognition API keys 32-character lowercase hexadecimal strings
try:
    result = r.recognize_bing(audio, key=BING_KEY, language = "en-US", show_all = False)
    print("Microsoft Bing Voice Recognition thinks you said " + result)
except sr.UnknownValueError:
    print("Microsoft Bing Voice Recognition could not understand audio")
except sr.RequestError as e:
    print("Could not request results from Microsoft Bing Voice Recognition service; {0}".format(e))

# def text2int(textnum, numwords={}):
#     if not numwords:
#       units = [
#         "zero", "one", "two", "three", "four", "five", "six", "seven", "eight",
#         "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen",
#         "sixteen", "seventeen", "eighteen", "nineteen",
#       ]

#       tens = ["", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"]

#       scales = ["hundred", "thousand", "million", "billion", "trillion"]

#       numwords["and"] = (1, 0)
#       for idx, word in enumerate(units):    numwords[word] = (1, idx)
#       for idx, word in enumerate(tens):     numwords[word] = (1, idx * 10)
#       for idx, word in enumerate(scales):   numwords[word] = (10 ** (idx * 3 or 2), 0)

#     current = result = 0
#     for word in textnum.split()

#         scale, increment = numwords[word]
#         current = current * scale + increment
#         if scale > 100:
#             result += current
#             current = 0

#     return result + current

# print(text2int(result))

# Microsoft Bing Voice Recognition thinks you said ford junk vernon miles at eight zero zero bella rosa street suite three zero zero san antonio texas seven eight two zero seven your next individual hearing date is april thirteen two thousand seventeen at one PM before it jumps vernon miles at eight zero zero delarossa street suite three zero zero san antonio texas seven eight two zero seven william next hearing date press one for case processing information<commit_msg>ceating a class for Bingtranscribe<commit_after>#!/usr/bin/env python3

import speech_recognition as sr
from secrets import *

# obtain path to "english.wav" in the same folder as this script
from os import path
class BingTranscribe(object):

    def __init__(self):
        self.bing_key = bing_speech_api_key
        self.language="en-US"
        self.audio_file = path.join(path.dirname(path.realpath(__file__)), sample_wav)

    def transcribe_audio_object(self, audio_object):
            try:
                r = sr.Recognizer()
                return r.recognize_bing(audio_object, key=self.bing_key, language = self.language, show_all = False)
            except sr.UnknownValueError:
                print("Microsoft Bing Voice Recognition could not understand audio")
            except sr.RequestError as e:
                print("Could not request results from Microsoft Bing Voice Recognition service; {0}".format(e))

    def transcribe_audio_file_path(self, audio_file_path):
        r = sr.Recognizer()
        try:
            with sr.AudioFile(audio_file_path) as source:
                audio = r.record(source)
                return self.transcribe_audio_object(audio)
        except sr.UnknownValueError as e:
            print("{0}".format(e))

foo = BingTranscribe()
result=foo.transcribe_audio_file_path((foo.audio_file))
print(result)

# def text2int(textnum, numwords={}):
#     if not numwords:
#       units = [
#         "zero", "one", "two", "three", "four", "five", "six", "seven", "eight",
#         "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen",
#         "sixteen", "seventeen", "eighteen", "nineteen",
#       ]

#       tens = ["", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"]

#       scales = ["hundred", "thousand", "million", "billion", "trillion"]

#       numwords["and"] = (1, 0)
#       for idx, word in enumerate(units):    numwords[word] = (1, idx)
#       for idx, word in enumerate(tens):     numwords[word] = (1, idx * 10)
#       for idx, word in enumerate(scales):   numwords[word] = (10 ** (idx * 3 or 2), 0)

#     current = result = 0
#     for word in textnum.split()

#         scale, increment = numwords[word]
#         current = current * scale + increment
#         if scale > 100:
#             result += current
#             current = 0

#     return result + current

# print(text2int(result))

# Microsoft Bing Voice Recognition thinks you said ford junk vernon miles at eight zero zero bella rosa street suite three zero zero san antonio texas seven eight two zero seven your next individual hearing date is april thirteen two thousand seventeen at one PM before it jumps vernon miles at eight zero zero delarossa street suite three zero zero san antonio texas seven eight two zero seven william next hearing date press one for case processing information<|endoftext|>


<commit_before>import os
import warnings

from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.test import TestCase
from django.test.utils import override_settings

from . import DATA_DIR

class BrowserTest(TestCase):
    def test_browser(self):
        url = reverse('graphite.browser.views.browser')
        response = self.client.get(url)
        self.assertContains(response, 'Graphite Browser')

    def test_header(self):
        self.assertEqual(User.objects.count(), 0)
        url = reverse('graphite.browser.views.header')
        response = self.client.get(url)
        self.assertContains(response, 'Graphite Browser Header')

        # Graphite has created a default user
        self.assertEqual(User.objects.get().username, 'default')

    # Django 1.6.1 on python 2.7 throws a few warnings for this test
    # that need to be silenced to calm my inner OCD monster (@SEJeff)
    @override_settings(INDEX_FILE=os.path.join(DATA_DIR, 'index'))
    def test_search(self):
        url = reverse('graphite.browser.views.search')

        with warnings.catch_warnings():
            warnings.simplefilter('ignore', DeprecationWarning)
            response = self.client.post(url)
        self.assertEqual(response.content, '')

        # simple query
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', DeprecationWarning)
            response = self.client.post(url, {'query': 'collectd'})
        self.assertEqual(response.content.split(',')[0],
                         'collectd.test.df-root.df_complex-free')

        # No match
        response = self.client.post(url, {'query': 'other'})
        self.assertEqual(response.content, '')

        # Multiple terms (OR)
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', DeprecationWarning)
            response = self.client.post(url, {'query': 'midterm shortterm'})
        self.assertEqual(response.content.split(','),
                         ['collectd.test.load.load.midterm',
                          'collectd.test.load.load.shortterm'])
<commit_msg>Remove warning silencing code<commit_after>import os

from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.test import TestCase
from django.test.utils import override_settings

from . import DATA_DIR

class BrowserTest(TestCase):
    def test_browser(self):
        url = reverse('graphite.browser.views.browser')
        response = self.client.get(url)
        self.assertContains(response, 'Graphite Browser')

    def test_header(self):
        self.assertEqual(User.objects.count(), 0)
        url = reverse('graphite.browser.views.header')
        response = self.client.get(url)
        self.assertContains(response, 'Graphite Browser Header')

        # Graphite has created a default user
        self.assertEqual(User.objects.get().username, 'default')

    @override_settings(INDEX_FILE=os.path.join(DATA_DIR, 'index'))
    def test_search(self):
        url = reverse('graphite.browser.views.search')

        response = self.client.post(url)
        self.assertEqual(response.content, '')

        # simple query
        response = self.client.post(url, {'query': 'collectd'})
        self.assertEqual(response.content.split(',')[0],
                         'collectd.test.df-root.df_complex-free')

        # No match
        response = self.client.post(url, {'query': 'other'})
        self.assertEqual(response.content, '')

        # Multiple terms (OR)
        response = self.client.post(url, {'query': 'midterm shortterm'})
        self.assertEqual(response.content.split(','),
                         ['collectd.test.load.load.midterm',
                          'collectd.test.load.load.shortterm'])
<|endoftext|>


<commit_before>from django.core.management import BaseCommand

from corehq.apps.change_feed import data_sources, topics
from corehq.apps.change_feed.producer import producer
from corehq.form_processor.utils import should_use_sql_backend
from dimagi.utils.chunked import chunked

from casexml.apps.case.models import CommCareCase
from corehq.doctypemigrations.continuous_migrate import bulk_get_revs
from corehq.apps.hqcase.management.commands.backfill_couch_forms_and_cases import (
    publish_change, create_case_change_meta
)
from pillowtop.feed.interface import ChangeMeta

class Command(BaseCommand):
    """
    Republish case changes. Meant to be used in conjunction with stale_cases_in_es command
            $ ./manage.py republish_couch_case_changes <DOMAIN> <case_ids.txt>
    """

    def add_arguments(self, parser):
        parser.add_argument('domain')
        parser.add_argument('case_ids_file')

    def handle(self, domain, case_ids_file, *args, **options):
        case_ids = _get_case_ids(case_ids_file)
        _publish_cases(domain, case_ids)

def _get_case_ids(case_ids_file):
    with open(case_ids_file, 'r') as f:
        lines = f.readlines()
        return [l.split(',')[0].strip() for l in lines]

def _publish_cases(domain, case_ids):
    if should_use_sql_backend(domain):
        _publish_cases_for_sql(domain, case_ids)
    else:
        _publish_cases_for_couch(domain, case_ids)

def _publish_cases_for_couch(domain, case_ids):
    for ids in chunked(case_ids, 500):
        doc_id_rev_list = bulk_get_revs(CommCareCase.get_db(), ids)
        for doc_id, doc_rev in doc_id_rev_list:
            publish_change(
                create_case_change_meta(domain, doc_id, doc_rev)
            )

def _publish_cases_for_sql(domain, case_ids):
    for case_id in case_ids:
        producer.send_change(
            topics.CASE_SQL,
            _change_meta_for_sql_case(domain, case_id)
        )

def _change_meta_for_sql_case(domain, case_id):
    return ChangeMeta(
        document_id=case_id,
        data_source_type=data_sources.SOURCE_SQL,
        data_source_name=data_sources.CASE_SQL,
        document_type='CommCareCase',
        document_subtype=None,  # should be case.type or form.xmlns, but not used anywhere
        domain=domain,
        is_deletion=False,
    )
<commit_msg>add case types to republish script<commit_after>from collections import defaultdict

from django.core.management import BaseCommand

from corehq.apps.change_feed import data_sources, topics
from corehq.apps.change_feed.producer import producer
from corehq.form_processor.backends.sql.dbaccessors import ShardAccessor
from corehq.form_processor.models import CommCareCaseSQL
from corehq.form_processor.utils import should_use_sql_backend
from corehq.sql_db.util import get_db_aliases_for_partitioned_query
from dimagi.utils.chunked import chunked

from casexml.apps.case.models import CommCareCase
from corehq.doctypemigrations.continuous_migrate import bulk_get_revs
from corehq.apps.hqcase.management.commands.backfill_couch_forms_and_cases import (
    publish_change, create_case_change_meta
)
from pillowtop.feed.interface import ChangeMeta

class Command(BaseCommand):
    """
    Republish case changes. Meant to be used in conjunction with stale_cases_in_es command
            $ ./manage.py republish_couch_case_changes <DOMAIN> <case_ids.txt>
    """

    def add_arguments(self, parser):
        parser.add_argument('domain')
        parser.add_argument('case_ids_file')

    def handle(self, domain, case_ids_file, *args, **options):
        case_ids = _get_case_ids(case_ids_file)
        _publish_cases(domain, case_ids)

def _get_case_ids(case_ids_file):
    with open(case_ids_file, 'r') as f:
        lines = f.readlines()
        return [l.split(',')[0].strip() for l in lines]

def _publish_cases(domain, case_ids):
    if should_use_sql_backend(domain):
        _publish_cases_for_sql(domain, case_ids)
    else:
        _publish_cases_for_couch(domain, case_ids)

def _publish_cases_for_couch(domain, case_ids):
    for ids in chunked(case_ids, 500):
        doc_id_rev_list = bulk_get_revs(CommCareCase.get_db(), ids)
        for doc_id, doc_rev in doc_id_rev_list:
            publish_change(
                create_case_change_meta(domain, doc_id, doc_rev)
            )

def _publish_cases_for_sql(domain, case_ids):
    for id_chunk in chunked(case_ids, 10000):
        # databases will contain a mapping of shard database ids to case_ids in that DB
        databases = ShardAccessor.get_docs_by_database(id_chunk)
        for db, doc_ids in databases.items():
            results = CommCareCaseSQL.objects.using(db).filter(
                case_id__in=doc_ids,
            ).values_list('case_id', 'type')
            # make sure we found the same number of IDs
            assert len(results) == len(doc_ids)
            for case_id, case_type in results:
                print(f'case_id: {case_id} {case_type}')
                producer.send_change(
                    topics.CASE_SQL,
                    _change_meta_for_sql_case(domain, case_id, case_type)
                )

def _change_meta_for_sql_case(domain, case_id, case_type):
    return ChangeMeta(
        document_id=case_id,
        data_source_type=data_sources.SOURCE_SQL,
        data_source_name=data_sources.CASE_SQL,
        document_type='CommCareCase',
        document_subtype=case_type,
        domain=domain,
        is_deletion=False,
    )
<|endoftext|>


<commit_before>import os
import shutil
from zipfile import ZipFile, BadZipFile

import pytest

from webcomix.comic import Comic, SPLASH_SETTINGS
from webcomix.supported_comics import supported_comics
from webcomix.tests.fake_websites.fixture import (
    three_webpages_uri,
    three_webpages_alt_text_uri,
    one_webpage_uri,
)

@pytest.fixture
def cleanup_test_directories():
    if os.path.isfile("xkcd.cbz"):
        os.remove("xkcd.cbz")
    if os.path.isfile("test.cbz"):
        os.remove("test.cbz")
    if os.path.isdir("xkcd"):
        shutil.rmtree("xkcd")
    if os.path.isdir("test"):
        shutil.rmtree("test")
    yield None
    if os.path.isdir("xkcd"):
        shutil.rmtree("xkcd")
    if os.path.isdir("test"):
        shutil.rmtree("test")
    if os.path.isfile("xkcd.cbz"):
        os.remove("xkcd.cbz")
    if os.path.isfile("test.cbz"):
        os.remove("test.cbz")

@pytest.fixture
def fake_downloaded_xkcd_comic():
    if os.path.isfile("xkcd.cbz"):
        os.remove("xkcd.cbz")
    if os.path.isdir("xkcd"):
        shutil.rmtree("xkcd")
    comic = Comic(
        "xkcd",
        "http://xkcd.com/1/",
        "//div[@id='comic']/img/@src",
        "//a[@rel='next']/@href",
        False,
    )
    os.mkdir("xkcd")
    for i in range(1, 6):
        with open("xkcd/{}.txt".format(i), "w") as image_file:
            image_file.write("testing {}".format(i))
    yield comic
    if os.path.isfile("xkcd.cbz"):
        os.remove("xkcd.cbz")
    if os.path.isdir("xkcd"):
        shutil.rmtree("xkcd")

def test_save_image_location():
    assert (
        Comic.save_image_location(
            "http://imgs.xkcd.com/comics/barrel_cropped_(1).jpg", 1, "foo"
        )
        == "foo/1.jpg"
    )
    assert Comic.save_image_location("", 1, "bar") == "bar/1"

def test_make_cbz(fake_downloaded_xkcd_comic):
    fake_downloaded_xkcd_comic.convert_to_cbz()
    with ZipFile("xkcd.cbz") as cbz_file:
        for i in range(1, 6):
            with cbz_file.open("{}.txt".format(i), "r") as image_file:
                assert image_file.read().decode("ascii") == "testing {}".format(i)

def test_make_cbz_corrupted_archive(mocker, capfd, fake_downloaded_xkcd_comic):
    mocker.patch.object(ZipFile, "testzip", return_value=mocker.ANY)
    with pytest.raises(BadZipFile):
        fake_downloaded_xkcd_comic.convert_to_cbz()

def test_download_runs_the_worker(mocker, cleanup_test_directories):
    mock_crawler_running = mocker.patch(
        "webcomix.scrapy.crawler_worker.CrawlerWorker.start"
    )
    comic = Comic(
        "xkcd",
        "http://xkcd.com/1/",
        "//div[@id='comic']//img/@src",
        "//a[@rel='next']/@href",
        False,
    )
    comic.download()
    assert mock_crawler_running.call_count == 1

def test_download_saves_the_files(cleanup_test_directories, three_webpages_uri):
    comic = Comic("test", three_webpages_uri, "//img/@src", "//a/@href")
    comic.download()
    path, dirs, files = next(os.walk("test"))
    assert len(files) == 2

def test_download_with_alt_text_saves_the_text(
    cleanup_test_directories, three_webpages_alt_text_uri
):
    comic = Comic(
        "test", three_webpages_alt_text_uri, "//img/@src", "//a/@href", "//img/@title"
    )
    comic.download()
    path, dirs, files = next(os.walk("test"))
    assert len(files) == 4

def test_download_does_not_add_crawlers_in_main_process(
    mocker, cleanup_test_directories, three_webpages_uri
):
    mocker.patch("webcomix.scrapy.crawler_worker.CrawlerWorker.start")
    mock_add_to_crawl = mocker.patch("scrapy.crawler.Crawler.crawl")
    comic = Comic("test", three_webpages_uri, "//img/@src", "//a/@href")
    comic.download()
    assert mock_add_to_crawl.call_count == 0

def test_convert_to_cbz_adds_all_files_to_cbz(
    cleanup_test_directories, three_webpages_uri
):
    comic = Comic("test", three_webpages_uri, "//img/@src", "//a/@href")
    comic.download()
    comic.convert_to_cbz()
    with ZipFile("{}.cbz".format(comic.name), mode="r") as cbz_file:
        assert len(cbz_file.infolist()) == 2

def test_verify_xpath():
    comic = Comic("xkcd", *supported_comics["xkcd"])
    assert comic.verify_xpath() == [
        {
            "page": 1,
            "url": "https://xkcd.com/1/",
            "image_urls": ["https://imgs.xkcd.com/comics/barrel_cropped_(1).jpg"],
            "alt_text": None,
        },
        {
            "page": 2,
            "url": "https://xkcd.com/2/",
            "image_urls": ["https://imgs.xkcd.com/comics/tree_cropped_(1).jpg"],
            "alt_text": None,
        },
        {
            "page": 3,
            "url": "https://xkcd.com/3/",
            "image_urls": ["https://imgs.xkcd.com/comics/island_color.jpg"],
            "alt_text": None,
        },
    ]

def test_verify_xpath_with_alt_text():
    comic = Comic("xkcd", *supported_comics["xkcd_alt"])
    assert comic.verify_xpath() == [
        {
            "page": 1,
            "url": "https://xkcd.com/1/",
            "image_urls": ["https://imgs.xkcd.com/comics/barrel_cropped_(1).jpg"],
            "alt_text": "Don't we all.",
        },
        {
            "page": 2,
            "url": "https://xkcd.com/2/",
            "image_urls": ["https://imgs.xkcd.com/comics/tree_cropped_(1).jpg"],
            "alt_text": "'Petit' being a reference to Le Petit Prince, which I only thought about halfway through the sketch",
        },
        {
            "page": 3,
            "url": "https://xkcd.com/3/",
            "image_urls": ["https://imgs.xkcd.com/comics/island_color.jpg"],
            "alt_text": "Hello, island",
        },
    ]

def test_verify_xpath_only_verifies_one_page_with_single_page(one_webpage_uri):
    comic = Comic("test", one_webpage_uri, "//img/@src", "//a/@href", single_page=True)
    actual = comic.verify_xpath()
    assert len(actual) == 1
    assert len(actual[0]["image_urls"]) == 2

def test_download_will_run_javascript_settings_if_javascript(mocker):
    mocker.patch("os.path.isdir")
    mock_crawler_worker = mocker.patch("webcomix.comic.CrawlerWorker")
    comic = Comic(
        mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY, True
    )
    comic.download()
    settings = mock_crawler_worker.call_args_list[0][0][0]
    assert all(setting in settings for setting in SPLASH_SETTINGS)

def test_download_will_not_run_javascript_settings_if_not_javascript(mocker):
    mocker.patch("os.path.isdir")
    mock_crawler_worker = mocker.patch("webcomix.comic.CrawlerWorker")
    comic = Comic(mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY)
    comic.download()
    settings = mock_crawler_worker.call_args_list[0][0][0]
    assert all(setting not in settings for setting in SPLASH_SETTINGS)

def test_verify_xpath_will_run_javascript_settings_if_javascript(mocker):
    mocker.patch("os.path.isdir")
    mock_crawler_worker = mocker.patch("webcomix.comic.CrawlerWorker")
    comic = Comic(
        mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY, True
    )
    comic.verify_xpath()
    settings = mock_crawler_worker.call_args_list[0][0][0]
    assert all(setting in settings for setting in SPLASH_SETTINGS)

def test_verify_xpath_will_not_run_javascript_settings_if_not_javascript(mocker):
    mocker.patch("os.path.isdir")
    mock_crawler_worker = mocker.patch("webcomix.comic.CrawlerWorker")
    comic = Comic(mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY, False)
    comic.verify_xpath()
    settings = mock_crawler_worker.call_args_list[0][0][0]
    assert all(setting not in settings for setting in SPLASH_SETTINGS)
<commit_msg>Add test for save_image_filename<commit_after>import os
import shutil
from zipfile import ZipFile, BadZipFile

import pytest

from webcomix.comic import Comic, SPLASH_SETTINGS
from webcomix.supported_comics import supported_comics
from webcomix.tests.fake_websites.fixture import (
    three_webpages_uri,
    three_webpages_alt_text_uri,
    one_webpage_uri,
)

@pytest.fixture
def cleanup_test_directories():
    if os.path.isfile("xkcd.cbz"):
        os.remove("xkcd.cbz")
    if os.path.isfile("test.cbz"):
        os.remove("test.cbz")
    if os.path.isdir("xkcd"):
        shutil.rmtree("xkcd")
    if os.path.isdir("test"):
        shutil.rmtree("test")
    yield None
    if os.path.isdir("xkcd"):
        shutil.rmtree("xkcd")
    if os.path.isdir("test"):
        shutil.rmtree("test")
    if os.path.isfile("xkcd.cbz"):
        os.remove("xkcd.cbz")
    if os.path.isfile("test.cbz"):
        os.remove("test.cbz")

@pytest.fixture
def fake_downloaded_xkcd_comic():
    if os.path.isfile("xkcd.cbz"):
        os.remove("xkcd.cbz")
    if os.path.isdir("xkcd"):
        shutil.rmtree("xkcd")
    comic = Comic(
        "xkcd",
        "http://xkcd.com/1/",
        "//div[@id='comic']/img/@src",
        "//a[@rel='next']/@href",
        False,
    )
    os.mkdir("xkcd")
    for i in range(1, 6):
        with open("xkcd/{}.txt".format(i), "w") as image_file:
            image_file.write("testing {}".format(i))
    yield comic
    if os.path.isfile("xkcd.cbz"):
        os.remove("xkcd.cbz")
    if os.path.isdir("xkcd"):
        shutil.rmtree("xkcd")

def test_save_image_location():
    assert (
        Comic.save_image_location(
            "http://imgs.xkcd.com/comics/barrel_cropped_(1).jpg", 1, "foo"
        )
        == "foo/1.jpg"
    )
    assert Comic.save_image_location("", 1, "bar") == "bar/1"

def test_save_image_filename_with_title_present():
    assert (
        Comic.save_image_filename(
            "http://imgs.xkcd.com/comics/barrel_cropped_(1).jpg", 1, True, "foo"
        )
        == "foo-1.jpg"
    )

def test_make_cbz(fake_downloaded_xkcd_comic):
    fake_downloaded_xkcd_comic.convert_to_cbz()
    with ZipFile("xkcd.cbz") as cbz_file:
        for i in range(1, 6):
            with cbz_file.open("{}.txt".format(i), "r") as image_file:
                assert image_file.read().decode("ascii") == "testing {}".format(i)

def test_make_cbz_corrupted_archive(mocker, capfd, fake_downloaded_xkcd_comic):
    mocker.patch.object(ZipFile, "testzip", return_value=mocker.ANY)
    with pytest.raises(BadZipFile):
        fake_downloaded_xkcd_comic.convert_to_cbz()

def test_download_runs_the_worker(mocker, cleanup_test_directories):
    mock_crawler_running = mocker.patch(
        "webcomix.scrapy.crawler_worker.CrawlerWorker.start"
    )
    comic = Comic(
        "xkcd",
        "http://xkcd.com/1/",
        "//div[@id='comic']//img/@src",
        "//a[@rel='next']/@href",
        False,
    )
    comic.download()
    assert mock_crawler_running.call_count == 1

def test_download_saves_the_files(cleanup_test_directories, three_webpages_uri):
    comic = Comic("test", three_webpages_uri, "//img/@src", "//a/@href")
    comic.download()
    path, dirs, files = next(os.walk("test"))
    assert len(files) == 2

def test_download_with_alt_text_saves_the_text(
    cleanup_test_directories, three_webpages_alt_text_uri
):
    comic = Comic(
        "test", three_webpages_alt_text_uri, "//img/@src", "//a/@href", "//img/@title"
    )
    comic.download()
    path, dirs, files = next(os.walk("test"))
    assert len(files) == 4

def test_download_does_not_add_crawlers_in_main_process(
    mocker, cleanup_test_directories, three_webpages_uri
):
    mocker.patch("webcomix.scrapy.crawler_worker.CrawlerWorker.start")
    mock_add_to_crawl = mocker.patch("scrapy.crawler.Crawler.crawl")
    comic = Comic("test", three_webpages_uri, "//img/@src", "//a/@href")
    comic.download()
    assert mock_add_to_crawl.call_count == 0

def test_convert_to_cbz_adds_all_files_to_cbz(
    cleanup_test_directories, three_webpages_uri
):
    comic = Comic("test", three_webpages_uri, "//img/@src", "//a/@href")
    comic.download()
    comic.convert_to_cbz()
    with ZipFile("{}.cbz".format(comic.name), mode="r") as cbz_file:
        assert len(cbz_file.infolist()) == 2

def test_verify_xpath():
    comic = Comic("xkcd", *supported_comics["xkcd"])
    assert comic.verify_xpath() == [
        {
            "page": 1,
            "url": "https://xkcd.com/1/",
            "image_urls": ["https://imgs.xkcd.com/comics/barrel_cropped_(1).jpg"],
            "alt_text": None,
        },
        {
            "page": 2,
            "url": "https://xkcd.com/2/",
            "image_urls": ["https://imgs.xkcd.com/comics/tree_cropped_(1).jpg"],
            "alt_text": None,
        },
        {
            "page": 3,
            "url": "https://xkcd.com/3/",
            "image_urls": ["https://imgs.xkcd.com/comics/island_color.jpg"],
            "alt_text": None,
        },
    ]

def test_verify_xpath_with_alt_text():
    comic = Comic("xkcd", *supported_comics["xkcd_alt"])
    assert comic.verify_xpath() == [
        {
            "page": 1,
            "url": "https://xkcd.com/1/",
            "image_urls": ["https://imgs.xkcd.com/comics/barrel_cropped_(1).jpg"],
            "alt_text": "Don't we all.",
        },
        {
            "page": 2,
            "url": "https://xkcd.com/2/",
            "image_urls": ["https://imgs.xkcd.com/comics/tree_cropped_(1).jpg"],
            "alt_text": "'Petit' being a reference to Le Petit Prince, which I only thought about halfway through the sketch",
        },
        {
            "page": 3,
            "url": "https://xkcd.com/3/",
            "image_urls": ["https://imgs.xkcd.com/comics/island_color.jpg"],
            "alt_text": "Hello, island",
        },
    ]

def test_verify_xpath_only_verifies_one_page_with_single_page(one_webpage_uri):
    comic = Comic("test", one_webpage_uri, "//img/@src", "//a/@href", single_page=True)
    actual = comic.verify_xpath()
    assert len(actual) == 1
    assert len(actual[0]["image_urls"]) == 2

def test_download_will_run_javascript_settings_if_javascript(mocker):
    mocker.patch("os.path.isdir")
    mock_crawler_worker = mocker.patch("webcomix.comic.CrawlerWorker")
    comic = Comic(
        mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY, True
    )
    comic.download()
    settings = mock_crawler_worker.call_args_list[0][0][0]
    assert all(setting in settings for setting in SPLASH_SETTINGS)

def test_download_will_not_run_javascript_settings_if_not_javascript(mocker):
    mocker.patch("os.path.isdir")
    mock_crawler_worker = mocker.patch("webcomix.comic.CrawlerWorker")
    comic = Comic(mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY)
    comic.download()
    settings = mock_crawler_worker.call_args_list[0][0][0]
    assert all(setting not in settings for setting in SPLASH_SETTINGS)

def test_verify_xpath_will_run_javascript_settings_if_javascript(mocker):
    mocker.patch("os.path.isdir")
    mock_crawler_worker = mocker.patch("webcomix.comic.CrawlerWorker")
    comic = Comic(
        mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY, True
    )
    comic.verify_xpath()
    settings = mock_crawler_worker.call_args_list[0][0][0]
    assert all(setting in settings for setting in SPLASH_SETTINGS)

def test_verify_xpath_will_not_run_javascript_settings_if_not_javascript(mocker):
    mocker.patch("os.path.isdir")
    mock_crawler_worker = mocker.patch("webcomix.comic.CrawlerWorker")
    comic = Comic(mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY, False)
    comic.verify_xpath()
    settings = mock_crawler_worker.call_args_list[0][0][0]
    assert all(setting not in settings for setting in SPLASH_SETTINGS)
<|endoftext|>


<commit_before># coding=utf-8
from __future__ import absolute_import
from __future__ import unicode_literals

from django.db.models import Q
from django.test import TestCase
from datetime import time
from io import BytesIO
import re
import tempfile

from couchexport.export import export_raw
from couchexport.models import Format

from corehq.apps.data_interfaces.models import (
    AutomaticUpdateRule,
    CreateScheduleInstanceActionDefinition,
)
from corehq.apps.data_interfaces.tests.util import create_empty_rule
from corehq.apps.domain.models import Domain
from corehq.messaging.scheduling.forms import ContentForm
from corehq.messaging.scheduling.models import (
    EmailContent,
    SMSContent,
    TimedEvent,
    TimedSchedule,
)
from corehq.apps.users.models import CommCareUser
from corehq.messaging.scheduling.scheduling_partitioned.dbaccessors import delete_case_schedule_instance
from corehq.messaging.scheduling.scheduling_partitioned.models import CaseTimedScheduleInstance
from corehq.messaging.scheduling.tests.util import delete_timed_schedules
from corehq.messaging.scheduling.view_helpers import get_conditional_alert_rows, upload_conditional_alert_workbook
from corehq.sql_db.util import run_query_across_partitioned_databases
from corehq.util.workbook_json.excel import get_workbook

class TestBulkConditionalAlerts(TestCase):
    domain = 'bulk-conditional-alert-test'

    EMAIL_RULE = 'email_rule'
    UNTRANSLATED_RULE = 'untranslated_rule'
    DAILY_RULE = 'daily_rule'
    WEEKLY_RULE = 'weekly_rule'

    @classmethod
    def setUpClass(cls):
        super(TestBulkConditionalAlerts, cls).setUpClass()
        cls.domain_obj = Domain(
            name=cls.domain,
            default_timezone='America/New_York',
        )
        cls.domain_obj.save()
        cls.user = CommCareUser.create(cls.domain, 'test1', 'abc')
        cls.langs = ['en', 'es']

    def setUp(self):
        self._rules = {
            self.EMAIL_RULE: self._add_daily_rule(EmailContent(
                subject={'*': 'You just won something'},
                message={'*': 'This is a scam'},
            )),
            self.UNTRANSLATED_RULE: self._add_daily_rule(SMSContent(message={
                '*': 'Joan',
            })),
            self.DAILY_RULE: self._add_daily_rule(SMSContent(message={
                'en': 'Diamonds and Rust',
                'es': 'Diamantes y Óxido',
            })),
            self.WEEKLY_RULE: self._add_weekly_rule(SMSContent(message={
                'en': 'It\'s Too Late',
                'es': 'Es Demasiado Tarde',
            })),
        }

    @classmethod
    def tearDownClass(cls):
        cls.user.delete()
        cls.domain_obj.delete()
        super(TestBulkConditionalAlerts, cls).tearDownClass()

    def tearDown(self):
        for rule in AutomaticUpdateRule.objects.filter(domain=self.domain):
            rule.hard_delete()

        for instance in run_query_across_partitioned_databases(CaseTimedScheduleInstance, Q(domain=self.domain)):
            delete_case_schedule_instance(instance)

        delete_timed_schedules(self.domain)

    def _get_rule(self, type):
        return AutomaticUpdateRule.objects.get(id=self._rules[type].id)

    def _assertPatternIn(self, pattern, collection):
        self.assertTrue(any(re.match(pattern, item) for item in collection))

    def _add_daily_rule(self, content):
        schedule = TimedSchedule.create_simple_daily_schedule(
            self.domain,
            TimedEvent(time=time(9, 0)),
            content
        )
        return self._add_rule(schedule)

    def _add_weekly_rule(self, content):
        schedule = TimedSchedule.create_simple_weekly_schedule(
            self.domain,
            TimedEvent(time=time(12, 0)),
            content,
            [0, 4],
            0,
            total_iterations=3,
            repeat_every=2,
        )
        return self._add_rule(schedule)

    def _add_rule(self, schedule):
        rule = create_empty_rule(self.domain, AutomaticUpdateRule.WORKFLOW_SCHEDULING)
        self.addCleanup(rule.delete)
        rule.add_action(
            CreateScheduleInstanceActionDefinition,
            timed_schedule_id=schedule.schedule_id,
            recipients=(('CommCareUser', self.user.get_id),)
        )
        rule.save()
        return rule

    def test_download(self):
        (translated_rows, untranslated_rows) = get_conditional_alert_rows(self.domain, self.langs)

        self.assertEqual(len(translated_rows), 2)
        self.assertEqual(len(untranslated_rows), 1)

        rows_by_id = {row[0]: row[1:] for row in translated_rows + untranslated_rows}
        self.assertListEqual(rows_by_id[self._get_rule(self.UNTRANSLATED_RULE).id],
                                       ['test', 'person', 'Joan'])
        self.assertListEqual(rows_by_id[self._get_rule(self.DAILY_RULE).id],
                                       ['test', 'person', 'Diamonds and Rust', 'Diamantes y Óxido'])
        self.assertListEqual(rows_by_id[self._get_rule(self.WEEKLY_RULE).id],
                                       ['test', 'person', 'It\'s Too Late', 'Es Demasiado Tarde'])

    def test_upload(self):
        headers = (
            ("translated", ("id", "name", "case_type", "message_en", "message_es")),
            ("not translated", ("id", "name", "case_type", "message")),
        )
        data = (
            ("translated", (
                (self._get_rule(self.DAILY_RULE).id, 'test daily', 'song', 'Rustier', 'Más Oxidado'),
                (self._get_rule(self.UNTRANSLATED_RULE).id, 'test wrong sheet', 'wrong', 'wrong'),
                (self._get_rule(self.EMAIL_RULE).id, 'test email', 'song', 'Email content', 'does not fit'),
                (1000, 'Not a rule', 'person'),
                (self._get_rule(self.WEEKLY_RULE).id, 'test weekly', 'song', 'It\'s On Time', 'Está a Tiempo'),
            )),
            ("not translated", (
                (self._get_rule(self.UNTRANSLATED_RULE).id, 'test untranslated', 'song', 'Joanie'),
                (self._get_rule(self.DAILY_RULE).id, 'test wrong sheet', 'wrong', 'wrong', 'wrong'),
            )),
        )
        file = BytesIO()
        export_raw(headers, data, file, format=Format.XLS_2007)

        with tempfile.TemporaryFile(suffix='.xlsx') as f:
            f.write(file.getvalue())
            f.seek(0)
            workbook = get_workbook(f)
            msgs = [m[1] for m in upload_conditional_alert_workbook(self.domain, self.langs, workbook)]

            self.assertEqual(len(msgs), 6)
            self._assertPatternIn(r"Rule in row 3 with id \d+ does not belong in 'translated' sheet.", msgs)
            self._assertPatternIn(r"Row 4 in 'translated' sheet, with rule id \d+, does not use SMS content", msgs)
            self._assertPatternIn(r"Could not find rule for row 5 in 'translated' sheet, with id \d+", msgs)
            self.assertIn("Updated 2 rule(s) in 'translated' sheet", msgs)
            self._assertPatternIn(r"Rule in row 3 with id \d+ does not belong in 'not translated' sheet.", msgs)
            self.assertIn("Updated 1 rule(s) in 'not translated' sheet", msgs)

            untranslated_rule = self._get_rule(self.UNTRANSLATED_RULE)
            self.assertEqual(untranslated_rule.name, 'test untranslated')
            self.assertEqual(untranslated_rule.case_type, 'song')
            untranslated_content = untranslated_rule.get_messaging_rule_schedule().memoized_events[0].content
            self.assertEqual(untranslated_content.message, {
                '*': 'Joanie',
            })

            # TODO: assert that schedules were preserved
            daily_rule = self._get_rule(self.DAILY_RULE)
            self.assertEqual(daily_rule.name, 'test daily')
            self.assertEqual(daily_rule.case_type, 'song')
            daily_content = daily_rule.get_messaging_rule_schedule().memoized_events[0].content
            self.assertEqual(daily_content.message, {
                'en': 'Rustier',
                'es': 'Más Oxidado',
            })

            weekly_rule = self._get_rule(self.WEEKLY_RULE)
            weekly_content = weekly_rule.get_messaging_rule_schedule().memoized_events[0].content
            self.assertEqual(weekly_content.message, {
                'en': 'It\'s On Time',
                'es': 'Está a Tiempo',
            })
<commit_msg>Added tests to verify schedules are preserved during upload<commit_after># coding=utf-8
from __future__ import absolute_import
from __future__ import unicode_literals

from django.db.models import Q
from django.test import TestCase
from datetime import time
from io import BytesIO
import re
import tempfile

from couchexport.export import export_raw
from couchexport.models import Format

from corehq.apps.data_interfaces.models import (
    AutomaticUpdateRule,
    CreateScheduleInstanceActionDefinition,
)
from corehq.apps.data_interfaces.tests.util import create_empty_rule
from corehq.apps.domain.models import Domain
from corehq.messaging.scheduling.forms import ContentForm
from corehq.messaging.scheduling.models import (
    EmailContent,
    SMSContent,
    TimedEvent,
    TimedSchedule,
)
from corehq.apps.users.models import CommCareUser
from corehq.messaging.scheduling.scheduling_partitioned.dbaccessors import delete_case_schedule_instance
from corehq.messaging.scheduling.scheduling_partitioned.models import CaseTimedScheduleInstance
from corehq.messaging.scheduling.tests.util import delete_timed_schedules
from corehq.messaging.scheduling.view_helpers import get_conditional_alert_rows, upload_conditional_alert_workbook
from corehq.sql_db.util import run_query_across_partitioned_databases
from corehq.util.workbook_json.excel import get_workbook

class TestBulkConditionalAlerts(TestCase):
    domain = 'bulk-conditional-alert-test'

    EMAIL_RULE = 'email_rule'
    UNTRANSLATED_RULE = 'untranslated_rule'
    DAILY_RULE = 'daily_rule'
    WEEKLY_RULE = 'weekly_rule'

    @classmethod
    def setUpClass(cls):
        super(TestBulkConditionalAlerts, cls).setUpClass()
        cls.domain_obj = Domain(
            name=cls.domain,
            default_timezone='America/New_York',
        )
        cls.domain_obj.save()
        cls.user = CommCareUser.create(cls.domain, 'test1', 'abc')
        cls.langs = ['en', 'es']

    def setUp(self):
        self._rules = {
            self.EMAIL_RULE: self._add_daily_rule(EmailContent(
                subject={'*': 'You just won something'},
                message={'*': 'This is a scam'},
            )),
            self.UNTRANSLATED_RULE: self._add_daily_rule(SMSContent(message={
                '*': 'Joan',
            })),
            self.DAILY_RULE: self._add_daily_rule(SMSContent(message={
                'en': 'Diamonds and Rust',
                'es': 'Diamantes y Óxido',
            })),
            self.WEEKLY_RULE: self._add_weekly_rule(SMSContent(message={
                'en': 'It\'s Too Late',
                'es': 'Es Demasiado Tarde',
            })),
        }

    @classmethod
    def tearDownClass(cls):
        cls.user.delete()
        cls.domain_obj.delete()
        super(TestBulkConditionalAlerts, cls).tearDownClass()

    def tearDown(self):
        for rule in AutomaticUpdateRule.objects.filter(domain=self.domain):
            rule.hard_delete()

        for instance in run_query_across_partitioned_databases(CaseTimedScheduleInstance, Q(domain=self.domain)):
            delete_case_schedule_instance(instance)

        delete_timed_schedules(self.domain)

    def _get_rule(self, type):
        return AutomaticUpdateRule.objects.get(id=self._rules[type].id)

    def _assertPatternIn(self, pattern, collection):
        self.assertTrue(any(re.match(pattern, item) for item in collection))

    def _assertTimedScheduleEventsEqual(self, schedule1, schedule2):
        self.assertEqual(len(schedule1.memoized_events), len(schedule2.memoized_events))
        for i in range(len(schedule1.memoized_events)):
            event1 = schedule1.memoized_events[i]
            event2 = schedule2.memoized_events[i]
            self.assertTrue(isinstance(event1, TimedEvent))
            self.assertTrue(isinstance(event2, TimedEvent))
            self.assertEqual(event1.day, event2.day)
            self.assertEqual(event1.get_scheduling_info(), event2.get_scheduling_info())

    def _add_daily_rule(self, content):
        schedule = TimedSchedule.create_simple_daily_schedule(
            self.domain,
            TimedEvent(time=time(9, 0)),
            content
        )
        return self._add_rule(schedule)

    def _add_weekly_rule(self, content):
        schedule = TimedSchedule.create_simple_weekly_schedule(
            self.domain,
            TimedEvent(time=time(12, 0)),
            content,
            [0, 4],
            0,
            total_iterations=3,
            repeat_every=2,
        )
        return self._add_rule(schedule)

    def _add_rule(self, schedule):
        rule = create_empty_rule(self.domain, AutomaticUpdateRule.WORKFLOW_SCHEDULING)
        self.addCleanup(rule.delete)
        rule.add_action(
            CreateScheduleInstanceActionDefinition,
            timed_schedule_id=schedule.schedule_id,
            recipients=(('CommCareUser', self.user.get_id),)
        )
        rule.save()
        return rule

    def test_download(self):
        (translated_rows, untranslated_rows) = get_conditional_alert_rows(self.domain, self.langs)

        self.assertEqual(len(translated_rows), 2)
        self.assertEqual(len(untranslated_rows), 1)

        rows_by_id = {row[0]: row[1:] for row in translated_rows + untranslated_rows}
        self.assertListEqual(rows_by_id[self._get_rule(self.UNTRANSLATED_RULE).id],
                                       ['test', 'person', 'Joan'])
        self.assertListEqual(rows_by_id[self._get_rule(self.DAILY_RULE).id],
                                       ['test', 'person', 'Diamonds and Rust', 'Diamantes y Óxido'])
        self.assertListEqual(rows_by_id[self._get_rule(self.WEEKLY_RULE).id],
                                       ['test', 'person', 'It\'s Too Late', 'Es Demasiado Tarde'])

    def test_upload(self):
        headers = (
            ("translated", ("id", "name", "case_type", "message_en", "message_es")),
            ("not translated", ("id", "name", "case_type", "message")),
        )
        data = (
            ("translated", (
                (self._get_rule(self.DAILY_RULE).id, 'test daily', 'song', 'Rustier', 'Más Oxidado'),
                (self._get_rule(self.UNTRANSLATED_RULE).id, 'test wrong sheet', 'wrong', 'wrong'),
                (self._get_rule(self.EMAIL_RULE).id, 'test email', 'song', 'Email content', 'does not fit'),
                (1000, 'Not a rule', 'person'),
                (self._get_rule(self.WEEKLY_RULE).id, 'test weekly', 'song', 'It\'s On Time', 'Está a Tiempo'),
            )),
            ("not translated", (
                (self._get_rule(self.UNTRANSLATED_RULE).id, 'test untranslated', 'song', 'Joanie'),
                (self._get_rule(self.DAILY_RULE).id, 'test wrong sheet', 'wrong', 'wrong', 'wrong'),
            )),
        )
        file = BytesIO()
        export_raw(headers, data, file, format=Format.XLS_2007)

        with tempfile.TemporaryFile(suffix='.xlsx') as f:
            f.write(file.getvalue())
            f.seek(0)
            workbook = get_workbook(f)

            old_daily_schedule = self._get_rule(self.DAILY_RULE).get_messaging_rule_schedule()
            old_weekly_schedule = self._get_rule(self.WEEKLY_RULE).get_messaging_rule_schedule()
            msgs = upload_conditional_alert_workbook(self.domain, self.langs, workbook)
            msgs = [m[1] for m in msgs]     # msgs is tuples of (type, message); ignore the type

            self.assertEqual(len(msgs), 6)
            self._assertPatternIn(r"Rule in row 3 with id \d+ does not belong in 'translated' sheet.", msgs)
            self._assertPatternIn(r"Row 4 in 'translated' sheet, with rule id \d+, does not use SMS content", msgs)
            self._assertPatternIn(r"Could not find rule for row 5 in 'translated' sheet, with id \d+", msgs)
            self.assertIn("Updated 2 rule(s) in 'translated' sheet", msgs)
            self._assertPatternIn(r"Rule in row 3 with id \d+ does not belong in 'not translated' sheet.", msgs)
            self.assertIn("Updated 1 rule(s) in 'not translated' sheet", msgs)

            untranslated_rule = self._get_rule(self.UNTRANSLATED_RULE)
            self.assertEqual(untranslated_rule.name, 'test untranslated')
            self.assertEqual(untranslated_rule.case_type, 'song')
            untranslated_content = untranslated_rule.get_messaging_rule_schedule().memoized_events[0].content
            self.assertEqual(untranslated_content.message, {
                '*': 'Joanie',
            })

            daily_rule = self._get_rule(self.DAILY_RULE)
            self.assertEqual(daily_rule.name, 'test daily')
            self.assertEqual(daily_rule.case_type, 'song')
            daily_schedule = daily_rule.get_messaging_rule_schedule()
            self._assertTimedScheduleEventsEqual(daily_schedule, old_daily_schedule)
            daily_content = daily_schedule.memoized_events[0].content
            self.assertEqual(daily_content.message, {
                'en': 'Rustier',
                'es': 'Más Oxidado',
            })

            weekly_rule = self._get_rule(self.WEEKLY_RULE)
            weekly_schedule = weekly_rule.get_messaging_rule_schedule()
            self._assertTimedScheduleEventsEqual(weekly_schedule, old_weekly_schedule)
            weekly_content = weekly_rule.get_messaging_rule_schedule().memoized_events[0].content
            self.assertEqual(weekly_content.message, {
                'en': 'It\'s On Time',
                'es': 'Está a Tiempo',
            })
<|endoftext|>


<commit_before>import os
import re

from django.http import HttpResponse
from django.conf import settings
from django.core.urlresolvers import reverse

def _sub_base_href(html, base_url):
    return re.sub(r'<base href=".*">', '<base href="%s">' % base_url, html)

def _sub_publish_url(html, publish_url):
    return re.sub(r'<meta name="publish-url" content=".*">',
                  '<meta name="publish-url" content="%s">' % publish_url,
                  html)

def _sub_remix_url(html, remix_url):
    return re.sub(r'<meta name="remix-url" content=".*">',
                  '<meta name="remix-url" content="%s">' % remix_url,
                  html)

def _frontend_html(base_url, publish_url, blank_url, remix_url):
    mydir = os.path.dirname(__file__)
    html = open(os.path.join(mydir, 'friendlycode', 'index.html')).read()
    html = html.replace("blank.html", blank_url)
    html = _sub_remix_url(html, remix_url)
    return _sub_base_href(_sub_publish_url(html, publish_url), base_url)

def _editor(request, remix_url):
    return HttpResponse(_frontend_html(
        base_url='%sfriendlycode/' % settings.MEDIA_URL,
        publish_url=request.build_absolute_uri("/")[:-1],
        blank_url=request.build_absolute_uri(reverse(blank_page)),
        remix_url=remix_url
        ))

BLANK_HTML = """<!doctype html>
<html>
  <head>
    <meta charset="utf-8">
    <title>Blank Page</title>
  </head>
  <body>
  </body>
</html>"""

def default_editor(request):
    return _editor(request,
                   '%sfriendlycode/default-content.html' % settings.MEDIA_URL)

def blank_page(request):
    response = HttpResponse(BLANK_HTML)
    response.no_frame_options = True
    return response

def editor(request, **kwargs):
    viewname = kwargs['remix']
    del kwargs['remix']
    to_remix = reverse(viewname, kwargs=kwargs)
    return _editor(request, to_remix)
<commit_msg>refactoring to maintain DRY in editor.views.<commit_after>import os
import re

from django.http import HttpResponse
from django.conf import settings
from django.core.urlresolvers import reverse

def _subvar(html, template, replacement):
    regexp = "(%s)(.*)(%s)" % tuple(template.split(".*"))
    return re.sub(regexp, r'\1' + replacement + r'\3', html)

def _sub_base_href(html, base_url):
    return _subvar(html, r'<base href=".*">', base_url)

def _sub_publish_url(html, publish_url):
    return _subvar(html, r'<meta name="publish-url" content=".*">',
                   publish_url)

def _sub_remix_url(html, remix_url):
    return _subvar(html, r'<meta name="remix-url" content=".*">', remix_url)

def _frontend_html(base_url, publish_url, blank_url, remix_url):
    mydir = os.path.dirname(__file__)
    html = open(os.path.join(mydir, 'friendlycode', 'index.html')).read()
    html = html.replace("blank.html", blank_url)
    html = _sub_remix_url(html, remix_url)
    return _sub_base_href(_sub_publish_url(html, publish_url), base_url)

def _editor(request, remix_url):
    return HttpResponse(_frontend_html(
        base_url='%sfriendlycode/' % settings.MEDIA_URL,
        publish_url=request.build_absolute_uri("/")[:-1],
        blank_url=request.build_absolute_uri(reverse(blank_page)),
        remix_url=remix_url
        ))

BLANK_HTML = """<!doctype html>
<html>
  <head>
    <meta charset="utf-8">
    <title>Blank Page</title>
  </head>
  <body>
  </body>
</html>"""

def default_editor(request):
    return _editor(request,
                   '%sfriendlycode/default-content.html' % settings.MEDIA_URL)

def blank_page(request):
    response = HttpResponse(BLANK_HTML)
    response.no_frame_options = True
    return response

def editor(request, **kwargs):
    viewname = kwargs['remix']
    del kwargs['remix']
    to_remix = reverse(viewname, kwargs=kwargs)
    return _editor(request, to_remix)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import json
import codecs
import httplib
import functools

import furl
from flask import request
from flask import redirect
from flask import make_response
from modularodm.exceptions import NoResultsFound

from framework.auth import Auth
from framework.sessions import session
from framework.sentry import log_exception
from framework.exceptions import HTTPError
from framework.render.tasks import build_rendered_html
from framework.auth.decorators import must_be_logged_in, must_be_signed

from website import mails
from website import settings
from website.project import decorators
from website.addons.base import exceptions
from website.models import User, Node, NodeLog
from website.util import rubeus
from website.project.utils import serialize_node
from website.project.decorators import must_be_valid_project, must_be_contributor_or_public

@decorators.must_have_permission('write')
@decorators.must_not_be_registration
def disable_addon(auth, **kwargs):
    node = kwargs['node'] or kwargs['project']

    addon_name = kwargs.get('addon')
    if addon_name is None:
        raise HTTPError(httplib.BAD_REQUEST)

    deleted = node.delete_addon(addon_name, auth)

    return {'deleted': deleted}

@must_be_logged_in
def get_addon_user_config(**kwargs):

    user = kwargs['auth'].user

    addon_name = kwargs.get('addon')
    if addon_name is None:
        raise HTTPError(httplib.BAD_REQUEST)

    addon = user.get_addon(addon_name)
    if addon is None:
        raise HTTPError(httplib.BAD_REQUEST)

    return addon.to_json(user)

def check_file_guid(guid):

    guid_url = '/{0}/'.format(guid._id)
    if not request.path.startswith(guid_url):
        url_split = request.url.split(guid.file_url)
        try:
            guid_url += url_split[1].lstrip('/')
        except IndexError:
            pass
        return guid_url
    return None

permission_map = {
    'create_folder': 'write',
    'revisions': 'read',
    'metadata': 'read',
    'download': 'read',
    'upload': 'write',
    'delete': 'write',
    'copy': 'write',
    'move': 'write',
}

def check_access(node, user, action, key=None):
    """Verify that user can perform requested action on resource. Raise appropriate
    error code if action cannot proceed.
    """
    permission = permission_map.get(action, None)
    if permission is None:
        raise HTTPError(httplib.BAD_REQUEST)
    if node.has_permission(user, permission):
        return True
    if permission == 'read':
        if node.is_public or key in node.private_link_keys_active:
            return True
    code = httplib.FORBIDDEN if user else httplib.UNAUTHORIZED
    raise HTTPError(code)

def make_auth(user):
    if user is not None:
        return {
            'id': user._id,
            'email': '{}@osf.io'.format(user._id),
            'name': user.fullname,
        }
    return {}

def restrict_addrs(*addrs):
    def wrapper(func):
        @functools.wraps(func)
        def wrapped(*args, **kwargs):
            remote = request.remote_addr
            if remote not in addrs:
                raise HTTPError(httplib.FORBIDDEN)
            return func(*args, **kwargs)
        return wrapped
    return wrapper

restrict_waterbutler = restrict_addrs(*settings.WATERBUTLER_ADDRS)

@restrict_waterbutler
def get_auth(**kwargs):
    try:
        action = request.args['action']
        node_id = request.args['nid']
        provider_name = request.args['provider']
    except KeyError:
        raise HTTPError(httplib.BAD_REQUEST)

    cookie = request.args.get('cookie')
    view_only = request.args.get('view_only')

    if 'auth_user_id' in session.data:
        user = User.load(session.data['auth_user_id'])
    elif cookie:
        user = User.from_cookie(cookie)
    else:
        user = None

    node = Node.load(node_id)
    if not node:
        raise HTTPError(httplib.NOT_FOUND)

    check_access(node, user, action, key=view_only)

    provider_settings = node.get_addon(provider_name)
    if not provider_settings:
        raise HTTPError(httplib.BAD_REQUEST)

    try:
        credentials = provider_settings.serialize_waterbutler_credentials()
        settings = provider_settings.serialize_waterbutler_settings()
    except exceptions.AddonError:
        log_exception()
        raise HTTPError(httplib.BAD_REQUEST)

    return {
        'auth': make_auth(user),
        'credentials': credentials,
        'settings': settings,
        'callback_url': node.api_url_for(
            'create_waterbutler_log',
            _absolute=True,
        ),
    }

LOG_ACTION_MAP = {
    'move': NodeLog.FILE_MOVED,
    'copy': NodeLog.FILE_COPIED,
    'create': NodeLog.FILE_ADDED,
    'update': NodeLog.FILE_UPDATED,
    'delete': NodeLog.FILE_REMOVED,
    'create_folder': NodeLog.FOLDER_CREATED,
}

@must_be_signed
@restrict_waterbutler
@must_be_valid_project
def create_waterbutler_log(payload, **kwargs):
    try:
        auth = payload['auth']
        action = LOG_ACTION_MAP[payload['action']]
    except KeyError:
        raise HTTPError(httplib.BAD_REQUEST)

    user = User.load(auth['id'])
    if user is None:
        raise HTTPError(httplib.BAD_REQUEST)

    auth = Auth(user=user)
    node = kwargs['node'] or kwargs['project']

    if action in (NodeLog.FILE_MOVED, NodeLog.FILE_COPIED):
        for bundle in ('source', 'destination'):
            for key in ('provider', 'materialized', 'name', 'nid'):
                if key not in payload[bundle]:
                    raise HTTPError(httplib.BAD_REQUEST)

        destination_node = node  # For clarity
        source_node = Node.load(payload['source']['nid'])

        source = source_node.get_addon(payload['source']['provider'])
        destination = node.get_addon(payload['destination']['provider'])

        payload['source']['addon'] = source.config.full_name
        payload['destination']['addon'] = destination.config.full_name

        payload['source']['path'] = payload['source']['materialized']
        payload['destination']['path'] = payload['destination']['materialized']

        payload.update({
            'node': destination_node._id,
            'project': destination_node.parent_id,
        })

        destination_node.add_log(
            action=action,
            auth=auth,
            params=payload
        )

        if payload.get('email') is True:
            mails.send_mail(
                user.username,
                mails.FILE_OPERATION_FAILED if payload.get('errors')
                else mails.FILE_OPERATION_SUCCESS,
                action=payload['action'],
                source_node=source_node,
                destination_node=destination_node,
                source_path=payload['source']['path'],
                destination_path=payload['source']['path'],
                source_addon=payload['source']['addon'],
                destination_addon=payload['destination']['addon'],
            )
    else:
        try:
            metadata = payload['metadata']
            node_addon = node.get_addon(payload['provider'])
        except KeyError:
            raise HTTPError(httplib.BAD_REQUEST)

        if node_addon is None:
            raise HTTPError(httplib.BAD_REQUEST)

        metadata['path'] = metadata['path'].lstrip('/')

        node_addon.create_waterbutler_log(auth, action, metadata)

    return {'status': 'success'}

def get_or_start_render(file_guid, start_render=True):
    try:
        file_guid.enrich()
    except exceptions.AddonEnrichmentError as error:
        return error.as_html()

    try:
        return codecs.open(file_guid.mfr_cache_path, 'r', 'utf-8').read()
    except IOError:
        if start_render:
            # Start rendering job if requested
            build_rendered_html(
                file_guid.mfr_download_url,
                file_guid.mfr_cache_path,
                file_guid.mfr_temp_path,
                file_guid.public_download_url
            )
    return None

@must_be_valid_project
def addon_view_or_download_file_legacy(**kwargs):
    query_params = request.args.to_dict()
    node = kwargs.get('node') or kwargs['project']

    action = query_params.pop('action', 'view')
    provider = kwargs.get('provider', 'osfstorage')

    if kwargs.get('path'):
        path = kwargs['path']
    elif kwargs.get('fid'):
        path = kwargs['fid']

    if 'download' in request.path or request.path.startswith('/api/v1/'):
        action = 'download'

    if kwargs.get('vid'):
        query_params['version'] = kwargs['vid']

    # If provider is OSFstorage, check existence of requested file in the filetree
    # This prevents invalid GUIDs from being created
    if provider == 'osfstorage':
        node_settings = node.get_addon('osfstorage')

        try:
            path = node_settings.root_node.find_child_by_name(path)._id
        except NoResultsFound:
            raise HTTPError(
                404, data=dict(
                    message_short='File not found',
                    message_long='You requested a file that does not exist.'
                )
            )

    return redirect(
        node.web_url_for(
            'addon_view_or_download_file',
            path=path,
            provider=provider,
            action=action,
            **query_params
        ),
        code=httplib.MOVED_PERMANENTLY
    )

@must_be_valid_project
@must_be_contributor_or_public
def addon_view_or_download_file(auth, path, provider, **kwargs):
    extras = request.args.to_dict()
    action = extras.get('action', 'view')
    node = kwargs.get('node') or kwargs['project']

    node_addon = node.get_addon(provider)

    if not path or not node_addon:
        raise HTTPError(httplib.BAD_REQUEST)

    if not node_addon.has_auth:
        raise HTTPError(httplib.FORBIDDEN)

    if not path.startswith('/'):
        path = '/' + path

    file_guid, created = node_addon.find_or_create_file_guid(path)

    if file_guid.guid_url != request.path:
        guid_url = furl.furl(file_guid.guid_url)
        guid_url.args.update(extras)
        return redirect(guid_url)

    file_guid.maybe_set_version(**extras)

    if request.method == 'HEAD':
        download_url = furl.furl(file_guid.download_url)
        download_url.args.update(extras)
        download_url.args['accept_url'] = 'false'
        return make_response(('', 200, {'Location': download_url.url}))

    if action == 'download':
        download_url = furl.furl(file_guid.download_url)
        download_url.args.update(extras)

        return redirect(download_url.url)

    return addon_view_file(auth, node, node_addon, file_guid, extras)

def addon_view_file(auth, node, node_addon, file_guid, extras):
    render_url = node.api_url_for(
        'addon_render_file',
        path=file_guid.waterbutler_path.lstrip('/'),
        provider=file_guid.provider,
        render=True,
        **extras
    )

    ret = serialize_node(node, auth, primary=True)

    # Disable OSF Storage file deletion in DISK_SAVING_MODE
    if settings.DISK_SAVING_MODE and node_addon.config.short_name == 'osfstorage':
        ret['user']['can_edit'] = False

    ret.update({
        'provider': file_guid.provider,
        'render_url': render_url,
        'file_path': file_guid.waterbutler_path,
        'files_url': node.web_url_for('collect_file_trees'),
        'rendered': get_or_start_render(file_guid),
        # Note: must be called after get_or_start_render. This is really only for github
        'extra': json.dumps(getattr(file_guid, 'extra', {})),
        #NOTE: get_or_start_render must be called first to populate name
        'file_name': getattr(file_guid, 'name', os.path.split(file_guid.waterbutler_path)[1]),
        'materialized_path': getattr(file_guid, 'materialized', file_guid.waterbutler_path),
    })

    ret.update(rubeus.collect_addon_assets(node))
    return ret

@must_be_valid_project
@must_be_contributor_or_public
def addon_render_file(auth, path, provider, **kwargs):
    node = kwargs.get('node') or kwargs['project']

    node_addon = node.get_addon(provider)

    if not path:
        raise HTTPError(httplib.BAD_REQUEST)

    if not node_addon:
        raise HTTPError(httplib.BAD_REQUEST, {
            'message_short': 'Bad Request',
            'message_long': 'The add-on containing this file is no longer attached to the {}.'.format(node.project_or_component)
        })

    if not node_addon.has_auth:
        raise HTTPError(httplib.UNAUTHORIZED, {
            'message_short': 'Unauthorized',
            'message_long': 'The add-on containing this file is no longer authorized.'
        })

    if not node_addon.complete:
        raise HTTPError(httplib.BAD_REQUEST, {
            'message_short': 'Bad Request',
            'message_long': 'The add-on containing this file is no longer configured.'
        })

    if not path.startswith('/'):
        path = '/' + path

    file_guid, created = node_addon.find_or_create_file_guid(path)

    file_guid.maybe_set_version(**request.args.to_dict())

    return get_or_start_render(file_guid)
<commit_msg>include a lot more data with copies and moves<commit_after># -*- coding: utf-8 -*-

import os
import json
import codecs
import httplib
import functools

import furl
from flask import request
from flask import redirect
from flask import make_response
from modularodm.exceptions import NoResultsFound

from framework.auth import Auth
from framework.sessions import session
from framework.sentry import log_exception
from framework.exceptions import HTTPError
from framework.render.tasks import build_rendered_html
from framework.auth.decorators import must_be_logged_in, must_be_signed

from website import mails
from website import settings
from website.project import decorators
from website.addons.base import exceptions
from website.models import User, Node, NodeLog
from website.util import rubeus
from website.project.utils import serialize_node
from website.project.decorators import must_be_valid_project, must_be_contributor_or_public

@decorators.must_have_permission('write')
@decorators.must_not_be_registration
def disable_addon(auth, **kwargs):
    node = kwargs['node'] or kwargs['project']

    addon_name = kwargs.get('addon')
    if addon_name is None:
        raise HTTPError(httplib.BAD_REQUEST)

    deleted = node.delete_addon(addon_name, auth)

    return {'deleted': deleted}

@must_be_logged_in
def get_addon_user_config(**kwargs):

    user = kwargs['auth'].user

    addon_name = kwargs.get('addon')
    if addon_name is None:
        raise HTTPError(httplib.BAD_REQUEST)

    addon = user.get_addon(addon_name)
    if addon is None:
        raise HTTPError(httplib.BAD_REQUEST)

    return addon.to_json(user)

def check_file_guid(guid):

    guid_url = '/{0}/'.format(guid._id)
    if not request.path.startswith(guid_url):
        url_split = request.url.split(guid.file_url)
        try:
            guid_url += url_split[1].lstrip('/')
        except IndexError:
            pass
        return guid_url
    return None

permission_map = {
    'create_folder': 'write',
    'revisions': 'read',
    'metadata': 'read',
    'download': 'read',
    'upload': 'write',
    'delete': 'write',
    'copy': 'write',
    'move': 'write',
}

def check_access(node, user, action, key=None):
    """Verify that user can perform requested action on resource. Raise appropriate
    error code if action cannot proceed.
    """
    permission = permission_map.get(action, None)
    if permission is None:
        raise HTTPError(httplib.BAD_REQUEST)
    if node.has_permission(user, permission):
        return True
    if permission == 'read':
        if node.is_public or key in node.private_link_keys_active:
            return True
    code = httplib.FORBIDDEN if user else httplib.UNAUTHORIZED
    raise HTTPError(code)

def make_auth(user):
    if user is not None:
        return {
            'id': user._id,
            'email': '{}@osf.io'.format(user._id),
            'name': user.fullname,
        }
    return {}

def restrict_addrs(*addrs):
    def wrapper(func):
        @functools.wraps(func)
        def wrapped(*args, **kwargs):
            remote = request.remote_addr
            if remote not in addrs:
                raise HTTPError(httplib.FORBIDDEN)
            return func(*args, **kwargs)
        return wrapped
    return wrapper

restrict_waterbutler = restrict_addrs(*settings.WATERBUTLER_ADDRS)

@restrict_waterbutler
def get_auth(**kwargs):
    try:
        action = request.args['action']
        node_id = request.args['nid']
        provider_name = request.args['provider']
    except KeyError:
        raise HTTPError(httplib.BAD_REQUEST)

    cookie = request.args.get('cookie')
    view_only = request.args.get('view_only')

    if 'auth_user_id' in session.data:
        user = User.load(session.data['auth_user_id'])
    elif cookie:
        user = User.from_cookie(cookie)
    else:
        user = None

    node = Node.load(node_id)
    if not node:
        raise HTTPError(httplib.NOT_FOUND)

    check_access(node, user, action, key=view_only)

    provider_settings = node.get_addon(provider_name)
    if not provider_settings:
        raise HTTPError(httplib.BAD_REQUEST)

    try:
        credentials = provider_settings.serialize_waterbutler_credentials()
        settings = provider_settings.serialize_waterbutler_settings()
    except exceptions.AddonError:
        log_exception()
        raise HTTPError(httplib.BAD_REQUEST)

    return {
        'auth': make_auth(user),
        'credentials': credentials,
        'settings': settings,
        'callback_url': node.api_url_for(
            'create_waterbutler_log',
            _absolute=True,
        ),
    }

LOG_ACTION_MAP = {
    'move': NodeLog.FILE_MOVED,
    'copy': NodeLog.FILE_COPIED,
    'create': NodeLog.FILE_ADDED,
    'update': NodeLog.FILE_UPDATED,
    'delete': NodeLog.FILE_REMOVED,
    'create_folder': NodeLog.FOLDER_CREATED,
}

@must_be_signed
@restrict_waterbutler
@must_be_valid_project
def create_waterbutler_log(payload, **kwargs):
    try:
        auth = payload['auth']
        action = LOG_ACTION_MAP[payload['action']]
    except KeyError:
        raise HTTPError(httplib.BAD_REQUEST)

    user = User.load(auth['id'])
    if user is None:
        raise HTTPError(httplib.BAD_REQUEST)

    auth = Auth(user=user)
    node = kwargs['node'] or kwargs['project']

    if action in (NodeLog.FILE_MOVED, NodeLog.FILE_COPIED):
        for bundle in ('source', 'destination'):
            for key in ('provider', 'materialized', 'name', 'nid'):
                if key not in payload[bundle]:
                    raise HTTPError(httplib.BAD_REQUEST)

        destination_node = node  # For clarity
        source_node = Node.load(payload['source']['nid'])

        source = source_node.get_addon(payload['source']['provider'])
        destination = node.get_addon(payload['destination']['provider'])

        payload['source'].update({
            'materialized': payload['source']['materialized'].lstrip('/'),
            'addon': source.config.full_name,
            'url': source_node.web_url_for(
                'addon_view_or_download_file',
                path=payload['source']['path'].lstrip('/'),
                provider=payload['source']['provider']
            ),
            'node': {
                '_id': source_node._id,
                'url': source_node.url,
                'title': source_node.title,
            }
        })

        payload['destination'].update({
            'materialized': payload['destination']['materialized'].lstrip('/'),
            'addon': destination.config.full_name,
            'url': destination_node.web_url_for(
                'addon_view_or_download_file',
                path=payload['destination']['path'].lstrip('/'),
                provider=payload['destination']['provider']
            ),
            'node': {
                '_id': destination_node._id,
                'url': destination_node.url,
                'title': destination_node.title,
            }
        })

        payload.update({
            'node': destination_node._id,
            'project': destination_node.parent_id,
        })

        destination_node.add_log(
            action=action,
            auth=auth,
            params=payload
        )

        if payload.get('email') is True:
            mails.send_mail(
                user.username,
                mails.FILE_OPERATION_FAILED if payload.get('errors')
                else mails.FILE_OPERATION_SUCCESS,
                action=payload['action'],
                source_node=source_node,
                destination_node=destination_node,
                source_path=payload['source']['path'],
                destination_path=payload['source']['path'],
                source_addon=payload['source']['addon'],
                destination_addon=payload['destination']['addon'],
            )
    else:
        try:
            metadata = payload['metadata']
            node_addon = node.get_addon(payload['provider'])
        except KeyError:
            raise HTTPError(httplib.BAD_REQUEST)

        if node_addon is None:
            raise HTTPError(httplib.BAD_REQUEST)

        metadata['path'] = metadata['path'].lstrip('/')

        node_addon.create_waterbutler_log(auth, action, metadata)

    return {'status': 'success'}

def get_or_start_render(file_guid, start_render=True):
    try:
        file_guid.enrich()
    except exceptions.AddonEnrichmentError as error:
        return error.as_html()

    try:
        return codecs.open(file_guid.mfr_cache_path, 'r', 'utf-8').read()
    except IOError:
        if start_render:
            # Start rendering job if requested
            build_rendered_html(
                file_guid.mfr_download_url,
                file_guid.mfr_cache_path,
                file_guid.mfr_temp_path,
                file_guid.public_download_url
            )
    return None

@must_be_valid_project
def addon_view_or_download_file_legacy(**kwargs):
    query_params = request.args.to_dict()
    node = kwargs.get('node') or kwargs['project']

    action = query_params.pop('action', 'view')
    provider = kwargs.get('provider', 'osfstorage')

    if kwargs.get('path'):
        path = kwargs['path']
    elif kwargs.get('fid'):
        path = kwargs['fid']

    if 'download' in request.path or request.path.startswith('/api/v1/'):
        action = 'download'

    if kwargs.get('vid'):
        query_params['version'] = kwargs['vid']

    # If provider is OSFstorage, check existence of requested file in the filetree
    # This prevents invalid GUIDs from being created
    if provider == 'osfstorage':
        node_settings = node.get_addon('osfstorage')

        try:
            path = node_settings.root_node.find_child_by_name(path)._id
        except NoResultsFound:
            raise HTTPError(
                404, data=dict(
                    message_short='File not found',
                    message_long='You requested a file that does not exist.'
                )
            )

    return redirect(
        node.web_url_for(
            'addon_view_or_download_file',
            path=path,
            provider=provider,
            action=action,
            **query_params
        ),
        code=httplib.MOVED_PERMANENTLY
    )

@must_be_valid_project
@must_be_contributor_or_public
def addon_view_or_download_file(auth, path, provider, **kwargs):
    extras = request.args.to_dict()
    action = extras.get('action', 'view')
    node = kwargs.get('node') or kwargs['project']

    node_addon = node.get_addon(provider)

    if not path or not node_addon:
        raise HTTPError(httplib.BAD_REQUEST)

    if not node_addon.has_auth:
        raise HTTPError(httplib.FORBIDDEN)

    if not path.startswith('/'):
        path = '/' + path

    file_guid, created = node_addon.find_or_create_file_guid(path)

    if file_guid.guid_url != request.path:
        guid_url = furl.furl(file_guid.guid_url)
        guid_url.args.update(extras)
        return redirect(guid_url)

    file_guid.maybe_set_version(**extras)

    if request.method == 'HEAD':
        download_url = furl.furl(file_guid.download_url)
        download_url.args.update(extras)
        download_url.args['accept_url'] = 'false'
        return make_response(('', 200, {'Location': download_url.url}))

    if action == 'download':
        download_url = furl.furl(file_guid.download_url)
        download_url.args.update(extras)

        return redirect(download_url.url)

    return addon_view_file(auth, node, node_addon, file_guid, extras)

def addon_view_file(auth, node, node_addon, file_guid, extras):
    render_url = node.api_url_for(
        'addon_render_file',
        path=file_guid.waterbutler_path.lstrip('/'),
        provider=file_guid.provider,
        render=True,
        **extras
    )

    ret = serialize_node(node, auth, primary=True)

    # Disable OSF Storage file deletion in DISK_SAVING_MODE
    if settings.DISK_SAVING_MODE and node_addon.config.short_name == 'osfstorage':
        ret['user']['can_edit'] = False

    ret.update({
        'provider': file_guid.provider,
        'render_url': render_url,
        'file_path': file_guid.waterbutler_path,
        'files_url': node.web_url_for('collect_file_trees'),
        'rendered': get_or_start_render(file_guid),
        # Note: must be called after get_or_start_render. This is really only for github
        'extra': json.dumps(getattr(file_guid, 'extra', {})),
        #NOTE: get_or_start_render must be called first to populate name
        'file_name': getattr(file_guid, 'name', os.path.split(file_guid.waterbutler_path)[1]),
        'materialized_path': getattr(file_guid, 'materialized', file_guid.waterbutler_path),
    })

    ret.update(rubeus.collect_addon_assets(node))
    return ret

@must_be_valid_project
@must_be_contributor_or_public
def addon_render_file(auth, path, provider, **kwargs):
    node = kwargs.get('node') or kwargs['project']

    node_addon = node.get_addon(provider)

    if not path:
        raise HTTPError(httplib.BAD_REQUEST)

    if not node_addon:
        raise HTTPError(httplib.BAD_REQUEST, {
            'message_short': 'Bad Request',
            'message_long': 'The add-on containing this file is no longer attached to the {}.'.format(node.project_or_component)
        })

    if not node_addon.has_auth:
        raise HTTPError(httplib.UNAUTHORIZED, {
            'message_short': 'Unauthorized',
            'message_long': 'The add-on containing this file is no longer authorized.'
        })

    if not node_addon.complete:
        raise HTTPError(httplib.BAD_REQUEST, {
            'message_short': 'Bad Request',
            'message_long': 'The add-on containing this file is no longer configured.'
        })

    if not path.startswith('/'):
        path = '/' + path

    file_guid, created = node_addon.find_or_create_file_guid(path)

    file_guid.maybe_set_version(**request.args.to_dict())

    return get_or_start_render(file_guid)
<|endoftext|>