<commit_before>"""
Module bdp_fe.jobconf.views

"""

import logging

from django import forms
from django.conf import settings
from django.contrib import auth, messages
from django.contrib.auth.decorators import login_required
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.core.urlresolvers import reverse
from django.http import Http404, HttpResponseNotFound
from django.shortcuts import get_object_or_404, redirect, render_to_response
from django.template import RequestContext

from bdp_fe.jobconf import data
from bdp_fe.jobconf.cluster import remote
from bdp_fe.jobconf.models import CustomJobModel, Job, JobModel
from bdp_fe.jobconf.views_util import (get_owned_job_or_40x, safe_int_param,
                                       retrieve_results)

LOGGER = logging.getLogger(__name__)
CLUSTER = remote.Cluster(settings.CLUSTER_CONF.get('host'),
                         settings.CLUSTER_CONF.get('port'))

@login_required
def list_jobs(request):
    job_id = safe_int_param(request.GET, 'run_job')
    if job_id:
        run_job(request, job_id)

    return render_to_response('job_listing.html', {
        'title': 'Job listing',
        'jobs': Job.objects.all(),
    }, context_instance=RequestContext(request))

@login_required
def view_results(request, job_id):
    job = get_owned_job_or_40x(request, job_id)
    if job.status != Job.SUCCESSFUL:
        raise Http404
    primary_key = job.results_primary_key
    results = retrieve_results(job_id, 'word')
    prototype_result = results[0]
    paginator = Paginator(results, 100)
    page = request.GET.get('page')
    if not page:
        page = 1
    try:
        paginated_results = paginator.page(page)
    except PageNotAnInteger:
        paginated_results = paginator.page(1)
    except EmptyPage:
        paginated_results = paginator.page(paginator.num_pages)

    return render_to_response('job_results.html',
                              {'title' : 'Results of job %s' % job_id,
                               'job_results' : paginated_results,
                               'prototype_result': prototype_result,
                               'hidden_keys': ['_id', 'job_id'],
                               'expand_types': ['dict', 'list'],
                               'primary_key': primary_key},
                              context_instance=RequestContext(request))

def run_job(request, job_id):
    try:
        job = Job.objects.get(id=job_id)
    except Job.DoesNotExist:
        messages.warning(request, "Cannot start job %d: not found" % job_id)
        LOGGER.warning("Job %d not found" % job_id)
        return

    if job.status != Job.CREATED:
        msg = "Cannot start job %s while in %s status" % (
            job.name, job.get_status_display())
        messages.warning(request, msg)
        LOGGER.warning(msg)
        return

    if job.start(CLUSTER):
        messages.info(request, "Job %s was started." % job.name)
    else:
        messages.warning(request, "Cannot start job %s." % job.name)

class NewJobForm(forms.Form):
    name = forms.CharField(max_length=Job.NAME_MAX_LENGTH)
    results_primary_key = forms.CharField(max_length=Job.RESULTS_PK_MAX_LENGTH)

@login_required
def new_job(request):
    if request.method == 'POST':
        form = NewJobForm(request.POST)
        if form.is_valid():
            job = Job(name=form.cleaned_data['name'],
                      user=request.user,
                      results_primary_key=request.user,
                      status=Job.UNCONFIGURED)
            job.save()
            model = CustomJobModel(job=job) # The only option for the moment
            model.save()
            return redirect(reverse('config_job', args=[job.id]))
    else:
        form = NewJobForm()

    return render_to_response('new_job.html', {
        'title': 'New job',
        'form': form,
    }, context_instance=RequestContext(request))

class UploadJarForm(forms.Form):
    file = forms.FileField()

@login_required
def config_job(request, job_id):
    job = get_owned_job_or_40x(request, job_id)
    if job.status != Job.UNCONFIGURED:
        raise Http404()
    model = JobModel.objects.get(job=job).customjobmodel
    if request.method == 'POST':
        form = UploadJarForm(request.POST, request.FILES)
        if form.is_valid() and model.jar_upload(request.FILES['file']):
            job.status = Job.CONFIGURED
            job.save()
            return redirect(reverse('upload_data', args=[job.id]))
        else:
            messages.info(request, 'JAR file upload failed')
    else:
        form = UploadJarForm()
    return render_to_response('upload_jar.html', {
        'title': 'Configure custom job',
        'job': job,
        'form': form,
    }, context_instance=RequestContext(request))

class UploadDataForm(forms.Form):
    file = forms.FileField()

@login_required
def upload_data(request, job_id):
    job = get_owned_job_or_40x(request, job_id)
    if job.status != Job.CONFIGURED:
        raise Http404()

    if request.method == 'POST':
        form = UploadDataForm(request.POST, request.FILES)
        if form.is_valid() and job.data_upload(request.FILES['file'],
                                               CLUSTER):
            Job.input_data = job.hdfs_data_path()
            job.save()
            return redirect(reverse('list_jobs'))
        else:
            messages.info(request, 'Data file upload failed')
    else:
        form = UploadDataForm()
    return render_to_response('upload_data.html', {
        'title': 'Upload job %s data' % job.name,
        'job': job,
        'form': form,
    }, context_instance=RequestContext(request))
<commit_msg>Get displayed primary key from URL<commit_after>"""
Module bdp_fe.jobconf.views

"""

import logging

from django import forms
from django.conf import settings
from django.contrib import auth, messages
from django.contrib.auth.decorators import login_required
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.core.urlresolvers import reverse
from django.http import Http404, HttpResponseNotFound
from django.shortcuts import get_object_or_404, redirect, render_to_response
from django.template import RequestContext

from bdp_fe.jobconf import data
from bdp_fe.jobconf.cluster import remote
from bdp_fe.jobconf.models import CustomJobModel, Job, JobModel
from bdp_fe.jobconf.views_util import (get_owned_job_or_40x, safe_int_param,
                                       retrieve_results)

LOGGER = logging.getLogger(__name__)
CLUSTER = remote.Cluster(settings.CLUSTER_CONF.get('host'),
                         settings.CLUSTER_CONF.get('port'))

@login_required
def list_jobs(request):
    job_id = safe_int_param(request.GET, 'run_job')
    if job_id:
        run_job(request, job_id)

    return render_to_response('job_listing.html', {
        'title': 'Job listing',
        'jobs': Job.objects.all(),
    }, context_instance=RequestContext(request))

@login_required
def view_results(request, job_id):
    job = get_owned_job_or_40x(request, job_id)
    if job.status != Job.SUCCESSFUL:
        raise Http404
    primary_key = request.GET.get('primary_key')
    results = retrieve_results(job_id, primary_key)
    prototype_result = results[0]
    if not primary_key:
        primary_key = prototype_result.document.keys()[0]
        for result in results:
            result.pk = primary_key
    paginator = Paginator(results, 100)
    page = request.GET.get('page')
    if not page:
        page = 1
    try:
        paginated_results = paginator.page(page)
    except PageNotAnInteger:
        paginated_results = paginator.page(1)
    except EmptyPage:
        paginated_results = paginator.page(paginator.num_pages)

    return render_to_response('job_results.html',
                              {'title' : 'Results of job %s' % job_id,
                               'job_results' : paginated_results,
                               'prototype_result': prototype_result,
                               'hidden_keys': ['_id', 'job_id'],
                               'expand_types': ['dict', 'list'],
                               'primary_key': primary_key},
                              context_instance=RequestContext(request))

def run_job(request, job_id):
    try:
        job = Job.objects.get(id=job_id)
    except Job.DoesNotExist:
        messages.warning(request, "Cannot start job %d: not found" % job_id)
        LOGGER.warning("Job %d not found" % job_id)
        return

    if job.status != Job.CREATED:
        msg = "Cannot start job %s while in %s status" % (
            job.name, job.get_status_display())
        messages.warning(request, msg)
        LOGGER.warning(msg)
        return

    if job.start(CLUSTER):
        messages.info(request, "Job %s was started." % job.name)
    else:
        messages.warning(request, "Cannot start job %s." % job.name)

class NewJobForm(forms.Form):
    name = forms.CharField(max_length=Job.NAME_MAX_LENGTH)
    results_primary_key = forms.CharField(max_length=Job.RESULTS_PK_MAX_LENGTH)

@login_required
def new_job(request):
    if request.method == 'POST':
        form = NewJobForm(request.POST)
        if form.is_valid():
            job = Job(name=form.cleaned_data['name'],
                      user=request.user,
                      results_primary_key=request.user,
                      status=Job.UNCONFIGURED)
            job.save()
            model = CustomJobModel(job=job) # The only option for the moment
            model.save()
            return redirect(reverse('config_job', args=[job.id]))
    else:
        form = NewJobForm()

    return render_to_response('new_job.html', {
        'title': 'New job',
        'form': form,
    }, context_instance=RequestContext(request))

class UploadJarForm(forms.Form):
    file = forms.FileField()

@login_required
def config_job(request, job_id):
    job = get_owned_job_or_40x(request, job_id)
    if job.status != Job.UNCONFIGURED:
        raise Http404()
    model = JobModel.objects.get(job=job).customjobmodel
    if request.method == 'POST':
        form = UploadJarForm(request.POST, request.FILES)
        if form.is_valid() and model.jar_upload(request.FILES['file']):
            job.status = Job.CONFIGURED
            job.save()
            return redirect(reverse('upload_data', args=[job.id]))
        else:
            messages.info(request, 'JAR file upload failed')
    else:
        form = UploadJarForm()
    return render_to_response('upload_jar.html', {
        'title': 'Configure custom job',
        'job': job,
        'form': form,
    }, context_instance=RequestContext(request))

class UploadDataForm(forms.Form):
    file = forms.FileField()

@login_required
def upload_data(request, job_id):
    job = get_owned_job_or_40x(request, job_id)
    if job.status != Job.CONFIGURED:
        raise Http404()

    if request.method == 'POST':
        form = UploadDataForm(request.POST, request.FILES)
        if form.is_valid() and job.data_upload(request.FILES['file'],
                                               CLUSTER):
            Job.input_data = job.hdfs_data_path()
            job.save()
            return redirect(reverse('list_jobs'))
        else:
            messages.info(request, 'Data file upload failed')
    else:
        form = UploadDataForm()
    return render_to_response('upload_data.html', {
        'title': 'Upload job %s data' % job.name,
        'job': job,
        'form': form,
    }, context_instance=RequestContext(request))
<|endoftext|>


<commit_before>import urllib
import re
import asyncio
from plugin_manager import BasePlugin
from rs_errors import CommandSyntaxError, UserPermissionError
from rs_utils import respond, is_positive
from command_dispatcher import Command
from discord import InvalidArgument
from traceback import format_exception, format_exc

class BotManagement(BasePlugin):
    name = "bot_management"

    @Command("Shutdown",
             doc="Shuts down the bot.",
             syntax="N/A",
             category="bot_management",
             perms={"manage_guild"})
    async def _shutdown(self, msg):
        await respond(msg, "**AFFIRMATIVE. SHUTTING DOWN.**")
        await self.client.stop_bot()

    @Command("Save",
             doc="Saves plugin storage data.",
             category="bot_management",
             perms={"manage_guild"})
    async def _save(self, msg):
        self.plugin_manager.shelve.sync()
        self.logger.debug("Writing to shelve...")
        await respond(msg, "**AFFIRMATIVE. Storage saved to disk.**")

    @Command("UpdateAvatar",
             doc="Updates the bot's avatar.",
             syntax="(URL)",
             category="bot_management",
             perms={"manage_guild"})
    async def _update_avatar(self, msg):
        if "bot_maintainers" not in self.config_manager.config:
            raise UserPermissionError("No bot maintainers are set!")
        elif msg.author.id not in self.config_manager.config.bot_maintainers:
            raise UserPermissionError
        url = " ".join(msg.content.split()[1:])
        if url:
            try:
                img = urllib.request.urlopen(url).read()
                await self.client.user.edit(avatar=img)
                await respond(msg, "**AVATAR UPDATED.**")
            except (urllib.request.URLError, ValueError):
                await respond(msg, "**WARNING: Invalid URL provided.**")
            except InvalidArgument:
                await respond(msg, "**NEGATIVE. Image must be a PNG or JPG.**")
        else:
            raise CommandSyntaxError("No URL provided.")

    @Command("UpdateName",
             doc="Updates the bot's (nick)name.",
             syntax="[change username?] (name)",
             category="bot_management",
             perms={"manage_guild"})
    async def _update_name(self, msg):
        args = msg.clean_content.split()[1:]
        edit_username = False
        if is_positive(args[0]):
            args.pop(0)
            edit_username = True
        elif args[0].lower() == "reset":
            await msg.guild.me.edit(nick=None)
            await respond(msg, "**ANALYSIS: Nickname reset.**")
            return
        newname = " ".join(args)
        if edit_username:
            if "bot_maintainers" not in self.config_manager.config:
                raise UserPermissionError("No bot maintainers are set!")
            if msg.author.id not in self.config_manager.config.bot_maintainers:
                raise UserPermissionError
            await self.client.user.edit(username=newname)
            await respond(msg, f"**ANALYSIS: Username changed to {newname} successfully.**")
        else:
            await msg.guild.me.edit(nick=newname)
            await respond(msg, f"**ANALYSIS: Nickname changed to {newname} successfully.**")

    @Command("Activate",
             doc="Activates an inactive plugin.",
             syntax="(plugin) [permanent]",
             category="bot_management",
             perms={"manage_guild"})
    async def _activate(self, msg):
        if "bot_maintainers" not in self.config_manager.config:
            raise UserPermissionError("No bot maintainers are set!")
        elif msg.author.id not in self.config_manager.config.bot_maintainers:
            raise UserPermissionError
        plgname = msg.content.split()[1]
        try:
            permanent = is_positive(msg.content.split()[2])
        except IndexError:
            permanent = False
        all_plugins = self.plugin_manager.plugins
        if plgname in all_plugins:
            if plgname not in self.plugins:
                if plgname in self.config_manager.config.disabled_plugins and permanent:
                    self.config_manager.config.disabled_plugins.remove(plgname)
                    self.config_manager.save_config()
                await self.plugin_manager.activate(plgname)
                await respond(msg, f"**ANALYSIS: Plugin {plgname} was activated successfully.**")
            else:
                await respond(msg, f"**ANALYSIS: Plugin {plgname} is already activated.**")
        else:
            await respond(msg, f"**WARNING: Could not find plugin {plgname}.**")

    @Command("Deactivate",
             doc="Deactivates an active plugin.",
             syntax="(plugin) [permanent]",
             category="bot_management",
             perms={"manage_guild"})
    async def _deactivate(self, msg):
        if "bot_maintainers" not in self.config_manager.config:
            raise UserPermissionError("No bot maintainers are set!")
        elif msg.author.id not in self.config_manager.config.bot_maintainers:
            raise UserPermissionError
        plgname = msg.content.split()[1].lower()
        try:
            permanent = is_positive(msg.content.split()[2])
        except IndexError:
            permanent = False
        if plgname == self.name:
            await respond(msg, f"**WARNING: Cannot deactivate {self.name}.**")
        elif plgname in self.plugins:
            if plgname not in self.config_manager.config.disabled_plugins and permanent:
                self.config_manager.config.disabled_plugins.append(plgname)
                self.config_manager.save_config()
            await self.plugin_manager.deactivate(plgname)
            await respond(msg, f"**ANALYSIS: Plugin {plgname} was deactivated successfully.**")
        else:
            await respond(msg, f"**ANALYSIS: Plugin {plgname} is not active.**")

    @Command("ListPlugins",
             doc="Lists all plugins and their activation status.",
             syntax="(plugin)",
             category="bot_management",
             perms={"manage_guild"})
    async def _list_plugins(self, msg):
        if "bot_maintainers" not in self.config_manager.config:
            raise UserPermissionError("No bot maintainers are set!")
        elif msg.author.id not in self.config_manager.config.bot_maintainers:
            raise UserPermissionError
        active_plgs = ", ".join(self.plugins.keys())
        if not active_plgs:
            active_plgs = "None."
        all_plgs = list(self.plugin_manager.plugins.keys())
        inactive_plgs = ", ".join([x for x in all_plgs if x not in self.plugins])
        if not inactive_plgs:
            inactive_plgs = "None."
        await respond(msg, f"**ANALYSIS: Plugins are as follows:**```\nActive: {active_plgs}\n"
                           f"Inactive: {inactive_plgs}\n```")

    @Command("GetConfig",
             doc="Gets the config value at the specified path. Use <server> to fill in the server ID.",
             syntax="(path/to/value)",
             category="bot_management",
             perms={"manage_guild"})
    async def _get_config(self, msg):
        if "bot_maintainers" not in self.config_manager.config:
            raise UserPermissionError("No bot maintainers are set!")
        elif msg.author.id not in self.config_manager.config.bot_maintainers:
            raise UserPermissionError
        conf = self.config_manager.config
        args = msg.clean_content.split()[1:]
        try:
            path = args[0]
        except IndexError:
            raise CommandSyntaxError("Missing path to config value.")
        path = path.replace("<server>", str(msg.guild.id))
        if path.startswith("/"):
            path = path[1:]
        path = path.split("/")
        val = conf
        for k in path:
            try:
                val = val[k]
            except TypeError:
                try:
                    i = int(k)
                    val = val[i]
                except ValueError:
                    raise CommandSyntaxError(f"{k} is not a valid integer.")
                except IndexError:
                    raise CommandSyntaxError(f"Path {args[0]} is invalid.")
            except KeyError:
                raise CommandSyntaxError(f"Path {args[0]} is invalid.")
        await respond(msg, f"**ANALYSIS: Value of {args[0]}:** `{val}`")

    @Command("SetConfig",
             doc="Edits the config key at the specified path. Use <server> to fill in the server ID.",
             syntax="(path/to/edit) (value)",
             category="bot_management",
             perms={"manage_guild"})
    async def _set_config(self, msg):
        if "bot_maintainers" not in self.config_manager.config:
            raise UserPermissionError("No bot maintainers are set!")
        elif msg.author.id not in self.config_manager.config.bot_maintainers:
            raise UserPermissionError
        conf = self.config_manager.config
        args = msg.clean_content.split()[1:]
        try:
            path = args[0]
            path = path.replace("<server>", str(msg.guild.id))
            if path.startswith("/"):
                path = path[1:]
            path = path.split("/")
            key = path.pop()
            value = " ".join(args[1:])
            if not value:
                raise CommandSyntaxError("Missing new config value.")
        except IndexError:
            raise CommandSyntaxError("Missing path to config value.")
        val = conf
        for k in path:
            try:
                val = val[k]
            except TypeError:
                try:
                    i = int(k)
                    val = val[i]
                except ValueError:
                    raise CommandSyntaxError(f"{k} is not a valid integer.")
                except IndexError:
                    raise CommandSyntaxError(f"Path {args[0]} is invalid.")
            except KeyError:
                raise CommandSyntaxError(f"Path {args[0]} is invalid.")
        try:
            orig = val[key]
        except TypeError:
            try:
                key = int(key)
                orig = val[key]
            except ValueError:
                raise CommandSyntaxError(f"{k} is not a valid integer.")
            except IndexError:
                raise CommandSyntaxError(f"Path {args[0]} is invalid.")
        except KeyError:
            raise CommandSyntaxError(f"Path {args[0]} is invalid.")
        if isinstance(orig, str):
            val[key] = value
        elif isinstance(orig, bool):
                val[key] = is_positive(value)
        elif isinstance(orig, int):
            try:
                value = int(value)
                val[key] = value
            except ValueError:
                raise CommandSyntaxError(f"{value} is not a valid integer.")
        elif isinstance(orig, float):
            try:
                value = float(value)
                val[key] = value
            except ValueError:
                raise CommandSyntaxError(f"{value} is not a valid floating-point number.")
        else:
            raise CommandSyntaxError(f"{args[0]} is an object or array.")
        self.config_manager.save_config()
        await respond(msg, f"**ANALYSIS: Config value {args[0]} edited to** `{value}` **successfully.**")

    @Command("LastError",
             doc="Gets the last error to occur in the specified context.",
             syntax="(command/event/unhandled)",
             category="debug",
             perms={"manage_guild"})
    async def _last_error(self, msg):
        try:
            args = msg.clean_content.split(" ", 1)[1].lower()
        except IndexError:
            raise CommandSyntaxError("No error context specified.")
        if args == "command":
            e = self.client.command_dispatcher.last_error
        elif args == "event":
            e = self.plugin_manager.last_error
        elif args == "unhandled":
            e = self.client.last_error
        else:
            raise CommandSyntaxError("Invalid error context.")
        if e:
            excstr = "\n".join(format_exception(*e))
            await respond(msg, f"**ANALYSIS: Last error in context {args}:** ```Python\n{excstr}\n```")
        else:
            await respond(msg, f"**ANALYSIS: No error in context {args}.**")

    @Command("Execute", "Exec", "Eval",
             doc="Executes the given Python code. Be careful, you can really break things with this!\n"
                 "Provided variables are `aef` (shorthand for asyncio.ensure_future) and `self.result` "
                 "(printed by the bot after execution).",
             syntax="(code in code block)",
             category="debug",
             perms={"manage_guild"},
             bot_maintainers_only=True)
    async def _execute(self, msg):
        try:
            arg = msg.content.split(" ", 1)[1]
            code = re.match(r"```.*?\n(.*)```", arg, re.S).group(1)
        except IndexError:
            raise CommandSyntaxError("No code provided.")
        except AttributeError:
            raise CommandSyntaxError("Code is not in valid code block.")
        # Convenience variables for printing results and ensure_future
        aef = asyncio.ensure_future
        self.res = None
        try:
            exec(code, globals(), locals())
        except:
            await respond(msg, f"**WARNING: Error occurred while executing code. Traceback:**\n"
                               f"```Py\n{format_exc()}\n```")
        if self.res is not None:
            await respond(msg, f"**ANALYSIS: Result: {self.res}**")
            self.res = None
<commit_msg>!eval now takes single-tick code blocks and understands line breaks.<commit_after>import urllib
import re
import asyncio
from plugin_manager import BasePlugin
from rs_errors import CommandSyntaxError, UserPermissionError
from rs_utils import respond, is_positive
from command_dispatcher import Command
from discord import InvalidArgument
from traceback import format_exception, format_exc

class BotManagement(BasePlugin):
    name = "bot_management"

    @Command("Shutdown",
             doc="Shuts down the bot.",
             syntax="N/A",
             category="bot_management",
             perms={"manage_guild"})
    async def _shutdown(self, msg):
        await respond(msg, "**AFFIRMATIVE. SHUTTING DOWN.**")
        await self.client.stop_bot()

    @Command("Save",
             doc="Saves plugin storage data.",
             category="bot_management",
             perms={"manage_guild"})
    async def _save(self, msg):
        self.plugin_manager.shelve.sync()
        self.logger.debug("Writing to shelve...")
        await respond(msg, "**AFFIRMATIVE. Storage saved to disk.**")

    @Command("UpdateAvatar",
             doc="Updates the bot's avatar.",
             syntax="(URL)",
             category="bot_management",
             perms={"manage_guild"})
    async def _update_avatar(self, msg):
        if "bot_maintainers" not in self.config_manager.config:
            raise UserPermissionError("No bot maintainers are set!")
        elif msg.author.id not in self.config_manager.config.bot_maintainers:
            raise UserPermissionError
        url = " ".join(msg.content.split()[1:])
        if url:
            try:
                img = urllib.request.urlopen(url).read()
                await self.client.user.edit(avatar=img)
                await respond(msg, "**AVATAR UPDATED.**")
            except (urllib.request.URLError, ValueError):
                await respond(msg, "**WARNING: Invalid URL provided.**")
            except InvalidArgument:
                await respond(msg, "**NEGATIVE. Image must be a PNG or JPG.**")
        else:
            raise CommandSyntaxError("No URL provided.")

    @Command("UpdateName",
             doc="Updates the bot's (nick)name.",
             syntax="[change username?] (name)",
             category="bot_management",
             perms={"manage_guild"})
    async def _update_name(self, msg):
        args = msg.clean_content.split()[1:]
        edit_username = False
        if is_positive(args[0]):
            args.pop(0)
            edit_username = True
        elif args[0].lower() == "reset":
            await msg.guild.me.edit(nick=None)
            await respond(msg, "**ANALYSIS: Nickname reset.**")
            return
        newname = " ".join(args)
        if edit_username:
            if "bot_maintainers" not in self.config_manager.config:
                raise UserPermissionError("No bot maintainers are set!")
            if msg.author.id not in self.config_manager.config.bot_maintainers:
                raise UserPermissionError
            await self.client.user.edit(username=newname)
            await respond(msg, f"**ANALYSIS: Username changed to {newname} successfully.**")
        else:
            await msg.guild.me.edit(nick=newname)
            await respond(msg, f"**ANALYSIS: Nickname changed to {newname} successfully.**")

    @Command("Activate",
             doc="Activates an inactive plugin.",
             syntax="(plugin) [permanent]",
             category="bot_management",
             perms={"manage_guild"})
    async def _activate(self, msg):
        if "bot_maintainers" not in self.config_manager.config:
            raise UserPermissionError("No bot maintainers are set!")
        elif msg.author.id not in self.config_manager.config.bot_maintainers:
            raise UserPermissionError
        plgname = msg.content.split()[1]
        try:
            permanent = is_positive(msg.content.split()[2])
        except IndexError:
            permanent = False
        all_plugins = self.plugin_manager.plugins
        if plgname in all_plugins:
            if plgname not in self.plugins:
                if plgname in self.config_manager.config.disabled_plugins and permanent:
                    self.config_manager.config.disabled_plugins.remove(plgname)
                    self.config_manager.save_config()
                await self.plugin_manager.activate(plgname)
                await respond(msg, f"**ANALYSIS: Plugin {plgname} was activated successfully.**")
            else:
                await respond(msg, f"**ANALYSIS: Plugin {plgname} is already activated.**")
        else:
            await respond(msg, f"**WARNING: Could not find plugin {plgname}.**")

    @Command("Deactivate",
             doc="Deactivates an active plugin.",
             syntax="(plugin) [permanent]",
             category="bot_management",
             perms={"manage_guild"})
    async def _deactivate(self, msg):
        if "bot_maintainers" not in self.config_manager.config:
            raise UserPermissionError("No bot maintainers are set!")
        elif msg.author.id not in self.config_manager.config.bot_maintainers:
            raise UserPermissionError
        plgname = msg.content.split()[1].lower()
        try:
            permanent = is_positive(msg.content.split()[2])
        except IndexError:
            permanent = False
        if plgname == self.name:
            await respond(msg, f"**WARNING: Cannot deactivate {self.name}.**")
        elif plgname in self.plugins:
            if plgname not in self.config_manager.config.disabled_plugins and permanent:
                self.config_manager.config.disabled_plugins.append(plgname)
                self.config_manager.save_config()
            await self.plugin_manager.deactivate(plgname)
            await respond(msg, f"**ANALYSIS: Plugin {plgname} was deactivated successfully.**")
        else:
            await respond(msg, f"**ANALYSIS: Plugin {plgname} is not active.**")

    @Command("ListPlugins",
             doc="Lists all plugins and their activation status.",
             syntax="(plugin)",
             category="bot_management",
             perms={"manage_guild"})
    async def _list_plugins(self, msg):
        if "bot_maintainers" not in self.config_manager.config:
            raise UserPermissionError("No bot maintainers are set!")
        elif msg.author.id not in self.config_manager.config.bot_maintainers:
            raise UserPermissionError
        active_plgs = ", ".join(self.plugins.keys())
        if not active_plgs:
            active_plgs = "None."
        all_plgs = list(self.plugin_manager.plugins.keys())
        inactive_plgs = ", ".join([x for x in all_plgs if x not in self.plugins])
        if not inactive_plgs:
            inactive_plgs = "None."
        await respond(msg, f"**ANALYSIS: Plugins are as follows:**```\nActive: {active_plgs}\n"
                           f"Inactive: {inactive_plgs}\n```")

    @Command("GetConfig",
             doc="Gets the config value at the specified path. Use <server> to fill in the server ID.",
             syntax="(path/to/value)",
             category="bot_management",
             perms={"manage_guild"})
    async def _get_config(self, msg):
        if "bot_maintainers" not in self.config_manager.config:
            raise UserPermissionError("No bot maintainers are set!")
        elif msg.author.id not in self.config_manager.config.bot_maintainers:
            raise UserPermissionError
        conf = self.config_manager.config
        args = msg.clean_content.split()[1:]
        try:
            path = args[0]
        except IndexError:
            raise CommandSyntaxError("Missing path to config value.")
        path = path.replace("<server>", str(msg.guild.id))
        if path.startswith("/"):
            path = path[1:]
        path = path.split("/")
        val = conf
        for k in path:
            try:
                val = val[k]
            except TypeError:
                try:
                    i = int(k)
                    val = val[i]
                except ValueError:
                    raise CommandSyntaxError(f"{k} is not a valid integer.")
                except IndexError:
                    raise CommandSyntaxError(f"Path {args[0]} is invalid.")
            except KeyError:
                raise CommandSyntaxError(f"Path {args[0]} is invalid.")
        await respond(msg, f"**ANALYSIS: Value of {args[0]}:** `{val}`")

    @Command("SetConfig",
             doc="Edits the config key at the specified path. Use <server> to fill in the server ID.",
             syntax="(path/to/edit) (value)",
             category="bot_management",
             perms={"manage_guild"})
    async def _set_config(self, msg):
        if "bot_maintainers" not in self.config_manager.config:
            raise UserPermissionError("No bot maintainers are set!")
        elif msg.author.id not in self.config_manager.config.bot_maintainers:
            raise UserPermissionError
        conf = self.config_manager.config
        args = msg.clean_content.split()[1:]
        try:
            path = args[0]
            path = path.replace("<server>", str(msg.guild.id))
            if path.startswith("/"):
                path = path[1:]
            path = path.split("/")
            key = path.pop()
            value = " ".join(args[1:])
            if not value:
                raise CommandSyntaxError("Missing new config value.")
        except IndexError:
            raise CommandSyntaxError("Missing path to config value.")
        val = conf
        for k in path:
            try:
                val = val[k]
            except TypeError:
                try:
                    i = int(k)
                    val = val[i]
                except ValueError:
                    raise CommandSyntaxError(f"{k} is not a valid integer.")
                except IndexError:
                    raise CommandSyntaxError(f"Path {args[0]} is invalid.")
            except KeyError:
                raise CommandSyntaxError(f"Path {args[0]} is invalid.")
        try:
            orig = val[key]
        except TypeError:
            try:
                key = int(key)
                orig = val[key]
            except ValueError:
                raise CommandSyntaxError(f"{k} is not a valid integer.")
            except IndexError:
                raise CommandSyntaxError(f"Path {args[0]} is invalid.")
        except KeyError:
            raise CommandSyntaxError(f"Path {args[0]} is invalid.")
        if isinstance(orig, str):
            val[key] = value
        elif isinstance(orig, bool):
                val[key] = is_positive(value)
        elif isinstance(orig, int):
            try:
                value = int(value)
                val[key] = value
            except ValueError:
                raise CommandSyntaxError(f"{value} is not a valid integer.")
        elif isinstance(orig, float):
            try:
                value = float(value)
                val[key] = value
            except ValueError:
                raise CommandSyntaxError(f"{value} is not a valid floating-point number.")
        else:
            raise CommandSyntaxError(f"{args[0]} is an object or array.")
        self.config_manager.save_config()
        await respond(msg, f"**ANALYSIS: Config value {args[0]} edited to** `{value}` **successfully.**")

    @Command("LastError",
             doc="Gets the last error to occur in the specified context.",
             syntax="(command/event/unhandled)",
             category="debug",
             perms={"manage_guild"})
    async def _last_error(self, msg):
        try:
            args = msg.clean_content.split(" ", 1)[1].lower()
        except IndexError:
            raise CommandSyntaxError("No error context specified.")
        if args == "command":
            e = self.client.command_dispatcher.last_error
        elif args == "event":
            e = self.plugin_manager.last_error
        elif args == "unhandled":
            e = self.client.last_error
        else:
            raise CommandSyntaxError("Invalid error context.")
        if e:
            excstr = "\n".join(format_exception(*e))
            await respond(msg, f"**ANALYSIS: Last error in context {args}:** ```Python\n{excstr}\n```")
        else:
            await respond(msg, f"**ANALYSIS: No error in context {args}.**")

    @Command("Execute", "Exec", "Eval",
             doc="Executes the given Python code. Be careful, you can really break things with this!\n"
                 "Provided variables are `aef` (shorthand for asyncio.ensure_future) and `self.result` "
                 "(printed by the bot after execution).",
             syntax="(code in code block)",
             category="debug",
             perms={"manage_guild"},
             bot_maintainers_only=True)
    async def _execute(self, msg):
        try:
            arg = re.split(r"\s+", msg.content, 1)[1]
            t_match = re.match(r"`([^\n\r`]+)`|```.*?\s+(.*)```", arg, re.DOTALL)
            code = t_match.group(1) or t_match.group(2)
        except IndexError:
            raise CommandSyntaxError("No code provided.")
        except AttributeError:
            raise CommandSyntaxError("Code is not in valid code block.")
        # Convenience variables for printing results and ensure_future
        aef = asyncio.ensure_future
        self.res = None
        try:
            exec(code, globals(), locals())
        except:
            await respond(msg, f"**WARNING: Error occurred while executing code. Traceback:**\n"
                               f"```Py\n{format_exc()}\n```")
        if self.res is not None:
            await respond(msg, f"**ANALYSIS: Result: {self.res}**")
            self.res = None
<|endoftext|>


<commit_before>from django.contrib import admin

from .models import (
    Contest,
    Contestant,
    Performance,
)

@admin.register(Contest)
class ContestAdmin(admin.ModelAdmin):
    save_on_top = True

@admin.register(Contestant)
class ContestantAdmin(admin.ModelAdmin):
    save_on_top = True

    fields = (
        'name', (
            'website',
            'facebook',
        ),
        'phone', (
            'lead',
            'tenor',
            'baritone',
            'bass',
        ),
        'director',
        'district',
        'prelim',
    )

    readonly_fields = (
        'name',
        'district',
        'prelim',
    )

    search_fields = (
        'name',
    )

    list_display = (
        'name',
        'website',
        'facebook',
        'phone',
        'prelim',
    )

    list_filter = (
        'contestant_type',
    )

@admin.register(Performance)
class PerformanceAdmin(admin.ModelAdmin):
    save_on_top = True

    fields = (
        'contestant',
        'contest',
        'contest_round',
        'appearance',
        'song1', (
            'mus1',
            'prs1',
            'sng1',
        ),
        'song2', (
            'mus2',
            'prs2',
            'sng2',
        ),
    )

    readonly_fields = (
        'contestant',
        'contest',
        'contest_round',
        'appearance',
    )

    list_display = (
        '__unicode__',
        'song1',
        'score1',
        'song2',
        'score2',
        'appearance',
    )

    list_filter = (
        'contest',
        'contest_round',
    )
<commit_msg>Add picture field to admin<commit_after>from django.contrib import admin

from .models import (
    Contest,
    Contestant,
    Performance,
)

@admin.register(Contest)
class ContestAdmin(admin.ModelAdmin):
    save_on_top = True

@admin.register(Contestant)
class ContestantAdmin(admin.ModelAdmin):
    save_on_top = True

    fields = (
        'name', (
            'website',
            'facebook',
        ),
        'phone', (
            'lead',
            'tenor',
            'baritone',
            'bass',
        ),
        'director',
        'district',
        'prelim',
        'picture',
    )

    readonly_fields = (
        'name',
        'district',
        'prelim',
    )

    search_fields = (
        'name',
    )

    list_display = (
        'name',
        'website',
        'facebook',
        'phone',
        'prelim',
    )

    list_filter = (
        'contestant_type',
    )

@admin.register(Performance)
class PerformanceAdmin(admin.ModelAdmin):
    save_on_top = True

    fields = (
        'contestant',
        'contest',
        'contest_round',
        'appearance',
        'song1', (
            'mus1',
            'prs1',
            'sng1',
        ),
        'song2', (
            'mus2',
            'prs2',
            'sng2',
        ),
    )

    readonly_fields = (
        'contestant',
        'contest',
        'contest_round',
        'appearance',
    )

    list_display = (
        '__unicode__',
        'song1',
        'score1',
        'song2',
        'score2',
        'appearance',
    )

    list_filter = (
        'contest',
        'contest_round',
    )
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Support a YAML file hosts.yml as external inventory in Ansible

# Copyright (C) 2012  Jeroen Hoekx <jeroen@hoekx.be>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
File format:

- <hostname>

or

- host: <hostname>
  vars:
  - myvar: value
  - myvbr: vblue
  groups:
  - mygroup1
  - mygroup2

or

- group: <groupname>
  vars:
  - groupvar: value
  hosts:
  - myhost1
  - myhost2
  groups:
  - subgroup1
  - subgroup2

Any statement except the first definition is optional.
"""

import json
import os
import sys

from optparse import OptionParser

import yaml

class Host():
    def __init__(self, name):
        self.name = name
        self.groups = []
        self.vars = {}
    def __repr__(self):
        return "Host('%s')"%(self.name)

    def set_variable(self, key, value):
        self.vars[key] = value

    def get_variables(self):
        result = {}
        for group in self.groups:
            for k,v in group.get_variables().items():
                result[k] = v
        for k, v in self.vars.items():
            result[k] = v
        return result

    def add_group(self, group):
        if group not in self.groups:
            self.groups.append(group)

class Group():
    def __init__(self, name):
        self.name = name
        self.hosts = []
        self.vars = {}
        self.subgroups = []
        self.parents = []
    def __repr__(self):
        return "Group('%s')"%(self.name)

    def get_hosts(self):
        """ List all hosts in this group, including subgroups """
        result = [ host for host in self.hosts ]
        for group in self.subgroups:
            for host in group.get_hosts():
                if host not in result:
                    result.append(host)
        return result

    def add_host(self, host):
        if host not in self.hosts:
            self.hosts.append(host)
            host.add_group(self)

    def add_subgroup(self, group):
        if group not in self.subgroups:
            self.subgroups.append(group)
            group.add_parent(self)

    def add_parent(self, group):
        if group not in self.parents:
            self.parents.append(group)

    def set_variable(self, key, value):
        self.vars[key] = value

    def get_variables(self):
        result = {}
        for group in self.parents:
            result.update( group.get_variables() )
        result.update(self.vars)
        return result

def find_group(name, groups):
    for group in groups:
        if name == group.name:
            return group

def parse_vars(vars, obj):
    ### vars can be a list of dicts or a dictionary
    if type(vars) == dict:
        for k,v in vars.items():
            obj.set_variable(k, v)
    elif type(vars) == list:
        for var in vars:
            k,v = var.items()[0]
            obj.set_variable(k, v)

def parse_yaml(yaml_hosts):
    groups = []

    all_hosts = Group('all')

    ungrouped = Group('ungrouped')
    groups.append(ungrouped)

    ### groups first, so hosts can be added to 'ungrouped' if necessary
    subgroups = []
    for entry in yaml_hosts:
        if 'group' in entry and type(entry)==dict:
            group = find_group(entry['group'], groups)
            if not group:
                group = Group(entry['group'])
                groups.append(group)

            if 'vars' in entry:
                parse_vars(entry['vars'], group)

            if 'hosts' in entry:
                for host_name in entry['hosts']:
                    host = None
                    for test_host in all_hosts.get_hosts():
                        if test_host.name == host_name:
                            host = test_host
                            break
                    else:
                        host = Host(host_name)
                        all_hosts.add_host(host)
                    group.add_host(host)

            if 'groups' in entry:
                for subgroup in entry['groups']:
                    subgroups.append((group.name, subgroup))

    for name, sub_name in subgroups:
        group = find_group(name, groups)
        subgroup = find_group(sub_name, groups)
        group.add_subgroup(subgroup)

    for entry in yaml_hosts:
        ### a host is either a dict or a single line definition
        if type(entry) in [str, unicode]:
            for test_host in all_hosts.get_hosts():
                if test_host.name == entry:
                    break
            else:
                host = Host(entry)
                all_hosts.add_host(host)
                ungrouped.add_host(host)

        elif 'host' in entry:
            host = None
            no_group = False
            for test_host in all_hosts.get_hosts():
                ### all hosts contains only hosts already in groups
                if test_host.name == entry['host']:
                    host = test_host
                    break
            else:
                host = Host(entry['host'])
                all_hosts.add_host(host)
                no_group = True

            if 'vars' in entry:
                parse_vars(entry['vars'], host)

            if 'groups' in entry:
                for test_group in groups:
                    if test_group.name in entry['groups']:
                        test_group.add_host(host)
                        all_hosts.add_host(host)
                        no_group = False

            if no_group:
                ungrouped.add_host(host)

    return groups, all_hosts

parser = OptionParser()
parser.add_option('-l', '--list', default=False, dest="list_hosts", action="store_true")
parser.add_option('-H', '--host', default=None, dest="host")
parser.add_option('-e', '--extra-vars', default=None, dest="extra")
options, args = parser.parse_args()

base_dir = os.path.dirname(os.path.realpath(__file__))
hosts_file = os.path.join(base_dir, 'hosts.yml')

with open(hosts_file) as f:
    yaml_hosts = yaml.safe_load( f.read() )

groups, all_hosts = parse_yaml(yaml_hosts)

if options.list_hosts == True:
    result = {}
    for group in groups:
        result[group.name] = [host.name for host in group.get_hosts()]
    print json.dumps(result)
    sys.exit(0)

if options.host is not None:
    result = {}
    host = None
    for test_host in all_hosts.get_hosts():
        if test_host.name == options.host:
            host = test_host
            break
    result = host.get_variables()
    if options.extra:
        k,v = options.extra.split("=")
        result[k] = v
    print json.dumps(result)
    sys.exit(0)

parser.print_help()
sys.exit(1)
<commit_msg>Remove YAML inventory example.<commit_after><|endoftext|>


<commit_before>from plumeria.command import CommandError

def parse_list(s, allow_spaces=True):
    s = s.strip()
    if '\n' in s:
        delimiter = '\n'
    elif ';' in s:
        delimiter = ';'
    elif ',' in s:
        delimiter = ','
    elif allow_spaces:
        delimiter = " "
    else:
        return [s]

    return list(filter(lambda s: len(s), map(lambda s: s.strip(), s.split(delimiter))))

def parse_numeric_list(s):
    numbers = parse_list(s)
    try:
        return list(map(lambda x: float(x), numbers))
    except ValueError as e:
        raise CommandError("All the entries must be numbers")

def build_list(items):
    return "\n".join(["\N{BULLET} " + str(i) for i in items])
<commit_msg>Remove bullets in parse_list.<commit_after>import re

from plumeria.command import CommandError

LIST_CLEAN_RE = re.compile("^\N{BULLET} +")

def parse_list(s, allow_spaces=True):
    s = s.strip()
    if '\n' in s:
        delimiter = '\n'
    elif ';' in s:
        delimiter = ';'
    elif ',' in s:
        delimiter = ','
    elif allow_spaces:
        delimiter = " "
    else:
        return [s]

    return list(filter(lambda s: len(s), map(lambda s: LIST_CLEAN_RE.sub('', s.strip()), s.split(delimiter))))

def parse_numeric_list(s):
    numbers = parse_list(s)
    try:
        return list(map(lambda x: float(x), numbers))
    except ValueError as e:
        raise CommandError("All the entries must be numbers")

def build_list(items):
    return "\n".join(["\N{BULLET} " + str(i) for i in items])
<|endoftext|>


<commit_before>from os import path
import datetime

from django.core.urlresolvers import reverse
from django.contrib.gis.geos import Point
from django.utils import timezone
from django_dynamic_fixture import G
from rest_framework import status

from main.models import Dataset, Site, Record
from main.tests.api import helpers

class TestPermission(helpers.BaseUserTestCase):
    # TODO
    pass

class TestGenericRecord(helpers.BaseUserTestCase):
    def _more_setup(self):
        self.fields = [
            {
                "name": "Column A",
                "type": "string",
                "constraints": helpers.NOT_REQUIRED_CONSTRAINTS
            },
            {
                "name": "Column B",
                "type": "string",
                "constraints": helpers.REQUIRED_CONSTRAINTS
            }
        ]
        self.data_package = helpers.create_data_package_from_fields(self.fields)
        self.ds = G(Dataset,
                    project=self.project_1,
                    type=Dataset.TYPE_GENERIC,
                    data_package=self.data_package)
        self.url = reverse('api:dataset-upload', kwargs={'pk': self.ds.pk})

    def test_upload_csv_happy_path(self):
        csv_data = [
            ['Column A', 'Column B'],
            ['A1', 'B1'],
            ['A2', 'B2']
        ]
        file_ = helpers.rows_to_csv_file(csv_data)
        client = self.custodian_1_client
        self.assertEquals(0, self.ds.record_queryset.count())
        file_name = path.basename(file_)
        with open(file_) as fp:
            data = {
                'file': fp,
                'strict': True  # upload in strict mode
            }
            resp = client.post(self.url, data=data, format='multipart')
            self.assertEquals(status.HTTP_200_OK, resp.status_code)
            # The records should be saved in order of the row
            qs = self.ds.record_queryset.order_by('pk')
            self.assertEquals(len(csv_data) - 1, qs.count())

            index = 0
            record = qs[index]
            expected_data = {
                'Column A': 'A1',
                'Column B': 'B1',
            }
            self.assertEquals(expected_data, record.data)
            # test that source_info contains the file_name and row_counter
            source_info = record.source_info
            self.assertIsNotNone(source_info)
            expected_info = {
                'file_name': file_name,
                'row': index + 2
            }
            self.assertEqual(source_info, expected_info)

            index = 1
            record = qs[index]
            expected_data = {
                'Column A': 'A2',
                'Column B': 'B2',
            }
            self.assertEquals(expected_data, record.data)
            # test that source_info contains the file_name and row_counter
            source_info = record.source_info
            self.assertIsNotNone(source_info)
            expected_info = {
                'file_name': file_name,
                'row': index + 2
            }
            self.assertEqual(source_info, expected_info)

            self.assertEquals(self.project_1.record_count, len(csv_data) - 1)
            self.assertEquals(self.ds.record_count, len(csv_data) - 1)

    def test_upload_xlsx_happy_path(self):
        csv_data = [
            ['Column A', 'Column B'],
            ['A1', 'B1'],
            ['A2', 'B2']
        ]
        file_ = helpers.rows_to_xlsx_file(csv_data)
        client = self.custodian_1_client
        self.assertEquals(0, self.ds.record_queryset.count())
        file_name = path.basename(file_)
        with open(file_, 'rb') as fp:
            data = {
                'file': fp,
                'strict': True  # upload in strict mode
            }
            resp = client.post(self.url, data=data, format='multipart')
            self.assertEquals(status.HTTP_200_OK, resp.status_code)
            # The records should be saved in order of the row
            qs = self.ds.record_queryset.order_by('pk')
            self.assertEquals(len(csv_data) - 1, qs.count())

            index = 0
            record = qs[index]
            expected_data = {
                'Column A': 'A1',
                'Column B': 'B1',
            }
            self.assertEquals(expected_data, record.data)
            # test that source_info contains the file_name and row_counter
            source_info = record.source_info
            self.assertIsNotNone(source_info)
            expected_info = {
                'file_name': file_name,
                'row': index + 2
            }
            self.assertEqual(source_info, expected_info)

            index = 1
            record = qs[index]
            expected_data = {
                'Column A': 'A2',
                'Column B': 'B2',
            }
            self.assertEquals(expected_data, record.data)
            # test that source_info contains the file_name and row_counter
            source_info = record.source_info
            self.assertIsNotNone(source_info)
            expected_info = {
                'file_name': file_name,
                'row': index + 2
            }
            self.assertEqual(source_info, expected_info)

            self.assertEquals(self.project_1.record_count, len(csv_data) - 1)
            self.assertEquals(self.ds.record_count, len(csv_data) - 1)

    def test_upload_blank_column(self):
        """ Blank column should be ignored"""
        csv_data = [
            ['Column A', '', '   ', 'Column B'],
            ['A1', '', 'something', 'B1'],
            ['A2', '', 'something', 'B2']
        ]
        file_ = helpers.rows_to_xlsx_file(csv_data)
        client = self.custodian_1_client
        self.assertEquals(0, self.ds.record_queryset.count())
        file_name = path.basename(file_)
        with open(file_, 'rb') as fp:
            data = {
                'file': fp,
                'strict': True  # upload in strict mode
            }
            resp = client.post(self.url, data=data, format='multipart')
            self.assertEquals(status.HTTP_200_OK, resp.status_code)
            # The records should be saved in order of the row
            qs = self.ds.record_queryset.order_by('pk')
            self.assertEquals(len(csv_data) - 1, qs.count())
            index = 0
            record = qs[index]
            expected_data = {
                'Column A': 'A1',
                'Column B': 'B1',
            }
            self.assertEquals(expected_data, record.data)
            # test that source_info contains the file_name and row_counter
            source_info = record.source_info
            self.assertIsNotNone(source_info)
            expected_info = {
                'file_name': file_name,
                'row': index + 2
            }
            self.assertEqual(source_info, expected_info)

            index = 1
            record = qs[index]
            expected_data = {
                'Column A': 'A2',
                'Column B': 'B2',
            }
            self.assertEquals(expected_data, record.data)
            # test that source_info contains the file_name and row_counter
            source_info = record.source_info
            self.assertIsNotNone(source_info)
            expected_info = {
                'file_name': file_name,
                'row': index + 2
            }
            self.assertEqual(source_info, expected_info)

            self.assertEquals(self.project_1.record_count, len(csv_data) - 1)
            self.assertEquals(self.ds.record_count, len(csv_data) - 1)

    def test_unicode(self):
        """
        Test that unicode characters works
        """
        csv_data = [
            [u'Column A', u'Column B'],
            [u'Some char: \u1234', u'The euro char: \u20ac']
        ]
        file_ = helpers.rows_to_xlsx_file(csv_data)
        client = self.custodian_1_client
        self.assertEquals(0, self.ds.record_queryset.count())
        with open(file_, 'rb') as fp:
            data = {
                'file': fp,
                'strict': False
            }
            resp = client.post(self.url, data=data, format='multipart')
            self.assertEquals(status.HTTP_200_OK, resp.status_code)
            # The records should be saved in order of the row
            qs = self.ds.record_queryset.order_by('pk')
            self.assertEquals(len(csv_data) - 1, qs.count())

            index = 0
            record = qs[index]
            expected_data = {
                'Column A': u'Some char: \u1234',
                'Column B': u'The euro char: \u20ac',
            }
            self.assertEquals(expected_data, record.data)

    def test_headers_are_trimmed_csv(self):
        """
        Test that if the user upload a csv with columns containing heading or trailing space, the parser will trimmed it and
        then compare to schema.
        """
        fields = ['What', 'When', 'Who']
        dataset = self._create_dataset_from_rows([
            fields
        ])
        schema = dataset.schema
        self.assertEquals(schema.headers, fields)

        # upload record
        csv_data = [
            ['What ', ' When', ' Who  '],
            ['Something', '2018-02-01', 'me'],
        ]
        file_ = helpers.rows_to_csv_file(csv_data)
        client = self.custodian_1_client
        url = reverse('api:dataset-upload', kwargs={'pk': dataset.pk})
        with open(file_) as fp:
            data = {
                'file': fp,
                'strict': True  # upload in strict mode
            }
            resp = client.post(url, data=data, format='multipart')
            self.assertEquals(status.HTTP_200_OK, resp.status_code)

            # verify stored data
            record = dataset.record_queryset.first()
            self.assertEquals(record.data.get('What'), 'Something')
            self.assertEquals(record.data.get('When'), '2018-02-01')
            self.assertEquals(record.data.get('Who'), 'me')
            # verify that the fields with space doesn't exists
            for f in csv_data[0]:
                self.assertIsNone(record.data.get(f))

    def test_headers_are_trimmed_xlsx(self):
        """
        Same as above but with an xlsx file
        """
        fields = ['What', 'When', 'Who']
        dataset = self._create_dataset_from_rows([
            fields
        ])
        schema = dataset.schema
        self.assertEquals(schema.headers, fields)

        # upload record
        csv_data = [
            ['What ', ' When', ' Who  '],
            ['Something', '2018-02-01', 'me'],
        ]
        file_ = helpers.rows_to_xlsx_file(csv_data)
        client = self.custodian_1_client
        url = reverse('api:dataset-upload', kwargs={'pk': dataset.pk})
        with open(file_) as fp:
            data = {
                'file': fp,
                'strict': True  # upload in strict mode
            }
            resp = client.post(url, data=data, format='multipart')
            self.assertEquals(status.HTTP_200_OK, resp.status_code)

            # verify stored data
            record = dataset.record_queryset.first()
            self.assertEquals(record.data.get('What'), 'Something')
            self.assertEquals(record.data.get('When'), '2018-02-01')
            self.assertEquals(record.data.get('Who'), 'me')
            # verify that the fields with space doesn't exists
            for f in csv_data[0]:
                self.assertIsNone(record.data.get(f))

class TestObservation(helpers.BaseUserTestCase):
    all_fields_nothing_required = [
        {
            "name": "What",
            "type": "string",
        },
        {
            "name": "When",
            "type": "date",
            "format": "any",
            "biosys": {
                'type': 'observationDate'
            }
        },
        {
            "name": 'Site',
            "type": "string",
            "biosys": {
                'type': 'siteCode'
            }
        },
        {
            "name": "Latitude",
            "type": "number",
            "biosys": {
                "type": "latitude"
            },
            "constraints": {
                "minimum": -90.0,
                "maximum": 90.0,
            }
        },
        {
            "name": "Longitude",
            "type": "number",
            "biosys": {
                "type": "longitude"
            },
            "constraints": {
                "minimum": -180.0,
                "maximum": 180.0,
            }
        },
        {
            "name": "Northing",
            "type": "number",
            "biosys": {
                "type": "northing"
            }
        },
        {
            "name": "Easting",
            "type": "number",
            "biosys": {
                "type": "easting"
            }
        },
        {
            "name": "Datum",
            "type": "string",
        },
        {
            "name": "Zone",
            "type": "integer",
        }
    ]

    def _more_setup(self):
        self.project = self.project_1
        self.client = self.custodian_1_client
        self.dataset = self._create_dataset_with_schema(
            self.project,
            self.client,
            self.all_fields_nothing_required,
            Dataset.TYPE_OBSERVATION
        )
        self.url = reverse('api:dataset-upload', kwargs={'pk': self.dataset.pk})
        # create a site
        site_geometry = Point(115.76, -32.0)
        payload = {
            'project': self.project.pk,
            'name': 'Cottesloe',
            'code': 'COT',
            'geometry': site_geometry.geojson
        }
        url = reverse('api:site-list')
        resp = self.client.post(url, data=payload, format='json')
        self.assertEquals(resp.status_code, status.HTTP_201_CREATED)
        self.site = Site.objects.filter(pk=resp.json().get('id')).first()
        self.assertIsNotNone(self.site)
        self.assertEquals(self.site.code, 'COT')

    def test_site_no_date(self):
        csv_data = [
            ['What', 'Site'],
            ['No Date', self.site.code]
        ]
        file_ = helpers.rows_to_xlsx_file(csv_data)
        client = self.custodian_1_client
        with open(file_, 'rb') as fp:
            data = {
                'file': fp,
                'strict': True  # upload in strict mode
            }
            resp = client.post(self.url, data=data, format='multipart')
            self.assertEquals(status.HTTP_200_OK, resp.status_code)
            records = self.dataset.record_queryset.all()
            self.assertEqual(len(records), 1)
            record = records[0]
            self.assertEqual(record.site, self.site)
            self.assertIsNone(record.datetime)
            self.assertEquals(record.geometry, self.site.geometry)

    def test_site_with_date(self):
        csv_data = [
            ['What', 'Site', 'When'],
            ['No Date', self.site.code, '04/06/2017']
        ]
        file_ = helpers.rows_to_xlsx_file(csv_data)
        client = self.custodian_1_client
        with open(file_, 'rb') as fp:
            data = {
                'file': fp,
                'strict': True  # upload in strict mode
            }
            resp = client.post(self.url, data=data, format='multipart')
            self.assertEquals(status.HTTP_200_OK, resp.status_code)
            records = self.dataset.record_queryset.all()
            self.assertEqual(len(records), 1)
            record = records[0]
            self.assertEqual(record.site, self.site)
            expected_date = datetime.date(2017, 6, 4)
            self.assertEqual(timezone.localtime(record.datetime).date(), expected_date)
            self.assertEquals(record.geometry, self.site.geometry)
<commit_msg>Added test to show that the API CRUD end-points doesn't trim the field names.<commit_after>from os import path
import datetime

from django.core.urlresolvers import reverse
from django.contrib.gis.geos import Point
from django.utils import timezone
from django_dynamic_fixture import G
from rest_framework import status

from main.models import Dataset, Site, Record
from main.tests.api import helpers

class TestPermission(helpers.BaseUserTestCase):
    # TODO
    pass

class TestGenericRecord(helpers.BaseUserTestCase):
    def _more_setup(self):
        self.fields = [
            {
                "name": "Column A",
                "type": "string",
                "constraints": helpers.NOT_REQUIRED_CONSTRAINTS
            },
            {
                "name": "Column B",
                "type": "string",
                "constraints": helpers.REQUIRED_CONSTRAINTS
            }
        ]
        self.data_package = helpers.create_data_package_from_fields(self.fields)
        self.ds = G(Dataset,
                    project=self.project_1,
                    type=Dataset.TYPE_GENERIC,
                    data_package=self.data_package)
        self.url = reverse('api:dataset-upload', kwargs={'pk': self.ds.pk})

    def test_upload_csv_happy_path(self):
        csv_data = [
            ['Column A', 'Column B'],
            ['A1', 'B1'],
            ['A2', 'B2']
        ]
        file_ = helpers.rows_to_csv_file(csv_data)
        client = self.custodian_1_client
        self.assertEquals(0, self.ds.record_queryset.count())
        file_name = path.basename(file_)
        with open(file_) as fp:
            data = {
                'file': fp,
                'strict': True  # upload in strict mode
            }
            resp = client.post(self.url, data=data, format='multipart')
            self.assertEquals(status.HTTP_200_OK, resp.status_code)
            # The records should be saved in order of the row
            qs = self.ds.record_queryset.order_by('pk')
            self.assertEquals(len(csv_data) - 1, qs.count())

            index = 0
            record = qs[index]
            expected_data = {
                'Column A': 'A1',
                'Column B': 'B1',
            }
            self.assertEquals(expected_data, record.data)
            # test that source_info contains the file_name and row_counter
            source_info = record.source_info
            self.assertIsNotNone(source_info)
            expected_info = {
                'file_name': file_name,
                'row': index + 2
            }
            self.assertEqual(source_info, expected_info)

            index = 1
            record = qs[index]
            expected_data = {
                'Column A': 'A2',
                'Column B': 'B2',
            }
            self.assertEquals(expected_data, record.data)
            # test that source_info contains the file_name and row_counter
            source_info = record.source_info
            self.assertIsNotNone(source_info)
            expected_info = {
                'file_name': file_name,
                'row': index + 2
            }
            self.assertEqual(source_info, expected_info)

            self.assertEquals(self.project_1.record_count, len(csv_data) - 1)
            self.assertEquals(self.ds.record_count, len(csv_data) - 1)

    def test_upload_xlsx_happy_path(self):
        csv_data = [
            ['Column A', 'Column B'],
            ['A1', 'B1'],
            ['A2', 'B2']
        ]
        file_ = helpers.rows_to_xlsx_file(csv_data)
        client = self.custodian_1_client
        self.assertEquals(0, self.ds.record_queryset.count())
        file_name = path.basename(file_)
        with open(file_, 'rb') as fp:
            data = {
                'file': fp,
                'strict': True  # upload in strict mode
            }
            resp = client.post(self.url, data=data, format='multipart')
            self.assertEquals(status.HTTP_200_OK, resp.status_code)
            # The records should be saved in order of the row
            qs = self.ds.record_queryset.order_by('pk')
            self.assertEquals(len(csv_data) - 1, qs.count())

            index = 0
            record = qs[index]
            expected_data = {
                'Column A': 'A1',
                'Column B': 'B1',
            }
            self.assertEquals(expected_data, record.data)
            # test that source_info contains the file_name and row_counter
            source_info = record.source_info
            self.assertIsNotNone(source_info)
            expected_info = {
                'file_name': file_name,
                'row': index + 2
            }
            self.assertEqual(source_info, expected_info)

            index = 1
            record = qs[index]
            expected_data = {
                'Column A': 'A2',
                'Column B': 'B2',
            }
            self.assertEquals(expected_data, record.data)
            # test that source_info contains the file_name and row_counter
            source_info = record.source_info
            self.assertIsNotNone(source_info)
            expected_info = {
                'file_name': file_name,
                'row': index + 2
            }
            self.assertEqual(source_info, expected_info)

            self.assertEquals(self.project_1.record_count, len(csv_data) - 1)
            self.assertEquals(self.ds.record_count, len(csv_data) - 1)

    def test_upload_blank_column(self):
        """ Blank column should be ignored"""
        csv_data = [
            ['Column A', '', '   ', 'Column B'],
            ['A1', '', 'something', 'B1'],
            ['A2', '', 'something', 'B2']
        ]
        file_ = helpers.rows_to_xlsx_file(csv_data)
        client = self.custodian_1_client
        self.assertEquals(0, self.ds.record_queryset.count())
        file_name = path.basename(file_)
        with open(file_, 'rb') as fp:
            data = {
                'file': fp,
                'strict': True  # upload in strict mode
            }
            resp = client.post(self.url, data=data, format='multipart')
            self.assertEquals(status.HTTP_200_OK, resp.status_code)
            # The records should be saved in order of the row
            qs = self.ds.record_queryset.order_by('pk')
            self.assertEquals(len(csv_data) - 1, qs.count())
            index = 0
            record = qs[index]
            expected_data = {
                'Column A': 'A1',
                'Column B': 'B1',
            }
            self.assertEquals(expected_data, record.data)
            # test that source_info contains the file_name and row_counter
            source_info = record.source_info
            self.assertIsNotNone(source_info)
            expected_info = {
                'file_name': file_name,
                'row': index + 2
            }
            self.assertEqual(source_info, expected_info)

            index = 1
            record = qs[index]
            expected_data = {
                'Column A': 'A2',
                'Column B': 'B2',
            }
            self.assertEquals(expected_data, record.data)
            # test that source_info contains the file_name and row_counter
            source_info = record.source_info
            self.assertIsNotNone(source_info)
            expected_info = {
                'file_name': file_name,
                'row': index + 2
            }
            self.assertEqual(source_info, expected_info)

            self.assertEquals(self.project_1.record_count, len(csv_data) - 1)
            self.assertEquals(self.ds.record_count, len(csv_data) - 1)

    def test_unicode(self):
        """
        Test that unicode characters works
        """
        csv_data = [
            [u'Column A', u'Column B'],
            [u'Some char: \u1234', u'The euro char: \u20ac']
        ]
        file_ = helpers.rows_to_xlsx_file(csv_data)
        client = self.custodian_1_client
        self.assertEquals(0, self.ds.record_queryset.count())
        with open(file_, 'rb') as fp:
            data = {
                'file': fp,
                'strict': False
            }
            resp = client.post(self.url, data=data, format='multipart')
            self.assertEquals(status.HTTP_200_OK, resp.status_code)
            # The records should be saved in order of the row
            qs = self.ds.record_queryset.order_by('pk')
            self.assertEquals(len(csv_data) - 1, qs.count())

            index = 0
            record = qs[index]
            expected_data = {
                'Column A': u'Some char: \u1234',
                'Column B': u'The euro char: \u20ac',
            }
            self.assertEquals(expected_data, record.data)

    def test_headers_are_trimmed_csv(self):
        """
        Test that if the user upload a csv with columns containing heading or trailing space, the parser will trimmed it and
        then compare to schema.
        """
        fields = ['What', 'When', 'Who']
        dataset = self._create_dataset_from_rows([
            fields
        ])
        schema = dataset.schema
        self.assertEquals(schema.headers, fields)

        # upload record
        csv_data = [
            ['What ', ' When', ' Who  '],
            ['Something', '2018-02-01', 'me'],
        ]
        file_ = helpers.rows_to_csv_file(csv_data)
        client = self.custodian_1_client
        url = reverse('api:dataset-upload', kwargs={'pk': dataset.pk})
        with open(file_) as fp:
            data = {
                'file': fp,
                'strict': True  # upload in strict mode
            }
            resp = client.post(url, data=data, format='multipart')
            self.assertEquals(status.HTTP_200_OK, resp.status_code)

            # verify stored data
            record = dataset.record_queryset.first()
            self.assertEquals(record.data.get('What'), 'Something')
            self.assertEquals(record.data.get('When'), '2018-02-01')
            self.assertEquals(record.data.get('Who'), 'me')
            # verify that the fields with space doesn't exists
            for f in csv_data[0]:
                self.assertIsNone(record.data.get(f))

    def test_headers_are_trimmed_xlsx(self):
        """
        Same as above but with an xlsx file
        """
        fields = ['What', 'When', 'Who']
        dataset = self._create_dataset_from_rows([
            fields
        ])
        schema = dataset.schema
        self.assertEquals(schema.headers, fields)

        # upload record
        csv_data = [
            ['What ', ' When', ' Who  '],
            ['Something', '2018-02-01', 'me'],
        ]
        file_ = helpers.rows_to_xlsx_file(csv_data)
        client = self.custodian_1_client
        url = reverse('api:dataset-upload', kwargs={'pk': dataset.pk})
        with open(file_) as fp:
            data = {
                'file': fp,
                'strict': True  # upload in strict mode
            }
            resp = client.post(url, data=data, format='multipart')
            self.assertEquals(status.HTTP_200_OK, resp.status_code)

            # verify stored data
            record = dataset.record_queryset.first()
            self.assertEquals(record.data.get('What'), 'Something')
            self.assertEquals(record.data.get('When'), '2018-02-01')
            self.assertEquals(record.data.get('Who'), 'me')
            # verify that the fields with space doesn't exists
            for f in csv_data[0]:
                self.assertIsNone(record.data.get(f))

    def test_headers_not_trimmed_with_api(self):
        """
        Contrary to the upload csv or xlsx when using the API in strict mode, it should not accept fields with header
        or trailing space
        see notes on https://decbugs.com/view.php?id=6863
        """
        fields = ['What', 'When', 'Who']
        dataset = self._create_dataset_from_rows([
            fields
        ])
        schema = dataset.schema
        self.assertEquals(schema.headers, fields)
        # create record with trailing and heading space
        data = {
            'What  ': 'Something',
            ' When': '2018-02-10',
            '  Who  ': 'me'
        }
        payload = {
            'dataset': dataset.pk,
            'data': data
        }
        client = self.custodian_1_client
        url = helpers.set_strict_mode(reverse('api:record-list'))
        resp = client.post(url, payload, format='json')
        self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST)

class TestObservation(helpers.BaseUserTestCase):
    all_fields_nothing_required = [
        {
            "name": "What",
            "type": "string",
        },
        {
            "name": "When",
            "type": "date",
            "format": "any",
            "biosys": {
                'type': 'observationDate'
            }
        },
        {
            "name": 'Site',
            "type": "string",
            "biosys": {
                'type': 'siteCode'
            }
        },
        {
            "name": "Latitude",
            "type": "number",
            "biosys": {
                "type": "latitude"
            },
            "constraints": {
                "minimum": -90.0,
                "maximum": 90.0,
            }
        },
        {
            "name": "Longitude",
            "type": "number",
            "biosys": {
                "type": "longitude"
            },
            "constraints": {
                "minimum": -180.0,
                "maximum": 180.0,
            }
        },
        {
            "name": "Northing",
            "type": "number",
            "biosys": {
                "type": "northing"
            }
        },
        {
            "name": "Easting",
            "type": "number",
            "biosys": {
                "type": "easting"
            }
        },
        {
            "name": "Datum",
            "type": "string",
        },
        {
            "name": "Zone",
            "type": "integer",
        }
    ]

    def _more_setup(self):
        self.project = self.project_1
        self.client = self.custodian_1_client
        self.dataset = self._create_dataset_with_schema(
            self.project,
            self.client,
            self.all_fields_nothing_required,
            Dataset.TYPE_OBSERVATION
        )
        self.url = reverse('api:dataset-upload', kwargs={'pk': self.dataset.pk})
        # create a site
        site_geometry = Point(115.76, -32.0)
        payload = {
            'project': self.project.pk,
            'name': 'Cottesloe',
            'code': 'COT',
            'geometry': site_geometry.geojson
        }
        url = reverse('api:site-list')
        resp = self.client.post(url, data=payload, format='json')
        self.assertEquals(resp.status_code, status.HTTP_201_CREATED)
        self.site = Site.objects.filter(pk=resp.json().get('id')).first()
        self.assertIsNotNone(self.site)
        self.assertEquals(self.site.code, 'COT')

    def test_site_no_date(self):
        csv_data = [
            ['What', 'Site'],
            ['No Date', self.site.code]
        ]
        file_ = helpers.rows_to_xlsx_file(csv_data)
        client = self.custodian_1_client
        with open(file_, 'rb') as fp:
            data = {
                'file': fp,
                'strict': True  # upload in strict mode
            }
            resp = client.post(self.url, data=data, format='multipart')
            self.assertEquals(status.HTTP_200_OK, resp.status_code)
            records = self.dataset.record_queryset.all()
            self.assertEqual(len(records), 1)
            record = records[0]
            self.assertEqual(record.site, self.site)
            self.assertIsNone(record.datetime)
            self.assertEquals(record.geometry, self.site.geometry)

    def test_site_with_date(self):
        csv_data = [
            ['What', 'Site', 'When'],
            ['No Date', self.site.code, '04/06/2017']
        ]
        file_ = helpers.rows_to_xlsx_file(csv_data)
        client = self.custodian_1_client
        with open(file_, 'rb') as fp:
            data = {
                'file': fp,
                'strict': True  # upload in strict mode
            }
            resp = client.post(self.url, data=data, format='multipart')
            self.assertEquals(status.HTTP_200_OK, resp.status_code)
            records = self.dataset.record_queryset.all()
            self.assertEqual(len(records), 1)
            record = records[0]
            self.assertEqual(record.site, self.site)
            expected_date = datetime.date(2017, 6, 4)
            self.assertEqual(timezone.localtime(record.datetime).date(), expected_date)
            self.assertEquals(record.geometry, self.site.geometry)
<|endoftext|>


<commit_before>"""Macro to use for loading the J2CL repository"""

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_jar")
load("@io_bazel_rules_closure//closure:repositories.bzl", "rules_closure_dependencies")
load("@bazel_skylib//lib:versions.bzl", "versions")

def setup_j2cl_workspace():
    """Load all dependencies needed for J2CL."""

    versions.check("0.24.0")  # The version J2CL currently have a CI setup for.

    rules_closure_dependencies(
        omit_com_google_protobuf = True,
        omit_com_google_auto_common = True,
    )

    native.maven_jar(
        name = "com_google_auto_common",
        artifact = "com.google.auto:auto-common:0.9",
    )

    native.maven_jar(
        name = "com_google_auto_service",
        artifact = "com.google.auto.service:auto-service:1.0-rc2",
    )

    # We cannot replace com_google_jsinterop_annotations so choose a different name
    http_archive(
        name = "com_google_jsinterop_annotations_head",
        urls = ["https://github.com/google/jsinterop-annotations/archive/master.zip"],
        strip_prefix = "jsinterop-annotations-master",
    )

    native.maven_jar(
        name = "org_apache_commons_collections",
        artifact = "commons-collections:commons-collections:3.2.2",
    )

    native.maven_jar(
        name = "org_apache_commons_lang2",
        artifact = "commons-lang:commons-lang:2.6",
    )

    native.maven_jar(
        name = "org_apache_commons_lang3",
        artifact = "org.apache.commons:commons-lang3:3.6",
    )

    native.maven_jar(
        name = "org_apache_commons_text",
        artifact = "org.apache.commons:commons-text:1.2",
    )

    native.maven_jar(
        name = "org_apache_velocity",
        artifact = "org.apache.velocity:velocity:1.7",
    )

    native.maven_jar(
        name = "org_junit",
        artifact = "junit:junit:4.12",
    )

    native.maven_jar(
        name = "com_google_testing_compile",
        artifact = "com.google.testing.compile:compile-testing:0.15",
    )

    native.maven_jar(
        name = "org_mockito",
        artifact = "org.mockito:mockito-all:1.9.5",
    )

    native.maven_jar(
        name = "com_google_truth",
        artifact = "com.google.truth:truth:1.0",
    )

    # TODO(b/135461024): for now J2CL uses a prepackaged version of javac. But in the future it
    # might be better to tie in to the Java platform in bazel and control the version there.
    native.maven_jar(
        name = "com_sun_tools_javac",
        artifact = "com.google.errorprone:javac:1.9.0-dev-r2644-1",
    )

    # Eclipse JARs listed at
    # http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/

    http_jar(
        name = "org_eclipse_jdt_content_type",
        url = "http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/org.eclipse.core.contenttype_3.7.300.v20190215-2048.jar",
        sha256 = "d4cee7a28a000a89863ab225c479cafc2fdf8638db2e89b383d6d792c5bfb866",
    )

    http_jar(
        name = "org_eclipse_jdt_jobs",
        url = "http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/org.eclipse.core.jobs_3.10.400.v20190506-1457.jar",
        sha256 = "31dce5a10508774c84ff06d5840ce90b04eeb5c1a22314abe1afbe0f402007e8",
    )

    http_jar(
        name = "org_eclipse_jdt_resources",
        url = "http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/org.eclipse.core.resources_3.13.400.v20190505-1655.jar",
        sha256 = "c7ea0920453e16dd72c43d7ee47b2d9cd45a5267c92984ad51e09cb6a8c4378d",
    )

    http_jar(
        name = "org_eclipse_jdt_runtime",
        url = "http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/org.eclipse.core.runtime_3.15.300.v20190508-0543.jar",
        sha256 = "a9dc5b43defbb9975136c31478eef9f66ca567be8f4b1d47814752173e18f299",
    )

    http_jar(
        name = "org_eclipse_jdt_equinox_common",
        url = "http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/org.eclipse.equinox.common_3.10.400.v20190516-1504.jar",
        sha256 = "85cde16fb71b9f25271916b11a5cdce50ddc79cb2974484085990d152e945661",
    )

    http_jar(
        name = "org_eclipse_jdt_equinox_preferences",
        url = "http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/org.eclipse.equinox.preferences_3.7.400.v20190516-1504.jar",
        sha256 = "298c1ee84e68f0eb011be5d55e4041525b682d034ae5ebc7dafdfcb86ef09d6f",
    )

    http_jar(
        name = "org_eclipse_jdt_compiler_apt",
        url = "http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/org.eclipse.jdt.apt.core_3.6.400.v20190328-1431.jar",
        sha256 = "589097f07dd37570117e078a56da8b754c22a4225fae09f1a09e5b6f81d58ba0",
    )

    http_jar(
        name = "org_eclipse_jdt_core",
        url = "http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/org.eclipse.jdt.core_3.18.0.v20190522-0428.jar",
        sha256 = "2a727df878a41bf63f2f6bd0a77fb327ffebd1b637f4b06f08cb41263ff96194",
    )

    http_jar(
        name = "org_eclipse_jdt_osgi",
        url = "http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/org.eclipse.osgi_3.14.0.v20190517-1309.jar",
        sha256 = "0a97384b1820f709695e76e9eca214b8bed59738206945113bb89c5fc5060e12",
    )

    http_jar(
        name = "org_eclipse_jdt_text",
        url = "http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/org.eclipse.text_3.8.200.v20190519-2344.jar",
        sha256 = "3c0747c95459cfadf6d11ef591452c98812a9208d40a9fe3719ba7dbf8a26132",
    )

    http_archive(
        name = "org_gwtproject_gwt",
        url = "https://gwt.googlesource.com/gwt/+archive/master.tar.gz",
    )

    http_archive(
        name = "org_jbox2d",
        strip_prefix = "jbox2d-jbox2d-2.2.1.1/jbox2d-library",
        urls = ["https://github.com/jbox2d/jbox2d/archive/jbox2d-2.2.1.1.zip"],
        sha256 = "088e5fc0f56c75f82c289c4721d9faf46a309e258d3ee647799622ef82e60303",
        patches = ["@com_google_j2cl//transpiler/javatests/com/google/j2cl/transpiler/integration/box2d:jbox2d.patch"],
        build_file_content = '''
filegroup(
    name = "j2cl_sources",
    srcs = glob(
        ["src/main/java/**/*.java"],
        exclude = [
            "**/StrictMath.java",
            "**/PlatformMathUtils.java",
            "**/Timer.java",
            "**/profile/**",
        ],
    ) + glob(["src/main/java/org/jbox2d/gwtemul/**/*.java"]),
    visibility = ["//visibility:public"],
)''',
    )

    # proto_library and java_proto_library rules implicitly depend on
    # @com_google_protobuf for protoc and proto runtimes.
    http_archive(
        name = "com_google_protobuf",
        strip_prefix = "protobuf-3.6.1.3",
        urls = ["https://github.com/google/protobuf/archive/v3.6.1.3.zip"],
        sha256 = "9510dd2afc29e7245e9e884336f848c8a6600a14ae726adb6befdb4f786f0be2",
    )

    # needed for protobuf
    native.bind(
        name = "guava",
        actual = "@com_google_guava",
    )

    # needed for protobuf
    native.bind(
        name = "gson",
        actual = "@com_google_code_gson",
    )
<commit_msg>Stop overriding protobuf dependency version.<commit_after>"""Macro to use for loading the J2CL repository"""

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_jar")
load("@io_bazel_rules_closure//closure:repositories.bzl", "rules_closure_dependencies")
load("@bazel_skylib//lib:versions.bzl", "versions")

def setup_j2cl_workspace():
    """Load all dependencies needed for J2CL."""

    versions.check("0.24.0")  # The version J2CL currently have a CI setup for.

    rules_closure_dependencies(
        omit_com_google_auto_common = True,
    )

    native.maven_jar(
        name = "com_google_auto_common",
        artifact = "com.google.auto:auto-common:0.9",
    )

    native.maven_jar(
        name = "com_google_auto_service",
        artifact = "com.google.auto.service:auto-service:1.0-rc2",
    )

    # We cannot replace com_google_jsinterop_annotations so choose a different name
    http_archive(
        name = "com_google_jsinterop_annotations_head",
        urls = ["https://github.com/google/jsinterop-annotations/archive/master.zip"],
        strip_prefix = "jsinterop-annotations-master",
    )

    native.maven_jar(
        name = "org_apache_commons_collections",
        artifact = "commons-collections:commons-collections:3.2.2",
    )

    native.maven_jar(
        name = "org_apache_commons_lang2",
        artifact = "commons-lang:commons-lang:2.6",
    )

    native.maven_jar(
        name = "org_apache_commons_lang3",
        artifact = "org.apache.commons:commons-lang3:3.6",
    )

    native.maven_jar(
        name = "org_apache_commons_text",
        artifact = "org.apache.commons:commons-text:1.2",
    )

    native.maven_jar(
        name = "org_apache_velocity",
        artifact = "org.apache.velocity:velocity:1.7",
    )

    native.maven_jar(
        name = "org_junit",
        artifact = "junit:junit:4.12",
    )

    native.maven_jar(
        name = "com_google_testing_compile",
        artifact = "com.google.testing.compile:compile-testing:0.15",
    )

    native.maven_jar(
        name = "org_mockito",
        artifact = "org.mockito:mockito-all:1.9.5",
    )

    native.maven_jar(
        name = "com_google_truth",
        artifact = "com.google.truth:truth:1.0",
    )

    # TODO(b/135461024): for now J2CL uses a prepackaged version of javac. But in the future it
    # might be better to tie in to the Java platform in bazel and control the version there.
    native.maven_jar(
        name = "com_sun_tools_javac",
        artifact = "com.google.errorprone:javac:1.9.0-dev-r2644-1",
    )

    # Eclipse JARs listed at
    # http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/

    http_jar(
        name = "org_eclipse_jdt_content_type",
        url = "http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/org.eclipse.core.contenttype_3.7.300.v20190215-2048.jar",
        sha256 = "d4cee7a28a000a89863ab225c479cafc2fdf8638db2e89b383d6d792c5bfb866",
    )

    http_jar(
        name = "org_eclipse_jdt_jobs",
        url = "http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/org.eclipse.core.jobs_3.10.400.v20190506-1457.jar",
        sha256 = "31dce5a10508774c84ff06d5840ce90b04eeb5c1a22314abe1afbe0f402007e8",
    )

    http_jar(
        name = "org_eclipse_jdt_resources",
        url = "http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/org.eclipse.core.resources_3.13.400.v20190505-1655.jar",
        sha256 = "c7ea0920453e16dd72c43d7ee47b2d9cd45a5267c92984ad51e09cb6a8c4378d",
    )

    http_jar(
        name = "org_eclipse_jdt_runtime",
        url = "http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/org.eclipse.core.runtime_3.15.300.v20190508-0543.jar",
        sha256 = "a9dc5b43defbb9975136c31478eef9f66ca567be8f4b1d47814752173e18f299",
    )

    http_jar(
        name = "org_eclipse_jdt_equinox_common",
        url = "http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/org.eclipse.equinox.common_3.10.400.v20190516-1504.jar",
        sha256 = "85cde16fb71b9f25271916b11a5cdce50ddc79cb2974484085990d152e945661",
    )

    http_jar(
        name = "org_eclipse_jdt_equinox_preferences",
        url = "http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/org.eclipse.equinox.preferences_3.7.400.v20190516-1504.jar",
        sha256 = "298c1ee84e68f0eb011be5d55e4041525b682d034ae5ebc7dafdfcb86ef09d6f",
    )

    http_jar(
        name = "org_eclipse_jdt_compiler_apt",
        url = "http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/org.eclipse.jdt.apt.core_3.6.400.v20190328-1431.jar",
        sha256 = "589097f07dd37570117e078a56da8b754c22a4225fae09f1a09e5b6f81d58ba0",
    )

    http_jar(
        name = "org_eclipse_jdt_core",
        url = "http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/org.eclipse.jdt.core_3.18.0.v20190522-0428.jar",
        sha256 = "2a727df878a41bf63f2f6bd0a77fb327ffebd1b637f4b06f08cb41263ff96194",
    )

    http_jar(
        name = "org_eclipse_jdt_osgi",
        url = "http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/org.eclipse.osgi_3.14.0.v20190517-1309.jar",
        sha256 = "0a97384b1820f709695e76e9eca214b8bed59738206945113bb89c5fc5060e12",
    )

    http_jar(
        name = "org_eclipse_jdt_text",
        url = "http://download.eclipse.org/eclipse/updates/4.12/R-4.12-201906051800/plugins/org.eclipse.text_3.8.200.v20190519-2344.jar",
        sha256 = "3c0747c95459cfadf6d11ef591452c98812a9208d40a9fe3719ba7dbf8a26132",
    )

    http_archive(
        name = "org_gwtproject_gwt",
        url = "https://gwt.googlesource.com/gwt/+archive/master.tar.gz",
    )

    http_archive(
        name = "org_jbox2d",
        strip_prefix = "jbox2d-jbox2d-2.2.1.1/jbox2d-library",
        urls = ["https://github.com/jbox2d/jbox2d/archive/jbox2d-2.2.1.1.zip"],
        sha256 = "088e5fc0f56c75f82c289c4721d9faf46a309e258d3ee647799622ef82e60303",
        patches = ["@com_google_j2cl//transpiler/javatests/com/google/j2cl/transpiler/integration/box2d:jbox2d.patch"],
        build_file_content = '''
filegroup(
    name = "j2cl_sources",
    srcs = glob(
        ["src/main/java/**/*.java"],
        exclude = [
            "**/StrictMath.java",
            "**/PlatformMathUtils.java",
            "**/Timer.java",
            "**/profile/**",
        ],
    ) + glob(["src/main/java/org/jbox2d/gwtemul/**/*.java"]),
    visibility = ["//visibility:public"],
)''',
    )

    # Required by protobuf_java_util
    native.bind(
        name = "guava",
        actual = "@com_google_guava",
    )

    # Required by protobuf_java_util
    native.bind(
        name = "gson",
        actual = "@com_google_code_gson",
    )

    # Required by protobuf_java_util
    native.bind(
        name = "error_prone_annotations",
        actual = "@com_google_errorprone_error_prone_annotations",
    )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Mon Oct  7 16:26:35 2013

@author: laure
"""

from sklearn import datasets

X, y = datasets.make_classification(n_samples=500,
                                    n_features=400000,
                                    n_informative=2,
                                    random_state=1)

Xy = dict(X=X, y=y)
## 2) Building workflow
## =======================================================
print " -> Pt2 : X and y created, building workflow"
from sklearn.svm import SVC
from epac import CV, Methods
cv_svm_local = CV(Methods(*[SVC(kernel="linear"),
                            SVC(kernel="rbf")]),
                  n_folds=3)
print " -> Pt3 : Workflow built, running"
cv_svm = None
n_proc = 2
# Running on the local machine
from epac import LocalEngine
local_engine = LocalEngine(cv_svm_local, num_processes=n_proc)
cv_svm = local_engine.run(**Xy)
<commit_msg>TEST: testing 200.000 features bug with sklearn<commit_after># -*- coding: utf-8 -*-
"""
Created on Mon Oct  7 16:26:35 2013

@author: laure
"""

from sklearn import datasets

X, y = datasets.make_classification(n_samples=500,
                                    n_features=200000,
                                    n_informative=2,
                                    random_state=1)

Xy = dict(X=X, y=y)
## 2) Building workflow
## =======================================================
print " -> Pt2 : X and y created, building workflow"
from sklearn import svm, cross_validation
kfold = cross_validation.KFold(n=len(X), n_folds=3)
svc = svm.SVC(C=1, kernel='linear')
print [svc.fit(X[train], y[train]).score(X[test], y[test]) for train, test in kfold]
#from epac import CV, Methods
#cv_svm_local = CV(Methods(*[svm.SVC(kernel="linear"),
#                            svm.SVC(kernel="rbf")]),
#                  n_folds=3)
print " -> Pt3 : Workflow built, running"
#cv_svm = None
#n_proc = 2
## Running on the local machine
#from epac import LocalEngine
#local_engine = LocalEngine(cv_svm_local, num_processes=n_proc)
#cv_svm = local_engine.run(**Xy)
<|endoftext|>