<commit_before>from django.views.generic import DetailView
from django.views.generic.list import ListView

from . import models
from wye.workshops.models import Workshop
from wye.base.constants import WorkshopStatus
from wye.organisations.models import Organisation

class ProfileView(DetailView):
    model = models.Profile
    template_name = 'profile/index.html'

    def get_context_data(self, *args, **kwargs):
        slug = self.kwargs['slug']
        self.object = models.Profile.objects.get(
            slug=slug)
        context = super(
            ProfileView, self).get_context_data(*args, **kwargs)
        return context

class UserDashboard(ListView):
    model = models.Profile
    template_name = 'profile/dashboard.html'

    def get_context_data(self, **kwargs):
        context = super(UserDashboard, self).get_context_data(**kwargs)
        user_profile = models.Profile.objects.get(user__id=self.request.user.id)
        for each_type in user_profile.get_user_type:
            if each_type == 'Tutor':
                context['workshop_list_tutor'] = Workshop.objects.filter(
                    presenter=self.request.user, status=WorkshopStatus.REQUESTED)
            if each_type == 'Regional-Lead':
                context['workshops_accepted'] = Workshop.objects.filter(
                    status=WorkshopStatus.ACCEPTED)
                context['workshops_pending'] = Workshop.objects.filter(
                    status=WorkshopStatus.REQUESTED)
                context['interested_tutors'] = models.Profile.objects.filter(
                    usertype__slug='Tutor',
                    interested_locations__name__in=user_profile.get_interested_locations).exclude(
                    user__id=self.request.user.id).count()
                context['interested_locations'] = Organisation.objects.filter(
                    location__name__in=user_profile.get_interested_locations).count()

        return context
<commit_msg>added details about college poc in the context<commit_after>from django.views.generic import DetailView
from django.views.generic.list import ListView

from . import models
from wye.workshops.models import Workshop
from wye.base.constants import WorkshopStatus
from wye.organisations.models import Organisation

class ProfileView(DetailView):
    model = models.Profile
    template_name = 'profile/index.html'

    def get_context_data(self, *args, **kwargs):
        slug = self.kwargs['slug']
        self.object = models.Profile.objects.get(
            slug=slug)
        context = super(
            ProfileView, self).get_context_data(*args, **kwargs)
        return context

class UserDashboard(ListView):
    model = models.Profile
    template_name = 'profile/dashboard.html'

    def get_context_data(self, **kwargs):
        context = super(UserDashboard, self).get_context_data(**kwargs)
        user_profile = models.Profile.objects.get(user__id=self.request.user.id)
        for each_type in user_profile.get_user_type:
            if each_type == 'Tutor':
                context['workshop_list_tutor'] = Workshop.objects.filter(
                    presenter=self.request.user, status=WorkshopStatus.REQUESTED)
            if each_type == 'Regional-Lead':
                context['workshops_accepted_under_rl'] = Workshop.objects.filter(
                    status=WorkshopStatus.ACCEPTED)
                context['workshops_pending_under_rl'] = Workshop.objects.filter(
                    status=WorkshopStatus.REQUESTED)
                context['interested_tutors'] = models.Profile.objects.filter(
                    usertype__slug='Tutor',
                    interested_locations__name__in=user_profile.get_interested_locations).exclude(
                    user__id=self.request.user.id).count()
                context['interested_locations'] = Organisation.objects.filter(
                    location__name__in=user_profile.get_interested_locations).count()
            if each_type == 'College-POC':
                context['organisation_users'] = models.Profile.objects.filter(
                    user__id__in=Organisation.objects.filter(
                        created_by__id=self.request.user.id).values_list(
                        'user', flat=True))
                context['workshop_requested_under_poc'] = Workshop.objects.filter(
                    status=WorkshopStatus.REQUESTED,
                    requester=Organisation.objects.filter(
                        created_by__id=self.request.user.id))
                context['workshops_accepted_under_poc'] = Workshop.objects.filter(
                    status=WorkshopStatus.ACCEPTED,
                    requester=Organisation.objects.filter(
                        created_by__id=self.request.user.id))

        return context
<|endoftext|>


<commit_before>import graphlab as gl
import re
import random
from copy import copy
import os
import graphlab.aggregate as agg
import array

import sys

model_name = "full-inet-small"
which_model = 4

print "Running model %d, %s" % (which_model, model_name)

alt_path = os.path.expanduser("~/data/tmp/")
if os.path.exists(alt_path):
    gl.set_runtime_config("GRAPHLAB_CACHE_FILE_LOCATIONS", alt_path)

model_path = "nn_256x256/models/"

X_train = gl.SFrame("image-sframes/train-%d/" % which_model)
X_valid = gl.SFrame("image-sframes/validation-%d/" % which_model)
X_test = gl.SFrame("image-sframes/test/")

network_str = '''
netconfig=start
layer[0->1] = conv
kernel_size = 8
  padding = 1
  stride = 4
  num_channels = 64
  random_type = xavier
layer[1->2] = max_pooling
  kernel_size = 3
  stride = 2
layer[2->3] = conv
  kernel_size = 3
  padding = 1
  stride = 2
  num_channels = 64
  random_type = xavier
layer[3->4] = max_pooling
  kernel_size = 3
  stride = 2
layer[4->5] = dropout
  threshold = 0.5
layer[5->6] = flatten
layer[6->7] = fullc
num_hidden_units = 128
  init_sigma = 0.01
layer[7->8] = dropout
  threshold = 0.5
layer[8->9] = sigmoid
layer[9->10] = fullc
  num_hidden_units = 128
  init_sigma = 0.01
layer[10->11] = fullc
  num_hidden_units = %d
  init_sigma = 0.01
layer[11->12] = softmax
netconfig=end

# input shape not including batch
input_shape = 3,256,256
batch_size = 100

## global parameters
init_random = gaussian

## learning parameters
learning_rate = 0.025
momentum = 0.9
l2_regularization = 0.0
divideby = 255
# end of config
''' % (5 if which_model == 0 else 2)

network = gl.deeplearning.NeuralNet(conf_str=network_str)

if os.path.exists("image-sframes/mean_image"):
    mean_image_sf = gl.SFrame("image-sframes/mean_image")
    mean_image = mean_image_sf["image"][0]
else:
    mean_image = X_train["image"].mean()
    mean_image_sf = gl.SFrame({"image" : [mean_image]})
    mean_image_sf.save("image-sframes/mean_image")

if which_model == 0:

    m = gl.classifier.neuralnet_classifier.create(
        X_train, features = ["image"], target = "level",
        network = network, mean_image = mean_image,
        device = "gpu", random_mirror=True, max_iterations = 25,
        validation_set=X_valid)

else:
    assert which_model in [1,2,3,4]

    X_train["class"] = (X_train["level"] >= which_model)
    X_valid["class"] = (X_valid["level"] >= which_model)

    # Downsample the less common class
    n_class_0 = (X_train["class"] == 0).sum()
    n_class_1 = (X_train["class"] == 1).sum()

    m = gl.classifier.neuralnet_classifier.create(
        X_train,
        features = ["image"], target = "class",
        network = network, mean_image = mean_image,
        device = "gpu", random_mirror=True, max_iterations = 25, validation_set=X_valid)

m.save(model_path + "gpu_model_%d-%s" % (which_model, model_name))

X_train["class_scores"] = \
  (m.predict_topk(X_train[["image"]], k= (5 if which_model == 0 else 2))\
   .unstack(["class", "score"], "scores").sort("row_id")["scores"])

X_test["class_scores"] = \
    (m.predict_topk(X_test[["image"]], k=(5 if which_model == 0 else 2))
     .unstack(["class", "score"], "scores").sort("row_id")["scores"])

X_train["features"] = m.extract_features(X_train[["image"]])
X_test["features"] = m.extract_features(X_test[["image"]])

def flatten_dict(d):
    out_d = {}
    def _add_to_dict(base, out_d, d):
        for k, v in d.iteritems():
            new_key = k if base is None else (base + '.' + str(k))
            if type(v) is dict:
                _add_to_dict(new_key, out_d, v)
            elif type(v) is array.array:
                for j, x in enumerate(v):
                    if x != 0:
                        out_d[new_key + ".%d" % j] = x
            else:
                out_d[new_key] = v
    _add_to_dict(None, out_d, d)
    return out_d

score_column = "scores_%d" % which_model
features_column = "features_%d" % which_model

Xt = X_train[["name", "source", "class_scores", "level", "features"]]
Xty = Xt.groupby(["name", "level"], {"cs" : agg.CONCAT("source", "class_scores")})
Xty[score_column] = Xty["cs"].apply(flatten_dict)

Xty2 = Xt.groupby("name", {"ft" : agg.CONCAT("source", "features")})
Xty2[features_column] = Xty2["ft"].apply(flatten_dict)

Xty = Xty.join(Xty2[["name", features_column]], on = "name")

Xty[["name", score_column, "level", features_column]].save(model_path + "/scores_train_%d" % which_model)

Xtst = X_test[["name", "source", "class_scores", "features"]]
Xtsty = Xtst.groupby("name", {"cs" : agg.CONCAT("source", "class_scores")})
Xtsty[score_column] = Xtsty["cs"].apply(flatten_dict)

Xtsty2 = Xtst.groupby("name", {"ft" : agg.CONCAT("source", "features")})
Xtsty2[features_column] = Xtsty2["ft"].apply(flatten_dict)

Xtsty = Xtsty.join(Xtsty2[["name", features_column]], on = "name")

Xtsty[["name", score_column, features_column]].save(model_path + "/scores_test_%d" % which_model)

<commit_msg>Updated NN archetecture.<commit_after>import graphlab as gl
import re
import random
from copy import copy
import os
import graphlab.aggregate as agg
import array

import sys

model_name = "full-inet-small"
which_model = 4

print "Running model %d, %s" % (which_model, model_name)

alt_path = os.path.expanduser("~/data/tmp/")
if os.path.exists(alt_path):
    gl.set_runtime_config("GRAPHLAB_CACHE_FILE_LOCATIONS", alt_path)

model_path = "nn_256x256/models/"

X_train = gl.SFrame("image-sframes/train-%d/" % which_model)
X_valid = gl.SFrame("image-sframes/validation-%d/" % which_model)
X_test = gl.SFrame("image-sframes/test/")

network_str = '''
netconfig=start
layer[0->1] = conv
kernel_size = 8
  padding = 1
  stride = 4
  num_channels = 64
  random_type = xavier
layer[1->2] = relu
layer[2->3] = max_pooling
  kernel_size = 3
  stride = 2
layer[3->4] = conv
  kernel_size = 3
  padding = 1
  stride = 2
  num_channels = 32
  random_type = xavier
layer[4->5] = relu
layer[5->6] = max_pooling
  kernel_size = 3
  stride = 2
layer[6->7] = dropout
  threshold = 0.5
layer[7->8] = flatten
layer[8->9] = fullc
num_hidden_units = 64
  init_sigma = 0.01
layer[9->10] = dropout
  threshold = 0.5
layer[10->11] = sigmoid
layer[11->12] = fullc
  num_hidden_units = %d
  init_sigma = 0.01
layer[12->13] = softmax
netconfig=end

# input shape not including batch
input_shape = 3,256,256
batch_size = 64

## global parameters
init_random = gaussian

## learning parameters
learning_rate = 0.025
momentum = 0.9
l2_regularization = 0.0001
divideby = 255
# end of config
''' % (5 if which_model == 0 else 2)

network = gl.deeplearning.NeuralNet(conf_str=network_str)

model_filename = model_path + "gpu_model_%d-%s" % (which_model, model_name)

if os.path.exists("image-sframes/mean_image"):
    mean_image_sf = gl.SFrame("image-sframes/mean_image")
    mean_image = mean_image_sf["image"][0]
else:
    mean_image = X_train["image"].mean()
    mean_image_sf = gl.SFrame({"image" : [mean_image]})
    mean_image_sf.save("image-sframes/mean_image")

if which_model == 0:

    m = gl.classifier.neuralnet_classifier.create(
        X_train, features = ["image"], target = "level",
        network = network, mean_image = mean_image,
        device = "gpu", random_mirror=True, max_iterations = 25,
        validation_set=X_valid,
        model_checkpoint_interval = 1,
        model_checkpoint_path = model_filename + "-checkpoint")

else:
    assert which_model in [1,2,3,4]

    X_train["class"] = (X_train["level"] >= which_model)
    X_valid["class"] = (X_valid["level"] >= which_model)

    # Downsample the less common class
    n_class_0 = (X_train["class"] == 0).sum()
    n_class_1 = (X_train["class"] == 1).sum()

    m = gl.classifier.neuralnet_classifier.create(
        X_train,
        features = ["image"], target = "class",
        network = network, mean_image = mean_image,
        model_checkpoint_path = model_filename + "-checkpoint",
        model_checkpoint_interval = 1,
        device = "gpu", random_mirror=True, max_iterations = 25, validation_set=X_valid)

m.save(model_path + "gpu_model_%d-%s" % (which_model, model_name))

X_train["class_scores"] = \
  (m.predict_topk(X_train[["image"]], k= (5 if which_model == 0 else 2))\
   .unstack(["class", "score"], "scores").sort("row_id")["scores"])

X_test["class_scores"] = \
    (m.predict_topk(X_test[["image"]], k=(5 if which_model == 0 else 2))
     .unstack(["class", "score"], "scores").sort("row_id")["scores"])

X_train["features"] = m.extract_features(X_train[["image"]])
X_test["features"] = m.extract_features(X_test[["image"]])

def flatten_dict(d):
    out_d = {}

    def _add_to_dict(base, out_d, d):

        if type(d) in [array.array, list]:
            for j, v in enumerate(d):
                new_key = str(j) if base is None else (base + ".%d" % j)
                _add_to_dict(new_key, out_d, v)

        elif type(d) is dict:
            for k, v in d.iteritems():
                new_key = k if base is None else (base + '.' + str(k))
                if type(v) in [dict, array.array, list]:
                    _add_to_dict(new_key, out_d, v)
                else:
                    out_d[new_key] = v
        else:
            out_d[base] = d

    _add_to_dict(None, out_d, d)

    return out_d

score_column = "scores_%d" % which_model
features_column = "features_%d" % which_model

Xt = X_train[["name", "source", "class_scores", "level", "features"]]
Xty = Xt.groupby(["name", "level"], {"cs" : agg.CONCAT("source", "class_scores")})
Xty[score_column] = Xty["cs"].apply(flatten_dict)

Xty2 = Xt.groupby("name", {"ft" : agg.CONCAT("source", "features")})
Xty2[features_column] = Xty2["ft"].apply(flatten_dict)

Xty = Xty.join(Xty2[["name", features_column]], on = "name")

Xty[["name", score_column, "level", features_column]].save(model_path + "/scores_train_%d" % which_model)

Xtst = X_test[["name", "source", "class_scores", "features"]]
Xtsty = Xtst.groupby("name", {"cs" : agg.CONCAT("source", "class_scores")})
Xtsty[score_column] = Xtsty["cs"].apply(flatten_dict)

Xtsty2 = Xtst.groupby("name", {"ft" : agg.CONCAT("source", "features")})
Xtsty2[features_column] = Xtsty2["ft"].apply(flatten_dict)

Xtsty = Xtsty.join(Xtsty2[["name", features_column]], on = "name")

Xtsty[["name", score_column, features_column]].save(model_path + "/scores_test_%d" % which_model)

<|endoftext|>


<commit_before>import subprocess
import logging
from flexget.plugin import *

log = logging.getLogger('adv_exec')

class PluginAdvExec:
    """
    Execute commands

    Example:

    adv_exec:
      on_start:
        event: echo "Started"
      on_input:
        for_entries: echo 'got %(title)s'
      on_output:
        for_accepted: echo 'accepted %(title)s - %(url)s > file

    You can use all (available) entry fields in the command.
    """

    NAME = 'adv_exec'

    def validator(self):
        from flexget import validator
        root = validator.factory('dict')

        def add(name):
            event = root.accept('dict', key=name)
            event.accept('text', key='event')
            event.accept('text', key='for_entries')
            event.accept('text', key='for_accepted')
            event.accept('text', key='for_rejected')
            event.accept('text', key='for_failed')

        for name in ['on_start', 'on_input', 'on_filter', 'on_output', 'on_end']:
            add(name)

        return root

    def on_feed_start(self, feed):
        self.execute(feed, 'on_start')

    def on_feed_input(self, feed):
        self.execute(feed, 'on_input')

    def on_feed_filter(self, feed):
        self.execute(feed, 'on_filter')

    def on_feed_output(self, feed):
        self.execute(feed, 'on_output')

    def on_feed_exit(self, feed):
        self.execute(feed, 'on_exit')

    def execute_cmd(self, cmd):
        p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, \
            stderr=subprocess.STDOUT, close_fds=False)
        (r, w) = (p.stdout, p.stdin)
        response = r.read()
        r.close()
        w.close()
        if response:
            log.info('Stdout: %s' % response)

    def execute(self, feed, event_name):
        if not event_name in feed.config[self.NAME]:
            log.debug('event %s not configured' % event_name)
            return

        name_map = {'for_entries': feed.entries, 'for_accepted': feed.accepted, \
            'for_rejected': feed.rejected, 'for_failed': feed.failed}

        for operation, entries in name_map.iteritems():
            if not operation in feed.config[self.NAME][event_name]:
                continue

            log.debug('running event_name: %s operation: %s entries: %s' % (event_name, operation, len(entries)))

            for entry in entries:
                try:
                    cmd = feed.config[self.NAME][event_name][operation] % entry
                except KeyError, e:
                    log.error('Entry %s does not have required field %s' % (entry['title'], e.message))
                    continue
                log.debug('event_name: %s operation: %s cmd: %s' % (event_name, operation, cmd))
                self.execute_cmd(cmd)

        # event keyword in this
        if 'event' in feed.config[self.NAME][event_name]:
            cmd = feed.config[self.NAME][event_name]['event']
            log.debug('event cmd: %s' % cmd)
            self.execute_cmd(cmd)

register_plugin(PluginAdvExec, 'adv_exec')
<commit_msg>properly escape and quote any substituted flexget value from within a call to the `adv_exec` plugin<commit_after>import subprocess
import logging
import shlex
import pipes
from flexget.plugin import *

log = logging.getLogger('adv_exec')

class PluginAdvExec:
    """
    Execute commands

    Example:

    adv_exec:
      on_start:
        event: echo "Started"
      on_input:
        for_entries: echo 'got %(title)s'
      on_output:
        for_accepted: echo 'accepted %(title)s - %(url)s > file

    You can use all (available) entry fields in the command.
    """

    NAME = 'adv_exec'

    def validator(self):
        from flexget import validator
        root = validator.factory('dict')

        def add(name):
            event = root.accept('dict', key=name)
            event.accept('text', key='event')
            event.accept('text', key='for_entries')
            event.accept('text', key='for_accepted')
            event.accept('text', key='for_rejected')
            event.accept('text', key='for_failed')

        for name in ['on_start', 'on_input', 'on_filter', 'on_output', 'on_end']:
            add(name)

        return root

    def on_feed_start(self, feed):
        self.execute(feed, 'on_start')

    def on_feed_input(self, feed):
        self.execute(feed, 'on_input')

    def on_feed_filter(self, feed):
        self.execute(feed, 'on_filter')

    def on_feed_output(self, feed):
        self.execute(feed, 'on_output')

    def on_feed_exit(self, feed):
        self.execute(feed, 'on_exit')

    def execute_cmd(self, cmd):
        p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, \
            stderr=subprocess.STDOUT, close_fds=False)
        (r, w) = (p.stdout, p.stdin)
        response = r.read()
        r.close()
        w.close()
        if response:
            log.info('Stdout: %s' % response)

    def execute(self, feed, event_name):
        if not event_name in feed.config[self.NAME]:
            log.debug('event %s not configured' % event_name)
            return

        name_map = {'for_entries': feed.entries, 'for_accepted': feed.accepted, \
            'for_rejected': feed.rejected, 'for_failed': feed.failed}

        for operation, entries in name_map.iteritems():
            if not operation in feed.config[self.NAME][event_name]:
                continue

            log.debug('running event_name: %s operation: %s entries: %s' % (event_name, operation, len(entries)))

            for entry in entries:
                try:
                    cmd = feed.config[self.NAME][event_name][operation] % entry

                    args = []
                    # shlex is documented to not work on unicode
                    for arg in shlex.split(cmd.encode('utf-8'), comments=True):
                        arg = unicode(arg, 'utf-8')
                        formatted = arg % entry
                        if formatted != arg:
                            arg = pipes.quote(formatted)
                        args.append(arg)
                    cmd = ' '.join(args)

                except KeyError, e:
                    log.error('Entry %s does not have required field %s' % (entry['title'], e.message))
                    continue
                log.debug('event_name: %s operation: %s cmd: %s' % (event_name, operation, cmd))
                self.execute_cmd(cmd)

        # event keyword in this
        if 'event' in feed.config[self.NAME][event_name]:
            cmd = feed.config[self.NAME][event_name]['event']
            log.debug('event cmd: %s' % cmd)
            self.execute_cmd(cmd)

register_plugin(PluginAdvExec, 'adv_exec')
<|endoftext|>


<commit_before>import os
from nose.tools import eq_
from pyexcel_cli.merge import merge
from click.testing import CliRunner
from pyexcel import get_book
from textwrap import dedent

def test_simple_option():
    runner = CliRunner()
    file_fixture = os.path.join("tests", "fixtures", "transcode_simple.csv")
    dir_fixture = os.path.join("tests", "fixtures", "file_dir")
    glob_fixture = os.path.join("tests", "fixtures", "glob_dir", "*")
    output = "test_simple_option.xls"
    result = runner.invoke(merge, [file_fixture, dir_fixture, glob_fixture,
                                   output])
    eq_(result.exit_code, 0)
    book = get_book(file_name=output)
    expected = dedent("""
    transcode_simple.csv:
    +---+---+---+
    | 1 | 2 | 3 |
    +---+---+---+
    merge_test.csv:
    +---+---+---+
    | 1 | 2 | 3 |
    +---+---+---+
    merge_test2.csv:
    +---+---+---+
    | 1 | 2 | 3 |
    +---+---+---+""").strip('\n')
    eq_(str(book), expected)
    os.unlink(output)

def test_stdout_option():
    runner = CliRunner()
    file_fixture = os.path.join("tests", "fixtures", "transcode_simple.csv")
    dir_fixture = os.path.join("tests", "fixtures", "file_dir")
    glob_fixture = os.path.join("tests", "fixtures", "glob_dir", "*")
    output = "-"
    result = runner.invoke(merge, ["--output-file-type", "csv",
                                   "--csv-lineterminator", "\n",
                                   "--csv-delimiter", ":",
                                   file_fixture, dir_fixture, glob_fixture,
                                   output])
    eq_(result.exit_code, 0)
    expected = dedent("""
    ---pyexcel:transcode_simple.csv---
    1:2:3
    ---pyexcel---
    ---pyexcel:merge_test.csv---
    1:2:3
    ---pyexcel---
    ---pyexcel:merge_test2.csv---
    1:2:3
    ---pyexcel---
    """).strip('\n') + '\n'
    eq_(result.output, expected)

def test_more_csv_options():
    runner = CliRunner()
    file_fixture = os.path.join("tests", "fixtures", "transcode_quoted.csv")
    output = "-"
    result = runner.invoke(merge, ["--output-file-type", "csv",
                                   "--csv-lineterminator", "\n",
                                   "--csv-delimiter", ":",
                                   "--csv-quoting", "minimal",
                                   file_fixture, output])
    eq_(result.exit_code, 0)
    expected = dedent("""
    hello:|world|:1:2
    3:4:5:6
    """).strip('\n') + '\n'
    eq_(result.output, expected)

def test_nothing_do_do():
    runner = CliRunner()
    glob_fixture = os.path.join("tests", "fixtures", "nothing_is_excel", "*")
    output = "test_simple_option.xls"
    result = runner.invoke(merge, [glob_fixture, output])
    expected = dedent("""
    Skipping tests{0}fixtures{0}nothing_is_excel{0}test.no.excel
    Nothing to be merged""").strip('\n') + '\n'
    eq_(result.exit_code, 0)
    eq_(result.output, expected.format(os.sep))
<commit_msg>debug travis build<commit_after>import os
from nose.tools import eq_
from pyexcel_cli.merge import merge
from click.testing import CliRunner
from pyexcel import get_book
from textwrap import dedent

def test_simple_option():
    runner = CliRunner()
    file_fixture = os.path.join("tests", "fixtures", "transcode_simple.csv")
    dir_fixture = os.path.join("tests", "fixtures", "file_dir")
    glob_fixture = os.path.join("tests", "fixtures", "glob_dir", "*")
    output = "test_simple_option.xls"
    result = runner.invoke(merge, [file_fixture, dir_fixture, glob_fixture,
                                   output])
    eq_(result.exit_code, 0)
    book = get_book(file_name=output)
    expected = dedent("""
    transcode_simple.csv:
    +---+---+---+
    | 1 | 2 | 3 |
    +---+---+---+
    merge_test.csv:
    +---+---+---+
    | 1 | 2 | 3 |
    +---+---+---+
    merge_test2.csv:
    +---+---+---+
    | 1 | 2 | 3 |
    +---+---+---+""").strip('\n')
    eq_(str(book), expected)
    os.unlink(output)

def test_stdout_option():
    runner = CliRunner()
    file_fixture = os.path.join("tests", "fixtures", "transcode_simple.csv")
    dir_fixture = os.path.join("tests", "fixtures", "file_dir")
    glob_fixture = os.path.join("tests", "fixtures", "glob_dir", "*")
    output = "-"
    result = runner.invoke(merge, ["--output-file-type", "csv",
                                   "--csv-lineterminator", "\n",
                                   "--csv-delimiter", ":",
                                   file_fixture, dir_fixture, glob_fixture,
                                   output])
    print(result.exc_info)
    eq_(result.exit_code, 0)
    expected = dedent("""
    ---pyexcel:transcode_simple.csv---
    1:2:3
    ---pyexcel---
    ---pyexcel:merge_test.csv---
    1:2:3
    ---pyexcel---
    ---pyexcel:merge_test2.csv---
    1:2:3
    ---pyexcel---
    """).strip('\n') + '\n'
    eq_(result.output, expected)

def test_more_csv_options():
    runner = CliRunner()
    file_fixture = os.path.join("tests", "fixtures", "transcode_quoted.csv")
    output = "-"
    result = runner.invoke(merge, ["--output-file-type", "csv",
                                   "--csv-lineterminator", "\n",
                                   "--csv-delimiter", ":",
                                   "--csv-quoting", "minimal",
                                   file_fixture, output])
    eq_(result.exit_code, 0)
    expected = dedent("""
    hello:|world|:1:2
    3:4:5:6
    """).strip('\n') + '\n'
    eq_(result.output, expected)

def test_nothing_do_do():
    runner = CliRunner()
    glob_fixture = os.path.join("tests", "fixtures", "nothing_is_excel", "*")
    output = "test_simple_option.xls"
    result = runner.invoke(merge, [glob_fixture, output])
    expected = dedent("""
    Skipping tests{0}fixtures{0}nothing_is_excel{0}test.no.excel
    Nothing to be merged""").strip('\n') + '\n'
    eq_(result.exit_code, 0)
    eq_(result.output, expected.format(os.sep))
<|endoftext|>


<commit_before> # Copyright Sourav Verma ABV_IIITMG

import requests
from bs4 import BeautifulSoup

url = "http://www.cricbuzz.com/cricket-match/live-scores"

def open_url(url):
    return requests.get(url).text  # returns html

def get_bsoup_object(html):
    return BeautifulSoup(html, "lxml")  # returns soup (BeautifulSoup's object)

def get_team_divs(soup):
    teams1 = soup.find_all('div', attrs = {'class': 'innings-info-1'})
    teams2 = soup.find_all('div', attrs = {'class': 'innings-info-2'})
    return get_team_scores(teams1), get_team_scores(teams2)

def get_team_scores(team_soup):
    team = list(map(lambda x: x.contents, team_soup))
    for t in team:
        t[1] = t[1].contents
    return [[t[0].strip(), t[1]] for t in team]

def main():
    t1, t2 = get_team_divs(get_bsoup_object(open_url(url)))
    for i in range(len(t1)):
print("%s. %s %s VS %s %s" %(i+1, t1[i][0], t1[i][1], t2[i][0], t2[i][1]))

<commit_msg>Main Function<commit_after>
 # Copyright Sourav Verma ABV_IIITMG

import requests
from bs4 import BeautifulSoup

url = "http://www.cricbuzz.com/cricket-match/live-scores"

def open_url(url):
    return requests.get(url).text  # returns html

def get_bsoup_object(html):
    return BeautifulSoup(html, "lxml")  # returns soup (BeautifulSoup's object)

def get_team_divs(soup):
    teams1 = soup.find_all('div', attrs = {'class': 'innings-info-1'})
    teams2 = soup.find_all('div', attrs = {'class': 'innings-info-2'})
    return get_team_scores(teams1), get_team_scores(teams2)

def get_team_scores(team_soup):
    team = list(map(lambda x: x.contents, team_soup))
    for t in team:
        t[1] = t[1].contents
    return [[t[0].strip(), t[1]] for t in team]

def main():
    t1, t2 = get_team_divs(get_bsoup_object(open_url(url)))
    for i in range(len(t1)):
print("%s. %s %s VS %s %s" %(i+1, t1[i][0], t1[i][1], t2[i][0], t2[i][1]))

<|endoftext|>


<commit_before>import warnings
import logging

from .base import BatchAttack
from .base import Attack
from .base import generator_decorator
from .saltandpepper import SaltAndPepperNoiseAttack
from .. import rng

class PointwiseAttack(BatchAttack):
    """Starts with an adversarial and performs a binary search between
    the adversarial and the original for each dimension of the input
    individually.

    References
    ----------
    .. [1] L. Schott, J. Rauber, M. Bethge, W. Brendel: "Towards the first
           adversarially robust neural network model on MNIST", ICLR (2019)
           https://arxiv.org/abs/1805.09190

    """

    @generator_decorator
    def as_generator(self, a,
                     starting_point=None, initialization_attack=None):

        """Starts with an adversarial and performs a binary search between
        the adversarial and the original for each dimension of the input
        individually.

        Parameters
        ----------
        input_or_adv : `numpy.ndarray` or :class:`Adversarial`
            The original, unperturbed input as a `numpy.ndarray` or
            an :class:`Adversarial` instance.
        label : int
            The reference label of the original input. Must be passed
            if `a` is a `numpy.ndarray`, must not be passed if `a` is
            an :class:`Adversarial` instance.
        unpack : bool
            If true, returns the adversarial input, otherwise returns
            the Adversarial object.
        starting_point : `numpy.ndarray`
            Adversarial input to use as a starting point, in particular
            for targeted attacks.
        initialization_attack : :class:`Attack`
            Attack to use to find a starting point. Defaults to
            SaltAndPepperNoiseAttack.

        """

        self._starting_point = starting_point
        self._initialization_attack = initialization_attack
        self.initialize_starting_point(a)

        if a.perturbed is None:
            warnings.warn(
                'Initialization failed. If the criterion is targeted,'
                ' it might be necessary to pass an explicit starting'
                ' point or targeted initialization attack.')
            return

        shape = a.unperturbed.shape
        N = a.unperturbed.size

        original = a.unperturbed.reshape(-1)
        x = a.perturbed.copy().reshape(-1)

        assert original.dtype == x.dtype

        while True:
            # draw random shuffling of all indices
            indices = list(range(N))
            rng.shuffle(indices)

            for index in indices:
                # change index
                old_value = x[index]
                new_value = original[index]
                if old_value == new_value:
                    continue
                x[index] = new_value

                # check if still adversarial
                _, is_adversarial = yield from a.forward_one(x.reshape(shape))

                # if adversarial, restart from there
                if is_adversarial:
                    logging.info('Reset value to original -> new distance:'
                                 ' {}'.format(a.distance))
                    break

                # if not, undo change
                x[index] = old_value
            else:
                # no index was successful
                break

        logging.info('Starting binary searches')

        while True:
            # draw random shuffling of all indices
            indices = list(range(N))
            rng.shuffle(indices)

            # whether that run through all values made any improvement
            improved = False

            logging.info('Starting new loop through all values')

            for index in indices:
                # change index
                old_value = x[index]
                original_value = original[index]
                if old_value == original_value:
                    continue
                x[index] = original_value

                # check if still adversarial
                _, is_adversarial = yield from a.forward_one(x.reshape(shape))

                # if adversarial, no binary search needed
                if is_adversarial:  # pragma: no cover
                    logging.info('Reset value at {} to original ->'
                                 ' new distance: {}'.format(
                                     index, a.distance))
                    improved = True
                else:
                    # binary search
                    adv_value = old_value
                    non_adv_value = original_value
                    best_adv_value = yield from self.binary_search(
                        a, x, index, adv_value, non_adv_value, shape)

                    if old_value != best_adv_value:
                        x[index] = best_adv_value
                        improved = True
                        logging.info('Set value at {} from {} to {}'
                                     ' (original has {}) ->'
                                     ' new distance: {}'.format(
                                         index, old_value, best_adv_value,
                                         original_value, a.distance))

            if not improved:
                # no improvement for any of the indices
                break

    def binary_search(self, a, x, index, adv_value, non_adv_value, shape):
        for i in range(10):
            next_value = (adv_value + non_adv_value) / 2
            x[index] = next_value
            _, is_adversarial = yield from a.forward_one(x.reshape(shape))
            if is_adversarial:
                adv_value = next_value
            else:
                non_adv_value = next_value
        return adv_value

    def initialize_starting_point(self, a):
        starting_point = self._starting_point
        init_attack = self._initialization_attack

        if a.perturbed is not None:
            if starting_point is not None:  # pragma: no cover
                warnings.warn(
                    'Ignoring starting_point because the attack'
                    ' is applied to a previously found adversarial.')
            if init_attack is not None:  # pragma: no cover
                warnings.warn(
                    'Ignoring initialization_attack because the attack'
                    ' is applied to a previously found adversarial.')
            return

        if starting_point is not None:
            yield from a.forward_one(starting_point)
            assert a.perturbed is not None, (
                'Invalid starting point provided. Please provide a starting '
                'point that is adversarial.')
            return

        if init_attack is None:
            init_attack = SaltAndPepperNoiseAttack
            logging.info(
                'Neither starting_point nor initialization_attack given.'
                ' Falling back to {} for initialization.'.format(
                    init_attack.__name__))

        if issubclass(init_attack, Attack) or\
                issubclass(init_attack, BatchAttack):
            # instantiate if necessary
            init_attack = init_attack()

        init_attack(a)
<commit_msg>Fix wrong attack initialization<commit_after>import warnings
import logging

from .base import BatchAttack
from .base import Attack
from .base import generator_decorator
from .saltandpepper import SaltAndPepperNoiseAttack
from .. import rng

class PointwiseAttack(BatchAttack):
    """Starts with an adversarial and performs a binary search between
    the adversarial and the original for each dimension of the input
    individually.

    References
    ----------
    .. [1] L. Schott, J. Rauber, M. Bethge, W. Brendel: "Towards the first
           adversarially robust neural network model on MNIST", ICLR (2019)
           https://arxiv.org/abs/1805.09190

    """

    @generator_decorator
    def as_generator(self, a,
                     starting_point=None, initialization_attack=None):

        """Starts with an adversarial and performs a binary search between
        the adversarial and the original for each dimension of the input
        individually.

        Parameters
        ----------
        input_or_adv : `numpy.ndarray` or :class:`Adversarial`
            The original, unperturbed input as a `numpy.ndarray` or
            an :class:`Adversarial` instance.
        label : int
            The reference label of the original input. Must be passed
            if `a` is a `numpy.ndarray`, must not be passed if `a` is
            an :class:`Adversarial` instance.
        unpack : bool
            If true, returns the adversarial input, otherwise returns
            the Adversarial object.
        starting_point : `numpy.ndarray`
            Adversarial input to use as a starting point, in particular
            for targeted attacks.
        initialization_attack : :class:`Attack`
            Attack to use to find a starting point. Defaults to
            SaltAndPepperNoiseAttack.

        """

        self._starting_point = starting_point
        self._initialization_attack = initialization_attack
        yield from self.initialize_starting_point(a)

        if a.perturbed is None:
            warnings.warn(
                'Initialization failed. If the criterion is targeted,'
                ' it might be necessary to pass an explicit starting'
                ' point or targeted initialization attack.')
            return

        shape = a.unperturbed.shape
        N = a.unperturbed.size

        original = a.unperturbed.reshape(-1)
        x = a.perturbed.copy().reshape(-1)

        assert original.dtype == x.dtype

        while True:
            # draw random shuffling of all indices
            indices = list(range(N))
            rng.shuffle(indices)

            for index in indices:
                # change index
                old_value = x[index]
                new_value = original[index]
                if old_value == new_value:
                    continue
                x[index] = new_value

                # check if still adversarial
                _, is_adversarial = yield from a.forward_one(x.reshape(shape))

                # if adversarial, restart from there
                if is_adversarial:
                    logging.info('Reset value to original -> new distance:'
                                 ' {}'.format(a.distance))
                    break

                # if not, undo change
                x[index] = old_value
            else:
                # no index was successful
                break

        logging.info('Starting binary searches')

        while True:
            # draw random shuffling of all indices
            indices = list(range(N))
            rng.shuffle(indices)

            # whether that run through all values made any improvement
            improved = False

            logging.info('Starting new loop through all values')

            for index in indices:
                # change index
                old_value = x[index]
                original_value = original[index]
                if old_value == original_value:
                    continue
                x[index] = original_value

                # check if still adversarial
                _, is_adversarial = yield from a.forward_one(x.reshape(shape))

                # if adversarial, no binary search needed
                if is_adversarial:  # pragma: no cover
                    logging.info('Reset value at {} to original ->'
                                 ' new distance: {}'.format(
                                     index, a.distance))
                    improved = True
                else:
                    # binary search
                    adv_value = old_value
                    non_adv_value = original_value
                    best_adv_value = yield from self.binary_search(
                        a, x, index, adv_value, non_adv_value, shape)

                    if old_value != best_adv_value:
                        x[index] = best_adv_value
                        improved = True
                        logging.info('Set value at {} from {} to {}'
                                     ' (original has {}) ->'
                                     ' new distance: {}'.format(
                                         index, old_value, best_adv_value,
                                         original_value, a.distance))

            if not improved:
                # no improvement for any of the indices
                break

    def binary_search(self, a, x, index, adv_value, non_adv_value, shape):
        for i in range(10):
            next_value = (adv_value + non_adv_value) / 2
            x[index] = next_value
            _, is_adversarial = yield from a.forward_one(x.reshape(shape))
            if is_adversarial:
                adv_value = next_value
            else:
                non_adv_value = next_value
        return adv_value

    def initialize_starting_point(self, a):
        starting_point = self._starting_point
        init_attack = self._initialization_attack

        if a.perturbed is not None:
            if starting_point is not None:  # pragma: no cover
                warnings.warn(
                    'Ignoring starting_point because the attack'
                    ' is applied to a previously found adversarial.')
            if init_attack is not None:  # pragma: no cover
                warnings.warn(
                    'Ignoring initialization_attack because the attack'
                    ' is applied to a previously found adversarial.')
            return

        if starting_point is not None:
            yield from a.forward_one(starting_point)
            assert a.perturbed is not None, (
                'Invalid starting point provided. Please provide a starting '
                'point that is adversarial.')
            return

        if init_attack is None:
            init_attack = SaltAndPepperNoiseAttack
            logging.info(
                'Neither starting_point nor initialization_attack given.'
                ' Falling back to {} for initialization.'.format(
                    init_attack.__name__))

        if issubclass(init_attack, Attack) or\
                issubclass(init_attack, BatchAttack):
            # instantiate if necessary
            init_attack = init_attack()

        yield from init_attack.as_generator(a)
<|endoftext|>


<commit_before>import asyncio
import concurrent.futures
import unittest

from unittest import mock

import janus

class TestMixedMode(unittest.TestCase):
    def setUp(self):
        self.loop = asyncio.new_event_loop()
        self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=5)
        self.loop.set_default_executor(self.executor)
        asyncio.set_event_loop(None)

    def tearDown(self):
        self.executor.shutdown()
        self.loop.close()

    def test_ctor_loop(self):
        loop = mock.Mock()
        q = janus.Queue(loop=loop)
        self.assertIs(q._loop, loop)

        q = janus.Queue(loop=self.loop)
        self.assertIs(q._loop, self.loop)

    def test_ctor_noloop(self):
        asyncio.set_event_loop(self.loop)
        q = janus.Queue()
        self.assertIs(q._loop, self.loop)

    def test_maxsize(self):
        q = janus.Queue(5, loop=self.loop)
        self.assertIs(5, q.maxsize)

    def test_maxsize_named_param(self):
        q = janus.Queue(maxsize=7, loop=self.loop)
        self.assertIs(7, q.maxsize)

    def test_maxsize_default(self):
        q = janus.Queue(loop=self.loop)
        self.assertIs(0, q.maxsize)

    def test_unfinished(self):
        q = janus.Queue(loop=self.loop)
        self.assertEqual(q.sync_q.unfinished_tasks, 0)
        self.assertEqual(q.async_q.unfinished_tasks, 0)
        q.sync_q.put(1)
        self.assertEqual(q.sync_q.unfinished_tasks, 1)
        self.assertEqual(q.async_q.unfinished_tasks, 1)
        q.sync_q.get()
        self.assertEqual(q.sync_q.unfinished_tasks, 1)
        self.assertEqual(q.async_q.unfinished_tasks, 1)
        q.sync_q.task_done()
        self.assertEqual(q.sync_q.unfinished_tasks, 0)
        self.assertEqual(q.async_q.unfinished_tasks, 0)

    def test_sync_put_async_get(self):
        q = janus.Queue(loop=self.loop)

        def threaded():
            for i in range(5):
                q.sync_q.put(i)

        @asyncio.coroutine
        def go():
            f = self.loop.run_in_executor(None, threaded)
            for i in range(5):
                val = yield from q.async_q.get()
                self.assertEqual(val, i)

            self.assertTrue(q.async_q.empty())

            yield from f

        for i in range(3):
            self.loop.run_until_complete(go())

    def test_sync_put_async_join(self):
        q = janus.Queue(loop=self.loop)

        for i in range(5):
            q.sync_q.put(i)

        @asyncio.coroutine
        def do_work():
            yield from asyncio.sleep(1)
            while True:
                yield from q.async_q.get()
                q.async_q.task_done()

        task = self.loop.create_task(do_work())

        @asyncio.coroutine
        def wait_for_empty_queue():
            while not q.async_q.empty():
                yield from q.async_q.join()
            task.cancel()

        self.loop.run_until_complete(wait_for_empty_queue())

    def test_async_put_sync_get(self):
        q = janus.Queue(loop=self.loop)

        def threaded():
            for i in range(5):
                val = q.sync_q.get()
                self.assertEqual(val, i)

        @asyncio.coroutine
        def go():
            f = self.loop.run_in_executor(None, threaded)
            for i in range(5):
                yield from q.async_q.put(i)

            yield from f
            self.assertTrue(q.async_q.empty())

        for i in range(3):
            self.loop.run_until_complete(go())

    def test_sync_join_async_done(self):
        q = janus.Queue(loop=self.loop)

        def threaded():
            for i in range(5):
                q.sync_q.put(i)
            q.sync_q.join()

        @asyncio.coroutine
        def go():
            f = self.loop.run_in_executor(None, threaded)
            for i in range(5):
                val = yield from q.async_q.get()
                self.assertEqual(val, i)
                q.async_q.task_done()

            self.assertTrue(q.async_q.empty())

            yield from f

        for i in range(3):
            self.loop.run_until_complete(go())

    def test_async_join_async_done(self):
        q = janus.Queue(loop=self.loop)

        def threaded():
            for i in range(5):
                val = q.sync_q.get()
                self.assertEqual(val, i)
                q.sync_q.task_done()

        @asyncio.coroutine
        def go():
            f = self.loop.run_in_executor(None, threaded)
            for i in range(5):
                yield from q.async_q.put(i)

            yield from q.async_q.join()

            yield from f
            self.assertTrue(q.async_q.empty())

        for i in range(3):
            self.loop.run_until_complete(go())

    def test_wait_without_closing(self):
        q = janus.Queue(loop=self.loop)

        with self.assertRaises(RuntimeError):
            self.loop.run_until_complete(q.wait_closed())

        q.close()
        self.loop.run_until_complete(q.wait_closed())

    def test_modifying_forbidden_after_closing(self):
        q = janus.Queue(loop=self.loop)
        q.close()

        with self.assertRaises(RuntimeError):
            q.sync_q.put(5)

        with self.assertRaises(RuntimeError):
            q.sync_q.get()

        with self.assertRaises(RuntimeError):
            q.sync_q.task_done()

        with self.assertRaises(RuntimeError):
            self.loop.run_until_complete(q.async_q.put(5))

        with self.assertRaises(RuntimeError):
            q.async_q.put_nowait(5)

        with self.assertRaises(RuntimeError):
            q.async_q.get_nowait()

        with self.assertRaises(RuntimeError):
            self.loop.run_until_complete(q.sync_q.task_done())

        self.loop.run_until_complete(q.wait_closed())

    def test_double_closing(self):
        q = janus.Queue(loop=self.loop)
        q.close()
        q.close()
        self.loop.run_until_complete(q.wait_closed())
<commit_msg>fix test<commit_after>import asyncio
import concurrent.futures
import unittest

from unittest import mock

import janus

class TestMixedMode(unittest.TestCase):
    def setUp(self):
        self.loop = asyncio.new_event_loop()
        self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=5)
        self.loop.set_default_executor(self.executor)
        asyncio.set_event_loop(None)

    def tearDown(self):
        self.executor.shutdown()
        self.loop.close()

    def test_ctor_loop(self):
        loop = mock.Mock()
        q = janus.Queue(loop=loop)
        self.assertIs(q._loop, loop)

        q = janus.Queue(loop=self.loop)
        self.assertIs(q._loop, self.loop)

    def test_ctor_noloop(self):
        asyncio.set_event_loop(self.loop)
        q = janus.Queue()
        self.assertIs(q._loop, self.loop)

    def test_maxsize(self):
        q = janus.Queue(5, loop=self.loop)
        self.assertIs(5, q.maxsize)

    def test_maxsize_named_param(self):
        q = janus.Queue(maxsize=7, loop=self.loop)
        self.assertIs(7, q.maxsize)

    def test_maxsize_default(self):
        q = janus.Queue(loop=self.loop)
        self.assertIs(0, q.maxsize)

    def test_unfinished(self):
        q = janus.Queue(loop=self.loop)
        self.assertEqual(q.sync_q.unfinished_tasks, 0)
        self.assertEqual(q.async_q.unfinished_tasks, 0)
        q.sync_q.put(1)
        self.assertEqual(q.sync_q.unfinished_tasks, 1)
        self.assertEqual(q.async_q.unfinished_tasks, 1)
        q.sync_q.get()
        self.assertEqual(q.sync_q.unfinished_tasks, 1)
        self.assertEqual(q.async_q.unfinished_tasks, 1)
        q.sync_q.task_done()
        self.assertEqual(q.sync_q.unfinished_tasks, 0)
        self.assertEqual(q.async_q.unfinished_tasks, 0)

    def test_sync_put_async_get(self):
        q = janus.Queue(loop=self.loop)

        def threaded():
            for i in range(5):
                q.sync_q.put(i)

        @asyncio.coroutine
        def go():
            f = self.loop.run_in_executor(None, threaded)
            for i in range(5):
                val = yield from q.async_q.get()
                self.assertEqual(val, i)

            self.assertTrue(q.async_q.empty())

            yield from f

        for i in range(3):
            self.loop.run_until_complete(go())

    def test_sync_put_async_join(self):
        q = janus.Queue(loop=self.loop)

        for i in range(5):
            q.sync_q.put(i)

        @asyncio.coroutine
        def do_work():
            yield from asyncio.sleep(1)
            while True:
                yield from q.async_q.get()
                q.async_q.task_done()

        task = self.loop.create_task(do_work())

        @asyncio.coroutine
        def wait_for_empty_queue():
            yield from q.async_q.join()
            task.cancel()

        self.loop.run_until_complete(wait_for_empty_queue())

    def test_async_put_sync_get(self):
        q = janus.Queue(loop=self.loop)

        def threaded():
            for i in range(5):
                val = q.sync_q.get()
                self.assertEqual(val, i)

        @asyncio.coroutine
        def go():
            f = self.loop.run_in_executor(None, threaded)
            for i in range(5):
                yield from q.async_q.put(i)

            yield from f
            self.assertTrue(q.async_q.empty())

        for i in range(3):
            self.loop.run_until_complete(go())

    def test_sync_join_async_done(self):
        q = janus.Queue(loop=self.loop)

        def threaded():
            for i in range(5):
                q.sync_q.put(i)
            q.sync_q.join()

        @asyncio.coroutine
        def go():
            f = self.loop.run_in_executor(None, threaded)
            for i in range(5):
                val = yield from q.async_q.get()
                self.assertEqual(val, i)
                q.async_q.task_done()

            self.assertTrue(q.async_q.empty())

            yield from f

        for i in range(3):
            self.loop.run_until_complete(go())

    def test_async_join_async_done(self):
        q = janus.Queue(loop=self.loop)

        def threaded():
            for i in range(5):
                val = q.sync_q.get()
                self.assertEqual(val, i)
                q.sync_q.task_done()

        @asyncio.coroutine
        def go():
            f = self.loop.run_in_executor(None, threaded)
            for i in range(5):
                yield from q.async_q.put(i)

            yield from q.async_q.join()

            yield from f
            self.assertTrue(q.async_q.empty())

        for i in range(3):
            self.loop.run_until_complete(go())

    def test_wait_without_closing(self):
        q = janus.Queue(loop=self.loop)

        with self.assertRaises(RuntimeError):
            self.loop.run_until_complete(q.wait_closed())

        q.close()
        self.loop.run_until_complete(q.wait_closed())

    def test_modifying_forbidden_after_closing(self):
        q = janus.Queue(loop=self.loop)
        q.close()

        with self.assertRaises(RuntimeError):
            q.sync_q.put(5)

        with self.assertRaises(RuntimeError):
            q.sync_q.get()

        with self.assertRaises(RuntimeError):
            q.sync_q.task_done()

        with self.assertRaises(RuntimeError):
            self.loop.run_until_complete(q.async_q.put(5))

        with self.assertRaises(RuntimeError):
            q.async_q.put_nowait(5)

        with self.assertRaises(RuntimeError):
            q.async_q.get_nowait()

        with self.assertRaises(RuntimeError):
            self.loop.run_until_complete(q.sync_q.task_done())

        self.loop.run_until_complete(q.wait_closed())

    def test_double_closing(self):
        q = janus.Queue(loop=self.loop)
        q.close()
        q.close()
        self.loop.run_until_complete(q.wait_closed())
<|endoftext|>


<commit_before>from rest_framework import serializers, relations
from . import models
from django.contrib.auth.models import User, Group, Permission
from taggit_serializer.serializers import TagListSerializerField, TaggitSerializer
import address

class UserSerializer(serializers.HyperlinkedModelSerializer):
    groups = serializers.SlugRelatedField(many=True, queryset=Group.objects.all(),
            slug_field='name')
    permissions = serializers.SerializerMethodField()

    def get_permissions(self, obj):
        return map(lambda x: x.name,
                Permission.objects.filter(group__in=obj.groups.all()))

    class Meta:
        model = User
        fields = ('email', 'id', 'is_staff', 'is_superuser', 'groups',
        'permissions')

class LocalitySerializer(serializers.ModelSerializer):
    class Meta:
        model = address.models.Locality
        fields = ('name', 'postal_code')

class AddressSerializer(serializers.ModelSerializer):
    class Meta:
        model = address.models.Address
        fields = ('raw', 'street_number', 'route', 'locality')

class TurfSerializer(serializers.HyperlinkedModelSerializer):
    locality = LocalitySerializer()

    class Meta:
        model = models.Turf
        fields = ('name', 'locality', 'url', 'id')

class TurfMembershipSerializer(serializers.ModelSerializer):
    name = serializers.CharField(source='turf.name', read_only=True)
    turf = serializers.PrimaryKeyRelatedField(read_only=True)
    url = serializers.HyperlinkedRelatedField(source='turf', queryset=models.Turf.objects.all(),
    view_name='turf-detail')

    class Meta:
        model = models.TurfMembership
        fields = ('name', 'joined_on', 'is_captain', 'turf', 'url')

class PersonStateSerializer(serializers.ModelSerializer):
    class Meta:
        model = models.PersonState
        fields = ('name', 'id')
        lookup_field = 'name'
        extra_kwargs = {
                'id': {'read_only': True}
        }

class PersonStateSerializer(serializers.ModelSerializer):
    class Meta:
        model = models.PersonState
        fields = ('name', 'id')
        lookup_field = 'name'
        extra_kwargs = {
                'id': {'read_only': True}
        }

class PersonSerializer(TaggitSerializer, serializers.HyperlinkedModelSerializer):
    tags = TagListSerializerField()
    current_turf = TurfSerializer(read_only=True)
    turf_memberships = TurfMembershipSerializer(many=True, read_only=True)
    state = serializers.SlugRelatedField(queryset=models.PersonState.objects.all(),
            slug_field='name')

    def to_internal_value(self, data):
        if 'state' in data:
            models.PersonState.objects.get_or_create(name=data.get('state'))
        return super(PersonSerializer, self).to_internal_value(data)

    class Meta:
        model = models.Person
        fields = ('name',  'id', 'email', 'created', 'url', 'tags',
        'geo', 'current_turf', 'turf_memberships', 'state', 'phone')

        lookup_field = 'email'
        extra_kwargs = {
                'url': {'lookup_field': 'email'},
                'id': {'source': 'email', 'read_only': True},
                'geo': {'read_only': True},
                'phone': {'write_only': True},
        }
<commit_msg>crm: serializers: support setting person addresses via api<commit_after>from rest_framework import serializers, relations
from . import models
from django.contrib.auth.models import User, Group, Permission
from taggit_serializer.serializers import TagListSerializerField, TaggitSerializer
import address

class UserSerializer(serializers.HyperlinkedModelSerializer):
    groups = serializers.SlugRelatedField(many=True, queryset=Group.objects.all(),
            slug_field='name')
    permissions = serializers.SerializerMethodField()

    def get_permissions(self, obj):
        return map(lambda x: x.name,
                Permission.objects.filter(group__in=obj.groups.all()))

    class Meta:
        model = User
        fields = ('email', 'id', 'is_staff', 'is_superuser', 'groups',
        'permissions')

class LocalitySerializer(serializers.ModelSerializer):
    class Meta:
        model = address.models.Locality
        fields = ('name', 'postal_code')

class AddressSerializer(serializers.ModelSerializer):
    class Meta:
        model = address.models.Address
        fields = ('raw', 'street_number', 'route', 'locality')

    def to_internal_value(self, data):
        if type(data) is dict:
            return super(AddressSerializer, self).to_internal_value(data)
        else:
            return super(AddressSerializer, self).to_internal_value({'raw':
                data})

class TurfSerializer(serializers.HyperlinkedModelSerializer):
    locality = LocalitySerializer()

    class Meta:
        model = models.Turf
        fields = ('name', 'locality', 'url', 'id')

class TurfMembershipSerializer(serializers.ModelSerializer):
    name = serializers.CharField(source='turf.name', read_only=True)
    turf = serializers.PrimaryKeyRelatedField(read_only=True)
    url = serializers.HyperlinkedRelatedField(source='turf', queryset=models.Turf.objects.all(),
    view_name='turf-detail')

    class Meta:
        model = models.TurfMembership
        fields = ('name', 'joined_on', 'is_captain', 'turf', 'url')

class PersonStateSerializer(serializers.ModelSerializer):
    class Meta:
        model = models.PersonState
        fields = ('name', 'id')
        lookup_field = 'name'
        extra_kwargs = {
                'id': {'read_only': True}
        }

class PersonStateSerializer(serializers.ModelSerializer):
    class Meta:
        model = models.PersonState
        fields = ('name', 'id')
        lookup_field = 'name'
        extra_kwargs = {
                'id': {'read_only': True}
        }

class PersonSerializer(TaggitSerializer, serializers.HyperlinkedModelSerializer):
    tags = TagListSerializerField()
    current_turf = TurfSerializer(read_only=True)
    turf_memberships = TurfMembershipSerializer(many=True, read_only=True)
    state = serializers.SlugRelatedField(queryset=models.PersonState.objects.all(),
            slug_field='name', required=False)
    address = AddressSerializer(write_only=True, required=False)

    def to_internal_value(self, data):
        if 'state' in data:
            models.PersonState.objects.get_or_create(name=data.get('state'))
        return super(PersonSerializer, self).to_internal_value(data)

    def update(self, instance, validated_data):
        if 'address' in validated_data:
            instance.address = validated_data.pop('address')
        instance.save()
        return instance

    def create(self, validated_data):
        valid_address = validated_data.pop('address', None)
        return models.Person.objects.create(address=valid_address, **validated_data)

    class Meta:
        model = models.Person
        fields = ('name',  'id', 'email', 'created', 'url', 'tags',
        'geo', 'current_turf', 'turf_memberships', 'state', 'address', 'phone')

        lookup_field = 'email'
        extra_kwargs = {
                'url': {'lookup_field': 'email'},
                'id': {'source': 'email', 'read_only': True},
                'geo': {'read_only': True},
                'phone': {'write_only': True},
        }
<|endoftext|>


<commit_before># coding: utf-8

from django.conf.urls.defaults import patterns

urlpatterns = patterns('',
    # Ximpia Templates
    (r'^jxTemplate/(?P<app>.*)/(?P<mode>.*)/(?P<tmplName>.*)$', 'ximpia.xpcore.views.jxTemplate'),
    (r'^jxAppTemplate/(?P<app>.*)$', 'ximpia.xpcore.views.jxAppTemplate'),
    # Exec Server action
    (r'^(?P<appSlug>[a-zA-Z0-9-]+)/do/(?P<actionSlug>[a-zA-Z0-9-]+)/(?P<actionAttrs>.*)$', 'ximpia.xpcore.views.execActionMsg'),
    # Show Server View.
    (r'^(?P<appSlug>[a-zA-Z0-9-]+)/(?P<viewSlug>[a-zA-Z0-9-]+)/(?P<viewAttrs>.*)$', 'ximpia.xpcore.views.showView'),
    # Ajax Urls
    (r'^jxSuggestList$', 'ximpia.xpcore.views.jxSuggestList'),
    (r'^jxJSON$', 'ximpia.xpcore.views.jxJSON'),
    (r'^jxService$', 'ximpia.xpcore.views.jxService'),
    (r'^jxDataQuery$', 'ximpia.xpcore.views.jxDataQuery'),
    (r'^jxSave$', 'ximpia.xpcore.views.jxSave'),
    (r'^jxDelete$', 'ximpia.xpcore.views.jxDelete'),
    (r'^jxSearchHeader$', 'ximpia.xpcore.views.jxSearchHeader'),
)
<commit_msg>application view url<commit_after># coding: utf-8

from django.conf.urls.defaults import patterns

urlpatterns = patterns('',
    # Ximpia Templates
    (r'^jxTemplate/(?P<app>.*)/(?P<mode>.*)/(?P<tmplName>.*)$', 'ximpia.xpcore.views.jxTemplate'),
    (r'^jxAppTemplate/(?P<app>.*)$', 'ximpia.xpcore.views.jxAppTemplate'),
    # Exec Server action
    (r'^(?P<appSlug>[a-zA-Z0-9-]+)/do/(?P<actionSlug>[a-zA-Z0-9-]+)/(?P<actionAttrs>.*)$', 'ximpia.xpcore.views.execActionMsg'),
    # Show Server View.
    (r'^(?P<appSlug>[a-zA-Z0-9-]+)/$', 'ximpia.xpcore.views.showView'),
    (r'^(?P<appSlug>[a-zA-Z0-9-]+)/(?P<viewSlug>[a-zA-Z0-9-]+)/(?P<viewAttrs>.*)$', 'ximpia.xpcore.views.showView'),
    # Ajax Urls
    (r'^jxSuggestList$', 'ximpia.xpcore.views.jxSuggestList'),
    (r'^jxJSON$', 'ximpia.xpcore.views.jxJSON'),
    (r'^jxService$', 'ximpia.xpcore.views.jxService'),
    (r'^jxDataQuery$', 'ximpia.xpcore.views.jxDataQuery'),
    (r'^jxSave$', 'ximpia.xpcore.views.jxSave'),
    (r'^jxDelete$', 'ximpia.xpcore.views.jxDelete'),
    (r'^jxSearchHeader$', 'ximpia.xpcore.views.jxSearchHeader'),
)
<|endoftext|>


<commit_before># Copyright 2010 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for model.py."""

from datetime import datetime
from google.appengine.ext import db
import unittest
import model
from utils import get_utcnow, set_utcnow_for_test

class ModelTests(unittest.TestCase):
    '''Test the loose odds and ends.'''

    def setUp(self):
        set_utcnow_for_test(datetime(2010, 1, 1))
        self.p1 = model.Person.create_original(
            'haiti',
            first_name='John',
            last_name='Smith',
            home_street='Washington St.',
            home_city='Los Angeles',
            home_state='California',
            home_postal_code='11111',
            home_neighborhood='Good Neighborhood',
            author_name='Alice Smith',
            author_phone='111-111-1111',
            author_email='alice.smith@gmail.com',
            source_url='https://www.source.com',
            source_date=datetime(2010, 1, 1),
            source_name='Source Name',
            entry_date=datetime(2010, 1, 1),
            expiry_date=datetime(2010, 2, 1),
            other='')
        self.p2 = model.Person.create_original(
            'haiti',
            first_name='Tzvika',
            last_name='Hartman',
            home_street='Herzl St.',
            home_city='Tel Aviv',
            home_state='Israel',
            entry_date=datetime(2010, 1, 1),
            expiry_date=datetime(2010, 3, 1),
            other='')
        self.key_p1 = db.put(self.p1)
        self.key_p2 = db.put(self.p2)

        self.n1_1 = model.Note.create_original(
            'haiti',
            person_record_id=self.p1.record_id,
            linked_person_record_id=self.p2.record_id,
            status=u'believed_missing',
            found=False,
            entry_date=get_utcnow(),
            source_date=datetime(2000, 1, 1))
        self.n1_2 = model.Note.create_original(
            'haiti',
            person_record_id=self.p1.record_id,
            found=True,
            entry_date=get_utcnow(),
            source_date=datetime(2000, 2, 2))
        self.key_n1_1 = db.put(self.n1_1)
        self.key_n1_2 = db.put(self.n1_2)

        # Update the Person entity according to the Note.
        self.p1.update_from_note(self.n1_1)
        self.p1.update_from_note(self.n1_2)
        db.put(self.p1)

    def tearDown(self):
        db.delete([self.key_p1, self.key_p2, self.key_n1_1, self.key_n1_2])

    def test_person(self):
        assert self.p1.first_name == 'John'
        assert self.p1.photo_url == ''
        assert self.p1.is_clone() == False
        assert model.Person.get('haiti', self.p1.record_id).record_id == \
            self.p1.record_id
        assert model.Person.get('haiti', self.p2.record_id).record_id == \
            self.p2.record_id
        assert model.Person.get('haiti', self.p1.record_id).record_id != \
            self.p2.record_id

        # Testing prefix properties
        assert hasattr(self.p1, 'first_name_n_')
        assert hasattr(self.p1, 'home_street_n1_')
        assert hasattr(self.p1, 'home_postal_code_n2_')

        # Testing indexing properties
        assert self.p1._fields_to_index_properties == \
            ['first_name', 'last_name']
        assert self.p1._fields_to_index_by_prefix_properties == \
            ['first_name', 'last_name']

        # Test propagation of Note fields to Person.
        assert self.p1.latest_status == u'believed_missing'  # from first note
        assert self.p1.latest_status_source_date == datetime(2000, 1, 1)
        assert self.p1.latest_found == True  # from second note
        assert self.p1.latest_found_source_date == datetime(2000, 2, 2)

        # Adding a Note with only 'found' should not affect 'last_status'.
        n1_3 = model.Note.create_original(
            'haiti', person_record_id=self.p1.record_id, found=False,
            entry_date=get_utcnow(), source_date=datetime(2000, 3, 3))
        self.p1.update_from_note(n1_3)
        assert self.p1.latest_status == u'believed_missing'
        assert self.p1.latest_status_source_date == datetime(2000, 1, 1)
        assert self.p1.latest_found == False
        assert self.p1.latest_found_source_date == datetime(2000, 3, 3)

        # Adding a Note with only 'status' should not affect 'last_found'.
        n1_4 = model.Note.create_original(
            'haiti', person_record_id=self.p1.record_id,
            found=None, status=u'is_note_author',
            entry_date=get_utcnow(),
            source_date=datetime(2000, 4, 4))
        self.p1.update_from_note(n1_4)
        assert self.p1.latest_status == u'is_note_author'
        assert self.p1.latest_status_source_date == datetime(2000, 4, 4)
        assert self.p1.latest_found == False
        assert self.p1.latest_found_source_date == datetime(2000, 3, 3)

        # Adding an older Note should not affect either field.
        n1_5 = model.Note.create_original(
            'haiti', person_record_id=self.p1.record_id,
            found=True, status=u'believed_alive',
            entry_date=get_utcnow(),
            source_date=datetime(2000, 1, 2))
        self.p1.update_from_note(n1_5)
        assert self.p1.latest_status == u'is_note_author'
        assert self.p1.latest_status_source_date == datetime(2000, 4, 4)
        assert self.p1.latest_found == False
        assert self.p1.latest_found_source_date == datetime(2000, 3, 3)

        # Adding a Note with a date in between should affect only one field.
        n1_6 = model.Note.create_original(
            'haiti', person_record_id=self.p1.record_id,
            found=True, status=u'believed_alive',
            entry_date=get_utcnow(),
            source_date=datetime(2000, 3, 4))
        self.p1.update_from_note(n1_6)
        assert self.p1.latest_status == u'is_note_author'
        assert self.p1.latest_status_source_date == datetime(2000, 4, 4)
        assert self.p1.latest_found == True
        assert self.p1.latest_found_source_date == datetime(2000, 3, 4)

    def test_note(self):
        assert self.n1_1.is_clone() == False
        notes = self.p1.get_notes()
        assert notes[0].record_id == self.n1_1.record_id
        assert notes[1].record_id == self.n1_2.record_id
        assert self.p1.get_linked_persons()[0].record_id == self.p2.record_id
        assert self.p2.get_linked_persons() == []

        assert model.Note.get('haiti', self.n1_1.record_id).record_id == \
            self.n1_1.record_id
        assert model.Note.get('haiti', self.n1_2.record_id).record_id == \
            self.n1_2.record_id

    def test_subscription(self):
        sd = 'haiti'
        email1 = 'one@example.com'
        email2 = 'two@example.com'
        s1 = model.Subscription.create(sd, self.p1.record_id, email1, 'fr')
        s2 = model.Subscription.create(sd, self.p1.record_id, email2, 'en')
        key_s1 = db.put(s1)
        key_s2 = db.put(s2)

        assert model.Subscription.get(sd, self.p1.record_id, email1) is not None
        assert model.Subscription.get(sd, self.p1.record_id, email2) is not None
        assert model.Subscription.get(sd, self.p2.record_id, email1) is None
        assert model.Subscription.get(sd, self.p2.record_id, email2) is None
        assert len(self.p1.get_subscriptions()) == 2
        assert len(self.p2.get_subscriptions()) == 0

        s3 = model.Subscription.create(sd, self.p1.record_id, email2, 'ar')
        key_s3 = db.put(s3)
        assert len(self.p1.get_subscriptions()) == 2
        assert model.Subscription.get(
            sd, self.p1.record_id, email2).language == 'ar'
        db.delete([key_s1, key_s2, key_s3])

    def test_past_due(self):
        """Make sure Person records are detected as past due correctly."""
        def assert_past_due_count(expected):
            assert len(list(model.Person.past_due_records())) == expected
        assert_past_due_count(0)
        set_utcnow_for_test(datetime(2010, 2, 15))
        assert_past_due_count(1)
        set_utcnow_for_test(datetime(2010, 3, 15))
        assert_past_due_count(2)

    def test_put_expiry_flags(self):
        # Try put_expiry_flags when the record has not expired yet.
        self.p1.put_expiry_flags()

        # Both entities should be unexpired.
        p1 = db.get(self.p1.key())
        assert not p1.is_expired
        assert p1.first_name == 'John'
        n1_1 = db.get(self.n1_1.key())
        assert not n1_1.is_expired

        # Advance past the expiry date and try again.
        set_utcnow_for_test(datetime(2010, 2, 3))
        p1.put_expiry_flags()

        # Both entities should be expired.
        p1 = db.get(self.p1.key())
        assert p1.is_expired
        assert p1.first_name == 'John'
        assert p1.source_date == datetime(2010, 2, 3)
        assert p1.entry_date == datetime(2010, 2, 3)
        assert p1.expiry_date == datetime(2010, 2, 1)
        n1_1 = db.get(self.n1_1.key())
        assert n1_1.is_expired

    def test_wipe_contents(self):
        # Advance past the expiry date.
        set_utcnow_for_test(datetime(2010, 2, 3))
        self.p1.put_expiry_flags()

        # Try wiping the contents.
        self.p1.wipe_contents()

        p1 = db.get(self.p1.key())
        assert p1.is_expired
        assert p1.first_name == None
        assert p1.source_date == datetime(2010, 2, 3)
        assert p1.entry_date == datetime(2010, 2, 3)
        assert p1.expiry_date == datetime(2010, 2, 1)
        assert not db.get(self.n1_1.key())

if __name__ == '__main__':
    unittest.main()
<commit_msg>Add a missing test for encode_count_name (which was added in 78e7336de74a).<commit_after># Copyright 2010 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for model.py."""

from datetime import datetime
from google.appengine.ext import db
import unittest
import model
from utils import get_utcnow, set_utcnow_for_test

class ModelTests(unittest.TestCase):
    '''Test the loose odds and ends.'''

    def setUp(self):
        set_utcnow_for_test(datetime(2010, 1, 1))
        self.p1 = model.Person.create_original(
            'haiti',
            first_name='John',
            last_name='Smith',
            home_street='Washington St.',
            home_city='Los Angeles',
            home_state='California',
            home_postal_code='11111',
            home_neighborhood='Good Neighborhood',
            author_name='Alice Smith',
            author_phone='111-111-1111',
            author_email='alice.smith@gmail.com',
            source_url='https://www.source.com',
            source_date=datetime(2010, 1, 1),
            source_name='Source Name',
            entry_date=datetime(2010, 1, 1),
            expiry_date=datetime(2010, 2, 1),
            other='')
        self.p2 = model.Person.create_original(
            'haiti',
            first_name='Tzvika',
            last_name='Hartman',
            home_street='Herzl St.',
            home_city='Tel Aviv',
            home_state='Israel',
            entry_date=datetime(2010, 1, 1),
            expiry_date=datetime(2010, 3, 1),
            other='')
        self.key_p1 = db.put(self.p1)
        self.key_p2 = db.put(self.p2)

        self.n1_1 = model.Note.create_original(
            'haiti',
            person_record_id=self.p1.record_id,
            linked_person_record_id=self.p2.record_id,
            status=u'believed_missing',
            found=False,
            entry_date=get_utcnow(),
            source_date=datetime(2000, 1, 1))
        self.n1_2 = model.Note.create_original(
            'haiti',
            person_record_id=self.p1.record_id,
            found=True,
            entry_date=get_utcnow(),
            source_date=datetime(2000, 2, 2))
        self.key_n1_1 = db.put(self.n1_1)
        self.key_n1_2 = db.put(self.n1_2)

        # Update the Person entity according to the Note.
        self.p1.update_from_note(self.n1_1)
        self.p1.update_from_note(self.n1_2)
        db.put(self.p1)

        self.to_delete = [self.p1, self.p2, self.n1_1, self.n1_2]

    def tearDown(self):
        db.delete(self.to_delete)

    def test_person(self):
        assert self.p1.first_name == 'John'
        assert self.p1.photo_url == ''
        assert self.p1.is_clone() == False
        assert model.Person.get('haiti', self.p1.record_id).record_id == \
            self.p1.record_id
        assert model.Person.get('haiti', self.p2.record_id).record_id == \
            self.p2.record_id
        assert model.Person.get('haiti', self.p1.record_id).record_id != \
            self.p2.record_id

        # Testing prefix properties
        assert hasattr(self.p1, 'first_name_n_')
        assert hasattr(self.p1, 'home_street_n1_')
        assert hasattr(self.p1, 'home_postal_code_n2_')

        # Testing indexing properties
        assert self.p1._fields_to_index_properties == \
            ['first_name', 'last_name']
        assert self.p1._fields_to_index_by_prefix_properties == \
            ['first_name', 'last_name']

        # Test propagation of Note fields to Person.
        assert self.p1.latest_status == u'believed_missing'  # from first note
        assert self.p1.latest_status_source_date == datetime(2000, 1, 1)
        assert self.p1.latest_found == True  # from second note
        assert self.p1.latest_found_source_date == datetime(2000, 2, 2)

        # Adding a Note with only 'found' should not affect 'last_status'.
        n1_3 = model.Note.create_original(
            'haiti', person_record_id=self.p1.record_id, found=False,
            entry_date=get_utcnow(), source_date=datetime(2000, 3, 3))
        self.p1.update_from_note(n1_3)
        assert self.p1.latest_status == u'believed_missing'
        assert self.p1.latest_status_source_date == datetime(2000, 1, 1)
        assert self.p1.latest_found == False
        assert self.p1.latest_found_source_date == datetime(2000, 3, 3)

        # Adding a Note with only 'status' should not affect 'last_found'.
        n1_4 = model.Note.create_original(
            'haiti', person_record_id=self.p1.record_id,
            found=None, status=u'is_note_author',
            entry_date=get_utcnow(),
            source_date=datetime(2000, 4, 4))
        self.p1.update_from_note(n1_4)
        assert self.p1.latest_status == u'is_note_author'
        assert self.p1.latest_status_source_date == datetime(2000, 4, 4)
        assert self.p1.latest_found == False
        assert self.p1.latest_found_source_date == datetime(2000, 3, 3)

        # Adding an older Note should not affect either field.
        n1_5 = model.Note.create_original(
            'haiti', person_record_id=self.p1.record_id,
            found=True, status=u'believed_alive',
            entry_date=get_utcnow(),
            source_date=datetime(2000, 1, 2))
        self.p1.update_from_note(n1_5)
        assert self.p1.latest_status == u'is_note_author'
        assert self.p1.latest_status_source_date == datetime(2000, 4, 4)
        assert self.p1.latest_found == False
        assert self.p1.latest_found_source_date == datetime(2000, 3, 3)

        # Adding a Note with a date in between should affect only one field.
        n1_6 = model.Note.create_original(
            'haiti', person_record_id=self.p1.record_id,
            found=True, status=u'believed_alive',
            entry_date=get_utcnow(),
            source_date=datetime(2000, 3, 4))
        self.p1.update_from_note(n1_6)
        assert self.p1.latest_status == u'is_note_author'
        assert self.p1.latest_status_source_date == datetime(2000, 4, 4)
        assert self.p1.latest_found == True
        assert self.p1.latest_found_source_date == datetime(2000, 3, 4)

    def test_note(self):
        assert self.n1_1.is_clone() == False
        notes = self.p1.get_notes()
        assert notes[0].record_id == self.n1_1.record_id
        assert notes[1].record_id == self.n1_2.record_id
        assert self.p1.get_linked_persons()[0].record_id == self.p2.record_id
        assert self.p2.get_linked_persons() == []

        assert model.Note.get('haiti', self.n1_1.record_id).record_id == \
            self.n1_1.record_id
        assert model.Note.get('haiti', self.n1_2.record_id).record_id == \
            self.n1_2.record_id

    def test_subscription(self):
        sd = 'haiti'
        email1 = 'one@example.com'
        email2 = 'two@example.com'
        s1 = model.Subscription.create(sd, self.p1.record_id, email1, 'fr')
        s2 = model.Subscription.create(sd, self.p1.record_id, email2, 'en')
        key_s1 = db.put(s1)
        key_s2 = db.put(s2)

        assert model.Subscription.get(sd, self.p1.record_id, email1) is not None
        assert model.Subscription.get(sd, self.p1.record_id, email2) is not None
        assert model.Subscription.get(sd, self.p2.record_id, email1) is None
        assert model.Subscription.get(sd, self.p2.record_id, email2) is None
        assert len(self.p1.get_subscriptions()) == 2
        assert len(self.p2.get_subscriptions()) == 0

        s3 = model.Subscription.create(sd, self.p1.record_id, email2, 'ar')
        key_s3 = db.put(s3)
        assert len(self.p1.get_subscriptions()) == 2
        assert model.Subscription.get(
            sd, self.p1.record_id, email2).language == 'ar'
        db.delete([key_s1, key_s2, key_s3])

    def test_past_due(self):
        """Make sure Person records are detected as past due correctly."""
        def assert_past_due_count(expected):
            assert len(list(model.Person.past_due_records())) == expected
        assert_past_due_count(0)
        set_utcnow_for_test(datetime(2010, 2, 15))
        assert_past_due_count(1)
        set_utcnow_for_test(datetime(2010, 3, 15))
        assert_past_due_count(2)

    def test_put_expiry_flags(self):
        # Try put_expiry_flags when the record has not expired yet.
        self.p1.put_expiry_flags()

        # Both entities should be unexpired.
        p1 = db.get(self.p1.key())
        assert not p1.is_expired
        assert p1.first_name == 'John'
        n1_1 = db.get(self.n1_1.key())
        assert not n1_1.is_expired

        # Advance past the expiry date and try again.
        set_utcnow_for_test(datetime(2010, 2, 3))
        p1.put_expiry_flags()

        # Both entities should be expired.
        p1 = db.get(self.p1.key())
        assert p1.is_expired
        assert p1.first_name == 'John'
        assert p1.source_date == datetime(2010, 2, 3)
        assert p1.entry_date == datetime(2010, 2, 3)
        assert p1.expiry_date == datetime(2010, 2, 1)
        n1_1 = db.get(self.n1_1.key())
        assert n1_1.is_expired

    def test_wipe_contents(self):
        # Advance past the expiry date.
        set_utcnow_for_test(datetime(2010, 2, 3))
        self.p1.put_expiry_flags()

        # Try wiping the contents.
        self.p1.wipe_contents()

        p1 = db.get(self.p1.key())
        assert p1.is_expired
        assert p1.first_name == None
        assert p1.source_date == datetime(2010, 2, 3)
        assert p1.entry_date == datetime(2010, 2, 3)
        assert p1.expiry_date == datetime(2010, 2, 1)
        assert not db.get(self.n1_1.key())

    def test_count_name_chars(self):
        """Regression test for arbitrary characters in a count_name."""
        counter = model.Counter.get_unfinished_or_create('haiti', 'person')
        counter.put()
        self.to_delete.append(counter)

        counter.increment(u'arbitrary \xef characters \u5e73 here')
        counter.put()  # without encode_count_name, this threw an exception

if __name__ == '__main__':
    unittest.main()
<|endoftext|>