<commit_before>"""
Test course_catalog serializers
"""
import pytest

from course_catalog import factories
from course_catalog.constants import OfferedBy, ListType
from course_catalog.factories import (
    CourseFactory,
    CourseTopicFactory,
    CoursePriceFactory,
    CourseInstructorFactory,
    BootcampFactory,
    ProgramFactory,
    UserListFactory,
    LearningResourceRunFactory,
    ProgramItemCourseFactory,
    UserListCourseFactory,
)
from course_catalog.models import FavoriteItem, UserListItem
from course_catalog.serializers import (
    CourseSerializer,
    BootcampSerializer,
    FavoriteItemSerializer,
    UserListSerializer,
    ProgramSerializer,
    LearningResourceRunSerializer,
    UserListItemSerializer,
    SimpleUserListSerializer,
    SimpleUserListItemSerializer,
)
from open_discussions.factories import UserFactory

pytestmark = pytest.mark.django_db

@pytest.mark.parametrize(
    "offered_by",
    [
        OfferedBy.mitx.value,
        OfferedBy.xpro.value,
        OfferedBy.micromasters.value,
        OfferedBy.ocw.value,
        OfferedBy.bootcamps.value,
    ],
)
def test_serialize_course_related_models(offered_by):
    """
    Verify that a serialized course contains attributes for related objects
    """
    course = CourseFactory(
        offered_by=offered_by, topics=CourseTopicFactory.create_batch(3)
    )
    serializer = CourseSerializer(course)
    assert len(serializer.data["topics"]) == 3
    assert "name" in serializer.data["topics"][0].keys()
    assert len(serializer.data["runs"]) == 3

def test_serialize_courserun_related_models():
    """
    Verify that a serialized course run contains attributes for related objects
    """
    courserun = LearningResourceRunFactory(
        prices=CoursePriceFactory.create_batch(2),
        instructors=CourseInstructorFactory.create_batch(2),
    )
    serializer = LearningResourceRunSerializer(courserun)
    assert len(serializer.data["prices"]) == 2
    for attr in ("mode", "price"):
        assert attr in serializer.data["prices"][0].keys()
    assert len(serializer.data["instructors"]) == 2
    for attr in ("first_name", "last_name", "full_name"):
        assert attr in serializer.data["instructors"][0].keys()

def test_serialize_bootcamp_related_models():
    """
    Verify that a serialized bootcamp contains attributes for related objects
    """
    bootcamp = BootcampFactory.create(topics=CourseTopicFactory.create_batch(3))
    serializer = BootcampSerializer(bootcamp)
    assert len(serializer.data["topics"]) == 3
    assert "name" in serializer.data["topics"][0].keys()
    assert len(serializer.data["runs"]) == 3

def test_serialize_program_related_models():
    """
    Verify that a serialized program contains attributes for related objects
    """
    program = ProgramFactory.create(topics=CourseTopicFactory.create_batch(3))
    ProgramItemCourseFactory.create_batch(4, program=program)
    serializer = ProgramSerializer(program)
    assert len(serializer.data["topics"]) == 3
    assert len(serializer.data["runs"]) == 1
    assert len(serializer.data["items"]) == 4
    assert "content_data" in serializer.data["items"][0].keys()

@pytest.mark.parametrize(
    "factory,valid_type",
    [
        ["CourseFactory", True],
        ["ProgramFactory", True],
        ["BootcampFactory", True],
        ["UserListFactory", True],
        ["VideoFactory", True],
        ["CourseTopicFactory", False],
    ],
)
def test_generic_foreign_key_serializer_classes(factory, valid_type):
    """
    Test that generic foreign key serializer properly accepts expected classes and rejects others
    """
    userlist = UserListFactory.create()
    kwargs = (
        {"user_list": UserListFactory.create()}
        if factory == "UserListCourseFactory"
        else {}
    )
    obj = getattr(factories, factory).create(**kwargs)
    list_item = UserListItem(user_list=userlist, item=obj)
    serializer = UserListItemSerializer(list_item)
    if valid_type:
        assert serializer.data.get("content_data").get("id") == obj.id
    else:
        with pytest.raises(Exception):
            assert serializer.data.get("content_data").get("id") == obj.id

@pytest.mark.parametrize(
    "list_type,valid",
    [
        [ListType.LIST.value, True],
        [ListType.LEARNING_PATH.value, True],
        ["bad_type", False],
        [None, False],
    ],
)
def test_userlist_serializer_validation(list_type, valid):
    """
    Test that the UserListSerializer validates list_type correctly
    """
    data = {"title": "My List", "list_type": list_type}
    serializer = UserListSerializer(data=data)
    assert serializer.is_valid() == valid

@pytest.mark.parametrize(
    "list_type,valid",
    [
        [ListType.LIST.value, True],
        [ListType.LEARNING_PATH.value, True],
        ["bad_type", False],
        [None, False],
    ],
)
def test_simple_userlist_serializer_validation(list_type, valid):
    """
    Test that the SimpleUserListSerializer validates list_type correctly
    """
    data = {"title": "My List", "list_type": list_type}
    serializer = SimpleUserListSerializer(data=data)
    assert serializer.is_valid() == valid

@pytest.mark.parametrize("object_exists", [True, False])
@pytest.mark.parametrize(
    "content_type,factory,valid_type",
    [
        ["course", "CourseFactory", True],
        ["program", "ProgramFactory", True],
        ["bootcamp", "BootcampFactory", True],
        ["video", "VideoFactory", True],
        ["userlist", "UserListFactory", True],
        ["user list item", "UserListCourseFactory", False],
        [None, "CourseFactory", False],
    ],
)
def test_userlistitem_serializer_validation(
    content_type, factory, valid_type, object_exists
):
    """
    Test that the UserListItemSerializer validates content_type and object correctly
    """
    userlist = UserListFactory.create()
    kwargs = (
        {"user_list": UserListFactory.create()}
        if factory == "UserListCourseFactory"
        else {}
    )
    # pylint:disable=redefined-builtin
    object_id = (
        getattr(factories, factory).create(**kwargs).id if object_exists else 1_001_001
    )
    data = {
        "content_type": content_type,
        "object_id": object_id,
        "user_list": userlist.id,
    }
    serializer = UserListItemSerializer(data=data)
    assert serializer.is_valid() == (valid_type and object_exists)

@pytest.mark.parametrize("is_null", [True, False])
def test_simpleuserlistitem_serializer_content_data(is_null):
    """
    Test that the SimpleUserListItemSerializer includes content_data with an image_src
    """
    course = CourseFactory.create()
    userlistitem = UserListCourseFactory.create(
        content_object=course, user_list=UserListFactory.create()
    )
    expected_content_data = {"image_src": course.image_src}
    if is_null:
        course.delete()
        expected_content_data = {"image_src": None}
    item_serializer = SimpleUserListItemSerializer(instance=userlistitem)
    assert item_serializer.data["content_data"] == expected_content_data
    list_serializer = SimpleUserListSerializer(instance=userlistitem.user_list)
    assert list_serializer.data["items"][0]["content_data"] == expected_content_data

def test_favorites_serializer():
    """
    Test that the favorite serializer generic foreign key works and also rejects unexpected classes
    """
    user = UserFactory.create()
    course = CourseFactory.create()
    bootcamp = BootcampFactory.create()
    user_list = UserListFactory.create(author=user)
    program = ProgramFactory.create()
    course_topic = CourseTopicFactory.create()

    favorite_item = FavoriteItem(user=user, item=course)
    serializer = FavoriteItemSerializer(favorite_item)
    assert serializer.data.get("content_data") == CourseSerializer(course).data

    favorite_item = FavoriteItem(user=user, item=bootcamp)
    serializer = FavoriteItemSerializer(favorite_item)
    assert serializer.data.get("content_data") == BootcampSerializer(bootcamp).data

    favorite_item = FavoriteItem(user=user, item=user_list)
    serializer = FavoriteItemSerializer(favorite_item)
    assert serializer.data.get("content_data") == UserListSerializer(user_list).data

    favorite_item = FavoriteItem(user=user, item=program)
    serializer = FavoriteItemSerializer(favorite_item)
    assert serializer.data.get("content_data") == ProgramSerializer(program).data

    favorite_item = FavoriteItem(user=user, item=course_topic)
    serializer = FavoriteItemSerializer(favorite_item)
    with pytest.raises(Exception):
        assert serializer.data.get("content_data").get("id") == course_topic.id
<commit_msg>Fix test (#2400)<commit_after>"""
Test course_catalog serializers
"""
import pytest

from course_catalog import factories
from course_catalog.constants import OfferedBy, ListType
from course_catalog.factories import (
    CourseFactory,
    CourseTopicFactory,
    CoursePriceFactory,
    CourseInstructorFactory,
    BootcampFactory,
    ProgramFactory,
    UserListFactory,
    LearningResourceRunFactory,
    ProgramItemCourseFactory,
    UserListCourseFactory,
)
from course_catalog.models import FavoriteItem, UserListItem
from course_catalog.serializers import (
    CourseSerializer,
    BootcampSerializer,
    FavoriteItemSerializer,
    UserListSerializer,
    ProgramSerializer,
    LearningResourceRunSerializer,
    UserListItemSerializer,
    SimpleUserListSerializer,
    SimpleUserListItemSerializer,
)
from open_discussions.factories import UserFactory

pytestmark = pytest.mark.django_db

@pytest.mark.parametrize(
    "offered_by",
    [
        OfferedBy.mitx.value,
        OfferedBy.xpro.value,
        OfferedBy.micromasters.value,
        OfferedBy.ocw.value,
        OfferedBy.bootcamps.value,
    ],
)
def test_serialize_course_related_models(offered_by):
    """
    Verify that a serialized course contains attributes for related objects
    """
    course = CourseFactory(
        offered_by=offered_by, topics=CourseTopicFactory.create_batch(3)
    )
    serializer = CourseSerializer(course)
    assert len(serializer.data["topics"]) == 3
    assert "name" in serializer.data["topics"][0].keys()
    assert len(serializer.data["runs"]) == 3

def test_serialize_courserun_related_models():
    """
    Verify that a serialized course run contains attributes for related objects
    """
    courserun = LearningResourceRunFactory(
        prices=CoursePriceFactory.create_batch(2),
        instructors=CourseInstructorFactory.create_batch(2),
    )
    serializer = LearningResourceRunSerializer(courserun)
    assert len(serializer.data["prices"]) == 2
    for attr in ("mode", "price"):
        assert attr in serializer.data["prices"][0].keys()
    assert len(serializer.data["instructors"]) == 2
    for attr in ("first_name", "last_name", "full_name"):
        assert attr in serializer.data["instructors"][0].keys()

def test_serialize_bootcamp_related_models():
    """
    Verify that a serialized bootcamp contains attributes for related objects
    """
    bootcamp = BootcampFactory.create(topics=CourseTopicFactory.create_batch(3))
    serializer = BootcampSerializer(bootcamp)
    assert len(serializer.data["topics"]) == 3
    assert "name" in serializer.data["topics"][0].keys()
    assert len(serializer.data["runs"]) == 3

def test_serialize_program_related_models():
    """
    Verify that a serialized program contains attributes for related objects
    """
    program = ProgramFactory.create(topics=CourseTopicFactory.create_batch(3))
    ProgramItemCourseFactory.create_batch(4, program=program)
    serializer = ProgramSerializer(program)
    assert len(serializer.data["topics"]) == 3
    assert len(serializer.data["runs"]) == 1
    assert len(serializer.data["items"]) == 4
    assert "content_data" in serializer.data["items"][0].keys()

@pytest.mark.parametrize(
    "factory,valid_type",
    [
        ["CourseFactory", True],
        ["ProgramFactory", True],
        ["BootcampFactory", True],
        ["UserListFactory", True],
        ["VideoFactory", True],
        ["CourseTopicFactory", False],
    ],
)
def test_generic_foreign_key_serializer_classes(factory, valid_type):
    """
    Test that generic foreign key serializer properly accepts expected classes and rejects others
    """
    userlist = UserListFactory.create()
    kwargs = (
        {"user_list": UserListFactory.create()}
        if factory == "UserListCourseFactory"
        else {}
    )
    obj = getattr(factories, factory).create(**kwargs)
    list_item = UserListItem(user_list=userlist, item=obj)
    serializer = UserListItemSerializer(list_item)
    if valid_type:
        assert serializer.data.get("content_data").get("id") == obj.id
    else:
        with pytest.raises(Exception):
            assert serializer.data.get("content_data").get("id") == obj.id

@pytest.mark.parametrize(
    "list_type,valid",
    [
        [ListType.LIST.value, True],
        [ListType.LEARNING_PATH.value, True],
        ["bad_type", False],
        [None, False],
    ],
)
def test_userlist_serializer_validation(list_type, valid):
    """
    Test that the UserListSerializer validates list_type correctly
    """
    data = {"title": "My List", "list_type": list_type}
    serializer = UserListSerializer(data=data)
    assert serializer.is_valid() == valid

@pytest.mark.parametrize(
    "list_type,valid",
    [
        [ListType.LIST.value, True],
        [ListType.LEARNING_PATH.value, True],
        ["bad_type", False],
        [None, False],
    ],
)
def test_simple_userlist_serializer_validation(list_type, valid):
    """
    Test that the SimpleUserListSerializer validates list_type correctly
    """
    data = {"title": "My List", "list_type": list_type}
    serializer = SimpleUserListSerializer(data=data)
    assert serializer.is_valid() == valid

@pytest.mark.parametrize("object_exists", [True, False])
@pytest.mark.parametrize(
    "content_type,factory,valid_type",
    [
        ["course", "CourseFactory", True],
        ["program", "ProgramFactory", True],
        ["bootcamp", "BootcampFactory", True],
        ["video", "VideoFactory", True],
        ["userlist", "UserListFactory", True],
        ["user list item", "UserListCourseFactory", False],
        [None, "CourseFactory", False],
    ],
)
def test_userlistitem_serializer_validation(
    content_type, factory, valid_type, object_exists
):
    """
    Test that the UserListItemSerializer validates content_type and object correctly
    """
    userlist = UserListFactory.create()
    kwargs = (
        {"user_list": UserListFactory.create()}
        if factory == "UserListCourseFactory"
        else {}
    )
    # pylint:disable=redefined-builtin
    object_id = (
        getattr(factories, factory).create(**kwargs).id if object_exists else 1_001_001
    )
    data = {
        "content_type": content_type,
        "object_id": object_id,
        "user_list": userlist.id,
    }
    serializer = UserListItemSerializer(data=data)
    assert serializer.is_valid() == (valid_type and object_exists)

@pytest.mark.parametrize("is_null", [True, False])
def test_simpleuserlistitem_serializer_content_data(is_null):
    """
    Test that the SimpleUserListItemSerializer includes content_data with an image_src
    """
    course = CourseFactory.create()
    userlistitem = UserListCourseFactory.create(
        content_object=course, user_list=UserListFactory.create()
    )
    expected_content_data = {"image_src": course.image_src}
    if is_null:
        course.image_src = None
        course.save()
        expected_content_data = {"image_src": None}
    item_serializer = SimpleUserListItemSerializer(instance=userlistitem)
    assert item_serializer.data["content_data"] == expected_content_data
    list_serializer = SimpleUserListSerializer(instance=userlistitem.user_list)
    assert list_serializer.data["items"][0]["content_data"] == expected_content_data

def test_favorites_serializer():
    """
    Test that the favorite serializer generic foreign key works and also rejects unexpected classes
    """
    user = UserFactory.create()
    course = CourseFactory.create()
    bootcamp = BootcampFactory.create()
    user_list = UserListFactory.create(author=user)
    program = ProgramFactory.create()
    course_topic = CourseTopicFactory.create()

    favorite_item = FavoriteItem(user=user, item=course)
    serializer = FavoriteItemSerializer(favorite_item)
    assert serializer.data.get("content_data") == CourseSerializer(course).data

    favorite_item = FavoriteItem(user=user, item=bootcamp)
    serializer = FavoriteItemSerializer(favorite_item)
    assert serializer.data.get("content_data") == BootcampSerializer(bootcamp).data

    favorite_item = FavoriteItem(user=user, item=user_list)
    serializer = FavoriteItemSerializer(favorite_item)
    assert serializer.data.get("content_data") == UserListSerializer(user_list).data

    favorite_item = FavoriteItem(user=user, item=program)
    serializer = FavoriteItemSerializer(favorite_item)
    assert serializer.data.get("content_data") == ProgramSerializer(program).data

    favorite_item = FavoriteItem(user=user, item=course_topic)
    serializer = FavoriteItemSerializer(favorite_item)
    with pytest.raises(Exception):
        assert serializer.data.get("content_data").get("id") == course_topic.id
<|endoftext|>


<commit_before>#!/bin/python2
# -*- coding: utf-8 -*-

import datetime
import glob
import urllib2
import os
import struct
import sys
import traceback
import util

# Delete the index to do away with old data that might remain otherwise
# Since no commit is executed here we avoid the empty index problem
def DeleteSolrIndex():
    try:
        request = urllib2.Request(
            "http://localhost:8080/solr/biblio/update?stream.body=%3Cdelete%3E%3Cquery%3E*:*%3C/query%3E%3C/delete%3E")
        response = urllib2.urlopen(request, timeout=60)
    except:
        util.SendEmail("MARC-21 Pipeline", "Failed to clear the SOLR index!", priority=1)
        sys.exit(-1)

def OptimizeSolrIndex():
    try:
        request = urllib2.Request(
            "http://localhost:8080/solr/biblio/update?optimize=true")
        response = urllib2.urlopen(request, timeout=1200)
    except:
        util.SendEmail("MARC-21 Pipeline", "Failed to optimize the SOLR index!", priority=1)
        sys.exit(-1)

solrmarc_log_summary = "/tmp/solrmarc_log.summary"

def ImportIntoVuFind(pattern, log_file_name):
    args = [ sorted(glob.glob(pattern), reverse=True)[0] ]
    if len(args) != 1:
        util.Error("\"" + pattern + "\" matched " + str(len(args))
                   + " files! (Should have matched exactly 1 file!)")
    DeleteSolrIndex()
    util.ExecOrDie("/usr/local/vufind/import-marc.sh", args, log_file_name)
    util.ExecOrDie("/usr/local/bin/summarize_logs", ["/usr/local/vufind/import/solrmarc.log", solrmarc_log_summary],
                   log_file_name)
    OptimizeSolrIndex()
    util.ExecOrDie("/usr/local/vufind/index-alphabetic-browse.sh", None, log_file_name)

def StartPipeline(pipeline_script_name, marc_title, conf):
    log_file_name = util.MakeLogFileName(pipeline_script_name, util.GetLogDirectory())
    util.ExecOrDie(pipeline_script_name, [ marc_title ], log_file_name)

    deletion_list_glob = "LOEPPN-[0-9][0-9][0-9][0-9][0-9][0-9]"
    most_recent_deletion_list = util.getMostRecentFileMatchingGlob(deletion_list_glob)
    if not most_recent_deletion_list:
        util.SendEmail("MARC-21 Pipeline", "Did not find any files matching \"" + deletion_list_glob + "\".",
                       priority=5)
    else:
        delete_solr_ids_args = [ util.default_email_recipient, most_recent_deletion_list ]
        util.ExecOrDie("/usr/local/bin/delete_solr_ids.sh", delete_solr_ids_args, log_file_name)

    log_file_name = util.MakeLogFileName("import_into_vufind", util.GetLogDirectory())
    ImportIntoVuFind(conf.get("FileNames", "title_marc_data"), log_file_name)

# Returns True if we have no timestamp file or if link_filename's creation time is more recent than
# the time found in the timestamp file.
def FoundNewBSZDataFile(link_filename):
    try:
        statinfo = os.stat(link_filename)
        file_creation_time = statinfo.st_ctime
    except OSError as e:
        util.Error("in FoundNewBSZDataFile: Symlink \"" + link_filename + "\" is missing or dangling!")
    old_timestamp = util.ReadTimestamp()
    return old_timestamp < file_creation_time

def Main():
    util.default_email_sender = "initiate_marc_pipeline@ub.uni-tuebingen.de"
    if len(sys.argv) != 3:
         util.SendEmail("MARC-21 Pipeline Kick-Off (Failure)",
                        "This script needs to be called with two arguments,\n"
                        + "the default email recipient and the name of the MARC-21\n"
                        + "pipeline script to be executed.\n", priority=1)
         sys.exit(-1)

    util.default_email_recipient = sys.argv[1]
    pipeline_script_name = sys.argv[2]
    if not os.access(pipeline_script_name, os.X_OK):
         util.SendEmail("MARC-21 Pipeline Kick-Off (Failure)", "Pipeline script not found or not executable: \""
                        + pipeline_script_name + "\"\n", priority=1)
         sys.exit(-1)
    conf = util.LoadConfigFile()
    link_name = conf.get("Misc", "link_name")
    if FoundNewBSZDataFile(link_name):
        bsz_data = util.ResolveSymlink(link_name)
        if not bsz_data.endswith(".tar.gz"):
            util.Error("BSZ data file must end in .tar.gz!")
        file_name_list = util.ExtractAndRenameBSZFiles(bsz_data)

        StartPipeline(pipeline_script_name, file_name_list[0], conf)
        util.SendEmail("MARC-21 Pipeline", "Pipeline completed successfully.", priority=5)
        util.WriteTimestamp()
    else:
        util.SendEmail("MARC-21 Pipeline Kick-Off", "No new data was found.", priority=5)

try:
    Main()
except Exception as e:
    error_msg =  "An unexpected error occurred: " + str(e) + "\n\n" + traceback.format_exc(20)
    util.SendEmail("MARC-21 Pipeline Kick-Off", error_msg, priority=1, attachment=solrmarc_log_summary)
    sys.stderr.write(error_msg)
<commit_msg>Also rotate the solrmarc.log file.<commit_after>#!/bin/python2
# -*- coding: utf-8 -*-

import datetime
import glob
import urllib2
import os
import struct
import sys
import traceback
import util

# Delete the index to do away with old data that might remain otherwise
# Since no commit is executed here we avoid the empty index problem
def DeleteSolrIndex():
    try:
        request = urllib2.Request(
            "http://localhost:8080/solr/biblio/update?stream.body=%3Cdelete%3E%3Cquery%3E*:*%3C/query%3E%3C/delete%3E")
        response = urllib2.urlopen(request, timeout=60)
    except:
        util.SendEmail("MARC-21 Pipeline", "Failed to clear the SOLR index!", priority=1)
        sys.exit(-1)

def OptimizeSolrIndex():
    try:
        request = urllib2.Request(
            "http://localhost:8080/solr/biblio/update?optimize=true")
        response = urllib2.urlopen(request, timeout=1200)
    except:
        util.SendEmail("MARC-21 Pipeline", "Failed to optimize the SOLR index!", priority=1)
        sys.exit(-1)

solrmarc_log_summary = "/tmp/solrmarc_log.summary"

def ImportIntoVuFind(pattern, log_file_name):
    args = [ sorted(glob.glob(pattern), reverse=True)[0] ]
    if len(args) != 1:
        util.Error("\"" + pattern + "\" matched " + str(len(args))
                   + " files! (Should have matched exactly 1 file!)")
    DeleteSolrIndex()
    util.ExecOrDie("/usr/local/vufind/import-marc.sh", args, log_file_name)
    util.ExecOrDie("/usr/local/bin/summarize_logs", ["/usr/local/vufind/import/solrmarc.log", solrmarc_log_summary],
                   log_file_name)
    util.ExecOrDie("/usr/local/bin/log_rotate ", ["/usr/local/vufind/import/", "solrmarc\\.log"], log_file_name)
    OptimizeSolrIndex()
    util.ExecOrDie("/usr/local/vufind/index-alphabetic-browse.sh", None, log_file_name)

def StartPipeline(pipeline_script_name, marc_title, conf):
    log_file_name = util.MakeLogFileName(pipeline_script_name, util.GetLogDirectory())
    util.ExecOrDie(pipeline_script_name, [ marc_title ], log_file_name)

    deletion_list_glob = "LOEPPN-[0-9][0-9][0-9][0-9][0-9][0-9]"
    most_recent_deletion_list = util.getMostRecentFileMatchingGlob(deletion_list_glob)
    if not most_recent_deletion_list:
        util.SendEmail("MARC-21 Pipeline", "Did not find any files matching \"" + deletion_list_glob + "\".",
                       priority=5)
    else:
        delete_solr_ids_args = [ util.default_email_recipient, most_recent_deletion_list ]
        util.ExecOrDie("/usr/local/bin/delete_solr_ids.sh", delete_solr_ids_args, log_file_name)

    log_file_name = util.MakeLogFileName("import_into_vufind", util.GetLogDirectory())
    ImportIntoVuFind(conf.get("FileNames", "title_marc_data"), log_file_name)

# Returns True if we have no timestamp file or if link_filename's creation time is more recent than
# the time found in the timestamp file.
def FoundNewBSZDataFile(link_filename):
    try:
        statinfo = os.stat(link_filename)
        file_creation_time = statinfo.st_ctime
    except OSError as e:
        util.Error("in FoundNewBSZDataFile: Symlink \"" + link_filename + "\" is missing or dangling!")
    old_timestamp = util.ReadTimestamp()
    return old_timestamp < file_creation_time

def Main():
    util.default_email_sender = "initiate_marc_pipeline@ub.uni-tuebingen.de"
    if len(sys.argv) != 3:
         util.SendEmail("MARC-21 Pipeline Kick-Off (Failure)",
                        "This script needs to be called with two arguments,\n"
                        + "the default email recipient and the name of the MARC-21\n"
                        + "pipeline script to be executed.\n", priority=1)
         sys.exit(-1)

    util.default_email_recipient = sys.argv[1]
    pipeline_script_name = sys.argv[2]
    if not os.access(pipeline_script_name, os.X_OK):
         util.SendEmail("MARC-21 Pipeline Kick-Off (Failure)", "Pipeline script not found or not executable: \""
                        + pipeline_script_name + "\"\n", priority=1)
         sys.exit(-1)
    conf = util.LoadConfigFile()
    link_name = conf.get("Misc", "link_name")
    if FoundNewBSZDataFile(link_name):
        bsz_data = util.ResolveSymlink(link_name)
        if not bsz_data.endswith(".tar.gz"):
            util.Error("BSZ data file must end in .tar.gz!")
        file_name_list = util.ExtractAndRenameBSZFiles(bsz_data)

        StartPipeline(pipeline_script_name, file_name_list[0], conf)
        util.SendEmail("MARC-21 Pipeline", "Pipeline completed successfully.", priority=5)
        util.WriteTimestamp()
    else:
        util.SendEmail("MARC-21 Pipeline Kick-Off", "No new data was found.", priority=5)

try:
    Main()
except Exception as e:
    error_msg =  "An unexpected error occurred: " + str(e) + "\n\n" + traceback.format_exc(20)
    util.SendEmail("MARC-21 Pipeline Kick-Off", error_msg, priority=1, attachment=solrmarc_log_summary)
    sys.stderr.write(error_msg)
<|endoftext|>


<commit_before>import sys
import operator
import csb.test as test
import csb.pyutils as utils

@test.unit
class TestShell(test.Case):

    def setUp(self):

        super(TestShell, self).setUp()
        self.output = utils.Shell.run('echo TeST')

    def testSTDOUT(self):
        self.assertEquals(self.output.stdout.strip(), 'TeST')

    def testSTDERR(self):
        self.assertEquals(self.output.stderr, '')

    def testExitCode(self):
        self.assertEquals(self.output.code, 0)

@test.unit
class TestDeepCopy(test.Case):

    def runTest(self):

        rec = sys.getrecursionlimit()
        obj = ['X']
        copy = utils.deepcopy(obj, recursion=(rec + 1))

        self.assertEquals(obj, copy)
        self.assertNotEquals(id(obj), id(copy))

@test.unit
class TestEnum(test.Case):

    def setUp(self):

        super(TestEnum, self).setUp()
        self.enum = utils.enum('A', 'B', C=66)

    def testConstructor(self):
        self.assertRaises(AttributeError, utils.enum, '3')
        self.assertRaises(ValueError, utils.enum, 'A', 'A')
        self.assertRaises(ValueError, utils.enum, A=1, B=1)

    def testComparison(self):
        self.assertEqual(self.enum.A, 0)
        self.assertEqual(self.enum.C, 66)
        self.assertFalse(self.enum.C is 66)
        self.assertFalse(isinstance(self.enum.A, int))

    def testStr(self):
        self.assertEqual(str(self.enum.A), '0')
        self.assertEqual(int(self.enum.A), 0)
        self.assertEqual(self.enum.A.value, 0)

    def testRepr(self):
        self.assertEqual(repr(self.enum.A), 'A')
        self.assertEqual(self.enum.A.name, 'A')

    def testMembers(self):
        result = utils.Enum.members(self.enum)
        members = [self.enum.A, self.enum.B, self.enum.C]
        self.assertEquals(set(result), set(members))

    def testNames(self):
        result = utils.Enum.names(self.enum)
        names = map(repr, [self.enum.A, self.enum.B, self.enum.C])
        names2 = map(operator.attrgetter('name'), [self.enum.A, self.enum.B, self.enum.C])
        self.assertEquals(set(result), set(names))
        self.assertEquals(set(result), set(names2))

    def testValues(self):
        result = utils.Enum.values(self.enum)
        values = map(int, [self.enum.A, self.enum.B, self.enum.C])
        values2 = map(operator.attrgetter('value'), [self.enum.A, self.enum.B, self.enum.C])
        self.assertEquals(set(result), set(values))
        self.assertEquals(set(result), set(values2))

    def testParseValue(self):
        item = utils.Enum.parse(self.enum, 66)
        self.assertTrue(item is self.enum.C)
        self.assertRaises(utils.EnumValueError, utils.Enum.parse, self.enum, 111)
        self.assertRaises(utils.EnumValueError, utils.Enum.parse, self.enum, '0')

    def testParseName(self):
        item = utils.Enum.parsename(self.enum, 'A')
        self.assertTrue(item is self.enum.A)
        self.assertRaises(utils.EnumMemberError, utils.Enum.parsename, self.enum, 'XXX')

    def testIsMember(self):
        self.assertTrue(utils.Enum.ismember(self.enum.A, self.enum))

@test.unit
class TestDictionaryContainer(test.Case):

    def setUp(self):

        super(TestDictionaryContainer, self).setUp()

        self.dict = utils.OrderedDict({'A': 1, 'B': 2})
        self.keys = ('A', 'B', 'C', 'D', 'Z')
        self.new = utils.DictionaryContainer
        self.test = utils.DictionaryContainer(items=self.dict, restrict=self.keys)

    def testConstructor(self):
        new = utils.DictionaryContainer(items=self.dict, restrict=self.keys)
        self.assertEqual(dict(new), dict(self.dict))
        self.assertEqual(list(new), list(self.dict))    # True if the dictionary is ordered
        self.assertEqual(new.length, len(self.dict))
        self.assertRaises(utils.InvalidKeyError, self.new, items={'X': 1}, restrict=self.keys)

    def testAppend(self):
        self.test.append('C', 1)
        self.assertTrue('C' in self.test)
        self.assertRaises(utils.DuplicateKeyError, self.test.append, key='C', item=1)
        self.assertRaises(utils.InvalidKeyError, self.test.append, key='X', item=1)

    def testRemove(self):
        self.assertTrue('A' in self.test)
        self.test._remove('A')
        self.assertFalse('A' in self.test)

    def testGetitem(self):
        self.assertEqual(self.test['B'], self.dict['B'])
        self.assertRaises(utils.ItemNotFoundError, lambda k: self.test[k], 66)

    def testSet(self):
        new = utils.DictionaryContainer(items=self.dict, restrict=self.keys)
        new._set({'Z': 6})
        self.assertTrue('Z' in new)
        self.assertFalse('A' in new)

    def testUpdate(self):
        new = utils.DictionaryContainer(items=self.dict, restrict=self.keys)

        new._update({'A': 7})
        self.assertTrue('A' in new)
        self.assertEquals(new['A'], 7)

        self.assertRaises(utils.ItemNotFoundError, new._update, {'Z': 0})

@test.unit
class TestCollectionContainer(test.Case):

    def setUp(self):

        super(TestCollectionContainer, self).setUp()

        self.items = [11, 22, 33]
        self.start = 5
        self.new = utils.CollectionContainer
        self.test = utils.CollectionContainer(items=self.items, type=int, start_index=self.start)

    def testConstructor(self):
        new = utils.CollectionContainer(items=self.items, type=int, start_index=self.start)
        self.assertEqual(list(new), list(self.items))
        self.assertEqual(new.length, len(self.items))
        self.assertRaises(TypeError, self.new, items=['S', 1.2], type=int)

    def testGetitem(self):
        self.assertEqual(self.test[self.start], self.items[0])
        self.assertEqual(self.test[self.test.start_index], self.items[0])
        self.assertEqual(self.test[self.test.last_index], self.items[-1])
        self.assertEqual(self.test[-1], self.items[-1])
        self.assertEqual(self.test[-1 : -2], self.items[-1 : -2])

        def get(i):
            return self.test[i]

        self.assertRaises(utils.CollectionIndexError, get, self.start + len(self.items))  # i = end + 1
        if self.start > 0:
            self.assertRaises(utils.CollectionIndexError, get, self.start - 1)            # i = start - 1, i >= 0

    def testIndices(self):
        new = self.new(items=self.items, type=int, start_index=self.start)
        self.assertEqual(new.start_index, self.start)
        self.assertEqual(new.last_index, self.start + len(self.items) - 1)
        self.assertEqual(new.length, len(self.items))

    def testAppend(self):
        rank = self.test.append(44)
        self.assertTrue(44 in self.test)
        self.assertEqual(rank, self.test.last_index)
        self.assertRaises(TypeError, self.test.append, item='S')

    def testUpdate(self):
        new = self.new(items=self.items, type=int, start_index=self.start)

        new._update([98, 99])
        self.assertTrue(98 in new)
        self.assertEquals(new.length, 2)

        self.assertRaises(TypeError, new._update, ['S'])

@test.unit
class TestAbstractContainer(test.Case):

    def setUp(self):

        super(TestAbstractContainer, self).setUp()

        dictitems = {'A': 1, 'B': 2}
        self.items = dictitems

        listitems = [11, 22]
        start = 5
        self.listitems = listitems
        self.start = start

        class DummyDict(utils.AbstractContainer):
            @property
            def _children(self):
                return utils.DictionaryContainer(dictitems)

        class DummyList(utils.AbstractContainer):
            @property
            def _children(self):
                return utils.CollectionContainer(listitems, start_index=start)

        self.dict = DummyDict()
        self.list = DummyList()

    def testGetitem(self):
        self.assertEqual(self.dict['A'], 1)
        self.assertRaises(utils.ItemNotFoundError, lambda k: self.dict[k], 'X')

        self.assertEqual(self.list[self.start], self.listitems[0])
        if self.start > 0:
            self.assertRaises(utils.CollectionIndexError, lambda i: self.list[i], 0)

    def testIterator(self):
        self.assertEqual(list(self.items), list(self.dict))
        self.assertEqual(list(self.listitems), list(self.list))

@test.unit
class TestAbstractNIContainer(test.Case):

    def setUp(self):

        super(TestAbstractNIContainer, self).setUp()

        items = list(set([11, 22, 33]))
        start = 1

        self.items = items
        self.start = start

        class Dummy(utils.AbstractNIContainer):
            @property
            def _children(self):
                return utils.CollectionContainer(items, start_index=start)

        self.test = Dummy()

    def testGetitem(self):
        assert self.start > 0

        self.assertEqual(self.test[0], self.items[0])
        self.assertEqual(self.test[1:2], self.items[1:2])
        self.assertEqual(self.test[0], self.test._children[self.start])
        self.assertNotEqual(self.test[1], self.test._children[1])

        self.assertRaises(IndexError, lambda i: self.test[i], 9999999)

    def testIterator(self):
        self.assertEqual(list(self.items), list(self.test))

if __name__ == '__main__':

    test.Console()        <commit_msg>Added TestOrderedDict<commit_after>import sys
import operator
import csb.test as test
import csb.pyutils as utils

@test.unit
class TestShell(test.Case):

    def setUp(self):

        super(TestShell, self).setUp()
        self.output = utils.Shell.run('echo TeST')

    def testSTDOUT(self):
        self.assertEquals(self.output.stdout.strip(), 'TeST')

    def testSTDERR(self):
        self.assertEquals(self.output.stderr, '')

    def testExitCode(self):
        self.assertEquals(self.output.code, 0)

@test.unit
class TestDeepCopy(test.Case):

    def runTest(self):

        rec = sys.getrecursionlimit()
        obj = ['X']
        copy = utils.deepcopy(obj, recursion=(rec + 1))

        self.assertEquals(obj, copy)
        self.assertNotEquals(id(obj), id(copy))

@test.unit
class TestEnum(test.Case):

    def setUp(self):

        super(TestEnum, self).setUp()
        self.enum = utils.enum('A', 'B', C=66)

    def testConstructor(self):
        self.assertRaises(AttributeError, utils.enum, '3')
        self.assertRaises(ValueError, utils.enum, 'A', 'A')
        self.assertRaises(ValueError, utils.enum, A=1, B=1)

    def testComparison(self):
        self.assertEqual(self.enum.A, 0)
        self.assertEqual(self.enum.C, 66)
        self.assertFalse(self.enum.C is 66)
        self.assertFalse(isinstance(self.enum.A, int))

    def testStr(self):
        self.assertEqual(str(self.enum.A), '0')
        self.assertEqual(int(self.enum.A), 0)
        self.assertEqual(self.enum.A.value, 0)

    def testRepr(self):
        self.assertEqual(repr(self.enum.A), 'A')
        self.assertEqual(self.enum.A.name, 'A')

    def testMembers(self):
        result = utils.Enum.members(self.enum)
        members = [self.enum.A, self.enum.B, self.enum.C]
        self.assertEquals(set(result), set(members))

    def testNames(self):
        result = utils.Enum.names(self.enum)
        names = map(repr, [self.enum.A, self.enum.B, self.enum.C])
        names2 = map(operator.attrgetter('name'), [self.enum.A, self.enum.B, self.enum.C])
        self.assertEquals(set(result), set(names))
        self.assertEquals(set(result), set(names2))

    def testValues(self):
        result = utils.Enum.values(self.enum)
        values = map(int, [self.enum.A, self.enum.B, self.enum.C])
        values2 = map(operator.attrgetter('value'), [self.enum.A, self.enum.B, self.enum.C])
        self.assertEquals(set(result), set(values))
        self.assertEquals(set(result), set(values2))

    def testParseValue(self):
        item = utils.Enum.parse(self.enum, 66)
        self.assertTrue(item is self.enum.C)
        self.assertRaises(utils.EnumValueError, utils.Enum.parse, self.enum, 111)
        self.assertRaises(utils.EnumValueError, utils.Enum.parse, self.enum, '0')

    def testParseName(self):
        item = utils.Enum.parsename(self.enum, 'A')
        self.assertTrue(item is self.enum.A)
        self.assertRaises(utils.EnumMemberError, utils.Enum.parsename, self.enum, 'XXX')

    def testIsMember(self):
        self.assertTrue(utils.Enum.ismember(self.enum.A, self.enum))

@test.unit
class TestDictionaryContainer(test.Case):

    def setUp(self):

        super(TestDictionaryContainer, self).setUp()

        self.dict = utils.OrderedDict({'A': 1, 'B': 2})
        self.keys = ('A', 'B', 'C', 'D', 'Z')
        self.new = utils.DictionaryContainer
        self.test = utils.DictionaryContainer(items=self.dict, restrict=self.keys)

    def testConstructor(self):
        new = utils.DictionaryContainer(items=self.dict, restrict=self.keys)
        self.assertEqual(dict(new), dict(self.dict))
        self.assertEqual(list(new), list(self.dict))    # True if the dictionary is ordered
        self.assertEqual(new.length, len(self.dict))
        self.assertRaises(utils.InvalidKeyError, self.new, items={'X': 1}, restrict=self.keys)

    def testAppend(self):
        self.test.append('C', 1)
        self.assertTrue('C' in self.test)
        self.assertRaises(utils.DuplicateKeyError, self.test.append, key='C', item=1)
        self.assertRaises(utils.InvalidKeyError, self.test.append, key='X', item=1)

    def testRemove(self):
        self.assertTrue('A' in self.test)
        self.test._remove('A')
        self.assertFalse('A' in self.test)

    def testGetitem(self):
        self.assertEqual(self.test['B'], self.dict['B'])
        self.assertRaises(utils.ItemNotFoundError, lambda k: self.test[k], 66)

    def testSet(self):
        new = utils.DictionaryContainer(items=self.dict, restrict=self.keys)
        new._set({'Z': 6})
        self.assertTrue('Z' in new)
        self.assertFalse('A' in new)

    def testUpdate(self):
        new = utils.DictionaryContainer(items=self.dict, restrict=self.keys)

        new._update({'A': 7})
        self.assertTrue('A' in new)
        self.assertEquals(new['A'], 7)

        self.assertRaises(utils.ItemNotFoundError, new._update, {'Z': 0})

@test.unit
class TestCollectionContainer(test.Case):

    def setUp(self):

        super(TestCollectionContainer, self).setUp()

        self.items = [11, 22, 33]
        self.start = 5
        self.new = utils.CollectionContainer
        self.test = utils.CollectionContainer(items=self.items, type=int, start_index=self.start)

    def testConstructor(self):
        new = utils.CollectionContainer(items=self.items, type=int, start_index=self.start)
        self.assertEqual(list(new), list(self.items))
        self.assertEqual(new.length, len(self.items))
        self.assertRaises(TypeError, self.new, items=['S', 1.2], type=int)

    def testGetitem(self):
        self.assertEqual(self.test[self.start], self.items[0])
        self.assertEqual(self.test[self.test.start_index], self.items[0])
        self.assertEqual(self.test[self.test.last_index], self.items[-1])
        self.assertEqual(self.test[-1], self.items[-1])
        self.assertEqual(self.test[-1 : -2], self.items[-1 : -2])

        def get(i):
            return self.test[i]

        self.assertRaises(utils.CollectionIndexError, get, self.start + len(self.items))  # i = end + 1
        if self.start > 0:
            self.assertRaises(utils.CollectionIndexError, get, self.start - 1)            # i = start - 1, i >= 0

    def testIndices(self):
        new = self.new(items=self.items, type=int, start_index=self.start)
        self.assertEqual(new.start_index, self.start)
        self.assertEqual(new.last_index, self.start + len(self.items) - 1)
        self.assertEqual(new.length, len(self.items))

    def testAppend(self):
        rank = self.test.append(44)
        self.assertTrue(44 in self.test)
        self.assertEqual(rank, self.test.last_index)
        self.assertRaises(TypeError, self.test.append, item='S')

    def testUpdate(self):
        new = self.new(items=self.items, type=int, start_index=self.start)

        new._update([98, 99])
        self.assertTrue(98 in new)
        self.assertEquals(new.length, 2)

        self.assertRaises(TypeError, new._update, ['S'])

@test.unit
class TestAbstractContainer(test.Case):

    def setUp(self):

        super(TestAbstractContainer, self).setUp()

        dictitems = {'A': 1, 'B': 2}
        self.items = dictitems

        listitems = [11, 22]
        start = 5
        self.listitems = listitems
        self.start = start

        class DummyDict(utils.AbstractContainer):
            @property
            def _children(self):
                return utils.DictionaryContainer(dictitems)

        class DummyList(utils.AbstractContainer):
            @property
            def _children(self):
                return utils.CollectionContainer(listitems, start_index=start)

        self.dict = DummyDict()
        self.list = DummyList()

    def testGetitem(self):
        self.assertEqual(self.dict['A'], 1)
        self.assertRaises(utils.ItemNotFoundError, lambda k: self.dict[k], 'X')

        self.assertEqual(self.list[self.start], self.listitems[0])
        if self.start > 0:
            self.assertRaises(utils.CollectionIndexError, lambda i: self.list[i], 0)

    def testIterator(self):
        self.assertEqual(list(self.items), list(self.dict))
        self.assertEqual(list(self.listitems), list(self.list))

@test.unit
class TestAbstractNIContainer(test.Case):

    def setUp(self):

        super(TestAbstractNIContainer, self).setUp()

        items = list(set([11, 22, 33]))
        start = 1

        self.items = items
        self.start = start

        class Dummy(utils.AbstractNIContainer):
            @property
            def _children(self):
                return utils.CollectionContainer(items, start_index=start)

        self.test = Dummy()

    def testGetitem(self):
        assert self.start > 0

        self.assertEqual(self.test[0], self.items[0])
        self.assertEqual(self.test[1:2], self.items[1:2])
        self.assertEqual(self.test[0], self.test._children[self.start])
        self.assertNotEqual(self.test[1], self.test._children[1])

        self.assertRaises(IndexError, lambda i: self.test[i], 9999999)

    def testIterator(self):
        self.assertEqual(list(self.items), list(self.test))

@test.functional
class TestOrderedDict(test.Case):

    def runTest(self):

        items = [('G', 4), ('A', 2), ('C', 8), ('B', 7)]
        odict = utils.OrderedDict(items)

        self.assertEquals(odict.items(), items)

        for i, k in enumerate(odict):
            self.assertEqual(k, items[i][0])
            self.assertEqual(odict[k], items[i][1])

if __name__ == '__main__':

    test.Console()
<|endoftext|>


<commit_before>"""
Tinman Cache Module
"""
__author__ = "Gavin M. Roy"
__email__ = "gmr@myyearbook.com"
__date__ = "2010-03-10"
__version__ = 0.2

from logging import debug

# Module wide dictionary to hold the cached values in
local_cache = dict()

# Cache Decorator
def memoize(fn):

    def wrapper(*args):
        # Our module wide local_cache
        global local_cache

        # Get the class name for the key
        key = repr(args[0])

        # Add the arguments to the key
        for value in args[1:]:
            key += ':%s' % str(value)

        debug('memoize: %s' % key)
        if key in local_cache:
            debug('memoize hit: %s' % key)
            return local_cache[key]

        # Call and return the original function
        value = fn(*args)

        # Set the Value
        debug('memoize set: %s' % key)
        local_cache[key] = value

        # Return the value
        return value

    return wrapper

def flush():
    """
    Flush all of the attributes in the cache
    """
    global local_cache
    local_cache = dict()
<commit_msg>Make memoize cache all the output of RequestHandler.write by monkey patching the module into the write and finish code flow<commit_after>"""
Tinman Cache Module
"""
__author__ = "Gavin M. Roy"
__email__ = "gmr@myyearbook.com"
__date__ = "2010-03-10"
__version__ = 0.2

from functools import wraps
from logging import debug

# Module wide dictionary to hold the cached values in
local_cache = dict()

def memoize_write(*args):

    # Append the value if the key exists, otherwise just set it
    if args[0].tinman_memoize_key in local_cache:
        debug('memoize append: %s' % key)
        local_cache[args[0].tinman_memoize_key] += args[1]
    else:
        debug('memoize set: %s' % key)
        local_cache[args[0].tinman_memoize_key] = args[1]

    # Call the monkey patched RequestHandler.write
    args[0]._write(args[1])

def memoize_finish(*args):

    # If they passed in a last chunk, run the write
    if len(args) > 1:
        memoize_write(args)

    # Un-Monkey-patch
    args[0].write = args[0]._write
    args[0].finish = args[0]._finish

    # Remove the monkey patched attributes
    del args[0]._write
    del args[0]._finish

    # Call the RequestHandler.finish
    args[0].finish()

# Cache Decorator
def memoize(method):

    @wraps(method)
    def wrapper(*args, **kwargs):

        # Our module wide local_cache
        global local_cache

        if not hasattr(args[0], 'write'):
            raise AttributeError("Could not find the ")

        # Get the class name for the key
        key = repr(args[0])

        # Add the arguments to the key
        for value in args[1:]:
            key += ':%s' % str(value)

        debug('memoize: %s' % key)

        # See if the key is in cache and if so, send it
        if key in local_cache:
            debug('memoize hit: %s' % key)
            return self.finish(local_cache[key])

        # Assign our key
        args[0].tinman_memoize_key = key

        # Monkey-patch the write and finish functions
        args[0]._write = args[0].write
        args[0]._finish = args[0].finish
        args[0].write = memoize_write
        args[0].finish = memoize_finish

        # Return the value
        return method(*args, **kwargs)

    return wrapper

def flush():
    """
    Flush all of the attributes in the cache
    """
    global local_cache
    local_cache = dict()
<|endoftext|>


<commit_before>
import string

from cupy.core import _kernel

def _get_cub_reduction_function_code(
        name, block_size, segment_size, reduce_type, params, arginfos, identity,
        pre_map_expr, reduce_expr, post_map_expr,
        type_map, input_expr, output_expr, preamble, options):
    print("\n**************** I AM HERE ******************")
    #module_code = string.Template('''
    module_code = '''
#include <cub/block/block_reduce.cuh>
#include <cub/block/block_load.cuh>
//#include <inttypes.h>

${type_preamble}
${preamble}

//TODO(leofang): this should be auto-tuned based on CUDA arch?
#define ITEMS_PER_THREAD 4

#define POST_MAP(a) (${post_map_expr})

#define SEGMENT_SIZE ${segment_size}

typedef ${reduce_type} _type_reduce;

struct _reduction_op
{
    __device__ __forceinline__ _type_reduce operator()(const _type_reduce &a, const _type_reduce &b) const
    {
        return ${reduce_expr};
    }
};

extern "C" {
__global__ void ${name}(${params}) {
  unsigned int _tid = threadIdx.x;
  unsigned int _bid = blockIdx.x * blockDim.x + _tid;
  //printf("%d\\n", _bid);

  // Specialize BlockReduce type for our thread block
  typedef cub::BlockReduce<_type_reduce, ${block_size}> BlockReduceT;

  // Shared memory
  __shared__ typename BlockReduceT::TempStorage temp_storage;

  // Attempt: ignore CIndexer _in_ind and _out_ind?
  const int in_ndim = _raw_in0.ndim;
  const int out_ndim = _raw_out0.ndim;
  const ptrdiff_t* in_shape = _raw_in0.shape();
  const ptrdiff_t* out_shape = _raw_out0.shape();
  const ptrdiff_t* in_strides = _raw_in0.strides();
  const ptrdiff_t* out_strides = _raw_out0.strides();
  if (_bid == 0) {
    printf("%i %i\\n(", in_ndim, out_ndim);
    for(int i = 0; i<in_ndim; i++) {
       printf("%d, ", in_shape[i]);
    }
    printf("\\b\\b)\\n(");

    for(int i = 0; i<in_ndim; i++) {
       printf("%d, ", in_strides[i]);
    }
    printf("\\b\\b)\\n\\n(");

    for(int i = 0; i<out_ndim; i++) {
       printf("%d, ", out_shape[i]);
    }
    printf("\\b\\b)\\n(");

    for(int i = 0; i<out_ndim; i++) {
       printf("%d, ", out_strides[i]);
    }
    printf("\\b\\b)\\n");
  }

  // Declare reduction operation
  _reduction_op op;

  // input & output raw pointers
  // TODO: relax the type requriments?
  _type_reduce* _in0 = (_type_reduce*)&(_raw_in0[0]);
  _type_reduce* _out0 = (_type_reduce*)&(_raw_out0[0]);
'''

    # TODO: if map_expr == 'in0', use CUB load, else use for loop
    if pre_map_expr == 'in0':
        module_code += '''
  // Per-thread tile data
  _type_reduce _sdata[ITEMS_PER_THREAD] = {${identity}};

  // each block handles the reduction of 1 segment
  _type_reduce _block_out = ${identity};
  for (size_t i = 0; i < SEGMENT_SIZE; i += blockDim.x * ITEMS_PER_THREAD) {
      cub::LoadDirectBlocked(_tid, _in0 + blockIdx.x * SEGMENT_SIZE, _sdata, SEGMENT_SIZE);

      //for (size_t i = 0; i<ITEMS_PER_THREAD; i++)
      //    printf("_bid: %d, local items: %f\\n", _bid, _sdata[i]);
'''
    else:  # TODO
        raise NotImplementedError
        module_code += '''
'''

    module_code += '''
      // Compute block reduction
      _type_reduce aggregate = BlockReduceT(temp_storage).Reduce(_sdata, op);

      if (_tid == 0)
          _block_out = op(_block_out, aggregate);
  }

  if (_tid == 0) {
      printf("_bid: %d (blockIdx.x: %d), block out: %f\\n", _bid, blockIdx.x, _block_out);

      type_out0_raw& out0 = *(_out0 + blockIdx.x);
      POST_MAP(_block_out);
  }
}
} // extern C'''
    module_code = string.Template(module_code).substitute(
        name=name,  # used
        block_size=block_size,  # used
        segment_size=segment_size,  # used
        reduce_type=reduce_type,  # used
        params=_kernel._get_kernel_params(params, arginfos),  # used
        identity=identity,
        reduce_expr=reduce_expr,  # used
        pre_map_expr=pre_map_expr,
        post_map_expr=post_map_expr,  # used
        type_preamble=type_map.get_typedef_code(),  # used
        input_expr=input_expr,
        output_expr=output_expr,
        preamble=preamble)  # used
    print('\n', module_code, '\n')
    return module_code
<commit_msg>[WIP] fixed the bug with the last tile!<commit_after>
import string

from cupy.core import _kernel

def _get_cub_reduction_function_code(
        name, block_size, segment_size, reduce_type, params, arginfos, identity,
        pre_map_expr, reduce_expr, post_map_expr,
        type_map, input_expr, output_expr, preamble, options):
    print("\n**************** I AM HERE ******************")
    #module_code = string.Template('''
    module_code = '''
#include <cub/block/block_reduce.cuh>
#include <cub/block/block_load.cuh>
//#include <inttypes.h>

${type_preamble}
${preamble}

//TODO(leofang): this should be auto-tuned based on CUDA arch?
#define ITEMS_PER_THREAD 4

#define POST_MAP(a) (${post_map_expr})

#define SEGMENT_SIZE ${segment_size}

typedef ${reduce_type} _type_reduce;

struct _reduction_op
{
    __device__ __forceinline__ _type_reduce operator()(const _type_reduce &a, const _type_reduce &b) const
    {
        return ${reduce_expr};
    }
};

extern "C" {
__global__ void ${name}(${params}) {
  unsigned int _tid = threadIdx.x;
  unsigned int _bid = blockIdx.x * blockDim.x + _tid;
  //printf("%d\\n", _bid);

  // Specialize BlockReduce type for our thread block
  typedef cub::BlockReduce<_type_reduce, ${block_size}> BlockReduceT;

  // Shared memory
  __shared__ typename BlockReduceT::TempStorage temp_storage;

  // Attempt: ignore CIndexer _in_ind and _out_ind?
//  const int in_ndim = _raw_in0.ndim;
//  const int out_ndim = _raw_out0.ndim;
//  const ptrdiff_t* in_shape = _raw_in0.shape();
//  const ptrdiff_t* out_shape = _raw_out0.shape();
//  const ptrdiff_t* in_strides = _raw_in0.strides();
//  const ptrdiff_t* out_strides = _raw_out0.strides();
//  if (_bid == 0) {
//    printf("%i %i\\n(", in_ndim, out_ndim);
//    for(int i = 0; i<in_ndim; i++) {
//       printf("%d, ", in_shape[i]);
//    }
//    printf("\\b\\b)\\n(");
//
//    for(int i = 0; i<in_ndim; i++) {
//       printf("%d, ", in_strides[i]);
//    }
//    printf("\\b\\b)\\n\\n(");
//
//    for(int i = 0; i<out_ndim; i++) {
//       printf("%d, ", out_shape[i]);
//    }
//    printf("\\b\\b)\\n(");
//
//    for(int i = 0; i<out_ndim; i++) {
//       printf("%d, ", out_strides[i]);
//    }
//    printf("\\b\\b)\\n");
//  }

  // Declare reduction operation
  _reduction_op op;

  // input & output raw pointers
  // TODO: relax the type requriments?
  _type_reduce* _in0 = (_type_reduce*)&(_raw_in0[0]);
  _type_reduce* _out0 = (_type_reduce*)&(_raw_out0[0]);

  // Per-thread tile data
  _type_reduce _sdata[ITEMS_PER_THREAD] = {${identity}};

  // each block handles the reduction of 1 segment
  _type_reduce _block_out = ${identity};
  size_t i = 0;  // tile head within the segment
  int tile_size = (blockDim.x * ITEMS_PER_THREAD < SEGMENT_SIZE ? blockDim.x * ITEMS_PER_THREAD : SEGMENT_SIZE);
  for (size_t i = 0; i < SEGMENT_SIZE; i += blockDim.x * ITEMS_PER_THREAD) {
'''

    # TODO: if map_expr == 'in0', use CUB load, else use for loop
    if pre_map_expr == 'in0':
        module_code += '''

      if (SEGMENT_SIZE - i < tile_size)  // for the last tile
          tile_size = SEGMENT_SIZE - i;
      //if (_bid == 0) {
      //    printf("i: %d\\n", i);
      //    printf("SEGMENT_SIZE: %d\\n", SEGMENT_SIZE);
      //    printf("tile_size: %d\\n\\n", tile_size);
      //}
      cub::LoadDirectBlocked(_tid, _in0 + blockIdx.x * SEGMENT_SIZE + i, _sdata, tile_size, ${identity});

      //for (size_t i = 0; i<ITEMS_PER_THREAD; i++)
      //    printf("_bid: %d, local items: %f\\n", _bid, _sdata[i]);
'''
    else:  # TODO
        raise NotImplementedError
        module_code += '''
'''

    module_code += '''
      // Compute block reduction
      _type_reduce aggregate = BlockReduceT(temp_storage).Reduce(_sdata, op);

      if (_tid == 0)
          _block_out = op(_block_out, aggregate);

      __syncthreads();  // for reusing temp_storage
  }
'''

#    # TODO: if map_expr == 'in0', use CUB load, else use for loop
#    if pre_map_expr == 'in0':
#        module_code += '''
#//  if (i  // last tile
#//  cub::LoadDirectBlocked(_tid, _in0 + blockIdx.x * SEGMENT_SIZE + i, _sdata, SEGMENT_SIZE - i);
#//
#//  //for (size_t i = 0; i<ITEMS_PER_THREAD; i++)
#//  //    printf("_bid: %d, local items: %f\\n", _bid, _sdata[i]);
#'''
#    else:  # TODO
#        raise NotImplementedError
#        module_code += '''
#'''

    module_code += '''
//      // Compute block reduction
//      _type_reduce aggregate = BlockReduceT(temp_storage).Reduce(_sdata, op);
//
//      if (_tid == 0)
//          _block_out = op(_block_out, aggregate);
//  }

  if (_tid == 0) {
      //printf("_bid: %d (blockIdx.x: %d), block out: %f\\n", _bid, blockIdx.x, _block_out);

      type_out0_raw& out0 = *(_out0 + blockIdx.x);
      POST_MAP(_block_out);
  }
} // kernel end
} // extern C
'''
    module_code = string.Template(module_code).substitute(
        name=name,  # used
        block_size=block_size,  # used
        segment_size=segment_size,  # used
        reduce_type=reduce_type,  # used
        params=_kernel._get_kernel_params(params, arginfos),  # used
        identity=identity,
        reduce_expr=reduce_expr,  # used
        pre_map_expr=pre_map_expr,
        post_map_expr=post_map_expr,  # used
        type_preamble=type_map.get_typedef_code(),  # used
        input_expr=input_expr,
        output_expr=output_expr,
        preamble=preamble)  # used
    print('\n', module_code, '\n')
    return module_code
<|endoftext|>


<commit_before># coding: utf-8

from __future__ import unicode_literals, division

import os
import shutil
from unittest import TestCase
from unittest.mock import patch
import unittest

from custodian.qchem.jobs import QCJob
from pymatgen.io.qchem.inputs import QCInput
from pymatgen.io.qchem.outputs import QCOutput

__author__ = "Samuel Blau"
__copyright__ = "Copyright 2018, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Samuel Blau"
__email__ = "samblau1@gmail.com"
__status__ = "Alpha"
__date__ = "6/6/18"
__credits__ = "Shyam Dwaraknath"

test_dir = os.path.join(
    os.path.dirname(__file__), "..", "..", "..", "test_files", "qchem",
    "new_test_files", "FF_working")

scr_dir = os.path.join(test_dir, "scr")
cwd = os.getcwd()

class QCJobTest(TestCase):

    def test_defaults(self):
        with patch("custodian.qchem.new_jobs.os.putenv") as putenv_patch:
            myjob = QCJob(qchem_command="qchem")
            self.assertEqual(myjob.current_command, ["qchem", "-nt", "32", "mol.qin", "mol.qout"])
            myjob.setup()
            self.assertEqual(putenv_patch.call_args_list[0][0][0], "QCSCRATCH")
            self.assertEqual(putenv_patch.call_args_list[0][0][1], "/dev/shm/qcscratch/")
            self.assertEqual(putenv_patch.call_args_list[1][0][0], "QCTHREADS")
            self.assertEqual(putenv_patch.call_args_list[1][0][1], "32")
            self.assertEqual(putenv_patch.call_args_list[2][0][0], "OMP_NUM_THREADS")
            self.assertEqual(putenv_patch.call_args_list[2][0][1], "32")

    def test_not_defaults(self):
        with patch("custodian.qchem.new_jobs.os.putenv") as putenv_patch:
            myjob = QCJob(qchem_command="qchem -slurm", multimode="mpi", input_file="different.qin", output_file="not_default.qout", max_cores=12, scratch_dir="/not/default/scratch/")
            self.assertEqual(myjob.current_command, ["qchem", "-slurm", "-np", "12", "different.qin", "not_default.qout"])
            myjob.setup()
            self.assertEqual(putenv_patch.call_args_list[0][0][0], "QCSCRATCH")
            self.assertEqual(putenv_patch.call_args_list[0][0][1], "/not/default/scratch/")

class OptFFTest(TestCase):
    def setUp(self):
        os.makedirs(scr_dir)
        shutil.copyfile(os.path.join(test_dir,"test.qin"),os.path.join(scr_dir,"test.qin"))
        shutil.copyfile(os.path.join(test_dir,"test.qout.opt_0"),os.path.join(scr_dir,"test.qout.opt_0"))
        shutil.copyfile(os.path.join(test_dir,"test.qout.freq_0"),os.path.join(scr_dir,"test.qout.freq_0"))
        shutil.copyfile(os.path.join(test_dir,"test.qout.opt_1"),os.path.join(scr_dir,"test.qout.opt_1"))
        shutil.copyfile(os.path.join(test_dir,"test.qout.freq_1"),os.path.join(scr_dir,"test.qout.freq_1"))
        os.chdir(scr_dir)

    def tearDown(self):
        os.chdir(cwd)
        shutil.rmtree(scr_dir)

    def test_OptFF(self):
        myjob = QCJob.opt_with_frequency_flattener(qchem_command="qchem", input_file="test.qin", output_file="test.qout")
        expected_next = QCJob(
                        qchem_command="qchem",
                        multimode="openmp",
                        input_file="test.qin",
                        output_file="test.qout",
                        suffix=".opt_0").as_dict()
        self.assertEqual(next(myjob).as_dict(),expected_next)
        expected_next = QCJob(
                        qchem_command="qchem",
                        multimode="openmp",
                        input_file="test.qin",
                        output_file="test.qout",
                        suffix=".freq_0").as_dict()
        self.assertEqual(next(myjob).as_dict(),expected_next)
        self.assertEqual(QCInput.from_file(os.path.join(test_dir,"test.qin.freq_0")).as_dict(),QCInput.from_file(os.path.join(scr_dir,"test.qin")).as_dict())
        expected_next = QCJob(
                        qchem_command="qchem",
                        multimode="openmp",
                        input_file="test.qin",
                        output_file="test.qout",
                        suffix=".opt_1").as_dict()
        self.assertEqual(next(myjob).as_dict(),expected_next)
        self.assertEqual(QCInput.from_file(os.path.join(test_dir,"test.qin.opt_1")).as_dict(),QCInput.from_file(os.path.join(scr_dir,"test.qin")).as_dict())
        expected_next = QCJob(
                        qchem_command="qchem",
                        multimode="openmp",
                        input_file="test.qin",
                        output_file="test.qout",
                        suffix=".freq_1").as_dict()
        self.assertEqual(next(myjob).as_dict(),expected_next)
        self.assertEqual(QCInput.from_file(os.path.join(test_dir,"test.qin.freq_1")).as_dict(),QCInput.from_file(os.path.join(scr_dir,"test.qin")).as_dict())
        self.assertRaises(StopIteration,myjob.__next__)

if __name__ == "__main__":
    unittest.main()
<commit_msg>Bug fix<commit_after># coding: utf-8

from __future__ import unicode_literals, division

import os
import shutil
from unittest import TestCase
from unittest.mock import patch
import unittest

from custodian.qchem.jobs import QCJob
from pymatgen.io.qchem.inputs import QCInput
from pymatgen.io.qchem.outputs import QCOutput

__author__ = "Samuel Blau"
__copyright__ = "Copyright 2018, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Samuel Blau"
__email__ = "samblau1@gmail.com"
__status__ = "Alpha"
__date__ = "6/6/18"
__credits__ = "Shyam Dwaraknath"

test_dir = os.path.join(
    os.path.dirname(__file__), "..", "..", "..", "test_files", "qchem",
    "new_test_files", "FF_working")

scr_dir = os.path.join(test_dir, "scr")
cwd = os.getcwd()

class QCJobTest(TestCase):

    def test_defaults(self):
        with patch("custodian.qchem.jobs.os.putenv") as putenv_patch:
            myjob = QCJob(qchem_command="qchem")
            self.assertEqual(myjob.current_command, ["qchem", "-nt", "32", "mol.qin", "mol.qout"])
            myjob.setup()
            self.assertEqual(putenv_patch.call_args_list[0][0][0], "QCSCRATCH")
            self.assertEqual(putenv_patch.call_args_list[0][0][1], "/dev/shm/qcscratch/")
            self.assertEqual(putenv_patch.call_args_list[1][0][0], "QCTHREADS")
            self.assertEqual(putenv_patch.call_args_list[1][0][1], "32")
            self.assertEqual(putenv_patch.call_args_list[2][0][0], "OMP_NUM_THREADS")
            self.assertEqual(putenv_patch.call_args_list[2][0][1], "32")

    def test_not_defaults(self):
        with patch("custodian.qchem.jobs.os.putenv") as putenv_patch:
            myjob = QCJob(qchem_command="qchem -slurm", multimode="mpi", input_file="different.qin", output_file="not_default.qout", max_cores=12, scratch_dir="/not/default/scratch/")
            self.assertEqual(myjob.current_command, ["qchem", "-slurm", "-np", "12", "different.qin", "not_default.qout"])
            myjob.setup()
            self.assertEqual(putenv_patch.call_args_list[0][0][0], "QCSCRATCH")
            self.assertEqual(putenv_patch.call_args_list[0][0][1], "/not/default/scratch/")

class OptFFTest(TestCase):
    def setUp(self):
        os.makedirs(scr_dir)
        shutil.copyfile(os.path.join(test_dir,"test.qin"),os.path.join(scr_dir,"test.qin"))
        shutil.copyfile(os.path.join(test_dir,"test.qout.opt_0"),os.path.join(scr_dir,"test.qout.opt_0"))
        shutil.copyfile(os.path.join(test_dir,"test.qout.freq_0"),os.path.join(scr_dir,"test.qout.freq_0"))
        shutil.copyfile(os.path.join(test_dir,"test.qout.opt_1"),os.path.join(scr_dir,"test.qout.opt_1"))
        shutil.copyfile(os.path.join(test_dir,"test.qout.freq_1"),os.path.join(scr_dir,"test.qout.freq_1"))
        os.chdir(scr_dir)

    def tearDown(self):
        os.chdir(cwd)
        shutil.rmtree(scr_dir)

    def test_OptFF(self):
        myjob = QCJob.opt_with_frequency_flattener(qchem_command="qchem", input_file="test.qin", output_file="test.qout")
        expected_next = QCJob(
                        qchem_command="qchem",
                        multimode="openmp",
                        input_file="test.qin",
                        output_file="test.qout",
                        suffix=".opt_0").as_dict()
        self.assertEqual(next(myjob).as_dict(),expected_next)
        expected_next = QCJob(
                        qchem_command="qchem",
                        multimode="openmp",
                        input_file="test.qin",
                        output_file="test.qout",
                        suffix=".freq_0").as_dict()
        self.assertEqual(next(myjob).as_dict(),expected_next)
        self.assertEqual(QCInput.from_file(os.path.join(test_dir,"test.qin.freq_0")).as_dict(),QCInput.from_file(os.path.join(scr_dir,"test.qin")).as_dict())
        expected_next = QCJob(
                        qchem_command="qchem",
                        multimode="openmp",
                        input_file="test.qin",
                        output_file="test.qout",
                        suffix=".opt_1").as_dict()
        self.assertEqual(next(myjob).as_dict(),expected_next)
        self.assertEqual(QCInput.from_file(os.path.join(test_dir,"test.qin.opt_1")).as_dict(),QCInput.from_file(os.path.join(scr_dir,"test.qin")).as_dict())
        expected_next = QCJob(
                        qchem_command="qchem",
                        multimode="openmp",
                        input_file="test.qin",
                        output_file="test.qout",
                        suffix=".freq_1").as_dict()
        self.assertEqual(next(myjob).as_dict(),expected_next)
        self.assertEqual(QCInput.from_file(os.path.join(test_dir,"test.qin.freq_1")).as_dict(),QCInput.from_file(os.path.join(scr_dir,"test.qin")).as_dict())
        self.assertRaises(StopIteration,myjob.__next__)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>from datetime import timedelta
import datetime
from django.utils.translation import ugettext_noop
from django.utils import html
from casexml.apps.case.models import CommCareCase
from django.core.urlresolvers import reverse, NoReverseMatch
from django.utils.translation import ugettext as _
from corehq.apps.reports.standard.cases.basic import CaseListReport

from corehq.apps.api.es import ReportCaseES
from corehq.apps.reports.standard import CustomProjectReport
from corehq.apps.reports.datatables import DataTablesHeader, DataTablesColumn
from corehq.apps.reports.standard.cases.data_sources import CaseDisplay
from corehq.pillows.base import restore_property_dict
from corehq.util.timezones.conversions import PhoneTime
from dimagi.utils.dates import force_to_date
from dimagi.utils.decorators.memoized import memoized
from dimagi.utils.parsing import json_format_date

def visit_completion_counter(case):
    mother_counter = 0
    child_counter = 0
    case_obj = CommCareCase.get(case['_id'])
    baby_case = [c for c in case_obj.get_subcases().all() if c.type == 'baby']
    for i in range(1, 8):
        if "pp_%s_done" % i in case:
            val = case["pp_%s_done" % i]
            try:
                if val.lower() == 'yes':
                    mother_counter += 1
                elif int(float(val)) == 1:
                    mother_counter += 1
            except ValueError:
                pass
        if baby_case and "bb_pp_%s_done" % i in baby_case[0]:
            val = baby_case[0]["bb_pp_%s_done" % i]
            try:
                if val.lower() == 'yes':
                    child_counter += 1
                elif int(float(val)) == 1:
                    child_counter += 1
            except ValueError:
                pass

    return mother_counter if mother_counter > child_counter else child_counter

class HNBCReportDisplay(CaseDisplay):

    @property
    def dob(self):
        if 'date_birth' not in self.case:
            return '---'
        else:
            return self.report.date_to_json(self.case['date_birth'])

    @property
    def visit_completion(self):
        return "%s/7" % visit_completion_counter(self.case)

    @property
    def delivery(self):
        if 'place_birth' not in self.case:
            return '---'
        else:
            if "at_home" == self.case['place_birth']:
                return _('Home')
            elif "in_hospital" == self.case['place_birth']:
                return _('Hospital')
            else:
                return _('Other')

    @property
    def case_link(self):
        case_id, case_name = self.case['_id'], self.case['mother_name']
        try:
            return html.mark_safe("%s" % (
                html.escape(reverse('crs_details_report', args=[self.report.domain, case_id, self.report.slug])),
                html.escape(case_name),
            ))
        except NoReverseMatch:
            return "%s (bad ID format)" % case_name

    @property
    def baby_name(self):
        case = CommCareCase.get(self.case['_id'])

        baby_case = [c for c in case.get_subcases().all() if c.type == 'baby']
        if baby_case:
            return baby_case[0].name
        else:
            return '---'

class BaseHNBCReport(CustomProjectReport, CaseListReport):

    fields = ['custom.apps.crs_reports.fields.SelectBlockField',
              'custom.apps.crs_reports.fields.SelectSubCenterField', # Todo: Currently there is no data about it in case
              'custom.apps.crs_reports.fields.SelectASHAField']

    ajax_pagination = True
    include_inactive = True
    module_name = 'crs_reports'
    report_template_name = None

    @property
    @memoized
    def case_es(self):
        return ReportCaseES(self.domain)

    def build_es_query(self, case_type=None, afilter=None, status=None):

        def _domain_term():
            return {"term": {"domain.exact": self.domain}}

        subterms = [_domain_term(), afilter] if afilter else [_domain_term()]
        if case_type:
            subterms.append({"term": {"type.exact": case_type}})

        if status:
            subterms.append({"term": {"closed": (status == 'closed')}})

        es_query = {
            'query': {
                'filtered': {
                    'query': {"match_all": {}},
                    'filter': {'and': subterms}
                }
            },
            'sort': self.get_sorting_block(),
            'from': self.pagination.start,
            'size': self.pagination.count,
        }

        return es_query

    @property
    @memoized
    def es_results(self):
        query = self.build_es_query(case_type=self.case_type, afilter=self.case_filter, status=self.case_status)
        return self.case_es.run_query(query)

    @property
    def headers(self):
        headers = DataTablesHeader(
            DataTablesColumn(_("Mother Name"), prop_name="mother_name.#value"),
            DataTablesColumn(_("Baby Name"), sortable=False),
            DataTablesColumn(_("CHW Name"), prop_name="owner_display", sortable=False),
            DataTablesColumn(_("Date of Delivery"),  prop_name="date_birth.#value"),
            DataTablesColumn(_("PNC Visit Completion"), sortable=False),
            DataTablesColumn(_("Delivery"), prop_name="place_birth"),
        )
        return headers

    @property
    def rows(self):
        case_displays = (HNBCReportDisplay(self, restore_property_dict(self.get_case(case)))
                         for case in self.es_results['hits'].get('hits', []))

        for disp in case_displays:
            yield [
                disp.case_link,
                disp.baby_name,
                disp.owner_display,
                disp.dob,
                disp.visit_completion,
                disp.delivery,
            ]

    @property
    @memoized
    def rendered_report_title(self):
        if not self.individual:
            self.name = _("%(report_name)s for (0-42 days after delivery)") % {
                "report_name": _(self.name)
            }
        return self.name

    def base_filters(self):
        block = self.request_params.get('block', '')
        individual = self.request_params.get('individual', '')
        filters = []

        if block:
            filters.append({'term': {'block.#value': block}})
        if individual:
            filters.append({'term': {'owner_id': individual}})
        return filters

    def date_to_json(self, date):
        if date:
            try:
                date = force_to_date(date)
                return (PhoneTime(date, self.timezone).user_time(self.timezone)
                        .ui_string('%d-%m-%Y'))
            except ValueError:
                return ''
        else:
            return ''

class HBNCMotherReport(BaseHNBCReport):

    name = ugettext_noop('Mother HBNC Form')
    slug = 'hbnc_mother_report'
    report_template_name = 'mothers_form_reports_template'
    default_case_type = 'pregnant_mother'

    @property
    def case_filter(self):
        now = datetime.datetime.utcnow()
        fromdate = now - timedelta(days=42)
        filters = BaseHNBCReport.base_filters(self)
        filters.append({'term': {'pp_case_filter.#value': "1"}})
        filters.append({'range': {'date_birth.#value': {"gte": json_format_date(fromdate)}}})
        status = self.request_params.get('PNC_status', '')

        or_stmt = []

        if status:
            if status == 'On Time':
                for i in range(1, 8):
                    filters.append({'term': {'case_pp_%s_done.#value' % i: 'yes'}})
            else:
                for i in range(1, 8):
                    or_stmt.append({"not": {'term': {'case_pp_%s_done.#value' % i: 'yes'}}})
                or_stmt = {'or': or_stmt}
                filters.append(or_stmt)

        return {'and': filters} if filters else {}

<commit_msg>case 182780: force to datetime object<commit_after>from datetime import timedelta
import datetime
from django.utils.translation import ugettext_noop
from django.utils import html
from casexml.apps.case.models import CommCareCase
from django.core.urlresolvers import reverse, NoReverseMatch
from django.utils.translation import ugettext as _
from corehq.apps.reports.standard.cases.basic import CaseListReport

from corehq.apps.api.es import ReportCaseES
from corehq.apps.reports.standard import CustomProjectReport
from corehq.apps.reports.datatables import DataTablesHeader, DataTablesColumn
from corehq.apps.reports.standard.cases.data_sources import CaseDisplay
from corehq.pillows.base import restore_property_dict
from corehq.util.timezones.conversions import PhoneTime
from dimagi.utils.dates import force_to_datetime
from dimagi.utils.decorators.memoized import memoized
from dimagi.utils.parsing import json_format_date

def visit_completion_counter(case):
    mother_counter = 0
    child_counter = 0
    case_obj = CommCareCase.get(case['_id'])
    baby_case = [c for c in case_obj.get_subcases().all() if c.type == 'baby']
    for i in range(1, 8):
        if "pp_%s_done" % i in case:
            val = case["pp_%s_done" % i]
            try:
                if val.lower() == 'yes':
                    mother_counter += 1
                elif int(float(val)) == 1:
                    mother_counter += 1
            except ValueError:
                pass
        if baby_case and "bb_pp_%s_done" % i in baby_case[0]:
            val = baby_case[0]["bb_pp_%s_done" % i]
            try:
                if val.lower() == 'yes':
                    child_counter += 1
                elif int(float(val)) == 1:
                    child_counter += 1
            except ValueError:
                pass

    return mother_counter if mother_counter > child_counter else child_counter

class HNBCReportDisplay(CaseDisplay):

    @property
    def dob(self):
        if 'date_birth' not in self.case:
            return '---'
        else:
            return self.report.date_to_json(self.case['date_birth'])

    @property
    def visit_completion(self):
        return "%s/7" % visit_completion_counter(self.case)

    @property
    def delivery(self):
        if 'place_birth' not in self.case:
            return '---'
        else:
            if "at_home" == self.case['place_birth']:
                return _('Home')
            elif "in_hospital" == self.case['place_birth']:
                return _('Hospital')
            else:
                return _('Other')

    @property
    def case_link(self):
        case_id, case_name = self.case['_id'], self.case['mother_name']
        try:
            return html.mark_safe("%s" % (
                html.escape(reverse('crs_details_report', args=[self.report.domain, case_id, self.report.slug])),
                html.escape(case_name),
            ))
        except NoReverseMatch:
            return "%s (bad ID format)" % case_name

    @property
    def baby_name(self):
        case = CommCareCase.get(self.case['_id'])

        baby_case = [c for c in case.get_subcases().all() if c.type == 'baby']
        if baby_case:
            return baby_case[0].name
        else:
            return '---'

class BaseHNBCReport(CustomProjectReport, CaseListReport):

    fields = ['custom.apps.crs_reports.fields.SelectBlockField',
              'custom.apps.crs_reports.fields.SelectSubCenterField', # Todo: Currently there is no data about it in case
              'custom.apps.crs_reports.fields.SelectASHAField']

    ajax_pagination = True
    include_inactive = True
    module_name = 'crs_reports'
    report_template_name = None

    @property
    @memoized
    def case_es(self):
        return ReportCaseES(self.domain)

    def build_es_query(self, case_type=None, afilter=None, status=None):

        def _domain_term():
            return {"term": {"domain.exact": self.domain}}

        subterms = [_domain_term(), afilter] if afilter else [_domain_term()]
        if case_type:
            subterms.append({"term": {"type.exact": case_type}})

        if status:
            subterms.append({"term": {"closed": (status == 'closed')}})

        es_query = {
            'query': {
                'filtered': {
                    'query': {"match_all": {}},
                    'filter': {'and': subterms}
                }
            },
            'sort': self.get_sorting_block(),
            'from': self.pagination.start,
            'size': self.pagination.count,
        }

        return es_query

    @property
    @memoized
    def es_results(self):
        query = self.build_es_query(case_type=self.case_type, afilter=self.case_filter, status=self.case_status)
        return self.case_es.run_query(query)

    @property
    def headers(self):
        headers = DataTablesHeader(
            DataTablesColumn(_("Mother Name"), prop_name="mother_name.#value"),
            DataTablesColumn(_("Baby Name"), sortable=False),
            DataTablesColumn(_("CHW Name"), prop_name="owner_display", sortable=False),
            DataTablesColumn(_("Date of Delivery"),  prop_name="date_birth.#value"),
            DataTablesColumn(_("PNC Visit Completion"), sortable=False),
            DataTablesColumn(_("Delivery"), prop_name="place_birth"),
        )
        return headers

    @property
    def rows(self):
        case_displays = (HNBCReportDisplay(self, restore_property_dict(self.get_case(case)))
                         for case in self.es_results['hits'].get('hits', []))

        for disp in case_displays:
            yield [
                disp.case_link,
                disp.baby_name,
                disp.owner_display,
                disp.dob,
                disp.visit_completion,
                disp.delivery,
            ]

    @property
    @memoized
    def rendered_report_title(self):
        if not self.individual:
            self.name = _("%(report_name)s for (0-42 days after delivery)") % {
                "report_name": _(self.name)
            }
        return self.name

    def base_filters(self):
        block = self.request_params.get('block', '')
        individual = self.request_params.get('individual', '')
        filters = []

        if block:
            filters.append({'term': {'block.#value': block}})
        if individual:
            filters.append({'term': {'owner_id': individual}})
        return filters

    def date_to_json(self, date):
        if date:
            try:
                date = force_to_datetime(date)
                return (PhoneTime(date, self.timezone).user_time(self.timezone)
                        .ui_string('%d-%m-%Y'))
            except ValueError:
                return ''
        else:
            return ''

class HBNCMotherReport(BaseHNBCReport):

    name = ugettext_noop('Mother HBNC Form')
    slug = 'hbnc_mother_report'
    report_template_name = 'mothers_form_reports_template'
    default_case_type = 'pregnant_mother'

    @property
    def case_filter(self):
        now = datetime.datetime.utcnow()
        fromdate = now - timedelta(days=42)
        filters = BaseHNBCReport.base_filters(self)
        filters.append({'term': {'pp_case_filter.#value': "1"}})
        filters.append({'range': {'date_birth.#value': {"gte": json_format_date(fromdate)}}})
        status = self.request_params.get('PNC_status', '')

        or_stmt = []

        if status:
            if status == 'On Time':
                for i in range(1, 8):
                    filters.append({'term': {'case_pp_%s_done.#value' % i: 'yes'}})
            else:
                for i in range(1, 8):
                    or_stmt.append({"not": {'term': {'case_pp_%s_done.#value' % i: 'yes'}}})
                or_stmt = {'or': or_stmt}
                filters.append(or_stmt)

        return {'and': filters} if filters else {}

<|endoftext|>


<commit_before># Copyright (c) 2015, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

from mixbox import entities
from mixbox import fields

import cybox.bindings.win_volume_object as win_volume_binding
from cybox.objects.volume_object import Volume
from cybox.common import String

class WindowsVolumeAttributesList(entities.EntityList):
    _binding = win_volume_binding
    _binding_class = win_volume_binding.WindowsVolumeAttributesListType
    _binding_var = "Attribute"
    _contained_type = String
    _namespace = "http://cybox.mitre.org/objects#WinVolumeObject-2"

class WinVolume(Volume):
    _binding = win_volume_binding
    _binding_class = win_volume_binding.WindowsVolumeObjectType
    _namespace = "http://cybox.mitre.org/objects#WinVolumeObject-2"
    _XSI_NS = "WinVolumeObj"
    _XSI_TYPE = "WindowsVolumeObjectType"

    attributes_list = fields.TypedField("Attributes_List", WindowsVolumeAttributesList)
    drive_letter = fields.TypedField("Drive_Letter", String)
    drive_Type = fields.TypedField("Drive_Type", String)
<commit_msg>Fixed capitalization on a TypedField field name. Addresses python-cybox issue #269.<commit_after># Copyright (c) 2015, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

from mixbox import entities
from mixbox import fields

import cybox.bindings.win_volume_object as win_volume_binding
from cybox.objects.volume_object import Volume
from cybox.common import String

class WindowsVolumeAttributesList(entities.EntityList):
    _binding = win_volume_binding
    _binding_class = win_volume_binding.WindowsVolumeAttributesListType
    _binding_var = "Attribute"
    _contained_type = String
    _namespace = "http://cybox.mitre.org/objects#WinVolumeObject-2"

class WinVolume(Volume):
    _binding = win_volume_binding
    _binding_class = win_volume_binding.WindowsVolumeObjectType
    _namespace = "http://cybox.mitre.org/objects#WinVolumeObject-2"
    _XSI_NS = "WinVolumeObj"
    _XSI_TYPE = "WindowsVolumeObjectType"

    attributes_list = fields.TypedField("Attributes_List", WindowsVolumeAttributesList)
    drive_letter = fields.TypedField("Drive_Letter", String)
    drive_type = fields.TypedField("Drive_Type", String)
<|endoftext|>