<commit_before>from __future__ import absolute_import
from __future__ import unicode_literals
from couchdbkit.exceptions import ResourceNotFound
from datetime import datetime

from casexml.apps.case.dbaccessors import (
    get_extension_case_ids,
    get_indexed_case_ids,
    get_all_reverse_indices_info,
    get_open_case_ids_in_domain,
    get_reverse_indexed_cases,
    get_related_indices,
)
from casexml.apps.case.models import CommCareCase
from casexml.apps.case.util import get_case_xform_ids, iter_cases
from casexml.apps.stock.models import StockTransaction
from corehq.apps.users.util import SYSTEM_USER_ID
from corehq.apps.commtrack.models import StockState
from corehq.apps.hqcase.dbaccessors import (
    get_case_ids_in_domain,
    get_open_case_ids,
    get_closed_case_ids,
    get_case_ids_in_domain_by_owner,
    get_cases_in_domain_by_external_id,
    get_deleted_case_ids_by_owner,
    get_all_case_owner_ids)
from corehq.apps.hqcase.utils import get_case_by_domain_hq_user_id
from corehq.dbaccessors.couchapps.cases_by_server_date.by_owner_server_modified_on import \
    get_case_ids_modified_with_owner_since
from corehq.dbaccessors.couchapps.cases_by_server_date.by_server_modified_on import \
    get_last_modified_dates
from corehq.form_processor.exceptions import AttachmentNotFound, LedgerValueNotFound
from corehq.form_processor.interfaces.dbaccessors import (
    AbstractCaseAccessor, AbstractFormAccessor, AttachmentContent,
    AbstractLedgerAccessor)
from couchforms.dbaccessors import (
    get_forms_by_type,
    get_deleted_form_ids_for_user,
    get_form_ids_for_user,
    get_forms_by_id,
    get_form_ids_by_type,
    get_form_ids_by_xmlns,
)
from couchforms.models import XFormInstance, doc_types, XFormOperation
from dimagi.utils.couch.database import iter_docs
from dimagi.utils.parsing import json_format_datetime
import six

class FormAccessorCouch(AbstractFormAccessor):

    @staticmethod
    def form_exists(form_id, domain=None):
        if not domain:
            return XFormInstance.get_db().doc_exist(form_id)
        else:
            try:
                xform = XFormInstance.get(form_id)
            except ResourceNotFound:
                return False

            return xform.domain == domain

    @staticmethod
    def get_form(form_id):
        return XFormInstance.get(form_id)

    @staticmethod
    def get_forms(form_ids, ordered=False):
        return get_forms_by_id(form_ids)

    @staticmethod
    def get_form_ids_in_domain_by_type(domain, type_):
        return get_form_ids_by_type(domain, type_)

    @staticmethod
    def get_forms_by_type(domain, type_, limit, recent_first=False):
        return get_forms_by_type(domain, type_, recent_first, limit)

    @staticmethod
    def get_with_attachments(form_id):
        doc = XFormInstance.get_db().get(form_id, attachments=True)
        doc = doc_types()[doc['doc_type']].wrap(doc)
        if doc.external_blobs:
            for name, meta in six.iteritems(doc.external_blobs):
                with doc.fetch_attachment(name, stream=True) as content:
                    doc.deferred_put_attachment(
                        content,
                        name,
                        content_type=meta.content_type,
                        content_length=meta.content_length,
                    )
        else:
            # xforms are expected to at least have the XML attachment
            raise ResourceNotFound(msg="XForm attachment missing: {}".format(form_id))
        return doc

    @staticmethod
    def get_attachment_content(form_id, attachment_id):
        return _get_attachment_content(XFormInstance, form_id, attachment_id)

    @staticmethod
    def save_new_form(form):
        form.save()

    @staticmethod
    def update_form_problem_and_state(form):
        form.save()

    @staticmethod
    def get_deleted_form_ids_for_user(domain, user_id):
        return get_deleted_form_ids_for_user(user_id)

    @staticmethod
    def get_form_ids_for_user(domain, user_id):
        return get_form_ids_for_user(domain, user_id)

    @staticmethod
    def soft_delete_forms(domain, form_ids, deletion_date=None, deletion_id=None):
        def _form_delete(doc):
            doc['server_modified_on'] = json_format_datetime(datetime.utcnow())
        return _soft_delete(XFormInstance.get_db(), form_ids, deletion_date, deletion_id, _form_delete)

    @staticmethod
    def soft_undelete_forms(domain, form_ids):
        def _form_undelete(doc):
            doc['server_modified_on'] = json_format_datetime(datetime.utcnow())
        return _soft_undelete(XFormInstance.get_db(), form_ids, _form_undelete)

    @staticmethod
    def modify_attachment_xml_and_metadata(form_data, form_attachment_new_xml):
        form_data.put_attachment(form_attachment_new_xml, name="form.xml", content_type='text/xml')
        operation = XFormOperation(user_id=SYSTEM_USER_ID, date=datetime.utcnow(),
                                   operation='gdpr_scrub')
        form_data.history.append(operation)  # TODO: should this show in Form History tab? it doesn't
        form_data.save()

    @staticmethod
    def iter_form_ids_by_xmlns(domain, xmlns=None):
        return get_form_ids_by_xmlns(domain, xmlns)

class CaseAccessorCouch(AbstractCaseAccessor):

    @staticmethod
    def get_case(case_id):
        return CommCareCase.get(case_id)

    @staticmethod
    def get_cases(case_ids, ordered=False, prefetched_indices=None):
        # prefetched_indices is ignored sinces cases already have them
        return [
            CommCareCase.wrap(doc) for doc in iter_docs(
                CommCareCase.get_db(),
                case_ids
            )
        ]

    @staticmethod
    def get_related_indices(domain, case_ids, exclude_indices):
        return get_related_indices(domain, case_ids, exclude_indices)

    @staticmethod
    def get_closed_and_deleted_ids(domain, case_ids):
        """Get the subset of given list of case ids that are closed or deleted

        WARNING this is inefficient (better version in SQL).
        """
        return [(case.case_id, case.closed, case.is_deleted)
            for case in iter_cases(case_ids)
            if case.domain == domain and (case.closed or case.is_deleted)]

    @staticmethod
    def get_modified_case_ids(accessor, case_ids, sync_log):
        """Get the subset of given list of case ids that have been modified
        since sync date/log id

        WARNING this is inefficient (better version in SQL).
        """
        return [case.case_id
            for case in accessor.iter_cases(case_ids)
            if not case.is_deleted and case.modified_since_sync(sync_log)]

    @staticmethod
    def case_exists(case_id):
        return CommCareCase.get_db().doc_exist(case_id)

    @staticmethod
    def get_case_xform_ids(case_id):
        return get_case_xform_ids(case_id)

    @staticmethod
    def get_case_ids_in_domain(domain, type=None):
        return get_case_ids_in_domain(domain, type=type)

    @staticmethod
    def get_case_ids_in_domain_by_owners(domain, owner_ids, closed=None):
        return get_case_ids_in_domain_by_owner(domain, owner_id__in=owner_ids, closed=closed)

    @staticmethod
    def get_open_case_ids_for_owner(domain, owner_id):
        return get_open_case_ids(domain, owner_id)

    @staticmethod
    def get_closed_case_ids_for_owner(domain, owner_id):
        return get_closed_case_ids(domain, owner_id)

    @staticmethod
    def get_open_case_ids_in_domain_by_type(domain, case_type, owner_ids=None):
        owner_ids = owner_ids if owner_ids else [None]
        return [
            case_id
            for owner_id in owner_ids
            for case_id in get_open_case_ids_in_domain(domain, type=case_type, owner_id=owner_id)
        ]

    @staticmethod
    def get_case_ids_modified_with_owner_since(domain, owner_id, reference_date):
        return get_case_ids_modified_with_owner_since(domain, owner_id, reference_date)

    @staticmethod
    def get_extension_case_ids(domain, case_ids, include_closed=True):
        # include_closed ignored for couch
        return get_extension_case_ids(domain, case_ids)

    @staticmethod
    def get_indexed_case_ids(domain, case_ids):
        return get_indexed_case_ids(domain, case_ids)

    @staticmethod
    def get_reverse_indexed_cases(domain, case_ids, case_types=None, is_closed=None):
        return [case for case in get_reverse_indexed_cases(domain, case_ids)
                if (not case_types or case.type in case_types)
                and (is_closed is None or case.closed == is_closed)]

    @staticmethod
    def get_last_modified_dates(domain, case_ids):
        return get_last_modified_dates(domain, case_ids)

    @staticmethod
    def get_all_reverse_indices_info(domain, case_ids):
        return get_all_reverse_indices_info(domain, case_ids)

    @staticmethod
    def get_attachment_content(case_id, attachment_id):
        return _get_attachment_content(CommCareCase, case_id, attachment_id)

    @staticmethod
    def get_case_by_domain_hq_user_id(domain, user_id, case_type):
        return get_case_by_domain_hq_user_id(domain, user_id, case_type)

    @staticmethod
    def get_cases_by_external_id(domain, external_id, case_type=None):
        cases = get_cases_in_domain_by_external_id(domain, external_id)
        if case_type:
            return [case for case in cases if case.type == case_type]
        return cases

    @staticmethod
    def soft_delete_cases(domain, case_ids, deletion_date=None, deletion_id=None):
        return _soft_delete(CommCareCase.get_db(), case_ids, deletion_date, deletion_id)

    @staticmethod
    def soft_undelete_cases(domain, case_ids):
        return _soft_undelete(CommCareCase.get_db(), case_ids)

    @staticmethod
    def get_deleted_case_ids_by_owner(domain, owner_id):
        return get_deleted_case_ids_by_owner(owner_id)

    @staticmethod
    def get_case_owner_ids(domain):
        return get_all_case_owner_ids(domain)

class LedgerAccessorCouch(AbstractLedgerAccessor):

    @staticmethod
    def get_transactions_for_consumption(domain, case_id, product_id, section_id, window_start, window_end):
        from casexml.apps.stock.models import StockTransaction
        db_transactions = StockTransaction.objects.filter(
            case_id=case_id, product_id=product_id,
            report__date__gt=window_start,
            report__date__lte=window_end,
            section_id=section_id,
        ).order_by('report__date', 'pk')

        first = True
        for db_tx in db_transactions:
            # for the very first transaction, include the previous one if there as well
            # to capture the data on the edge of the window
            if first:
                previous = db_tx.get_previous_transaction()
                if previous:
                    yield previous
                first = False

            yield db_tx

    @staticmethod
    def get_ledger_value(case_id, section_id, entry_id):
        try:
            return StockState.objects.get(case_id=case_id, section_id=section_id, product_id=entry_id)
        except StockState.DoesNotExist:
            raise LedgerValueNotFound

    @staticmethod
    def get_ledger_transactions_for_case(case_id, section_id=None, entry_id=None):
        query = StockTransaction.objects.filter(case_id=case_id)
        if entry_id:
            query = query.filter(product_id=entry_id)

        if section_id:
            query.filter(section_id=section_id)

        return query.order_by('report__date', 'pk')

    @staticmethod
    def get_latest_transaction(case_id, section_id, entry_id):
        return StockTransaction.latest(case_id, section_id, entry_id)

    @staticmethod
    def get_ledger_values_for_case(case_id):
        from corehq.apps.commtrack.models import StockState

        return StockState.objects.filter(case_id=case_id)

    @staticmethod
    def get_current_ledger_state(case_ids, ensure_form_id=False):
        from casexml.apps.stock.utils import get_current_ledger_state
        return get_current_ledger_state(case_ids, ensure_form_id=ensure_form_id)

def _get_attachment_content(doc_class, doc_id, attachment_id):
    try:
        doc = doc_class.get(doc_id)
        resp = doc.fetch_attachment(attachment_id, stream=True)
        content_type = doc.blobs[attachment_id].content_type
    except ResourceNotFound:
        raise AttachmentNotFound(attachment_id)

    return AttachmentContent(content_type, resp)

def _soft_delete(db, doc_ids, deletion_date=None, deletion_id=None, custom_delete=None):
    from dimagi.utils.couch.undo import DELETED_SUFFIX
    deletion_date = json_format_datetime(deletion_date or datetime.utcnow())

    def delete(doc):
        doc['doc_type'] += DELETED_SUFFIX
        doc['-deletion_id'] = deletion_id
        doc['-deletion_date'] = deletion_date

        if custom_delete:
            custom_delete(doc)

        return doc

    return _operate_on_docs(db, doc_ids, delete)

def _soft_undelete(db, doc_ids, custom_undelete=None):
    from dimagi.utils.couch.undo import DELETED_SUFFIX

    def undelete(doc):

        doc_type = doc['doc_type']
        if doc_type.endswith(DELETED_SUFFIX):
            doc['doc_type'] = doc_type[:-len(DELETED_SUFFIX)]

        if '-deletion_id' in doc:
            del doc['-deletion_id']
        if '-deletion_date' in doc:
            del doc['-deletion_date']

        if custom_undelete:
            custom_undelete(doc)
        return doc

    return _operate_on_docs(db, doc_ids, undelete)

def _operate_on_docs(db, doc_ids, operation_fn):
    docs_to_save = []
    for doc in iter_docs(db, doc_ids):
        doc = operation_fn(doc)
        docs_to_save.append(doc)

        if len(docs_to_save) % 1000 == 0:
            db.bulk_save(docs_to_save)
            docs_to_save = []

    if docs_to_save:
        db.bulk_save(docs_to_save)

    return len(doc_ids)
<commit_msg>Delete outdated TODO<commit_after>from __future__ import absolute_import
from __future__ import unicode_literals
from couchdbkit.exceptions import ResourceNotFound
from datetime import datetime

from casexml.apps.case.dbaccessors import (
    get_extension_case_ids,
    get_indexed_case_ids,
    get_all_reverse_indices_info,
    get_open_case_ids_in_domain,
    get_reverse_indexed_cases,
    get_related_indices,
)
from casexml.apps.case.models import CommCareCase
from casexml.apps.case.util import get_case_xform_ids, iter_cases
from casexml.apps.stock.models import StockTransaction
from corehq.apps.users.util import SYSTEM_USER_ID
from corehq.apps.commtrack.models import StockState
from corehq.apps.hqcase.dbaccessors import (
    get_case_ids_in_domain,
    get_open_case_ids,
    get_closed_case_ids,
    get_case_ids_in_domain_by_owner,
    get_cases_in_domain_by_external_id,
    get_deleted_case_ids_by_owner,
    get_all_case_owner_ids)
from corehq.apps.hqcase.utils import get_case_by_domain_hq_user_id
from corehq.dbaccessors.couchapps.cases_by_server_date.by_owner_server_modified_on import \
    get_case_ids_modified_with_owner_since
from corehq.dbaccessors.couchapps.cases_by_server_date.by_server_modified_on import \
    get_last_modified_dates
from corehq.form_processor.exceptions import AttachmentNotFound, LedgerValueNotFound
from corehq.form_processor.interfaces.dbaccessors import (
    AbstractCaseAccessor, AbstractFormAccessor, AttachmentContent,
    AbstractLedgerAccessor)
from couchforms.dbaccessors import (
    get_forms_by_type,
    get_deleted_form_ids_for_user,
    get_form_ids_for_user,
    get_forms_by_id,
    get_form_ids_by_type,
    get_form_ids_by_xmlns,
)
from couchforms.models import XFormInstance, doc_types, XFormOperation
from dimagi.utils.couch.database import iter_docs
from dimagi.utils.parsing import json_format_datetime
import six

class FormAccessorCouch(AbstractFormAccessor):

    @staticmethod
    def form_exists(form_id, domain=None):
        if not domain:
            return XFormInstance.get_db().doc_exist(form_id)
        else:
            try:
                xform = XFormInstance.get(form_id)
            except ResourceNotFound:
                return False

            return xform.domain == domain

    @staticmethod
    def get_form(form_id):
        return XFormInstance.get(form_id)

    @staticmethod
    def get_forms(form_ids, ordered=False):
        return get_forms_by_id(form_ids)

    @staticmethod
    def get_form_ids_in_domain_by_type(domain, type_):
        return get_form_ids_by_type(domain, type_)

    @staticmethod
    def get_forms_by_type(domain, type_, limit, recent_first=False):
        return get_forms_by_type(domain, type_, recent_first, limit)

    @staticmethod
    def get_with_attachments(form_id):
        doc = XFormInstance.get_db().get(form_id, attachments=True)
        doc = doc_types()[doc['doc_type']].wrap(doc)
        if doc.external_blobs:
            for name, meta in six.iteritems(doc.external_blobs):
                with doc.fetch_attachment(name, stream=True) as content:
                    doc.deferred_put_attachment(
                        content,
                        name,
                        content_type=meta.content_type,
                        content_length=meta.content_length,
                    )
        else:
            # xforms are expected to at least have the XML attachment
            raise ResourceNotFound(msg="XForm attachment missing: {}".format(form_id))
        return doc

    @staticmethod
    def get_attachment_content(form_id, attachment_id):
        return _get_attachment_content(XFormInstance, form_id, attachment_id)

    @staticmethod
    def save_new_form(form):
        form.save()

    @staticmethod
    def update_form_problem_and_state(form):
        form.save()

    @staticmethod
    def get_deleted_form_ids_for_user(domain, user_id):
        return get_deleted_form_ids_for_user(user_id)

    @staticmethod
    def get_form_ids_for_user(domain, user_id):
        return get_form_ids_for_user(domain, user_id)

    @staticmethod
    def soft_delete_forms(domain, form_ids, deletion_date=None, deletion_id=None):
        def _form_delete(doc):
            doc['server_modified_on'] = json_format_datetime(datetime.utcnow())
        return _soft_delete(XFormInstance.get_db(), form_ids, deletion_date, deletion_id, _form_delete)

    @staticmethod
    def soft_undelete_forms(domain, form_ids):
        def _form_undelete(doc):
            doc['server_modified_on'] = json_format_datetime(datetime.utcnow())
        return _soft_undelete(XFormInstance.get_db(), form_ids, _form_undelete)

    @staticmethod
    def modify_attachment_xml_and_metadata(form_data, form_attachment_new_xml):
        form_data.put_attachment(form_attachment_new_xml, name="form.xml", content_type='text/xml')
        operation = XFormOperation(user_id=SYSTEM_USER_ID, date=datetime.utcnow(),
                                   operation='gdpr_scrub')
        form_data.history.append(operation)
        form_data.save()

    @staticmethod
    def iter_form_ids_by_xmlns(domain, xmlns=None):
        return get_form_ids_by_xmlns(domain, xmlns)

class CaseAccessorCouch(AbstractCaseAccessor):

    @staticmethod
    def get_case(case_id):
        return CommCareCase.get(case_id)

    @staticmethod
    def get_cases(case_ids, ordered=False, prefetched_indices=None):
        # prefetched_indices is ignored sinces cases already have them
        return [
            CommCareCase.wrap(doc) for doc in iter_docs(
                CommCareCase.get_db(),
                case_ids
            )
        ]

    @staticmethod
    def get_related_indices(domain, case_ids, exclude_indices):
        return get_related_indices(domain, case_ids, exclude_indices)

    @staticmethod
    def get_closed_and_deleted_ids(domain, case_ids):
        """Get the subset of given list of case ids that are closed or deleted

        WARNING this is inefficient (better version in SQL).
        """
        return [(case.case_id, case.closed, case.is_deleted)
            for case in iter_cases(case_ids)
            if case.domain == domain and (case.closed or case.is_deleted)]

    @staticmethod
    def get_modified_case_ids(accessor, case_ids, sync_log):
        """Get the subset of given list of case ids that have been modified
        since sync date/log id

        WARNING this is inefficient (better version in SQL).
        """
        return [case.case_id
            for case in accessor.iter_cases(case_ids)
            if not case.is_deleted and case.modified_since_sync(sync_log)]

    @staticmethod
    def case_exists(case_id):
        return CommCareCase.get_db().doc_exist(case_id)

    @staticmethod
    def get_case_xform_ids(case_id):
        return get_case_xform_ids(case_id)

    @staticmethod
    def get_case_ids_in_domain(domain, type=None):
        return get_case_ids_in_domain(domain, type=type)

    @staticmethod
    def get_case_ids_in_domain_by_owners(domain, owner_ids, closed=None):
        return get_case_ids_in_domain_by_owner(domain, owner_id__in=owner_ids, closed=closed)

    @staticmethod
    def get_open_case_ids_for_owner(domain, owner_id):
        return get_open_case_ids(domain, owner_id)

    @staticmethod
    def get_closed_case_ids_for_owner(domain, owner_id):
        return get_closed_case_ids(domain, owner_id)

    @staticmethod
    def get_open_case_ids_in_domain_by_type(domain, case_type, owner_ids=None):
        owner_ids = owner_ids if owner_ids else [None]
        return [
            case_id
            for owner_id in owner_ids
            for case_id in get_open_case_ids_in_domain(domain, type=case_type, owner_id=owner_id)
        ]

    @staticmethod
    def get_case_ids_modified_with_owner_since(domain, owner_id, reference_date):
        return get_case_ids_modified_with_owner_since(domain, owner_id, reference_date)

    @staticmethod
    def get_extension_case_ids(domain, case_ids, include_closed=True):
        # include_closed ignored for couch
        return get_extension_case_ids(domain, case_ids)

    @staticmethod
    def get_indexed_case_ids(domain, case_ids):
        return get_indexed_case_ids(domain, case_ids)

    @staticmethod
    def get_reverse_indexed_cases(domain, case_ids, case_types=None, is_closed=None):
        return [case for case in get_reverse_indexed_cases(domain, case_ids)
                if (not case_types or case.type in case_types)
                and (is_closed is None or case.closed == is_closed)]

    @staticmethod
    def get_last_modified_dates(domain, case_ids):
        return get_last_modified_dates(domain, case_ids)

    @staticmethod
    def get_all_reverse_indices_info(domain, case_ids):
        return get_all_reverse_indices_info(domain, case_ids)

    @staticmethod
    def get_attachment_content(case_id, attachment_id):
        return _get_attachment_content(CommCareCase, case_id, attachment_id)

    @staticmethod
    def get_case_by_domain_hq_user_id(domain, user_id, case_type):
        return get_case_by_domain_hq_user_id(domain, user_id, case_type)

    @staticmethod
    def get_cases_by_external_id(domain, external_id, case_type=None):
        cases = get_cases_in_domain_by_external_id(domain, external_id)
        if case_type:
            return [case for case in cases if case.type == case_type]
        return cases

    @staticmethod
    def soft_delete_cases(domain, case_ids, deletion_date=None, deletion_id=None):
        return _soft_delete(CommCareCase.get_db(), case_ids, deletion_date, deletion_id)

    @staticmethod
    def soft_undelete_cases(domain, case_ids):
        return _soft_undelete(CommCareCase.get_db(), case_ids)

    @staticmethod
    def get_deleted_case_ids_by_owner(domain, owner_id):
        return get_deleted_case_ids_by_owner(owner_id)

    @staticmethod
    def get_case_owner_ids(domain):
        return get_all_case_owner_ids(domain)

class LedgerAccessorCouch(AbstractLedgerAccessor):

    @staticmethod
    def get_transactions_for_consumption(domain, case_id, product_id, section_id, window_start, window_end):
        from casexml.apps.stock.models import StockTransaction
        db_transactions = StockTransaction.objects.filter(
            case_id=case_id, product_id=product_id,
            report__date__gt=window_start,
            report__date__lte=window_end,
            section_id=section_id,
        ).order_by('report__date', 'pk')

        first = True
        for db_tx in db_transactions:
            # for the very first transaction, include the previous one if there as well
            # to capture the data on the edge of the window
            if first:
                previous = db_tx.get_previous_transaction()
                if previous:
                    yield previous
                first = False

            yield db_tx

    @staticmethod
    def get_ledger_value(case_id, section_id, entry_id):
        try:
            return StockState.objects.get(case_id=case_id, section_id=section_id, product_id=entry_id)
        except StockState.DoesNotExist:
            raise LedgerValueNotFound

    @staticmethod
    def get_ledger_transactions_for_case(case_id, section_id=None, entry_id=None):
        query = StockTransaction.objects.filter(case_id=case_id)
        if entry_id:
            query = query.filter(product_id=entry_id)

        if section_id:
            query.filter(section_id=section_id)

        return query.order_by('report__date', 'pk')

    @staticmethod
    def get_latest_transaction(case_id, section_id, entry_id):
        return StockTransaction.latest(case_id, section_id, entry_id)

    @staticmethod
    def get_ledger_values_for_case(case_id):
        from corehq.apps.commtrack.models import StockState

        return StockState.objects.filter(case_id=case_id)

    @staticmethod
    def get_current_ledger_state(case_ids, ensure_form_id=False):
        from casexml.apps.stock.utils import get_current_ledger_state
        return get_current_ledger_state(case_ids, ensure_form_id=ensure_form_id)

def _get_attachment_content(doc_class, doc_id, attachment_id):
    try:
        doc = doc_class.get(doc_id)
        resp = doc.fetch_attachment(attachment_id, stream=True)
        content_type = doc.blobs[attachment_id].content_type
    except ResourceNotFound:
        raise AttachmentNotFound(attachment_id)

    return AttachmentContent(content_type, resp)

def _soft_delete(db, doc_ids, deletion_date=None, deletion_id=None, custom_delete=None):
    from dimagi.utils.couch.undo import DELETED_SUFFIX
    deletion_date = json_format_datetime(deletion_date or datetime.utcnow())

    def delete(doc):
        doc['doc_type'] += DELETED_SUFFIX
        doc['-deletion_id'] = deletion_id
        doc['-deletion_date'] = deletion_date

        if custom_delete:
            custom_delete(doc)

        return doc

    return _operate_on_docs(db, doc_ids, delete)

def _soft_undelete(db, doc_ids, custom_undelete=None):
    from dimagi.utils.couch.undo import DELETED_SUFFIX

    def undelete(doc):

        doc_type = doc['doc_type']
        if doc_type.endswith(DELETED_SUFFIX):
            doc['doc_type'] = doc_type[:-len(DELETED_SUFFIX)]

        if '-deletion_id' in doc:
            del doc['-deletion_id']
        if '-deletion_date' in doc:
            del doc['-deletion_date']

        if custom_undelete:
            custom_undelete(doc)
        return doc

    return _operate_on_docs(db, doc_ids, undelete)

def _operate_on_docs(db, doc_ids, operation_fn):
    docs_to_save = []
    for doc in iter_docs(db, doc_ids):
        doc = operation_fn(doc)
        docs_to_save.append(doc)

        if len(docs_to_save) % 1000 == 0:
            db.bulk_save(docs_to_save)
            docs_to_save = []

    if docs_to_save:
        db.bulk_save(docs_to_save)

    return len(doc_ids)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# SymPy documentation build configuration file, created by
# sphinx-quickstart.py on Sat Mar 22 19:34:32 2008.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# The contents of this file are pickled, so don't put values in the namespace
# that aren't pickleable (module imports are okay, they're removed automatically).
#
# All configuration values have a default value; values that are commented out
# serve to show the default value.

import sys

# If your extensions are in another directory, add it here.
sys.path.extend(['../sympy', 'ext'])

# General configuration
# ---------------------

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.addons.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'mathjax']

# MathJax file, which is free to use.  See https://bitbucket.org/kevindunn/sphinx-extension-mathjax/wiki/Home
mathjax_path = 'http://mathjax.connectmv.com/MathJax.js'

# Add any paths that contain templates here, relative to this directory.
templates_path = ['.templates']

# The suffix of source filenames.
source_suffix = '.txt'

# The master toctree document.
master_doc = 'index'

# General substitutions.
project = 'SymPy'
copyright = '2008, 2009, 2010, 2011 SymPy Development Team'

# The default replacements for |version| and |release|, also used in various
# other places throughout the built documents.
#
# The short X.Y version.
version = '0.7.0'
# The full version, including alpha/beta/rc tags.
release = '0.7.0-git'

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
today_fmt = '%B %d, %Y'

# List of documents that shouldn't be included in the build.
#unused_docs = []

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# Options for HTML output
# -----------------------

# The style sheet to use for HTML and HTML Help pages. A file of that name
# must exist either in Sphinx' static/ path, or in one of the custom paths
# given in html_static_path.
html_style = 'default.css'

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
html_last_updated_fmt = '%b %d, %Y'

html_logo = '_static/sympylogo.png'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Content template for the index page.
#html_index = ''

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_use_modindex = True

# If true, the reST sources are included in the HTML build as _sources/<name>.
#html_copy_source = True

# Output file base name for HTML help builder.
htmlhelp_basename = 'SymPydoc'

# Options for LaTeX output
# ------------------------

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, document class [howto/manual]).
latex_documents = [('index', 'sympy.tex', 'SymPy Documentation',
                        'SymPy Development Team', 'manual')]

# Additional stuff for the LaTeX preamble.
#latex_preamble = ''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_use_modindex = True

default_role = 'math'
pngmath_divpng_args = ['-gamma 1.5','-D 110']
pngmath_latex_preamble =  '\\usepackage{amsmath}\n'+\
              '\\usepackage{bm}\n'+\
              '\\usepackage{amsfonts}\n'+\
              '\\usepackage{amssymb}\n'+\
              '\\setlength{\\parindent}{0pt}\n'+\
              '\\newcommand{\\bfrac}[2]{\\displaystyle\\frac{#1}{#2}}\n'+\
              '\\newcommand{\\lp}{\\left (}\n'+\
              '\\newcommand{\\rp}{\\right )}\n'+\
              '\\newcommand{\\half}{\\frac{1}{2}}\n'+\
              '\\newcommand{\\llt}{\\left <}\n'+\
              '\\newcommand{\\rgt}{\\right >}\n'+\
              '\\newcommand{\\abs}[1]{\\left |{#1}\\right | }\n'+\
              '\\newcommand{\\pdiff}[2]{\\bfrac{\\partial {#1}}{\\partial {#2}}}\n'+\
              '\\newcommand{\\lbrc}{\\left \\{}\n'+\
              '\\newcommand{\\rbrc}{\\right \\}}\n'+\
              '\\newcommand{\\W}{\\wedge}\n'+\
              '\\newcommand{\\R}{\\dagger}\n'+\
              '\\newcommand{\\lbrk}{\\left [}\n'+\
              '\\newcommand{\\rbrk}{\\right ]}\n'+\
              '\\newcommand{\\proj}[2]{\\llt {#1} \\rgt_{#2}}\n'+\
              '\\newcommand{\\bs}{$\\backslash$}\n'+\
              '\\newcommand{\\sinf}[1]{\\sin\\lp{#1}\\rp}\n'+\
              '\\newcommand{\\cosf}[1]{\\cos\\lp{#1}\\rp}\n'+\
              '\\newcommand{\\ebh}{\\hat{\\bm{e}}}\n'

<commit_msg>Update the MathJax cdn to the one recommended at mathjax.org<commit_after># -*- coding: utf-8 -*-
#
# SymPy documentation build configuration file, created by
# sphinx-quickstart.py on Sat Mar 22 19:34:32 2008.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# The contents of this file are pickled, so don't put values in the namespace
# that aren't pickleable (module imports are okay, they're removed automatically).
#
# All configuration values have a default value; values that are commented out
# serve to show the default value.

import sys

# If your extensions are in another directory, add it here.
sys.path.extend(['../sympy', 'ext'])

# General configuration
# ---------------------

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.addons.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'mathjax']

# MathJax file, which is free to use.  See https://bitbucket.org/kevindunn/sphinx-extension-mathjax/wiki/Home
mathjax_path = 'http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML-full'

# Add any paths that contain templates here, relative to this directory.
templates_path = ['.templates']

# The suffix of source filenames.
source_suffix = '.txt'

# The master toctree document.
master_doc = 'index'

# General substitutions.
project = 'SymPy'
copyright = '2008, 2009, 2010, 2011 SymPy Development Team'

# The default replacements for |version| and |release|, also used in various
# other places throughout the built documents.
#
# The short X.Y version.
version = '0.7.0'
# The full version, including alpha/beta/rc tags.
release = '0.7.0-git'

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
today_fmt = '%B %d, %Y'

# List of documents that shouldn't be included in the build.
#unused_docs = []

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# Options for HTML output
# -----------------------

# The style sheet to use for HTML and HTML Help pages. A file of that name
# must exist either in Sphinx' static/ path, or in one of the custom paths
# given in html_static_path.
html_style = 'default.css'

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
html_last_updated_fmt = '%b %d, %Y'

html_logo = '_static/sympylogo.png'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Content template for the index page.
#html_index = ''

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_use_modindex = True

# If true, the reST sources are included in the HTML build as _sources/<name>.
#html_copy_source = True

# Output file base name for HTML help builder.
htmlhelp_basename = 'SymPydoc'

# Options for LaTeX output
# ------------------------

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, document class [howto/manual]).
latex_documents = [('index', 'sympy.tex', 'SymPy Documentation',
                        'SymPy Development Team', 'manual')]

# Additional stuff for the LaTeX preamble.
#latex_preamble = ''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_use_modindex = True

default_role = 'math'
pngmath_divpng_args = ['-gamma 1.5','-D 110']
pngmath_latex_preamble =  '\\usepackage{amsmath}\n'+\
              '\\usepackage{bm}\n'+\
              '\\usepackage{amsfonts}\n'+\
              '\\usepackage{amssymb}\n'+\
              '\\setlength{\\parindent}{0pt}\n'+\
              '\\newcommand{\\bfrac}[2]{\\displaystyle\\frac{#1}{#2}}\n'+\
              '\\newcommand{\\lp}{\\left (}\n'+\
              '\\newcommand{\\rp}{\\right )}\n'+\
              '\\newcommand{\\half}{\\frac{1}{2}}\n'+\
              '\\newcommand{\\llt}{\\left <}\n'+\
              '\\newcommand{\\rgt}{\\right >}\n'+\
              '\\newcommand{\\abs}[1]{\\left |{#1}\\right | }\n'+\
              '\\newcommand{\\pdiff}[2]{\\bfrac{\\partial {#1}}{\\partial {#2}}}\n'+\
              '\\newcommand{\\lbrc}{\\left \\{}\n'+\
              '\\newcommand{\\rbrc}{\\right \\}}\n'+\
              '\\newcommand{\\W}{\\wedge}\n'+\
              '\\newcommand{\\R}{\\dagger}\n'+\
              '\\newcommand{\\lbrk}{\\left [}\n'+\
              '\\newcommand{\\rbrk}{\\right ]}\n'+\
              '\\newcommand{\\proj}[2]{\\llt {#1} \\rgt_{#2}}\n'+\
              '\\newcommand{\\bs}{$\\backslash$}\n'+\
              '\\newcommand{\\sinf}[1]{\\sin\\lp{#1}\\rp}\n'+\
              '\\newcommand{\\cosf}[1]{\\cos\\lp{#1}\\rp}\n'+\
              '\\newcommand{\\ebh}{\\hat{\\bm{e}}}\n'

<|endoftext|>


<commit_before>from robot.libraries.BuiltIn import BuiltIn
import re

class BasePage(object):
    _object_name = None

    def __init__(self, object_name=None):
        if object_name:
            self._object_name = object_name

    @property
    def object_name(self):
        object_name = self._object_name

        # the length check is to skip objects from a different namespace
        # like foobar__otherpackageobject__c
        if object_name is not None:
            parts = object_name.split("__")
            if len(parts) == 2 and parts[-1] == "c":
                # get_namespace_prefix already takes care of returning an actual
                # prefix or an empty string depending on whether the package is managed
                object_name = "{}{}".format(
                    self.cumulusci.get_namespace_prefix(), object_name
                )
        return object_name

    @property
    def builtin(self):
        """Returns an instance of robot framework's BuiltIn library"""
        return BuiltIn()

    @property
    def cumulusci(self):
        """Returns the instance of the imported CumulusCI library"""
        return self.builtin.get_library_instance("cumulusci.robotframework.CumulusCI")

    @property
    def salesforce(self):
        """Returns the instance of the imported Salesforce library"""
        return self.builtin.get_library_instance("cumulusci.robotframework.Salesforce")

    @property
    def selenium(self):
        """Returns the instance of the imported SeleniumLibrary library"""
        return self.builtin.get_library_instance("SeleniumLibrary")

    def _get_object(self, **kwargs):
        """Get the object associated with the given keyword arguments.

        This performs a salesforce query. It will raise an exception unless
        exactly one result is returned from the query.
        """
        results = self.salesforce.salesforce_query(self.object_name, **kwargs)
        if len(results) == 0:
            human_friendly_args = ", ".join(
                ["{}={}".format(key, kwargs[key]) for key in kwargs]
            )
            raise Exception(
                "no {} matches {}".format(self.object_name, human_friendly_args)
            )
        elif len(results) > 1:
            raise Exception("Query returned {} objects".format(len(results)))
        else:
            return results[0]

class ListingPage(BasePage):
    def _go_to_page(self, filter_name=None):
        url_template = "{root}/lightning/o/{object_name}/list"
        url = url_template.format(
            root=self.cumulusci.org.lightning_base_url, object_name=self.object_name
        )
        if filter_name:
            url += "?filterName={}".format(filter_name)
        self.selenium.go_to(url)
        self.salesforce.wait_until_loading_is_complete()

    def _is_current_page(self):
        self.selenium.location_should_contain(
            "/lightning/o/{}/list".format(self.object_name)
        )

class HomePage(BasePage):
    def _go_to_page(self, filter_name=None):
        url_template = "{root}/lightning/o/{object_name}/home"
        url = url_template.format(
            root=self.cumulusci.org.lightning_base_url, object_name=self.object_name
        )
        self.selenium.go_to(url)
        self.salesforce.wait_until_loading_is_complete()

    def _is_current_page(self):
        self.selenium.location_should_contain(
            "/lightning/o/{}/home".format(self.object_name)
        )

class DetailPage(BasePage):
    def _go_to_page(self, object_id=None, **kwargs):
        """Go to the detail page for the given object.

        You may pass in an object id, or you may pass in keyword arguments
        which can be used to look up the object.
        """

        if kwargs and object_id:
            raise Exception("Specify an object id or keyword arguments, but not both")

        if kwargs:
            # note: this will raise an exception if no object is found,
            # or if multiple objects are found.
            object_id = self._get_object(**kwargs)["Id"]

        url_template = "{root}/lightning/r/{object_name}/{object_id}/view"
        url = url_template.format(
            root=self.cumulusci.org.lightning_base_url,
            object_name=self.object_name,
            object_id=object_id,
        )
        self.selenium.go_to(url)
        self.salesforce.wait_until_loading_is_complete()

    def _is_current_page(self, **kwargs):
        """Verify we are on a detail page.

        If keyword arguments are present, this function will go a query
        on the given parameters, assert that the query returns a single
        result, and the verify that the returned object id is part of the url.
        """
        if kwargs:
            # do a lookup to get the object i
            object_id = self._get_object(**kwargs)["Id"]
            pattern = r"/lightning/r/{}/{}/view$".format(self.object_name, object_id)
        else:
            # no kwargs means we should just verify we are on a detail
            # page without regard to which object
            pattern = r"/lightning/r/{}/.*/view$".format(self.object_name)

        location = self.selenium.get_location()
        if not re.search(pattern, location):
            raise Exception(
                "Location '{}' didn't match pattern {}".format(location, pattern)
            )
<commit_msg>Added the keyword log_current_page object to the base keyword This is implemented in the base class rather than in PageObjects or Salesforce since it is only useful if you are using page objects in your test.<commit_after>from robot.libraries.BuiltIn import BuiltIn
import re

class BasePage(object):
    _object_name = None

    def __init__(self, object_name=None):
        if object_name:
            self._object_name = object_name

    @property
    def object_name(self):
        object_name = self._object_name

        # the length check is to skip objects from a different namespace
        # like foobar__otherpackageobject__c
        if object_name is not None:
            parts = object_name.split("__")
            if len(parts) == 2 and parts[-1] == "c":
                # get_namespace_prefix already takes care of returning an actual
                # prefix or an empty string depending on whether the package is managed
                object_name = "{}{}".format(
                    self.cumulusci.get_namespace_prefix(), object_name
                )
        return object_name

    @property
    def builtin(self):
        """Returns an instance of robot framework's BuiltIn library"""
        return BuiltIn()

    @property
    def cumulusci(self):
        """Returns the instance of the imported CumulusCI library"""
        return self.builtin.get_library_instance("cumulusci.robotframework.CumulusCI")

    @property
    def salesforce(self):
        """Returns the instance of the imported Salesforce library"""
        return self.builtin.get_library_instance("cumulusci.robotframework.Salesforce")

    @property
    def selenium(self):
        """Returns the instance of the imported SeleniumLibrary library"""
        return self.builtin.get_library_instance("SeleniumLibrary")

    def _get_object(self, **kwargs):
        """Get the object associated with the given keyword arguments.

        This performs a salesforce query. It will raise an exception unless
        exactly one result is returned from the query.
        """
        results = self.salesforce.salesforce_query(self.object_name, **kwargs)
        if len(results) == 0:
            human_friendly_args = ", ".join(
                ["{}={}".format(key, kwargs[key]) for key in kwargs]
            )
            raise Exception(
                "no {} matches {}".format(self.object_name, human_friendly_args)
            )
        elif len(results) > 1:
            raise Exception("Query returned {} objects".format(len(results)))
        else:
            return results[0]

    def log_current_page_object(self):
        """Logs the name of the current page object

        The current page object is also returned as an object
        """
        polib = self.builtin.get_library_instance(
            "cumulusci.robotframework.PageObjects"
        )
        if polib.current_page_object is None:
            # this should not be possible, since this keyword is only available
            # if you've loaded a page object. Still, better safe then sorry.
            self.builtin.log("no page object has been loaded")
            return None
        else:
            pobj = polib.current_page_object

            self.builtin.log(
                "current page object: {}".format(polib.current_page_object)
            )
            return pobj

class ListingPage(BasePage):
    def _go_to_page(self, filter_name=None):
        url_template = "{root}/lightning/o/{object_name}/list"
        url = url_template.format(
            root=self.cumulusci.org.lightning_base_url, object_name=self.object_name
        )
        if filter_name:
            url += "?filterName={}".format(filter_name)
        self.selenium.go_to(url)
        self.salesforce.wait_until_loading_is_complete()

    def _is_current_page(self):
        self.selenium.location_should_contain(
            "/lightning/o/{}/list".format(self.object_name)
        )

class HomePage(BasePage):
    def _go_to_page(self, filter_name=None):
        url_template = "{root}/lightning/o/{object_name}/home"
        url = url_template.format(
            root=self.cumulusci.org.lightning_base_url, object_name=self.object_name
        )
        self.selenium.go_to(url)
        self.salesforce.wait_until_loading_is_complete()

    def _is_current_page(self):
        self.selenium.location_should_contain(
            "/lightning/o/{}/home".format(self.object_name)
        )

class DetailPage(BasePage):
    def _go_to_page(self, object_id=None, **kwargs):
        """Go to the detail page for the given object.

        You may pass in an object id, or you may pass in keyword arguments
        which can be used to look up the object.
        """

        if kwargs and object_id:
            raise Exception("Specify an object id or keyword arguments, but not both")

        if kwargs:
            # note: this will raise an exception if no object is found,
            # or if multiple objects are found.
            object_id = self._get_object(**kwargs)["Id"]

        url_template = "{root}/lightning/r/{object_name}/{object_id}/view"
        url = url_template.format(
            root=self.cumulusci.org.lightning_base_url,
            object_name=self.object_name,
            object_id=object_id,
        )
        self.selenium.go_to(url)
        self.salesforce.wait_until_loading_is_complete()

    def _is_current_page(self, **kwargs):
        """Verify we are on a detail page.

        If keyword arguments are present, this function will go a query
        on the given parameters, assert that the query returns a single
        result, and the verify that the returned object id is part of the url.
        """
        if kwargs:
            # do a lookup to get the object i
            object_id = self._get_object(**kwargs)["Id"]
            pattern = r"/lightning/r/{}/{}/view$".format(self.object_name, object_id)
        else:
            # no kwargs means we should just verify we are on a detail
            # page without regard to which object
            pattern = r"/lightning/r/{}/.*/view$".format(self.object_name)

        location = self.selenium.get_location()
        if not re.search(pattern, location):
            raise Exception(
                "Location '{}' didn't match pattern {}".format(location, pattern)
            )
<|endoftext|>


<commit_before>from fabric.api import task, local
import os
import glob

def build():
    os.chdir('command_line_tools')
    for filename in glob.glob("*.ipynb"):
        basename = os.path.splitext(filename)[0]
        local('ipython nbconvert --config=../ipython_nbconvert_config.py --profile-dir=/tmp --output "{}" "{}"'.format(basename, filename))
    os.chdir('..')

    os.chdir('user_guide')
    for filename in glob.glob("*.ipynb"):
        basename = os.path.splitext(filename)[0]
        local('ipython nbconvert --config=../ipython_nbconvert_config.py --profile-dir=/tmp --output "{}" "{}"'.format(basename, filename))
    os.chdir('..')

@task
def clean():
    local('rm -f user_guide/release_example/StudentNotebook.ipynb')
    local('rm -f user_guide/grade_example/GradedNotebookBitdiddle.ipynb')
    local('rm -f user_guide/grade_example/GradedNotebookBitdiddle.html')

@task(default=True)
def all():
    clean()
    build()
<commit_msg>Update documentation fabfile<commit_after>from fabric.api import task, local
import os
import glob

def build():
    os.chdir('command_line_tools')
    for filename in glob.glob("*.ipynb"):
        basename = os.path.splitext(filename)[0]
        local('ipython nbconvert --config=../ipython_nbconvert_config.py --profile-dir=/tmp --output "{}" "{}"'.format(basename, filename))
    os.chdir('..')

    os.chdir('user_guide')
    for filename in glob.glob("*.ipynb"):
        basename = os.path.splitext(filename)[0]
        local('ipython nbconvert --config=../ipython_nbconvert_config.py --profile-dir=/tmp --output "{}" "{}"'.format(basename, filename))
    os.chdir('..')

@task
def clean():
    local('rm -f user_guide/release_example/StudentNotebook.ipynb')
    local('rm -f user_guide/grade_example/StudentNotebook*.autograded.ipynb')
    local('rm -f user_guide/grade_example/StudentNotebook*.autograded.html')

@task(default=True)
def all():
    clean()
    build()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import pickle
import sys

import cherrypy
from formencode import Invalid
from genshi.filters import HTMLFormFiller
from paste.evalexception.middleware import EvalException

from geddit.form import SubmissionForm, CommentForm
from geddit.lib import template
from geddit.model import Submission, Comment

class Root(object):

    def __init__(self, data):
        self.data = data
        self.submission_lookup = {}
        self.comment_lookup = {}
        for submission in self.data:
            self.submission_lookup[submission.code] = submission
            for comment in submission.comments:
                self.comment_lookup[comment.code] = comment
                def _add_replies(comment):
                    for reply in comment.replies:
                        self.comment_lookup[reply.code] = reply
                _add_replies(comment)

    @cherrypy.expose
    @template.output('index.html', method='html', doctype='html')
    def index(self):
        return template.render(submissions=self.data)

    @cherrypy.expose
    @template.output('info.html', method='html', doctype='html')
    def info(self, code):
        submission = self.submission_lookup.get(code)
        if not submission:
            raise cherrypy.NotFound()
        return template.render(submission=submission)

    @cherrypy.expose
    @template.output('submit.html', method='html', doctype='html')
    def submit(self, cancel=False, **data):
        if cherrypy.request.method == 'POST':
            if cancel:
                raise cherrypy.HTTPRedirect('/')
            form = SubmissionForm()
            try:
                data = form.to_python(data)
                submission = Submission(**data)
                self.data.append(submission)
                raise cherrypy.HTTPRedirect('/')
            except Invalid, e:
                errors = e.unpack_errors()
        else:
            errors = {}

        return template.render(errors=errors) | HTMLFormFiller(data=data)

    @cherrypy.expose
    @template.output('comment.html', method='html', doctype='html')
    def comment(self, code, cancel=False, **data):
        submission = self.submission_lookup.get(code)
        if not submission:
            raise cherrypy.NotFound()
        if cherrypy.request.method == 'POST':
            if cancel:
                raise cherrypy.HTTPRedirect('/info/%s' % submission.code)
            form = CommentForm()
            try:
                data = form.to_python(data)
                comment = submission.add_comment(**data)
                self.comment_lookup[comment.code] = comment
                raise cherrypy.HTTPRedirect('/')
            except Invalid, e:
                errors = e.unpack_errors()
        else:
            errors = {}

        return template.render(submission=submission, comment=None,
                               errors=errors)

    @cherrypy.expose
    @template.output('comment.html', method='html', doctype='html')
    def reply(self, code, cancel=False, **data):
        comment = self.comment_lookup.get(code)
        submission = comment.submission
        if not comment:
            raise cherrypy.NotFound()
        if cherrypy.request.method == 'POST':
            if cancel:
                raise cherrypy.HTTPRedirect('/info/%s' % submission.code)
            form = CommentForm()
            try:
                data = form.to_python(data)
                comment = comment.add_reply(**data)
                self.comment_lookup[comment.code] = comment
                raise cherrypy.HTTPRedirect('/')
            except Invalid, e:
                errors = e.unpack_errors()
        else:
            errors = {}

        return template.render(submission=submission, comment=comment,
                               errors=errors)

def main(filename):
    # load data from the pickle file, or initialize it to an empty list
    if os.path.exists(filename):
        fileobj = open(filename, 'rb')
        try:
            data = pickle.load(fileobj)
        finally:
            fileobj.close()
    else:
        data = []

    def _save_data():
        # save data back to the pickle file
        fileobj = open(filename, 'wb')
        try:
            pickle.dump(data, fileobj)
        finally:
            fileobj.close()
    cherrypy.engine.on_stop_engine_list.append(_save_data)

    # Some global configuration; note that this could be moved into a configuration file
    cherrypy.config.update({
        'request.throw_errors': True,
        'tools.encode.on': True, 'tools.encode.encoding': 'utf-8',
        'tools.decode.on': True,
        'tools.trailing_slash.on': True,
        'tools.staticdir.root': os.path.abspath(os.path.dirname(__file__)),
    })

    # Initialize the application, and add EvalException for more helpful error messages
    app = cherrypy.Application(Root(data))
    app.wsgiapp.pipeline.append(('paste_exc', EvalException))
    cherrypy.quickstart(app, '/', {
        '/media': {
            'tools.staticdir.on': True,
            'tools.staticdir.dir': 'static'
        }
    })

if __name__ == '__main__':
    import formencode
    formencode.api.set_stdtranslation(languages=['en'])
    main(sys.argv[1])
<commit_msg>GenshiTutorial: fix storing of submissions.<commit_after>#!/usr/bin/env python

import os
import pickle
import sys

import cherrypy
from formencode import Invalid
from genshi.filters import HTMLFormFiller
from paste.evalexception.middleware import EvalException

from geddit.form import SubmissionForm, CommentForm
from geddit.lib import template
from geddit.model import Submission, Comment

class Root(object):

    def __init__(self, data):
        self.data = data
        self._submission_lookup = {}
        self._comment_lookup = {}
        for submission in self.data:
            self._submission_lookup[submission.code] = submission
            for comment in submission.comments:
                self._comment_lookup[comment.code] = comment
                def _add_replies(comment):
                    for reply in comment.replies:
                        self._comment_lookup[reply.code] = reply
                _add_replies(comment)

    @cherrypy.expose
    @template.output('index.html', method='html', doctype='html')
    def index(self):
        return template.render(submissions=self.data)

    @cherrypy.expose
    @template.output('info.html', method='html', doctype='html')
    def info(self, code):
        submission = self._submission_lookup.get(code)
        if not submission:
            raise cherrypy.NotFound()
        return template.render(submission=submission)

    @cherrypy.expose
    @template.output('submit.html', method='html', doctype='html')
    def submit(self, cancel=False, **data):
        if cherrypy.request.method == 'POST':
            if cancel:
                raise cherrypy.HTTPRedirect('/')
            form = SubmissionForm()
            try:
                data = form.to_python(data)
                submission = Submission(**data)
                self.data.append(submission)
                self._comment_lookup[comment.code] = comment
                raise cherrypy.HTTPRedirect('/')
            except Invalid, e:
                errors = e.unpack_errors()
        else:
            errors = {}

        return template.render(errors=errors) | HTMLFormFiller(data=data)

    @cherrypy.expose
    @template.output('comment.html', method='html', doctype='html')
    def comment(self, code, cancel=False, **data):
        submission = self._submission_lookup.get(code)
        if not submission:
            raise cherrypy.NotFound()
        if cherrypy.request.method == 'POST':
            if cancel:
                raise cherrypy.HTTPRedirect('/info/%s' % submission.code)
            form = CommentForm()
            try:
                data = form.to_python(data)
                comment = submission.add_comment(**data)
                self._comment_lookup[comment.code] = comment
                raise cherrypy.HTTPRedirect('/')
            except Invalid, e:
                errors = e.unpack_errors()
        else:
            errors = {}

        return template.render(submission=submission, comment=None,
                               errors=errors)

    @cherrypy.expose
    @template.output('comment.html', method='html', doctype='html')
    def reply(self, code, cancel=False, **data):
        comment = self._comment_lookup.get(code)
        submission = comment.submission
        if not comment:
            raise cherrypy.NotFound()
        if cherrypy.request.method == 'POST':
            if cancel:
                raise cherrypy.HTTPRedirect('/info/%s' % submission.code)
            form = CommentForm()
            try:
                data = form.to_python(data)
                comment = comment.add_reply(**data)
                self._comment_lookup[comment.code] = comment
                raise cherrypy.HTTPRedirect('/')
            except Invalid, e:
                errors = e.unpack_errors()
        else:
            errors = {}

        return template.render(submission=submission, comment=comment,
                               errors=errors)

def main(filename):
    # load data from the pickle file, or initialize it to an empty list
    if os.path.exists(filename):
        fileobj = open(filename, 'rb')
        try:
            data = pickle.load(fileobj)
        finally:
            fileobj.close()
    else:
        data = []

    def _save_data():
        # save data back to the pickle file
        fileobj = open(filename, 'wb')
        try:
            pickle.dump(data, fileobj)
        finally:
            fileobj.close()
    cherrypy.engine.on_stop_engine_list.append(_save_data)

    # Some global configuration; note that this could be moved into a configuration file
    cherrypy.config.update({
        'request.throw_errors': True,
        'tools.encode.on': True, 'tools.encode.encoding': 'utf-8',
        'tools.decode.on': True,
        'tools.trailing_slash.on': True,
        'tools.staticdir.root': os.path.abspath(os.path.dirname(__file__)),
    })

    # Initialize the application, and add EvalException for more helpful error messages
    app = cherrypy.Application(Root(data))
    app.wsgiapp.pipeline.append(('paste_exc', EvalException))
    cherrypy.quickstart(app, '/', {
        '/media': {
            'tools.staticdir.on': True,
            'tools.staticdir.dir': 'static'
        }
    })

if __name__ == '__main__':
    import formencode
    formencode.api.set_stdtranslation(languages=['en'])
    main(sys.argv[1])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import schedule
import time
import sys
import os
import random
from glob import glob   # ->added to make pics upload -> see job8
import threading        # ->added to make multithreadening possible -> see fn run_threaded

sys.path.append(os.path.join(sys.path[0], '../../'))
from instabot import Bot

import config

bot = Bot(comments_file=config.COMMENTS_FILE,
          blacklist_file=config.BLACKLIST_FILE,
          whitelist_file=config.WHITELIST_FILE,
          friends_file=config.FRIENDS_FILE)
bot.login()
bot.logger.info("ULTIMATE script. 24hours save")

random_user_file = bot.read_list_from_file(config.USERS_FILE)
random_hashtag_file = bot.read_list_from_file(config.HASHTAGS_FILE)
photo_captions = bot.read_list_from_file(config.PHOTO_CAPTIONS_FILE)

# to get pics and autopost it
posted_pic_list = instabot.utils.file(config.POSTED_PICS_FILE).list

# Get the filenames of the photos in the path ->
pics = [os.path.basename(x) for x in glob(config.PICS_PATH + "/*.jpg")]
pics = sorted(pics)

# Return a random value from a list, used in various jobs below
def get_random(from_list):
    _random = random.choice(from_list)
    return _random

def stats():
    bot.save_user_stats(bot.user_id)

def job1():
    bot.like_hashtag(get_random(random_hashtag_file), amount=int(700 / 24))

def job2():
    bot.like_timeline(amount=int(300 / 24))

def job3():
    bot.like_followers(get_random(random_user_file), nlikes=3)

def job4():
    bot.follow_followers(get_random(random_user_file), nfollows=config.NUMBER_OF_FOLLOWERS_TO_FOLLOW)

def job5():
    bot.comment_medias(bot.get_timeline_medias())

def job6():
    bot.unfollow_non_followers(n_to_unfollows=config.NUMBER_OF_NON_FOLLOWERS_TO_UNFOLLOW)

def job7():
    bot.follow_users(bot.get_hashtag_users(get_random(random_hashtag_file)))

def job8():  # Comment posts with an hashtag in HASHTAGS_FILE
    hashtag = get_random(random_hashtag_file)
    bot.logger.info("Commenting on hashtag: " + hashtag)
    bot.comment_hashtag(hashtag)

def job9():  # Automatically post a pic in 'pics' folder
    try:
        for pic in pics:
            if pic in posted_pic_list:
                continue

            caption = get_random(photo_captions)
            full_caption = caption + "\n" + config.FOLLOW_MESSAGE
            bot.logger.info("Uploading pic with caption: " + caption)
            bot.upload_photo(config.PICS_PATH + pic, caption=full_caption)
            if bot.last_response.status_code != 200:
                bot.logger.error("Something went wrong, read the following ->\n")
                bot.logger.error(bot.last_response)
                break

            if pic not in posted_pic_list:
                # After posting a pic, comment it with all the hashtags specified
                # In config.PICS_HASHTAGS
                posted_pic_list.append(pic)
                with open('pics.txt', 'a') as f:
                    f.write(pic + "\n")
                bot.logger.info("Succesfully uploaded: " + pic)
                bot.logger.info("Commenting uploaded photo with hashtags...")
                medias = bot.get_your_medias()
                last_photo = medias[0]  # Get the last photo posted
                bot.comment(last_photo, config.PICS_HASHTAGS)
                break
    except Exception as e:
        bot.logger.error("Couldn't upload pic")
        bot.logger.error(str(e))

def job10():  # put non followers on blacklist
    try:
        bot.logger.info("Creating non-followers list")
        followings = set(bot.following)  # getting following
        followers = set(bot.followers)  # getting followers
        friends = bot.friends_file.set  # same whitelist (just user ids)
        non_followers = followings - followers - friends
        for user_id in non_followers:
            bot.blacklist_file.append(user_id, allow_duplicates=False)
        bot.logger.info("Done.")
    except Exception as e:
        bot.logger.error("Couldn't update blacklist")
        bot.logger.error(str(e))

# function to make threads -> details here http://bit.ly/faq_schedule
def run_threaded(job_fn):
    job_thread = threading.Thread(target=job_fn)
    job_thread.start()

schedule.every(1).hour.do(run_threaded, stats)              # get stats
schedule.every(8).hours.do(run_threaded, job1)              # like hashtag
schedule.every(2).hours.do(run_threaded, job2)              # like timeline
schedule.every(1).days.at("16:00").do(run_threaded, job3)   # like followers of users from file
schedule.every(2).days.at("11:00").do(run_threaded, job4)   # follow followers
schedule.every(16).hours.do(run_threaded, job5)             # comment medias
schedule.every(1).days.at("08:00").do(run_threaded, job6)   # unfollow non-followers
schedule.every(12).hours.do(run_threaded, job7)             # follow users from hashtag from file
schedule.every(6).hours.do(run_threaded, job8)              # comment hashtag
schedule.every(1).days.at("21:28").do(run_threaded, job9)   # upload pics
schedule.every(4).days.at("07:50").do(run_threaded, job10)  # non-followers blacklist

while True:
    schedule.run_pending()
    time.sleep(1)
<commit_msg>Sort imports and remove unnecessary comments<commit_after># -*- coding: utf-8 -*-

from glob import glob
import os
import random
import sys
import threading
import time

sys.path.append(os.path.join(sys.path[0], '../../'))
import schedule
from instabot import Bot

import config

bot = Bot(comments_file=config.COMMENTS_FILE,
          blacklist_file=config.BLACKLIST_FILE,
          whitelist_file=config.WHITELIST_FILE,
          friends_file=config.FRIENDS_FILE)
bot.login()
bot.logger.info("ULTIMATE script. 24hours save")

random_user_file = bot.read_list_from_file(config.USERS_FILE)
random_hashtag_file = bot.read_list_from_file(config.HASHTAGS_FILE)
photo_captions = bot.read_list_from_file(config.PHOTO_CAPTIONS_FILE)

# to get pics and autopost it
posted_pic_list = instabot.utils.file(config.POSTED_PICS_FILE).list

# Get the filenames of the photos in the path ->
pics = [os.path.basename(x) for x in glob(config.PICS_PATH + "/*.jpg")]
pics = sorted(pics)

# Return a random value from a list, used in various jobs below
def get_random(from_list):
    _random = random.choice(from_list)
    return _random

def stats():
    bot.save_user_stats(bot.user_id)

def job1():
    bot.like_hashtag(get_random(random_hashtag_file), amount=int(700 / 24))

def job2():
    bot.like_timeline(amount=int(300 / 24))

def job3():
    bot.like_followers(get_random(random_user_file), nlikes=3)

def job4():
    bot.follow_followers(get_random(random_user_file), nfollows=config.NUMBER_OF_FOLLOWERS_TO_FOLLOW)

def job5():
    bot.comment_medias(bot.get_timeline_medias())

def job6():
    bot.unfollow_non_followers(n_to_unfollows=config.NUMBER_OF_NON_FOLLOWERS_TO_UNFOLLOW)

def job7():
    bot.follow_users(bot.get_hashtag_users(get_random(random_hashtag_file)))

def job8():  # Comment posts with an hashtag in HASHTAGS_FILE
    hashtag = get_random(random_hashtag_file)
    bot.logger.info("Commenting on hashtag: " + hashtag)
    bot.comment_hashtag(hashtag)

def job9():  # Automatically post a pic in 'pics' folder
    try:
        for pic in pics:
            if pic in posted_pic_list:
                continue

            caption = get_random(photo_captions)
            full_caption = caption + "\n" + config.FOLLOW_MESSAGE
            bot.logger.info("Uploading pic with caption: " + caption)
            bot.upload_photo(config.PICS_PATH + pic, caption=full_caption)
            if bot.last_response.status_code != 200:
                bot.logger.error("Something went wrong, read the following ->\n")
                bot.logger.error(bot.last_response)
                break

            if pic not in posted_pic_list:
                # After posting a pic, comment it with all the hashtags specified
                # In config.PICS_HASHTAGS
                posted_pic_list.append(pic)
                with open('pics.txt', 'a') as f:
                    f.write(pic + "\n")
                bot.logger.info("Succesfully uploaded: " + pic)
                bot.logger.info("Commenting uploaded photo with hashtags...")
                medias = bot.get_your_medias()
                last_photo = medias[0]  # Get the last photo posted
                bot.comment(last_photo, config.PICS_HASHTAGS)
                break
    except Exception as e:
        bot.logger.error("Couldn't upload pic")
        bot.logger.error(str(e))

def job10():  # put non followers on blacklist
    try:
        bot.logger.info("Creating non-followers list")
        followings = set(bot.following)  # getting following
        followers = set(bot.followers)  # getting followers
        friends = bot.friends_file.set  # same whitelist (just user ids)
        non_followers = followings - followers - friends
        for user_id in non_followers:
            bot.blacklist_file.append(user_id, allow_duplicates=False)
        bot.logger.info("Done.")
    except Exception as e:
        bot.logger.error("Couldn't update blacklist")
        bot.logger.error(str(e))

# function to make threads -> details here http://bit.ly/faq_schedule
def run_threaded(job_fn):
    job_thread = threading.Thread(target=job_fn)
    job_thread.start()

schedule.every(1).hour.do(run_threaded, stats)              # get stats
schedule.every(8).hours.do(run_threaded, job1)              # like hashtag
schedule.every(2).hours.do(run_threaded, job2)              # like timeline
schedule.every(1).days.at("16:00").do(run_threaded, job3)   # like followers of users from file
schedule.every(2).days.at("11:00").do(run_threaded, job4)   # follow followers
schedule.every(16).hours.do(run_threaded, job5)             # comment medias
schedule.every(1).days.at("08:00").do(run_threaded, job6)   # unfollow non-followers
schedule.every(12).hours.do(run_threaded, job7)             # follow users from hashtag from file
schedule.every(6).hours.do(run_threaded, job8)              # comment hashtag
schedule.every(1).days.at("21:28").do(run_threaded, job9)   # upload pics
schedule.every(4).days.at("07:50").do(run_threaded, job10)  # non-followers blacklist

while True:
    schedule.run_pending()
    time.sleep(1)
<|endoftext|>