<commit_before>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str

from datetime import datetime

import boto3
from sqlalchemy import func

from indra.db.util import get_primary_db

def __report_stat(report_str, fname=None, do_print=True):
    if do_print:
        print(report_str)
    if fname is not None:
        with open(fname, 'a+') as f:
            f.write(report_str + '\n')
    return

def get_text_ref_stats(fname=None, db=None):
    if db is None:
        db = get_primary_db()
    tr_tc_link = db.TextRef.id == db.TextContent.text_ref_id
    tc_rdng_link = db.TextContent.id == db.Reading.text_content_id
    __report_stat("Text ref statistics:", fname)
    __report_stat("--------------------", fname)
    total_refs = db.count(db.TextRef)
    __report_stat('Total number of text refs: %d' % total_refs, fname)
    refs_with_content = db.count(db.TextContent.text_ref_id)
    __report_stat('Total number of refs with content: %d' % refs_with_content,
                  fname)
    refs_by_type = _report_groups(db, db.TextContent, 'text_ref_id',
                                  'text_type', fname)
    __report_stat(('Number of refs with only abstract: %d'
                   % (refs_with_content-refs_by_type['fulltext'])), fname)
    refs_with_reading = db.count(db.TextContent.text_ref_id,
                                 *db.link(db.TextContent, db.Reading))
    __report_stat('Number of refs that have been read: %d' % refs_with_reading,
                  fname)
    _report_groups(db, db.TextContent, 'text_ref_id', 'text_type', fname,
                   tc_rdng_link)
    return

def get_text_content_stats(fname=None, db=None):
    if db is None:
        db = get_primary_db()
    tc_rdng_link = db.TextContent.id == db.Reading.text_content_id
    __report_stat("\nText Content statistics:", fname)
    __report_stat('------------------------', fname)
    total_content = db.count(db.TextContent)
    __report_stat("Total number of text content entries: %d" % total_content)
    latest_updates = (db.session.query(db.Updates.source,
                                       func.max(db.Updates.datetime))
                      .group_by(db.Updates.source)
                      .all())
    __report_stat(("Latest updates:\n    %s"
                   % '\n    '.join(['%s: %s' % (s, d)
                                    for s, d in latest_updates])),
                  fname
                  )
    content_read = db.count(db.Reading.text_content_id)
    __report_stat("Total content read: %d" % content_read, fname)
    fulltext_content = db.count(db.TextContent,
                                db.TextContent.text_type == 'fulltext')
    __report_stat("Number of fulltext entries: %d" % fulltext_content, fname)
    fulltext_read = db.count(db.TextContent,
                             db.TextContent.text_type == 'fulltext',
                             tc_rdng_link)
    __report_stat("Number of fulltext entries read: %d" % fulltext_read, fname)
    _report_groups(db, db.TextContent, 'id', 'source', fname)
    _report_groups(db, db.TextContent, 'id', 'source', fname, tc_rdng_link)
    return

def _report_groups(db, tbl, count_attr, group_attr, fname, *filters):
    """Report on the number of rows by group."""
    col_obj = getattr(tbl, group_attr)
    count_obj = getattr(tbl, count_attr)
    q = db.session.query(col_obj, count_obj)
    if filters:
        q = q.filter(*filters)
    content_by_group = (q.distinct()
                         .group_by(col_obj)
                         .all())
    broad_format = "{table} by {group}:\n    {values}"
    value_strs = ['%s: %d' % (s, n) for s, n in content_by_group]
    value_str = '\n    '.join(value_strs)
    report_str = broad_format.format(group=group_attr, values=value_str,
                                     table=tbl.__tablename__)
    __report_stat(report_str, fname)
    return {s: n for s, n in content_by_group}

def get_readings_stats(fname=None, db=None):
    if db is None:
        db = get_primary_db()

    __report_stat('\nReading statistics:', fname)
    __report_stat('-------------------', fname)
    rdg_q = db.filter_query(db.Reading)
    __report_stat('Total number or readings: %d' % rdg_q.count(), fname)
    # There may be a way to do this more neatly with a group_by clause, however
    # the naive way of doing it leaves us with a miscount due to indistinct.
    reader_versions = (db.session.query(db.Reading.reader_version)
                       .distinct().all())
    sources = db.session.query(db.TextContent.source).distinct().all()
    stats = ''
    for rv, in reader_versions:
        for src, in sources:
            cnt = db.filter_query(
                db.Reading,
                db.TextContent.id == db.Reading.text_content_id,
                db.TextContent.source == src,
                db.Reading.reader_version == rv
                ).distinct().count()
            stats += '    Readings by %s from %s: %d\n' % (rv, src, cnt)
    __report_stat("Readings by reader version and content source:\n%s" % stats,
                  fname)
    return

def get_statements_stats(fname=None, db=None, indra_version=None):
    if db is None:
        db = get_primary_db()
    tc_rdng_link = db.TextContent.id == db.Reading.text_content_id
    stmt_rdng_link = db.Reading.id == db.RawStatements.reader_ref

    __report_stat('\nStatement Statistics:', fname)
    __report_stat('---------------------', fname)
    stmt_q = db.filter_query(db.RawStatements)
    if indra_version is not None:
        stmt_q = stmt_q.filter(db.RawStatements.indra_version == indra_version)
    __report_stat("Total number of statments: %d" % stmt_q.count(), fname)
    readers = db.session.query(db.Reading.reader).distinct().all()
    sources = db.session.query(db.TextContent.source).distinct().all()
    stats = ''
    for reader, in readers:
        for src, in sources:
            cnt = stmt_q.filter(
                stmt_rdng_link,
                tc_rdng_link,
                db.Reading.reader == reader,
                db.TextContent.source == src
                ).distinct().count()
            stats += ('    Statements from %s reading %s: %d\n'
                      % (reader, src, cnt))
    __report_stat("Statements by reader and content source:\n%s" % stats,
                  fname)
    if indra_version is None:
        statements_by_db_source = (
            db.session.query(db.DBInfo.db_name, func.count(db.RawStatements.id))
            .filter(db.RawStatements.db_ref == db.DBInfo.id)
            .distinct()
            .group_by(db.DBInfo.db_name)
            .all()
            )
        __report_stat(("Statements by database:\n    %s"
                       % '\n    '.join(['%s: %d' % (s, n)
                                        for s, n in statements_by_db_source])),
                      fname
                      )
        statements_by_indra_version = (
            db.session.query(db.RawStatements.indra_version,
                             func.count(db.RawStatements.id))
            .group_by(db.RawStatements.indra_version)
            .all()
            )
        __report_stat(("Number of statements by indra version:\n    %s"
                       % '\n    '.join(['%s: %d' % (s, n) for s, n
                                        in statements_by_indra_version])),
                      fname
                      )
    return

def get_pa_statement_stats(fname=None, db=None):
    if db is None:
        db = get_primary_db()
    __report_stat('\nStatement Statistics:', fname)
    __report_stat('---------------------', fname)
    stmt_q = db.filter_query(db.PAStatements)
    __report_stat("Total number of statments: %d" % stmt_q.count(), fname)
    statements_produced_by_indra_version = (
        db.session.query(db.PAStatements.indra_version,
                         func.count(db.PAStatements.id))
          .group_by(db.PAStatements.indra_version)
          .all()
    )
    __report_stat(("Number of statements by indra version:\n    %s"
                   % '\n    '.join(['%s: %d' % (s, n) for s, n
                                    in statements_produced_by_indra_version])),
                  fname
                  )
    return

def get_db_statistics(fname=None, db=None, tables=None):
    """Get statistics on the contents of the database"""
    if db is None:
        db = get_primary_db()

    task_dict = {
        'text_ref': get_text_ref_stats,
        'text_content': get_text_content_stats,
        'readings': get_readings_stats,
        'statements': get_statements_stats,
        'pa_statements': get_pa_statement_stats
        }

    task_order = ['text_ref', 'text_content', 'readings', 'statements',
                  'pa_statements']

    # Get the statistics
    if tables is None:
        for task_name in task_order:
            stat_meth = task_dict[task_name]
            stat_meth(fname, db)
    else:
        table_set = set(tables)
        for task_name in [tn for tn in task_order if tn in table_set]:
            task_dict[task_name](fname, db)

    return

def get_all_db_statistics_s3():
    utcnow = datetime.utcnow()
    fname = "Primary_Database_Status_Report_%s.txt" % utcnow.strftime("%Y%m%d")
    print("Creating report in: %s." % fname)

    print("\nBegin Report============\n")
    get_db_statistics(fname)
    print("\nEnd Report==============\n")

    print("Saving record to s3.")
    s3 = boto3.client('s3')
    with open(fname, 'rb') as f:
        s3.put_object(Body=f, Bucket='bigmech', Key='indra-db/reports/%s' % fname)
    return

if __name__ == '__main__':
    get_all_db_statistics_s3()
<commit_msg>Tidy up a bit.<commit_after>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str

from datetime import datetime

import boto3
from sqlalchemy import func

from indra.db.util import get_primary_db

def __report_stat(report_str, fname=None, do_print=True):
    if do_print:
        print(report_str)
    if fname is not None:
        with open(fname, 'a+') as f:
            f.write(report_str + '\n')
    return

def _report_groups(db, tbl, count_attr, group_attr, fname, *filters):
    """Report on the number of rows by group."""
    col_obj = getattr(tbl, group_attr)
    count_obj = getattr(tbl, count_attr)
    q = db.session.query(col_obj, count_obj)
    if filters:
        q = q.filter(*filters)
    content_by_group = (q.distinct()
                        .group_by(col_obj)
                        .all())
    broad_format = "{table} by {group}:\n    {values}"
    value_strs = ['%s: %d' % (s, n) for s, n in content_by_group]
    value_str = '\n    '.join(value_strs)
    report_str = broad_format.format(group=group_attr, values=value_str,
                                     table=tbl.__tablename__)
    __report_stat(report_str, fname)
    return {s: n for s, n in content_by_group}

def get_text_ref_stats(fname=None, db=None):
    if db is None:
        db = get_primary_db()
    tc_rdng_link = db.TextContent.id == db.Reading.text_content_id
    __report_stat("Text ref statistics:", fname)
    __report_stat("--------------------", fname)
    total_refs = db.count(db.TextRef)
    __report_stat('Total number of text refs: %d' % total_refs, fname)
    refs_with_content = db.count(db.TextContent.text_ref_id)
    __report_stat('Total number of refs with content: %d' % refs_with_content,
                  fname)
    refs_by_type = _report_groups(db, db.TextContent, 'text_ref_id',
                                  'text_type', fname)
    __report_stat(('Number of refs with only abstract: %d'
                   % (refs_with_content-refs_by_type['fulltext'])), fname)
    refs_with_reading = db.count(db.TextContent.text_ref_id,
                                 tc_rdng_link)
    __report_stat('Number of refs that have been read: %d' % refs_with_reading,
                  fname)
    _report_groups(db, db.TextContent, 'text_ref_id', 'text_type', fname,
                   tc_rdng_link)
    return

def get_text_content_stats(fname=None, db=None):
    if db is None:
        db = get_primary_db()
    tc_rdng_link = db.TextContent.id == db.Reading.text_content_id
    __report_stat("\nText Content statistics:", fname)
    __report_stat('------------------------', fname)
    total_content = db.count(db.TextContent)
    __report_stat("Total number of text content entries: %d" % total_content)
    latest_updates = (db.session.query(db.Updates.source,
                                       func.max(db.Updates.datetime))
                      .group_by(db.Updates.source)
                      .all())
    __report_stat(("Latest updates:\n    %s"
                   % '\n    '.join(['%s: %s' % (s, d)
                                    for s, d in latest_updates])),
                  fname
                  )
    content_read = db.count(db.Reading.text_content_id)
    __report_stat("Total content read: %d" % content_read, fname)
    fulltext_content = db.count(db.TextContent,
                                db.TextContent.text_type == 'fulltext')
    __report_stat("Number of fulltext entries: %d" % fulltext_content, fname)
    fulltext_read = db.count(db.TextContent,
                             db.TextContent.text_type == 'fulltext',
                             tc_rdng_link)
    __report_stat("Number of fulltext entries read: %d" % fulltext_read, fname)
    _report_groups(db, db.TextContent, 'id', 'source', fname)
    _report_groups(db, db.TextContent, 'id', 'source', fname, tc_rdng_link)
    return

def get_readings_stats(fname=None, db=None):
    if db is None:
        db = get_primary_db()

    __report_stat('\nReading statistics:', fname)
    __report_stat('-------------------', fname)
    rdg_q = db.filter_query(db.Reading)
    __report_stat('Total number or readings: %d' % rdg_q.count(), fname)
    # There may be a way to do this more neatly with a group_by clause, however
    # the naive way of doing it leaves us with a miscount due to indistinct.
    reader_versions = (db.session.query(db.Reading.reader_version)
                       .distinct().all())
    sources = db.session.query(db.TextContent.source).distinct().all()
    stats = ''
    for rv, in reader_versions:
        for src, in sources:
            cnt = db.filter_query(
                db.Reading,
                db.TextContent.id == db.Reading.text_content_id,
                db.TextContent.source == src,
                db.Reading.reader_version == rv
                ).distinct().count()
            stats += '    Readings by %s from %s: %d\n' % (rv, src, cnt)
    __report_stat("Readings by reader version and content source:\n%s" % stats,
                  fname)
    return

def get_statements_stats(fname=None, db=None, indra_version=None):
    if db is None:
        db = get_primary_db()
    tc_rdng_link = db.TextContent.id == db.Reading.text_content_id
    stmt_rdng_link = db.Reading.id == db.RawStatements.reader_ref

    __report_stat('\nStatement Statistics:', fname)
    __report_stat('---------------------', fname)
    stmt_q = db.filter_query(db.RawStatements)
    if indra_version is not None:
        stmt_q = stmt_q.filter(db.RawStatements.indra_version == indra_version)
    __report_stat("Total number of statments: %d" % stmt_q.count(), fname)
    readers = db.session.query(db.Reading.reader).distinct().all()
    sources = db.session.query(db.TextContent.source).distinct().all()
    stats = ''
    for reader, in readers:
        for src, in sources:
            cnt = stmt_q.filter(
                stmt_rdng_link,
                tc_rdng_link,
                db.Reading.reader == reader,
                db.TextContent.source == src
                ).distinct().count()
            stats += ('    Statements from %s reading %s: %d\n'
                      % (reader, src, cnt))
    __report_stat("Statements by reader and content source:\n%s" % stats,
                  fname)
    if indra_version is None:
        statements_by_db_source = (
            db.session.query(db.DBInfo.db_name, func.count(db.RawStatements.id))
            .filter(db.RawStatements.db_ref == db.DBInfo.id)
            .distinct()
            .group_by(db.DBInfo.db_name)
            .all()
            )
        __report_stat(("Statements by database:\n    %s"
                       % '\n    '.join(['%s: %d' % (s, n)
                                        for s, n in statements_by_db_source])),
                      fname
                      )
        statements_by_indra_version = (
            db.session.query(db.RawStatements.indra_version,
                             func.count(db.RawStatements.id))
            .group_by(db.RawStatements.indra_version)
            .all()
            )
        __report_stat(("Number of statements by indra version:\n    %s"
                       % '\n    '.join(['%s: %d' % (s, n) for s, n
                                        in statements_by_indra_version])),
                      fname
                      )
    return

def get_pa_statement_stats(fname=None, db=None):
    if db is None:
        db = get_primary_db()
    __report_stat('\nStatement Statistics:', fname)
    __report_stat('---------------------', fname)
    stmt_q = db.filter_query(db.PAStatements)
    __report_stat("Total number of statments: %d" % stmt_q.count(), fname)
    statements_produced_by_indra_version = (
        db.session.query(db.PAStatements.indra_version,
                         func.count(db.PAStatements.id))
          .group_by(db.PAStatements.indra_version)
          .all()
    )
    __report_stat(("Number of statements by indra version:\n    %s"
                   % '\n    '.join(['%s: %d' % (s, n) for s, n
                                    in statements_produced_by_indra_version])),
                  fname
                  )
    return

def get_db_statistics(fname=None, db=None, tables=None):
    """Get statistics on the contents of the database"""
    if db is None:
        db = get_primary_db()

    task_dict = {
        'text_ref': get_text_ref_stats,
        'text_content': get_text_content_stats,
        'readings': get_readings_stats,
        'statements': get_statements_stats,
        'pa_statements': get_pa_statement_stats
        }

    task_order = ['text_ref', 'text_content', 'readings', 'statements',
                  'pa_statements']

    # Get the statistics
    if tables is None:
        for task_name in task_order:
            stat_meth = task_dict[task_name]
            stat_meth(fname, db)
    else:
        table_set = set(tables)
        for task_name in [tn for tn in task_order if tn in table_set]:
            task_dict[task_name](fname, db)

    return

def get_all_db_statistics_s3():
    utcnow = datetime.utcnow()
    fname = "Primary_Database_Status_Report_%s.txt" % utcnow.strftime("%Y%m%d")
    print("Creating report in: %s." % fname)

    print("\nBegin Report============\n")
    get_db_statistics(fname)
    print("\nEnd Report==============\n")

    print("Saving record to s3.")
    s3 = boto3.client('s3')
    with open(fname, 'rb') as f:
        s3.put_object(Body=f, Bucket='bigmech', Key='indra-db/reports/%s' % fname)
    return

if __name__ == '__main__':
    get_all_db_statistics_s3()
<|endoftext|>


<commit_before>#
# Newfies-Dialer License
# http://www.newfies-dialer.org
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Copyright (C) 2011-2012 Star2Billing S.L.
#
# The Initial Developer of the Original Code is
# Arezqui Belaid <info@star2billing.com>
#

from celery.task import Task, PeriodicTask
from dialer_campaign.models import Campaign, CampaignSubscriber
from dialer_campaign.function_def import user_dialer_setting
from dialer_cdr.models import Callrequest, VoIPCall
from celery.decorators import task
from django.db import IntegrityError
from datetime import datetime, timedelta
from time import sleep
from django.conf import settings
#from celery.task.http import HttpDispatchTask
#from common_functions import isint

if settings.DIALERDEBUG:
    Timelaps = 5
else:
    Timelaps = 60

@task(default_retry_delay=30 * 60)  # retry in 30 minutes.
def add(x, y):
    """This task adds 2 numbers.
    For instance (1, 2) will return '3'

    **Attributes**:

        * ``x`` -
        * ``y`` -

    **Troubleshoot**:

        * python manage.py shell_plus
        * from  dialer_campaign import tasks
        * mytask = tasks.add.delay(1,5)
        * mytask.result
    """

    print("Executing task id %r, args: %r kwargs: %r" % (
        add.request.id, add.request.args, add.request.kwargs))
    try:
        return x + y
    except Exception, exc:
        self.retry(exc=exc, countdown=60)  # override the default and
                                           # retry in 1 minute

#TODO: Put a priority on this task
@task()
def check_campaign_pendingcall(campaign_id):
    """This will execute the outbound calls in the campaign

    **Attributes**:

        * ``campaign_id`` - Campaign ID
    """
    logger = check_campaign_pendingcall.get_logger()
    logger.info("TASK :: check_campaign_pendingcall = %s" % str(campaign_id))

    try:
        obj_campaign = Campaign.objects.get(id=campaign_id)
    except:
        logger.error('Can\'t find this campaign')
        return False

    #TODO: Control the Speed
    #if there is many task pending we should slow down
    frequency = obj_campaign.frequency # default 10 calls per minutes

    dialer_set = user_dialer_setting(obj_campaign.user)

    # default call_type
    call_type = 1
    # Check campaign's maxretry
    if obj_campaign.maxretry == 0:
        call_type = 2

    # Check user's dialer setting maxretry
    if dialer_set:
        if dialer_set.maxretry == 0:
            call_type = 2

        # check frequency to control the Speed
        #if dialer_set.frequency:
        #    frequency = 20

    #Speed
    #check if the other tasks send for this campaign finished to be ran

    #Get the subscriber of this campaign
    # get_pending_subscriber get Max 1000 records
    list_subscriber = obj_campaign.get_pending_subscriber_update(
                            frequency,
                            6 # Update to In Process
                            )
    #if list_subscriber:
    #    print len(list_subscriber)

    try:
        no_subscriber = list_subscriber.count()
    except AttributeError:
        no_subscriber = 0

    if no_subscriber == 0:
        logger.info("No Subscriber to proceed on this campaign")
        return False

    #find how to dispatch them in the current minutes
    time_to_wait = 60.0 / no_subscriber

    for elem_camp_subscriber in list_subscriber:
        """Loop on Subscriber and start the initcall task"""
        logger.info("Add CallRequest for Subscriber (%s) & wait (%s) " %
                        (str(elem_camp_subscriber.id), str(time_to_wait)))

        #Check if the contact is authorized
        if not obj_campaign.is_authorized_contact(elem_camp_subscriber.contact.contact):
            logger.error("Error : Contact not authorized")
            elem_camp_subscriber.status = 7 # Update to Not Authorized
            elem_camp_subscriber.save()
            return True

        #Create a Callrequest Instance to track the call task
        new_callrequest = Callrequest(status=1, # PENDING
                            call_type=call_type,
                            call_time=datetime.now(),
                            timeout=obj_campaign.calltimeout,
                            callerid=obj_campaign.callerid,
                            phone_number=elem_camp_subscriber.contact.contact,
                            campaign=obj_campaign,
                            aleg_gateway=obj_campaign.aleg_gateway,
                            content_type=obj_campaign.content_type,
                            object_id=obj_campaign.object_id,
                            user=obj_campaign.user,
                            extra_data=obj_campaign.extra_data,
                            timelimit=obj_campaign.callmaxduration,
                            campaign_subscriber=elem_camp_subscriber)
        new_callrequest.save()

        #Todo Check if it's a good practice / implement a PID algorithm
        if no_subscriber > 1:
            sleep(time_to_wait)

class campaign_running(PeriodicTask):
    """A periodic task that checks the campaign, create and tasks the calls

    **Usage**:

        campaign_running.delay()
    """
    run_every = timedelta(seconds=Timelaps)
    #NOTE : until we implement a PID Controller :
    #http://en.wikipedia.org/wiki/PID_controller

    #The campaign have to run every minutes in order to control the number
    # of calls per minute. Cons : new calls might delay 60seconds
    #run_every = timedelta(seconds=60)

    def run(self, **kwargs):
        logger = self.get_logger(**kwargs)
        logger.warning( "TASK :: campaign_running")

        for campaign in Campaign.objects.get_running_campaign():
            logger.debug("=> Campaign name %s (id:%s)" % (campaign.name,
                                                         campaign.id))

            check_campaign_pendingcall.delay(campaign.id)

class campaign_spool_contact(PeriodicTask):
    """A periodic task that checks the campaign, add subscribers

    **Usage**:

        campaign_spool_contact.delay()
    """

    run_every = timedelta(seconds=15)
    #The campaign have to run every minutes in order to control the number
    # of calls per minute. Cons : new calls might delay 60seconds
    #run_every = timedelta(seconds=60)

    def run(self, **kwargs):
        logger = self.get_logger(**kwargs)
        logger.info( "TASK :: campaign_spool_contact")

        for campaign in Campaign.objects.get_running_campaign():
            logger.debug("=> Spool Contact : Campaign name %s (id:%s)" % (campaign.name,
                                                         str(campaign.id)))

            collect_subscriber_optimized.delay(campaign.id)

@task()
def collect_subscriber_optimized(campaign_id):
    """This task will collect all the subscribers

    **Attributes**:

        * ``campaign_id`` - Campaign ID
    """
    logger = collect_subscriber_optimized.get_logger()
    logger.debug("Collect subscribers for the campaign = %s" % str(campaign_id))

    #Retrieve the list of active contact
    obj_campaign = Campaign.objects.get(id=campaign_id)

    list_phonebook = obj_campaign.phonebook.all()

    for item_phonebook in list_phonebook:
        phonebook_id = item_phonebook.id

        # check if phonebook_id is missing in imported_phonebook list
        if not str(phonebook_id) in obj_campaign.imported_phonebook.split(','):
            #Run import
            import_phonebook.delay(obj_campaign.id, phonebook_id)

def importcontact_custom_sql(campaign_id, phonebook_id):
    from django.db import connection, transaction
    cursor = connection.cursor()

    # Call PL-SQL stored procedure
    CampaignSubscriber.importcontact_pl_sql(campaign_id, phonebook_id)

    #TODO : Rewrite this using PL SQL

    print "ISSUE HERE : REPLACE WITH PL SQL"
    # Data insert operation - commit required
    sqlimport = "INSERT INTO dialer_campaign_subscriber (contact_id, campaign_id, duplicate_contact, status, created_date, updated_date) "\
                "SELECT id, %d, contact, 1, NOW(), NOW() FROM dialer_contact WHERE phonebook_id=%d" % (campaign_id, phonebook_id)
    print sqlimport
    print "************************"
    cursor.execute(sqlimport)
    transaction.commit_unless_managed()

    return True

@task()
def import_phonebook(campaign_id, phonebook_id):
    """
    Read all the contact from phonebook_id and insert them into campaignsubscriber
    """
    logger = import_phonebook.get_logger()
    logger.info( "\n\n\nTASK :: import_phonebook")

    #TODO: Add a semafore

    obj_campaign = Campaign.objects.get(id=campaign_id)

    #Faster method, ask the Database to do the job
    importcontact_custom_sql(campaign_id, phonebook_id)

    #Add the phonebook id to the imported list
    if obj_campaign.imported_phonebook == '':
        sep = ''
    else:
        sep = ','
    obj_campaign.imported_phonebook = obj_campaign.imported_phonebook + '%s%d' % (sep, phonebook_id)

@task()
def collect_subscriber(campaign_id):
    """This task will collect all the subscribers

    **Attributes**:

        * ``campaign_id`` - Campaign ID
    """
    logger = collect_subscriber.get_logger()
    logger.debug("Collect subscribers for the campaign = %s" % str(campaign_id))

    #Retrieve the list of active contact
    obj_campaign = Campaign.objects.get(id=campaign_id)
    list_contact = obj_campaign.get_active_contact_no_subscriber()

    if not list_contact:
        logger.debug("No new contact or phonebook to import into \
            this campaign.")
        return True
    else:
        #Create CampaignSubscribers for each new active contact
        for elem_contact in list_contact:
            try:
                CampaignSubscriber.objects.create(
                                    contact=elem_contact,
                                    status=1, #START
                                    duplicate_contact=elem_contact.contact,
                                    campaign=obj_campaign)
            except IntegrityError, e:
                #We don't stop if it fails to add a subscriber to one campaign
                logger.error("IntegrityError to create CampaignSubscriber "\
                    "contact_id=%s - Error:%s" % (elem_contact.id, e))

    return True

class campaign_expire_check(PeriodicTask):
    """A periodic task that checks the campaign expiration

    **Usage**:

        campaign_expire_check.delay()
    """
    run_every = timedelta(seconds=300)
    #The campaign have to run every minutes in order to control the number
    # of calls per minute. Cons : new calls might delay 60seconds
    #run_every = timedelta(seconds=60)

    def run(self, **kwargs):
        logger = self.get_logger(**kwargs)
        logger.info( "TASK :: campaign_expire_check")
        from dialer_campaign.views import common_campaign_status
        for campaign in Campaign.objects.get_expired_campaign():
            logger.debug("=> Campaign name %s (id:%s)" % (campaign.name,
                                                         campaign.id))
            common_campaign_status(campaign.id, 4)

<commit_msg>missing save<commit_after>#
# Newfies-Dialer License
# http://www.newfies-dialer.org
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Copyright (C) 2011-2012 Star2Billing S.L.
#
# The Initial Developer of the Original Code is
# Arezqui Belaid <info@star2billing.com>
#

from celery.task import Task, PeriodicTask
from dialer_campaign.models import Campaign, CampaignSubscriber
from dialer_campaign.function_def import user_dialer_setting
from dialer_cdr.models import Callrequest, VoIPCall
from celery.decorators import task
from django.db import IntegrityError
from datetime import datetime, timedelta
from time import sleep
from django.conf import settings
#from celery.task.http import HttpDispatchTask
#from common_functions import isint

if settings.DIALERDEBUG:
    Timelaps = 5
else:
    Timelaps = 60

@task(default_retry_delay=30 * 60)  # retry in 30 minutes.
def add(x, y):
    """This task adds 2 numbers.
    For instance (1, 2) will return '3'

    **Attributes**:

        * ``x`` -
        * ``y`` -

    **Troubleshoot**:

        * python manage.py shell_plus
        * from  dialer_campaign import tasks
        * mytask = tasks.add.delay(1,5)
        * mytask.result
    """

    print("Executing task id %r, args: %r kwargs: %r" % (
        add.request.id, add.request.args, add.request.kwargs))
    try:
        return x + y
    except Exception, exc:
        self.retry(exc=exc, countdown=60)  # override the default and
                                           # retry in 1 minute

#TODO: Put a priority on this task
@task()
def check_campaign_pendingcall(campaign_id):
    """This will execute the outbound calls in the campaign

    **Attributes**:

        * ``campaign_id`` - Campaign ID
    """
    logger = check_campaign_pendingcall.get_logger()
    logger.info("TASK :: check_campaign_pendingcall = %s" % str(campaign_id))

    try:
        obj_campaign = Campaign.objects.get(id=campaign_id)
    except:
        logger.error('Can\'t find this campaign')
        return False

    #TODO: Control the Speed
    #if there is many task pending we should slow down
    frequency = obj_campaign.frequency # default 10 calls per minutes

    dialer_set = user_dialer_setting(obj_campaign.user)

    # default call_type
    call_type = 1
    # Check campaign's maxretry
    if obj_campaign.maxretry == 0:
        call_type = 2

    # Check user's dialer setting maxretry
    if dialer_set:
        if dialer_set.maxretry == 0:
            call_type = 2

        # check frequency to control the Speed
        #if dialer_set.frequency:
        #    frequency = 20

    #Speed
    #check if the other tasks send for this campaign finished to be ran

    #Get the subscriber of this campaign
    # get_pending_subscriber get Max 1000 records
    list_subscriber = obj_campaign.get_pending_subscriber_update(
                            frequency,
                            6 # Update to In Process
                            )
    #if list_subscriber:
    #    print len(list_subscriber)

    try:
        no_subscriber = list_subscriber.count()
    except AttributeError:
        no_subscriber = 0

    if no_subscriber == 0:
        logger.info("No Subscriber to proceed on this campaign")
        return False

    #find how to dispatch them in the current minutes
    time_to_wait = 60.0 / no_subscriber

    for elem_camp_subscriber in list_subscriber:
        """Loop on Subscriber and start the initcall task"""
        logger.info("Add CallRequest for Subscriber (%s) & wait (%s) " %
                        (str(elem_camp_subscriber.id), str(time_to_wait)))

        #Check if the contact is authorized
        if not obj_campaign.is_authorized_contact(elem_camp_subscriber.contact.contact):
            logger.error("Error : Contact not authorized")
            elem_camp_subscriber.status = 7 # Update to Not Authorized
            elem_camp_subscriber.save()
            return True

        #Create a Callrequest Instance to track the call task
        new_callrequest = Callrequest(status=1, # PENDING
                            call_type=call_type,
                            call_time=datetime.now(),
                            timeout=obj_campaign.calltimeout,
                            callerid=obj_campaign.callerid,
                            phone_number=elem_camp_subscriber.contact.contact,
                            campaign=obj_campaign,
                            aleg_gateway=obj_campaign.aleg_gateway,
                            content_type=obj_campaign.content_type,
                            object_id=obj_campaign.object_id,
                            user=obj_campaign.user,
                            extra_data=obj_campaign.extra_data,
                            timelimit=obj_campaign.callmaxduration,
                            campaign_subscriber=elem_camp_subscriber)
        new_callrequest.save()

        #Todo Check if it's a good practice / implement a PID algorithm
        if no_subscriber > 1:
            sleep(time_to_wait)

class campaign_running(PeriodicTask):
    """A periodic task that checks the campaign, create and tasks the calls

    **Usage**:

        campaign_running.delay()
    """
    run_every = timedelta(seconds=Timelaps)
    #NOTE : until we implement a PID Controller :
    #http://en.wikipedia.org/wiki/PID_controller

    #The campaign have to run every minutes in order to control the number
    # of calls per minute. Cons : new calls might delay 60seconds
    #run_every = timedelta(seconds=60)

    def run(self, **kwargs):
        logger = self.get_logger(**kwargs)
        logger.warning( "TASK :: campaign_running")

        for campaign in Campaign.objects.get_running_campaign():
            logger.debug("=> Campaign name %s (id:%s)" % (campaign.name,
                                                         campaign.id))

            check_campaign_pendingcall.delay(campaign.id)

class campaign_spool_contact(PeriodicTask):
    """A periodic task that checks the campaign, add subscribers

    **Usage**:

        campaign_spool_contact.delay()
    """

    run_every = timedelta(seconds=15)
    #The campaign have to run every minutes in order to control the number
    # of calls per minute. Cons : new calls might delay 60seconds
    #run_every = timedelta(seconds=60)

    def run(self, **kwargs):
        logger = self.get_logger(**kwargs)
        logger.info( "TASK :: campaign_spool_contact")

        for campaign in Campaign.objects.get_running_campaign():
            logger.debug("=> Spool Contact : Campaign name %s (id:%s)" % (campaign.name,
                                                         str(campaign.id)))

            collect_subscriber_optimized.delay(campaign.id)

@task()
def collect_subscriber_optimized(campaign_id):
    """This task will collect all the subscribers

    **Attributes**:

        * ``campaign_id`` - Campaign ID
    """
    logger = collect_subscriber_optimized.get_logger()
    logger.debug("Collect subscribers for the campaign = %s" % str(campaign_id))

    #Retrieve the list of active contact
    obj_campaign = Campaign.objects.get(id=campaign_id)

    list_phonebook = obj_campaign.phonebook.all()

    for item_phonebook in list_phonebook:
        phonebook_id = item_phonebook.id

        # check if phonebook_id is missing in imported_phonebook list
        if not str(phonebook_id) in obj_campaign.imported_phonebook.split(','):
            #Run import
            import_phonebook.delay(obj_campaign.id, phonebook_id)

def importcontact_custom_sql(campaign_id, phonebook_id):
    from django.db import connection, transaction
    cursor = connection.cursor()

    # Call PL-SQL stored procedure
    CampaignSubscriber.importcontact_pl_sql(campaign_id, phonebook_id)

    #TODO : Rewrite this using PL SQL

    print "ISSUE HERE : REPLACE WITH PL SQL"
    # Data insert operation - commit required
    sqlimport = "INSERT INTO dialer_campaign_subscriber (contact_id, campaign_id, duplicate_contact, status, created_date, updated_date) "\
                "SELECT id, %d, contact, 1, NOW(), NOW() FROM dialer_contact WHERE phonebook_id=%d" % (campaign_id, phonebook_id)
    print sqlimport
    print "************************"
    cursor.execute(sqlimport)
    transaction.commit_unless_managed()

    return True

@task()
def import_phonebook(campaign_id, phonebook_id):
    """
    Read all the contact from phonebook_id and insert them into campaignsubscriber
    """
    logger = import_phonebook.get_logger()
    logger.info( "\n\n\nTASK :: import_phonebook")

    #TODO: Add a semafore

    obj_campaign = Campaign.objects.get(id=campaign_id)

    #Faster method, ask the Database to do the job
    importcontact_custom_sql(campaign_id, phonebook_id)

    #Add the phonebook id to the imported list
    if obj_campaign.imported_phonebook == '':
        sep = ''
    else:
        sep = ','
    obj_campaign.imported_phonebook = obj_campaign.imported_phonebook + '%s%d' % (sep, phonebook_id)
    obj_campaign.save()

@task()
def collect_subscriber(campaign_id):
    """This task will collect all the subscribers

    **Attributes**:

        * ``campaign_id`` - Campaign ID
    """
    logger = collect_subscriber.get_logger()
    logger.debug("Collect subscribers for the campaign = %s" % str(campaign_id))

    #Retrieve the list of active contact
    obj_campaign = Campaign.objects.get(id=campaign_id)
    list_contact = obj_campaign.get_active_contact_no_subscriber()

    if not list_contact:
        logger.debug("No new contact or phonebook to import into \
            this campaign.")
        return True
    else:
        #Create CampaignSubscribers for each new active contact
        for elem_contact in list_contact:
            try:
                CampaignSubscriber.objects.create(
                                    contact=elem_contact,
                                    status=1, #START
                                    duplicate_contact=elem_contact.contact,
                                    campaign=obj_campaign)
            except IntegrityError, e:
                #We don't stop if it fails to add a subscriber to one campaign
                logger.error("IntegrityError to create CampaignSubscriber "\
                    "contact_id=%s - Error:%s" % (elem_contact.id, e))

    return True

class campaign_expire_check(PeriodicTask):
    """A periodic task that checks the campaign expiration

    **Usage**:

        campaign_expire_check.delay()
    """
    run_every = timedelta(seconds=300)
    #The campaign have to run every minutes in order to control the number
    # of calls per minute. Cons : new calls might delay 60seconds
    #run_every = timedelta(seconds=60)

    def run(self, **kwargs):
        logger = self.get_logger(**kwargs)
        logger.info( "TASK :: campaign_expire_check")
        from dialer_campaign.views import common_campaign_status
        for campaign in Campaign.objects.get_expired_campaign():
            logger.debug("=> Campaign name %s (id:%s)" % (campaign.name,
                                                         campaign.id))
            common_campaign_status(campaign.id, 4)

<|endoftext|>


<commit_before>from csp import settings

def send_activation_email_gmail(email, host, activation_key):
    """
        This sends the activation link to the user using Gmail, the content will be moved to template files

        Keyword Arguments:
        host -- the domain of the website
        activation_key -- the key which activates the account
    """
    import smtplib

    subject, from_email, to = 'Daemo Account Activation', settings.EMAIL_SENDER, email
    activation_url = 'http://' + host + '/account-activation/' + activation_key
    text_content = 'Hello, \n ' \
                   'Activate your account by clicking the following link: \n' + activation_url + \
                   '\nGreetings, \nDaemo Team'

    html_content = '
Hello,
' \
                   '
Activate your account by clicking the following link:
' \
                   '' + activation_url + '
' \
                                                                          '

 Greetings,
 Daemo Team'
    message = 'Subject: %s\n\n%s' % (subject, text_content)
    try:
        server = smtplib.SMTP("smtp.gmail.com", 587)  # or port 465 doesn't seem to work!
        server.ehlo()
        server.starttls()
        server.login(settings.EMAIL_SENDER_DEV, settings.EMAIL_SENDER_PASSWORD_DEV)
        server.sendmail(from_email, to, message)
        server.close()
    except:
        print("Error sending activation email")

def send_activation_email_sendgrid(email, host, activation_key):
    """
        This sends the activation link to the user using sendgrid, the content will be moved to template files

        Keyword Arguments:
        host -- the domain of the website
        activation_key -- the key which activates the account
    """
    # from django.core.mail import EmailMultiAlternatives
    import smtplib

    subject, from_email, to = 'Daemo Account Activation', settings.EMAIL_SENDER, email
    activation_url = 'http://' + host + '/account-activation/' + activation_key
    text_content = 'Hello, \n ' \
                   'Activate your account by clicking the following link: \n' + activation_url + \
                   '\nGreetings, \nDaemo Team'

    html_content = '
Hello,
' \
                   '
Activate your account by clicking the following link:
' \
                   '' + activation_url + '
' \
                                                                          '

 Greetings,
 Daemo Team'
    send_grid(to, subject, text_content, html_content)

def send_grid(to, subject, text, html=None):
    import sendgrid
    sg = sendgrid.SendGridClient(settings.SENDGRID_API_KEY)

    message = sendgrid.Mail()
    message.add_to('<%s>'%to)
    message.set_subject(subject)
    message.set_text(text)
    message.set_html(html)
    message.set_from('Daemo Team <%s>'%settings.EMAIL_SENDER)
    status, msg = sg.send(message)
    return status, msg<commit_msg>send reset_email method<commit_after>from csp import settings

def send_activation_email_gmail(email, host, activation_key):
    """
        This sends the activation link to the user using Gmail, the content will be moved to template files

        Keyword Arguments:
        host -- the domain of the website
        activation_key -- the key which activates the account
    """
    import smtplib

    subject, from_email, to = 'Daemo Account Activation', settings.EMAIL_SENDER, email
    activation_url = 'http://' + host + '/account-activation/' + activation_key
    text_content = 'Hello, \n ' \
                   'Activate your account by clicking the following link: \n' + activation_url + \
                   '\nGreetings, \nDaemo Team'

    html_content = '
Hello,
' \
                   '
Activate your account by clicking the following link:
' \
                   '' + activation_url + '
' \
                                                                          '

 Greetings,
 Daemo Team'
    message = 'Subject: %s\n\n%s' % (subject, text_content)
    try:
        server = smtplib.SMTP("smtp.gmail.com", 587)  # or port 465 doesn't seem to work!
        server.ehlo()
        server.starttls()
        server.login(settings.EMAIL_SENDER_DEV, settings.EMAIL_SENDER_PASSWORD_DEV)
        server.sendmail(from_email, to, message)
        server.close()
    except:
        print("Error sending activation email")

def send_activation_email_sendgrid(email, host, activation_key):
    """
        This sends the activation link to the user using sendgrid, the content will be moved to template files

        Keyword Arguments:
        host -- the domain of the website
        activation_key -- the key which activates the account
    """
    # from django.core.mail import EmailMultiAlternatives
    import smtplib

    subject, from_email, to = 'Daemo Account Activation', settings.EMAIL_SENDER, email
    activation_url = 'http://' + host + '/account-activation/' + activation_key
    text_content = 'Hello, \n ' \
                   'Activate your account by clicking the following link: \n' + activation_url + \
                   '\nGreetings, \nDaemo Team'

    html_content = '
Hello,
' \
                   '
Activate your account by clicking the following link:
' \
                   '' + activation_url + '
' \
                                                                          '

 Greetings,
 Daemo Team'
    send_grid(to, subject, text_content, html_content)

def send_grid(to, subject, text, html=None):
    import sendgrid

    sg = sendgrid.SendGridClient(settings.SENDGRID_API_KEY)

    message = sendgrid.Mail()
    message.add_to('<%s>' % to)
    message.set_subject(subject)
    message.set_text(text)
    message.set_html(html)
    message.set_from('Daemo Team <%s>' % settings.EMAIL_SENDER)
    status, msg = sg.send(message)
    return status, msg

def send_password_reset_email(email, host, reset_key):
    """
        This sends the email to the user
        The email includes two links, one for changing the password and the other for discarding the forgot password request.
    """

    subject, from_email, to = 'Daemo Password Reset', settings.EMAIL_SENDER, email
    reset_url = 'https://' + host + '/reset-password/' + reset_key
    text_content = 'Hello, \n ' \
                   'Please reset your password using the following link: \n' + reset_url + '/1'' \
                   ''\nIf you did not request a password reset please click the following link: ' + reset_url + '/0'' \
                   ''\nGreetings, \nDaemo Team'
    html_content = '
Hello,
' \
                   '
Please reset your password using the following link:
' \
                   '' + reset_url + '/1' + '
'" \
                   ""

If you didn't request a password reset please click the following link:
"+\
                   '' + reset_url + '/0' +\
                   '

 Greetings,
 Daemo Team'
    send_grid(to, subject, text_content, html_content)
<|endoftext|>


<commit_before>import os
import sys
from sqlobject import *

from muzicast.const import DB_FILE

# NOTE: When adding a class to this file, make a relevant
# entry in init_meta to initialize the database
connection_string = 'sqlite://' + DB_FILE
if sys.platform == 'win32':
    connection_string = 'sqlite:/' + os.path.normpath(DB_FILE).replace(':', '|/', 1)

connection = connectionForURI(connection_string)
sqlhub.processConnection = connection

class Artist(SQLObject):
    name = UnicodeCol()
    albums = MultipleJoin('Album')

class Album(SQLObject):
    name = UnicodeCol()
    artist = ForeignKey('Artist')
    image = UnicodeCol()
    tracks = MultipleJoin('Track')

class Genre(SQLObject):
    name = UnicodeCol()

class Composer(SQLObject):
    name = UnicodeCol()

class Track(SQLObject):
    url = UnicodeCol()
    artist = ForeignKey('Artist')
    album = ForeignKey('Album')
    genre = ForeignKey('Genre')
    composer = ForeignKey('Composer')
    year = IntCol()
    title = UnicodeCol()
    comment = UnicodeCol()
    tracknumber = IntCol()
    discnumber = IntCol()
    bitrate = IntCol()
    duration = IntCol()
    filesize = IntCol()
    filetype = StringCol()
    bpm = IntCol()
    createdate = DateTimeCol()
    modifydate = DateTimeCol()

def init_meta():
    """
    Creates metadata tables.

    Should be run before any transactions
    are performed.
    NOTE: This should be called only once,
    otherwise it will OVERWRITE the earlier data.
    """
    Artist.createTable()
    Album.createTable()
    Genre.createTable()
    Composer.createTable()
    Track.createTable()
    # XXX: Add more HERE
    return True

#TODO(nikhil) statistics, playlists tables
<commit_msg>Added statistics tables<commit_after>import os
import sys
from sqlobject import *

from muzicast.const import DB_FILE

# NOTE: When adding a class to this file, make a relevant
# entry in init_meta to initialize the database
connection_string = 'sqlite://' + DB_FILE
if sys.platform == 'win32':
    connection_string = 'sqlite:/' + os.path.normpath(DB_FILE).replace(':', '|/', 1)

connection = connectionForURI(connection_string)
sqlhub.processConnection = connection

class Artist(SQLObject):
    name = UnicodeCol()
    albums = MultipleJoin('Album')

class Album(SQLObject):
    name = UnicodeCol()
    artist = ForeignKey('Artist')
    image = UnicodeCol()
    tracks = MultipleJoin('Track')

class Genre(SQLObject):
    name = UnicodeCol()

class Composer(SQLObject):
    name = UnicodeCol()

class Track(SQLObject):
    url = UnicodeCol()
    artist = ForeignKey('Artist')
    album = ForeignKey('Album')
    genre = ForeignKey('Genre')
    composer = ForeignKey('Composer')
    year = IntCol()
    title = UnicodeCol()
    comment = UnicodeCol()
    tracknumber = IntCol()
    discnumber = IntCol()
    bitrate = IntCol()
    duration = IntCol()
    filesize = IntCol()
    filetype = StringCol()
    bpm = IntCol()
    createdate = DateTimeCol()
    modifydate = DateTimeCol()

class TrackStatistics(SQLObject):
    track = ForeignKey('Track')
    last_played = DateTimeCol()
    play_count = IntCol()

class AlbumStatistics(SQLObject):
    album = ForeignKey('Album')
    last_played = DateTimeCol()
    play_count = IntCol()

class ArtistStatistics(SQLObject):
    artist = ForeignKey('Artist')
    last_played = DateTimeCol()
    play_count = IntCol()

class GenreStatistics(SQLObject):
    genre = ForeignKey('Genre')
    last_played = DateTimeCol()
    play_count = IntCol()

def init_meta():
    """
    Creates metadata tables.

    Should be run before any transactions
    are performed.
    NOTE: This should be called only once,
    otherwise it will OVERWRITE the earlier data.
    """
    Artist.createTable()
    Album.createTable()
    Genre.createTable()
    Composer.createTable()
    Track.createTable()
    TrackStatistics.createTable()
    AlbumStatistics.createTable()
    ArtistStatistics.createTable()
    GenreStatistics.createTable()
    # XXX: Add more HERE
    return True

#TODO(nikhil) statistics, playlists tables
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
daily work, clean expired items and its place in queues in memory
"""
# @author chengdujin
# @contact chengdujin@gmail.com
# @created Aug. 22, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append("..")

from bson.objectid import ObjectId
from config import Collection, db
from config import logger
from config import rclient
import time
import cleaner

def _is_zombie(item):
    """
    zombies are items in memory but not in database
    """
    if not item:
        return True

    try:
        document_name = item['language']
        document = Collection(db, document_name)
        item_id = ObjectId(item['_id'])
        found = document.find_one({'_id': item_id})
        if not found:
            return True
        else:
            return False
    except Exception as k:
        logger.error(str(k))
        return True

def clean_by_item(item_id):
    """
    remove item-related things in memory
    """
    if not item_id:
        return None
    if not rclient.exists(item_id):
        return None
    # it's possible item_id in queues be visited
    # success returns 1, else 0
    return rclient.delete(item_id)

def clean_by_items(news_list, item_ids):
    """
    removed a group of expired items in memory
    the item might be in redis, but void
    """
    for item_id in item_ids:
        # remove id from queues
        rclient.zrem(news_list, item_id)
        # delete id from redis, if it's available
        clean_by_item(item_id)

def clean():
    """
    remove expired items from queues in memory
    walk through all redis content
    """
    logger.info('... cleaning memory ...')
    try:
        news_lists = rclient.keys('news::*')
        for news_list in news_lists:
            # get the total number of a news list
            news_list_count = rclient.zcard(news_list)
            # get all the ids in a news list
            if news_list_count:
                news_ids = rclient.zrange(news_list, 0, news_list_count)
                for news_id in news_ids:
                    # make sure every item is touched
                    if not rclient.exists(news_id):
                        rclient.zrem(news_list, news_id)
                    else:
                        news_item_string = rclient.get(news_id)
                        if news_item_string:
                            news_item = eval(news_item_string)
                            news_updated = float(news_item['updated'])

                            # WTF, remove it
                            if cleaner.is_overdue(news_updated):
                                rclient.zrem(news_list, news_id)
                                rclient.delete(news_id)
                            else:  # check if this is zombie
                                if _is_zombie(news_item):
                                    rclient.zrem(news_list, news_id)
                                    rclient.delete(news_id)
                        else:
                            rclient.zrem(news_list, news_id)
                            rclient.delete(news_id)
                return True
            else:
                logger.error('Nothing in the list')
                return False
    except Exception as k:
        logger.error(str(k))
        return False

if __name__ == "__main__":
    clean()
<commit_msg>changed one return for safe operation<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
daily work, clean expired items and its place in queues in memory
"""
# @author chengdujin
# @contact chengdujin@gmail.com
# @created Aug. 22, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append("..")

from bson.objectid import ObjectId
from config import Collection, db
from config import logger
from config import rclient
import time
import cleaner

def _is_zombie(item):
    """
    zombies are items in memory but not in database
    """
    if not item:
        return True

    try:
        document_name = item['language']
        document = Collection(db, document_name)
        item_id = ObjectId(item['_id'])
        found = document.find_one({'_id': item_id})
        if not found:
            return True
        else:
            return False
    except Exception as k:
        logger.error(str(k))
        return False

def clean_by_item(item_id):
    """
    remove item-related things in memory
    """
    if not item_id:
        return None
    if not rclient.exists(item_id):
        return None
    # it's possible item_id in queues be visited
    # success returns 1, else 0
    return rclient.delete(item_id)

def clean_by_items(news_list, item_ids):
    """
    removed a group of expired items in memory
    the item might be in redis, but void
    """
    for item_id in item_ids:
        # remove id from queues
        rclient.zrem(news_list, item_id)
        # delete id from redis, if it's available
        clean_by_item(item_id)

def clean():
    """
    remove expired items from queues in memory
    walk through all redis content
    """
    logger.info('... cleaning memory ...')
    try:
        news_lists = rclient.keys('news::*')
        for news_list in news_lists:
            # get the total number of a news list
            news_list_count = rclient.zcard(news_list)
            # get all the ids in a news list
            if news_list_count:
                news_ids = rclient.zrange(news_list, 0, news_list_count)
                for news_id in news_ids:
                    # make sure every item is touched
                    if not rclient.exists(news_id):
                        rclient.zrem(news_list, news_id)
                    else:
                        news_item_string = rclient.get(news_id)
                        if news_item_string:
                            news_item = eval(news_item_string)
                            news_updated = float(news_item['updated'])

                            # WTF, remove it
                            if cleaner.is_overdue(news_updated):
                                rclient.zrem(news_list, news_id)
                                rclient.delete(news_id)
                            else:  # check if this is zombie
                                if _is_zombie(news_item):
                                    rclient.zrem(news_list, news_id)
                                    rclient.delete(news_id)
                        else:
                            rclient.zrem(news_list, news_id)
                            rclient.delete(news_id)
                return True
            else:
                logger.error('Nothing in the list')
                return False
    except Exception as k:
        logger.error(str(k))
        return False

if __name__ == "__main__":
    clean()
<|endoftext|>


<commit_before>from django.db import models
from django.utils.encoding import force_bytes
from django.utils import timezone

# dependent on icalendar package - pip install icalendar
from icalendar import Calendar, Event, vDatetime, LocalTimezone
from datetime import datetime, timedelta
import urllib.request, urllib.error, urllib.parse
import os
from csc_new import settings

import dateutil.rrule as rrule

# Create your models here.
class ExamReview(models.Model):
    title = models.CharField(max_length=100)
    questions = models.FileField(upload_to="exam_reviews")
    answers = models.FileField(upload_to="exam_reviews")
    last_modified = models.DateTimeField(auto_now_add=True, auto_now=True)

    def __str__(self):
        return '%s' % (self.title)

    def delete(self, *args, **kwargs):
        os.remove(os.path.join(settings.MEDIA_ROOT, str(self.questions)))
        os.remove(os.path.join(settings.MEDIA_ROOT, str(self.answers)))
        super(ExamReview, self).delete(*args, **kwargs)

class Photo(models.Model):
    title = models.CharField(max_length=100)
    desc = models.CharField(max_length=255)
    src = models.FileField(upload_to="photos")

    def __str__(self):
        return self.title

    def delete(self, *args, **kwargs):
        os.remove(os.path.join(settings.MEDIA_ROOT, str(self.src)))
        super(Photo, self).delete(*args, **kwargs)

# RenderableEvent - holds an event
class RenderableEvent(models.Model):
    __slots__ = ('summary', 'start_date', 'start_time', 'end_time', 'desc', 'pureTime', 'location')

    def __init__(self, summ, sdate, stime, etime, d, stimePure, loc):
        self.summary = summ
        self.start_date = sdate
        self.start_time = stime
        self.end_time = etime
        self.desc = d
        self.pureTime = stimePure
        self.location = loc

    def __str__(self):
        return self.summary + " " + self.start_date + " " + self.start_time + " " + self.end_time + " " + self.location

# RenderableEvents - holds all events
class RenderableEvents(models.Model):
    __slots__ = ('events')

    def __init__(self):
        self.events = []

    def getEvents(self):
        icalFile = urllib.request.urlopen(
            'http://www.google.com/calendar/ical/calendar%40csc.cs.rit.edu/public/basic.ics')
        ical = Calendar.from_ical(icalFile.read())

        lt = LocalTimezone()

        for thing in ical.walk():

            eventtime = thing.get('dtstart')
            if eventtime != None:
                offset = lt.utcoffset(eventtime.dt)

            loc = thing.get('location')
            if (loc == None) or (loc == "") or (loc == "TBD"):
                loc = "TBD"

            if thing.name == "VEVENT" and eventtime.dt.replace(tzinfo=None) + offset > datetime.today() - timedelta(
                    days=1):

                event = RenderableEvent(
                    thing.get('summary'),
                    (eventtime.dt.replace(tzinfo=None) + offset).strftime("%m/%d/%Y"),
                    (eventtime.dt.replace(tzinfo=None) + offset).strftime("%I:%M %p"),
                    (thing.get('dtend').dt.replace(tzinfo=None) + offset).strftime("%I:%M %p"),
                    thing.get('description'),
                    (eventtime.dt.replace(tzinfo=None) + offset),
                    loc)

                self.events.append(event)

            elif thing.name == "VEVENT" and thing.get('RRULE') is not None:
                repeats = list(rrule.rrulestr(thing.get('RRULE').to_ical().decode('unicode_escape'), ignoretz=True,
                                              dtstart=datetime.now()))
                if (len(repeats) <= 0):
                    continue

		if(thing.get('summary') == 'General Meeting!'):
                    continue

                self.events.append(
                    RenderableEvent(thing.get('summary'), (repeats[0].replace(tzinfo=None)).strftime("%m/%d/%Y"),
                                    (thing.get('dtstart').dt.replace(tzinfo=None)).strftime("%I:%M %p"),
                                    (thing.get('dtend').dt.replace(tzinfo=None)).strftime("%I:%M %p"),
                                    thing.get('description'),
                                    (repeats[0].replace(tzinfo=None)), loc))

        # Sort events by date and time!
        self.events = sorted(self.events, key=lambda renderable_event: renderable_event.pureTime)

        icalFile.close()

<commit_msg>fixed indentation issue in models<commit_after>from django.db import models
from django.utils.encoding import force_bytes
from django.utils import timezone

# dependent on icalendar package - pip install icalendar
from icalendar import Calendar, Event, vDatetime, LocalTimezone
from datetime import datetime, timedelta
import urllib.request, urllib.error, urllib.parse
import os
from csc_new import settings

import dateutil.rrule as rrule

# Create your models here.
class ExamReview(models.Model):
    title = models.CharField(max_length=100)
    questions = models.FileField(upload_to="exam_reviews")
    answers = models.FileField(upload_to="exam_reviews")
    last_modified = models.DateTimeField(auto_now_add=True, auto_now=True)

    def __str__(self):
        return '%s' % (self.title)

    def delete(self, *args, **kwargs):
        os.remove(os.path.join(settings.MEDIA_ROOT, str(self.questions)))
        os.remove(os.path.join(settings.MEDIA_ROOT, str(self.answers)))
        super(ExamReview, self).delete(*args, **kwargs)

class Photo(models.Model):
    title = models.CharField(max_length=100)
    desc = models.CharField(max_length=255)
    src = models.FileField(upload_to="photos")

    def __str__(self):
        return self.title

    def delete(self, *args, **kwargs):
        os.remove(os.path.join(settings.MEDIA_ROOT, str(self.src)))
        super(Photo, self).delete(*args, **kwargs)

# RenderableEvent - holds an event
class RenderableEvent(models.Model):
    __slots__ = ('summary', 'start_date', 'start_time', 'end_time', 'desc', 'pureTime', 'location')

    def __init__(self, summ, sdate, stime, etime, d, stimePure, loc):
        self.summary = summ
        self.start_date = sdate
        self.start_time = stime
        self.end_time = etime
        self.desc = d
        self.pureTime = stimePure
        self.location = loc

    def __str__(self):
        return self.summary + " " + self.start_date + " " + self.start_time + " " + self.end_time + " " + self.location

# RenderableEvents - holds all events
class RenderableEvents(models.Model):
    __slots__ = ('events')

    def __init__(self):
        self.events = []

    def getEvents(self):
        icalFile = urllib.request.urlopen(
            'http://www.google.com/calendar/ical/calendar%40csc.cs.rit.edu/public/basic.ics')
        ical = Calendar.from_ical(icalFile.read())

        lt = LocalTimezone()

        for thing in ical.walk():

            eventtime = thing.get('dtstart')
            if eventtime != None:
                offset = lt.utcoffset(eventtime.dt)

            loc = thing.get('location')
            if (loc == None) or (loc == "") or (loc == "TBD"):
                loc = "TBD"

            if thing.name == "VEVENT" and eventtime.dt.replace(tzinfo=None) + offset > datetime.today() - timedelta(
                    days=1):

                event = RenderableEvent(
                    thing.get('summary'),
                    (eventtime.dt.replace(tzinfo=None) + offset).strftime("%m/%d/%Y"),
                    (eventtime.dt.replace(tzinfo=None) + offset).strftime("%I:%M %p"),
                    (thing.get('dtend').dt.replace(tzinfo=None) + offset).strftime("%I:%M %p"),
                    thing.get('description'),
                    (eventtime.dt.replace(tzinfo=None) + offset),
                    loc)

                self.events.append(event)

            elif thing.name == "VEVENT" and thing.get('RRULE') is not None:
                repeats = list(rrule.rrulestr(thing.get('RRULE').to_ical().decode('unicode_escape'), ignoretz=True,
                                              dtstart=datetime.now()))
                if (len(repeats) <= 0):
                    continue

		if(thing.get('summary') == 'General Meeting!'):
			continue

                self.events.append(
                    RenderableEvent(thing.get('summary'), (repeats[0].replace(tzinfo=None)).strftime("%m/%d/%Y"),
                                    (thing.get('dtstart').dt.replace(tzinfo=None)).strftime("%I:%M %p"),
                                    (thing.get('dtend').dt.replace(tzinfo=None)).strftime("%I:%M %p"),
                                    thing.get('description'),
                                    (repeats[0].replace(tzinfo=None)), loc))

        # Sort events by date and time!
        self.events = sorted(self.events, key=lambda renderable_event: renderable_event.pureTime)

        icalFile.close()

<|endoftext|>


<commit_before>from collections import deque
from random import randint

from sympy import Mul, Basic
from sympy.matrices import Matrix, eye
from sympy.physics.quantum.gate import (X, Y, Z, H, S, T, CNOT,
        IdentityGate, gate_simp)
from sympy.physics.quantum.represent import represent

__all__ = [
    'GateIdentity',
    'is_scalar_matrix',
    'bfs_identity_search',
    'random_identity_search'
]

class GateIdentity(Basic):
    """Wrapper class for circuits that reduce to a scalar value."""

    def __new__(cls, circuit):
        # circuit should be a tuple
        obj = Basic.__new__(cls, circuit)
        obj._circuit = circuit
        return obj

    @property
    def circuit(self):
        return self._circuit

    def generate_gate_rules():
        raise NotImplementedError(
            "Generation of gate rules from identity not implemented."
        )

    def __str__(self):
        """Returns the string of gates in a tuple."""
        return str(self.circuit)

# Dynamically generate and concatenate a list of all possible
# sympy gate objects in given space.
def construct_gate_list(numqubits):
    Xs = [X(i) for i in xrange(numqubits)]
    Ys = [Y(i) for i in xrange(numqubits)]
    Zs = [Z(i) for i in xrange(numqubits)]
    Hs = [H(i) for i in xrange(numqubits)]
    Ss = [S(i) for i in xrange(numqubits)]
    Ts = [T(i) for i in xrange(numqubits)]
    CNOTs = [CNOT(i,j) for i in xrange(numqubits)
                           for j in xrange(numqubits) if i != j]

    gate_list = Xs+Ys+Zs+Hs+Ss+Ts+CNOTs

    return gate_list

# Dynamically generate and concatenate a list of all possible scipy.sparse gate matrices in given space.
def construct_matrix_list(numqubits):
    xs = [represent(X(i), nqubits=numqubits, format='scipy.sparse')
          for i in xrange(numqubits)]
    ys = [represent(Y(i), nqubits=numqubits, format='scipy.sparse')
          for i in xrange(numqubits)]
    zs = [represent(Z(i), nqubits=numqubits, format='scipy.sparse')
          for i in xrange(numqubits)]
    hs = [represent(H(i), nqubits=numqubits, format='scipy.sparse')
          for i in xrange(numqubits)]
    ss = [represent(S(i), nqubits=numqubits, format='scipy.sparse')
          for i in xrange(numqubits)]
    ts = [represent(T(i), nqubits=numqubits, format='scipy.sparse')
          for i in xrange(numqubits)]
    cnots = [represent(CNOT(i,j), nqubits=numqubits, format='scipy.sparse')
             for i in xrange(numqubits) for j in xrange(numqubits) if i != j]

    matrix_list = xs+ys+zs+hs+ss+ts+cnots

    return matrix_list

def is_scalar_matrix(matrix):
    """Checks if given scipy.sparse matrix is a scalar matrix."""

    if list(matrix.nonzero()[0]) == list(matrix.nonzero()[1]):
        diag = list(matrix.diagonal())
        if diag.count(diag[0]) == len(diag):
            return True
    return False

def write_to_file(identityset, filename, append=False):
    """Write gate identities to a specified file."""

    # File is truncated unless given a flag to not truncate.
    identities = open(filename, 'a') if append else open(filename, 'w+')

    for identity in identityset:
        identities.write(str(identity) + "\n")

    identities.close()

def bfs_identity_search(gate_list, numqubits, max_depth=0):
    # Breadth first search might be more efficient because it eliminates
    # a search down paths like ZZZZZZ or XXXXYY.
    # Returns the set of gate identities from the list.

    # If max depth of a path isn't given, use the length of the gate_list
    if (max_depth == 0):
        max_depth = len(gate_list)

    # Root of BFS tree is an IdentityGate(0)
    id_gate = IdentityGate(0)
    queue = deque([(id_gate,)])

    # Create an empty set of gate identities
    ids = set()

    # Begin searching for gate identities in given space.
    while (queue.__len__() > 0):
        current_circuit = queue.popleft()

        for next_gate in gate_list:
            new_circuit = current_circuit + (next_gate,)
            matrix_version = represent(Mul(*new_circuit), nqubits=numqubits,
                                       format='scipy.sparse')

            # In many cases when the matrix is a scalar value,
            # the evaluated matrix will actually be an integer
            if (isinstance(matrix_version, int)):
                # When adding a gate identity, remove the
                # identity gate at the beginning of the tuple
                ids.add(GateIdentity(new_circuit[1:]))

            # If a matrix is equivalent to a scalar value is found
            elif (is_scalar_matrix(matrix_version)):
                # When adding a gate identity, remove the
                # identity gate at the beginning of the tuple
                ids.add(GateIdentity(new_circuit[1:]))

            # Number of operators in the circuit gives the
            # number of gates in the circuit
            elif (len(new_circuit) < max_depth + 1):
                queue.append(new_circuit)

    return ids

def random_identity_search(gate_list, numgates, numqubits):
    """Randomly selects numgates from gate_list and checks if it is
       a gate identity.

       If the circuit is a gate identity, the circuit is returned;
       Otherwise, None is returned
    """

    gate_size = len(gate_list)
    circuit = IdentityGate(0)

    for i in range(numgates):
        next_gate = gate_list[randint(0, gate_size - 1)]
        circuit = initial_circuit*next_gate

    matrix_version = represent(circuit, nqubits=numqubits,
                               format='scipy.sparse')

    return circuit if is_scalar_matrix(matrix_version) else None
<commit_msg>Removed write_to_file because no longer needed. Use len(collection) instead of private function. Starting to write a function to generate permutation of gate sequence (recursive and nonrecursive).<commit_after>from collections import deque
from random import randint

from sympy import Mul, Basic
from sympy.matrices import Matrix, eye
from sympy.physics.quantum.gate import (X, Y, Z, H, S, T, CNOT,
        IdentityGate, gate_simp)
from sympy.physics.quantum.represent import represent

__all__ = [
    'generate_gate_rules'
    'GateIdentity',
    'is_scalar_matrix',
    'bfs_identity_search',
    'random_identity_search'
]

def generate_gate_rules_recursive(gate_seq, recurse_pt):

    seq = list(gate_seq)
    for i in range(recurse_pt, len(gate_seq)):
        generate_gate_rules_recursive(seq, i)

    return None

def generate_gate_rules(gate_seq):
    # Not recursive version - unsure if interpreter
    # optimizes recursion
    return None

class GateIdentity(Basic):
    """Wrapper class for circuits that reduce to a scalar value."""

    def __new__(cls, circuit):
        # circuit should be a tuple
        obj = Basic.__new__(cls, circuit)
        obj._circuit = circuit
        obj._gate_rules = generate_gate_rules(circuit)

        return obj

    @property
    def circuit(self):
        return self._circuit

    @property
    def gate_rules(self):
        return self._gate_rules

    def __str__(self):
        """Returns the string of gates in a tuple."""
        return str(self.circuit)

# Dynamically generate and concatenate a list of all possible
# sympy gate objects in given space.
def construct_gate_list(numqubits):
    Xs = [X(i) for i in xrange(numqubits)]
    Ys = [Y(i) for i in xrange(numqubits)]
    Zs = [Z(i) for i in xrange(numqubits)]
    Hs = [H(i) for i in xrange(numqubits)]
    Ss = [S(i) for i in xrange(numqubits)]
    Ts = [T(i) for i in xrange(numqubits)]
    CNOTs = [CNOT(i,j) for i in xrange(numqubits)
                           for j in xrange(numqubits) if i != j]

    gate_list = Xs+Ys+Zs+Hs+Ss+Ts+CNOTs

    return gate_list

# Dynamically generate and concatenate a list of all possible
# scipy.sparse gate matrices in given space.
def construct_matrix_list(numqubits):
    xs = [represent(X(i), nqubits=numqubits, format='scipy.sparse')
          for i in xrange(numqubits)]
    ys = [represent(Y(i), nqubits=numqubits, format='scipy.sparse')
          for i in xrange(numqubits)]
    zs = [represent(Z(i), nqubits=numqubits, format='scipy.sparse')
          for i in xrange(numqubits)]
    hs = [represent(H(i), nqubits=numqubits, format='scipy.sparse')
          for i in xrange(numqubits)]
    ss = [represent(S(i), nqubits=numqubits, format='scipy.sparse')
          for i in xrange(numqubits)]
    ts = [represent(T(i), nqubits=numqubits, format='scipy.sparse')
          for i in xrange(numqubits)]
    cnots = [represent(CNOT(i,j), nqubits=numqubits, format='scipy.sparse')
             for i in xrange(numqubits) for j in xrange(numqubits) if i != j]

    matrix_list = xs+ys+zs+hs+ss+ts+cnots

    return matrix_list

def is_scalar_matrix(matrix):
    """Checks if given scipy.sparse matrix is a scalar matrix."""

    if (list(matrix.nonzero()[0]) == list(matrix.nonzero()[1])):
        diag = list(matrix.diagonal())
        if (diag.count(diag[0]) == len(diag)):
            return True
    return False

def bfs_identity_search(gate_list, numqubits, max_depth=0):
    # Breadth first search might be more efficient because it eliminates
    # a search down paths like ZZZZZZ or XXXXYY.
    # Returns the set of gate identities from the list.

    # If max depth of a path isn't given, use the length of the gate_list
    if (max_depth == 0):
        max_depth = len(gate_list)

    # Root of BFS tree is an IdentityGate(0)
    id_gate = IdentityGate(0)
    queue = deque([(id_gate,)])

    # Create an empty set of gate identities
    ids = set()

    # Begin searching for gate identities in given space.
    while (len(queue) > 0):
        current_circuit = queue.popleft()

        for next_gate in gate_list:
            new_circuit = current_circuit + (next_gate,)
            matrix_version = represent(Mul(*new_circuit), nqubits=numqubits,
                                       format='scipy.sparse')

            # In many cases when the matrix is a scalar value,
            # the evaluated matrix will actually be an integer
            if (isinstance(matrix_version, int)):
                # When adding a gate identity, remove the
                # identity gate at the beginning of the tuple
                ids.add(GateIdentity(new_circuit[1:]))

            # If a matrix is equivalent to a scalar value is found
            elif (is_scalar_matrix(matrix_version)):
                # When adding a gate identity, remove the
                # identity gate at the beginning of the tuple
                ids.add(GateIdentity(new_circuit[1:]))

            # Number of operators in the circuit gives the
            # number of gates in the circuit
            elif (len(new_circuit) < max_depth + 1):
                queue.append(new_circuit)

    return ids

def random_identity_search(gate_list, numgates, numqubits):
    """Randomly selects numgates from gate_list and checks if it is
       a gate identity.

       If the circuit is a gate identity, the circuit is returned;
       Otherwise, None is returned
    """

    gate_size = len(gate_list)
    circuit = IdentityGate(0)

    for i in range(numgates):
        next_gate = gate_list[randint(0, gate_size - 1)]
        circuit = initial_circuit*next_gate

    matrix_version = represent(circuit, nqubits=numqubits,
                               format='scipy.sparse')

    return circuit if is_scalar_matrix(matrix_version) else None
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Change directory to provide relative paths for doctests
   >>> import os
   >>> filepath = os.path.dirname( os.path.realpath( __file__ ) )
   >>> datadir = os.path.realpath(os.path.join(filepath, '../../testing/data'))
   >>> os.chdir(datadir)
"""
from nipype.interfaces.base import (
    TraitedSpec, BaseInterface, BaseInterfaceInputSpec,
    File, isdefined, traits)
from nipype.utils.filemanip import split_filename
import os.path as op
import nibabel as nb
import nibabel.trackvis as nbt
from nipype.utils.misc import package_check
import warnings

from ... import logging
iflogger = logging.getLogger('interface')

have_dipy = True
try:
    package_check('dipy', version='0.6.0')
except Exception, e:
    have_dipy = False
else:
    from dipy.tracking.utils import density_map

class TrackDensityMapInputSpec(TraitedSpec):
    in_file = File(exists=True, mandatory=True,
                   desc='The input TrackVis track file')
    reference = File(exists=True,
                     desc='A reference file to define RAS coordinates space')
    points_space = traits.Enum('rasmm', 'voxel', None, usedefault=True,
                               desc='coordinates of trk file')

    voxel_dims = traits.List(traits.Float, minlen=3, maxlen=3,
                             desc='The size of each voxel in mm.')
    data_dims = traits.List(traits.Int, minlen=3, maxlen=3,
                            desc='The size of the image in voxels.')
    out_filename = File('tdi.nii', usedefault=True,
                        desc=('The output filename for the tracks in TrackVis '
                              '(.trk) format'))

class TrackDensityMapOutputSpec(TraitedSpec):
    out_file = File(exists=True)

class TrackDensityMap(BaseInterface):

    """
    Creates a tract density image from a TrackVis track file using functions
    from dipy

    Example
    -------

    >>> import nipype.interfaces.dipy as dipy
    >>> trk2tdi = dipy.TrackDensityMap()
    >>> trk2tdi.inputs.in_file = 'converted.trk'
    >>> trk2tdi.run()                                   # doctest: +SKIP

    """
    input_spec = TrackDensityMapInputSpec
    output_spec = TrackDensityMapOutputSpec

    def _run_interface(self, runtime):
        tracks, header = nbt.read(self.inputs.in_file)
        streams = ((ii[0]) for ii in tracks)

        if isdefined(self.inputs.reference):
            refnii = nb.load(self.inputs.reference)
            affine = refnii.get_affine()
            data_dims = refnii.get_shape()[:3]
            kwargs = dict(affine=affine)
        else:
            iflogger.warn(('voxel_dims and data_dims are deprecated'
                           'as of dipy 0.7.1. Please use reference '
                           'input instead'))

            if not isdefined(self.inputs.data_dims):
                data_dims = header['dim']
            else:
                data_dims = self.inputs.data_dims
            if not isdefined(self.inputs.voxel_dims):
                voxel_size = header['voxel_size']
            else:
                voxel_size = self.inputs.voxel_dims

            affine = header['vox_to_ras']
            kwargs = dict(voxel_size=voxel_size)

        data = density_map(streams, data_dims, **kwargs)
        data = data.astype(np.min_scalar_type(data.max()))
        img = nb.Nifti1Image(data, affine)
        out_file = op.abspath(self.inputs.out_filename)
        nb.save(img, out_file)

        iflogger.info(
            ('Track density map saved as {i}, size={d}, '
             'dimensions={v}').format(
                i=out_file,
                d=img.get_shape(),
                v=img.get_header().get_zooms()))
        return runtime

    def _list_outputs(self):
        outputs = self._outputs().get()
        outputs['out_file'] = op.abspath(self.inputs.out_filename)
        return outputs
<commit_msg>fix missing import in dipy<commit_after># -*- coding: utf-8 -*-
"""
Change directory to provide relative paths for doctests
   >>> import os
   >>> filepath = os.path.dirname( os.path.realpath( __file__ ) )
   >>> datadir = os.path.realpath(os.path.join(filepath, '../../testing/data'))
   >>> os.chdir(datadir)
"""
from nipype.interfaces.base import (
    TraitedSpec, BaseInterface, BaseInterfaceInputSpec,
    File, isdefined, traits)
from nipype.utils.filemanip import split_filename
import os.path as op
import nibabel as nb
import nibabel.trackvis as nbt
from nipype.utils.misc import package_check
import warnings

from ... import logging
iflogger = logging.getLogger('interface')

have_dipy = True
try:
    package_check('dipy', version='0.6.0')
except Exception, e:
    have_dipy = False
else:
    from dipy.tracking.utils import density_map

class TrackDensityMapInputSpec(TraitedSpec):
    in_file = File(exists=True, mandatory=True,
                   desc='The input TrackVis track file')
    reference = File(exists=True,
                     desc='A reference file to define RAS coordinates space')
    points_space = traits.Enum('rasmm', 'voxel', None, usedefault=True,
                               desc='coordinates of trk file')

    voxel_dims = traits.List(traits.Float, minlen=3, maxlen=3,
                             desc='The size of each voxel in mm.')
    data_dims = traits.List(traits.Int, minlen=3, maxlen=3,
                            desc='The size of the image in voxels.')
    out_filename = File('tdi.nii', usedefault=True,
                        desc=('The output filename for the tracks in TrackVis '
                              '(.trk) format'))

class TrackDensityMapOutputSpec(TraitedSpec):
    out_file = File(exists=True)

class TrackDensityMap(BaseInterface):

    """
    Creates a tract density image from a TrackVis track file using functions
    from dipy

    Example
    -------

    >>> import nipype.interfaces.dipy as dipy
    >>> trk2tdi = dipy.TrackDensityMap()
    >>> trk2tdi.inputs.in_file = 'converted.trk'
    >>> trk2tdi.run()                                   # doctest: +SKIP

    """
    input_spec = TrackDensityMapInputSpec
    output_spec = TrackDensityMapOutputSpec

    def _run_interface(self, runtime):
        from numpy import min_scalar_type
        tracks, header = nbt.read(self.inputs.in_file)
        streams = ((ii[0]) for ii in tracks)

        if isdefined(self.inputs.reference):
            refnii = nb.load(self.inputs.reference)
            affine = refnii.get_affine()
            data_dims = refnii.get_shape()[:3]
            kwargs = dict(affine=affine)
        else:
            iflogger.warn(('voxel_dims and data_dims are deprecated'
                           'as of dipy 0.7.1. Please use reference '
                           'input instead'))

            if not isdefined(self.inputs.data_dims):
                data_dims = header['dim']
            else:
                data_dims = self.inputs.data_dims
            if not isdefined(self.inputs.voxel_dims):
                voxel_size = header['voxel_size']
            else:
                voxel_size = self.inputs.voxel_dims

            affine = header['vox_to_ras']
            kwargs = dict(voxel_size=voxel_size)

        data = density_map(streams, data_dims, **kwargs)
        data = data.astype(min_scalar_type(data.max()))
        img = nb.Nifti1Image(data, affine)
        out_file = op.abspath(self.inputs.out_filename)
        nb.save(img, out_file)

        iflogger.info(
            ('Track density map saved as {i}, size={d}, '
             'dimensions={v}').format(
                i=out_file,
                d=img.get_shape(),
                v=img.get_header().get_zooms()))
        return runtime

    def _list_outputs(self):
        outputs = self._outputs().get()
        outputs['out_file'] = op.abspath(self.inputs.out_filename)
        return outputs
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import os.path
import re
import subprocess
import sys
import threading
import time
import tempfile

from nimp.utilities.logging import *
from nimp.utilities.system import *
from nimp.utilities.windows_utilities import *

#-------------------------------------------------------------------------------
def _default_log_callback(line, log_level):
    log_message(log_level, False, line)

#-------------------------------------------------------------------------------
def _sanitize_command(command):
    # If we’re running under MSYS, leading slashes in command line arguments
    # will be treated as a path, so we need to escape them, except if the given
    # argument is indeed a file
    if is_msys():
        tmp = []
        for x in command:
            if len(x) > 5 and (os.path.isfile(x) or os.path.isdir(x)):
                tmp.append(x)
            else:
                tmp.append(re.sub('^/', '//', x))
        command = tmp

    return command

#-------------------------------------------------------------------------------
def capture_process_output(directory, command, input = None, encoding = 'utf-8'):
    command = _sanitize_command(command)
    log_verbose("Running “{0}” in “{1}”", " ".join(command), os.path.abspath(directory))
    process = subprocess.Popen(command,
                               cwd    = directory,
                               stdout = subprocess.PIPE,
                               stderr = subprocess.PIPE,
                               stdin  = subprocess.PIPE)
    output, error = process.communicate(input.encode(encoding) if input else None)

    return process.wait(), output.decode(encoding), error.decode(encoding)

#-------------------------------------------------------------------------------
def call_process(directory, command, stdout_callback = _default_log_callback,
                                     stderr_callback = None,
                                     heartbeat = 0):
    command = _sanitize_command(command)
    log_verbose("Running “{0}” in “{1}”", " ".join(command), os.path.abspath(directory))

    disable_win32_dialogs()

    debug_pipe = OutputDebugStringLogger()

    # The bufsize = 1 is important; if we don’t bufferise the
    # output, we’re going to make the callee lag a lot.
    process = subprocess.Popen(command,
                               cwd     = directory,
                               stdout  = subprocess.PIPE,
                               stderr  = subprocess.PIPE,
                               stdin   = None,
                               bufsize = 1)
    debug_pipe.attach(process.pid)
    debug_pipe.start()

    def heartbeat_worker(heartbeat):
        t = time.monotonic()
        while process:
            if heartbeat > 0 and time.monotonic() > t + heartbeat:
                log_verbose("Keepalive for {0}", command[0])
                t += heartbeat
            time.sleep(0.050)

    def output_worker(user_callback, log_function, pipe):
        output_buffer = ""
        # FIXME: it would be better to use while process.poll() == None
        # here, but thread safety issues in Python < 3.4 prevent it.
        while process:
            try:
                for line in iter(pipe.readline, ''):
                    try:
                        line = line.decode("utf-8")
                    except:
                        line = line.decode("cp850")

                    if line == '':
                        break

                    line = line.replace("{", "{{").replace("}", "}}")
                    line = line.rstrip('\r\n')

                    user_callback(line, log_function)

                # Sleep for 10 milliseconds if there was no data,
                # or we’ll hog the CPU.
                time.sleep(0.010)

            except ValueError:
                return

    if not stderr_callback:
        stderr_callback = stdout_callback

    log_thread_args = [ (stdout_callback, LOG_LEVEL_VERBOSE, process.stdout),
                        (stderr_callback, LOG_LEVEL_ERROR,   process.stderr),
                        (stderr_callback, LOG_LEVEL_VERBOSE, debug_pipe.output)]
    worker_threads = [ threading.Thread(target = output_worker, args = args) for args in log_thread_args ]
    # Send keepalive to stderr if requested
    if heartbeat > 0:
        worker_threads += [ threading.Thread(target = heartbeat_worker, args = (heartbeat, )) ]

    for thread in worker_threads:
        thread.start()

    try:
        process_return = process.wait()
    finally:
        process = None
        debug_pipe.stop()
        for thread in worker_threads:
            thread.join()

    if process_return == 0:
        log = log_verbose
    else:
        log = log_error
    log("Program “{0}” finished with exit code {1}", command[0], process_return)

    return process_return

<commit_msg>Minor improvement to the commandline heuristics.<commit_after># -*- coding: utf-8 -*-

import os
import os.path
import re
import subprocess
import sys
import threading
import time
import tempfile

from nimp.utilities.logging import *
from nimp.utilities.system import *
from nimp.utilities.windows_utilities import *

#-------------------------------------------------------------------------------
def _default_log_callback(line, log_level):
    log_message(log_level, False, line)

#-------------------------------------------------------------------------------
def _sanitize_command(command):
    # If we’re running under MSYS, leading slashes in command line arguments
    # will be treated as a path, so we need to escape them, except if the given
    # argument is indeed a file
    if is_msys():
        tmp = []
        for x in command:
            if x[0] == '/':
                # If the argument starts with /, we may wish to rewrite it
                if len(x) >= 2 and x[1].isalpha() and x[2] == '/':
                    # Looks like a path with a drive letter, keep it that way
                    pass
                elif len(x) > 5 and (os.path.isfile(x) or os.path.isdir(x)):
                    pass
                else:
                    x = '/' + x
            tmp.append(x)
        command = tmp

    return command

#-------------------------------------------------------------------------------
def capture_process_output(directory, command, input = None, encoding = 'utf-8'):
    command = _sanitize_command(command)
    log_verbose("Running “{0}” in “{1}”", " ".join(command), os.path.abspath(directory))
    process = subprocess.Popen(command,
                               cwd    = directory,
                               stdout = subprocess.PIPE,
                               stderr = subprocess.PIPE,
                               stdin  = subprocess.PIPE)
    output, error = process.communicate(input.encode(encoding) if input else None)

    return process.wait(), output.decode(encoding), error.decode(encoding)

#-------------------------------------------------------------------------------
def call_process(directory, command, stdout_callback = _default_log_callback,
                                     stderr_callback = None,
                                     heartbeat = 0):
    command = _sanitize_command(command)
    log_verbose("Running “{0}” in “{1}”", " ".join(command), os.path.abspath(directory))

    disable_win32_dialogs()

    debug_pipe = OutputDebugStringLogger()

    # The bufsize = 1 is important; if we don’t bufferise the
    # output, we’re going to make the callee lag a lot.
    process = subprocess.Popen(command,
                               cwd     = directory,
                               stdout  = subprocess.PIPE,
                               stderr  = subprocess.PIPE,
                               stdin   = None,
                               bufsize = 1)
    debug_pipe.attach(process.pid)
    debug_pipe.start()

    def heartbeat_worker(heartbeat):
        t = time.monotonic()
        while process:
            if heartbeat > 0 and time.monotonic() > t + heartbeat:
                log_verbose("Keepalive for {0}", command[0])
                t += heartbeat
            time.sleep(0.050)

    def output_worker(user_callback, log_function, pipe):
        output_buffer = ""
        # FIXME: it would be better to use while process.poll() == None
        # here, but thread safety issues in Python < 3.4 prevent it.
        while process:
            try:
                for line in iter(pipe.readline, ''):
                    try:
                        line = line.decode("utf-8")
                    except:
                        line = line.decode("cp850")

                    if line == '':
                        break

                    line = line.replace("{", "{{").replace("}", "}}")
                    line = line.rstrip('\r\n')

                    user_callback(line, log_function)

                # Sleep for 10 milliseconds if there was no data,
                # or we’ll hog the CPU.
                time.sleep(0.010)

            except ValueError:
                return

    if not stderr_callback:
        stderr_callback = stdout_callback

    log_thread_args = [ (stdout_callback, LOG_LEVEL_VERBOSE, process.stdout),
                        (stderr_callback, LOG_LEVEL_ERROR,   process.stderr),
                        (stderr_callback, LOG_LEVEL_VERBOSE, debug_pipe.output)]
    worker_threads = [ threading.Thread(target = output_worker, args = args) for args in log_thread_args ]
    # Send keepalive to stderr if requested
    if heartbeat > 0:
        worker_threads += [ threading.Thread(target = heartbeat_worker, args = (heartbeat, )) ]

    for thread in worker_threads:
        thread.start()

    try:
        process_return = process.wait()
    finally:
        process = None
        debug_pipe.stop()
        for thread in worker_threads:
            thread.join()

    if process_return == 0:
        log = log_verbose
    else:
        log = log_error
    log("Program “{0}” finished with exit code {1}", command[0], process_return)

    return process_return

<|endoftext|>