<commit_before>import logging
import multiprocessing
import multiprocessing.queues
import sys
import threading
import time

import numpy

from smqtk.utils import SmqtkObject

__author__ = 'paul.tunison@kitware.com'

__all__ = [
    'elements_to_matrix',
]

def elements_to_matrix(descr_elements, mat=None, procs=None, buffer_factor=2,
                       report_interval=None):
    """
    Add to or create a numpy matrix, adding to it the vector data contained in
    a sequence of DescriptorElement instances using asynchronous processing.

    If ``mat`` is provided, its shape must equal:
        ( len(descr_elements) , descr_elements[0].size )

    :param descr_elements: Sequence of DescriptorElement objects to transform
        into a matrix. Each element should contain descriptor vectors of the
        same size.
    :type descr_elements:
        collections.Sequence[smqtk.representation.DescriptorElement]

    :param mat: Optionally pre-constructed numpy matrix of the appropriate shape
        to as loaded vectors into. If supplied this must have rows of the shape:
        (nDescriptors, nFeatures). If this is not supplied, we create a new
        matrix to insert vectors into.
    :type mat: None | numpy.core.multiarray.ndarray

    :param procs: Optional specification of the number of cores to use. If None,
        we will attempt to use all available cores.
    :type procs: None | int | long

    :param buffer_factor: Multiplier against the number of processes used to
        limit the growth size of the result queue coming from worker processes.
    :type buffer_factor: float

    :param report_interval: Optional interval in seconds for debug logging to
        occur reporting about conversion speed. This should be greater than 0
        if this debug logging is desired.
    :type report_interval: None | float

    :return: Created or input matrix.
    :rtype: numpy.core.multiarray.ndarray

    """
    log = logging.getLogger(__name__)

    # Create/check matrix
    if mat is None:
        shp = (len(descr_elements),
               descr_elements[0].vector().size)
        log.debug("Creating new matrix with shape: %s", shp)
        mat = numpy.ndarray(shp, float)
    else:
        assert mat.shape[0] == len(descr_elements)
        assert mat.shape[1] == descr_elements[0].vector().size

    if procs is None:
        procs = multiprocessing.cpu_count()

    in_q = multiprocessing.Queue()
    out_q = multiprocessing.Queue(int(procs * buffer_factor))
    log.debug("Output queue size: %d", out_q._maxsize)

    # Workers for async extraction
    log.debug("constructing worker processes")
    workers = [_ElemVectorExtractorProcess(i, in_q, out_q) for i in xrange(procs)]

    in_queue_t = _FeedQueueThread(descr_elements, in_q, mat, len(workers))

    try:
        # Start worker processes
        log.debug("starting worker processes")
        for w in workers:
            w.daemon = True
            w.start()

        log.debug("Sending work packets")
        in_queue_t.start()

        # Collect work from async
        log.debug("Aggregating async results")
        terminals_collected = 0
        f = 0
        lt = t = time.time()
        while terminals_collected < len(workers):
            packet = out_q.get()
            if packet is None:
                terminals_collected += 1
            else:
                r, v = packet
                mat[r] = v

                f += 1
                if report_interval and time.time() - lt >= report_interval:
                    log.debug("Rows per second: %f, Total: %d",
                              f / (time.time() - t), f)
                    lt = time.time()

        log.debug("Closing output queue")
        out_q.close()

        # All work should be exhausted at this point
        if sys.platform == 'darwin':
            # qsize doesn't work on OSX
            # Try to get something from each queue, expecting an empty exception
            try:
                in_q.get(block=False)
            except multiprocessing.queues.Empty:
                pass
            else:
                raise AssertionError("In queue not empty")
            try:
                out_q.get(block=False)
            except multiprocessing.queues.Empty:
                pass
            else:
                raise AssertionError("Out queue not empty")
        else:
            assert in_q.qsize() == 0, "In queue not empty"
            assert out_q.qsize() == 0, "Out queue not empty"

        log.debug("Done")
        return mat
    finally:
        log.debug("Stopping/Joining queue feeder thread")
        in_queue_t.stop()
        in_queue_t.join()

        # Forcibly terminate worker processes if still alive
        log.debug("Joining/Terminating workers")
        for w in workers:
            if w.is_alive():
                w.terminate()
            w.join()

        log.debug("Cleaning queues")
        for q in (in_q, out_q):
            q.close()
            q.join_thread()

class _FeedQueueThread (SmqtkObject, threading.Thread):

    def __init__(self, descr_elements, q, out_mat, num_terminal_packets):
        super(_FeedQueueThread, self).__init__()

        self.num_terminal_packets = num_terminal_packets
        self.out_mat = out_mat
        self.q = q
        self.descr_elements = descr_elements

        self._stop = threading.Event()

    def stop(self):
        self._stop.set()

    def stopped(self):
        return self._stop.isSet()

    def run(self):
        # Special case for in-memory storage of descriptors
        from smqtk.representation.descriptor_element.local_elements \
            import DescriptorMemoryElement

        for r, d in enumerate(self.descr_elements):
            if isinstance(d, DescriptorMemoryElement):
                self.out_mat[r] = d.vector()
            else:
                self.q.put((r, d))

            # If we're told to stop, immediately quit out of processing
            if self.stopped():
                break

        self._log.debug("Sending in-queue terminal packets")
        for _ in xrange(self.num_terminal_packets):
            self.q.put(None)
        self._log.debug("Closing in-queue")
        self.q.close()

class _ElemVectorExtractorProcess (SmqtkObject, multiprocessing.Process):
    """
    Helper process for extracting DescriptorElement vectors on a separate
    process. This terminates with a None packet fed to in_q. Otherwise, in_q
    values are expected to be (row, element) pairs. Tuples of the form
    (row, vector) are published to the out_q.

    Terminal value: None

    """

    def __init__(self, i, in_q, out_q):
        super(_ElemVectorExtractorProcess, self)\
            .__init__(name='[w%d]' % i)
        self._log.debug("Making worker (%d, %s, %s)", i, in_q, out_q)
        self.i = i
        self.in_q = in_q
        self.out_q = out_q

    def run(self):
        packet = self.in_q.get()
        while packet is not None:
            row, elem = packet
            v = elem.vector()
            self.out_q.put((row, v))
            packet = self.in_q.get()
        self.out_q.put(None)
<commit_msg>Added thread/process mode option to element-2-matrix function<commit_after>import logging
import multiprocessing
import multiprocessing.queues
from Queue import Queue
import sys
import threading
import time

import numpy

from smqtk.utils import SmqtkObject

__author__ = 'paul.tunison@kitware.com'

__all__ = [
    'elements_to_matrix',
]

def elements_to_matrix(descr_elements, mat=None, procs=None, buffer_factor=2,
                       report_interval=None, use_multiprocessing=False):
    """
    Add to or create a numpy matrix, adding to it the vector data contained in
    a sequence of DescriptorElement instances using asynchronous processing.

    If ``mat`` is provided, its shape must equal:
        ( len(descr_elements) , descr_elements[0].size )

    :param descr_elements: Sequence of DescriptorElement objects to transform
        into a matrix. Each element should contain descriptor vectors of the
        same size.
    :type descr_elements:
        collections.Sequence[smqtk.representation.DescriptorElement]

    :param mat: Optionally pre-constructed numpy matrix of the appropriate shape
        to as loaded vectors into. If supplied this must have rows of the shape:
        (nDescriptors, nFeatures). If this is not supplied, we create a new
        matrix to insert vectors into.
    :type mat: None | numpy.core.multiarray.ndarray

    :param procs: Optional specification of the number of threads/cores to use.
        If None, we will attempt to use all available threads/cores.
    :type procs: None | int | long

    :param buffer_factor: Multiplier against the number of processes used to
        limit the growth size of the result queue coming from worker processes.
    :type buffer_factor: float

    :param report_interval: Optional interval in seconds for debug logging to
        occur reporting about conversion speed. This should be greater than 0
        if this debug logging is desired.
    :type report_interval: None | float

    :param use_multiprocessing: Whether or not to use discrete processes as the
        parallelization agent vs python threads.
    :type use_multiprocessing: bool

    :return: Created or input matrix.
    :rtype: numpy.core.multiarray.ndarray

    """
    log = logging.getLogger(__name__)

    # Create/check matrix
    if mat is None:
        shp = (len(descr_elements),
               descr_elements[0].vector().size)
        log.debug("Creating new matrix with shape: %s", shp)
        mat = numpy.ndarray(shp, float)
    else:
        assert mat.shape[0] == len(descr_elements)
        assert mat.shape[1] == descr_elements[0].vector().size

    if procs is None:
        procs = multiprocessing.cpu_count()

    # Choose parallel types
    if use_multiprocessing:
        queue_t = multiprocessing.Queue
        worker_t = _ElemVectorExtractorProcess
    else:
        queue_t = Queue
        worker_t = _ElemVectorExtractorThread

    in_q = queue_t()
    out_q = queue_t(int(procs * buffer_factor))
    # Workers for async extraction
    log.debug("constructing worker processes")
    workers = [worker_t(i, in_q, out_q) for i in xrange(procs)]

    in_queue_t = _FeedQueueThread(descr_elements, in_q, mat, len(workers))

    try:
        # Start worker processes
        log.debug("starting worker processes")
        for w in workers:
            w.daemon = True
            w.start()

        log.debug("Sending work packets")
        in_queue_t.start()

        # Collect work from async
        log.debug("Aggregating async results")
        terminals_collected = 0
        f = 0
        lt = t = time.time()
        while terminals_collected < len(workers):
            packet = out_q.get()
            if packet is None:
                terminals_collected += 1
            else:
                r, v = packet
                mat[r] = v

                f += 1
                if report_interval and time.time() - lt >= report_interval:
                    log.debug("Rows per second: %f, Total: %d",
                              f / (time.time() - t), f)
                    lt = time.time()

        # All work should be exhausted at this point
        if use_multiprocessing and sys.platform == 'darwin':
            # multiprocessing.Queue.qsize doesn't work on OSX
            # Try to get something from each queue, expecting an empty exception
            try:
                in_q.get(block=False)
            except multiprocessing.queues.Empty:
                pass
            else:
                raise AssertionError("In queue not empty")
            try:
                out_q.get(block=False)
            except multiprocessing.queues.Empty:
                pass
            else:
                raise AssertionError("Out queue not empty")
        else:
            assert in_q.qsize() == 0, "In queue not empty"
            assert out_q.qsize() == 0, "Out queue not empty"

        return mat
    finally:
        log.debug("Stopping/Joining queue feeder thread")
        in_queue_t.stop()
        in_queue_t.join()

        if use_multiprocessing:
            # Forcibly terminate worker processes if still alive
            log.debug("Joining/Terminating process workers")
            for w in workers:
                if w.is_alive():
                    w.terminate()
                w.join()

            log.debug("Cleaning multiprocess queues")
            for q in (in_q, out_q):
                q.close()
                q.join_thread()
        else:
            log.debug("Stopping/Joining threaded workers")
            for w in workers:
                w.stop()
                w.join()

        log.debug("Done")

class _FeedQueueThread (SmqtkObject, threading.Thread):

    def __init__(self, descr_elements, q, out_mat, num_terminal_packets):
        super(_FeedQueueThread, self).__init__()

        self.num_terminal_packets = num_terminal_packets
        self.out_mat = out_mat
        self.q = q
        self.descr_elements = descr_elements

        self._stop = threading.Event()

    def stop(self):
        self._stop.set()

    def stopped(self):
        return self._stop.isSet()

    def run(self):
        try:
            # Special case for in-memory storage of descriptors
            from smqtk.representation.descriptor_element.local_elements \
                import DescriptorMemoryElement

            for r, d in enumerate(self.descr_elements):
                if isinstance(d, DescriptorMemoryElement):
                    self.out_mat[r] = d.vector()
                else:
                    self.q.put((r, d))

                # If we're told to stop, immediately quit out of processing
                if self.stopped():
                    break
        except KeyboardInterrupt:
            pass
        finally:
            self._log.debug("Sending in-queue terminal packets")
            for _ in xrange(self.num_terminal_packets):
                self.q.put(None)
            self._log.debug("Closing in-queue")

class _ElemVectorExtractorProcess (SmqtkObject, multiprocessing.Process):
    """
    Helper process for extracting DescriptorElement vectors on a separate
    process. This terminates with a None packet fed to in_q. Otherwise, in_q
    values are expected to be (row, element) pairs. Tuples of the form
    (row, vector) are published to the out_q.

    Terminal value: None

    """

    def __init__(self, i, in_q, out_q):
        super(_ElemVectorExtractorProcess, self)\
            .__init__(name='[w%d]' % i)
        self._log.debug("Making process worker (%d, %s, %s)", i, in_q, out_q)
        self.i = i
        self.in_q = in_q
        self.out_q = out_q

    def run(self):
        try:
            packet = self.in_q.get()
            while packet is not None:
                row, elem = packet
                v = elem.vector()
                self.out_q.put((row, v))
                packet = self.in_q.get()
            self.out_q.put(None)
        except KeyboardInterrupt:
            pass

class _ElemVectorExtractorThread (SmqtkObject, threading.Thread):
    """
    Helper process for extracting DescriptorElement vectors on a separate
    process. This terminates with a None packet fed to in_q. Otherwise, in_q
    values are expected to be (row, element) pairs. Tuples of the form
    (row, vector) are published to the out_q.

    Terminal value: None

    """

    def __init__(self, i, in_q, out_q):
        SmqtkObject.__init__(self)
        threading.Thread.__init__(self, name='[w%d]' % i)

        self._log.debug("Making thread worker (%d, %s, %s)", i, in_q, out_q)
        self.i = i
        self.in_q = in_q
        self.out_q = out_q

        self._stop = threading.Event()

    def stop(self):
        self._stop.set()

    def stopped(self):
        return self._stop.isSet()

    def run(self):
        try:
            packet = self.in_q.get()
            while packet is not None and not self.stopped():
                row, elem = packet
                v = elem.vector()
                self.out_q.put((row, v))
                packet = self.in_q.get()
            self.out_q.put(None, 1.0)
        except KeyboardInterrupt:
            pass
<|endoftext|>


<commit_before>from os.path import join, split, dirname
import os
import sys
from distutils.dep_util import newer

def msvc_version():
    """Return the msvc version used to build the running python, None if not
    built with MSVC."""
    msc_pos = sys.version.find('MSC v.')
    if msc_pos != -1:
        return sys.version[msc_pos+6:msc_pos+10]
    return None

def configuration(parent_package='',top_path=None):
    from numpy.distutils.misc_util import Configuration, get_mathlibs
    config = Configuration('random',parent_package,top_path)

    def generate_libraries(ext, build_dir):
        config_cmd = config.get_config_cmd()
        libs = get_mathlibs()
        tc = testcode_wincrypt()
        if config_cmd.try_run(tc):
            libs.append('Advapi32')
        ext.libraries.extend(libs)
        return None

    def generate_config_h(ext, build_dir):
        defs = []
        target = join(build_dir, "mtrand", 'config.h')
        dir = dirname(target)
        if not os.path.exists(dir):
            os.makedirs(dir)

        if newer(__file__, target):
            target_f = open(target, 'a')
            for d in defs:
                if isinstance(d, str):
                    target_f.write('#define %s\n' % (d))
            target_f.close()

    libs = []
    # Configure mtrand
    config.add_extension('mtrand',
                         sources=[join('mtrand', x) for x in
                                  ['mtrand.c', 'randomkit.c', 'initarray.c',
                                   'distributions.c']]+[generate_libraries]
                                   + [generate_config_h],
                         libraries=libs,
                         depends = [join('mtrand','*.h'),
                                    join('mtrand','*.pyx'),
                                    join('mtrand','*.pxi'),
                                    ]
                        )

    config.add_data_files(('.', join('mtrand', 'randomkit.h')))
    config.add_data_dir('tests')

    return config

def testcode_wincrypt():
    return """\
/* check to see if _WIN32 is defined */
int main(int argc, char *argv[])
{
#ifdef _WIN32
    return 0;
#else
    return 1;
#endif
}
"""

if __name__ == '__main__':
    from numpy.distutils.core import setup
    setup(configuration=configuration)
<commit_msg>Add a macro to know whether we need the mingw workaround for _ftime + add a define for the MSVCR version.<commit_after>from os.path import join, split, dirname
import os
import sys
from distutils.dep_util import newer
from numpy.distutils.misc_util import msvc_runtime_library

def msvc_version():
    """Return the msvc version used to build the running python, None if not
    built with MSVC."""
    msc_pos = sys.version.find('MSC v.')
    if msc_pos != -1:
        return sys.version[msc_pos+6:msc_pos+10]
    return None

def msvcrt_to_hex(msvc):
    major = msvc / 100
    minor = msvc - major * 100
    return hex(major * 256 + minor)

def configuration(parent_package='',top_path=None):
    from numpy.distutils.misc_util import Configuration, get_mathlibs
    config = Configuration('random',parent_package,top_path)

    def generate_libraries(ext, build_dir):
        config_cmd = config.get_config_cmd()
        libs = get_mathlibs()
        tc = testcode_wincrypt()
        if config_cmd.try_run(tc):
            libs.append('Advapi32')
        ext.libraries.extend(libs)
        return None

    def generate_config_h(ext, build_dir):
        defs = []
        target = join(build_dir, "mtrand", 'config.h')
        dir = dirname(target)
        if not os.path.exists(dir):
            os.makedirs(dir)

        msv = msvc_version()
        if msv and msv >= 1400:
            msvcrt = msvc_runtime_library()
            if msvcrt is None:
                raise ValueError("Discrepancy between " \
                                 "msvc_runtime_library " \
                                 "and our msvc detection scheme ?"
            hmsvc = msvc_to_hex(msvcrt)
            defs.append("NPY_NEEDS_MINGW_TIME_WORKAROUND")
            defs.append(("NPY_MSVCRT_VERSION", str(hmsvc)))

        if newer(__file__, target):
            target_f = open(target, 'a')
            for d in defs:
                if isinstance(d, str):
                    target_f.write('#define %s\n' % (d))
            target_f.close()

    libs = []
    # Configure mtrand
    config.add_extension('mtrand',
                         sources=[join('mtrand', x) for x in
                                  ['mtrand.c', 'randomkit.c', 'initarray.c',
                                   'distributions.c']]+[generate_libraries]
                                   + [generate_config_h],
                         libraries=libs,
                         depends = [join('mtrand','*.h'),
                                    join('mtrand','*.pyx'),
                                    join('mtrand','*.pxi'),
                                    ]
                        )

    config.add_data_files(('.', join('mtrand', 'randomkit.h')))
    config.add_data_dir('tests')

    return config

def testcode_wincrypt():
    return """\
/* check to see if _WIN32 is defined */
int main(int argc, char *argv[])
{
#ifdef _WIN32
    return 0;
#else
    return 1;
#endif
}
"""

if __name__ == '__main__':
    from numpy.distutils.core import setup
    setup(configuration=configuration)
<|endoftext|>


<commit_before># -----------------------------------------------------------------------------
# Copyright (c) 2014--, The Qiita Development Team.
#
# Distributed under the terms of the BSD 3-clause License.
#
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------
from __future__ import division
from collections import namedtuple
from json import dumps

from tornado.web import authenticated, HTTPError
from tornado.gen import coroutine, Task

from qiita_core.exceptions import IncompetentQiitaDeveloperError
from qiita_db.user import User
from qiita_db.study import Study, StudyPerson
from qiita_pet.handlers.base_handlers import BaseHandler
from qiita_pet.handlers.util import study_person_linkifier, pubmed_linkifier

def _get_shared_links_for_study(study):
    shared = []
    for person in study.shared_with:
        person = User(person)
        name = person.info['name']
        email = person.email
        # Name is optional, so default to email if non existant
        if name:
            shared.append(study_person_linkifier(
                (email, name)))
        else:
            shared.append(study_person_linkifier(
                (email, email)))
    return ", ".join(shared)

def _build_study_info(studytype, user=None):
    """builds list of namedtuples for study listings"""
    if studytype == "private":
        studylist = user.user_studies
    elif studytype == "shared":
        studylist = user.shared_studies
    elif studytype == "public":
        studylist = Study.get_by_status('public')
    else:
        raise IncompetentQiitaDeveloperError("Must use private, shared, "
                                             "or public!")

    StudyTuple = namedtuple('StudyInfo', 'id title meta_complete '
                            'num_samples_collected shared num_raw_data pi '
                            'pmids owner status')

    infolist = []
    for s_id in studylist:
        study = Study(s_id)
        status = study.status
        # Just passing the email address as the name here, since
        # name is not a required field in qiita.qiita_user
        owner = study_person_linkifier((study.owner, study.owner))
        info = study.info
        PI = StudyPerson(info['principal_investigator_id'])
        PI = study_person_linkifier((PI.email, PI.name))
        pmids = ", ".join([pubmed_linkifier([pmid])
                           for pmid in study.pmids])
        shared = _get_shared_links_for_study(study)
        infolist.append(StudyTuple(study.id, study.title,
                                   info["metadata_complete"],
                                   info["number_samples_collected"],
                                   shared, len(study.raw_data()),
                                   PI, pmids, owner, status))
    return infolist

def _check_owner(user, study):
    """make sure user is the owner of the study requested"""
    if not user.id == study.owner:
        raise HTTPError(403, "User %s does not own study %d" %
                        (user, study.id))

class PrivateStudiesHandler(BaseHandler):
    @authenticated
    @coroutine
    def get(self):
        self.write(self.render_string('waiting.html'))
        self.flush()
        user = self.current_user
        user_studies = yield Task(self._get_private, user)
        shared_studies = yield Task(self._get_shared, user)
        all_emails_except_current = yield Task(self._get_all_emails)
        all_emails_except_current.remove(self.current_user.id)
        self.render('private_studies.html',
                    user_studies=user_studies, shared_studies=shared_studies,
                    all_emails_except_current=all_emails_except_current)

    def _get_private(self, user, callback):
        callback(_build_study_info("private", user))

    def _get_shared(self, user, callback):
        """builds list of tuples for studies that are shared with user"""
        callback(_build_study_info("shared", user))

    def _get_all_emails(self, callback):
        callback(list(User.iter()))

class PublicStudiesHandler(BaseHandler):
    @authenticated
    @coroutine
    def get(self):
        self.write(self.render_string('waiting.html'))
        self.flush()
        public_studies = yield Task(self._get_public)
        self.render('public_studies.html',
                    public_studies=public_studies)

    def _get_public(self, callback):
        """builds list of tuples for studies that are public"""
        callback(_build_study_info("public"))

class StudyApprovalList(BaseHandler):
    @authenticated
    def get(self):
        user = self.current_user
        if user.level != 'admin':
            raise HTTPError(403, 'User %s is not admin' % self.current_user)

        parsed_studies = []
        for sid in Study.get_by_status('awaiting_approval'):
            study = Study(sid)
            parsed_studies.append((study.id, study.title, study.owner))

        self.render('admin_approval.html',
                    study_info=parsed_studies)

class ShareStudyAJAX(BaseHandler):
    def _get_shared_for_study(self, study, callback):
        shared_links = _get_shared_links_for_study(study)
        users = study.shared_with
        callback((users, shared_links))

    def _share(self, study, user, callback):
        user = User(user)
        callback(study.share(user))

    def _unshare(self, study, user, callback):
        user = User(user)
        callback(study.unshare(user))

    @authenticated
    @coroutine
    def get(self):
        study_id = int(self.get_argument('study_id'))
        study = Study(study_id)
        _check_owner(self.current_user, study)

        selected = self.get_argument('selected', None)
        deselected = self.get_argument('deselected', None)

        if selected is not None:
            yield Task(self._share, study, selected)
        if deselected is not None:
            yield Task(self._unshare, study, deselected)

        users, links = yield Task(self._get_shared_for_study, study)

        self.write(dumps({'users': users, 'links': links}))
<commit_msg>one more<commit_after># -----------------------------------------------------------------------------
# Copyright (c) 2014--, The Qiita Development Team.
#
# Distributed under the terms of the BSD 3-clause License.
#
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------
from __future__ import division
from collections import namedtuple
from json import dumps

from tornado.web import authenticated, HTTPError
from tornado.gen import coroutine, Task

from qiita_core.exceptions import IncompetentQiitaDeveloperError
from qiita_db.user import User
from qiita_db.study import Study, StudyPerson
from qiita_pet.handlers.base_handlers import BaseHandler
from qiita_pet.handlers.util import study_person_linkifier, pubmed_linkifier

def _get_shared_links_for_study(study):
    shared = []
    for person in study.shared_with:
        person = User(person)
        name = person.info['name']
        email = person.email
        # Name is optional, so default to email if non existant
        if name:
            shared.append(study_person_linkifier(
                (email, name)))
        else:
            shared.append(study_person_linkifier(
                (email, email)))
    return ", ".join(shared)

def _build_study_info(studytype, user=None):
    """builds list of namedtuples for study listings"""
    if studytype == "private":
        studylist = user.user_studies
    elif studytype == "shared":
        studylist = user.shared_studies
    elif studytype == "public":
        studylist = Study.get_by_status('public')
    else:
        raise IncompetentQiitaDeveloperError("Must use private, shared, "
                                             "or public!")

    StudyTuple = namedtuple('StudyInfo', 'id title meta_complete '
                            'num_samples_collected shared num_raw_data pi '
                            'pmids owner status')

    infolist = []
    for s_id in studylist:
        study = Study(s_id)
        status = study.status
        # Just passing the email address as the name here, since
        # name is not a required field in qiita.qiita_user
        owner = study_person_linkifier((study.owner, study.owner))
        info = study.info
        PI = StudyPerson(info['principal_investigator_id'])
        PI = study_person_linkifier((PI.email, PI.name))
        pmids = ", ".join([pubmed_linkifier([pmid])
                           for pmid in study.pmids])
        shared = _get_shared_links_for_study(study)
        infolist.append(StudyTuple(study.id, study.title,
                                   info["metadata_complete"],
                                   info["number_samples_collected"],
                                   shared, len(study.raw_data()),
                                   PI, pmids, owner, status))
    return infolist

def _check_owner(user, study):
    """make sure user is the owner of the study requested"""
    if not user.id == study.owner:
        raise HTTPError(403, "User %s does not own study %d" %
                        (user.id, study.id))

class PrivateStudiesHandler(BaseHandler):
    @authenticated
    @coroutine
    def get(self):
        self.write(self.render_string('waiting.html'))
        self.flush()
        user = self.current_user
        user_studies = yield Task(self._get_private, user)
        shared_studies = yield Task(self._get_shared, user)
        all_emails_except_current = yield Task(self._get_all_emails)
        all_emails_except_current.remove(self.current_user.id)
        self.render('private_studies.html',
                    user_studies=user_studies, shared_studies=shared_studies,
                    all_emails_except_current=all_emails_except_current)

    def _get_private(self, user, callback):
        callback(_build_study_info("private", user))

    def _get_shared(self, user, callback):
        """builds list of tuples for studies that are shared with user"""
        callback(_build_study_info("shared", user))

    def _get_all_emails(self, callback):
        callback(list(User.iter()))

class PublicStudiesHandler(BaseHandler):
    @authenticated
    @coroutine
    def get(self):
        self.write(self.render_string('waiting.html'))
        self.flush()
        public_studies = yield Task(self._get_public)
        self.render('public_studies.html',
                    public_studies=public_studies)

    def _get_public(self, callback):
        """builds list of tuples for studies that are public"""
        callback(_build_study_info("public"))

class StudyApprovalList(BaseHandler):
    @authenticated
    def get(self):
        user = self.current_user
        if user.level != 'admin':
            raise HTTPError(403, 'User %s is not admin' % self.current_user)

        parsed_studies = []
        for sid in Study.get_by_status('awaiting_approval'):
            study = Study(sid)
            parsed_studies.append((study.id, study.title, study.owner))

        self.render('admin_approval.html',
                    study_info=parsed_studies)

class ShareStudyAJAX(BaseHandler):
    def _get_shared_for_study(self, study, callback):
        shared_links = _get_shared_links_for_study(study)
        users = study.shared_with
        callback((users, shared_links))

    def _share(self, study, user, callback):
        user = User(user)
        callback(study.share(user))

    def _unshare(self, study, user, callback):
        user = User(user)
        callback(study.unshare(user))

    @authenticated
    @coroutine
    def get(self):
        study_id = int(self.get_argument('study_id'))
        study = Study(study_id)
        _check_owner(self.current_user, study)

        selected = self.get_argument('selected', None)
        deselected = self.get_argument('deselected', None)

        if selected is not None:
            yield Task(self._share, study, selected)
        if deselected is not None:
            yield Task(self._unshare, study, deselected)

        users, links = yield Task(self._get_shared_for_study, study)

        self.write(dumps({'users': users, 'links': links}))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Module: default
# Created on: 13.01.2017

# import local classes
if __package__ is None:
    import sys
    import os
    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
    from resources.lib.NetflixSession import NetflixSession
    from resources.lib.KodiHelper import KodiHelper
    from resources.lib.Navigation import Navigation
    from resources.lib.Library import Library
else:
    from .resources.lib.NetflixSession import NetflixSession
    from .resources.lib.KodiHelper import KodiHelper
    from .resources.lib.Navigation import Navigation
    from .resources.lib.Library import Library

# Setup plugin
plugin_handle = int(sys.argv[1])
base_url = sys.argv[0]

# init plugin libs
kodi_helper = KodiHelper(
    plugin_handle=plugin_handle,
    base_url=base_url
)
netflix_session = NetflixSession(
    cookie_path=kodi_helper.cookie_path,
    data_path=kodi_helper.data_path,
    log_fn=kodi_helper.log
)
library = Library(
    base_url=base_url,
    #root_folder=kodi_helper.base_data_path,
    root_folder='/Users/asciidisco/Desktop/lib',
    library_settings=kodi_helper.get_custom_library_settings(),
    log_fn=kodi_helper.log
)
navigation = Navigation(
    netflix_session=netflix_session,
    kodi_helper=kodi_helper,
    library=library,
    base_url=base_url,
    log_fn=kodi_helper.log
)
kodi_helper.set_library(library=library)

if __name__ == '__main__':
    # Call the router function and pass the plugin call parameters to it.
    # We use string slicing to trim the leading '?' from the plugin call paramstring
    navigation.router(paramstring=sys.argv[2][1:])
<commit_msg>fix(addon): Remove temporary path<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Module: default
# Created on: 13.01.2017

# import local classes
if __package__ is None:
    import sys
    import os
    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
    from resources.lib.NetflixSession import NetflixSession
    from resources.lib.KodiHelper import KodiHelper
    from resources.lib.Navigation import Navigation
    from resources.lib.Library import Library
else:
    from .resources.lib.NetflixSession import NetflixSession
    from .resources.lib.KodiHelper import KodiHelper
    from .resources.lib.Navigation import Navigation
    from .resources.lib.Library import Library

# Setup plugin
plugin_handle = int(sys.argv[1])
base_url = sys.argv[0]

# init plugin libs
kodi_helper = KodiHelper(
    plugin_handle=plugin_handle,
    base_url=base_url
)
netflix_session = NetflixSession(
    cookie_path=kodi_helper.cookie_path,
    data_path=kodi_helper.data_path,
    log_fn=kodi_helper.log
)
library = Library(
    base_url=base_url,
    root_folder=kodi_helper.base_data_path,
    library_settings=kodi_helper.get_custom_library_settings(),
    log_fn=kodi_helper.log
)
navigation = Navigation(
    netflix_session=netflix_session,
    kodi_helper=kodi_helper,
    library=library,
    base_url=base_url,
    log_fn=kodi_helper.log
)
kodi_helper.set_library(library=library)

if __name__ == '__main__':
    # Call the router function and pass the plugin call parameters to it.
    # We use string slicing to trim the leading '?' from the plugin call paramstring
    navigation.router(paramstring=sys.argv[2][1:])
<|endoftext|>


<commit_before>
import sys
import os
import time

from pycrosscall import ctypes
ctypes.windll.start_session(parameter = {'log_level': 10})

_call_demo_routine_ = ctypes.windll.LoadLibrary('demo_dll.dll').simple_demo_routine
print('DLL and routine loaded!')

_call_demo_routine_.argtypes = [
	ctypes.c_float,
	ctypes.c_float
	]
print('Set argument types!')

_call_demo_routine_.restype = ctypes.c_float
print('Set return value type!')

return_value = _call_demo_routine_(20.0, 1.07)
print('Called!')
try:
	print('Got "%f".' % return_value)
except:
	print('Got no return value!')
<commit_msg>example can run on wine and unix<commit_after>
import sys
import os
import time
from sys import platform

if True in [platform.startswith(os_name) for os_name in ['linux', 'darwin', 'freebsd']]:

	from pycrosscall import ctypes
	ctypes.windll.start_session(parameter = {'log_level': 10})

elif platform.startswith('win'):

	import ctypes

else:

	raise # TODO unsupported platform

_call_demo_routine_ = ctypes.windll.LoadLibrary('demo_dll.dll').simple_demo_routine
print('DLL and routine loaded!')

_call_demo_routine_.argtypes = [
	ctypes.c_float,
	ctypes.c_float
	]
print('Set argument types!')

_call_demo_routine_.restype = ctypes.c_float
print('Set return value type!')

return_value = _call_demo_routine_(20.0, 1.07)
print('Called!')
try:
	print('Got "%f".' % return_value)
except:
	print('Got no return value!')
<|endoftext|>


<commit_before>"""
code to initialize the remote side of a gateway once the io is created
"""
import os
import inspect
import execnet
from execnet import gateway_base
from execnet.gateway import Gateway
importdir = os.path.dirname(os.path.dirname(execnet.__file__))

class HostNotFound(Exception):
    pass

def bootstrap_import(io, spec):
    sendexec(
        io,
        "import sys",
        "sys.path.insert(0, %r)" % importdir,
        "from execnet.gateway_base import serve, init_popen_io, get_execmodel",
        "sys.stdout.write('1')",
        "sys.stdout.flush()",
        "execmodel = get_execmodel(%r)" % spec.execmodel,
        "serve(init_popen_io(execmodel), id='%s-slave')" % spec.id,
    )
    s = io.read(1)
    assert s == "1".encode('ascii'), repr(s)

def bootstrap_exec(io, spec):
    try:
        sendexec(
            io,
            inspect.getsource(gateway_base),
            "execmodel = get_execmodel(%r)" % spec.execmodel,
            'io = init_popen_io(execmodel)',
            "io.write('1'.encode('ascii'))",
            "serve(io, id='%s-slave')" % spec.id,
        )
        s = io.read(1)
        assert s == "1".encode('ascii')
    except EOFError:
        ret = io.wait()
        if ret == 255:
            raise HostNotFound(io.remoteaddress)

def bootstrap_socket(io, id):
    # XXX: switch to spec
    from execnet.gateway_socket import SocketIO

    sendexec(
        io,
        inspect.getsource(gateway_base),
        'import socket',
        inspect.getsource(SocketIO),
        "try: execmodel",
        "except NameError:",
        "   execmodel = get_execmodel('thread')",
        "io = SocketIO(clientsock, execmodel)",
        "io.write('1'.encode('ascii'))",
        "serve(io, id='%s-slave')" % id,
    )
    s = io.read(1)
    assert s == "1".encode('ascii')

def sendexec(io, *sources):
    source = "\n".join(sources)
    io.write((repr(source) + "\n").encode('ascii'))

def fix_pid_for_jython_popen(gw):
    """
    fix for jython 2.5.1
    """
    spec, io = gw.spec, gw._io
    if spec.popen and not spec.via:
        # XXX: handle the case of remote being jython
        #      and not having the popen pid
        if io.popen.pid is None:
            io.popen.pid = gw.remote_exec(
                "import os; channel.send(os.getpid())").receive()

def bootstrap(io, spec):
    if spec.popen:
        if spec.via or spec.python:
            bootstrap_exec(io, spec)
        else:
            bootstrap_import(io, spec)
    elif spec.ssh or spec.vagrant_ssh:
        bootstrap_exec(io, spec)
    elif spec.socket:
        bootstrap_socket(io, spec)
    else:
        raise ValueError('unknown gateway type, cant bootstrap')
    gw = Gateway(io, spec)
    fix_pid_for_jython_popen(gw)
    return gw
<commit_msg>FIX: be more gentle about hacking the path<commit_after>"""
code to initialize the remote side of a gateway once the io is created
"""
import os
import inspect
import execnet
from execnet import gateway_base
from execnet.gateway import Gateway
importdir = os.path.dirname(os.path.dirname(execnet.__file__))

class HostNotFound(Exception):
    pass

def bootstrap_import(io, spec):
    sendexec(
        io,
        "import sys",
        "if %r not in sys.path:" % importdir,
        "    sys.path.insert(0, %r)" % importdir,
        "from execnet.gateway_base import serve, init_popen_io, get_execmodel",
        "sys.stdout.write('1')",
        "sys.stdout.flush()",
        "execmodel = get_execmodel(%r)" % spec.execmodel,
        "serve(init_popen_io(execmodel), id='%s-slave')" % spec.id,
    )
    s = io.read(1)
    assert s == "1".encode('ascii'), repr(s)

def bootstrap_exec(io, spec):
    try:
        sendexec(
            io,
            inspect.getsource(gateway_base),
            "execmodel = get_execmodel(%r)" % spec.execmodel,
            'io = init_popen_io(execmodel)',
            "io.write('1'.encode('ascii'))",
            "serve(io, id='%s-slave')" % spec.id,
        )
        s = io.read(1)
        assert s == "1".encode('ascii')
    except EOFError:
        ret = io.wait()
        if ret == 255:
            raise HostNotFound(io.remoteaddress)

def bootstrap_socket(io, id):
    # XXX: switch to spec
    from execnet.gateway_socket import SocketIO

    sendexec(
        io,
        inspect.getsource(gateway_base),
        'import socket',
        inspect.getsource(SocketIO),
        "try: execmodel",
        "except NameError:",
        "   execmodel = get_execmodel('thread')",
        "io = SocketIO(clientsock, execmodel)",
        "io.write('1'.encode('ascii'))",
        "serve(io, id='%s-slave')" % id,
    )
    s = io.read(1)
    assert s == "1".encode('ascii')

def sendexec(io, *sources):
    source = "\n".join(sources)
    io.write((repr(source) + "\n").encode('ascii'))

def fix_pid_for_jython_popen(gw):
    """
    fix for jython 2.5.1
    """
    spec, io = gw.spec, gw._io
    if spec.popen and not spec.via:
        # XXX: handle the case of remote being jython
        #      and not having the popen pid
        if io.popen.pid is None:
            io.popen.pid = gw.remote_exec(
                "import os; channel.send(os.getpid())").receive()

def bootstrap(io, spec):
    if spec.popen:
        if spec.via or spec.python:
            bootstrap_exec(io, spec)
        else:
            bootstrap_import(io, spec)
    elif spec.ssh or spec.vagrant_ssh:
        bootstrap_exec(io, spec)
    elif spec.socket:
        bootstrap_socket(io, spec)
    else:
        raise ValueError('unknown gateway type, cant bootstrap')
    gw = Gateway(io, spec)
    fix_pid_for_jython_popen(gw)
    return gw
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import unicode_literals

import time

from django.test import TestCase

from mock import MagicMock, patch

from .forms import BatchUploadForm
from .models import Observation
from .utils import jd_now
from stars.models import Star
from variablestars.tests import BaseTestCase

class ObservationModelTestCase(BaseTestCase):
    """
    Tests for Observation model.
    """

    def test_str(self):
        """
        Check for string representation of an observation.
        """
        observation = Observation(
            observer=self.observer,
            star=self.star,
            jd=2456567.2550,
            magnitude=8.5,
        )
        expected = "%s %s %s" % (
            observation.star, observation.jd, observation.magnitude,
        )
        self.assertEqual(str(observation), expected)

    def test_update_star_observations_count(self):
        """
        Check that creating new observations updates star's observations_count
        denormalized field.
        """
        self.assertEqual(self.star.observations_count, 10)
        observation = Observation.objects.create(
            observer=self.observer,
            star=self.star,
            jd=2456567.2550,
            magnitude=8.5,
        )
        star = Star.objects.get(pk=self.star.pk)
        self.assertEqual(star.observations_count, 11)
        observation.delete()
        star = Star.objects.get(pk=self.star.pk)
        self.assertEqual(star.observations_count, 10)

class ObservationManagerTestCase(BaseTestCase):
    def test_top_stars(self):
        expected = [
            {'star_id': self.star.id, 'star__name': self.star.name, 'observations_count': 10},
            {'star_id': self.periodic_star.id, 'star__name': self.periodic_star.name, 'observations_count': 8},
        ]
        top_stars = list(Observation.objects.top_stars())
        self.assertEqual(top_stars, expected)

    def test_top_observers(self):
        """
        Check that top_observers() returns a list of people ordered by
        the number of observations.
        """
        expected = [
            {
                'observer_id': self.observer.id,
                'observer__aavso_code': self.observer.aavso_code,
                'observations_count': 15,
            },
            {
                'observer_id': self.observer2.id,
                'observer__aavso_code': self.observer2.aavso_code,
                'observations_count': 3,
            },
        ]
        top_observers = list(Observation.objects.top_observers())
        self.assertEqual(top_observers, expected)

    def test_recent_observations(self):
        observations = Observation.objects.recent_observations()
        self.assertEqual(observations[0].observer, self.observer)

class JdNowTestCase(TestCase):
    """
    Tests for ``observations.utils.jd_now`` function.
    """
    def test_unix_epoch_jd(self):
        """
        Check JD value at the beggining of Unix epoch.
        """
        mock_time = MagicMock()
        mock_time.return_value = 0.0
        with patch.object(time, 'time', mock_time):
            jd = jd_now()
            self.assertEqual(jd, 2440587.5)

class BatchUploadFormTestCase(BaseTestCase):
    def setUp(self):
        super(BatchUploadFormTestCase, self).setUp()
        self.row = {
            'name': self.star.name,
            'magnitude': '6.6',
            'date': str(jd_now()),
            'comp1': '65',
            'comp2': '71',
            'chart': '',
            'comment_code': '',
            'notes': 'test2',
        }

    def test_parse_row(self):
        form = BatchUploadForm()
        form.process_row(self.row, self.observer)
        observation = Observation.objects.get(star=self.star, notes='test2')
        self.assertEqual(observation.comp1, self.row['comp1'])
        self.assertEqual(observation.comp2, self.row['comp2'])
<commit_msg>Tests for BatchUploadForm.normalize_star_name.<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals

import time

from django.test import TestCase

from mock import MagicMock, patch

from .forms import BatchUploadForm
from .models import Observation
from .utils import jd_now
from stars.models import Star
from variablestars.tests import BaseTestCase

class ObservationModelTestCase(BaseTestCase):
    """
    Tests for Observation model.
    """

    def test_str(self):
        """
        Check for string representation of an observation.
        """
        observation = Observation(
            observer=self.observer,
            star=self.star,
            jd=2456567.2550,
            magnitude=8.5,
        )
        expected = "%s %s %s" % (
            observation.star, observation.jd, observation.magnitude,
        )
        self.assertEqual(str(observation), expected)

    def test_update_star_observations_count(self):
        """
        Check that creating new observations updates star's observations_count
        denormalized field.
        """
        self.assertEqual(self.star.observations_count, 10)
        observation = Observation.objects.create(
            observer=self.observer,
            star=self.star,
            jd=2456567.2550,
            magnitude=8.5,
        )
        star = Star.objects.get(pk=self.star.pk)
        self.assertEqual(star.observations_count, 11)
        observation.delete()
        star = Star.objects.get(pk=self.star.pk)
        self.assertEqual(star.observations_count, 10)

class ObservationManagerTestCase(BaseTestCase):
    def test_top_stars(self):
        expected = [
            {'star_id': self.star.id, 'star__name': self.star.name, 'observations_count': 10},
            {'star_id': self.periodic_star.id, 'star__name': self.periodic_star.name, 'observations_count': 8},
        ]
        top_stars = list(Observation.objects.top_stars())
        self.assertEqual(top_stars, expected)

    def test_top_observers(self):
        """
        Check that top_observers() returns a list of people ordered by
        the number of observations.
        """
        expected = [
            {
                'observer_id': self.observer.id,
                'observer__aavso_code': self.observer.aavso_code,
                'observations_count': 15,
            },
            {
                'observer_id': self.observer2.id,
                'observer__aavso_code': self.observer2.aavso_code,
                'observations_count': 3,
            },
        ]
        top_observers = list(Observation.objects.top_observers())
        self.assertEqual(top_observers, expected)

    def test_recent_observations(self):
        observations = Observation.objects.recent_observations()
        self.assertEqual(observations[0].observer, self.observer)

class JdNowTestCase(TestCase):
    """
    Tests for ``observations.utils.jd_now`` function.
    """
    def test_unix_epoch_jd(self):
        """
        Check JD value at the beggining of Unix epoch.
        """
        mock_time = MagicMock()
        mock_time.return_value = 0.0
        with patch.object(time, 'time', mock_time):
            jd = jd_now()
            self.assertEqual(jd, 2440587.5)

class BatchUploadFormTestCase(BaseTestCase):
    def setUp(self):
        super(BatchUploadFormTestCase, self).setUp()
        self.row = {
            'name': self.star.name,
            'magnitude': '6.6',
            'date': str(jd_now()),
            'comp1': '65',
            'comp2': '71',
            'chart': '',
            'comment_code': '',
            'notes': 'test2',
        }

    def test_normalize_star_name(self):
        form = BatchUploadForm()
        self.assertEqual(form.normalize_star_name('RR LYR'), 'RR LYR')
        self.assertEqual(form.normalize_star_name('V1339 CYG'), 'V1339 CYG')
        self.assertEqual(form.normalize_star_name('V838 MON'), 'V0838 MON')

    def test_parse_row(self):
        form = BatchUploadForm()
        form.process_row(self.row, self.observer)
        observation = Observation.objects.get(star=self.star, notes='test2')
        self.assertEqual(observation.comp1, self.row['comp1'])
        self.assertEqual(observation.comp2, self.row['comp2'])
<|endoftext|>


<commit_before>from django.core.exceptions import PermissionDenied
from django.core.urlresolvers import reverse
from django.shortcuts import redirect, render
from django.template import Context, loader
from django.views.generic import UpdateView, DetailView
from django.views.generic.edit import FormView
from django.views.generic.list import ListView

from wye.base.constants import WorkshopStatus
from wye.base.emailer_html import send_email_to_list, send_email_to_id
from wye.organisations.models import Organisation
from wye.profiles.models import Profile
from wye.workshops.models import Workshop

from .forms import UserProfileForm, ContactUsForm

def profile_view(request, slug):
    try:
        p = Profile.objects.get(user__username=slug)
    except Profile.DoesNotExist:
        return render(request, 'error.html', {"message": "Profile does not exist"})
    return render(request, 'profile/index.html', {'object': p})

class UserDashboard(ListView):
    model = Profile
    template_name = 'profile/dashboard.html'

    def dispatch(self, request, *args, **kwargs):
        user_profile = Profile.objects.get(
            user__id=self.request.user.id)
        if not user_profile.get_user_type:
            return redirect('profiles:profile-edit', slug=request.user.username)
        return super(UserDashboard, self).dispatch(request, *args, **kwargs)

    def get_context_data(self, **kwargs):
        context = super(UserDashboard, self).get_context_data(**kwargs)
        user_profile = Profile.objects.get(user=self.request.user)
        workshop_all = Workshop.objects.all()
        accept_workshops = workshop_all.filter(
            status=WorkshopStatus.ACCEPTED)
        requested_workshops = workshop_all.filter(
            status=WorkshopStatus.REQUESTED)
        organisation_all = Organisation.objects.all()
        for each_type in user_profile.get_user_type:
            if each_type == 'tutor':
                context['is_tutor'] = True
                context['workshop_requested_tutor'] = accept_workshops.filter(
                    presenter=self.request.user)
                context['workshop_completed_tutor'] = requested_workshops.filter(
                    presenter=self.request.user)
            if each_type == 'lead':
                context['is_regional_lead'] = True
                context['workshops_accepted_under_rl'] = accept_workshops
                context['workshops_pending_under_rl'] = requested_workshops
                context['interested_tutors'] = Profile.objects.filter(
                    usertype__slug='tutor',
                    interested_locations__name__in=user_profile.get_interested_locations).exclude(
                    user=self.request.user).count()
                context['interested_locations'] = organisation_all.filter(
                    location__name__in=user_profile.get_interested_locations).count()

            if each_type == 'poc':
                context['is_college_poc'] = True
                context['users_organisation'] = organisation_all.filter(
                    user=self.request.user)
                context['workshop_requested_under_poc'] = workshop_all.filter(
                    requester__id__in=organisation_all.values_list(
                        'id', flat=True))
                context['workshops_accepted_under_poc'] = workshop_all.filter(
                    status=WorkshopStatus.ACCEPTED,
                    requester__id__in=organisation_all.values_list(
                        'id', flat=True))

            if each_type == 'admin':
                context['is_admin'] = True
                context['workshops_by_status'] = workshop_all.order_by(
                    'status')
                context['workshops_by_region'] = workshop_all.order_by(
                    'location')

        return context

class ProfileEditView(UpdateView):
    model = Profile
    template_name = 'profile/update.html'
    form_class = UserProfileForm
    slug_field = 'user__username'

    def form_valid(self, form):
        return super(ProfileEditView, self).form_valid(form)

    def get_success_url(self):
        return reverse('profiles:profile-page', kwargs={'slug': self.object.slug})

    def dispatch(self, *args, **kwargs):
        if self.request.user.pk == self.get_object().pk:
            return super(ProfileEditView, self).dispatch(*args, **kwargs)
        else:
            raise PermissionDenied

class ContactFormView(FormView):
    form_class = ContactUsForm
    template_name = 'contact.html'
    success_url = '/thankyou'

    def form_valid(self, form):

        email_context = Context({
            'contact_name': form.cleaned_data['name'],
            'contact_email': form.cleaned_data['email'],
            'comments': form.cleaned_data['comments'],
            'conatct_number': form.cleaned_data['contact_number'],
            'feedback_type': form.cleaned_data['feedback_type']
        })

        subject = "PythonExpress Feedback by %s" % (form.cleaned_data['name'])
        text_body = loader.get_template(
            'email_messages/contactus/message.txt').render(email_context)
        email_body = loader.get_template(
            'email_messages/contactus/message.html').render(email_context)
        user_subject = '[PythonExpress] Feedback Received'
        user_text_body = loader.get_template(
            'email_messages/contactus/message_user.txt').render(email_context)
        user_email_body = loader.get_template(
            'email_messages/contactus/message_user.html').render(email_context)

        try:
            regional_lead = Profile.objects.filter(
                usertype__slug__in=['lead', 'admin']).values_list(
                'user__email', flat=True)
            send_email_to_list(
                subject,
                users_list=regional_lead,
                body=email_body,
                text_body=text_body)

            send_email_to_id(
                user_subject,
                body=user_email_body,
                email_id=form.cleaned_data['email'],
                text_body=user_text_body)
        except Exception as e:
            print(e)

        return super(ContactFormView, self).form_valid(form)
<commit_msg>Remove unused import<commit_after># -*- coding: utf-8 -*-

from django.core.exceptions import PermissionDenied
from django.core.urlresolvers import reverse
from django.shortcuts import redirect, render
from django.template import Context, loader
from django.views.generic import UpdateView
from django.views.generic.edit import FormView
from django.views.generic.list import ListView

from wye.base.constants import WorkshopStatus
from wye.base.emailer_html import send_email_to_id, send_email_to_list
from wye.organisations.models import Organisation
from wye.profiles.models import Profile
from wye.workshops.models import Workshop

from .forms import ContactUsForm, UserProfileForm

def profile_view(request, slug):
    try:
        p = Profile.objects.get(user__username=slug)
    except Profile.DoesNotExist:
        return render(request, 'error.html', {"message": "Profile does not exist"})
    return render(request, 'profile/index.html', {'object': p})

class UserDashboard(ListView):
    model = Profile
    template_name = 'profile/dashboard.html'

    def dispatch(self, request, *args, **kwargs):
        user_profile = Profile.objects.get(
            user__id=self.request.user.id)
        if not user_profile.get_user_type:
            return redirect('profiles:profile-edit', slug=request.user.username)
        return super(UserDashboard, self).dispatch(request, *args, **kwargs)

    def get_context_data(self, **kwargs):
        context = super(UserDashboard, self).get_context_data(**kwargs)
        user_profile = Profile.objects.get(user=self.request.user)
        workshop_all = Workshop.objects.all()
        accept_workshops = workshop_all.filter(
            status=WorkshopStatus.ACCEPTED)
        requested_workshops = workshop_all.filter(
            status=WorkshopStatus.REQUESTED)
        organisation_all = Organisation.objects.all()
        for each_type in user_profile.get_user_type:
            if each_type == 'tutor':
                context['is_tutor'] = True
                context['workshop_requested_tutor'] = accept_workshops.filter(
                    presenter=self.request.user)
                context['workshop_completed_tutor'] = requested_workshops.filter(
                    presenter=self.request.user)
            if each_type == 'lead':
                context['is_regional_lead'] = True
                context['workshops_accepted_under_rl'] = accept_workshops
                context['workshops_pending_under_rl'] = requested_workshops
                context['interested_tutors'] = Profile.objects.filter(
                    usertype__slug='tutor',
                    interested_locations__name__in=user_profile.get_interested_locations).exclude(
                    user=self.request.user).count()
                context['interested_locations'] = organisation_all.filter(
                    location__name__in=user_profile.get_interested_locations).count()

            if each_type == 'poc':
                context['is_college_poc'] = True
                context['users_organisation'] = organisation_all.filter(
                    user=self.request.user)
                context['workshop_requested_under_poc'] = workshop_all.filter(
                    requester__id__in=organisation_all.values_list(
                        'id', flat=True))
                context['workshops_accepted_under_poc'] = workshop_all.filter(
                    status=WorkshopStatus.ACCEPTED,
                    requester__id__in=organisation_all.values_list(
                        'id', flat=True))

            if each_type == 'admin':
                context['is_admin'] = True
                context['workshops_by_status'] = workshop_all.order_by(
                    'status')
                context['workshops_by_region'] = workshop_all.order_by(
                    'location')

        return context

class ProfileEditView(UpdateView):
    model = Profile
    template_name = 'profile/update.html'
    form_class = UserProfileForm
    slug_field = 'user__username'

    def form_valid(self, form):
        return super(ProfileEditView, self).form_valid(form)

    def get_success_url(self):
        return reverse('profiles:profile-page', kwargs={'slug': self.object.slug})

    def dispatch(self, *args, **kwargs):
        if self.request.user.pk == self.get_object().pk:
            return super(ProfileEditView, self).dispatch(*args, **kwargs)
        else:
            raise PermissionDenied

class ContactFormView(FormView):
    form_class = ContactUsForm
    template_name = 'contact.html'
    success_url = '/thankyou'

    def form_valid(self, form):

        email_context = Context({
            'contact_name': form.cleaned_data['name'],
            'contact_email': form.cleaned_data['email'],
            'comments': form.cleaned_data['comments'],
            'conatct_number': form.cleaned_data['contact_number'],
            'feedback_type': form.cleaned_data['feedback_type']
        })

        subject = "PythonExpress Feedback by %s" % (form.cleaned_data['name'])
        text_body = loader.get_template(
            'email_messages/contactus/message.txt').render(email_context)
        email_body = loader.get_template(
            'email_messages/contactus/message.html').render(email_context)
        user_subject = '[PythonExpress] Feedback Received'
        user_text_body = loader.get_template(
            'email_messages/contactus/message_user.txt').render(email_context)
        user_email_body = loader.get_template(
            'email_messages/contactus/message_user.html').render(email_context)

        try:
            regional_lead = Profile.objects.filter(
                usertype__slug__in=['lead', 'admin']).values_list(
                'user__email', flat=True)
            send_email_to_list(
                subject,
                users_list=regional_lead,
                body=email_body,
                text_body=text_body)

            send_email_to_id(
                user_subject,
                body=user_email_body,
                email_id=form.cleaned_data['email'],
                text_body=user_text_body)
        except Exception as e:
            print(e)

        return super(ContactFormView, self).form_valid(form)
<|endoftext|>


<commit_before>import logging
import os
import sys
# Submodule. Necessary explicit import
from logging import handlers

LOGGER_NAME = 'storj-python-gui-log'
LOG_FILENAME = 'test.log'
PATH = os.path.join(os.getcwd(), LOG_FILENAME)

logger = logging.getLogger(LOGGER_NAME)
logger.addHandler(logging.StreamHandler(sys.stdout))
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')

# Logs written to file
# Rotating file logger. Max size 3 MB per 5 log files
fileHandler = handlers.RotatingFileHandler(PATH,
                                           maxBytes=(1048576 * 3),
                                           backupCount=5)
fileHandler.setFormatter(formatter)
# fileHandler.setLevel(logging.WARNING)
fileHandler.setLevel(logging.DEBUG)  # TODO: change to WARNING
logger.addHandler(fileHandler)

# Logs to stdout
consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(formatter)
consoleHandler.setLevel(logging.DEBUG)
logger.addHandler(consoleHandler)
<commit_msg>write only warning logs to file<commit_after>import logging
import os
import sys
# Submodule. Necessary explicit import
from logging import handlers

LOGGER_NAME = 'storj-python-gui-log'
LOG_FILENAME = 'test.log'
PATH = os.path.join(os.getcwd(), LOG_FILENAME)

logger = logging.getLogger(LOGGER_NAME)
logger.addHandler(logging.StreamHandler(sys.stdout))
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')

# Logs written to file
# Rotating file logger. Max size 3 MB per 5 log files
fileHandler = handlers.RotatingFileHandler(PATH,
                                           maxBytes=(1048576 * 3),
                                           backupCount=5)
fileHandler.setFormatter(formatter)
fileHandler.setLevel(logging.WARNING)
# fileHandler.setLevel(logging.DEBUG)  # TODO: change to WARNING
logger.addHandler(fileHandler)

# Logs to stdout
consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(formatter)
consoleHandler.setLevel(logging.DEBUG)
logger.addHandler(consoleHandler)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import unicode_literals

import time

from django.test import TestCase

from mock import MagicMock, patch

from .models import Observation
from .utils import jd_now
from stars.models import Star
from variablestars.tests import BaseTestCase

class ObservationModelTestCase(BaseTestCase):
    """
    Tests for Observation model.
    """

    def test_str(self):
        """
        Check for string representation of an observation.
        """
        observation = Observation(
            observer=self.observer,
            star=self.star,
            jd=2456567.2550,
            magnitude=8.5,
        )
        expected = "%s %s %s" % (
            observation.star, observation.jd, observation.magnitude,
        )
        self.assertEqual(str(observation), expected)

    def test_update_star_observations_count(self):
        """
        Check that creating new observations updates star's observations_count
        denormalized field.
        """
        self.assertEqual(self.star.observations_count, 10)
        observation = Observation.objects.create(
            observer=self.observer,
            star=self.star,
            jd=2456567.2550,
            magnitude=8.5,
        )
        star = Star.objects.get(pk=self.star.pk)
        self.assertEqual(star.observations_count, 11)
        observation.delete()
        star = Star.objects.get(pk=self.star.pk)
        self.assertEqual(star.observations_count, 10)

class ObservationManagerTestCase(BaseTestCase):
    def test_top_stars(self):
        expected = [
            {'star_id': self.star.id, 'star__name': self.star.name, 'observations_count': 10},
            {'star_id': self.periodic_star.id, 'star__name': self.periodic_star.name, 'observations_count': 8},
        ]
        top_stars = list(Observation.objects.top_stars())
        self.assertEqual(top_stars, expected)

    def test_top_observers(self):
        """
        Check that top_observers() returns a list of people ordered by
        the number of observations.
        """
        expected = [
            {
                'observer_id': self.observer.id,
                'observer__aavso_code': self.observer.aavso_code,
                'observations_count': 15,
            },
            {
                'observer_id': self.observer2.id,
                'observer__aavso_code': self.observer2.aavso_code,
                'observations_count': 3,
            },
        ]
        top_observers = list(Observation.objects.top_observers())
        self.assertEqual(top_observers, expected)

    def test_recent_observations(self):
        observations = Observation.objects.recent_observations()
        self.assertEqual(observations[0].observer, self.observer)

class JdNowTestCase(TestCase):
    """
    Tests for ``observations.utils.jd_now`` function.
    """
    def test_unix_epoch_jd(self):
        """
        Check JD value at the beggining of Unix epoch.
        """
        mock_time = MagicMock()
        mock_time.return_value = 0.0
        with patch.object(time, 'time', mock_time):
            jd = jd_now()
            self.assertEqual(jd, 2440587.5)
<commit_msg>Initial tests for parsing AAVSO input.<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals

import time

from django.test import TestCase

from mock import MagicMock, patch

from .forms import BatchUploadForm
from .models import Observation
from .utils import jd_now
from stars.models import Star
from variablestars.tests import BaseTestCase

class ObservationModelTestCase(BaseTestCase):
    """
    Tests for Observation model.
    """

    def test_str(self):
        """
        Check for string representation of an observation.
        """
        observation = Observation(
            observer=self.observer,
            star=self.star,
            jd=2456567.2550,
            magnitude=8.5,
        )
        expected = "%s %s %s" % (
            observation.star, observation.jd, observation.magnitude,
        )
        self.assertEqual(str(observation), expected)

    def test_update_star_observations_count(self):
        """
        Check that creating new observations updates star's observations_count
        denormalized field.
        """
        self.assertEqual(self.star.observations_count, 10)
        observation = Observation.objects.create(
            observer=self.observer,
            star=self.star,
            jd=2456567.2550,
            magnitude=8.5,
        )
        star = Star.objects.get(pk=self.star.pk)
        self.assertEqual(star.observations_count, 11)
        observation.delete()
        star = Star.objects.get(pk=self.star.pk)
        self.assertEqual(star.observations_count, 10)

class ObservationManagerTestCase(BaseTestCase):
    def test_top_stars(self):
        expected = [
            {'star_id': self.star.id, 'star__name': self.star.name, 'observations_count': 10},
            {'star_id': self.periodic_star.id, 'star__name': self.periodic_star.name, 'observations_count': 8},
        ]
        top_stars = list(Observation.objects.top_stars())
        self.assertEqual(top_stars, expected)

    def test_top_observers(self):
        """
        Check that top_observers() returns a list of people ordered by
        the number of observations.
        """
        expected = [
            {
                'observer_id': self.observer.id,
                'observer__aavso_code': self.observer.aavso_code,
                'observations_count': 15,
            },
            {
                'observer_id': self.observer2.id,
                'observer__aavso_code': self.observer2.aavso_code,
                'observations_count': 3,
            },
        ]
        top_observers = list(Observation.objects.top_observers())
        self.assertEqual(top_observers, expected)

    def test_recent_observations(self):
        observations = Observation.objects.recent_observations()
        self.assertEqual(observations[0].observer, self.observer)

class JdNowTestCase(TestCase):
    """
    Tests for ``observations.utils.jd_now`` function.
    """
    def test_unix_epoch_jd(self):
        """
        Check JD value at the beggining of Unix epoch.
        """
        mock_time = MagicMock()
        mock_time.return_value = 0.0
        with patch.object(time, 'time', mock_time):
            jd = jd_now()
            self.assertEqual(jd, 2440587.5)

class BatchUploadFormTestCase(BaseTestCase):
    def setUp(self):
        super(BatchUploadFormTestCase, self).setUp()
        self.row = {
            'name': self.star.name,
            'magnitude': '6.6',
            'date': str(jd_now()),
            'comp1': '65',
            'comp2': '71',
            'chart': '',
            'comment_code': '',
            'notes': 'test2',
        }

    def test_parse_row(self):
        form = BatchUploadForm()
        form.process_row(self.row, self.observer)
        observation = Observation.objects.get(star=self.star, notes='test2')
        self.assertEqual(observation.comp1, self.row['comp1'])
        self.assertEqual(observation.comp2, self.row['comp2'])
<|endoftext|>


<commit_before># This script opens up all of the graphs from disk, and performs learning on
# them. This script is best run on the Rous cluster, where all of the graphs
# are located on disk (it occupies ~200GB of space, because pickling is
# not efficient.

from graphfp.layers import FingerprintLayer, LinearRegressionLayer,\
    GraphConvLayer
from graphfp.utils import initialize_network, batch_sample, train_test_split
from graphfp.optimizers import adam
from graphfp.flatten import flatten
from time import time
from autograd import grad
# from joblib import Parallel, delayed
# from collections import defaultdict
from tqdm import tqdm

import json
import pickle as pkl
import os
import pandas as pd
import autograd.numpy as np
import sys
# import multiprocessing

def read_graphs(cpu, list_of_projects):
    graphs = []
    print(cpu, len(list_of_projects))
    for i, (seqid, project) in enumerate(list_of_projects):
        print(i, seqid, project)
        try:
            with open('../../data/batch_models/{0}/model_01.pkl'.format(
                    project), 'rb') as f:
                p = pkl.load(f)
                p.graph['input_shape'] = p.nodes(data=True)[0][1][
                    'features'].shape
                p.graph['project'] = project
                p.graph['seqid'] = seqid
                graphs.append(p)

        except:
            print('did not add graph for {0}'.format(project))
            pass
    return graphs

def read_data():
    """
    Reads all of the protein graphs into memory from disk.
    Also reads in the data table into memory.
    """
    print('Reading in the model data.')
    with open('../../data/batch_summary.json') as f:
        model_data = json.load(f)

    # Read in the quantitative data
    print('Reading in quantitative drug resistance data.')
    protease_data = pd.read_csv(
        '../../data/hiv_data/hiv-protease-data-expanded.csv',
        index_col=0)

    print('Processing drug resistance data.')
    drug_data = protease_data.dropna(subset=['FPV'])[['FPV', 'seqid']]
    drug_data['FPV'] = drug_data['FPV'].apply(np.log10)

    proj_titles = {c['title']: c['code'] for c in model_data['projects']}
    # Divide the proj_titles into n_cpu jobs.

    # print('Dividing project titles across CPUs...')
    # n_cpus = multiprocessing.cpu_count()
    # cpu_jobs = defaultdict(list)
    # for i, (seqid, project) in enumerate(proj_titles.items()):
    #     cpu_idx = i % n_cpus
    #     print('assigning task {0} into cpu {1}'.format(seqid, cpu_idx))
    #     cpu_jobs[cpu_idx].append((seqid, project))

    # print('Parallelizing jobs...')
    # results = Parallel(n_jobs=-1)(delayed(read_graphs)(cpu, list_of_projects)
    #                               for cpu, list_of_projects in cpu_jobs.items()
    #                               )
    # print('Collating results into a single list...')
    # all_graphs = []
    # for result in results:
    #     all_graphs.extend(result)
    # print('There are a total of {0} graphs'.format(len(all_graphs)))

    # # Finally, clean up any None
    # all_graphs = [p for p in all_graphs if p is not None]

    all_graphs = []
    n_graphs = len(drug_data)
    for i, (seqid, project) in tqdm(enumerate(proj_titles.items())):
        if len(all_graphs) < n_graphs and\
                seqid in drug_data['seqid'].values:
            print(seqid, project)
            # We use the try/except pattern, just in case there's some
            # problem with graph reading.
            try:
                with open('../../data/batch_models/{0}/model_01.pkl'.format(
                        project), 'rb') as f:
                    p = pkl.load(f)
                    p.graph['input_shape'] = p.nodes(data=True)[0][1][
                        'features'].shape
                    print(p.graph['input_shape'])
                    p.graph['project'] = project
                    p.graph['seqid'] = seqid
                    all_graphs.append(p)
            except:
                print('did not add graph for {0}'.format(project))
                pass

    return all_graphs, drug_data

def train_loss(wb_vect, unflattener, cv=False, batch=True, batch_size=100,
               debug=False):
    """
    Training loss is MSE.

    We pass in a flattened parameter vector and its unflattener.
    """
    wb_struct = unflattener(wb_vect)

    if batch:
        batch_size = batch_size
    else:
        batch_size = len(graphs)

    if cv and not batch:
        samp_graphs, samp_inputs = batch_sample(test_graphs,
                                                input_shape,
                                                batch_size=100)
    else:
        samp_graphs, samp_inputs = batch_sample(graphs,
                                                input_shape,
                                                batch_size=batch_size)

    preds = predict(wb_struct, samp_inputs, samp_graphs)
    graph_ids = [g.graph['seqid'] for g in samp_graphs]
    graph_scores = drug_data.set_index('seqid').ix[graph_ids]['FPV'].values.\
        reshape(preds.shape)
    # print(graph_scores)
    assert preds.shape == graph_scores.shape

    mse = np.mean(np.power(preds - graph_scores, 2))

    if debug:
        print(graph_ids)
        print('Predictions:')
        print(preds)
        print('Mean: {0}'.format(np.mean(preds)))
        print('')
        print('Actual')
        print(graph_scores)
        print('Mean: {0}'.format(np.mean(graph_scores)))
        print('')
        print('Difference')
        print(preds - graph_scores)
        print('Mean Squared Error: {0}'.format(mse))
        print('')

    return mse

def predict(wb_struct, inputs, graphs):
    """
    Makes predictions by running the forward pass over all of the layers.

    Parameters:
    ===========
    - wb_struct: a dictionary of weights and biases stored for each layer.
    - inputs: the input data matrix. should be one row per graph.
    - graphs: a list of all graphs.
    """
    curr_inputs = inputs

    for i, layer in enumerate(layers):
        wb = wb_struct['layer{0}_{1}'.format(i, layer)]
        curr_inputs = layer.forward_pass(wb, curr_inputs, graphs)
    return curr_inputs

def callback(wb, i):
    """
    Any function you want to run at each iteration of the optimization.
    """
    wb_vect, wb_unflattener = flatten(wb)
    print('Iteration: {0}'.format(i))
    print('Training Loss: ')

    # Record training set train_loss
    tl = train_loss(wb_vect, wb_unflattener, batch=True, cv=False,
                    batch_size=100)
    print(tl)
    trainloss.append(tl)

    # Record the preds vs. actual for the training set.
    batch_size = 10
    samp_graphs, samp_inputs = batch_sample(graphs, input_shape,
                                            batch_size)

    preds = predict(wb, samp_inputs, samp_graphs)
    graph_ids = [g.graph['seqid'] for g in samp_graphs]
    graph_scores = drug_data.set_index('seqid').ix[graph_ids]['FPV'].\
        values.reshape(preds.shape)

    preds_vs_actual[i] = dict()
    preds_vs_actual[i]['preds'] = preds
    preds_vs_actual[i]['actual'] = graph_scores

    # Record cross-validation train_loss
    cv_tl = train_loss(wb_vect, wb_unflattener, cv=True, batch=False)
    trainloss_cv.append(cv_tl)

    # Record the preds vs. actual for 10 test_graphs
    samp_graphs, samp_inputs = batch_sample(test_graphs, input_shape,
                                            batch_size)
    preds_cv = predict(wb, samp_inputs, samp_graphs)
    graph_ids = [g.graph['seqid'] for g in samp_graphs]
    graph_scores_cv = drug_data.set_index('seqid').ix[graph_ids]['FPV'].\
        values.reshape(preds_cv.shape)

    preds_vs_actual_cv[i] = dict()
    preds_vs_actual_cv[i]['preds'] = preds_cv
    preds_vs_actual_cv[i]['actual'] = graph_scores_cv

    print('cross-validated training loss: {0}'.format(cv_tl))

    # Report on the expected time left.
    time_elapsed = time() - start
    print('Total time: {0} min {1} sec'.format(int(time_elapsed / 60),
          time_elapsed % 60))
    time_left = (num_iters - i + 1) * (time_elapsed / (i + 1))
    print('Expected time left: {0} min {1} sec'.format(int(time_left / 60),
          time_left % 60))

    print('')

if __name__ == '__main__':

    num_iters = int(sys.argv[1])

    start = time()

    # Read in all of the graphs
    all_graphs, drug_data = read_data()
    print('total num of graphs: {0}'.format(len(all_graphs)))

    # Split the graphs into a training and testing set.
    # Also reads in the data table into memory.
    graphs, test_graphs = train_test_split(all_graphs, test_fraction=0.2)

    # Define the gradient function
    grad_tl = grad(train_loss)

    # Specify neural network shape and meta-parameters
    input_shape = graphs[0].graph['input_shape']
    print('input shape: {0}'.format(input_shape))
    layers = [GraphConvLayer((input_shape[1], input_shape[1])),
              GraphConvLayer((input_shape[1], input_shape[1])),
              FingerprintLayer(input_shape[1]),
              LinearRegressionLayer((input_shape[1], 1)),
              ]

    # Initialize neural network weights and baises, as well as an empty
    # container for holding the training losses.
    wb_all = initialize_network(input_shape, graphs, layers)
    trainloss = list()
    preds_vs_actual = dict()
    trainloss_cv = list()
    preds_vs_actual_cv = dict()

    # Train the neural network on the data.
    wb_vect, wb_unflattener = adam(grad_tl, wb_all, callback=callback,
                                   num_iters=num_iters)
    wb_all = wb_unflattener(wb_vect)

    # Write training losses and weights/biases to disk.
    if not os.path.exists('outputs'):
        os.mkdir('outputs')

    handle = 'all-graphs_{0}-iters'.format(num_iters)

    with open('outputs/{0}_trainloss.pkl'.format(handle), 'wb') as f:
        pkl.dump(trainloss, f)

    with open('outputs/{0}_wbs.pkl'.format(handle), 'wb') as f:
        pkl.dump(wb_all, f)

    with open('outputs/{0}_predsactual.pkl'.format(handle), 'wb') as f:
        pkl.dump(preds_vs_actual, f)

    with open('outputs/{0}_trainloss_cv.pkl'.format(handle), 'wb') as f:
        pkl.dump(trainloss_cv, f)

    with open('outputs/{0}_predsactual_cv.pkl'.format(handle), 'wb') as f:
        pkl.dump(preds_vs_actual_cv, f)
<commit_msg>changed all batch sizes to 1 for experimentation with profiler<commit_after># This script opens up all of the graphs from disk, and performs learning on
# them. This script is best run on the Rous cluster, where all of the graphs
# are located on disk (it occupies ~200GB of space, because pickling is
# not efficient.

from graphfp.layers import FingerprintLayer, LinearRegressionLayer,\
    GraphConvLayer
from graphfp.utils import initialize_network, batch_sample, train_test_split
from graphfp.optimizers import adam
from graphfp.flatten import flatten
from time import time
from autograd import grad
# from joblib import Parallel, delayed
# from collections import defaultdict
from tqdm import tqdm

import json
import pickle as pkl
import os
import pandas as pd
import autograd.numpy as np
import sys
# import multiprocessing

def read_data():
    """
    Reads all of the protein graphs into memory from disk.
    Also reads in the data table into memory.
    """
    print('Reading in the model data.')
    with open('../../data/batch_summary.json') as f:
        model_data = json.load(f)

    # Read in the quantitative data
    print('Reading in quantitative drug resistance data.')
    protease_data = pd.read_csv(
        '../../data/hiv_data/hiv-protease-data-expanded.csv',
        index_col=0)

    print('Processing drug resistance data.')
    drug_data = protease_data.dropna(subset=['FPV'])[['FPV', 'seqid']]
    drug_data['FPV'] = drug_data['FPV'].apply(np.log10)

    proj_titles = {c['title']: c['code'] for c in model_data['projects']}

    all_graphs = []
    n_graphs = 200
    for i, (seqid, project) in tqdm(enumerate(proj_titles.items())):
        if len(all_graphs) < n_graphs and\
                seqid in drug_data['seqid'].values:
            print(seqid, project)
            # We use the try/except pattern, just in case there's some
            # problem with graph reading.
            try:
                with open('../../data/batch_models/{0}/model_01.pkl'.format(
                        project), 'rb') as f:
                    p = pkl.load(f)
                    p.graph['input_shape'] = p.nodes(data=True)[0][1][
                        'features'].shape
                    print(p.graph['input_shape'])
                    p.graph['project'] = project
                    p.graph['seqid'] = seqid
                    all_graphs.append(p)
            except:
                print('did not add graph for {0}'.format(project))
                pass

    return all_graphs, drug_data

def train_loss(wb_vect, unflattener, cv=False, batch=True, batch_size=1,
               debug=False):
    """
    Training loss is MSE.

    We pass in a flattened parameter vector and its unflattener.
    """
    wb_struct = unflattener(wb_vect)

    if batch:
        batch_size = batch_size
    else:
        batch_size = len(graphs)

    if cv and not batch:
        samp_graphs, samp_inputs = batch_sample(test_graphs,
                                                input_shape,
                                                batch_size=1)
    else:
        samp_graphs, samp_inputs = batch_sample(graphs,
                                                input_shape,
                                                batch_size=batch_size)

    preds = predict(wb_struct, samp_inputs, samp_graphs)
    graph_ids = [g.graph['seqid'] for g in samp_graphs]
    graph_scores = drug_data.set_index('seqid').ix[graph_ids]['FPV'].values.\
        reshape(preds.shape)
    # print(graph_scores)
    assert preds.shape == graph_scores.shape

    mse = np.mean(np.power(preds - graph_scores, 2))

    if debug:
        print(graph_ids)
        print('Predictions:')
        print(preds)
        print('Mean: {0}'.format(np.mean(preds)))
        print('')
        print('Actual')
        print(graph_scores)
        print('Mean: {0}'.format(np.mean(graph_scores)))
        print('')
        print('Difference')
        print(preds - graph_scores)
        print('Mean Squared Error: {0}'.format(mse))
        print('')

    return mse

def predict(wb_struct, inputs, graphs):
    """
    Makes predictions by running the forward pass over all of the layers.

    Parameters:
    ===========
    - wb_struct: a dictionary of weights and biases stored for each layer.
    - inputs: the input data matrix. should be one row per graph.
    - graphs: a list of all graphs.
    """
    curr_inputs = inputs

    for i, layer in enumerate(layers):
        wb = wb_struct['layer{0}_{1}'.format(i, layer)]
        curr_inputs = layer.forward_pass(wb, curr_inputs, graphs)
    return curr_inputs

def callback(wb, i):
    """
    Any function you want to run at each iteration of the optimization.
    """
    wb_vect, wb_unflattener = flatten(wb)
    print('Iteration: {0}'.format(i))
    print('Training Loss: ')

    # Record training set train_loss
    tl = train_loss(wb_vect, wb_unflattener, batch=True, cv=False,
                    batch_size=1)
    print(tl)
    trainloss.append(tl)

    # Record the preds vs. actual for the training set.
    batch_size = 1
    samp_graphs, samp_inputs = batch_sample(graphs, input_shape,
                                            batch_size)

    preds = predict(wb, samp_inputs, samp_graphs)
    graph_ids = [g.graph['seqid'] for g in samp_graphs]
    graph_scores = drug_data.set_index('seqid').ix[graph_ids]['FPV'].\
        values.reshape(preds.shape)

    preds_vs_actual[i] = dict()
    preds_vs_actual[i]['preds'] = preds
    preds_vs_actual[i]['actual'] = graph_scores

    # Record cross-validation train_loss
    cv_tl = train_loss(wb_vect, wb_unflattener, cv=True, batch=False)
    trainloss_cv.append(cv_tl)

    # Record the preds vs. actual for 10 test_graphs
    samp_graphs, samp_inputs = batch_sample(test_graphs, input_shape,
                                            batch_size)
    preds_cv = predict(wb, samp_inputs, samp_graphs)
    graph_ids = [g.graph['seqid'] for g in samp_graphs]
    graph_scores_cv = drug_data.set_index('seqid').ix[graph_ids]['FPV'].\
        values.reshape(preds_cv.shape)

    preds_vs_actual_cv[i] = dict()
    preds_vs_actual_cv[i]['preds'] = preds_cv
    preds_vs_actual_cv[i]['actual'] = graph_scores_cv

    print('cross-validated training loss: {0}'.format(cv_tl))

    # Report on the expected time left.
    time_elapsed = time() - start
    print('Total time: {0} min {1} sec'.format(int(time_elapsed / 60),
          time_elapsed % 60))
    time_left = (num_iters - i + 1) * (time_elapsed / (i + 1))
    print('Expected time left: {0} min {1} sec'.format(int(time_left / 60),
          time_left % 60))

    print('')

if __name__ == '__main__':

    num_iters = int(sys.argv[1])

    start = time()

    # Read in all of the graphs
    all_graphs, drug_data = read_data()
    print('total num of graphs: {0}'.format(len(all_graphs)))

    # Split the graphs into a training and testing set.
    # Also reads in the data table into memory.
    graphs, test_graphs = train_test_split(all_graphs, test_fraction=0.2)

    # Define the gradient function
    grad_tl = grad(train_loss)

    # Specify neural network shape and meta-parameters
    input_shape = graphs[0].graph['input_shape']
    print('input shape: {0}'.format(input_shape))
    layers = [GraphConvLayer((input_shape[1], input_shape[1])),
              GraphConvLayer((input_shape[1], input_shape[1])),
              FingerprintLayer(input_shape[1]),
              LinearRegressionLayer((input_shape[1], 1)),
              ]

    # Initialize neural network weights and baises, as well as an empty
    # container for holding the training losses.
    wb_all = initialize_network(input_shape, graphs, layers)
    trainloss = list()
    preds_vs_actual = dict()
    trainloss_cv = list()
    preds_vs_actual_cv = dict()

    # Train the neural network on the data.
    wb_vect, wb_unflattener = adam(grad_tl, wb_all, callback=callback,
                                   num_iters=num_iters)
    wb_all = wb_unflattener(wb_vect)

    # Write training losses and weights/biases to disk.
    if not os.path.exists('outputs'):
        os.mkdir('outputs')

    handle = 'all-graphs_{0}-iters'.format(num_iters)

    with open('outputs/{0}_trainloss.pkl'.format(handle), 'wb') as f:
        pkl.dump(trainloss, f)

    with open('outputs/{0}_wbs.pkl'.format(handle), 'wb') as f:
        pkl.dump(wb_all, f)

    with open('outputs/{0}_predsactual.pkl'.format(handle), 'wb') as f:
        pkl.dump(preds_vs_actual, f)

    with open('outputs/{0}_trainloss_cv.pkl'.format(handle), 'wb') as f:
        pkl.dump(trainloss_cv, f)

    with open('outputs/{0}_predsactual_cv.pkl'.format(handle), 'wb') as f:
        pkl.dump(preds_vs_actual_cv, f)
<|endoftext|>