<commit_before>#
# Population MCMC
#
# This file is part of PINTS.
#  Copyright (c) 2017, University of Oxford.
#  For licensing information, see the LICENSE file distributed with the PINTS
#  software package.
#
# Some code in this file was adapted from Myokit (see http://myokit.org)
#
from __future__ import absolute_import, division
from __future__ import print_function, unicode_literals
import pints
import numpy as np

class PopulationMCMC(pints.SingleChainMCMC):
    """
    Creates a chain of samples from a target distribution, using the population
    MCMC (simulated tempering) routine described in algorithm 1 in [1].

    The algorithm goes through the following steps (after initialising ``N``
    chains):

    1. Mutation: randomly select chain ``i`` and update the chain using a
    Markov kernel that admits ``p_i`` as its invariant distribution.

    2. Exchange: Select another chain ``j`` at random from the remaining and
    swap the parameter vector of ``i`` and ``j`` with probability
    ``min(1, A)``,

    ``A = p_i(x_j) * p_j(x_i) / (p_i(x_i) * p_j(x_j))``

    where ``x_i`` and ``x_j`` are the current values of chains ``i`` and ``j``,
    respectively, where ``p_i = p(theta|data) ^ (1 - T_i)``, where
    ``p(theta|data)`` is the target distribution and ``T_i`` is bounded between
    ``[0, 1]`` and represents a tempering parameter.

    We use a range of ``T = (0,delta_T,...,1)``, where
    ``delta_T = 1 / num_temperatures``, and the chain with ``T_i = 0`` is the
    one whose target distribution we want to sample.

    *Extends:* :class:`SingleChainMCMC`

    [1] "On population-based simulation for static inference", Ajay Jasra,
    David A. Stephens and Christopher C. Holmes, Statistical Computing, 2007.
    """
    def __init__(self, x0, sigma0=None):
        super(PopulationMCMC, self).__init__(x0, sigma0)

        # Set initial state
        self._running = False

        # Current points, and the logpdf of those points (_not_ the tempered
        # version!)
        self._current = None
        self._current_log_pdf = None

        # Single proposed point
        self._proposed = None

        # Inner chains
        self._chains = None

        #
        # Default settings
        #
        self._method = pints.AdaptiveCovarianceMCMC

        # Temperature schedule
        self._schedule = None
        self.set_temperature_schedule()

        #
        # Logging
        #
        self._have_exchanged = False

    def ask(self):
        """ See :meth:`SingleChainMCMC.ask()`. """
        # Initialise on first call
        if not self._running:
            self._initialise()

        # Propose new points
        if self._proposed is None:

            # Select chains to update and exchange/crossover
            n = len(self._chains)
            self._i, self._j = np.random.choice(n, 2, replace=False)

            # Propose new point
            self._proposed = self._chains[self._i].ask()

        # Return proposed point
        return self._proposed

    def _initialise(self):
        """
        Initialises the routine before the first iteration.
        """
        if self._running:
            raise RuntimeError('Already initialised.')

        # Create inner chains
        n = len(self._schedule)
        self._chains = [self._method(self._x0, self._sigma0) for i in range(n)]

        # Propose initial point
        self._current = None
        self._current_log_pdf = None
        self._proposed = self._x0

        # Next chain to update
        self._i = 0

        # Next chain to exchange / crossover with
        self._j = 1

        # Ask all inner chains for first point (should be x0, so ignore!)
        for chain in self._chains:
            chain.ask()

        # Update sampler state
        self._running = True

    def _log_init(self, logger):
        """ See :meth:`Loggable._log_init()`. """
        n = len(self._schedule)
        logger.add_counter('i', max_value=n)
        logger.add_counter('j', max_value=n)
        logger.add_string('Ex.', width=3)

    def _log_write(self, logger):
        """ See :meth:`Loggable._log_write()`. """
        logger.log(self._i)
        logger.log(self._j)
        logger.log('yes' if self._have_exchanged else 'no')

    def name(self):
        """ See :meth:`pints.MCMCSampler.name()`. """
        return 'Population MCMC'

    def temperature_schedule(self):
        """
        Returns the temperature schedule used in the tempering algorithm. Each
        temperature ``T`` pertains to particular chain whose stationary
        distribution is ``p(theta|data) ^ (1 - T)``.
        """
        return self._schedule

    def replace(self, x, fx):
        """
        Not implemented for this method!
        """
        raise NotImplementedError

    def set_temperature_schedule(self, schedule=10):
        """
        Sets a temperature schedule.

        If ``schedule`` is an ``int`` it is interpreted as the number of
        temperatures and a schedule is generated accordingly.

        If ``schedule`` is a list (or array) it is interpreted as a custom
        temperature schedule.
        """
        if self._running:
            raise RuntimeError(
                'Temperature schedule cannot be changed during run.')

        # Check type of schedule argument
        if np.isscalar(schedule):

            # Set using int
            schedule = int(schedule)
            if schedule < 2:
                raise ValueError(
                    'A schedule must contain at least two temperatures.')
            self._schedule = np.linspace(0, 0.95, schedule)
            self._schedule.setflags(write=False)

        else:

            # Set to custom schedule
            schedule = pints.vector(schedule)
            if len(schedule) < 2:
                raise ValueError(
                    'A schedule must contain at least two temperatures.')
            if schedule[0] != 0:
                raise ValueError(
                    'First element of temperature schedule must be 0.')

            # Check vector elements all between 0 and 1 (inclusive)
            if np.any(schedule < 0):
                raise ValueError('Temperatures must be non-negative.')
            if np.any(schedule > 1):
                raise ValueError('Temperatures cannot exceed 1.')

            # Store
            self._schedule = schedule

    def tell(self, fx):
        """ See :meth:`pints.SingleChainMCMC.tell()`. """
        # Check if we had a proposal
        if self._proposed is None:
            raise RuntimeError('Tell called before proposal was set.')

        # Ensure fx is a float
        fx = float(fx)

        # First point?
        if self._current is None:
            # Pass to inner chains (ignore returned x0)
            for chain in self._chains:
                chain.tell(fx)

            # Always accept
            n = len(self._chains)
            self._current = np.array([self._x0] * n)

            # Store untempered logpdfs
            self._current_log_pdf = np.array([fx] * n)

            # Clear proposal
            self._proposed = None

            # Return first point for chain 0
            sample = np.array(self._current[0], copy=False)
            sample.setflags(write=False)
            return sample

        # Perform mutation step (update one chain)

        # Update chain, get new sample
        sample = self._chains[self._i].tell(fx * (1 - self._schedule[self._i]))

        # Update current sample and untempered log pdf
        if np.any(sample != self._current[self._i]):
            self._current[self._i] = sample
            self._current_log_pdf[self._i] = fx

        # Clear proposal
        self._proposed = None

        # Perform exchange step

        # Calculate current tempered likelihoods
        pii = (1 - self._schedule[self._i]) * self._current_log_pdf[self._i]
        pjj = (1 - self._schedule[self._j]) * self._current_log_pdf[self._j]

        # Calculate proposed log likelihoods
        pij = (1 - self._schedule[self._i]) * self._current_log_pdf[self._j]
        pji = (1 - self._schedule[self._j]) * self._current_log_pdf[self._i]

        # Accept/reject exchange step
        self._have_exchanged = False
        if np.isfinite(pij) and np.isfinite(pji):
            u = np.log(np.random.uniform(0, 1))
            if u < pij + pji - (pii + pjj):
                self._chains[self._i].replace(self._current[self._j], pij)
                self._chains[self._j].replace(self._current[self._j], pji)
                self._have_exchanged = True

        # Return new point for chain 0
        sample = np.array(self._current[0], copy=False)
        sample.setflags(write=False)
        return sample

<commit_msg>Fixed issues with population mcmc. Closes #475.<commit_after>#
# Population MCMC
#
# This file is part of PINTS.
#  Copyright (c) 2017, University of Oxford.
#  For licensing information, see the LICENSE file distributed with the PINTS
#  software package.
#
# Some code in this file was adapted from Myokit (see http://myokit.org)
#
from __future__ import absolute_import, division
from __future__ import print_function, unicode_literals
import pints
import numpy as np

class PopulationMCMC(pints.SingleChainMCMC):
    """
    Creates a chain of samples from a target distribution, using the population
    MCMC (simulated tempering) routine described in algorithm 1 in [1].

    The algorithm goes through the following steps (after initialising ``N``
    chains):

    1. Mutation: randomly select chain ``i`` and update the chain using a
    Markov kernel that admits ``p_i`` as its invariant distribution.

    2. Exchange: Select another chain ``j`` at random from the remaining and
    swap the parameter vector of ``i`` and ``j`` with probability
    ``min(1, A)``,

    ``A = p_i(x_j) * p_j(x_i) / (p_i(x_i) * p_j(x_j))``

    where ``x_i`` and ``x_j`` are the current values of chains ``i`` and ``j``,
    respectively, where ``p_i = p(theta|data) ^ (1 - T_i)``, where
    ``p(theta|data)`` is the target distribution and ``T_i`` is bounded between
    ``[0, 1]`` and represents a tempering parameter.

    We use a range of ``T = (0,delta_T,...,1)``, where
    ``delta_T = 1 / num_temperatures``, and the chain with ``T_i = 0`` is the
    one whose target distribution we want to sample.

    *Extends:* :class:`SingleChainMCMC`

    [1] "On population-based simulation for static inference", Ajay Jasra,
    David A. Stephens and Christopher C. Holmes, Statistical Computing, 2007.
    """
    def __init__(self, x0, sigma0=None):
        super(PopulationMCMC, self).__init__(x0, sigma0)

        # Set initial state
        self._running = False

        # Current points, and the logpdf of those points (_not_ the tempered
        # version!)
        self._current = None
        self._current_log_pdf = None

        # Single proposed point
        self._proposed = None

        # Inner chains
        self._chains = None

        #
        # Default settings
        #
        self._method = pints.AdaptiveCovarianceMCMC
        self._needs_initial_phase = True
        self._in_initial_phase = True

        # Temperature schedule
        self._schedule = None
        self.set_temperature_schedule()

        #
        # Logging
        #
        self._have_exchanged = False

    def ask(self):
        """ See :meth:`SingleChainMCMC.ask()`. """
        # Initialise on first call
        if not self._running:
            self._initialise()

        # Propose new points
        if self._proposed is None:

            # Select chains to update and exchange/crossover
            n = len(self._chains)
            self._i, self._j = np.random.choice(n, 2, replace=False)

            # Propose new point
            self._proposed = self._chains[self._i].ask()

        # Return proposed point
        return self._proposed

    def in_initial_phase(self):
        """
        See :meth:`MCMCSampling.in_initial_phase()`.
        """
        return self._in_initial_phase

    def _initialise(self):
        """
        Initialises the routine before the first iteration.
        """
        if self._running:
            raise RuntimeError('Already initialised.')

        # Create inner chains
        n = len(self._schedule)
        self._chains = [self._method(self._x0, self._sigma0) for i in range(n)]

        # Set initial phase for methods that need it
        if self._needs_initial_phase:
            for chain in self._chains:
                chain.set_initial_phase(self._in_initial_phase)

        # Propose initial point
        self._current = None
        self._current_log_pdf = None
        self._proposed = self._x0

        # Next chain to update
        self._i = 0

        # Next chain to exchange / crossover with
        self._j = 1

        # Ask all inner chains for first point (should be x0, so ignore!)
        for chain in self._chains:
            chain.ask()

        # Update sampler state
        self._running = True

    def _log_init(self, logger):
        """ See :meth:`Loggable._log_init()`. """
        n = len(self._schedule)
        logger.add_counter('i', max_value=n)
        logger.add_counter('j', max_value=n)
        logger.add_string('Ex.', width=3)

    def _log_write(self, logger):
        """ See :meth:`Loggable._log_write()`. """
        logger.log(self._i)
        logger.log(self._j)
        logger.log('yes' if self._have_exchanged else 'no')

    def name(self):
        """ See :meth:`pints.MCMCSampler.name()`. """
        return 'Population MCMC'

    def needs_initial_phase(self):
        """ See :meth:`pints.MCMCSampler.needs_initial_phase()`. """
        return self._needs_initial_phase

    def replace(self, x, fx):
        """
        Not implemented for this method!
        """
        raise NotImplementedError

    def set_initial_phase(self, phase):
        """
        See :meth:`MCMCSampling.set_initial_phase()`.
        """
        self._in_initial_phase = bool(phase)
        if self._running:
            for chain in self._chains:
                chain.set_initial_phase(self._in_initial_phase)

    def set_temperature_schedule(self, schedule=10):
        """
        Sets a temperature schedule.

        If ``schedule`` is an ``int`` it is interpreted as the number of
        temperatures and a schedule is generated accordingly.

        If ``schedule`` is a list (or array) it is interpreted as a custom
        temperature schedule.
        """
        if self._running:
            raise RuntimeError(
                'Temperature schedule cannot be changed during run.')

        # Check type of schedule argument
        if np.isscalar(schedule):

            # Set using int
            schedule = int(schedule)
            if schedule < 2:
                raise ValueError(
                    'A schedule must contain at least two temperatures.')
            self._schedule = np.linspace(0, 0.95, schedule)
            self._schedule.setflags(write=False)

        else:

            # Set to custom schedule
            schedule = pints.vector(schedule)
            if len(schedule) < 2:
                raise ValueError(
                    'A schedule must contain at least two temperatures.')
            if schedule[0] != 0:
                raise ValueError(
                    'First element of temperature schedule must be 0.')

            # Check vector elements all between 0 and 1 (inclusive)
            if np.any(schedule < 0):
                raise ValueError('Temperatures must be non-negative.')
            if np.any(schedule > 1):
                raise ValueError('Temperatures cannot exceed 1.')

            # Store
            self._schedule = schedule

    def tell(self, fx):
        """ See :meth:`pints.SingleChainMCMC.tell()`. """
        # Check if we had a proposal
        if self._proposed is None:
            raise RuntimeError('Tell called before proposal was set.')

        # Ensure fx is a float
        fx = float(fx)

        # First point?
        if self._current is None:
            # Pass to inner chains (ignore returned x0)
            for chain in self._chains:
                chain.tell(fx)

            # Always accept
            n = len(self._chains)
            self._current = np.array([self._x0] * n)

            # Store untempered logpdfs
            self._current_log_pdf = np.array([fx] * n)

            # Clear proposal
            self._proposed = None

            # Return first point for chain 0
            sample = np.array(self._current[0], copy=False)
            sample.setflags(write=False)
            return sample

        # Perform mutation step (update one chain)

        # Update chain, get new sample
        sample = self._chains[self._i].tell(fx * (1 - self._schedule[self._i]))

        # Update current sample and untempered log pdf
        if np.any(sample != self._current[self._i]):
            self._current[self._i] = sample
            self._current_log_pdf[self._i] = fx

        # Clear proposal
        self._proposed = None

        # Perform exchange step

        # Calculate current tempered likelihoods
        pii = (1 - self._schedule[self._i]) * self._current_log_pdf[self._i]
        pjj = (1 - self._schedule[self._j]) * self._current_log_pdf[self._j]

        # Calculate proposed log likelihoods
        pij = (1 - self._schedule[self._i]) * self._current_log_pdf[self._j]
        pji = (1 - self._schedule[self._j]) * self._current_log_pdf[self._i]

        # Accept/reject exchange step
        self._have_exchanged = False
        if np.isfinite(pij) and np.isfinite(pji):
            u = np.log(np.random.uniform(0, 1))
            if u < pij + pji - (pii + pjj):
                self._chains[self._i].replace(self._current[self._j], pij)
                self._chains[self._j].replace(self._current[self._j], pji)
                self._have_exchanged = True

        # Return new point for chain 0
        sample = np.array(self._current[0], copy=False)
        sample.setflags(write=False)
        return sample

    def temperature_schedule(self):
        """
        Returns the temperature schedule used in the tempering algorithm. Each
        temperature ``T`` pertains to particular chain whose stationary
        distribution is ``p(theta|data) ^ (1 - T)``.
        """
        return self._schedule

<|endoftext|>


<commit_before>import os, re
import subprocess
import time
import pandas as pd
import numpy as np
from StringIO import StringIO

def write_slurm_script(filename, cmd, **batch_options):
    with open(filename, 'w') as fout:
        fout.write('#!/bin/bash\n')
        for opts in batch_options.items():
            fout.write('#SBATCH --{0}={1}\n'.format(*opts))

        fout.write('\n')
        fout.write('{0}\n'.format(cmd))

def get_job_status(jobid, wait=30):
    """Returns status of slurm job <jobid>

    Currently parses output of `sacct`.  Perhaps would
    be a good idea to move this to pyslurm (though this would
    add a dependency.)
    """
    cmd = 'scontrol show job {0}'.format(jobid)
    try:
        output = subprocess.check_output(cmd, shell=True)
        m = re.search('JobState=(\w+)', output)
    except subprocess.CalledProcessError:
        m = False

    status = None
    if m:
        status = m.group(1)
    else:
        repeat = 0
        while not m and repeat < wait:
            cmd = 'sacct -b -j {0}'.format(jobid)
            output = subprocess.check_output(cmd, shell=True)
            m = re.search('{0}\s+([A-Z]+)'.format(jobid), output)
            time.sleep(1)
            repeat += 1
        if m:
            status = m.group(1)

    if status is None:
        raise ValueError('Job not found: {0}'.format(jobid))
    else:
        return status

def get_pipeline_status(name, info=('jobid','State','Elapsed','start','end','exitcode')):
    pipe_logfile = os.path.join('{0}_output'.format(name), 'pipe.log')

    # Split up by ====
    with open(pipe_logfile) as fin:
        file_str = fin.read()

    pattern = re.compile('='*30 + '\n' + '((\S+ \d+\n)+)')
    m = re.findall(pattern, file_str)

    run_lists = [mm[0].splitlines() for mm in m]
    results = []
    for run in run_lists:
        jobs, ids = zip(*[l.split() for l in run])
        id_str = ','.join(ids)
        info_str = ','.join(info)

        ids = np.array(ids).astype(int)

        cmd = 'sacct -j {0} --format {1}'.format(id_str, info_str)
        o = subprocess.check_output(cmd, shell=True)

        # Filter to make sure only one line per job is kept
        keep_lines = []
        for l in o.splitlines():
            x = l.split()
            if re.search('^\d+$', x[0]):
                keep_lines.append(l)

        o = '\n'.join(keep_lines)

        df = pd.read_table(StringIO(o), header=None, names=info, delim_whitespace=True,
                            index_col=0)

        # Add jobs; this will also add null rows for jobs without slurmdb entry
        df['job'] = None
        for i,j in zip(ids, jobs):
            if i not in df.index:
                df.ix[i] = None
            df.ix[i, 'job'] = j

        # Reorder columns to make job name first
        new_cols = ['job'] + list(df.columns[:-1])
        df = df.ix[:, new_cols]
        results.append(df)

    return results<commit_msg>debug trace<commit_after>import os, re
import subprocess
import time
import pandas as pd
import numpy as np
from StringIO import StringIO
import pbd

def write_slurm_script(filename, cmd, **batch_options):
    with open(filename, 'w') as fout:
        fout.write('#!/bin/bash\n')
        for opts in batch_options.items():
            fout.write('#SBATCH --{0}={1}\n'.format(*opts))

        fout.write('\n')
        fout.write('{0}\n'.format(cmd))

def get_job_status(jobid, wait=30):
    """Returns status of slurm job <jobid>

    Currently parses output of `sacct`.  Perhaps would
    be a good idea to move this to pyslurm (though this would
    add a dependency.)
    """
    cmd = 'scontrol show job {0}'.format(jobid)
    try:
        output = subprocess.check_output(cmd, shell=True)
        m = re.search('JobState=(\w+)', output)
    except subprocess.CalledProcessError:
        m = False

    status = None
    if m:
        status = m.group(1)
    else:
        repeat = 0
        while not m and repeat < wait:
            cmd = 'sacct -b -j {0}'.format(jobid)
            output = subprocess.check_output(cmd, shell=True)
            m = re.search('{0}\s+([A-Z]+)'.format(jobid), output)
            time.sleep(1)
            repeat += 1
        if m:
            status = m.group(1)

    if status is None:
        raise ValueError('Job not found: {0}'.format(jobid))
    else:
        return status

def get_pipeline_status(name, info=('jobid','State','Elapsed','start','end','exitcode')):
    pipe_logfile = os.path.join('{0}_output'.format(name), 'pipe.log')

    # Split up by ====
    with open(pipe_logfile) as fin:
        file_str = fin.read()

    pattern = re.compile('='*30 + '\n' + '((\S+ \d+\n)+)')
    m = re.findall(pattern, file_str)

    run_lists = [mm[0].splitlines() for mm in m]
    results = []
    for run in run_lists:
        jobs, ids = zip(*[l.split() for l in run])
        id_str = ','.join(ids)
        info_str = ','.join(info)

        ids = np.array(ids).astype(int)

        cmd = 'sacct -j {0} --format {1}'.format(id_str, info_str)
        o = subprocess.check_output(cmd, shell=True)

        # Filter to make sure only one line per job is kept
        keep_lines = []
        for l in o.splitlines():
            x = l.split()
            if re.search('^\d+$', x[0]):
                keep_lines.append(l)

        o = '\n'.join(keep_lines)

        df = pd.read_table(StringIO(o), header=None, names=info, delim_whitespace=True,
                            index_col=0)

        # Add jobs; this will also add null rows for jobs without slurmdb entry
        df['job'] = None
        for i,j in zip(ids, jobs):
            if i not in df.index:
                try:
                    df.ix[i] = None
                except:
                    pdb.set_trace()
            df.ix[i, 'job'] = j

        # Reorder columns to make job name first
        new_cols = ['job'] + list(df.columns[:-1])
        df = df.ix[:, new_cols]
        results.append(df)

    return results<|endoftext|>


<commit_before>import os, re
import subprocess
import time

def write_slurm_script(filename, cmd, **batch_options):
    with open(filename, 'w') as fout:
        fout.write('#!/bin/bash\n')
        for opts in batch_options.items():
            fout.write('#SBATCH --{0}={1}\n'.format(*opts))

        fout.write('\n')
        fout.write('{0}\n'.format(cmd))

def get_job_status(jobid, wait=30):
    """Returns status of slurm job <jobid>

    Currently parses output of `sacct`.  Perhaps would
    be a good idea to move this to pyslurm (though this would
    add a dependency.)
    """
    cmd = 'scontrol show job {0}'.format(jobid)
    output = subprocess.check_output(cmd, shell=True)
    m = re.search('JobState=(\w+)', output)
    status = None
    if m:
        status = m.group(1)
    else:
        repeat = 0
        while not m and repeat < wait:
            cmd = 'sacct -b -j {0}'.format(jobid)
            output = subprocess.check_output(cmd, shell=True)
            m = re.search('{0}\s+([A-Z]+)'.format(jobid), output)
            time.sleep(1)
            repeat += 1
        if m:
            status = m.group(1)

    if status is None:
        raise ValueError('Job not found: {0}'.format(jobid))
    else:
        return status

def get_pipeline_status(name):
    pipe_logfile = os.path.join('{0}_output'.format(name), 'pipe.log')

    # Split up by ====
    with open(pipe_logfile) as fin:
        file_str = fin.read()

    pattern = re.compile('='*30 + '\n' + '(TEST\n)?' + '(20\d\d-\d\d-\d\d.*)\n' + '.*' + '='*30, flags=re.DOTALL)# + '\n' + '(.*)' + '($|=)')
    m = re.findall(pattern, file_str)
    return pattern, m, file_str<commit_msg>returning whole pattern<commit_after>import os, re
import subprocess
import time

def write_slurm_script(filename, cmd, **batch_options):
    with open(filename, 'w') as fout:
        fout.write('#!/bin/bash\n')
        for opts in batch_options.items():
            fout.write('#SBATCH --{0}={1}\n'.format(*opts))

        fout.write('\n')
        fout.write('{0}\n'.format(cmd))

def get_job_status(jobid, wait=30):
    """Returns status of slurm job <jobid>

    Currently parses output of `sacct`.  Perhaps would
    be a good idea to move this to pyslurm (though this would
    add a dependency.)
    """
    cmd = 'scontrol show job {0}'.format(jobid)
    output = subprocess.check_output(cmd, shell=True)
    m = re.search('JobState=(\w+)', output)
    status = None
    if m:
        status = m.group(1)
    else:
        repeat = 0
        while not m and repeat < wait:
            cmd = 'sacct -b -j {0}'.format(jobid)
            output = subprocess.check_output(cmd, shell=True)
            m = re.search('{0}\s+([A-Z]+)'.format(jobid), output)
            time.sleep(1)
            repeat += 1
        if m:
            status = m.group(1)

    if status is None:
        raise ValueError('Job not found: {0}'.format(jobid))
    else:
        return status

def get_pipeline_status(name):
    pipe_logfile = os.path.join('{0}_output'.format(name), 'pipe.log')

    # Split up by ====
    with open(pipe_logfile) as fin:
        file_str = fin.read()

    pattern = re.compile('='*30 + '\n' + '(TEST\n)?' + '(20\d\d-\d\d-\d\d.*)\n' + \
                         '.*' + '='*30 + '\n' + '(.*)' + '($|=)', flags=re.DOTALL)
    m = re.findall(pattern, file_str)
    return m<|endoftext|>


<commit_before>#!/usr/bin/env python

'''
Get greengenes taxonomies. Given an otu table with otu ids in the first column, search through
the greengenes taxonomy list. Output the taxonomies in order.

If the input database is a pickle, just load that dictionary.
'''

import sys, argparse, re, cPickle as pickle

def table_ids(fn):
    '''get the otu ids from the otu table with filename fn'''
    with open(fn) as f:
        ids = [line.split()[0] for line in f]

    # remove the first item, which is "OTU_ID"
    ids.pop(0)

    return ids

def uc_ids(fn):
    with open(fn) as f:
        ids = [line.split()[-1] for line in f]

    return ids

def list_ids(fn):
    with open(fn) as f:
        ids = [line.strip() for line in f]

    return ids

def taxa_dictionary(fn, ids):
    '''get the second field in lines whose first fields match ids'''
    # populate a hash otu_id => taxonomy
    d = {}
    with open(fn) as f:
        for line in f:
            fields = line.split()
            otu = fields[0]
            tax = " ".join(fields[1:])
            if otu in ids:
                d[otu] = tax

    return d

if __name__ == '__main__':
    # parse command line arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('db', help='input database')
    input_type = parser.add_mutually_exclusive_group(required=True)
    input_type.add_argument('-t', '--table', default=None, help='input otu table')
    input_type.add_argument('-u', '--uc', default=None, help='input uc file')
    input_type.add_argument('-l', '--list', default=None, help='input plain text list')

    parser.add_argument('-i', '--no_match_id', default=None, help='OTU ID for no match (default "no_match" for table/list; "*" for uc)')
    parser.add_argument('-x', '--no_match_tax', default='k__; p__; c__; o__; f__; g__; s__', help='taxonomy for unmatched OTU ID (default is QIIME taxonomy format)')
    args = parser.parse_args()

    # depending on the input type, adjust the no match id and parsing function
    if args.table is not None:
        ids = table_ids(args.table)
        if args.no_match_id is None: args.no_match_id = 'no_match'
    elif args.uc is not None:
        ids = uc_ids(args.uc)
        if args.no_match_id is None: args.no_match_id = '*'
    elif args.list is not None:
        ids = list_ids(args.list)
        if args.no_match_id is None: args.no_match_id = 'no_match'

    # check if the database file ends in .pkl or .pickle
    # if it is, used a pickled dictionary
    # otherwise, just search line by line
    if re.search("\.(pkl|pickle)$", args.db):
        with open(args.db, 'rb') as f:
            d = pickle.load(f)
    else:
        d = taxa_dictionary(args.db, ids)

    d[args.no_match_id] = args.no_match_tax

    print "\n".join([d[i] for i in ids])<commit_msg>get tax: an option for a single word<commit_after>#!/usr/bin/env python

'''
Get greengenes taxonomies. Given an otu table with otu ids in the first column, search through
the greengenes taxonomy list. Output the taxonomies in order.

If the input database is a pickle, just load that dictionary.
'''

import sys, argparse, re, cPickle as pickle

def table_ids(fn):
    '''get the otu ids from the otu table with filename fn'''
    with open(fn) as f:
        ids = [line.split()[0] for line in f]

    # remove the first item, which is "OTU_ID"
    ids.pop(0)

    return ids

def uc_ids(fn):
    with open(fn) as f:
        ids = [line.split()[-1] for line in f]

    return ids

def list_ids(fn):
    with open(fn) as f:
        ids = [line.strip() for line in f]

    return ids

def taxa_dictionary(fn, ids):
    '''get the second field in lines whose first fields match ids'''
    # populate a hash otu_id => taxonomy
    d = {}
    with open(fn) as f:
        for line in f:
            fields = line.split()
            otu = fields[0]
            tax = " ".join(fields[1:])
            if otu in ids:
                d[otu] = tax

    return d

if __name__ == '__main__':
    # parse command line arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('db', help='input database')
    input_type = parser.add_mutually_exclusive_group(required=True)
    input_type.add_argument('-t', '--table', default=None, help='input otu table')
    input_type.add_argument('-u', '--uc', default=None, help='input uc file')
    input_type.add_argument('-l', '--list', default=None, help='input plain text list')
    input_type.add_argument('-w', '--word', default=None, help='input a single ID')

    parser.add_argument('-i', '--no_match_id', default=None, help='OTU ID for no match (default "no_match" for table/list; "*" for uc)')
    parser.add_argument('-x', '--no_match_tax', default='k__; p__; c__; o__; f__; g__; s__', help='taxonomy for unmatched OTU ID (default is QIIME taxonomy format)')
    args = parser.parse_args()

    # depending on the input type, adjust the no match id and parsing function
    if args.table is not None:
        ids = table_ids(args.table)
        if args.no_match_id is None: args.no_match_id = 'no_match'
    elif args.uc is not None:
        ids = uc_ids(args.uc)
        if args.no_match_id is None: args.no_match_id = '*'
    elif args.list is not None:
        ids = list_ids(args.list)
        if args.no_match_id is None: args.no_match_id = 'no_match'
    elif args.word is not None:
        ids = [args.word]

    # check if the database file ends in .pkl or .pickle
    # if it is, used a pickled dictionary
    # otherwise, just search line by line
    if re.search("\.(pkl|pickle)$", args.db):
        with open(args.db, 'rb') as f:
            d = pickle.load(f)
    else:
        d = taxa_dictionary(args.db, ids)

    d[args.no_match_id] = args.no_match_tax

    print "\n".join([d[i] for i in ids])<|endoftext|>


<commit_before>"""Coach and Student classes"""

import requests
import requests_oauthlib
import pandas as pd
import urllib
import yaml

class Coach(object):

    OAUTH_DIR = 'oauth_data'
    with open('keys.yml', 'r') as f:
        APP_KEYS = yaml.load(f)

    CONSUMER_KEY = APP_KEYS['consumer_key']
    CONSUMER_SECRET = APP_KEYS['consumer_secret']
    khan_students_url = 'http://www.khanacademy.org/api/v1/user/students'
    khan_user_url = 'http://www.khanacademy.org/api/v1/user'

    def __init__(self, coach, access_token=None, access_token_secret=None):
        self.coach = coach
        # get stored access tokens
        oauth_data = __import__(
            'pkaaw.%s.%s.tokens' % (Coach.OAUTH_DIR, coach),
            fromlist=['tokens']
        )

        if access_token is None:
            self.final_token = oauth_data.FINAL_OAUTH_TOKEN
        else:
            self.final_token = access_token

        if access_token_secret is None:
            self.final_token_secret = oauth_data.FINAL_OAUTH_TOKEN_SECRET
        else:
            self.final_token_secret = access_token_secret

        self.oauth = requests_oauthlib.OAuth1(
            client_key=Coach.CONSUMER_KEY,
            client_secret=Coach.CONSUMER_SECRET,
            resource_owner_key=self.final_token,
            resource_owner_secret=self.final_token_secret,
            signature_type='auth_header',
            signature_method='PLAINTEXT'
        )

    def get_oauth(self):
        return self.oauth

    def make_request(self, target_url):
        """Returns respons from target_url"""
        r = requests.get(url=target_url, auth=self.oauth)
        return r

    def get_students(self):
        """Returns list of the coach's students' user ids"""
        r = self.make_request(Coach.khan_students_url)
        students = [student['user_id'] for student in r.json()]

        return students

class Student(object):

    DATA_DIR = 'khan_data'
    khan_user_url = 'http://www.khanacademy.org/api/v1/user'
    khan_exercises_url = 'http://www.khanacademy.org/api/v1/user/exercises'
    khan_exercise_states_url = ('http://www.khanacademy.org/api/v1/user/' +
                                'exercises/progress_changes')
    khan_badges_url = 'http://www.khanacademy.org/api/v1/badges'
    khan_exercise_metadata_url = 'http://www.khanacademy.org/api/v1/exercises'

    def __init__(self, coach, user_id):
        self.coach = Coach(coach)
        self.id = user_id
        self.meta = pd.read_csv(
            'pkaaw/%s/topic_metadata.csv' % Student.DATA_DIR
        )
        self.exers = self.meta['id'][self.meta['1'] == 'math'].tolist()
        self.user_link = urllib.urlencode({'userId': self.id})

    def make_request(self, target_url):
        """Returns respons from target_url"""
        r = requests.get(url=target_url, auth=self.coach.get_oauth())

        return r

    def get_details(self):
        stu_url = '%s?%s' % (Student.khan_user_url, self.user_link)
        r = self.make_request(stu_url)
        inner_dict = {}
        # if response is valid, build a dict of key fields
        if r.status_code == requests.codes.ok:
            resp_dict = r.json()
            # convert keys to string from unicode
            resp_dict = dict([(str(k), v) for k, v in resp_dict.items()])
            # build an inner dict with desired fields
            inner_dict = {
                # collapse json to string
                'student': self.id,
                'all_proficient_exercises': ', '.join(map(
                    str, resp_dict['all_proficient_exercises'])
                ),
                'badge_lev0': resp_dict['badge_counts'][u'0'],
                'badge_lev1': resp_dict['badge_counts'][u'1'],
                'badge_lev2': resp_dict['badge_counts'][u'2'],
                'badge_lev3': resp_dict['badge_counts'][u'3'],
                'badge_lev4': resp_dict['badge_counts'][u'4'],
                'badge_lev5': resp_dict['badge_counts'][u'5'],
                'coaches': ', '.join(map(str, resp_dict['coaches'])),
                'first_visit': str(resp_dict['first_visit']),
                'joined': str(resp_dict['joined']),
                'registration_date': str(resp_dict['registration_date']),
                'nickname': resp_dict['nickname'].encode('ascii', 'ignore'),
                'points': resp_dict['points'],
                'proficient_exercises': ', '.join(map(
                    str, resp_dict['proficient_exercises'])
                ),
                'profile_root': str(resp_dict['profile_root']),
                'total_seconds_watched': resp_dict['total_seconds_watched'],
                'username': resp_dict['username'].encode('ascii', 'ignore'),
            }
            identity_email = [
                str(x)
                for x in resp_dict['auth_emails']
                if x.endswith("@teamstudents.org")
            ]

            inner_dict['identity_email'] = ', '.join(map(str, identity_email))

        return inner_dict

    def get_student_badges(self):
        """Gets earned status and count for every student/badge."""
        stu_badge_url = '%s?%s' % (Student.khan_badges_url, self.user_link)
        r = self.make_request(stu_badge_url)
        if r.status_code == requests.codes.ok:
            # returns a list of badges
            badge_list = r.json()
            stu_list = []
            stu_list_detail = []
            for badge in badge_list:
                # make a dict of each badge
                dict([(str(k), v) for k, v in badge.items()])
                int_dict = {
                    'student': self.id,
                    'slug': str(badge['slug']),
                    'owned': badge['is_owned'],
                    'count': 0
                }
                # API only returns user badges if badge is owned
                if badge['is_owned']:
                    try:
                        int_dict['count'] = len(badge['user_badges'])
                        # iterate over user badges and store date & context
                        # (badges can be earned multiple times)
                        for detail in badge['user_badges']:
                            dict([(str(k), v) for k, v in detail.items()])
                            int_detail_dict = {
                                'student': self.id,
                                'slug': str(badge['slug']),
                                'date_earned': detail['date'],
                                'context': str(detail['target_context_name'])
                            }
                            stu_list_detail.append(int_detail_dict)
                    except Exception, e:
                        print e
                        int_dict['count'] = 1
                    # append to student dict
                    stu_list.append(int_dict)

        return stu_list

    def get_composite_exercises(self):
        """Gets exercise status for every student/exercise."""
        stu_exercise_url = '%s?%s' % (Student.khan_exercises_url,
                                      self.user_link)
        r = self.make_request(stu_exercise_url)
        stu_list = []
        if r.status_code == requests.codes.ok:
            resp = r.json()
            for ex in resp:
                ex_dict = dict([(str(k), v) for k, v in ex.items()])
                int_dict = {
                    'student': self.id,
                    'exercise': str(ex_dict['exercise']),
                    'maximum_exercise_progress_dt': ex_dict[
                        'maximum_exercise_progress_dt'
                    ],
                    'streak': ex_dict['streak'],
                    'progress': ex_dict['exercise_progress']['level'],
                    'practiced_date': ex_dict['practiced_date'],
                    'proficient_date': ex_dict['proficient_date'],
                    'total_done': ex_dict['total_done'],
                    'struggling': ex_dict['exercise_states']['struggling'],
                    'proficient': ex_dict['exercise_states']['proficient'],
                    'practiced': ex_dict['exercise_states']['practiced'],
                    'mastered': ex_dict['exercise_states']['mastered'],
                    'level': str(
                        ex_dict['maximum_exercise_progress']['level']
                    ),
                    'total_correct': ex_dict['total_correct'],
                    'last_done': ex_dict['last_done'],
                    'longest_streak': ex_dict['longest_streak']
                }
                stu_list.append(int_dict)

            stu_list = [
                stu_dict
                for stu_dict in stu_list
                if stu_dict['exercise'] in self.exers
            ]

        return stu_list

    def get_exercise_states(self):
        """Gets exercise state changes for every student/exercise"""
        stu_states_url = '%s?%s' % (Student.khan_exercise_states_url,
                                    self.user_link)
        r = self.make_request(stu_states_url)
        stu_list = []
        if r.status_code == requests.codes.ok:
            resp = r.json()
            for ex_st in resp:
                change_dict = dict([(str(k), v) for k, v in ex_st.items()])
                int_dict = {
                    'student': self.id,
                    'exercise': str(change_dict['exercise_name']),
                    'date': change_dict['date'],
                    'change_type': None,
                    'exercise_status': change_dict['to_progress']['level'],
                    'mastery_flag': change_dict['to_progress']['mastered']
                }
                stu_list.append(int_dict)

            stu_list = [
                item_dict
                for item_dict in stu_list
                if item_dict['exercise'] in self.exers
            ]

        return stu_list
<commit_msg>python 3.4 compatability<commit_after>"""Coach and Student classes"""

import requests
import requests_oauthlib
import pandas as pd
import urllib
import yaml
import six.moves

class Coach(object):

    OAUTH_DIR = 'oauth_data'
    with open('keys.yml', 'r') as f:
        APP_KEYS = yaml.load(f)

    CONSUMER_KEY = APP_KEYS['consumer_key']
    CONSUMER_SECRET = APP_KEYS['consumer_secret']
    khan_students_url = 'http://www.khanacademy.org/api/v1/user/students'
    khan_user_url = 'http://www.khanacademy.org/api/v1/user'

    def __init__(self, coach, access_token=None, access_token_secret=None):
        self.coach = coach
        # get stored access tokens
        oauth_data = __import__(
            'pkaaw.%s.%s.tokens' % (Coach.OAUTH_DIR, coach),
            fromlist=['tokens']
        )

        if access_token is None:
            self.final_token = oauth_data.FINAL_OAUTH_TOKEN
        else:
            self.final_token = access_token

        if access_token_secret is None:
            self.final_token_secret = oauth_data.FINAL_OAUTH_TOKEN_SECRET
        else:
            self.final_token_secret = access_token_secret

        self.oauth = requests_oauthlib.OAuth1(
            client_key=Coach.CONSUMER_KEY,
            client_secret=Coach.CONSUMER_SECRET,
            resource_owner_key=self.final_token,
            resource_owner_secret=self.final_token_secret,
            signature_type='auth_header',
            signature_method='PLAINTEXT'
        )

    def get_oauth(self):
        return self.oauth

    def make_request(self, target_url):
        """Returns respons from target_url"""
        r = requests.get(url=target_url, auth=self.oauth)
        return r

    def get_students(self):
        """Returns list of the coach's students' user ids"""
        r = self.make_request(Coach.khan_students_url)
        students = [student['user_id'] for student in r.json()]

        return students

class Student(object):

    DATA_DIR = 'khan_data'
    khan_user_url = 'http://www.khanacademy.org/api/v1/user'
    khan_exercises_url = 'http://www.khanacademy.org/api/v1/user/exercises'
    khan_exercise_states_url = ('http://www.khanacademy.org/api/v1/user/' +
                                'exercises/progress_changes')
    khan_badges_url = 'http://www.khanacademy.org/api/v1/badges'
    khan_exercise_metadata_url = 'http://www.khanacademy.org/api/v1/exercises'

    def __init__(self, coach, user_id):
        self.coach = Coach(coach)
        self.id = user_id
        self.meta = pd.read_csv(
            'pkaaw/%s/topic_metadata.csv' % Student.DATA_DIR
        )
        self.exers = self.meta['id'][self.meta['1'] == 'math'].tolist()
        self.user_link = urllib.urlencode({'userId': self.id})

    def make_request(self, target_url):
        """Returns respons from target_url"""
        r = requests.get(url=target_url, auth=self.coach.get_oauth())

        return r

    def get_details(self):
        stu_url = '%s?%s' % (Student.khan_user_url, self.user_link)
        r = self.make_request(stu_url)
        inner_dict = {}
        # if response is valid, build a dict of key fields
        if r.status_code == requests.codes.ok:
            resp_dict = r.json()
            # convert keys to string from unicode
            resp_dict = dict([(str(k), v) for k, v in resp_dict.items()])
            # build an inner dict with desired fields
            inner_dict = {
                # collapse json to string
                'student': self.id,
                'all_proficient_exercises': ', '.join(map(
                    str, resp_dict['all_proficient_exercises'])
                ),
                'badge_lev0': resp_dict['badge_counts'][u'0'],
                'badge_lev1': resp_dict['badge_counts'][u'1'],
                'badge_lev2': resp_dict['badge_counts'][u'2'],
                'badge_lev3': resp_dict['badge_counts'][u'3'],
                'badge_lev4': resp_dict['badge_counts'][u'4'],
                'badge_lev5': resp_dict['badge_counts'][u'5'],
                'coaches': ', '.join(map(str, resp_dict['coaches'])),
                'first_visit': str(resp_dict['first_visit']),
                'joined': str(resp_dict['joined']),
                'registration_date': str(resp_dict['registration_date']),
                'nickname': resp_dict['nickname'].encode('ascii', 'ignore'),
                'points': resp_dict['points'],
                'proficient_exercises': ', '.join(map(
                    str, resp_dict['proficient_exercises'])
                ),
                'profile_root': str(resp_dict['profile_root']),
                'total_seconds_watched': resp_dict['total_seconds_watched'],
                'username': resp_dict['username'].encode('ascii', 'ignore'),
            }
            identity_email = [
                str(x)
                for x in resp_dict['auth_emails']
                if x.endswith("@teamstudents.org")
            ]

            inner_dict['identity_email'] = ', '.join(map(str, identity_email))

        return inner_dict

    def get_student_badges(self):
        """Gets earned status and count for every student/badge."""
        stu_badge_url = '%s?%s' % (Student.khan_badges_url, self.user_link)
        r = self.make_request(stu_badge_url)
        if r.status_code == requests.codes.ok:
            # returns a list of badges
            badge_list = r.json()
            stu_list = []
            stu_list_detail = []
            for badge in badge_list:
                # make a dict of each badge
                dict([(str(k), v) for k, v in badge.items()])
                int_dict = {
                    'student': self.id,
                    'slug': str(badge['slug']),
                    'owned': badge['is_owned'],
                    'count': 0
                }
                # API only returns user badges if badge is owned
                if badge['is_owned']:
                    try:
                        int_dict['count'] = len(badge['user_badges'])
                        # iterate over user badges and store date & context
                        # (badges can be earned multiple times)
                        for detail in badge['user_badges']:
                            dict([(str(k), v) for k, v in detail.items()])
                            int_detail_dict = {
                                'student': self.id,
                                'slug': str(badge['slug']),
                                'date_earned': detail['date'],
                                'context': str(detail['target_context_name'])
                            }
                            stu_list_detail.append(int_detail_dict)
                    except Exception, e:
                        six.print_(e)
                        int_dict['count'] = 1
                    # append to student dict
                    stu_list.append(int_dict)

        return stu_list

    def get_composite_exercises(self):
        """Gets exercise status for every student/exercise."""
        stu_exercise_url = '%s?%s' % (Student.khan_exercises_url,
                                      self.user_link)
        r = self.make_request(stu_exercise_url)
        stu_list = []
        if r.status_code == requests.codes.ok:
            resp = r.json()
            for ex in resp:
                ex_dict = dict([(str(k), v) for k, v in ex.items()])
                int_dict = {
                    'student': self.id,
                    'exercise': str(ex_dict['exercise']),
                    'maximum_exercise_progress_dt': ex_dict[
                        'maximum_exercise_progress_dt'
                    ],
                    'streak': ex_dict['streak'],
                    'progress': ex_dict['exercise_progress']['level'],
                    'practiced_date': ex_dict['practiced_date'],
                    'proficient_date': ex_dict['proficient_date'],
                    'total_done': ex_dict['total_done'],
                    'struggling': ex_dict['exercise_states']['struggling'],
                    'proficient': ex_dict['exercise_states']['proficient'],
                    'practiced': ex_dict['exercise_states']['practiced'],
                    'mastered': ex_dict['exercise_states']['mastered'],
                    'level': str(
                        ex_dict['maximum_exercise_progress']['level']
                    ),
                    'total_correct': ex_dict['total_correct'],
                    'last_done': ex_dict['last_done'],
                    'longest_streak': ex_dict['longest_streak']
                }
                stu_list.append(int_dict)

            stu_list = [
                stu_dict
                for stu_dict in stu_list
                if stu_dict['exercise'] in self.exers
            ]

        return stu_list

    def get_exercise_states(self):
        """Gets exercise state changes for every student/exercise"""
        stu_states_url = '%s?%s' % (Student.khan_exercise_states_url,
                                    self.user_link)
        r = self.make_request(stu_states_url)
        stu_list = []
        if r.status_code == requests.codes.ok:
            resp = r.json()
            for ex_st in resp:
                change_dict = dict([(str(k), v) for k, v in ex_st.items()])
                int_dict = {
                    'student': self.id,
                    'exercise': str(change_dict['exercise_name']),
                    'date': change_dict['date'],
                    'change_type': None,
                    'exercise_status': change_dict['to_progress']['level'],
                    'mastery_flag': change_dict['to_progress']['mastered']
                }
                stu_list.append(int_dict)

            stu_list = [
                item_dict
                for item_dict in stu_list
                if item_dict['exercise'] in self.exers
            ]

        return stu_list
<|endoftext|>


<commit_before>#!/usr/bin/env python

from os import getcwd, path

import logging.handlers

LOG_FILE = "logs/bot.log"

def get_logger(prefix=None):
  """Build and return a logger for formatted stream and file output.

  :param prefix: Optional path from CWD to dir that contains logs dir.
  :type prefix: string.
  :returns: The constructed logging object.

  """

  # Setup path to log output. Allows usage from any subpackage.
  if prefix is None:
    prefix = "../" * getcwd().split("/bot")[1].count("/")
  qual_log_file = prefix + LOG_FILE

  # Build logger
  logger = logging.getLogger(__name__)
  logger.setLevel(logging.DEBUG)

  # Check if log exists and should therefore be rolled
  needRoll = False
  if path.isfile(qual_log_file):
    needRoll = True

  # Build file output formatter
  file_formatter = logging.Formatter("%(asctime)s | %(levelname)s | "
                                     "%(filename)s | %(funcName)s | "
                                     "%(lineno)d | %(message)s")

  # Build stream output formatter
  stream_formatter = logging.Formatter("%(filename)s | %(funcName)s | "
                                       "%(lineno)d | %(levelname)s | "
                                       "%(message)s")

  # Build file handler (for output log output to files)
  file_handler = logging.handlers.RotatingFileHandler(qual_log_file, mode="a",
                                                      backupCount=50,
                                                      delay=True)
  file_handler.setLevel(logging.DEBUG)
  file_handler.setFormatter(file_formatter)

  # Build stream handler (for output to stdout)
  stream_handler = logging.StreamHandler()
  stream_handler.setLevel(logging.WARN)
  stream_handler.setFormatter(stream_formatter)

  # Add handlers to logger
  logger.addHandler(file_handler)
  logger.addHandler(stream_handler)

  # This is a stale log, so roll it
  if needRoll:
    logger.handlers[0].doRollover()

  logger.debug("Logger built")
  return logger
<commit_msg>2-space indent to 4-space (PEP8)<commit_after>#!/usr/bin/env python

from os import getcwd, path

import logging.handlers

LOG_FILE = "logs/bot.log"

def get_logger(prefix=None):
    """Build and return a logger for formatted stream and file output.

    :param prefix: Optional path from CWD to dir that contains logs dir.
    :type prefix: string.
    :returns: The constructed logging object.

    """

    # Setup path to log output. Allows usage from any subpackage.
    if prefix is None:
    prefix = "../" * getcwd().split("/bot")[1].count("/")
    qual_log_file = prefix + LOG_FILE

    # Build logger
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.DEBUG)

    # Check if log exists and should therefore be rolled
    needRoll = False
    if path.isfile(qual_log_file):
    needRoll = True

    # Build file output formatter
    file_formatter = logging.Formatter("%(asctime)s | %(levelname)s | "
                                     "%(filename)s | %(funcName)s | "
                                     "%(lineno)d | %(message)s")

    # Build stream output formatter
    stream_formatter = logging.Formatter("%(filename)s | %(funcName)s | "
                                       "%(lineno)d | %(levelname)s | "
                                       "%(message)s")

    # Build file handler (for output log output to files)
    file_handler = logging.handlers.RotatingFileHandler(qual_log_file, mode="a",
                                                      backupCount=50,
                                                      delay=True)
    file_handler.setLevel(logging.DEBUG)
    file_handler.setFormatter(file_formatter)

    # Build stream handler (for output to stdout)
    stream_handler = logging.StreamHandler()
    stream_handler.setLevel(logging.WARN)
    stream_handler.setFormatter(stream_formatter)

    # Add handlers to logger
    logger.addHandler(file_handler)
    logger.addHandler(stream_handler)

    # This is a stale log, so roll it
    if needRoll:
    logger.handlers[0].doRollover()

    logger.debug("Logger built")
    return logger
<|endoftext|>


<commit_before>#!/usr/bin/env python

# stdlib
import tempfile
import time
import os
import shutil
import sys
import uuid

# pypi
import sh
from retask.queue import Queue
from sqlalchemy import engine_from_config
from pyramid.paster import (
    get_appsettings,
    setup_logging,
)

import pep8

# local
import pep8bot.githubutils as gh
import pep8bot.models as m

class directory(object):
    """ pushd/popd context manager. """
    def __init__(self, newPath):
        self.newPath = newPath

    def __enter__(self, *args, **kw):
        self.savedPath = os.getcwd()
        os.chdir(self.newPath)

    def __exit__(self, *args, **kw):
        os.chdir(self.savedPath)

class WebReport(pep8.StandardReport):
    """ Collect errors as the pep8check runs so we can save them in the DB """

    def __init__(self, *args, **kw):
        super(WebReport, self).__init__(*args, **kw)
        self._deferred_print = []
        self._all_errors = []

    def init_file(self, *args, **kw):
        """Signal a new file."""

        for items in self._deferred_print:
            self._all_errors.append(list(items) + [self.filename])

        return super(WebReport, self).init_file(*args, **kw)

class Worker(object):
    """ Represents the worker process.  Waits for tasks to come in from the
    webapp and then acts on them.
    """

    def __init__(self, config_uri):
        setup_logging(config_uri)
        settings = get_appsettings(config_uri, name="pep8bot")
        engine = engine_from_config(settings, 'sqlalchemy.')
        m.DBSession.configure(bind=engine)

        # TODO -- pass in redis params from config, hostname, etc.
        self.queue = Queue('commits')
        self.queue.connect()
        # TODO -- set both of these with the config file.
        # Use pyramid tools to load config.
        self.sleep_interval = 3
        self.scratch_dir = "/home/threebean/scratch/pep8bot-scratch"
        try:
            os.makedirs(self.scratch_dir)
        except OSError:
            pass  # Assume that the scratch_dir already exists.

    def run(self):
        while True:
            task = self.queue.wait()
            time.sleep(self.sleep_interval)
            data = task.data

            repo = data['reponame']
            owner = data['username']
            commits = data['commits']
            _user = m.User.query.filter_by(username=owner).one()

            token = _user.oauth_access_token
            if not token:
                for u in _user.users:
                    if u.oauth_access_token:
                        token = u.oauth_access_token

            clone_url = data['clone_url']

            self.working_dir = tempfile.mkdtemp(
                prefix=owner + '-' + repo,
                dir=self.scratch_dir,
            )

            print "** Cloning to", self.working_dir
            print sh.git.clone(clone_url, self.working_dir)

            lookup = {
                "success": "PEP8bot says \"OK\"",
                "failure": "PEP8bot detected {n} errors",
                "pending": "Still waiting on PEP8bot check",
                "error": "PEP8bot ran into trouble",
            }
            print "** Processing commits."
            for sha in commits:
                # TODO -- what about hash collisions?
                _commit = m.Commit.query.filter_by(sha=sha).first()
                import transaction
                try:
                    print "** Processing files on commit", sha
                    with directory(self.working_dir):
                        print sh.git.checkout(sha)

                    # TODO -- only process those in modified and added
                    infiles = []
                    for root, dirs, files in os.walk(self.working_dir):

                        if '.git' in root:
                            continue

                        infiles.extend([
                            root + "/" + fname
                            for fname in files
                            if fname.endswith(".py")
                        ])

                    # TODO -- document that the user can keep a .config/pep8
                    # file in their project dir.
                    pep8style = pep8.StyleGuide(
                        reporter=WebReport,
                        config_file="./.config/pep8",
                    )
                    result = pep8style.check_files(infiles)

                    if result.total_errors > 0:
                        status = "failure"
                    else:
                        status = "success"

                    desc = lookup[status].format(n=result.total_errors)
                    _commit.status = status
                    _commit.pep8_error_count = result.total_errors

                    errors = []
                    url_tmpl = "https://github.com/{owner}/{repo}/blob" + \
                            "/{sha}/{path}#L{lno}"
                    fmt = "{path}:{row}:{col}: " + \
                            "{code} {text}"
                    for lno, oset, code, text, d, fname in result._all_errors:

                        # Turn absolute path into relative path
                        path = fname[len(self.working_dir)+1:]

                        errors.append(fmt.format(
                            path=path,
                            row=result.line_offset + lno,
                            col=oset + 1,
                            code=code,
                            text=text,
                            url=url_tmpl.format(**locals()),
                        ))

                    _commit.pep8_errors = '\n'.join(errors)
                    gh.post_status(owner, repo, sha, status, token, desc)
                except Exception:
                    status = "error"
                    desc = lookup[status]
                    _commit.status = status
                    gh.post_status(owner, repo, sha, status, token, desc)
                    raise
                finally:
                    transaction.commit()

def usage(argv):
    cmd = os.path.basename(argv[0])
    print('usage: %s <config_uri>\n'
          '(example: "%s development.ini")' % (cmd, cmd))
    sys.exit(1)

def worker(config_filename):
    w = Worker(config_filename)
    try:
        w.run()
    except KeyboardInterrupt:
        pass

def main(argv=sys.argv):
    if len(argv) != 2:
        usage(argv)
    worker(sys.argv[1])

if __name__ == '__main__':
    main()
<commit_msg>More sleeping, slightly better (:/) logging<commit_after>#!/usr/bin/env python

# stdlib
import tempfile
import time
import os
import shutil
import sys
import uuid

# pypi
import sh
from retask.queue import Queue
from sqlalchemy import engine_from_config
from pyramid.paster import (
    get_appsettings,
    setup_logging,
)

import pep8

# local
import pep8bot.githubutils as gh
import pep8bot.models as m

class directory(object):
    """ pushd/popd context manager. """
    def __init__(self, newPath):
        self.newPath = newPath

    def __enter__(self, *args, **kw):
        self.savedPath = os.getcwd()
        os.chdir(self.newPath)

    def __exit__(self, *args, **kw):
        os.chdir(self.savedPath)

class WebReport(pep8.StandardReport):
    """ Collect errors as the pep8check runs so we can save them in the DB """

    def __init__(self, *args, **kw):
        super(WebReport, self).__init__(*args, **kw)
        self._deferred_print = []
        self._all_errors = []

    def init_file(self, *args, **kw):
        """Signal a new file."""

        for items in self._deferred_print:
            self._all_errors.append(list(items) + [self.filename])

        return super(WebReport, self).init_file(*args, **kw)

class Worker(object):
    """ Represents the worker process.  Waits for tasks to come in from the
    webapp and then acts on them.
    """

    def __init__(self, config_uri):
        setup_logging(config_uri)
        settings = get_appsettings(config_uri, name="pep8bot")
        engine = engine_from_config(settings, 'sqlalchemy.')
        m.DBSession.configure(bind=engine)

        # TODO -- pass in redis params from config, hostname, etc.
        self.queue = Queue('commits')
        self.queue.connect()
        # TODO -- set both of these with the config file.
        # Use pyramid tools to load config.
        self.sleep_interval = 10
        self.scratch_dir = "/home/threebean/scratch/pep8bot-scratch"
        try:
            os.makedirs(self.scratch_dir)
        except OSError:
            pass  # Assume that the scratch_dir already exists.

    def run(self):
        while True:
            print "**", time.time(), "waiting on a task."
            task = self.queue.wait()
            print "**", time.time(), "popped a task off the queue.  Sleeping."
            time.sleep(self.sleep_interval)
            print "**", time.time(), "woke up."
            data = task.data

            repo = data['reponame']
            owner = data['username']
            commits = data['commits']
            _user = m.User.query.filter_by(username=owner).one()

            token = _user.oauth_access_token
            if not token:
                for u in _user.users:
                    if u.oauth_access_token:
                        token = u.oauth_access_token

            clone_url = data['clone_url']

            self.working_dir = tempfile.mkdtemp(
                prefix=owner + '-' + repo,
                dir=self.scratch_dir,
            )

            print "**", time.time(), "cloning to", self.working_dir
            print sh.git.clone(clone_url, self.working_dir)

            lookup = {
                "success": "PEP8bot says \"OK\"",
                "failure": "PEP8bot detected {n} errors",
                "pending": "Still waiting on PEP8bot check",
                "error": "PEP8bot ran into trouble",
            }
            print "** Processing commits."
            for sha in commits:
                # TODO -- what about hash collisions?
                _commit = m.Commit.query.filter_by(sha=sha).first()
                import transaction
                try:
                    print "** Processing files on commit", sha
                    with directory(self.working_dir):
                        print sh.git.checkout(sha)

                    # TODO -- only process those in modified and added
                    infiles = []
                    for root, dirs, files in os.walk(self.working_dir):

                        if '.git' in root:
                            continue

                        infiles.extend([
                            root + "/" + fname
                            for fname in files
                            if fname.endswith(".py")
                        ])

                    # TODO -- document that the user can keep a .config/pep8
                    # file in their project dir.
                    pep8style = pep8.StyleGuide(
                        reporter=WebReport,
                        config_file="./.config/pep8",
                    )
                    result = pep8style.check_files(infiles)

                    if result.total_errors > 0:
                        status = "failure"
                    else:
                        status = "success"

                    desc = lookup[status].format(n=result.total_errors)
                    _commit.status = status
                    _commit.pep8_error_count = result.total_errors

                    errors = []
                    url_tmpl = "https://github.com/{owner}/{repo}/blob" + \
                            "/{sha}/{path}#L{lno}"
                    fmt = "{path}:{row}:{col}: " + \
                            "{code} {text}"
                    for lno, oset, code, text, d, fname in result._all_errors:

                        # Turn absolute path into relative path
                        path = fname[len(self.working_dir)+1:]

                        errors.append(fmt.format(
                            path=path,
                            row=result.line_offset + lno,
                            col=oset + 1,
                            code=code,
                            text=text,
                            url=url_tmpl.format(**locals()),
                        ))

                    _commit.pep8_errors = '\n'.join(errors)
                    gh.post_status(owner, repo, sha, status, token, desc)
                except Exception:
                    status = "error"
                    desc = lookup[status]
                    _commit.status = status
                    gh.post_status(owner, repo, sha, status, token, desc)
                    raise
                finally:
                    transaction.commit()

def usage(argv):
    cmd = os.path.basename(argv[0])
    print('usage: %s <config_uri>\n'
          '(example: "%s development.ini")' % (cmd, cmd))
    sys.exit(1)

def worker(config_filename):
    w = Worker(config_filename)
    try:
        w.run()
    except KeyboardInterrupt:
        pass

def main(argv=sys.argv):
    if len(argv) != 2:
        usage(argv)
    worker(sys.argv[1])

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from django.test import TestCase
from django.urls import reverse

from places.models import Restaurant

# Create your tests here.

class HomeViewTestCase(TestCase):
    """HomeView test suite"""

    expected_url = '/places/'
    reverse_name = 'places:home'

    def test_desired_location(self):
        resp = self.client.get(self.expected_url)
        self.assertEqual(resp.status_code, 200)

    def test_desired_name(self):
        reverse_url = reverse(self.reverse_name)
        self.assertEquals(reverse_url, self.expected_url)

class RestaurantListViewTestCase(TestCase):
    """RestaurantListView test suite"""

    expected_url = '/places/restaurants/'
    reverse_name = 'places:restaurant-list'
    restaurant_count = 25

    @classmethod
    def setUpTestData(cls):
        for i in range(cls.restaurant_count):
            Restaurant.objects.create(
                name='Chipotle {rid}'.format(rid=i), address='Bogus {rid}'.format(rid=i),
                latitude=0.00, longitude=0.00,
                min_party=1, max_party=6,
                yelp_link='/chipotle-bogus-{rid}'.format(rid=i))

    def test_desired_location(self):
        resp = self.client.get(self.expected_url)
        self.assertEqual(resp.status_code, 200)

    def test_desired_name(self):
        reverse_url = reverse(self.reverse_name)
        self.assertEquals(reverse_url, self.expected_url)

    def test_desired_data(self):
        resp = self.client.get(self.expected_url)
        self.assertEqual(resp.status_code, 200)
        self.assertEqual(len(resp.context['restaurant_list']), self.restaurant_count)

class RestaurantDetailViewTestCase(TestCase):
    """RestaurantDetailView test suite"""

    restaurant_id = 1
    expected_url = '/places/restaurants/{rid}'.format(rid=restaurant_id)
    reverse_name = 'places:restaurant-detail'

    @classmethod
    def setUpTestData(cls):
        Restaurant.objects.create(
            name='Chick Fil A', address='Venus',
            latitude=0.00, longitude=0.00,
            min_party=3, max_party=8,
            yelp_link='/chick-fil-a-venus')

    def test_desired_location(self):
        resp = self.client.get(self.expected_url)
        self.assertEqual(resp.status_code, 200)

    def test_desired_name(self):
        reverse_url = reverse(self.reverse_name, args=(self.restaurant_id,))
        self.assertEquals(reverse_url, self.expected_url)

    def test_desired_data(self):
        resp = self.client.get(self.expected_url)
        self.assertEqual(resp.status_code, 200)
        restaurant = resp.context['restaurant']
        self.assertTrue(type(restaurant) == Restaurant)
        self.assertEqual(restaurant.name, 'Chick Fil A')
        self.assertEqual(restaurant.min_party, 3)
        self.assertEqual(restaurant.max_party, 8)
<commit_msg>Test corner cases for DetailView<commit_after>from django.test import TestCase
from django.urls import reverse

from places.models import Restaurant

# Create your tests here.

class HomeViewTestCase(TestCase):
    """HomeView test suite"""

    expected_url = '/places/'
    reverse_name = 'places:home'

    def test_desired_location(self):
        resp = self.client.get(self.expected_url)
        self.assertEqual(resp.status_code, 200)

    def test_desired_name(self):
        reverse_url = reverse(self.reverse_name)
        self.assertEquals(reverse_url, self.expected_url)

class RestaurantListViewTestCase(TestCase):
    """RestaurantListView test suite"""

    expected_url = '/places/restaurants/'
    reverse_name = 'places:restaurant-list'
    restaurant_count = 25

    @classmethod
    def setUpTestData(cls):
        for i in range(cls.restaurant_count):
            Restaurant.objects.create(
                name='Chipotle {rid}'.format(rid=i), address='Bogus {rid}'.format(rid=i),
                latitude=0.00, longitude=0.00,
                min_party=1, max_party=6,
                yelp_link='/chipotle-bogus-{rid}'.format(rid=i))

    def test_desired_location(self):
        resp = self.client.get(self.expected_url)
        self.assertEqual(resp.status_code, 200)

    def test_desired_name(self):
        reverse_url = reverse(self.reverse_name)
        self.assertEquals(reverse_url, self.expected_url)

    def test_desired_data(self):
        resp = self.client.get(self.expected_url)
        self.assertEqual(resp.status_code, 200)
        self.assertEqual(len(resp.context['restaurant_list']), self.restaurant_count)

class RestaurantDetailViewTestCase(TestCase):
    """RestaurantDetailView test suite"""

    reverse_name = 'places:restaurant-detail'

    @classmethod
    def setUpTestData(cls):
        Restaurant.objects.create(
            name='Chick Fil A', address='Venus',
            latitude=0.00, longitude=0.00,
            min_party=3, max_party=8,
            yelp_link='/chick-fil-a-venus')

    def setUp(self):
        self.good_id = 1
        self.bad_id = 2
        self.good_url = '/places/restaurants/{rid}'.format(rid=self.good_id)
        self.bad_url = '/places/restaurants/{rid}'.format(rid=self.bad_id)

    def test_desired_location(self):
        resp = self.client.get(self.good_url)
        self.assertEqual(resp.status_code, 200)

    def test_desired_name(self):
        reverse_url = reverse(self.reverse_name, args=(self.good_id,))
        self.assertEquals(reverse_url, self.good_url)

    def test_desired_data(self):
        resp = self.client.get(self.good_url)
        self.assertEqual(resp.status_code, 200)
        restaurant = resp.context['restaurant']
        self.assertTrue(type(restaurant) == Restaurant)
        self.assertEqual(restaurant.name, 'Chick Fil A')
        self.assertEqual(restaurant.min_party, 3)
        self.assertEqual(restaurant.max_party, 8)

    def test_bad_location(self):
        resp = self.client.get(self.bad_url)
        self.assertNotEqual(resp.status_code, 200)
        self.assertEqual(resp.status_code, 404)

    def test_bad_name(self):
        reverse_url = reverse(self.reverse_name, args=(self.bad_id,))
        self.assertEquals(reverse_url, self.bad_url)
<|endoftext|>


<commit_before>class PluginLoader:

    def load(self, directory):
        ret = []

        for plugin in directory.children:
            plugin_file = plugin.find(name=plugin.name, type='py')
            plugin_class = plugin_file.classes[plugin.name]

            self._check_attributes(plugin_class)

            plugin_instance = plugin_class()
            ret.append(plugin_instance)

        return ret

    def _check_attributes(self, klass):
        self._check_aliases(klass)
        self._check_matches(klass)
        self._check_priority(klass)

    def _check_aliases(self, klass):
        self._check_attribute(klass, 'aliases', '')

    def _check_matches(self, klass):
        def matches(self, expression):
            if isinstance(self.aliases, str):
                return expression == self.aliases

            return expression in self.aliases

        self._check_attribute(klass, 'matches', matches)

    def _check_priority(self, klass):
        self._check_attribute(klass, 'priority', 'normal')

        if klass.priority not in ('low', 'normal', 'high'):
            klass.priority = 'normal'

    def _check_attribute(self, klass, attribute, value):
        if not hasattr(klass, attribute):
            setattr(klass, attribute, value)
<commit_msg>Make plugin loader more robust<commit_after>class PluginLoader:

    def load(self, directory):
        ret = []

        for plugin in directory.children:
            plugin_file = plugin.find(name=plugin.name, type='py')

            if not plugin_file:
                continue

            plugin_class = plugin_file.classes[plugin.name]

            self._check_attributes(plugin_class)

            plugin_instance = plugin_class()
            ret.append(plugin_instance)

        return ret

    def _check_attributes(self, klass):
        self._check_aliases(klass)
        self._check_matches(klass)
        self._check_priority(klass)

    def _check_aliases(self, klass):
        self._check_attribute(klass, 'aliases', '')

    def _check_matches(self, klass):
        def matches(self, expression):
            if isinstance(self.aliases, str):
                return expression == self.aliases

            return expression in self.aliases

        self._check_attribute(klass, 'matches', matches)

    def _check_priority(self, klass):
        self._check_attribute(klass, 'priority', 'normal')

        if klass.priority not in ('low', 'normal', 'high'):
            klass.priority = 'normal'

    def _check_attribute(self, klass, attribute, value):
        if not hasattr(klass, attribute):
            setattr(klass, attribute, value)
<|endoftext|>


<commit_before>import astropy.units as u
import healpy as hp
import numpy as np
import numpy.random as random
import numpy.random.mtrand
from astropy.coordinates import SkyCoord, Longitude, Latitude
from mpi4py import MPI

from mpi_helper import r_print

comm = MPI.COMM_WORLD
enable_intersection = True
ar_map_nside = 2048

# use a different random seed for each MPI rank, but still use predictable results for easier debugging.
_ = np.random.mtrand.RandomState(comm.rank)

def ra_dec2ang(ra, dec):
    return (90. - dec) * np.pi / 180., ra / 180. * np.pi

def main_loop(ar_map, max_bin_angle, outer_disc_mean, outer_disc, max_angular_separation):
    ar_product = np.zeros(shape=num_bins)
    ar_weights = np.zeros(shape=num_bins)
    chosen_indices = np.random.choice(outer_disc, size=100, replace=False)
    for index in chosen_indices:
        vec_a = hp.pix2vec(ar_map_nside, index)
        inner_disc = hp.query_disc(ar_map_nside, vec=vec_a, radius=max_angular_separation.to(u.rad).value)
        if enable_intersection:
            inner_disc = np.intersect1d(inner_disc, outer_disc, assume_unique=True)
        vec_b = hp.pix2vec(ar_map_nside, inner_disc)
        ar_ang_dist_with_zero = hp.rotator.angdist(vec_b, vec_a)
        a = index
        b = inner_disc[ar_ang_dist_with_zero > 0]
        ar_ang_dist = ar_ang_dist_with_zero[ar_ang_dist_with_zero > 0]
        ar_bins_float = ar_ang_dist / max_bin_angle * num_bins  # type: np.ndarray
        ar_bins = ar_bins_float.astype(int)
        # TODO: filter NaNs earlier so that they don't decrease the correlation
        pair_product = np.nan_to_num((ar_map[a] - outer_disc_mean) * (ar_map[b] - outer_disc_mean))
        ar_product += np.bincount(ar_bins, weights=pair_product, minlength=num_bins)
        ar_weights += np.bincount(ar_bins, minlength=num_bins)

    return ar_product, ar_weights

# load Planck map on the root node
ar_map_shape = None
ar_map_0 = None
ar_map_0_log = None
if comm.rank == 0:
    ar_map_0 = hp.fitsfunc.read_map("../../data/COM_CompMap_Dust-DL07-AvMaps_2048_R2.00.fits", field=0)
    # ar_map_0_log = np.log(ar_map_0)

    # optionally add a mock signal to the map
    mock = False
    if mock:
        ar_mock = ar_map_0
        nside_signal = 32
        radius = hp.nside2resol(nside_signal) / 2 / np.sqrt(2)

        for i in range(hp.nside2npix(nside_signal)):
            vec1 = hp.pix2vec(nside_signal, i)
            mask = hp.query_disc(ar_map_nside, vec=vec1, radius=radius)
            ar_mock[mask] *= 100

        ar_mock /= np.sqrt(100)

# send the map to all other nodes
ar_map_local = comm.bcast(ar_map_0)

# initialize correlation bins
num_bins = 100
ar_product_total = np.zeros(shape=(10, num_bins))
ar_weights_total = np.zeros(shape=(10, num_bins))

# sky scan parameters
num_directions = 4
stripe_step_deg = 10

for current_direction_index in np.arange(num_directions):
    # start with a location within the BOSS field
    center_ra = 180.
    center_dec = 30.
    center_coord = SkyCoord(ra=center_ra * u.degree, dec=center_dec * u.degree)
    center_galactic = center_coord.galactic
    # scan the sky in galactic coordinates:
    galactic_l = Longitude(center_galactic.l + 0 * u.degree)
    # currently we just change the latitude
    galactic_b = Latitude(
        center_galactic.b - (current_direction_index - num_directions * 0.0) * stripe_step_deg * u.degree)
    r_print("galactic l-value:", galactic_l.value)
    r_print("galactic b-value:", galactic_b.value)
    # convert galactic coordinates to healpy theta/phi and create a unit vector:
    center_theta, center_phi = ra_dec2ang(ra=galactic_l.value, dec=galactic_b.value)
    vec = hp.ang2vec(theta=center_theta, phi=center_phi)
    r_print("unit vector:", vec)
    # get all pixels in the current disc
    current_disc_radius = 10 / 180. * np.pi
    disc = hp.query_disc(ar_map_nside, vec=vec, radius=current_disc_radius)
    r_print("disc has ", disc.shape[0], " pixels")

    max_angle_fixed = 5. / 180. * np.pi
    disc_mean = np.nanmean(ar_map_local[disc])
    ar_dec, ar_ra = hp.pix2ang(ar_map_nside, disc)
    global_max_angular_separation = 5. * u.degree  # type: u.Quantity

    # work in iterations so that we don't use too much memory
    for i in np.arange(1):
        # initialize bins for reduce operation
        ar_product_reduce = np.zeros(shape=num_bins)
        ar_weights_reduce = np.zeros(shape=num_bins)

        ar_product_local, ar_weights_local = main_loop(
            ar_map=ar_map_local, max_bin_angle=max_angle_fixed,
            outer_disc_mean=disc_mean, outer_disc=disc,
            max_angular_separation=global_max_angular_separation
        )

        # sum up bins from all nodes
        comm.Reduce(
            [ar_product_local, MPI.DOUBLE],
            [ar_product_reduce, MPI.DOUBLE],
            op=MPI.SUM, root=0)
        comm.Reduce(
            [ar_weights_local, MPI.DOUBLE],
            [ar_weights_reduce, MPI.DOUBLE],
            op=MPI.SUM, root=0)

        if comm.rank == 0:
            r_print("Finished direction ", current_direction_index, ", Iteration ", i)
            ar_product_total[current_direction_index] += ar_weights_reduce
            ar_weights_total[current_direction_index] += ar_weights_reduce
            r_print("total weight: ", ar_weights_total[current_direction_index].sum())

            angular_separation_bins = np.arange(num_bins, dtype=float) / num_bins * max_angle_fixed * 180. / np.pi
            np.savez(
                '../../data/planck_dust_correlation.npz', angular_separation_bins=angular_separation_bins,
                ar_product_total=ar_product_total, ar_weights_total=ar_weights_total)
<commit_msg>fix typo.<commit_after>import astropy.units as u
import healpy as hp
import numpy as np
import numpy.random as random
import numpy.random.mtrand
from astropy.coordinates import SkyCoord, Longitude, Latitude
from mpi4py import MPI

from mpi_helper import r_print

comm = MPI.COMM_WORLD
enable_intersection = True
ar_map_nside = 2048

# use a different random seed for each MPI rank, but still use predictable results for easier debugging.
_ = np.random.mtrand.RandomState(comm.rank)

def ra_dec2ang(ra, dec):
    return (90. - dec) * np.pi / 180., ra / 180. * np.pi

def main_loop(ar_map, max_bin_angle, outer_disc_mean, outer_disc, max_angular_separation):
    ar_product = np.zeros(shape=num_bins)
    ar_weights = np.zeros(shape=num_bins)
    chosen_indices = np.random.choice(outer_disc, size=100, replace=False)
    for index in chosen_indices:
        vec_a = hp.pix2vec(ar_map_nside, index)
        inner_disc = hp.query_disc(ar_map_nside, vec=vec_a, radius=max_angular_separation.to(u.rad).value)
        if enable_intersection:
            inner_disc = np.intersect1d(inner_disc, outer_disc, assume_unique=True)
        vec_b = hp.pix2vec(ar_map_nside, inner_disc)
        ar_ang_dist_with_zero = hp.rotator.angdist(vec_b, vec_a)
        a = index
        b = inner_disc[ar_ang_dist_with_zero > 0]
        ar_ang_dist = ar_ang_dist_with_zero[ar_ang_dist_with_zero > 0]
        ar_bins_float = ar_ang_dist / max_bin_angle * num_bins  # type: np.ndarray
        ar_bins = ar_bins_float.astype(int)
        # TODO: filter NaNs earlier so that they don't decrease the correlation
        pair_product = np.nan_to_num((ar_map[a] - outer_disc_mean) * (ar_map[b] - outer_disc_mean))
        ar_product += np.bincount(ar_bins, weights=pair_product, minlength=num_bins)
        ar_weights += np.bincount(ar_bins, minlength=num_bins)

    return ar_product, ar_weights

# load Planck map on the root node
ar_map_shape = None
ar_map_0 = None
ar_map_0_log = None
if comm.rank == 0:
    ar_map_0 = hp.fitsfunc.read_map("../../data/COM_CompMap_Dust-DL07-AvMaps_2048_R2.00.fits", field=0)
    # ar_map_0_log = np.log(ar_map_0)

    # optionally add a mock signal to the map
    mock = False
    if mock:
        ar_mock = ar_map_0
        nside_signal = 32
        radius = hp.nside2resol(nside_signal) / 2 / np.sqrt(2)

        for i in range(hp.nside2npix(nside_signal)):
            vec1 = hp.pix2vec(nside_signal, i)
            mask = hp.query_disc(ar_map_nside, vec=vec1, radius=radius)
            ar_mock[mask] *= 100

        ar_mock /= np.sqrt(100)

# send the map to all other nodes
ar_map_local = comm.bcast(ar_map_0)

# initialize correlation bins
num_bins = 100
ar_product_total = np.zeros(shape=(10, num_bins))
ar_weights_total = np.zeros(shape=(10, num_bins))

# sky scan parameters
num_directions = 4
stripe_step_deg = 10

for current_direction_index in np.arange(num_directions):
    # start with a location within the BOSS field
    center_ra = 180.
    center_dec = 30.
    center_coord = SkyCoord(ra=center_ra * u.degree, dec=center_dec * u.degree)
    center_galactic = center_coord.galactic
    # scan the sky in galactic coordinates:
    galactic_l = Longitude(center_galactic.l + 0 * u.degree)
    # currently we just change the latitude
    galactic_b = Latitude(
        center_galactic.b - (current_direction_index - num_directions * 0.0) * stripe_step_deg * u.degree)
    r_print("galactic l-value:", galactic_l.value)
    r_print("galactic b-value:", galactic_b.value)
    # convert galactic coordinates to healpy theta/phi and create a unit vector:
    center_theta, center_phi = ra_dec2ang(ra=galactic_l.value, dec=galactic_b.value)
    vec = hp.ang2vec(theta=center_theta, phi=center_phi)
    r_print("unit vector:", vec)
    # get all pixels in the current disc
    current_disc_radius = 10 / 180. * np.pi
    disc = hp.query_disc(ar_map_nside, vec=vec, radius=current_disc_radius)
    r_print("disc has ", disc.shape[0], " pixels")

    max_angle_fixed = 5. / 180. * np.pi
    disc_mean = np.nanmean(ar_map_local[disc])
    ar_dec, ar_ra = hp.pix2ang(ar_map_nside, disc)
    global_max_angular_separation = 5. * u.degree  # type: u.Quantity

    # work in iterations so that we don't use too much memory
    for i in np.arange(1):
        # initialize bins for reduce operation
        ar_product_reduce = np.zeros(shape=num_bins)
        ar_weights_reduce = np.zeros(shape=num_bins)

        ar_product_local, ar_weights_local = main_loop(
            ar_map=ar_map_local, max_bin_angle=max_angle_fixed,
            outer_disc_mean=disc_mean, outer_disc=disc,
            max_angular_separation=global_max_angular_separation
        )

        # sum up bins from all nodes
        comm.Reduce(
            [ar_product_local, MPI.DOUBLE],
            [ar_product_reduce, MPI.DOUBLE],
            op=MPI.SUM, root=0)
        comm.Reduce(
            [ar_weights_local, MPI.DOUBLE],
            [ar_weights_reduce, MPI.DOUBLE],
            op=MPI.SUM, root=0)

        if comm.rank == 0:
            r_print("Finished direction ", current_direction_index, ", Iteration ", i)
            ar_product_total[current_direction_index] += ar_product_reduce
            ar_weights_total[current_direction_index] += ar_weights_reduce
            r_print("total weight: ", ar_weights_total[current_direction_index].sum())

            angular_separation_bins = np.arange(num_bins, dtype=float) / num_bins * max_angle_fixed * 180. / np.pi
            np.savez(
                '../../data/planck_dust_correlation.npz', angular_separation_bins=angular_separation_bins,
                ar_product_total=ar_product_total, ar_weights_total=ar_weights_total)
<|endoftext|>


<commit_before>'''
The MIT License (MIT)

Copyright (c) 2014 NTHUOJ team

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
'''
import time
import random
import datetime
from django.http import Http404
from django.utils import timezone
from utils.log_info import get_logger
from contest.models import Contest
from django.shortcuts import render
from django.http import HttpResponse
from users.models import User, Notification
from django.template import RequestContext
from utils.user_info import validate_user

# Create your views here.
logger = get_logger()
def index(request, alert_info='none'):

    present = timezone.now()
    c_runnings = Contest.objects.filter(start_time__lt=present, end_time__gt=present)
    c_upcomings = Contest.objects.filter(start_time__gt=present)
    return render(request, 'index/index.html',
                {'c_runnings':c_runnings, 'c_upcomings':c_upcomings,
                'alert_info':alert_info},
                context_instance=RequestContext(request, processors=[custom_proc]))

def custom_404(request):
    return render(request, 'index/404.html', status=404)

def custom_500(request):
    return render(request, 'index/500.html',{'error_message':'error'}, status=500)

def base(request):
    return render(request, 'index/base.html',{},
                context_instance=RequestContext(request, processors=[custom_proc]))

def get_time(request):
    t = time.time()
    tstr = datetime.datetime.fromtimestamp(t).strftime('%Y/%m/%d %H:%M:%S')
    return HttpResponse(tstr)

def custom_proc(request):
<<<<<<< HEAD

    amount = Notification.objects.filter \
        (receiver=request.user, read=False).count()

=======
>>>>>>> d50af02af4ebd9f457894d38f459a14a5bd75877
    t = time.time()
    tstr = datetime.datetime.fromtimestamp(t).strftime('%Y/%m/%d %H:%M:%S')
    people = random.randint(100,999)
    return {
        'tstr': tstr,
        'people': people,
        'amount': amount
    }
<commit_msg>fix conflict<commit_after>'''
The MIT License (MIT)

Copyright (c) 2014 NTHUOJ team

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
'''
import time
import random
import datetime
from django.http import Http404
from django.utils import timezone
from utils.log_info import get_logger
from contest.models import Contest
from django.shortcuts import render
from django.http import HttpResponse
from users.models import User, Notification
from django.template import RequestContext
from utils.user_info import validate_user

# Create your views here.
logger = get_logger()
def index(request, alert_info='none'):

    present = timezone.now()
    c_runnings = Contest.objects.filter(start_time__lt=present, end_time__gt=present)
    c_upcomings = Contest.objects.filter(start_time__gt=present)
    return render(request, 'index/index.html',
                {'c_runnings':c_runnings, 'c_upcomings':c_upcomings,
                'alert_info':alert_info},
                context_instance=RequestContext(request, processors=[custom_proc]))

def custom_404(request):
    return render(request, 'index/404.html', status=404)

def custom_500(request):
    return render(request, 'index/500.html',{'error_message':'error'}, status=500)

def base(request):
    return render(request, 'index/base.html',{},
                context_instance=RequestContext(request, processors=[custom_proc]))

def get_time(request):
    t = time.time()
    tstr = datetime.datetime.fromtimestamp(t).strftime('%Y/%m/%d %H:%M:%S')
    return HttpResponse(tstr)

def custom_proc(request):

    amount = Notification.objects.filter \
        (receiver=request.user, read=False).count()

    t = time.time()
    tstr = datetime.datetime.fromtimestamp(t).strftime('%Y/%m/%d %H:%M:%S')
    people = random.randint(100,999)
    return {
        'tstr': tstr,
        'people': people,
        'amount': amount
    }
<|endoftext|>