<commit_before>import os.path
import re
from functools import total_ordering
import numpy as np

import dwi.asciifile
import dwi.files
import dwi.util

@total_ordering
class GleasonScore(object):
    def __init__(self, score):
        """Intialize with a sequence or a string like '3+4+5' (third digit is
        optional)."""
        if isinstance(score, str) or isinstance(score, unicode):
            s = score.split('+')
        else:
            s = list(score)
        if not 2 <= len(s) <= 3:
            raise Exception('Invalid gleason score: %s', score)
        if len(s) == 2:
            s.append(0)
        self.score = tuple(map(int, s))

    def __repr__(self):
        s = self.score
        if not s[-1]:
            s = s[0:-1] # Drop trailing zero.
        return '+'.join(map(str, s))

    def __hash__(self):
        return hash(self.score)

    def __iter__(self):
        return iter(self.score)

    def __eq__(self, other):
        return self.score == other.score

    def __lt__(self, other):
        return self.score < other.score

@total_ordering
class Patient(object):
    def __init__(self, num, name, scans, score):
        self.num = num
        self.name = name
        self.scans = scans
        self.score = score

    def __repr__(self):
        return repr(self.tuple())

    def __hash__(self):
        return hash(self.tuple())

    def __eq__(self, other):
        return self.tuple() == other.tuple()

    def __lt__(self, other):
        return self.tuple() < other.tuple()

    def tuple(self):
        return self.num, self.name, self.scans, self.score

def read_patients_file(filename):
    """Load a list of patients.

    Row format: num name scan,... score1+score2
    """
    patients = []
    p = re.compile(r'(\d+)\s+(\w+)\s+([\w,]+)\s+(\d\+\d(\+\d)?)')
    with open(filename, 'rU') as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            if line[0] == '#':
                continue
            m = p.match(line)
            if m:
                num, name, scans, score = m.groups()[0:4]
                num = int(num)
                name = name.lower()
                scans = sorted(scans.lower().split(','))
                score = GleasonScore(score)
                patient = Patient(num, name, scans, score)
                patients.append(patient)
            else:
                raise Exception('Invalid line in patients file: %s', line)
    return sorted(patients)

def scan_in_patients(patients, num, scan):
    """Is this scan listed in the patients sequence?"""
    for p in patients:
        if p.num == num and scan in p.scans:
            return True
    return False

def get_patient(patients, num):
    """Search a patient from sequence by patient number."""
    for p in patients:
        if p.num == num:
            return p
    raise Exception('Patient not found: {}'.format(num))

def get_gleason_scores(patients):
    """Get all separate Gleason scores, sorted."""
    scores = {p.score for p in patients}
    return sorted(scores)

def score_ord(scores, score):
    """Get Gleason score's ordinal number."""
    return sorted(scores).index(score)

def load_files(patients, filenames, pairs=False):
    """Load pmap files."""
    pmapfiles = []
    for f in filenames:
        num, scan = dwi.util.parse_num_scan(os.path.basename(f))
        if scan_in_patients(patients, num, scan):
            pmapfiles.append(f)
    afs = map(dwi.asciifile.AsciiFile, pmapfiles)
    if pairs:
        dwi.util.scan_pairs(afs)
    ids = [dwi.util.parse_num_scan(af.basename) for af in afs]
    pmaps = [af.a for af in afs]
    pmaps = np.array(pmaps)
    params = afs[0].params()
    assert pmaps.shape[-1] == len(params), 'Parameter name mismatch.'
    #print 'Filenames: %i, loaded: %i, lines: %i, columns: %i'\
    #        % (len(filenames), pmaps.shape[0], pmaps.shape[1], pmaps.shape[2])
    return pmaps, ids, params

def load_labels(patients, nums, labeltype='score'):
    """Load labels according to patient numbers."""
    gs = get_gleason_scores(patients)
    scores = [get_patient(patients, n).score for n in nums]
    if labeltype == 'score':
        # Use Gleason score.
        labels = scores
    elif labeltype == 'ord':
        # Use ordinal.
        labels = [score_ord(gs, s) for s in scores]
    elif labeltype == 'bin':
        # Is aggressive? (ROI1.)
        labels = [s > GleasonScore('3+4') for s in scores]
    elif labeltype == 'cancer':
        # Is cancer? (ROI1 vs 2, all true for ROI1.)
        labels = [1] * len(scores)
    else:
        raise Exception('Invalid parameter: %s' % labeltype)
    return labels

def read_pmaps(samplelist_file, patients_file, pmapdir, thresholds=['3+3'],
        voxel='all'):
    """Read pmaps labeled by their Gleason score.

    Label thresholds are maximum scores of each label group. Labels are ordinal
    of score if no thresholds provided."""
    # TODO Support for selecting measurements over scan pairs
    thresholds = map(GleasonScore, thresholds)
    samples = dwi.files.read_sample_list(samplelist_file)
    patientsinfo = read_patients_file(patients_file)
    data = []
    for sample in samples:
        case = sample['case']
        score = get_patient(patientsinfo, case).score
        if thresholds:
            label = sum(score > t for t in thresholds)
        else:
            label = score_ord(get_gleason_scores(patientsinfo), score)
        for scan in sample['scans']:
            pmap, params, pathname = read_pmap(pmapdir, case, scan, voxel=voxel)
            d = dict(case=case, scan=scan, score=score, label=label, pmap=pmap,
                    params=params, pathname=pathname)
            data.append(d)
            if pmap.shape != data[0]['pmap'].shape:
                raise Exception('Irregular shape: %s' % pathname)
            if params != data[0]['params']:
                raise Exception('Irregular params: %s' % pathname)
    return data

def grouping(data):
    """Return different scores, grouped scores, and their sample sizes.

    See read_pmaps()."""
    scores = [d['score'] for d in data]
    labels = [d['label'] for d in data]
    n_labels = max(labels) + 1
    groups = [[] for _ in range(n_labels)]
    for s, l in zip(scores, labels):
        groups[l].append(s)
    different_scores = sorted(set(scores))
    group_scores = [sorted(set(g)) for g in groups]
    group_sizes = [len(g) for g in groups]
    return different_scores, group_scores, group_sizes

def read_pmap(dirname, case, scan, voxel='all'):
    """Read single pmap."""
    d = dict(d=dirname, c=case, s=scan)
    path = dwi.util.sglob('{d}/{c}_*{s}*.txt'.format(**d))
    af = dwi.asciifile.AsciiFile(path)
    pmap = af.a
    params = af.params()
    if pmap.shape[-1] != len(params):
        # TODO Move to Asciifile initializer?
        raise Exception('Number of parameters mismatch: %s' % af.filename)
    # Select voxel to use.
    if voxel == 'all':
        pass # Use all voxels.
    elif voxel == 'mean':
        pmap = np.mean(pmap, axis=0, keepdims=True) # Use mean voxel.
    elif voxel == 'median':
        pmap = dwi.util.median(pmap, axis=0, keepdims=True) # Use median voxel.
    else:
        pmap = pmap[[int(voxel)]] # Use single voxel only.
    return pmap, params, af.filename
<commit_msg>refactoring<commit_after>import os.path
import re
from functools import total_ordering
import numpy as np

import dwi.asciifile
import dwi.files
import dwi.util

@total_ordering
class GleasonScore(object):
    def __init__(self, score):
        """Intialize with a sequence or a string like '3+4+5' (third digit is
        optional)."""
        if isinstance(score, str) or isinstance(score, unicode):
            s = score.split('+')
        else:
            s = list(score)
        if not 2 <= len(s) <= 3:
            raise Exception('Invalid gleason score: %s', score)
        if len(s) == 2:
            s.append(0)
        self.score = tuple(map(int, s))

    def __repr__(self):
        s = self.score
        if not s[-1]:
            s = s[0:-1] # Drop trailing zero.
        return '+'.join(map(str, s))

    def __hash__(self):
        return hash(self.score)

    def __iter__(self):
        return iter(self.score)

    def __eq__(self, other):
        return self.score == other.score

    def __lt__(self, other):
        return self.score < other.score

@total_ordering
class Patient(object):
    def __init__(self, num, name, scans, score):
        self.num = num
        self.name = name
        self.scans = scans
        self.score = score

    def __repr__(self):
        return repr(self.tuple())

    def __hash__(self):
        return hash(self.tuple())

    def __eq__(self, other):
        return self.tuple() == other.tuple()

    def __lt__(self, other):
        return self.tuple() < other.tuple()

    def tuple(self):
        return self.num, self.name, self.scans, self.score

def read_patients_file(filename):
    """Load a list of patients.

    Row format: num name scan,... score1+score2
    """
    patients = []
    p = re.compile(r'(\d+)\s+(\w+)\s+([\w,]+)\s+(\d\+\d(\+\d)?)')
    with open(filename, 'rU') as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            if line[0] == '#':
                continue
            m = p.match(line)
            if m:
                num, name, scans, score = m.groups()[0:4]
                num = int(num)
                name = name.lower()
                scans = sorted(scans.lower().split(','))
                score = GleasonScore(score)
                patient = Patient(num, name, scans, score)
                patients.append(patient)
            else:
                raise Exception('Invalid line in patients file: %s', line)
    return sorted(patients)

def scan_in_patients(patients, num, scan):
    """Is this scan listed in the patients sequence?"""
    for p in patients:
        if p.num == num and scan in p.scans:
            return True
    return False

def get_patient(patients, num):
    """Search a patient from sequence by patient number."""
    for p in patients:
        if p.num == num:
            return p
    raise Exception('Patient not found: {}'.format(num))

def get_gleason_scores(patients):
    """Get all separate Gleason scores, sorted."""
    return sorted({p.score for p in patients})

def score_ord(scores, score):
    """Get Gleason score's ordinal number."""
    return sorted(scores).index(score)

def load_files(patients, filenames, pairs=False):
    """Load pmap files."""
    pmapfiles = []
    for f in filenames:
        num, scan = dwi.util.parse_num_scan(os.path.basename(f))
        if scan_in_patients(patients, num, scan):
            pmapfiles.append(f)
    afs = map(dwi.asciifile.AsciiFile, pmapfiles)
    if pairs:
        dwi.util.scan_pairs(afs)
    ids = [dwi.util.parse_num_scan(af.basename) for af in afs]
    pmaps = [af.a for af in afs]
    pmaps = np.array(pmaps)
    params = afs[0].params()
    assert pmaps.shape[-1] == len(params), 'Parameter name mismatch.'
    #print 'Filenames: %i, loaded: %i, lines: %i, columns: %i'\
    #        % (len(filenames), pmaps.shape[0], pmaps.shape[1], pmaps.shape[2])
    return pmaps, ids, params

def load_labels(patients, nums, labeltype='score'):
    """Load labels according to patient numbers."""
    gs = get_gleason_scores(patients)
    scores = [get_patient(patients, n).score for n in nums]
    if labeltype == 'score':
        # Use Gleason score.
        labels = scores
    elif labeltype == 'ord':
        # Use ordinal.
        labels = [score_ord(gs, s) for s in scores]
    elif labeltype == 'bin':
        # Is aggressive? (ROI1.)
        labels = [s > GleasonScore('3+4') for s in scores]
    elif labeltype == 'cancer':
        # Is cancer? (ROI1 vs 2, all true for ROI1.)
        labels = [1] * len(scores)
    else:
        raise Exception('Invalid parameter: %s' % labeltype)
    return labels

def read_pmaps(samplelist_file, patients_file, pmapdir, thresholds=['3+3'],
        voxel='all'):
    """Read pmaps labeled by their Gleason score.

    Label thresholds are maximum scores of each label group. Labels are ordinal
    of score if no thresholds provided."""
    # TODO Support for selecting measurements over scan pairs
    thresholds = map(GleasonScore, thresholds)
    samples = dwi.files.read_sample_list(samplelist_file)
    patientsinfo = read_patients_file(patients_file)
    data = []
    for sample in samples:
        case = sample['case']
        score = get_patient(patientsinfo, case).score
        if thresholds:
            label = sum(score > t for t in thresholds)
        else:
            label = score_ord(get_gleason_scores(patientsinfo), score)
        for scan in sample['scans']:
            pmap, params, pathname = read_pmap(pmapdir, case, scan, voxel=voxel)
            d = dict(case=case, scan=scan, score=score, label=label, pmap=pmap,
                    params=params, pathname=pathname)
            data.append(d)
            if pmap.shape != data[0]['pmap'].shape:
                raise Exception('Irregular shape: %s' % pathname)
            if params != data[0]['params']:
                raise Exception('Irregular params: %s' % pathname)
    return data

def grouping(data):
    """Return different scores, grouped scores, and their sample sizes.

    See read_pmaps()."""
    scores = [d['score'] for d in data]
    labels = [d['label'] for d in data]
    n_labels = max(labels) + 1
    groups = [[] for _ in range(n_labels)]
    for s, l in zip(scores, labels):
        groups[l].append(s)
    different_scores = sorted(set(scores))
    group_scores = [sorted(set(g)) for g in groups]
    group_sizes = [len(g) for g in groups]
    return different_scores, group_scores, group_sizes

def read_pmap(dirname, case, scan, voxel='all'):
    """Read single pmap."""
    d = dict(d=dirname, c=case, s=scan)
    path = dwi.util.sglob('{d}/{c}_*{s}*.txt'.format(**d))
    af = dwi.asciifile.AsciiFile(path)
    pmap = af.a
    params = af.params()
    if pmap.shape[-1] != len(params):
        # TODO Move to Asciifile initializer?
        raise Exception('Number of parameters mismatch: %s' % af.filename)
    # Select voxel to use.
    if voxel == 'all':
        pass # Use all voxels.
    elif voxel == 'mean':
        pmap = np.mean(pmap, axis=0, keepdims=True) # Use mean voxel.
    elif voxel == 'median':
        pmap = dwi.util.median(pmap, axis=0, keepdims=True) # Use median voxel.
    else:
        pmap = pmap[[int(voxel)]] # Use single voxel only.
    return pmap, params, af.filename
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2010-2014, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""
Scenario calculator core functionality
"""
import collections
import random
import numpy

from openquake.nrmllib.hazard.parsers import RuptureModelParser

# HAZARDLIB
from openquake.hazardlib.calc import ground_motion_fields, filters
from openquake.hazardlib.imt import from_string
import openquake.hazardlib.gsim

from openquake.engine.calculators.hazard import general as haz_general
from openquake.engine.utils import tasks, general
from openquake.engine.db import models
from openquake.engine.input import source
from openquake.engine import writer
from openquake.engine.performance import EnginePerformanceMonitor

AVAILABLE_GSIMS = openquake.hazardlib.gsim.get_available_gsims()

@tasks.oqtask
def gmfs(job_id, seeds, sitecol, rupture, gmf_id, task_no):
    """
    A celery task wrapper function around :func:`compute_gmfs`.
    See :func:`compute_gmfs` for parameter definitions.
    """
    hc = models.HazardCalculation.objects.get(oqjob=job_id)
    imts = [from_string(x) for x in hc.intensity_measure_types]
    gsim = AVAILABLE_GSIMS[hc.gsim]()  # instantiate the GSIM class
    realizations = 1  # one realization for each seed
    correlation_model = haz_general.get_correl_model(hc)
    rup_filter = filters.rupture_site_distance_filter(hc.maximum_distance)

    cache = collections.defaultdict(list)  # {site_id, imt -> gmvs}
    inserter = writer.CacheInserter(models.GmfData, 1000)
    # insert GmfData in blocks of 1000 sites

    with EnginePerformanceMonitor('computing gmfs', job_id, gmfs):
        for task_seed in seeds:
            numpy.random.seed(task_seed)
            gmf_dict = ground_motion_fields(
                rupture, sitecol, imts, gsim, hc.truncation_level,
                realizations, correlation_model, rup_filter)
            for imt in imts:
                for site_id, gmv in zip(sitecol.sids, gmf_dict[imt]):
                    # float is needed below to convert 1x1 matrices
                    cache[site_id, imt].append(float(gmv))

    with EnginePerformanceMonitor('saving gmfs', job_id, gmfs):
        for site_id, imt in cache:
            inserter.add(
                models.GmfData(
                    gmf_id=gmf_id,
                    task_no=task_no,
                    imt=imt[0],
                    sa_period=imt[1],
                    sa_damping=imt[2],
                    site_id=site_id,
                    rupture_ids=None,
                    gmvs=cache[site_id, imt]))
        inserter.flush()

class ScenarioHazardCalculator(haz_general.BaseHazardCalculator):
    """
    Scenario hazard calculator. Computes ground motion fields.
    """

    core_calc_task = gmfs
    output = None  # defined in pre_execute

    def __init__(self, *args, **kwargs):
        super(ScenarioHazardCalculator, self).__init__(*args, **kwargs)
        self.gmf = None
        self.rupture = None

    def initialize_sources(self):
        """
        Get the rupture_model file from the job.ini file, and set the
        attribute self.rupture.
        """
        nrml = RuptureModelParser(self.hc.inputs['rupture_model']).parse()
        self.rupture = source.NrmlHazardlibConverter(self.hc)(nrml)

    def pre_execute(self):
        """
        Do pre-execution work. At the moment, this work entails:
        parsing and initializing sources, parsing and initializing the
        site model (if there is one), parsing vulnerability and
        exposure files, and generating logic tree realizations. (The
        latter piece basically defines the work to be done in the
        `execute` phase.)
        """
        self.parse_risk_models()
        self.initialize_sources()
        self.initialize_site_model()

        # create a record in the output table
        output = models.Output.objects.create(
            oq_job=self.job,
            display_name="GMF",
            output_type="gmf_scenario")

        # create an associated gmf record
        self.gmf = models.Gmf.objects.create(output=output)

    def task_arg_gen(self):
        """
        Yield a tuple of the form (job_id, sitecol, rupture_id, gmf_id,
        task_seed, num_realizations). `task_seed` will be used to seed
        numpy for temporal occurence sampling. Only a single task
        will be generated which is fine since the computation is fast
        anyway.
        """
        rnd = random.Random()
        rnd.seed(self.hc.random_seed)
        all_seeds = [rnd.randint(0, models.MAX_SINT_32)
                     for _ in xrange(self.hc.number_of_ground_motion_fields)]
        ss = general.SequenceSplitter(self.concurrent_tasks())
        for task_no, task_seeds in enumerate(ss.split(all_seeds)):
            yield (self.job.id, task_seeds, self.hc.site_collection,
                   self.rupture, self.gmf.id, task_no)
<commit_msg>Do the site filtering only once at the beginning<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2010-2014, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""
Scenario calculator core functionality
"""
import collections
import random
import numpy

from openquake.nrmllib.hazard.parsers import RuptureModelParser

# HAZARDLIB
from openquake.hazardlib.calc import ground_motion_fields, filters
from openquake.hazardlib.imt import from_string
import openquake.hazardlib.gsim

from openquake.engine.calculators.hazard import general as haz_general
from openquake.engine.utils import tasks, general
from openquake.engine.db import models
from openquake.engine.input import source
from openquake.engine import writer
from openquake.engine.performance import EnginePerformanceMonitor

AVAILABLE_GSIMS = openquake.hazardlib.gsim.get_available_gsims()

@tasks.oqtask
def gmfs(job_id, seeds, sitecol, rupture, gmf_id, task_no):
    """
    A celery task wrapper function around :func:`compute_gmfs`.
    See :func:`compute_gmfs` for parameter definitions.
    """
    hc = models.HazardCalculation.objects.get(oqjob=job_id)
    imts = [from_string(x) for x in hc.intensity_measure_types]
    gsim = AVAILABLE_GSIMS[hc.gsim]()  # instantiate the GSIM class
    realizations = 1  # one realization for each seed
    correlation_model = haz_general.get_correl_model(hc)

    cache = collections.defaultdict(list)  # {site_id, imt -> gmvs}
    inserter = writer.CacheInserter(models.GmfData, 1000)
    # insert GmfData in blocks of 1000 sites

    with EnginePerformanceMonitor('computing gmfs', job_id, gmfs):
        for task_seed in seeds:
            numpy.random.seed(task_seed)
            gmf_dict = ground_motion_fields(
                rupture, sitecol, imts, gsim, hc.truncation_level,
                realizations, correlation_model)
            for imt in imts:
                for site_id, gmv in zip(sitecol.sids, gmf_dict[imt]):
                    # float is needed below to convert 1x1 matrices
                    cache[site_id, imt].append(float(gmv))

    with EnginePerformanceMonitor('saving gmfs', job_id, gmfs):
        for site_id, imt in cache:
            inserter.add(
                models.GmfData(
                    gmf_id=gmf_id,
                    task_no=task_no,
                    imt=imt[0],
                    sa_period=imt[1],
                    sa_damping=imt[2],
                    site_id=site_id,
                    rupture_ids=None,
                    gmvs=cache[site_id, imt]))
        inserter.flush()

class ScenarioHazardCalculator(haz_general.BaseHazardCalculator):
    """
    Scenario hazard calculator. Computes ground motion fields.
    """

    core_calc_task = gmfs
    output = None  # defined in pre_execute

    def __init__(self, *args, **kwargs):
        super(ScenarioHazardCalculator, self).__init__(*args, **kwargs)
        self.gmf = None
        self.rupture = None

    def initialize_sources(self):
        """
        Get the rupture_model file from the job.ini file, and set the
        attribute self.rupture.
        """
        nrml = RuptureModelParser(self.hc.inputs['rupture_model']).parse()
        self.rupture = source.NrmlHazardlibConverter(self.hc)(nrml)

    def pre_execute(self):
        """
        Do pre-execution work. At the moment, this work entails:
        parsing and initializing sources, parsing and initializing the
        site model (if there is one), parsing vulnerability and
        exposure files, and generating logic tree realizations. (The
        latter piece basically defines the work to be done in the
        `execute` phase.)
        """
        self.parse_risk_models()
        self.initialize_sources()
        self.initialize_site_model()

        # create a record in the output table
        output = models.Output.objects.create(
            oq_job=self.job,
            display_name="GMF",
            output_type="gmf_scenario")

        # create an associated gmf record
        self.gmf = models.Gmf.objects.create(output=output)

    def task_arg_gen(self):
        """
        Yield a tuple of the form (job_id, sitecol, rupture_id, gmf_id,
        task_seed, num_realizations). `task_seed` will be used to seed
        numpy for temporal occurence sampling. Only a single task
        will be generated which is fine since the computation is fast
        anyway.
        """
        hc = self.hc
        if hc.maximum_distance:
            sites = filters.filter_sites_by_distance_to_rupture(
                self.rupture, hc.maximum_distance, hc.site_collection)
            if sites is None:
                raise RuntimeError(
                    'All sites where filtered out! '
                    'maximum_distance=%s km' % hc.maximum_distance)
        rnd = random.Random()
        rnd.seed(self.hc.random_seed)
        all_seeds = [rnd.randint(0, models.MAX_SINT_32)
                     for _ in xrange(self.hc.number_of_ground_motion_fields)]
        ss = general.SequenceSplitter(self.concurrent_tasks())
        for task_no, task_seeds in enumerate(ss.split(all_seeds)):
            yield (self.job.id, task_seeds, sites, self.rupture,
                   self.gmf.id, task_no)
<|endoftext|>


<commit_before>from django.conf import settings

def google_analytics(request):
    """
    Google analytics intergration
    :param request:
    :return:
    """
    ga_prop_id = getattr(settings, 'GOOGLE_ANALYTICS_PROPERTY_ID', False)
    ga_domain = getattr(settings, 'GOOGLE_ANALYTICS_DOMAIN', False)

    if not settings.DEBUG and ga_prop_id and ga_domain:
        return {
            'GOOGLE_ANALYTICS_PROPERTY_ID': ga_prop_id,
            'GOOGLE_ANALYTICS_DOMAIN': ga_domain
        }
    elif settings.DEBUG:
        return {
            'GOOGLE_ANALYTICS_PROPERTY_ID': 'development',
            'GOOGLE_ANALYTICS_DOMAIN': 'auto'
        }
<commit_msg>"master"<commit_after>from django.conf import settings

def google_analytics(request):
    """
    Google analytics intergration
    :param request:
    :return:
    """
    ga_prop_id = getattr(settings, 'GOOGLE_ANALYTICS_PROPERTY_ID', False)
    ga_domain = getattr(settings, 'GOOGLE_ANALYTICS_DOMAIN', False)

    state = True

    if not settings.DEBUG and ga_prop_id and ga_domain:
        return {
            'GOOGLE_ANALYTICS_PROPERTY_ID': ga_prop_id if state else '',
            'GOOGLE_ANALYTICS_DOMAIN': ga_domain if state else ''
        }
    elif settings.DEBUG:
        return {
            'GOOGLE_ANALYTICS_PROPERTY_ID': 'development',
            'GOOGLE_ANALYTICS_DOMAIN': 'auto'
        }
<|endoftext|>


<commit_before>from __future__ import print_function
from future.utils import itervalues
from future.builtins import str
from mrq.task import Task
from mrq.queue import Queue
from bson import ObjectId
from mrq.context import connections, get_current_config, get_current_job
from mrq.job import set_queues_size
from collections import defaultdict
from mrq.utils import group_iter
import datetime
import ujson as json

def get_task_cfg(taskpath):
    return get_current_config().get("tasks", {}).get(taskpath) or {}

class JobAction(Task):

    params = None
    collection = None

    def run(self, params):

        self.params = params
        self.collection = connections.mongodb_jobs.mrq_jobs

        query = self.build_query()

        return self.perform_action(
            self.params.get("action"), query, self.params.get("destination_queue")
        )

    def build_query(self):
        query = {}
        current_job = get_current_job()

        if self.params.get("id"):
            query["_id"] = ObjectId(self.params.get("id"))

        # TODO use redis for queue
        for k in [
                "queue",
                "status",
                "worker",
                "path",
                "dateretry",
                "exceptiontype"]:
            if self.params.get(k):
                if isinstance(self.params[k], (list, tuple)):
                    query[k] = {"$in": list(self.params[k])}
                else:
                    query[k] = self.params[k]
            if query.get("worker"):
                query["worker"] = ObjectId(query["worker"])

        if self.params.get("params"):
            params_dict = json.loads(self.params.get("params"))  # pylint: disable=no-member

            for key in params_dict:
                query["params.%s" % key] = params_dict[key]

        if current_job and "_id" not in query:
            query["_id"] = {"$lte": current_job.id}

        return query

    def perform_action(self, action, query, destination_queue):

        stats = {
            "requeued": 0,
            "cancelled": 0
        }

        if action == "cancel":

            default_job_timeout = get_current_config()["default_job_timeout"]

            # Finding the ttl here to expire is a bit hard because we may have mixed paths
            # and hence mixed ttls.
            # If we are cancelling by path, get this ttl
            if query.get("path"):
                result_ttl = get_task_cfg(query["path"]).get("result_ttl", default_job_timeout)

            # If not, get the maxmimum ttl of all tasks.
            else:

                tasks_defs = get_current_config().get("tasks", {})
                tasks_ttls = [cfg.get("result_ttl", 0) for cfg in itervalues(tasks_defs)]

                result_ttl = max([default_job_timeout] + tasks_ttls)

            now = datetime.datetime.utcnow()

            size_by_queues = defaultdict(int)
            if "queue" not in query:
                for job in self.collection.find(query, projection={"queue": 1}):
                    size_by_queues[job["queue"]] += 1

            ret = self.collection.update(query, {"$set": {
                "status": "cancel",
                "dateexpires": now + datetime.timedelta(seconds=result_ttl),
                "dateupdated": now
            }}, multi=True)
            stats["cancelled"] = ret["n"]

            if "queue" in query:
                if isinstance(query["queue"], str):
                    size_by_queues[query["queue"]] = ret["n"]
            set_queues_size(size_by_queues, action="decr")

            # Special case when emptying just by queue name: empty it directly!
            # In this case we could also loose some jobs that were queued after
            # the MongoDB update. They will be "lost" and requeued later like the other case
            # after the Redis BLPOP
            if list(query.keys()) == ["queue"]:
                Queue(query["queue"]).empty()

        elif action in ("requeue", "requeue_retry"):

            # Requeue task by groups of maximum 1k items (if all in the same
            # queue)
            cursor = self.collection.find(query, projection=["_id", "queue"])

            # We must freeze the list because queries below would change it.
            # This could not fit in memory, research adding {"stats": {"$ne":
            # "queued"}} in the query
            fetched_jobs = list(cursor)

            for jobs in group_iter(fetched_jobs, n=1000):

                jobs_by_queue = defaultdict(list)
                for job in jobs:
                    jobs_by_queue[job["queue"]].append(job["_id"])
                    stats["requeued"] += 1

                for queue in jobs_by_queue:
                    updates = {
                        "status": "queued",
                        "datequeued": datetime.datetime.utcnow(),
                        "dateupdated": datetime.datetime.utcnow()
                    }

                    if destination_queue is not None:
                        updates["queue"] = destination_queue

                    if action == "requeue":
                        updates["retry_count"] = 0

                    self.collection.update({
                        "_id": {"$in": jobs_by_queue[queue]}
                    }, {"$set": updates}, multi=True)

                set_queues_size({queue: len(jobs) for queue, jobs in jobs_by_queue.iteritems()})

        return stats
<commit_msg>fix dashboard actions on subqueues<commit_after>from __future__ import print_function
from future.utils import itervalues
from future.builtins import str
from mrq.task import Task
from mrq.queue import Queue
from bson import ObjectId
from mrq.context import connections, get_current_config, get_current_job
from mrq.job import set_queues_size
from collections import defaultdict
from mrq.utils import group_iter
import datetime
import ujson as json

def get_task_cfg(taskpath):
    return get_current_config().get("tasks", {}).get(taskpath) or {}

class JobAction(Task):

    params = None
    collection = None

    def run(self, params):

        self.params = params
        self.collection = connections.mongodb_jobs.mrq_jobs

        query = self.build_query()

        return self.perform_action(
            self.params.get("action"), query, self.params.get("destination_queue")
        )

    def build_query(self):
        query = {}
        current_job = get_current_job()

        if self.params.get("id"):
            query["_id"] = ObjectId(self.params.get("id"))

        # TODO use redis for queue
        for k in [
                "queue",
                "status",
                "worker",
                "path",
                "dateretry",
                "exceptiontype"]:
            if self.params.get(k):
                if isinstance(self.params[k], (list, tuple)):
                    query[k] = {"$in": list(self.params[k])}
                elif k == "queue" and self.params[k][-1] == "/":
                    query[k] = {"$regex": "^%s" % self.params[k]}
                else:
                    query[k] = self.params[k]
            if query.get("worker"):
                query["worker"] = ObjectId(query["worker"])

        if self.params.get("params"):
            params_dict = json.loads(self.params.get("params"))  # pylint: disable=no-member

            for key in params_dict:
                query["params.%s" % key] = params_dict[key]

        if current_job and "_id" not in query:
            query["_id"] = {"$lte": current_job.id}

        return query

    def perform_action(self, action, query, destination_queue):

        stats = {
            "requeued": 0,
            "cancelled": 0
        }

        if action == "cancel":

            default_job_timeout = get_current_config()["default_job_timeout"]

            # Finding the ttl here to expire is a bit hard because we may have mixed paths
            # and hence mixed ttls.
            # If we are cancelling by path, get this ttl
            if query.get("path"):
                result_ttl = get_task_cfg(query["path"]).get("result_ttl", default_job_timeout)

            # If not, get the maxmimum ttl of all tasks.
            else:

                tasks_defs = get_current_config().get("tasks", {})
                tasks_ttls = [cfg.get("result_ttl", 0) for cfg in itervalues(tasks_defs)]

                result_ttl = max([default_job_timeout] + tasks_ttls)

            now = datetime.datetime.utcnow()

            size_by_queues = defaultdict(int)
            if "queue" not in query:
                for job in self.collection.find(query, projection={"queue": 1}):
                    size_by_queues[job["queue"]] += 1

            ret = self.collection.update(query, {"$set": {
                "status": "cancel",
                "dateexpires": now + datetime.timedelta(seconds=result_ttl),
                "dateupdated": now
            }}, multi=True)
            stats["cancelled"] = ret["n"]

            if "queue" in query:
                if isinstance(query["queue"], str):
                    size_by_queues[query["queue"]] = ret["n"]
            set_queues_size(size_by_queues, action="decr")

            # Special case when emptying just by queue name: empty it directly!
            # In this case we could also loose some jobs that were queued after
            # the MongoDB update. They will be "lost" and requeued later like the other case
            # after the Redis BLPOP
            if list(query.keys()) == ["queue"]:
                Queue(query["queue"]).empty()

        elif action in ("requeue", "requeue_retry"):

            # Requeue task by groups of maximum 1k items (if all in the same
            # queue)
            cursor = self.collection.find(query, projection=["_id", "queue"])

            # We must freeze the list because queries below would change it.
            # This could not fit in memory, research adding {"stats": {"$ne":
            # "queued"}} in the query
            fetched_jobs = list(cursor)

            for jobs in group_iter(fetched_jobs, n=1000):

                jobs_by_queue = defaultdict(list)
                for job in jobs:
                    jobs_by_queue[job["queue"]].append(job["_id"])
                    stats["requeued"] += 1

                for queue in jobs_by_queue:
                    updates = {
                        "status": "queued",
                        "datequeued": datetime.datetime.utcnow(),
                        "dateupdated": datetime.datetime.utcnow()
                    }

                    if destination_queue is not None:
                        updates["queue"] = destination_queue

                    if action == "requeue":
                        updates["retry_count"] = 0

                    self.collection.update({
                        "_id": {"$in": jobs_by_queue[queue]}
                    }, {"$set": updates}, multi=True)

                set_queues_size({queue: len(jobs) for queue, jobs in jobs_by_queue.iteritems()})

        return stats
<|endoftext|>


<commit_before>import os
import sys
import zipfile
import urllib
import json
import requests

from flask import Flask
from flask import request

app = Flask('cloud-mdbn')

import MDBN

batch_output_dir = ''
batch_start_date_str = ''
BUSY = 10
FREE = 0
netStatus = FREE
verbose = False

def prepare_OV_TCGA_datafiles(config, datadir='data'):
    base_url = 'http://nar.oxfordjournals.org/content/suppl/2012/07/25/gks725.DC1/'
    archive = 'nar-00961-n-2012-File005.zip'

    datafiles = dict()
    for key in config["pathways"]:
        datafiles[key] = config['dbns'][key]["datafile"]

    if not os.path.isdir(datadir):
        os.mkdir(datadir)

    root_dir = os.getcwd()
    os.chdir(datadir)
    for name, datafile in datafiles.iteritems():
        if not os.path.isfile(datafile):
            if not os.path.isfile(archive):
                print('Downloading TCGA_Data from ' + base_url)
                testfile = urllib.URLopener()
                testfile.retrieve(base_url + archive, archive)
            zipfile.ZipFile(archive, 'r').extract(datafile)
    os.chdir(root_dir)
    return datafiles

def get_status():
    return netStatus

def set_status(s):
    global netStatus
    netStatus = s
    return netStatus

@app.route('/status')
def statusCmd():
    return 'ready' if get_status() == FREE else 'busy'

@app.route('/run/<uuid>', methods=['POST'])
def runCmd(uuid):
    if get_status() == FREE:
        config = request.json
        print(config)
        set_status(BUSY)
        print(get_status())
        datafiles = prepare_OV_TCGA_datafiles(config)
        MDBN.run(config, datafiles, verbose)
        set_status(FREE)
    return uuid

def main(argv, batch_dir_prefix='OV_Batch', config_filename='ov_config.json'):
    daemonized, port, config_filename, verbose = \
        MDBN.init(argv, batch_dir_prefix, config_filename)

    print (batch_output_dir)
    if not daemonized:
        with open('%s' % config_filename) as config_file:
            config = json.load(config_file)
            datafiles = prepare_OV_TCGA_datafiles(config)
            MDBN.run(config, datafiles, verbose)
    else:
        app.run(port=port, debug=False, threaded= True)

if __name__ == '__main__':
    main(sys.argv[1:])<commit_msg>Changed download location.<commit_after>import os
import sys
import zipfile
import urllib
import json
import requests

from flask import Flask
from flask import request

app = Flask('cloud-mdbn')

import MDBN

batch_output_dir = ''
batch_start_date_str = ''
BUSY = 10
FREE = 0
netStatus = FREE
verbose = False

def prepare_OV_TCGA_datafiles(config, datadir='data'):
    base_url = 'https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3479191/bin/'
    archive = 'supp_gks725_nar-00961-n-2012-File005.zip'

    datafiles = dict()
    for key in config["pathways"]:
        datafiles[key] = config['dbns'][key]["datafile"]

    if not os.path.isdir(datadir):
        os.mkdir(datadir)

    root_dir = os.getcwd()
    os.chdir(datadir)
    for name, datafile in datafiles.iteritems():
        if not os.path.isfile(datafile):
            if not os.path.isfile(archive):
                print('Downloading TCGA_Data from ' + base_url)
                testfile = urllib.URLopener()
                testfile.retrieve(base_url + archive, archive)
            zipfile.ZipFile(archive, 'r').extract(datafile)
    os.chdir(root_dir)
    return datafiles

def get_status():
    return netStatus

def set_status(s):
    global netStatus
    netStatus = s
    return netStatus

@app.route('/status')
def statusCmd():
    return 'ready' if get_status() == FREE else 'busy'

@app.route('/run/<uuid>', methods=['POST'])
def runCmd(uuid):
    if get_status() == FREE:
        config = request.json
        print(config)
        set_status(BUSY)
        print(get_status())
        datafiles = prepare_OV_TCGA_datafiles(config)
        MDBN.run(config, datafiles, verbose)
        set_status(FREE)
    return uuid

def main(argv, batch_dir_prefix='OV_Batch', config_filename='ov_config.json'):
    daemonized, port, config_filename, verbose = \
        MDBN.init(argv, batch_dir_prefix, config_filename)

    print (batch_output_dir)
    if not daemonized:
        with open('%s' % config_filename) as config_file:
            config = json.load(config_file)
            datafiles = prepare_OV_TCGA_datafiles(config)
            MDBN.run(config, datafiles, verbose)
    else:
        app.run(port=port, debug=False, threaded= True)

if __name__ == '__main__':
    main(sys.argv[1:])<|endoftext|>


<commit_before>"""High level parallel chip-seq analysis
"""
import os
import copy
import toolz as tz

import subprocess

from bcbio.log import logger
from bcbio import utils
from bcbio.pipeline import config_utils
from bcbio.pipeline import datadict as dd
from bcbio.provenance import do
from bcbio.distributed.transaction import file_transaction

def get_callers():
    """Get functions related to each caller"""
    from bcbio.chipseq import macs2
    return {"macs2": macs2.run}

def peakcall_prepare(data, run_parallel):
    """Entry point for doing peak calling"""
    caller_fns = get_callers()
    to_process = []
    for sample in data:
        mimic = copy.copy(sample[0])
        callers = dd.get_peakcaller(sample[0])
        if not isinstance(callers, list):
            callers = [callers]
        for caller in callers:
            if caller in caller_fns:
                mimic["peak_fn"] = caller
                name = dd.get_sample_name(mimic)
                mimic = _check(mimic, data)
                if mimic:
                    to_process.append(mimic)
                else:
                    logger.info("Skipping peak calling. No input sample for %s" % name)
    if to_process:
        after_process = run_parallel("peakcalling", to_process)
        data = _sync(data, after_process)
    return data

def calling(data):
    """Main function to parallelize peak calling."""
    chip_bam = dd.get_work_bam(data)
    input_bam = data.get("work_bam_input", None)
    caller_fn = get_callers()[data["peak_fn"]]
    name = dd.get_sample_name(data)
    out_dir = utils.safe_makedir(os.path.join(dd.get_work_dir(data), data["peak_fn"], name))
    encode_bed = tz.get_in(["genome_resources", "variation", "encode_blacklist"], data)
    # lcr_bed = utils.get_in(data, ("genome_resources", "variation", "lcr"))
    if encode_bed:
        chip_bam = _prepare_bam(chip_bam, encode_bed, data['config'])
        data["work_bam_filter"] = chip_bam
        input_bam = _prepare_bam(input_bam, encode_bed, data['config'])
        data["input_bam_filter"] = input_bam
    out_files = caller_fn(name, chip_bam, input_bam, dd.get_genome_build(data), out_dir,
                          dd.get_chip_method(data), data["resources"], data["config"])
    greylistdir = greylisting(data)
    data.update({"peaks_files": out_files})
    if greylistdir:
        data["greylist"] = greylistdir
    return [[data]]
    data["input_bam_filter"] = input_bam

def _prepare_bam(bam_file, bed_file, config):
    """Remove regions from bed files"""
    if not bam_file or not bed_file:
        return bam_file
    out_file = utils.append_stem(bam_file, '_filter')
    bedtools = config_utils.get_program("bedtools", config)
    if not utils.file_exists(out_file):
        with file_transaction(out_file) as tx_out:
            cmd = "{bedtools} subtract -nonamecheck -A -a {bam_file} -b {bed_file} > {tx_out}"
            do.run(cmd.format(**locals()), "Clean %s" % bam_file)
    return out_file

def _sync(original, processed):
    """
    Add output to data if run sucessfully.
    For now only macs2 is available, so no need
    to consider multiple callers.
    """
    for original_sample in original:
        original_sample[0]["peaks_files"] = {}
        for process_sample in processed:
            if dd.get_sample_name(original_sample[0]) == dd.get_sample_name(process_sample[0]):
                for key in ["peaks_file", "work_bam_filter", "input_bam_filter"]:
                    if process_sample[0].get(key):
                        original_sample[0][key] = process_sample[0][key]
    return original

def _check(sample, data):
    """Get input sample for each chip bam file."""
    if dd.get_chip_method(sample).lower() == "atac":
        return [sample]
    if dd.get_phenotype(sample) == "input":
        return None
    for origin in data:
        if dd.get_batch(sample) in (dd.get_batches(origin[0]) or []) and dd.get_phenotype(origin[0]) == "input":
            sample["work_bam_input"] = dd.get_work_bam(origin[0])
            return [sample]
    return [sample]

def _get_multiplier(samples):
    """Get multiplier to get jobs
       only for samples that have input
    """
    to_process = 1.0
    to_skip = 0
    for sample in samples:
        if dd.get_phenotype(sample[0]) == "chip":
            to_process += 1.0
        elif dd.get_chip_method(sample[0]).lower() == "atac":
            to_process += 1.0
        else:
            to_skip += 1.0
    mult = (to_process - to_skip) / len(samples)
    if mult <= 0:
        mult = 1 / len(samples)
    return max(mult, 1)

def greylisting(data):
    """
    Run ChIP-seq greylisting
    """
    input_bam = data.get("work_bam_input", None)
    if not input_bam:
        logger.info("No input BAM file detected, skipping greylisting.")
        return None
    try:
        greylister = config_utils.get_program("chipseq-greylist", data)
    except config_utils.CmdNotFound:
        logger.info("No greylister found, skipping greylisting.")
        return None
    greylistdir = os.path.join(os.path.dirname(input_bam), "greylist")
    if os.path.exists(greylistdir):
        return greylistdir
    cmd = "{greylister} --outdir {txgreylistdir} {input_bam}"
    message = "Running greylisting on %s." % input_bam
    with file_transaction(greylistdir) as txgreylistdir:
        utils.safe_makedir(txgreylistdir)
        try:
            do.run(cmd.format(**locals()), message)
        except subprocess.CalledProcessError as msg:
            if str(msg).find("Cannot take a larger sample than population when 'replace=False'") >= 0:
                logger.info("Skipping chipseq greylisting because of small sample size: %s"
                            % dd.get_sample_name(data))
                return None
    return greylistdir
<commit_msg>CHIP:Fix missing macs2 folder in final<commit_after>"""High level parallel chip-seq analysis
"""
import os
import copy
import toolz as tz

import subprocess

from bcbio.log import logger
from bcbio import utils
from bcbio.pipeline import config_utils
from bcbio.pipeline import datadict as dd
from bcbio.provenance import do
from bcbio.distributed.transaction import file_transaction

def get_callers():
    """Get functions related to each caller"""
    from bcbio.chipseq import macs2
    return {"macs2": macs2.run}

def peakcall_prepare(data, run_parallel):
    """Entry point for doing peak calling"""
    caller_fns = get_callers()
    to_process = []
    for sample in data:
        mimic = copy.copy(sample[0])
        callers = dd.get_peakcaller(sample[0])
        if not isinstance(callers, list):
            callers = [callers]
        for caller in callers:
            if caller in caller_fns:
                mimic["peak_fn"] = caller
                name = dd.get_sample_name(mimic)
                mimic = _check(mimic, data)
                if mimic:
                    to_process.append(mimic)
                else:
                    logger.info("Skipping peak calling. No input sample for %s" % name)
    if to_process:
        after_process = run_parallel("peakcalling", to_process)
        data = _sync(data, after_process)
    return data

def calling(data):
    """Main function to parallelize peak calling."""
    chip_bam = dd.get_work_bam(data)
    input_bam = data.get("work_bam_input", None)
    caller_fn = get_callers()[data["peak_fn"]]
    name = dd.get_sample_name(data)
    out_dir = utils.safe_makedir(os.path.join(dd.get_work_dir(data), data["peak_fn"], name))
    encode_bed = tz.get_in(["genome_resources", "variation", "encode_blacklist"], data)
    # lcr_bed = utils.get_in(data, ("genome_resources", "variation", "lcr"))
    if encode_bed:
        chip_bam = _prepare_bam(chip_bam, encode_bed, data['config'])
        data["work_bam_filter"] = chip_bam
        input_bam = _prepare_bam(input_bam, encode_bed, data['config'])
        data["input_bam_filter"] = input_bam
    out_files = caller_fn(name, chip_bam, input_bam, dd.get_genome_build(data), out_dir,
                          dd.get_chip_method(data), data["resources"], data["config"])
    greylistdir = greylisting(data)
    data.update({"peaks_files": out_files})
    data["input_bam_filter"] = input_bam
    if greylistdir:
        data["greylist"] = greylistdir
    return [[data]]

def _prepare_bam(bam_file, bed_file, config):
    """Remove regions from bed files"""
    if not bam_file or not bed_file:
        return bam_file
    out_file = utils.append_stem(bam_file, '_filter')
    bedtools = config_utils.get_program("bedtools", config)
    if not utils.file_exists(out_file):
        with file_transaction(out_file) as tx_out:
            cmd = "{bedtools} subtract -nonamecheck -A -a {bam_file} -b {bed_file} > {tx_out}"
            do.run(cmd.format(**locals()), "Clean %s" % bam_file)
    return out_file

def _sync(original, processed):
    """
    Add output to data if run sucessfully.
    For now only macs2 is available, so no need
    to consider multiple callers.
    """
    for original_sample in original:
        original_sample[0]["peaks_files"] = {}
        for process_sample in processed:
            if dd.get_sample_name(original_sample[0]) == dd.get_sample_name(process_sample[0]):
                for key in ["peaks_files", "work_bam_filter", "input_bam_filter"]:
                    if process_sample[0].get(key):
                        original_sample[0][key] = process_sample[0][key]
    return original

def _check(sample, data):
    """Get input sample for each chip bam file."""
    if dd.get_chip_method(sample).lower() == "atac":
        return [sample]
    if dd.get_phenotype(sample) == "input":
        return None
    for origin in data:
        if dd.get_batch(sample) in (dd.get_batches(origin[0]) or []) and dd.get_phenotype(origin[0]) == "input":
            sample["work_bam_input"] = dd.get_work_bam(origin[0])
            return [sample]
    return [sample]

def _get_multiplier(samples):
    """Get multiplier to get jobs
       only for samples that have input
    """
    to_process = 1.0
    to_skip = 0
    for sample in samples:
        if dd.get_phenotype(sample[0]) == "chip":
            to_process += 1.0
        elif dd.get_chip_method(sample[0]).lower() == "atac":
            to_process += 1.0
        else:
            to_skip += 1.0
    mult = (to_process - to_skip) / len(samples)
    if mult <= 0:
        mult = 1 / len(samples)
    return max(mult, 1)

def greylisting(data):
    """
    Run ChIP-seq greylisting
    """
    input_bam = data.get("work_bam_input", None)
    if not input_bam:
        logger.info("No input BAM file detected, skipping greylisting.")
        return None
    try:
        greylister = config_utils.get_program("chipseq-greylist", data)
    except config_utils.CmdNotFound:
        logger.info("No greylister found, skipping greylisting.")
        return None
    greylistdir = os.path.join(os.path.dirname(input_bam), "greylist")
    if os.path.exists(greylistdir):
        return greylistdir
    cmd = "{greylister} --outdir {txgreylistdir} {input_bam}"
    message = "Running greylisting on %s." % input_bam
    with file_transaction(greylistdir) as txgreylistdir:
        utils.safe_makedir(txgreylistdir)
        try:
            do.run(cmd.format(**locals()), message)
        except subprocess.CalledProcessError as msg:
            if str(msg).find("Cannot take a larger sample than population when 'replace=False'") >= 0:
                logger.info("Skipping chipseq greylisting because of small sample size: %s"
                            % dd.get_sample_name(data))
                return None
    return greylistdir
<|endoftext|>


<commit_before>"""
Forms and validation code for user registration.

"""

from django.contrib.auth.models import User
from django import forms
from django.utils.translation import ugettext_lazy as _

from registration.models import RegistrationProfile

# I put this on all required fields, because it's easier to pick up
# on them with CSS or JavaScript if they have a class of "required"
# in the HTML. Your mileage may vary. If/when Django ticket #3515
# lands in trunk, this will no longer be necessary.
attrs_dict = { 'class': 'required' }

class RegistrationForm(forms.Form):
    """
    Form for registering a new user account.

    Validates that the requested username is not already in use, and
    requires the password to be entered twice to catch typos.

    Subclasses should feel free to add any additional validation they
    need, but should avoid defining a ``save()`` method -- the actual
    saving of collected user data is delegated to the active
    registration backend.

    """
    username = forms.RegexField(regex=r'^\w+$',
                                max_length=30,
                                widget=forms.TextInput(attrs=attrs_dict),
                                label=_(u'username'),
                                error_messages={ 'invalid': _(u'This value must contain only letters, numbers and underscores.') })
    email = forms.EmailField(widget=forms.TextInput(attrs=dict(attrs_dict,
                                                               maxlength=75)),
                             label=_(u'email address'))
    password1 = forms.CharField(widget=forms.PasswordInput(attrs=attrs_dict, render_value=False),
                                label=_(u'password'))
    password2 = forms.CharField(widget=forms.PasswordInput(attrs=attrs_dict, render_value=False),
                                label=_(u'password (again)'))

    def clean_username(self):
        """
        Validate that the username is alphanumeric and is not already
        in use.

        """
        try:
            user = User.objects.get(username__iexact=self.cleaned_data['username'])
        except User.DoesNotExist:
            return self.cleaned_data['username']
        raise forms.ValidationError(_(u'This username is already taken. Please choose another.'))

    def clean(self):
        """
        Verifiy that the values entered into the two password fields
        match. Note that an error here will end up in
        ``non_field_errors()`` because it doesn't apply to a single
        field.

        """
        if 'password1' in self.cleaned_data and 'password2' in self.cleaned_data:
            if self.cleaned_data['password1'] != self.cleaned_data['password2']:
                raise forms.ValidationError(_(u'You must type the same password each time'))
        return self.cleaned_data

class RegistrationFormTermsOfService(RegistrationForm):
    """
    Subclass of ``RegistrationForm`` which adds a required checkbox
    for agreeing to a site's Terms of Service.

    """
    tos = forms.BooleanField(widget=forms.CheckboxInput(attrs=attrs_dict),
                             label=_(u'I have read and agree to the Terms of Service'),
                             error_messages={ 'required': _(u"You must agree to the terms to register") })

class RegistrationFormUniqueEmail(RegistrationForm):
    """
    Subclass of ``RegistrationForm`` which enforces uniqueness of
    email addresses.

    """
    def clean_email(self):
        """
        Validate that the supplied email address is unique for the
        site.

        """
        if User.objects.filter(email__iexact=self.cleaned_data['email']):
            raise forms.ValidationError(_(u'This email address is already in use. Please supply a different email address.'))
        return self.cleaned_data['email']

class RegistrationFormNoFreeEmail(RegistrationForm):
    """
    Subclass of ``RegistrationForm`` which disallows registration with
    email addresses from popular free webmail services; moderately
    useful for preventing automated spam registrations.

    To change the list of banned domains, subclass this form and
    override the attribute ``bad_domains``.

    """
    bad_domains = ['aim.com', 'aol.com', 'email.com', 'gmail.com',
                   'googlemail.com', 'hotmail.com', 'hushmail.com',
                   'msn.com', 'mail.ru', 'mailinator.com', 'live.com']

    def clean_email(self):
        """
        Check the supplied email address against a list of known free
        webmail domains.

        """
        email_domain = self.cleaned_data['email'].split('@')[1]
        if email_domain in self.bad_domains:
            raise forms.ValidationError(_(u'Registration using free email addresses is prohibited. Please supply a different email address.'))
        return self.cleaned_data['email']
<commit_msg>Make the list of disallowed free-email domains match the docs.<commit_after>"""
Forms and validation code for user registration.

"""

from django.contrib.auth.models import User
from django import forms
from django.utils.translation import ugettext_lazy as _

from registration.models import RegistrationProfile

# I put this on all required fields, because it's easier to pick up
# on them with CSS or JavaScript if they have a class of "required"
# in the HTML. Your mileage may vary. If/when Django ticket #3515
# lands in trunk, this will no longer be necessary.
attrs_dict = { 'class': 'required' }

class RegistrationForm(forms.Form):
    """
    Form for registering a new user account.

    Validates that the requested username is not already in use, and
    requires the password to be entered twice to catch typos.

    Subclasses should feel free to add any additional validation they
    need, but should avoid defining a ``save()`` method -- the actual
    saving of collected user data is delegated to the active
    registration backend.

    """
    username = forms.RegexField(regex=r'^\w+$',
                                max_length=30,
                                widget=forms.TextInput(attrs=attrs_dict),
                                label=_(u'username'),
                                error_messages={ 'invalid': _(u'This value must contain only letters, numbers and underscores.') })
    email = forms.EmailField(widget=forms.TextInput(attrs=dict(attrs_dict,
                                                               maxlength=75)),
                             label=_(u'email address'))
    password1 = forms.CharField(widget=forms.PasswordInput(attrs=attrs_dict, render_value=False),
                                label=_(u'password'))
    password2 = forms.CharField(widget=forms.PasswordInput(attrs=attrs_dict, render_value=False),
                                label=_(u'password (again)'))

    def clean_username(self):
        """
        Validate that the username is alphanumeric and is not already
        in use.

        """
        try:
            user = User.objects.get(username__iexact=self.cleaned_data['username'])
        except User.DoesNotExist:
            return self.cleaned_data['username']
        raise forms.ValidationError(_(u'This username is already taken. Please choose another.'))

    def clean(self):
        """
        Verifiy that the values entered into the two password fields
        match. Note that an error here will end up in
        ``non_field_errors()`` because it doesn't apply to a single
        field.

        """
        if 'password1' in self.cleaned_data and 'password2' in self.cleaned_data:
            if self.cleaned_data['password1'] != self.cleaned_data['password2']:
                raise forms.ValidationError(_(u'You must type the same password each time'))
        return self.cleaned_data

class RegistrationFormTermsOfService(RegistrationForm):
    """
    Subclass of ``RegistrationForm`` which adds a required checkbox
    for agreeing to a site's Terms of Service.

    """
    tos = forms.BooleanField(widget=forms.CheckboxInput(attrs=attrs_dict),
                             label=_(u'I have read and agree to the Terms of Service'),
                             error_messages={ 'required': _(u"You must agree to the terms to register") })

class RegistrationFormUniqueEmail(RegistrationForm):
    """
    Subclass of ``RegistrationForm`` which enforces uniqueness of
    email addresses.

    """
    def clean_email(self):
        """
        Validate that the supplied email address is unique for the
        site.

        """
        if User.objects.filter(email__iexact=self.cleaned_data['email']):
            raise forms.ValidationError(_(u'This email address is already in use. Please supply a different email address.'))
        return self.cleaned_data['email']

class RegistrationFormNoFreeEmail(RegistrationForm):
    """
    Subclass of ``RegistrationForm`` which disallows registration with
    email addresses from popular free webmail services; moderately
    useful for preventing automated spam registrations.

    To change the list of banned domains, subclass this form and
    override the attribute ``bad_domains``.

    """
    bad_domains = ['aim.com', 'aol.com', 'email.com', 'gmail.com',
                   'googlemail.com', 'hotmail.com', 'hushmail.com',
                   'msn.com', 'mail.ru', 'mailinator.com', 'live.com',
                   'yahoo.com']

    def clean_email(self):
        """
        Check the supplied email address against a list of known free
        webmail domains.

        """
        email_domain = self.cleaned_data['email'].split('@')[1]
        if email_domain in self.bad_domains:
            raise forms.ValidationError(_(u'Registration using free email addresses is prohibited. Please supply a different email address.'))
        return self.cleaned_data['email']
<|endoftext|>


<commit_before>"""
Common functions.
"""
import datetime
import logging
import struct
from xml.dom import minidom
from xml.etree import cElementTree

import numpy as np

from .constants import DEFAULT_DATE

def _demo_logger(self, msg, *args, **kwargs):
    """A custom logger for :class:`~msl.equipment.connection_demo.ConnectionDemo` objects."""
    if self.isEnabledFor(logging.DEMO):
        self._log(logging.DEMO, msg, args, **kwargs)

logger = logging.getLogger(__package__)

# create a demo logger level between INFO and WARNING
logging.DEMO = logging.INFO + 5
logging.addLevelName(logging.DEMO, 'DEMO')
logging.Logger.demo = _demo_logger

def convert_to_enum(obj, enum, prefix=None, to_upper=False, strict=True):
    """Convert `obj` to an :class:`~enum.Enum` member.

    Parameters
    ----------
    obj : :class:`object`
        Any object to be converted to the specified `enum`. Can be a
        value of member of the specified `enum`.
    enum : Type[:class:`~enum.Enum`]
        The :class:`~enum.Enum` object that `obj` should be converted to.
    prefix : :class:`str`, optional
        If `obj` is a :class:`str`, then ensures that `prefix` is included at
        the beginning of `obj` before converting `obj` to the `enum`.
    to_upper : :class:`bool`, optional
        If `obj` is a :class:`str`, then whether to change `obj` to
        be upper case before converting `obj` to the `enum`.
    strict : :class:`bool`, optional
        Whether errors should be raised. If :data:`False` and `obj` cannot
        be converted to `enum` then `obj` is returned and the error is
        logged.

    Returns
    -------
    :class:`~enum.Enum`
        The `enum` member.

    Raises
    ------
    ValueError
        If `obj` is not in `enum` and `strict` is :data:`True`.
    """
    try:
        return enum(obj)
    except ValueError:
        pass

    # then `obj` must the Enum member name
    name = '{}'.format(obj).replace(' ', '_')
    if prefix and not name.startswith(prefix):
        name = prefix + name
    if to_upper:
        name = name.upper()

    try:
        return enum[name]
    except KeyError:
        pass

    msg = 'Cannot create {} from {!r}'.format(enum, obj)
    if strict:
        raise ValueError(msg)

    logger.error(msg)
    return obj

def convert_to_primitive(text):
    """Convert text into a primitive value.

    Parameters
    ----------
    text : :class:`str` or :class:`bytes`
        The text to convert.

    Returns
    -------
    The `text` as a :data:`None`, :class:`bool`, :class:`int`,
    :class:`float` or :class:`complex` object. Returns the
    original `text` if it cannot be converted to any of these types.
    The text 0 and 1 get converted to an integer not a boolean.
    """
    try:
        upper = text.upper().strip()
    except AttributeError:
        return text

    if upper == 'NONE':
        return None
    if upper == 'TRUE':
        return True
    if upper == 'FALSE':
        return False

    # TODO could consider using ast.literal_eval if text representations of
    #  list, dict, set, tuple would like to be supported
    for typ in (int, float, complex):
        try:
            return typ(text)
        except (ValueError, TypeError):
            pass

    return text

def convert_to_date(obj, fmt='%Y-%m-%d', strict=True):
    """Convert an object to a :class:`datetime.date` object.

    Parameters
    ----------
    obj : :class:`datetime.date`, :class:`datetime.datetime` or :class:`str`
        Any object that can be converted to a :class:`datetime.date` object.
    fmt : :class:`str`
        If `obj` is a :class:`str` then the format to use to convert `obj` to a
        :class:`datetime.date`.
    strict : :class:`bool`, optional
        Whether errors should be raised. If :data:`False` and `obj` cannot
        be converted to :class:`datetime.date` then ``datetime.date(datetime.MINYEAR, 1, 1)``
        is returned and the error is logged.

    Returns
    -------
    :class:`datetime.date`
        A :class:`datetime.date` object.
    """
    if isinstance(obj, datetime.date):
        return obj

    if isinstance(obj, datetime.datetime):
        return obj.date()

    if obj is None:
        return DEFAULT_DATE

    try:
        return datetime.datetime.strptime(obj, fmt).date()
    except (ValueError, TypeError) as e:
        if strict:
            raise
        else:
            logger.error(e)
            return DEFAULT_DATE

def convert_to_xml_string(element, indent='  ', encoding='utf-8', fix_newlines=True):
    """Convert an XML :class:`~xml.etree.ElementTree.Element` in to a string
    with proper indentation.

    Parameters
    ----------
    element : :class:`~xml.etree.ElementTree.Element`
        The element to convert.
    indent : :class:`str`, optional
        The value to use for the indentation.
    encoding : :class:`str`, optional
        The encoding to use.
    fix_newlines : :class:`bool`, optional
        Whether to remove newlines inside text nodes.

    Returns
    -------
    :class:`str`
        The `element` as a pretty string. The returned value can be
        directly written to a file (i.e., it includes the XML declaration).

    Examples
    --------
    If the :class:`~xml.etree.ElementTree.Element` contains unicode
    characters then you should use the :mod:`codecs` module to create the file
    if you are using Python 2.7::

        import codecs
        with codecs.open('my_file.xml', mode='w', encoding='utf-8') as fp:
            fp.write(convert_to_xml_string(element))

    otherwise you can use the builtin :func:`open` function::

        with open('my_file.xml', mode='w', encoding='utf-8') as fp:
            fp.write(convert_to_xml_string(element))

    """
    parsed = minidom.parseString(cElementTree.tostring(element))
    pretty = parsed.toprettyxml(indent=indent, encoding=encoding).decode(encoding)
    if fix_newlines:
        return '\n'.join(s for s in pretty.splitlines() if s.strip())
    return pretty

def xml_element(tag, text=None, tail=None, **attributes):
    """Create a new XML element.

    Parameters
    ----------
    tag : :class:`str`
        The element's name.
    text : :class:`str`, optional
        The text before the first sub-element. Can either be a string or :data:`None`.
    tail : :class:`str`, optional
        The text after this element's end tag, but before the next sibling element's start tag.
    attributes
        All additional key-value pairs are included as XML attributes
        for the element. The value must be of type :class:`str`.

    Returns
    -------
    :class:`~xml.etree.ElementTree.Element`
        The new XML element.
    """
    element = cElementTree.Element(tag, **attributes)
    element.text = text
    element.tail = tail
    return element

def xml_comment(text):
    """Create a new XML comment element.

    Parameters
    ----------
    text : :class:`str`
        The comment.

    Returns
    -------
    :func:`~xml.etree.ElementTree.Comment`
        A special element that is an XML comment.
    """
    return cElementTree.Comment(text)

def to_bytes(iterable, fmt='ieee', dtype='<f'):
    """Convert an iterable of numbers into bytes.

    .. _IEEE 488.2-1992: https://standards.ieee.org/ieee/488.2/718/
    .. _HP 8530A: https://www.keysight.com/us/en/product/8530A/microwave-receiver.html#resources

    Parameters
    ----------
    iterable
        An object to convert to bytes. Must be a 1-dimensional sequence of elements
        (not a multidimensional array).
    fmt : :class:`str` or :data:`None`, optional
        The format to use to convert `iterable`. Possible values are:

        * ``''`` (empty string or :data:`None`) -- convert `iterable` to bytes
          without a header.
        * ``'ascii'`` -- comma-separated ASCII characters, see the
          `<PROGRAM DATA SEPARATOR>` standard that is defined in
          Section 7.4.2.2, `IEEE 488.2-1992`_.
        * ``'ieee'`` -- arbitrary block data for `SCPI` messages, see the
          `<DEFINITE LENGTH ARBITRARY BLOCK RESPONSE DATA>` standard that
          is defined in Section 8.7.9, `IEEE 488.2-1992`_.
          Syntax, ``#<length of num bytes value><num bytes><byte><byte><byte>...``.
        * ``'hp'`` -- the HP-IB data transfer standard, i.e., the `FORM#` command
          option. See the programming guide for an `HP 8530A`_ for more details.
          Syntax, ``#A<num bytes as uint16><byte><byte><byte>...``.

    dtype : :class:`str` or :class:`numpy.number`, optional
        The data type to cast each element in `iterable` to. If `fmt` is ``'ascii'``
        then `dtype` is used in :func:`format` to first convert each element
        in `iterable` to a string, and then it is encoded (e.g., ``'.2e'`` converts
        each element to scientific notation with two digits after the decimal point).
        If `dtype` includes a byte-order character, it is ignored. For all other
        values of `fmt`, the `dtype` can be a C-type or Python-type specification,
        for example, ``'H'``, ``'ushort'`` and ``numpy.uint16`` are equivalent
        specifications to cast each element to an `unsigned short`. If a byte-order
        character is specified then it is used, otherwise the native byte order
        of the CPU architecture is used.

    Returns
    -------
    :class:`bytes`
        The `iterable` converted to bytes.
    """
    if fmt == 'ascii':
        format_spec = dtype.lstrip('@=<>!')
        return ','.join(format(item, format_spec) for item in iterable).encode('ascii')

    if isinstance(iterable, np.ndarray):
        array = iterable.astype(dtype=dtype)
    else:
        array = np.fromiter(iterable, dtype=dtype, count=len(iterable))

    if fmt == 'ieee':
        nbytes = str(array.nbytes)
        len_nbytes = len(nbytes)
        if len_nbytes > 9:
            # The IEEE-488.2 format allows for 1 digit to specify the number
            # of bytes in the array. This is extremely unlikely to occur in
            # practice since the instrument would require > 1GB of memory.
            raise OverflowError('length too big for IEEE-488.2 specification')
        return '#{}{}'.format(len_nbytes, nbytes).encode() + array.tobytes()

    if fmt == 'hp':
        byteorder = array.dtype.byteorder
        if byteorder == '|':
            # | means not applicable for the dtype specified, assign little endian
            # this redefinition is also declared in from_bytes()
            byteorder = '<'
        return b'#A' + struct.pack(byteorder + 'H', array.nbytes) + array.tobytes()

    if not fmt:
        return array.tobytes()

    raise ValueError("Invalid format {!r} -- must be 'ascii', 'ieee', 'hp' or None".format(fmt))

def from_bytes(buffer, fmt='ieee', dtype='<f'):
    """Convert bytes into an array.

    Parameters
    ----------
    buffer : :class:`bytes` or :class:`bytearray`
        A byte buffer.
    fmt : :class:`str` or :data:`None`, optional
        The format that `buffer` is in.
        See :func:`.to_bytes` for more details.
    dtype : :class:`str` or :class:`numpy.number`, optional
        The data type of each element in `buffer`.
        See :func:`.to_bytes` for more details.

    Returns
    -------
    :class:`numpy.ndarray`
        The array.
    """
    if fmt == 'ieee':
        offset = buffer.find(b'#')
        if offset == -1:
            raise ValueError('Invalid IEEE-488.2 format, '
                             'cannot find # character')

        try:
            len_nbytes = int(buffer[offset+1:offset+2])
        except ValueError:
            len_nbytes = None

        if len_nbytes is None:
            raise ValueError('Invalid IEEE-488.2 format, '
                             'character after # is not an integer')

        if len_nbytes == 0:
            # <INDEFINITE LENGTH ARBITRARY BLOCK RESPONSE DATA>
            # Section 8.7.10, IEEE 488.2-1992
            nbytes = len(buffer) - offset

            # The standard states that the buffer must end in a NL (\n) character.
            # Not sure if a manufacturer will always honor this requirement, or if
            # it has already been stripped from the buffer
            if buffer.endswith(b'\n'):
                nbytes -= 1
        else:
            # <DEFINITE LENGTH ARBITRARY BLOCK RESPONSE DATA>
            # Section 8.7.9, IEEE 488.2-1992
            try:
                nbytes = int(buffer[offset + 2:offset + 2 + len_nbytes])
            except ValueError:
                nbytes = None

        if nbytes is None:
            raise ValueError('Invalid IEEE-488.2 format, '
                             'characters after #{} are not integers'.format(len_nbytes))

        dtype = np.dtype(dtype)
        offset += 2 + len_nbytes
        count = nbytes // dtype.itemsize
        return np.frombuffer(buffer, dtype=dtype, count=count, offset=offset)

    if fmt == 'hp':
        offset = buffer.find(b'#A')
        if offset == -1:
            raise ValueError('Invalid HP format, cannot find #A character')

        dtype = np.dtype(dtype)
        i, j = offset + 2, offset + 4

        byteorder = dtype.byteorder
        if byteorder == '|':
            # | means not applicable for the dtype specified, assign little endian
            # this redefinition is also declared in to_bytes()
            byteorder = '<'

        try:
            nbytes, = struct.unpack(byteorder + 'H', buffer[i:j])
        except struct.error:
            nbytes = None

        if nbytes is None:
            raise ValueError('Invalid HP format, '
                             'characters after #A are not an unsigned short integer')

        count = nbytes // dtype.itemsize
        return np.frombuffer(buffer, dtype=dtype, count=count, offset=j)

    if fmt == 'ascii':
        if isinstance(buffer, (bytes, bytearray)):
            buffer = buffer.decode('ascii')
        return np.fromstring(buffer, dtype=dtype, sep=',')

    if not fmt:
        return np.frombuffer(buffer, dtype=dtype)

    raise ValueError("Invalid format {!r} -- must be 'ascii', 'ieee', 'hp' or None".format(fmt))
<commit_msg>edit the docstrings of to_bytes and from_bytes<commit_after>"""
Common functions.
"""
import datetime
import logging
import struct
from xml.dom import minidom
from xml.etree import cElementTree

import numpy as np

from .constants import DEFAULT_DATE

def _demo_logger(self, msg, *args, **kwargs):
    """A custom logger for :class:`~msl.equipment.connection_demo.ConnectionDemo` objects."""
    if self.isEnabledFor(logging.DEMO):
        self._log(logging.DEMO, msg, args, **kwargs)

logger = logging.getLogger(__package__)

# create a demo logger level between INFO and WARNING
logging.DEMO = logging.INFO + 5
logging.addLevelName(logging.DEMO, 'DEMO')
logging.Logger.demo = _demo_logger

def convert_to_enum(obj, enum, prefix=None, to_upper=False, strict=True):
    """Convert `obj` to an :class:`~enum.Enum` member.

    Parameters
    ----------
    obj : :class:`object`
        Any object to be converted to the specified `enum`. Can be a
        value of member of the specified `enum`.
    enum : Type[:class:`~enum.Enum`]
        The :class:`~enum.Enum` object that `obj` should be converted to.
    prefix : :class:`str`, optional
        If `obj` is a :class:`str`, then ensures that `prefix` is included at
        the beginning of `obj` before converting `obj` to the `enum`.
    to_upper : :class:`bool`, optional
        If `obj` is a :class:`str`, then whether to change `obj` to
        be upper case before converting `obj` to the `enum`.
    strict : :class:`bool`, optional
        Whether errors should be raised. If :data:`False` and `obj` cannot
        be converted to `enum` then `obj` is returned and the error is
        logged.

    Returns
    -------
    :class:`~enum.Enum`
        The `enum` member.

    Raises
    ------
    ValueError
        If `obj` is not in `enum` and `strict` is :data:`True`.
    """
    try:
        return enum(obj)
    except ValueError:
        pass

    # then `obj` must the Enum member name
    name = '{}'.format(obj).replace(' ', '_')
    if prefix and not name.startswith(prefix):
        name = prefix + name
    if to_upper:
        name = name.upper()

    try:
        return enum[name]
    except KeyError:
        pass

    msg = 'Cannot create {} from {!r}'.format(enum, obj)
    if strict:
        raise ValueError(msg)

    logger.error(msg)
    return obj

def convert_to_primitive(text):
    """Convert text into a primitive value.

    Parameters
    ----------
    text : :class:`str` or :class:`bytes`
        The text to convert.

    Returns
    -------
    The `text` as a :data:`None`, :class:`bool`, :class:`int`,
    :class:`float` or :class:`complex` object. Returns the
    original `text` if it cannot be converted to any of these types.
    The text 0 and 1 get converted to an integer not a boolean.
    """
    try:
        upper = text.upper().strip()
    except AttributeError:
        return text

    if upper == 'NONE':
        return None
    if upper == 'TRUE':
        return True
    if upper == 'FALSE':
        return False

    # TODO could consider using ast.literal_eval if text representations of
    #  list, dict, set, tuple would like to be supported
    for typ in (int, float, complex):
        try:
            return typ(text)
        except (ValueError, TypeError):
            pass

    return text

def convert_to_date(obj, fmt='%Y-%m-%d', strict=True):
    """Convert an object to a :class:`datetime.date` object.

    Parameters
    ----------
    obj : :class:`datetime.date`, :class:`datetime.datetime` or :class:`str`
        Any object that can be converted to a :class:`datetime.date` object.
    fmt : :class:`str`
        If `obj` is a :class:`str` then the format to use to convert `obj` to a
        :class:`datetime.date`.
    strict : :class:`bool`, optional
        Whether errors should be raised. If :data:`False` and `obj` cannot
        be converted to :class:`datetime.date` then ``datetime.date(datetime.MINYEAR, 1, 1)``
        is returned and the error is logged.

    Returns
    -------
    :class:`datetime.date`
        A :class:`datetime.date` object.
    """
    if isinstance(obj, datetime.date):
        return obj

    if isinstance(obj, datetime.datetime):
        return obj.date()

    if obj is None:
        return DEFAULT_DATE

    try:
        return datetime.datetime.strptime(obj, fmt).date()
    except (ValueError, TypeError) as e:
        if strict:
            raise
        else:
            logger.error(e)
            return DEFAULT_DATE

def convert_to_xml_string(element, indent='  ', encoding='utf-8', fix_newlines=True):
    """Convert an XML :class:`~xml.etree.ElementTree.Element` in to a string
    with proper indentation.

    Parameters
    ----------
    element : :class:`~xml.etree.ElementTree.Element`
        The element to convert.
    indent : :class:`str`, optional
        The value to use for the indentation.
    encoding : :class:`str`, optional
        The encoding to use.
    fix_newlines : :class:`bool`, optional
        Whether to remove newlines inside text nodes.

    Returns
    -------
    :class:`str`
        The `element` as a pretty string. The returned value can be
        directly written to a file (i.e., it includes the XML declaration).

    Examples
    --------
    If the :class:`~xml.etree.ElementTree.Element` contains unicode
    characters then you should use the :mod:`codecs` module to create the file
    if you are using Python 2.7::

        import codecs
        with codecs.open('my_file.xml', mode='w', encoding='utf-8') as fp:
            fp.write(convert_to_xml_string(element))

    otherwise you can use the builtin :func:`open` function::

        with open('my_file.xml', mode='w', encoding='utf-8') as fp:
            fp.write(convert_to_xml_string(element))

    """
    parsed = minidom.parseString(cElementTree.tostring(element))
    pretty = parsed.toprettyxml(indent=indent, encoding=encoding).decode(encoding)
    if fix_newlines:
        return '\n'.join(s for s in pretty.splitlines() if s.strip())
    return pretty

def xml_element(tag, text=None, tail=None, **attributes):
    """Create a new XML element.

    Parameters
    ----------
    tag : :class:`str`
        The element's name.
    text : :class:`str`, optional
        The text before the first sub-element. Can either be a string or :data:`None`.
    tail : :class:`str`, optional
        The text after this element's end tag, but before the next sibling element's start tag.
    attributes
        All additional key-value pairs are included as XML attributes
        for the element. The value must be of type :class:`str`.

    Returns
    -------
    :class:`~xml.etree.ElementTree.Element`
        The new XML element.
    """
    element = cElementTree.Element(tag, **attributes)
    element.text = text
    element.tail = tail
    return element

def xml_comment(text):
    """Create a new XML comment element.

    Parameters
    ----------
    text : :class:`str`
        The comment.

    Returns
    -------
    :func:`~xml.etree.ElementTree.Comment`
        A special element that is an XML comment.
    """
    return cElementTree.Comment(text)

def to_bytes(iterable, fmt='ieee', dtype='<f'):
    """Convert an iterable of numbers into bytes.

    .. _IEEE 488.2-1992: https://standards.ieee.org/ieee/488.2/718/
    .. _HP 8530A: https://www.keysight.com/us/en/product/8530A/microwave-receiver.html#resources

    Parameters
    ----------
    iterable
        An object to convert to bytes. Must be a 1-dimensional sequence of elements
        (not a multidimensional array).
    fmt : :class:`str` or :data:`None`, optional
        The format to use to convert `iterable`. Possible values are:

        * ``''`` (empty string or :data:`None`) -- convert `iterable` to bytes
          without a header.

          .. centered:: None: ``<byte><byte><byte>...``

        * ``'ascii'`` -- comma-separated ASCII characters, see the
          `<PROGRAM DATA SEPARATOR>` standard that is defined in Section 7.4.2.2,
          `IEEE 488.2-1992`_.

          .. centered:: ascii: ``<string>,<string>,<string>,...``

        * ``'ieee'`` -- arbitrary block data for `SCPI` messages, see the
          `<DEFINITE LENGTH ARBITRARY BLOCK RESPONSE DATA>` standard that
          is defined in Section 8.7.9, `IEEE 488.2-1992`_.

          .. centered:: ieee: ``#<length of num bytes value><num bytes><byte><byte><byte>...``

        * ``'hp'`` -- the HP-IB data transfer standard, i.e., the `FORM#` command
          option. See the programming guide for an `HP 8530A`_ for more details.

          .. centered:: hp: ``#A<num bytes as uint16><byte><byte><byte>...``

    dtype : :class:`str` or :class:`numpy.number`, optional
        The data type to cast each element in `iterable` to. If `fmt` is ``'ascii'``
        then `dtype` is used in :func:`format` to first convert each element
        in `iterable` to a string, and then it is encoded (e.g., ``'.2e'`` converts
        each element to scientific notation with two digits after the decimal point).
        If `dtype` includes a byte-order character, it is ignored. For all other
        values of `fmt`, the `dtype` can be a C-type or Python-type specification,
        for example, ``'H'``, ``'uint16'`` and :class:`numpy.ushort` are equivalent
        specifications to cast each element to an `unsigned short`. If a byte-order
        character is specified then it is used, otherwise the native byte order
        of the CPU architecture is used.

    Returns
    -------
    :class:`bytes`
        The `iterable` converted to bytes.
    """
    if fmt == 'ascii':
        format_spec = dtype.lstrip('@=<>!')
        return ','.join(format(item, format_spec) for item in iterable).encode('ascii')

    if isinstance(iterable, np.ndarray):
        array = iterable.astype(dtype=dtype)
    else:
        array = np.fromiter(iterable, dtype=dtype, count=len(iterable))

    if fmt == 'ieee':
        nbytes = str(array.nbytes)
        len_nbytes = len(nbytes)
        if len_nbytes > 9:
            # The IEEE-488.2 format allows for 1 digit to specify the number
            # of bytes in the array. This is extremely unlikely to occur in
            # practice since the instrument would require > 1GB of memory.
            raise OverflowError('length too big for IEEE-488.2 specification')
        return '#{}{}'.format(len_nbytes, nbytes).encode() + array.tobytes()

    if fmt == 'hp':
        byteorder = array.dtype.byteorder
        if byteorder == '|':
            # | means not applicable for the dtype specified, assign little endian
            # this redefinition is also declared in from_bytes()
            byteorder = '<'
        return b'#A' + struct.pack(byteorder + 'H', array.nbytes) + array.tobytes()

    if not fmt:
        return array.tobytes()

    raise ValueError("Invalid format {!r} -- must be 'ascii', 'ieee', 'hp' or None".format(fmt))

def from_bytes(buffer, fmt='ieee', dtype='<f'):
    """Convert bytes into an array.

    Parameters
    ----------
    buffer : :class:`bytes`, :class:`bytearray` or :class:`str`
        A byte buffer. Can be of type :class:`str`, but only if `fmt`
        equals ``'ascii'``.
    fmt : :class:`str` or :data:`None`, optional
        The format that `buffer` is in.
        See :func:`.to_bytes` for more details.
    dtype : :class:`str` or :class:`numpy.number`, optional
        The data type of each element in `buffer`.
        See :func:`.to_bytes` for more details.

    Returns
    -------
    :class:`numpy.ndarray`
        The array.
    """
    if fmt == 'ieee':
        offset = buffer.find(b'#')
        if offset == -1:
            raise ValueError('Invalid IEEE-488.2 format, '
                             'cannot find # character')

        try:
            len_nbytes = int(buffer[offset+1:offset+2])
        except ValueError:
            len_nbytes = None

        if len_nbytes is None:
            raise ValueError('Invalid IEEE-488.2 format, '
                             'character after # is not an integer')

        if len_nbytes == 0:
            # <INDEFINITE LENGTH ARBITRARY BLOCK RESPONSE DATA>
            # Section 8.7.10, IEEE 488.2-1992
            nbytes = len(buffer) - offset

            # The standard states that the buffer must end in a NL (\n) character.
            # Not sure if a manufacturer will always honor this requirement, or if
            # it has already been stripped from the buffer
            if buffer.endswith(b'\n'):
                nbytes -= 1
        else:
            # <DEFINITE LENGTH ARBITRARY BLOCK RESPONSE DATA>
            # Section 8.7.9, IEEE 488.2-1992
            try:
                nbytes = int(buffer[offset + 2:offset + 2 + len_nbytes])
            except ValueError:
                nbytes = None

        if nbytes is None:
            raise ValueError('Invalid IEEE-488.2 format, '
                             'characters after #{} are not integers'.format(len_nbytes))

        dtype = np.dtype(dtype)
        offset += 2 + len_nbytes
        count = nbytes // dtype.itemsize
        return np.frombuffer(buffer, dtype=dtype, count=count, offset=offset)

    if fmt == 'hp':
        offset = buffer.find(b'#A')
        if offset == -1:
            raise ValueError('Invalid HP format, cannot find #A character')

        dtype = np.dtype(dtype)
        i, j = offset + 2, offset + 4

        byteorder = dtype.byteorder
        if byteorder == '|':
            # | means not applicable for the dtype specified, assign little endian
            # this redefinition is also declared in to_bytes()
            byteorder = '<'

        try:
            nbytes, = struct.unpack(byteorder + 'H', buffer[i:j])
        except struct.error:
            nbytes = None

        if nbytes is None:
            raise ValueError('Invalid HP format, '
                             'characters after #A are not an unsigned short integer')

        count = nbytes // dtype.itemsize
        return np.frombuffer(buffer, dtype=dtype, count=count, offset=j)

    if fmt == 'ascii':
        if isinstance(buffer, (bytes, bytearray)):
            buffer = buffer.decode('ascii')
        return np.fromstring(buffer, dtype=dtype, sep=',')

    if not fmt:
        return np.frombuffer(buffer, dtype=dtype)

    raise ValueError("Invalid format {!r} -- must be 'ascii', 'ieee', 'hp' or None".format(fmt))
<|endoftext|>