<commit_before>#!/usr/bin/env ipy
#

import operator
import os
import sys
import tempfile
import unittest

sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
from excel import *
from test_utils import *

class ExcelWorkbooksUnitTest(unittest.TestCase):

    def test_create(self):
        filename = getTestFileName("test_create.xlsx")

        wb = ExcelApplication().Workbooks(filename)
        wb.Close()
        self.assertTrue(os.path.isfile(filename))

    def test_open(self):
        filename = getTestFileName("test_open.xlsx")

        wb = ExcelApplication().Workbooks(filename)
        wb.Close()
        self.assertTrue(os.path.isfile(filename))

        wb = ExcelApplication().Workbooks.open(filename)
        wb.Close()
        self.assertTrue(os.path.isfile(filename))

class ExcelWorksheetsUnitTest(unittest.TestCase):

    def test_create(self):
        filename = getTestFileName("test_worksheet_create.xlsx")

        wb = ExcelApplication().Workbooks(filename)
        wsheets = wb.Worksheets
        self.assertEqual(wsheets.Count, 3)

        ws = wsheets.create('test')
        self.assertEqual(wsheets.Count, 4)

        wb.Close()

class ExcelWorksheetUnitTest(unittest.TestCase):

    def test_UsedRange(self):
        filename = getTestFileName("test_used_range.xlsx")

        wb = ExcelApplication().Workbooks(filename)
        ws = wb.Worksheets('test')

        ws.Cells(2,3).text = "hello"
        self.assertEqual(ws.Cells(2,3).text, "hello")

        ws.Cells(11,22).text = "world"

        rng = ws.UsedRange
        self.assertEqual(rng.Row, 2)
        self.assertEqual(rng.Column, 3)

        print("Rows: ", rng.Rows.Count)
        print("Columns: ", rng.Columns.Count)

        wb.Close()

    def _Cells(self):
        ws = ExcelApplication().Workbooks('test\\test1.xlsx').Worksheets('test1')
        cell = ws.Cells(4,3)

        self.assertEqual(cell.Row, 4)
        self.assertEqual(cell.Column, 3)
        self.assertEqual(cell.Rows.Count, 1)
        self.assertEqual(cell.Columns.Count, 1)

    def _CellValue(self):
        rng = ExcelApplication().Workbooks('test\\test1.xlsx').Worksheets('test1').UsedRange

        cell = rng.Cells(1,1)
        self.assertEqual(cell.text, "TopLeft")

        cell = rng.Cells(rng.Rows.Count, rng.Columns.Count)
        self.assertEqual(cell.text, "BottomRight")

        cell = rng.Cells(1, rng.Columns.Count)
        cell.text = "TopRight"
        self.assertEqual(cell.text, "TopRight")
        cell.value = None
        self.assertEqual(cell.text, "")

    def _Row_Column(self):
        col = ExcelApplication().Workbooks('test\\test1.xlsx').Worksheets('test1').UsedRange.Columns

########################################################################
# Main
########################################################################

if __name__ == '__main__':
    logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
    unittest.main(verbosity=2)
<commit_msg>added test<commit_after>#!/usr/bin/env ipy
#

import operator
import os
import sys
import tempfile
import unittest

sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
from excel import *
from test_utils import *

class ExcelWorkbooksUnitTest(unittest.TestCase):

    def test_create(self):
        filename = getTestFileName("test_create.xlsx")

        wb = ExcelApplication().Workbooks(filename)
        wb.Close()
        self.assertTrue(os.path.isfile(filename))

    def test_open(self):
        filename = getTestFileName("test_open.xlsx")

        wb = ExcelApplication().Workbooks(filename)
        wb.Close()
        self.assertTrue(os.path.isfile(filename))

        wb = ExcelApplication().Workbooks.open(filename)
        wb.Close()
        self.assertTrue(os.path.isfile(filename))

    def test_multi_open(self):
        filename1 = getTestFileName("test_multi1.xlsx")
        filename2 = getTestFileName("test_multi2.xlsx")
        filename3 = getTestFileName("test_multi3.xlsx")
        filename4 = getTestFileName("test_multi4.xlsx")

        wb1 = ExcelApplication().Workbooks(filename1)
        wb2 = ExcelApplication().Workbooks(filename2)
        wb3 = ExcelApplication().Workbooks(filename3)
        wb4 = ExcelApplication().Workbooks(filename4)

        wb1.Close()
        wb2.Close()
        wb3.Close()
        wb4.Close()

        self.assertTrue(os.path.isfile(filename1))
        self.assertTrue(os.path.isfile(filename2))
        self.assertTrue(os.path.isfile(filename3))
        self.assertTrue(os.path.isfile(filename4))

class ExcelWorksheetsUnitTest(unittest.TestCase):

    def test_create(self):
        filename = getTestFileName("test_worksheet_create.xlsx")

        wb = ExcelApplication().Workbooks(filename)
        wsheets = wb.Worksheets
        self.assertEqual(wsheets.Count, 3)

        ws = wsheets.create('test')
        self.assertEqual(wsheets.Count, 4)

        wb.Close()

class ExcelWorksheetUnitTest(unittest.TestCase):

    def test_UsedRange(self):
        filename = getTestFileName("test_used_range.xlsx")

        wb = ExcelApplication().Workbooks(filename)
        ws = wb.Worksheets('test')

        ws.Cells(2,3).text = "hello"
        self.assertEqual(ws.Cells(2,3).text, "hello")

        ws.Cells(11,22).text = "world"

        rng = ws.UsedRange
        self.assertEqual(rng.Row, 2)
        self.assertEqual(rng.Column, 3)

        print("Rows: ", rng.Rows.Count)
        print("Columns: ", rng.Columns.Count)

        wb.Close()

    def _Cells(self):
        ws = ExcelApplication().Workbooks('test\\test1.xlsx').Worksheets('test1')
        cell = ws.Cells(4,3)

        self.assertEqual(cell.Row, 4)
        self.assertEqual(cell.Column, 3)
        self.assertEqual(cell.Rows.Count, 1)
        self.assertEqual(cell.Columns.Count, 1)

    def _CellValue(self):
        rng = ExcelApplication().Workbooks('test\\test1.xlsx').Worksheets('test1').UsedRange

        cell = rng.Cells(1,1)
        self.assertEqual(cell.text, "TopLeft")

        cell = rng.Cells(rng.Rows.Count, rng.Columns.Count)
        self.assertEqual(cell.text, "BottomRight")

        cell = rng.Cells(1, rng.Columns.Count)
        cell.text = "TopRight"
        self.assertEqual(cell.text, "TopRight")
        cell.value = None
        self.assertEqual(cell.text, "")

    def _Row_Column(self):
        col = ExcelApplication().Workbooks('test\\test1.xlsx').Worksheets('test1').UsedRange.Columns

########################################################################
# Main
########################################################################

if __name__ == '__main__':
    logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
    unittest.main(verbosity=2)
<|endoftext|>


<commit_before>"""Structural variant calling with Delly

https://github.com/tobiasrausch/delly
"""
import copy
import os
import subprocess

import vcf

from bcbio import bam, utils
from bcbio.distributed.multi import run_multicore, zeromq_aware_logging
from bcbio.distributed.transaction import file_transaction
from bcbio.pipeline import datadict as dd
from bcbio.provenance import do
from bcbio.structural import shared as sshared
from bcbio.variation import vcfutils

def _get_full_exclude_file(items, work_bams, work_dir):
    base_file = os.path.join(work_dir, "%s-svs" % (os.path.splitext(os.path.basename(work_bams[0]))[0]))
    return sshared.prepare_exclude_file(items, base_file)

def _delly_exclude_file(items, base_file, chrom):
    """Prepare a delly-specific exclude file eliminating chromosomes.
    Delly wants excluded chromosomes listed as just the chromosome, with no coordinates.
    """
    base_exclude = sshared.prepare_exclude_file(items, base_file, chrom)
    out_file = "%s-delly%s" % utils.splitext_plus(base_exclude)
    with file_transaction(items[0], out_file) as tx_out_file:
        with open(tx_out_file, "w") as out_handle:
            with open(base_exclude) as in_handle:
                for line in in_handle:
                    parts = line.split("\t")
                    if parts[0] == chrom:
                        out_handle.write(line)
                    else:
                        out_handle.write("%s\n" % parts[0])
    return out_file

@utils.map_wrap
@zeromq_aware_logging
def _run_delly(bam_files, chrom, ref_file, work_dir, items):
    """Run delly, calling structural variations for the specified type.
    """
    batch = sshared.get_cur_batch(items)
    ext = "-%s-svs" % batch if batch else "-svs"
    out_file = os.path.join(work_dir, "%s%s-%s.bcf"
                            % (os.path.splitext(os.path.basename(bam_files[0]))[0], ext, chrom))
    final_file = "%s.vcf.gz" % (utils.splitext_plus(out_file)[0])
    cores = min(utils.get_in(items[0], ("config", "algorithm", "num_cores"), 1),
                len(bam_files))
    if not utils.file_exists(out_file) and not utils.file_exists(final_file):
        with file_transaction(items[0], out_file) as tx_out_file:
            if sshared.has_variant_regions(items, out_file, chrom):
                exclude = ["-x", _delly_exclude_file(items, out_file, chrom)]
                # uses -n to skip small indel detection for speed, not yet optimized:
                # https://github.com/dellytools/delly/issues/36
                cmd = ["delly", "call", "--noindels", "-g", ref_file, "-o", tx_out_file] + exclude + bam_files
                multi_cmd = "export OMP_NUM_THREADS=%s && export LC_ALL=C && " % cores
                try:
                    do.run(multi_cmd + " ".join(cmd), "delly structural variant")
                except subprocess.CalledProcessError as msg:
                    # delly returns an error exit code if there are no variants
                    if "No structural variants found" not in str(msg):
                        raise
    return [_bgzip_and_clean(out_file, items)]

def _bgzip_and_clean(bcf_file, items):
    """Create a clean bgzipped VCF output file from bcf for downstream processing.

    Also corrects problems with missing likelihoods: https://github.com/dellytools/delly/issues/37
    GATK does not like missing GLs like '.,.,.'. This converts them to the recognized '.'
    """
    out_file = "%s.vcf.gz" % (utils.splitext_plus(bcf_file)[0])
    if not utils.file_exists(out_file):
        with file_transaction(items[0], out_file) as tx_out_file:
            if not utils.file_exists(bcf_file):
                vcfutils.write_empty_vcf(tx_out_file, samples=[dd.get_sample_name(d) for d in items])
            else:
                cmd = ("bcftools view {bcf_file} | sed 's/\.,\.,\././' | bgzip -c > {tx_out_file}")
                do.run(cmd.format(**locals()), "Convert and clean delly output")
    return vcfutils.bgzip_and_index(out_file, items[0]["config"])

@utils.map_wrap
@zeromq_aware_logging
def _prep_subsampled_bams(data, work_dir):
    """Prepare a subsampled BAM file with discordants from samblaster and minimal correct pairs.

    This attempts to minimize run times by pre-extracting useful reads mixed
    with subsampled normal pairs to estimate paired end distributions:

    https://groups.google.com/d/msg/delly-users/xmia4lwOd1Q/uaajoBkahAIJ

    Subsamples correctly aligned reads to 100 million based on speedseq defaults and
    evaluations on NA12878 whole genome data:

    https://github.com/cc2qe/speedseq/blob/ca624ba9affb0bd0fb88834ca896e9122639ec94/bin/speedseq#L1102

    XXX Currently not used as new versions of delly do not get good sensitivity
    with downsampled BAMs.
    """
    sr_bam, disc_bam = sshared.get_split_discordants(data, work_dir)
    ds_bam = bam.downsample(dd.get_align_bam(data), data, 1e8,
                            read_filter="-F 'not secondary_alignment and proper_pair'",
                            always_run=True, work_dir=work_dir)
    out_bam = "%s-final%s" % utils.splitext_plus(ds_bam)
    if not utils.file_exists(out_bam):
        bam.merge([ds_bam, sr_bam, disc_bam], out_bam, data["config"])
    bam.index(out_bam, data["config"])
    return [out_bam]

def _delly_count_evidence_filter(in_file, data):
    """Filter delly outputs based on read support (DV) and evidence (split and paired).

    We require DV > 4 and either both paired end and split read evidence or
    5 or more evidence for either individually.
    """
    filtname = "DVSupport"
    filtdoc = "FMT/DV < 4 || (SR < 1 && PE < 5) || (SR < 5 && PE < 1)"
    out_file = "%s-filter%s" % utils.splitext_plus(in_file)
    cur_out_file = out_file.replace(".vcf.gz", ".vcf")
    if not utils.file_exists(out_file):
        with file_transaction(data, cur_out_file) as tx_out_file:
            with utils.open_gzipsafe(in_file) as in_handle:
                with open(tx_out_file, "w") as out_handle:
                    inp = vcf.Reader(in_handle, in_file)
                    inp.filters["DVSupport"] = vcf.parser._Filter(filtname, filtdoc)
                    outp = vcf.Writer(out_handle, inp)
                    for rec in inp:
                        sr = rec.INFO.get("SR", 0)
                        pe = rec.INFO.get("PE", 0)
                        call = rec.samples[0].data
                        dv = call.DV if hasattr(call, "DV") else 0
                        if dv < 4 or (sr < 1 and pe < 5) or (sr < 5 and pe < 1):
                            rec.add_filter(filtname)
                        outp.write_record(rec)
    if out_file.endswith(".vcf.gz"):
        out_file = vcfutils.bgzip_and_index(cur_out_file, data["config"])
    return out_file

def run(items):
    """Perform detection of structural variations with delly.

    Performs post-call filtering with a custom filter tuned based
    on NA12878 Moleculo and PacBio data, using calls prepared by
    @ryanlayer and @cc2qe

    Filters using the high quality variant pairs (DV) compared with
    high quality reference pairs (DR).
    """
    work_dir = utils.safe_makedir(os.path.join(items[0]["dirs"]["work"], "structural",
                                               dd.get_sample_name(items[0]), "delly"))
    # Add core request for delly
    config = copy.deepcopy(items[0]["config"])
    delly_config = utils.get_in(config, ("resources", "delly"), {})
    delly_config["cores"] = 1
    config["resources"]["delly"] = delly_config
    parallel = {"type": "local", "cores": config["algorithm"].get("num_cores", 1),
                "progs": ["delly"]}
    work_bams = [dd.get_align_bam(d) for d in items]
    ref_file = dd.get_ref_file(items[0])
    exclude_file = _get_full_exclude_file(items, work_bams, work_dir)
    bytype_vcfs = run_multicore(_run_delly,
                                [(work_bams, chrom, ref_file, work_dir, items)
                                 for chrom in sshared.get_sv_chroms(items, exclude_file)],
                                config, parallel)
    out_file = "%s.vcf.gz" % sshared.outname_from_inputs(bytype_vcfs)
    combo_vcf = vcfutils.combine_variant_files(bytype_vcfs, out_file, ref_file, config)
    out = []
    for data in items:
        if "sv" not in data:
            data["sv"] = []
        base, ext = utils.splitext_plus(combo_vcf)
        final_vcf = sshared.finalize_sv(combo_vcf, data, items)
        if final_vcf:
            delly_vcf = _delly_count_evidence_filter(final_vcf, data)
            data["sv"].append({"variantcaller": "delly", "vrn_file": delly_vcf,
                               "exclude": exclude_file})
        out.append(data)
    return out
<commit_msg>delly: 0.7.9 release no longer as `--noindels`<commit_after>"""Structural variant calling with Delly

https://github.com/tobiasrausch/delly
"""
import copy
import os
import subprocess

import vcf

from bcbio import bam, utils
from bcbio.distributed.multi import run_multicore, zeromq_aware_logging
from bcbio.distributed.transaction import file_transaction
from bcbio.pipeline import datadict as dd
from bcbio.provenance import do
from bcbio.structural import shared as sshared
from bcbio.variation import vcfutils

def _get_full_exclude_file(items, work_bams, work_dir):
    base_file = os.path.join(work_dir, "%s-svs" % (os.path.splitext(os.path.basename(work_bams[0]))[0]))
    return sshared.prepare_exclude_file(items, base_file)

def _delly_exclude_file(items, base_file, chrom):
    """Prepare a delly-specific exclude file eliminating chromosomes.
    Delly wants excluded chromosomes listed as just the chromosome, with no coordinates.
    """
    base_exclude = sshared.prepare_exclude_file(items, base_file, chrom)
    out_file = "%s-delly%s" % utils.splitext_plus(base_exclude)
    with file_transaction(items[0], out_file) as tx_out_file:
        with open(tx_out_file, "w") as out_handle:
            with open(base_exclude) as in_handle:
                for line in in_handle:
                    parts = line.split("\t")
                    if parts[0] == chrom:
                        out_handle.write(line)
                    else:
                        out_handle.write("%s\n" % parts[0])
    return out_file

@utils.map_wrap
@zeromq_aware_logging
def _run_delly(bam_files, chrom, ref_file, work_dir, items):
    """Run delly, calling structural variations for the specified type.
    """
    batch = sshared.get_cur_batch(items)
    ext = "-%s-svs" % batch if batch else "-svs"
    out_file = os.path.join(work_dir, "%s%s-%s.bcf"
                            % (os.path.splitext(os.path.basename(bam_files[0]))[0], ext, chrom))
    final_file = "%s.vcf.gz" % (utils.splitext_plus(out_file)[0])
    cores = min(utils.get_in(items[0], ("config", "algorithm", "num_cores"), 1),
                len(bam_files))
    if not utils.file_exists(out_file) and not utils.file_exists(final_file):
        with file_transaction(items[0], out_file) as tx_out_file:
            if sshared.has_variant_regions(items, out_file, chrom):
                exclude = ["-x", _delly_exclude_file(items, out_file, chrom)]
                cmd = ["delly", "call", "-g", ref_file, "-o", tx_out_file] + exclude + bam_files
                multi_cmd = "export OMP_NUM_THREADS=%s && export LC_ALL=C && " % cores
                try:
                    do.run(multi_cmd + " ".join(cmd), "delly structural variant")
                except subprocess.CalledProcessError as msg:
                    # delly returns an error exit code if there are no variants
                    if "No structural variants found" not in str(msg):
                        raise
    return [_bgzip_and_clean(out_file, items)]

def _bgzip_and_clean(bcf_file, items):
    """Create a clean bgzipped VCF output file from bcf for downstream processing.

    Also corrects problems with missing likelihoods: https://github.com/dellytools/delly/issues/37
    GATK does not like missing GLs like '.,.,.'. This converts them to the recognized '.'
    """
    out_file = "%s.vcf.gz" % (utils.splitext_plus(bcf_file)[0])
    if not utils.file_exists(out_file):
        with file_transaction(items[0], out_file) as tx_out_file:
            if not utils.file_exists(bcf_file):
                vcfutils.write_empty_vcf(tx_out_file, samples=[dd.get_sample_name(d) for d in items])
            else:
                cmd = ("bcftools view {bcf_file} | sed 's/\.,\.,\././' | bgzip -c > {tx_out_file}")
                do.run(cmd.format(**locals()), "Convert and clean delly output")
    return vcfutils.bgzip_and_index(out_file, items[0]["config"])

@utils.map_wrap
@zeromq_aware_logging
def _prep_subsampled_bams(data, work_dir):
    """Prepare a subsampled BAM file with discordants from samblaster and minimal correct pairs.

    This attempts to minimize run times by pre-extracting useful reads mixed
    with subsampled normal pairs to estimate paired end distributions:

    https://groups.google.com/d/msg/delly-users/xmia4lwOd1Q/uaajoBkahAIJ

    Subsamples correctly aligned reads to 100 million based on speedseq defaults and
    evaluations on NA12878 whole genome data:

    https://github.com/cc2qe/speedseq/blob/ca624ba9affb0bd0fb88834ca896e9122639ec94/bin/speedseq#L1102

    XXX Currently not used as new versions of delly do not get good sensitivity
    with downsampled BAMs.
    """
    sr_bam, disc_bam = sshared.get_split_discordants(data, work_dir)
    ds_bam = bam.downsample(dd.get_align_bam(data), data, 1e8,
                            read_filter="-F 'not secondary_alignment and proper_pair'",
                            always_run=True, work_dir=work_dir)
    out_bam = "%s-final%s" % utils.splitext_plus(ds_bam)
    if not utils.file_exists(out_bam):
        bam.merge([ds_bam, sr_bam, disc_bam], out_bam, data["config"])
    bam.index(out_bam, data["config"])
    return [out_bam]

def _delly_count_evidence_filter(in_file, data):
    """Filter delly outputs based on read support (DV) and evidence (split and paired).

    We require DV > 4 and either both paired end and split read evidence or
    5 or more evidence for either individually.
    """
    filtname = "DVSupport"
    filtdoc = "FMT/DV < 4 || (SR < 1 && PE < 5) || (SR < 5 && PE < 1)"
    out_file = "%s-filter%s" % utils.splitext_plus(in_file)
    cur_out_file = out_file.replace(".vcf.gz", ".vcf")
    if not utils.file_exists(out_file):
        with file_transaction(data, cur_out_file) as tx_out_file:
            with utils.open_gzipsafe(in_file) as in_handle:
                with open(tx_out_file, "w") as out_handle:
                    inp = vcf.Reader(in_handle, in_file)
                    inp.filters["DVSupport"] = vcf.parser._Filter(filtname, filtdoc)
                    outp = vcf.Writer(out_handle, inp)
                    for rec in inp:
                        sr = rec.INFO.get("SR", 0)
                        pe = rec.INFO.get("PE", 0)
                        call = rec.samples[0].data
                        dv = call.DV if hasattr(call, "DV") else 0
                        if dv < 4 or (sr < 1 and pe < 5) or (sr < 5 and pe < 1):
                            rec.add_filter(filtname)
                        outp.write_record(rec)
    if out_file.endswith(".vcf.gz"):
        out_file = vcfutils.bgzip_and_index(cur_out_file, data["config"])
    return out_file

def run(items):
    """Perform detection of structural variations with delly.

    Performs post-call filtering with a custom filter tuned based
    on NA12878 Moleculo and PacBio data, using calls prepared by
    @ryanlayer and @cc2qe

    Filters using the high quality variant pairs (DV) compared with
    high quality reference pairs (DR).
    """
    work_dir = utils.safe_makedir(os.path.join(items[0]["dirs"]["work"], "structural",
                                               dd.get_sample_name(items[0]), "delly"))
    # Add core request for delly
    config = copy.deepcopy(items[0]["config"])
    delly_config = utils.get_in(config, ("resources", "delly"), {})
    delly_config["cores"] = 1
    config["resources"]["delly"] = delly_config
    parallel = {"type": "local", "cores": config["algorithm"].get("num_cores", 1),
                "progs": ["delly"]}
    work_bams = [dd.get_align_bam(d) for d in items]
    ref_file = dd.get_ref_file(items[0])
    exclude_file = _get_full_exclude_file(items, work_bams, work_dir)
    bytype_vcfs = run_multicore(_run_delly,
                                [(work_bams, chrom, ref_file, work_dir, items)
                                 for chrom in sshared.get_sv_chroms(items, exclude_file)],
                                config, parallel)
    out_file = "%s.vcf.gz" % sshared.outname_from_inputs(bytype_vcfs)
    combo_vcf = vcfutils.combine_variant_files(bytype_vcfs, out_file, ref_file, config)
    out = []
    for data in items:
        if "sv" not in data:
            data["sv"] = []
        base, ext = utils.splitext_plus(combo_vcf)
        final_vcf = sshared.finalize_sv(combo_vcf, data, items)
        if final_vcf:
            delly_vcf = _delly_count_evidence_filter(final_vcf, data)
            data["sv"].append({"variantcaller": "delly", "vrn_file": delly_vcf,
                               "exclude": exclude_file})
        out.append(data)
    return out
<|endoftext|>


<commit_before>from django.test import TestCase
from django.test.client import Client

from projects.models import Project
from teams.models import Team

class TestAPIProjectListSearch(TestCase):
    def setUp(self):
        self.test_string = "test_string"
        self.team = Team.objects.create_team(
            name="test_team_name",
        )
        self.project_with_test_string = Project.objects.create_project(
            name="test_project_name_with" + self.test_string,
            team=self.team,
        )
        self.project_without_test_string = Project.objects.create_project(
            name="test_project_name_without",   # +self.test_string
                                                # intentionally excluded
                                                # for valid search result
            team=self.team,
        )

        self.client = Client()
        self.response = self.client.get("/api/projects/?search=" + self.test_string)

    def test_api_project_list_search_should_return_valid_result(self):
        self.assertContains(self.response, self.project_with_test_string.name)
        self.assertNotContains(self.response, self.project_without_test_string.name)
<commit_msg>team field removed<commit_after>from django.test import TestCase
from django.test.client import Client

from projects.models import Project
from teams.models import Team

class TestAPIProjectListSearch(TestCase):
    def setUp(self):
        self.test_string = "test_string"
        self.team = Team.objects.create_team(
            name="test_team_name",
        )
        self.project_with_test_string = Project.objects.create_project(
            name="test_project_name_with" + self.test_string,
        )
        self.project_without_test_string = Project.objects.create_project(
            name="test_project_name_without",   # +self.test_string
                                                # intentionally excluded
                                                # for valid search result
        )

        self.client = Client()
        self.response = self.client.get("/api/projects/?search=" + self.test_string)

    def test_api_project_list_search_should_return_valid_result(self):
        self.assertContains(self.response, self.project_with_test_string.name)
        self.assertNotContains(self.response, self.project_without_test_string.name)
<|endoftext|>


<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
Define the Data-table Text Interchange Format DTIF which allows for reading and
writing all the meta data associated with an astropy Table object.
"""

import re
import json

from ...utils import OrderedDict

from . import core

def _decode_list(data):
    rv = []
    for item in data:
        if isinstance(item, unicode):
            item = item.encode('utf-8')
        elif isinstance(item, list):
            item = _decode_list(item)
        elif isinstance(item, dict):
            item = _decode_dict(item)
        rv.append(item)
    return rv

def _decode_dict(data):
    rv = OrderedDict()
    for key, value in data.iteritems():
        if isinstance(key, unicode):
            key = key.encode('utf-8')
        if isinstance(value, unicode):
            value = value.encode('utf-8')
        elif isinstance(value, list):
            value = _decode_list(value)
        elif isinstance(value, dict):
            value = _decode_dict(value)
        rv[key] = value
    return rv

def _get_col_attributes(col):
    """
    Extract information from a column (apart from the values) that is required
    to fully serialize the column.
    """
    attrs = OrderedDict()
    attrs['name'] = col.name
    attrs['unit'] = str(col.unit)
    attrs['format'] = col.format
    attrs['description'] = col.description
    attrs['type'] = col.dtype.type.__name__
    attrs['meta'] = col.meta

    return attrs

class DtifHeader(core.BaseHeader):
    """Header class for which the column definition line starts with the
    comment character.  See the :class:`CommentedHeader` class  for an example.
    """
    def process_lines(self, lines):
        """Return only non-blank lines that start with the comment regexp.  For these
        lines strip out the matching characters and leading/trailing whitespace."""
        re_comment = re.compile(self.comment)
        for line in lines:
            match = re_comment.match(line)
            if match:
                out = line[match.end():].strip()
                if out:
                    yield out

    def write(self, lines):
        """
        Write header information in the DTIF ASCII format.  This format
        starts with a delimiter separated list of the column names in order
        to make this format readable by humans and simple csv-type readers.
        It then encodes the full table meta and column attributes and meta
        as JSON and pretty-prints this in the header.  Finally the delimited
        column names are repeated again, for humans and readers that look
        for the *last* comment line as defining the column names.
        """
        meta = OrderedDict()
        meta['version'] = 1.0
        meta['schema'] = 'astropy.table'
        meta['table_meta'] = self.table_meta
        meta['columns'] = [_get_col_attributes(col) for col in self.cols]

        outs = ['<DTIF encoding=ascii>']
        meta_json = json.dumps(meta, indent=2, separators=(',', ': '))
        outs.extend(meta_json.splitlines())

        lines.extend([self.write_comment + line for line in outs])
        lines.append(self.splitter.join([x.name for x in self.cols]))

    def get_cols(self, lines):
        """Initialize the header Column objects from the table ``lines``.

        :param lines: list of table lines
        :returns: None (but sets self.cols)
        """
        # Extract non-blank comment (header) lines with comment character striped
        lines = list(self.process_lines(lines))

        m = re.match(r'< \s* DTIF \s+ encoding=(\S+) \s* >', lines[0], re.VERBOSE)
        if not m:
            raise core.InconsistentTableError('DTIF file must start with "# <DTIF encoding=[encoding]>')
        encoding = m.group(1)  # Nothing done with encoding at the moment

        # Now actually load the JSON data structure into `meta`
        meta_json = '\n'.join(lines[1:])
        meta = json.loads(meta_json, object_pairs_hook=OrderedDict)
        meta = _decode_dict(meta)
        self.table_meta = meta['table_meta']

        # Create the list of io.ascii column objects from `meta`
        meta_cols = OrderedDict((x['name'], x) for x in meta['columns'])
        self.names = meta_cols.keys()
        self._set_cols_from_names()  # BaseHeader method to create self.cols

        # Transfer attributes from the column descriptor stored in the input
        # header JSON metadata to the new columns to create this table.
        for col in self.cols:
            for attr in ('description', 'format', 'unit', 'meta'):
                setattr(col, attr, meta_cols[col.name][attr])
            col.dtype = meta_cols[col.name]['type']

class Dtif(core.BaseReader):
    """Read a file where the column names are given in a line that begins with
    the header comment character. `header_start` can be used to specify the
    line index of column names, and it can be a negative index (for example -1
    for the last commented line).  The default delimiter is the <space>
    character.::

      # col1 col2 col3
      # Comment line
      1 2 3
      4 5 6
    """
    _format_name = 'dtif'
    _description = 'Data-table Text Interchange Format'

    def __init__(self):
        core.BaseReader.__init__(self)
        self.header = DtifHeader()
        self.header.data = self.data
        self.data.header = self.header
        self.header.splitter.delimiter = ' '
        self.data.splitter.delimiter = ' '
        self.header.start_line = 0
        self.data.start_line = 1
        self.header.comment = r'\s*#'
        self.header.write_comment = '# '
        self.data.comment = r'\s*#'
        self.data.write_comment = '# '
<commit_msg>Change from JSON to YAML (WIP)<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
Define the Data-table Text Interchange Format DTIF which allows for reading and
writing all the meta data associated with an astropy Table object.
"""

import re
import yaml

from ...utils import OrderedDict

from . import core

def _decode_list(data):
    rv = []
    for item in data:
        if isinstance(item, unicode):
            item = item.encode('utf-8')
        elif isinstance(item, list):
            item = _decode_list(item)
        elif isinstance(item, dict):
            item = _decode_dict(item)
        rv.append(item)
    return rv

def _decode_dict(data):
    rv = OrderedDict()
    for key, value in data.iteritems():
        if isinstance(key, unicode):
            key = key.encode('utf-8')
        if isinstance(value, unicode):
            value = value.encode('utf-8')
        elif isinstance(value, list):
            value = _decode_list(value)
        elif isinstance(value, dict):
            value = _decode_dict(value)
        rv[key] = value
    return rv

def _get_col_attributes(col):
    """
    Extract information from a column (apart from the values) that is required
    to fully serialize the column.
    """
    attrs = OrderedDict()
    attrs['name'] = col.name
    attrs['unit'] = str(col.unit)
    attrs['format'] = col.format
    attrs['description'] = col.description
    attrs['type'] = col.dtype.type.__name__
    attrs['meta'] = col.meta

    return attrs

class DtifHeader(core.BaseHeader):
    """Header class for which the column definition line starts with the
    comment character.  See the :class:`CommentedHeader` class  for an example.
    """
    def process_lines(self, lines):
        """Return only non-blank lines that start with the comment regexp.  For these
        lines strip out the matching characters and leading/trailing whitespace."""
        re_comment = re.compile(self.comment)
        for line in lines:
            match = re_comment.match(line)
            if match:
                out = line[match.end():].strip()
                if out:
                    yield out

    def write(self, lines):
        """
        Write header information in the DTIF ASCII format.  This format
        starts with a delimiter separated list of the column names in order
        to make this format readable by humans and simple csv-type readers.
        It then encodes the full table meta and column attributes and meta
        as YAML and pretty-prints this in the header.  Finally the delimited
        column names are repeated again, for humans and readers that look
        for the *last* comment line as defining the column names.
        """
        meta = OrderedDict()
        meta['version'] = 1.0
        meta['schema'] = 'astropy.table'
        meta['table_meta'] = self.table_meta
        meta['columns'] = [_get_col_attributes(col) for col in self.cols]

        outs = ['<DTIF encoding=ascii>']
        meta_yaml = yaml.dump(meta)  # , indent=2, separators=(',', ': '))
        outs.extend(meta_yaml.splitlines())

        lines.extend([self.write_comment + line for line in outs])
        lines.append(self.splitter.join([x.name for x in self.cols]))

    def get_cols(self, lines):
        """Initialize the header Column objects from the table ``lines``.

        :param lines: list of table lines
        :returns: None (but sets self.cols)
        """
        # Extract non-blank comment (header) lines with comment character striped
        lines = list(self.process_lines(lines))

        m = re.match(r'< \s* DTIF \s+ encoding=(\S+) \s* >', lines[0], re.VERBOSE)
        if not m:
            raise core.InconsistentTableError('DTIF file must start with "# <DTIF encoding=[encoding]>')
        # encoding = m.group(1)  # Nothing done with encoding at the moment

        # Now actually load the YAML data structure into `meta`
        meta_yaml = '\n'.join(lines[1:])
        meta = yaml.loads(meta_yaml, object_pairs_hook=OrderedDict)
        meta = _decode_dict(meta)
        self.table_meta = meta['table_meta']

        # Create the list of io.ascii column objects from `meta`
        meta_cols = OrderedDict((x['name'], x) for x in meta['columns'])
        self.names = meta_cols.keys()
        self._set_cols_from_names()  # BaseHeader method to create self.cols

        # Transfer attributes from the column descriptor stored in the input
        # header YAML metadata to the new columns to create this table.
        for col in self.cols:
            for attr in ('description', 'format', 'unit', 'meta'):
                setattr(col, attr, meta_cols[col.name][attr])
            col.dtype = meta_cols[col.name]['type']

class Dtif(core.BaseReader):
    """Read a file where the column names are given in a line that begins with
    the header comment character. `header_start` can be used to specify the
    line index of column names, and it can be a negative index (for example -1
    for the last commented line).  The default delimiter is the <space>
    character.::

      # col1 col2 col3
      # Comment line
      1 2 3
      4 5 6
    """
    _format_name = 'dtif'
    _description = 'Data-table Text Interchange Format'

    def __init__(self):
        core.BaseReader.__init__(self)
        self.header = DtifHeader()
        self.header.data = self.data
        self.data.header = self.header
        self.header.splitter.delimiter = ' '
        self.data.splitter.delimiter = ' '
        self.header.start_line = 0
        self.data.start_line = 1
        self.header.comment = r'\s*#'
        self.header.write_comment = '# '
        self.data.comment = r'\s*#'
        self.data.write_comment = '# '
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    openslides.participant.api
    ~~~~~~~~~~~~~~~~~~~~~~~~~~

    Useful functions for the participant app.

    :copyright: 2011, 2012 by OpenSlides team, see AUTHORS.
    :license: GNU GPL, see LICENSE for more details.
"""

from random import choice
import string

from django.contrib.auth.models import User

def gen_password():
    """
    generates a random passwort.
    """
    chars = string.letters + string.digits
    newpassword = ''
    for i in range(8):
        newpassword += choice(chars)
    return newpassword

def gen_username(first_name, last_name):
    """
    generates the username for new users.
    """
    testname = "%s%s" % (first_name, last_name)
    try:
        User.objects.get(username=testname)
    except User.DoesNotExist:
        return testname
    i = 0
    while True:
        i += 1
        testname = "%s%s%s" % (first_name, last_name, i)
        try:
            User.objects.get(username=testname)
        except User.DoesNotExist:
            return testname
<commit_msg>use "forname lastname" as default username, with a space<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    openslides.participant.api
    ~~~~~~~~~~~~~~~~~~~~~~~~~~

    Useful functions for the participant app.

    :copyright: 2011, 2012 by OpenSlides team, see AUTHORS.
    :license: GNU GPL, see LICENSE for more details.
"""

from random import choice
import string

from django.contrib.auth.models import User

def gen_password():
    """
    generates a random passwort.
    """
    chars = string.letters + string.digits
    newpassword = ''
    for i in range(8):
        newpassword += choice(chars)
    return newpassword

def gen_username(first_name, last_name):
    """
    generates the username for new users.
    """
    testname = "%s %s" % (first_name.strip(), last_name.strip())
    try:
        User.objects.get(username=testname)
    except User.DoesNotExist:
        return testname
    i = 0
    while True:
        i += 1
        testname = "%s%s%s" % (first_name, last_name, i)
        try:
            User.objects.get(username=testname)
        except User.DoesNotExist:
            return testname
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Example command: ./import_registrar.py donut_courses_FA2018-19_180515.txt 2018 FA

import csv
import re
import sys
from donut.pymysql_connection import make_db
from donut.modules.courses import helpers

COURSE_MATCH = r'^(?P<dept>[A-Za-z/]+) *0*(?P<number>[0-9]+[A-Z]*)$'

db = make_db('dev')

_, filename, year, term = sys.argv
term = helpers.TERMS[term]
courses = {}  # map of course numbers to courses data
instructors = set()
grades_types = set()
sections = {}  # map of (course number, section) to section data
with open(filename) as f:
    tsv_reader = csv.reader(f, delimiter='\t')
    headers = None
    for row in tsv_reader:
        if not headers:
            headers = row
            continue

        row_data = dict(zip(headers, row))
        course = row_data['Course Number']
        if course not in courses:
            try:
                match = re.search(COURSE_MATCH, course)
                dept = match.group('dept')
                number = match.group('number').lower()
                units = row_data['Units']
                if units == '+': units = '0-0-0'
                lecture, lab, homework = map(float, units.split('-'))
                courses[course] = {
                    'department': dept,
                    'course_number': number,
                    'name': row_data['Course Name'],
                    'units_lecture': lecture,
                    'units_lab': lab,
                    'units_homework': homework
                }
            except e:
                print('Error parsing course:', row_data, file=sys.stderr)
                print(e, file=sys.stderr)
        try:
            instructor = row_data['Instructor']
            instructors.add(instructor)
            grades_type = row_data['Grades']
            grades_types.add(grades_type)
            section = int(row_data['Section Number'])
            times = row_data['Times']
            locations = row_data['Locations'].strip()
            section_key = (course, section)
            section_data = sections.get(section_key)
            if section_data:
                section_data['times'] += '\n' + times
                section_data['locations'] += '\n' + locations
            else:
                sections[section_key] = {
                    'course': course,
                    'section_number': section,
                    'instructor': instructor,
                    'grades_type': grades_type,
                    'times': times,
                    'locations': locations
                }
        except e:
            print('Error parsing section:', row_data, file=sys.stderr)
            print(e, file=sys.stderr)

with db.cursor() as cursor:
    query = """
        INSERT INTO courses (
            year, term,
            department, course_number,
            name,
            units_lecture, units_lab, units_homework
        ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
        ON DUPLICATE KEY UPDATE course_id = course_id
    """
    for course in courses.values():
        cursor.execute(
            query, (year, term, course['department'], course['course_number'],
                    course['name'], course['units_lecture'],
                    course['units_lab'], course['units_homework']))

    instructor_ids = {}
    for instructor in instructors:
        cursor.execute(
            'SELECT instructor_id FROM instructors WHERE instructor = %s LIMIT 1',
            instructor)
        instructor_id = cursor.fetchone()
        if instructor_id:
            instructor_ids[instructor] = instructor_id['instructor_id']
        else:
            cursor.execute('INSERT INTO instructors (instructor) VALUES (%s)',
                           instructor)
            instructor_ids[instructor] = cursor.lastrowid

    grades_types_ids = {}
    for grades_type in grades_types:
        cursor.execute(
            'SELECT grades_type_id FROM grades_types WHERE grades_type = %s',
            grades_type)
        grades_type_id = cursor.fetchone()
        if grades_type_id:
            grades_types_ids[grades_type] = grades_type_id['grades_type_id']
        else:
            cursor.execute(
                'INSERT INTO grades_types (grades_type) VALUES (%s)',
                grades_type)
            grades_types_ids[grades_type] = cursor.lastrowid

    query = """
        INSERT INTO sections (
            course_id,
            section_number,
            instructor_id, grades_type_id,
            times, locations
        ) VALUES (
            (SELECT course_id FROM courses WHERE year = %s AND term = %s AND department = %s AND course_number = %s LIMIT 1),
            %s,
            %s, %s,
            %s, %s
        ) ON DUPLICATE KEY UPDATE section_number = section_number
    """
    for section in sections.values():
        course = courses[section['course']]
        cursor.execute(
            query,
            (year, term, course['department'], course['course_number'],
             section['section_number'], instructor_ids[section['instructor']],
             grades_types_ids[section['grades_type']], section['times'],
             section['locations']))
<commit_msg>Improve documentation of registrar import script<commit_after>#!/usr/bin/env python
"""
Imports the Registrar's Office's exports of courses for a given term.
Never removes courses/sections and will not overwrite existing ones.
Expects file to be a TSV value with the following columns:
"Course Number"
"Course Name"
"Department"
"Instructor"
"Grades"
"Units"
"Section Number"
"Times"
"Locations"

Example row:
"CS 001"	"Introduction to Computer Programming"	"CS"	"Vanier, M"	"PASS-FAIL"	"3-4-2"	"01"	"MWF 14:00 - 14:55"	"Institute Auditorium BCK"

Note that there is a separate row for each time of each section of each course.
The insertion into our database is straightforward, with a few exceptions:
- We separate the course number into department and number
- Leading zeros are removed from the course number and it is lowercased
- If the units value is "+", we change it to "0-0-0"
- Units are split into lecture, lab, and homework units
  (note we use float since some courses have fractional units)
- Remove whitespace from locations (IDK why some are " YHC")
- Append locations and times from separate rows for the same section
- Make lists of all grades types (e.g. "PASS-FAIL") and instructors
  (e.g. "Vanier, M") and add to grades_types and instructors tables
"""

import argparse
import csv
import re
import sys
from donut.pymysql_connection import make_db
from donut.modules.courses.helpers import TERMS

COURSE_MATCH = r'^(?P<dept>[A-Za-z/]+) *0*(?P<number>[0-9]+[A-Z]*)$'

db = make_db('dev')

parser = argparse.ArgumentParser()
parser.add_argument(
    'file',
    help='Path to registrar file, e.g. donut_courses_WI2018-19_181113.txt')
parser.add_argument(
    'year', type=int, help='Year of the term to import (e.g. 2019)')
parser.add_argument('term', choices=TERMS, help='Term to import (e.g. WI)')
args = parser.parse_args()

year = args.year
term = TERMS[args.term]
courses = {}  # map of course numbers to courses data
instructors = set()
grades_types = set()
sections = {}  # map of (course number, section) to section data
with open(args.file) as f:
    tsv_reader = csv.reader(f, delimiter='\t')
    headers = None
    for row in tsv_reader:
        if not headers:
            headers = row
            continue

        row_data = dict(zip(headers, row))
        course = row_data['Course Number']
        if course not in courses:
            try:
                match = re.search(COURSE_MATCH, course)
                dept = match.group('dept')
                number = match.group('number').lower()
                units = row_data['Units']
                if units == '+': units = '0-0-0'
                lecture, lab, homework = map(float, units.split('-'))
                courses[course] = {
                    'department': dept,
                    'course_number': number,
                    'name': row_data['Course Name'],
                    'units_lecture': lecture,
                    'units_lab': lab,
                    'units_homework': homework
                }
            except e:
                print('Error parsing course:', row_data, file=sys.stderr)
                print(e, file=sys.stderr)
        try:
            instructor = row_data['Instructor']
            instructors.add(instructor)
            grades_type = row_data['Grades']
            grades_types.add(grades_type)
            section = int(row_data['Section Number'])
            times = row_data['Times']
            locations = row_data['Locations'].strip()
            section_key = course, section
            section_data = sections.get(section_key)
            if section_data:
                section_data['times'] += '\n' + times
                section_data['locations'] += '\n' + locations
            else:
                sections[section_key] = {
                    'course': course,
                    'section_number': section,
                    'instructor': instructor,
                    'grades_type': grades_type,
                    'times': times,
                    'locations': locations
                }
        except e:
            print('Error parsing section:', row_data, file=sys.stderr)
            print(e, file=sys.stderr)

with db.cursor() as cursor:
    course_ids = {}
    query = """
        INSERT INTO courses (
            year, term,
            department, course_number,
            name,
            units_lecture, units_lab, units_homework
        ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
        ON DUPLICATE KEY UPDATE course_id = course_id
    """
    for number, course in courses.items():
        cursor.execute(
            query, (year, term, course['department'], course['course_number'],
                    course['name'], course['units_lecture'],
                    course['units_lab'], course['units_homework']))
        if cursor.lastrowid:  # 0 if no insert occured
            course_ids[number] = cursor.lastrowid

    def add_possibilities(name, values):
        ids = {}
        for value in values:
            cursor.execute('SELECT ' + name + '_id AS id FROM ' + name +
                           's WHERE ' + name + ' = %s LIMIT 1', value)
            value_id = cursor.fetchone()
            if value_id:
                ids[value] = value_id['id']
            else:
                cursor.execute(
                    'INSERT INTO ' + name + 's (' + name + ') VALUES (%s)',
                    value)
                ids[value] = cursor.lastrowid
        return ids

    instructor_ids = add_possibilities('instructor', instructors)
    grades_types_ids = add_possibilities('grades_type', grades_types)

    query = """
        INSERT INTO sections (
            course_id,
            section_number,
            instructor_id, grades_type_id,
            times, locations
        ) VALUES (
            %s,
            %s,
            %s, %s,
            %s, %s
        ) ON DUPLICATE KEY UPDATE section_number = section_number
    """
    for section in sections.values():
        course_id = course_ids.get(section['course'])
        if not course_id: continue

        cursor.execute(query, (course_id, section['section_number'],
                               instructor_ids[section['instructor']],
                               grades_types_ids[section['grades_type']],
                               section['times'], section['locations']))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#@created: 09.10.2013
#@author: Aleksey Komissarov
#@contact: ad3002@gmail.com

 import Image, ImageDraw, ImageFont

# fix for Windows-specific bug with png extension registering
from PIL.PngImagePlugin import _save, PngImageFile, _accept
Image.register_open("PNG", PngImageFile, _accept)
Image.register_save("PNG", _save)
Image.register_extension("PNG", ".png")
Image.register_mime("PNG", "image/png")

def draw_vertical_chromosome(draw, x, y, length, bands=None, chr_width=100, name=None, scale=1, stars=None):
    '''
    Draw chromosome
    @param draw: ImageDraw object
    @param x: x coordinate of left top corner
    @param y: y coordinate of left top corner
    @param length: chromosome length
    @param bands: bands list of ideograms
    @param chr_width: chromosome width in px
    @param name: chromosome name
    @param scale: scaling, default value - 1
    @param stars: list of star positions
    '''
    arcbbox = (x, y, x + chr_width, y + chr_width)
    draw.arc(arcbbox, 180, 0, fill='#000000')
    y += chr_width/2
    arcbbox = (x, y + length-chr_width/2, x+chr_width, y + length+chr_width/2)
    draw.arc(arcbbox, 0, 180, fill='#000000')
    rectbbox = [x, y, x + chr_width, y + length]
    draw.rectangle(rectbbox, fill="#ffffff", outline="#000000")
    draw.line((x,y, x+chr_width, y), fill="#ffffff")
    draw.line((x,y + length, x+chr_width, y + length), fill="#ffffff")

    if name:
        draw_legend(draw, name, x, y+length+chr_width/2+25, chr_width)
    for band in bands:
        y, length, color, outline = band
        y += chr_width/2
        rectbbox = [x+1, y, x + chr_width-1, y + length]
        draw.rectangle(rectbbox, fill=color, outline=outline)
    if stars:
        for y in stars:
            y += chr_width/2
            draw_legend(draw, "*", x+chr_width+10, y, None, font_size=35)

def draw_horizontal_chromosome(draw, x, y, length, bands=None, chr_width=100, name=None, scale=1, stars=None):
    '''
    Draw chromosome
    @param draw: ImageDraw object
    @param x: x coordinate of left top corner
    @param y: y coordinate of left top corner
    @param length: chromosome length
    @param bands: bands list of ideograms
    @param chr_width: chromosome width in px
    @param name: chromosome name
    @param scale: scaling, default value - 1
    @param stars: list of star positions
    '''
    raise NotImplemented

def draw_centromere():
    raise NotImplemented

def draw_telomere():
    raise NotImplemented

def draw_legend(draw, text, x, y, width, font_size=35, font_file="../fonts/arialbd.ttf", text_color="#000000"):
    ''' Draw legend on image.
    @param draw: ImageDraw object
    @param x: x coordinate of left top corner
    @param y: y coordinate of left top corner
    @param width: text width
    @param font_size: text font size, default value - 35
    @param font_file: path to ttf file, default value - ../fonts/arialbd.ttf
    @param text_color: text color, default value - #000000
    '''
    font = ImageFont.truetype(font_file, font_size)
    if width:
        w, h = draw.textsize(text, font)
        x += (width - w)/2
    draw.text((x, y), text, font=font, fill=text_color)

def draw_full_legend(draw, x, y, names, colors_vals):
    ''' Draw legend for color annotated images
    @param draw: ImageDraw object
    @param x: x coordinate of left top corner
    @param y: y coordinate of left top corner
    @param names: list of names
    @param colors_vals: dictionary from name to color
    '''

    # draw borders
    rectbbox = [x-50, y+35, x + 350, y +1230]
    draw.rectangle(rectbbox, fill="#ffffff", outline="#dddddd")
    rectbbox = [x-50+1, y+35+1, x + 350-1, y +1230-1]
    draw.rectangle(rectbbox, fill="#ffffff", outline="#dddddd")

    # draw legend
    for i, title in enumerate(names):
        y +=  70
        color = colors_vals[title]
        rectbbox = [x, y, x + 50, y + 50]
        draw.rectangle(rectbbox, fill=color, outline="#000000")
        draw_legend(draw, title, x + 80, y, None, font_size = 42)

<commit_msg>draw_chromosome_scheme function<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#@created: 09.10.2013
#@author: Aleksey Komissarov
#@contact: ad3002@gmail.com

 import Image, ImageDraw, ImageFont

# fix for Windows-specific bug with png extension registering
from PIL.PngImagePlugin import _save, PngImageFile, _accept
Image.register_open("PNG", PngImageFile, _accept)
Image.register_save("PNG", _save)
Image.register_extension("PNG", ".png")
Image.register_mime("PNG", "image/png")

def draw_vertical_chromosome(draw, x, y, length, bands=None, chr_width=100, name=None, scale=1, stars=None):
    '''
    Draw chromosome
    @param draw: ImageDraw object
    @param x: x coordinate of left top corner
    @param y: y coordinate of left top corner
    @param length: chromosome length
    @param bands: bands list of ideograms
    @param chr_width: chromosome width in px
    @param name: chromosome name
    @param scale: scaling, default value - 1
    @param stars: list of star positions
    '''
    arcbbox = (x, y, x + chr_width, y + chr_width)
    draw.arc(arcbbox, 180, 0, fill='#000000')
    y += chr_width/2
    arcbbox = (x, y + length-chr_width/2, x+chr_width, y + length+chr_width/2)
    draw.arc(arcbbox, 0, 180, fill='#000000')
    rectbbox = [x, y, x + chr_width, y + length]
    draw.rectangle(rectbbox, fill="#ffffff", outline="#000000")
    draw.line((x,y, x+chr_width, y), fill="#ffffff")
    draw.line((x,y + length, x+chr_width, y + length), fill="#ffffff")

    if name:
        draw_legend(draw, name, x, y+length+chr_width/2+25, chr_width)
    for band in bands:
        y, length, color, outline = band
        y += chr_width/2
        rectbbox = [x+1, y, x + chr_width-1, y + length]
        draw.rectangle(rectbbox, fill=color, outline=outline)
    if stars:
        for y in stars:
            y += chr_width/2
            draw_legend(draw, "*", x+chr_width+10, y, None, font_size=35)

def draw_horizontal_chromosome(draw, x, y, length, bands=None, chr_width=100, name=None, scale=1, stars=None):
    '''
    Draw chromosome
    @param draw: ImageDraw object
    @param x: x coordinate of left top corner
    @param y: y coordinate of left top corner
    @param length: chromosome length
    @param bands: bands list of ideograms
    @param chr_width: chromosome width in px
    @param name: chromosome name
    @param scale: scaling, default value - 1
    @param stars: list of star positions
    '''
    raise NotImplemented

def draw_centromere():
    raise NotImplemented

def draw_telomere():
    raise NotImplemented

def draw_legend(draw, text, x, y, width, font_size=35, font_file="../fonts/arialbd.ttf", text_color="#000000"):
    ''' Draw legend on image.
    @param draw: ImageDraw object
    @param x: x coordinate of left top corner
    @param y: y coordinate of left top corner
    @param width: text width
    @param font_size: text font size, default value - 35
    @param font_file: path to ttf file, default value - ../fonts/arialbd.ttf
    @param text_color: text color, default value - #000000
    '''
    font = ImageFont.truetype(font_file, font_size)
    if width:
        w, h = draw.textsize(text, font)
        x += (width - w)/2
    draw.text((x, y), text, font=font, fill=text_color)

def draw_full_legend(draw, x, y, names, colors_vals):
    ''' Draw legend for color annotated images
    @param draw: ImageDraw object
    @param x: x coordinate of left top corner
    @param y: y coordinate of left top corner
    @param names: list of names
    @param colors_vals: dictionary from name to color
    '''

    # draw borders
    rectbbox = [x-50, y+35, x + 350, y +1230]
    draw.rectangle(rectbbox, fill="#ffffff", outline="#dddddd")
    rectbbox = [x-50+1, y+35+1, x + 350-1, y +1230-1]
    draw.rectangle(rectbbox, fill="#ffffff", outline="#dddddd")

    # draw legend
    for i, title in enumerate(names):
        y +=  70
        color = colors_vals[title]
        rectbbox = [x, y, x + 50, y + 50]
        draw.rectangle(rectbbox, fill=color, outline="#000000")
        draw_legend(draw, title, x + 80, y, None, font_size = 42)

def draw_chromosome_scheme(draw, x, y):
    ''' Draw chromosome scheme with cen and tel positions.
    @param draw: ImageDraw object
    @param x: x coordinate of left top corner
    @param y: y coordinate of left top corner
    '''
    bands = [
        (y+130, 100, "#cccccc","#cccccc"),
        (y+240, 280, "#eeeeee","#eeeeee"),
        (y+530, 100, "#cccccc","#cccccc"),
    ]
    draw_chromosome(draw, x-10, y+130, 500, bands=bands, chr_width=70)
    draw_legend(draw, "PeriCen", x + 90, y+180, None, font_size = 42)
    draw_legend(draw, "Arm", x + 90, y+380, None, font_size = 42)
    draw_legend(draw, "PeriTel", x + 90, y+580, None, font_size = 42)<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import unicode_literals
from datetime import datetime, date, time
from decimal import Decimal
import json
import os
import uuid
import re
import six
import sqlalchemy

from mock import patch
from six.moves import zip

from casexml.apps.case.models import CommCareCase
from corehq.apps.app_manager.xform_builder import XFormBuilder
from corehq.apps.change_feed import data_sources
from corehq.apps.userreports.models import DataSourceConfiguration, ReportConfiguration
from corehq.sql_db.connections import connection_manager
from dimagi.utils.parsing import json_format_datetime
from pillowtop.feed.interface import Change, ChangeMeta

from io import open

def get_sample_report_config():
    folder = os.path.join(os.path.dirname(__file__), 'data', 'configs')
    sample_file = os.path.join(folder, 'sample_report_config.json')
    with open(sample_file, encoding='utf-8') as f:
        structure = json.loads(f.read())
        return ReportConfiguration.wrap(structure)

def get_sample_data_source():
    folder = os.path.join(os.path.dirname(__file__), 'data', 'configs')
    sample_file = os.path.join(folder, 'sample_data_source.json')
    with open(sample_file, encoding='utf-8') as f:
        structure = json.loads(f.read())
        return DataSourceConfiguration.wrap(structure)

def get_data_source_with_related_doc_type():
    folder = os.path.join(os.path.dirname(__file__), 'data', 'configs')
    sample_file = os.path.join(folder, 'parent_child_data_source.json')
    with open(sample_file, encoding='utf-8') as f:
        structure = json.loads(f.read())
        return DataSourceConfiguration.wrap(structure)

def get_sample_doc_and_indicators(fake_time_now=None):
    if fake_time_now is None:
        fake_time_now = datetime.utcnow()
    date_opened = datetime(2014, 6, 21)
    sample_doc = dict(
        _id=uuid.uuid4().hex,
        opened_on=json_format_datetime(date_opened),
        owner_id='some-user-id',
        doc_type="CommCareCase",
        domain='user-reports',
        name='sample name',
        type='ticket',
        category='bug',
        tags='easy-win public',
        is_starred='yes',
        estimate=2.3,
        priority=4,
    )
    expected_indicators = {
        'doc_id': sample_doc['_id'],
        'repeat_iteration': 0,
        'date': date_opened,
        'owner': 'some-user-id',
        'count': 1,
        'category_bug': 1, 'category_feature': 0, 'category_app': 0, 'category_schedule': 0,
        'tags_easy-win': 1, 'tags_potential-dupe': 0, 'tags_roadmap': 0, 'tags_public': 1,
        'is_starred': 1,
        'estimate': Decimal(2.3),
        'priority': 4,
        'inserted_at': fake_time_now,
    }
    return sample_doc, expected_indicators

def doc_to_change(doc):
    return Change(
        id=doc['_id'],
        sequence_id='0',
        document=doc,
        metadata=ChangeMeta(
            document_id=doc['_id'],
            data_source_type=data_sources.SOURCE_COUCH,
            data_source_name=CommCareCase.get_db().dbname,
            document_type=doc['doc_type'],
            document_subtype=doc.get('type'),
            domain=doc['domain'],
            is_deletion=False,
        )
    )

def domain_lite(name):
    from corehq.apps.callcenter.utils import DomainLite
    return DomainLite(name, None, None, True)

def post_run_with_sql_backend(fn, *args, **kwargs):
    fn.doCleanups()
    fn.tearDown()

def pre_run_with_es_backend(fn, *args, **kwargs):
    fn.setUp()

def mock_datasource_config():
    return patch('corehq.apps.userreports.reports.data_source.get_datasource_config',
                 return_value=(get_sample_data_source(), None))

def get_simple_xform():
    xform = XFormBuilder()
    xform.new_question('first_name', 'First Name', data_type='string')
    xform.new_question('last_name', 'Last Name', data_type='string')
    xform.new_question('children', 'Children', data_type='int')
    xform.new_question('dob', 'Date of Birth', data_type='date')
    xform.new_question('state', 'State', data_type='select', choices={
        'MA': 'MA',
        'MN': 'MN',
        'VT': 'VT',
    })
    return xform.tostring()

def load_data_from_db(table_name):
    def _convert_decimal_to_string(value):
        value_str = str(value)
        p = re.compile('0E-(?P<zeros>[0-9]+)')
        match = p.match(value_str)
        if match:
            return '0.{}'.format(int(match.group('zeros')) * '0')
        else:
            return value_str

    engine = connection_manager.get_session_helper('default').engine
    metadata = sqlalchemy.MetaData(bind=engine)
    metadata.reflect(bind=engine)
    table = metadata.tables[table_name]
    columns = [
        column.name
        for column in table.columns
    ]
    with engine.begin() as connection:
        for row in list(connection.execute(table.select())):
            row = list(row)
            for idx, value in enumerate(row):
                if isinstance(value, date):
                    row[idx] = value.strftime('%Y-%m-%d')
                elif isinstance(value, time):
                    row[idx] = value.strftime("%H:%M:%S.%f").rstrip('0').rstrip('.')
                elif isinstance(value, six.integer_types):
                    row[idx] = str(value)
                elif isinstance(value, (float, Decimal)):
                    row[idx] = _convert_decimal_to_string(row[idx])
                elif isinstance(value, six.string_types):
                    row[idx] = value.encode('utf-8')
                elif value is None:
                    row[idx] = ''
            yield dict(zip(columns, row))
<commit_msg>add missing args<commit_after>from __future__ import absolute_import
from __future__ import unicode_literals
from datetime import datetime, date, time
from decimal import Decimal
import json
import os
import uuid
import re
import six
import sqlalchemy

from mock import patch
from six.moves import zip

from casexml.apps.case.models import CommCareCase
from corehq.apps.app_manager.xform_builder import XFormBuilder
from corehq.apps.change_feed import data_sources
from corehq.apps.userreports.models import DataSourceConfiguration, ReportConfiguration
from corehq.sql_db.connections import connection_manager
from dimagi.utils.parsing import json_format_datetime
from pillowtop.feed.interface import Change, ChangeMeta

from io import open

def get_sample_report_config():
    folder = os.path.join(os.path.dirname(__file__), 'data', 'configs')
    sample_file = os.path.join(folder, 'sample_report_config.json')
    with open(sample_file, encoding='utf-8') as f:
        structure = json.loads(f.read())
        return ReportConfiguration.wrap(structure)

def get_sample_data_source():
    folder = os.path.join(os.path.dirname(__file__), 'data', 'configs')
    sample_file = os.path.join(folder, 'sample_data_source.json')
    with open(sample_file, encoding='utf-8') as f:
        structure = json.loads(f.read())
        return DataSourceConfiguration.wrap(structure)

def get_data_source_with_related_doc_type():
    folder = os.path.join(os.path.dirname(__file__), 'data', 'configs')
    sample_file = os.path.join(folder, 'parent_child_data_source.json')
    with open(sample_file, encoding='utf-8') as f:
        structure = json.loads(f.read())
        return DataSourceConfiguration.wrap(structure)

def get_sample_doc_and_indicators(fake_time_now=None):
    if fake_time_now is None:
        fake_time_now = datetime.utcnow()
    date_opened = datetime(2014, 6, 21)
    sample_doc = dict(
        _id=uuid.uuid4().hex,
        opened_on=json_format_datetime(date_opened),
        owner_id='some-user-id',
        doc_type="CommCareCase",
        domain='user-reports',
        name='sample name',
        type='ticket',
        category='bug',
        tags='easy-win public',
        is_starred='yes',
        estimate=2.3,
        priority=4,
    )
    expected_indicators = {
        'doc_id': sample_doc['_id'],
        'repeat_iteration': 0,
        'date': date_opened,
        'owner': 'some-user-id',
        'count': 1,
        'category_bug': 1, 'category_feature': 0, 'category_app': 0, 'category_schedule': 0,
        'tags_easy-win': 1, 'tags_potential-dupe': 0, 'tags_roadmap': 0, 'tags_public': 1,
        'is_starred': 1,
        'estimate': Decimal(2.3),
        'priority': 4,
        'inserted_at': fake_time_now,
    }
    return sample_doc, expected_indicators

def doc_to_change(doc):
    return Change(
        id=doc['_id'],
        sequence_id='0',
        document=doc,
        metadata=ChangeMeta(
            document_id=doc['_id'],
            data_source_type=data_sources.SOURCE_COUCH,
            data_source_name=CommCareCase.get_db().dbname,
            document_type=doc['doc_type'],
            document_subtype=doc.get('type'),
            domain=doc['domain'],
            is_deletion=False,
        )
    )

def domain_lite(name):
    from corehq.apps.callcenter.utils import DomainLite
    return DomainLite(name, None, None, True, True, True, True)

def post_run_with_sql_backend(fn, *args, **kwargs):
    fn.doCleanups()
    fn.tearDown()

def pre_run_with_es_backend(fn, *args, **kwargs):
    fn.setUp()

def mock_datasource_config():
    return patch('corehq.apps.userreports.reports.data_source.get_datasource_config',
                 return_value=(get_sample_data_source(), None))

def get_simple_xform():
    xform = XFormBuilder()
    xform.new_question('first_name', 'First Name', data_type='string')
    xform.new_question('last_name', 'Last Name', data_type='string')
    xform.new_question('children', 'Children', data_type='int')
    xform.new_question('dob', 'Date of Birth', data_type='date')
    xform.new_question('state', 'State', data_type='select', choices={
        'MA': 'MA',
        'MN': 'MN',
        'VT': 'VT',
    })
    return xform.tostring()

def load_data_from_db(table_name):
    def _convert_decimal_to_string(value):
        value_str = str(value)
        p = re.compile('0E-(?P<zeros>[0-9]+)')
        match = p.match(value_str)
        if match:
            return '0.{}'.format(int(match.group('zeros')) * '0')
        else:
            return value_str

    engine = connection_manager.get_session_helper('default').engine
    metadata = sqlalchemy.MetaData(bind=engine)
    metadata.reflect(bind=engine)
    table = metadata.tables[table_name]
    columns = [
        column.name
        for column in table.columns
    ]
    with engine.begin() as connection:
        for row in list(connection.execute(table.select())):
            row = list(row)
            for idx, value in enumerate(row):
                if isinstance(value, date):
                    row[idx] = value.strftime('%Y-%m-%d')
                elif isinstance(value, time):
                    row[idx] = value.strftime("%H:%M:%S.%f").rstrip('0').rstrip('.')
                elif isinstance(value, six.integer_types):
                    row[idx] = str(value)
                elif isinstance(value, (float, Decimal)):
                    row[idx] = _convert_decimal_to_string(row[idx])
                elif isinstance(value, six.string_types):
                    row[idx] = value.encode('utf-8')
                elif value is None:
                    row[idx] = ''
            yield dict(zip(columns, row))
<|endoftext|>


<commit_before>"""\
OpenSpending test module
========================

Run the OpenSpending test suite by running

    nosetests

in the root of the repository, while in an active virtualenv. See
doc/install.rst for more information.
"""

import os
import sys

from paste.deploy import appconfig

from openspending import mongo
from helpers import clean_all

__all__ = ['TestCase', 'DatabaseTestCase']

here_dir = os.getcwd()
config = appconfig('config:test.ini', relative_to=here_dir)
mongo.configure(config)

class TestCase(object):
    def setup(self):
        pass

    def teardown(self):
        pass

class DatabaseTestCase(TestCase):
    def teardown(self):
        clean_all()
        super(DatabaseTestCase, self).teardown()<commit_msg>Use config given on command line<commit_after>"""\
OpenSpending test module
========================

Run the OpenSpending test suite by running

    nosetests

in the root of the repository, while in an active virtualenv. See
doc/install.rst for more information.
"""

import os
import sys

from pylons import config

from openspending import mongo
from .helpers import clean_all

__all__ = ['TestCase', 'DatabaseTestCase']

mongo.configure(config)

class TestCase(object):
    def setup(self):
        pass

    def teardown(self):
        pass

class DatabaseTestCase(TestCase):
    def teardown(self):
        clean_all()
        super(DatabaseTestCase, self).teardown()<|endoftext|>