<commit_before>import logging
logger = logging.getLogger(__name__)

def rpc_visibility(request, element):
    user = request.user
    if user.can("see", element):
        return True

def rpc_check_notifications(request):
    from rdrf.models import Notification
    user = request.user
    results = []
    notifications = Notification.objects.filter(
        to_username=user.username, seen=False).order_by('-created')
    for notification in notifications:
        results.append({"message": notification.message,
                        "from_user": notification.from_username, "link": notification.link})
    return results

def rpc_dismiss_notification(request, notification_id):
    from rdrf.models import Notification
    status = False
    try:
        notification = Notification.objects.get(pk=int(notification_id))
        notification.seen = True
        notification.save()
        status = True
    except Exception as ex:
        logger.error("could not mark notification with id %s as seen: %s" %
                     (notification_id, ex))
    return status

def rpc_fh_patient_is_index(request, patient_id):
    from registry.patients.models import Patient
    patient = Patient.objects.get(pk=patient_id)
    if patient.in_registry("fh"):
        is_index = patient.get_form_value(
            "fh", "ClinicalData", "fhDateSection", "CDEIndexOrRelative") != "fh_is_relative"
        return is_index
    else:
        return False

# Molecular Data Validation
def rpc_validate_dna(request, field_value):
    from rdrf.genetic_validation import GeneticValidator, GeneticType
    validator = GeneticValidator()
    return validator.validate(field_value, GeneticType.DNA)

def rpc_validate_exon(request, field_value):
    from rdrf.genetic_validation import GeneticValidator, GeneticType
    validator = GeneticValidator()
    return validator.validate(field_value, GeneticType.EXON)

def rpc_validate_rna(request, field_value):
    from rdrf.genetic_validation import GeneticValidator, GeneticType
    validator = GeneticValidator()
    return validator.validate(field_value, GeneticType.RNA)

def rpc_validate_protein(request, field_value):
    from rdrf.genetic_validation import GeneticValidator, GeneticType
    validator = GeneticValidator()
    return validator.validate(field_value, GeneticType.PROTEIN)

def rpc_reporting_command(request, queryId, registry_id, command, arg):
    # 2 possible commands/invocations client side from report definition screen:
    # get_field_data: used to build all the checkboxes for client
    # get_projection: process the checked checkboxes and get json representation
    # of the selected mongo fields ( used to build temp table)
    from rdrf.reporting_table import MongoFieldSelector
    from rdrf.models import Registry
    from explorer.models import Query
    user = request.user
    if queryId == "new":
        query_model = None
    else:
        query_model = Query.objects.get(pk=int(queryId))

    registry_model = Registry.objects.get(pk=int(registry_id))
    if command == "get_projection":
        checkbox_ids = arg["checkbox_ids"]
        longitudinal_ids = arg['longitudinal_ids']
        field_selector = MongoFieldSelector(user, registry_model, query_model, checkbox_ids, longitudinal_ids)
        result = field_selector.projections_json
        logger.debug("projections_json = %s" % result)
        return result
    elif command == "get_field_data":
        field_selector = MongoFieldSelector(user, registry_model, query_model)
        return field_selector.field_data
    else:
        raise Exception("unknown command: %s" % command)

# questionnaire handling

def rpc_load_matched_patient_data(request, patient_id, questionnaire_response_id):
    """
    Try to return any existing data for a patient corresponding the filled in values
    of a questionnaire filled out by on the questionnaire interface
    NB. The curator is responsible for matching an existing patient to the incoming
    questionnaire data.
    See RDR-1229 for a description of the use case.

    The existing data returned is the existing questionnaire values for this matched patient ( not the data
    provided in the questionnaire response itself - which potentially may overwrite the matched data if
    the curator indicates in the approval GUI.
    """
    from registry.patients.models import Patient
    from rdrf.models import QuestionnaireResponse
    from rdrf.questionnaires import Questionnaire

    questionnaire_response_model = QuestionnaireResponse.objects.get(pk=questionnaire_response_id)
    patient_model = Patient.objects.get(pk=patient_id)
    registry_model = questionnaire_response_model.registry
    questionnaire = Questionnaire(registry_model, questionnaire_response_model)
    existing_data = questionnaire.existing_data(patient_model)

    return { "link": existing_data.link,
             "name": existing_data.name,
             "questions": existing_data.questions}

def rpc_update_selected_cdes_from_questionnaire(request, patient_id, questionnaire_response_id, questionnaire_checked_ids):
    from registry.patients.models import Patient
    from rdrf.models import QuestionnaireResponse
    from rdrf.questionnaires import Questionnaire
    from django.db import transaction
    user = request.user
    questionnaire_response_model = QuestionnaireResponse.objects.get(pk=questionnaire_response_id)
    patient_model = Patient.objects.get(pk=patient_id)
    registry_model = questionnaire_response_model.registry
    questionnaire = Questionnaire(registry_model, questionnaire_response_model)
    mongo_data_before_update = patient_model.get_dynamic_data(registry_model)

    should_revert = False

    data_to_update = [question for question in questionnaire.questions if question.src_id in questionnaire_checked_ids]
    try:
        with transaction.atomic():
            errors = questionnaire.update_patient(patient_model, data_to_update)
            if len(errors) > 0:
                raise Exception("Errors occurred during update: %s" % ",".join(errors))
    except Exception, ex:
        should_revert = True
        logger.error("Update patient failed: rolled back: %s" % ex)

    if not should_revert:
        questionnaire_response_model.processed = True
        questionnaire_response_model.patient_id = patient_model.pk
        questionnaire_response_model.save()

        return {"status": "success", "message": "Patient updated successfully"}
    else:
        logger.info("Reverting to original mongo record for patient %s" % patient_id)
        patient_model.update_dynamic_data(registry_model, mongo_data_before_update)

        return {"status": "fail", "message": ",".join(errors)}

def rpc_create_patient_from_questionnaire(request, questionnaire_response_id):
    from rdrf.models import QuestionnaireResponse, Registry
    from rdrf.questionnaires import PatientCreator, PatientCreatorError
    from rdrf.dynamic_data import DynamicDataWrapper
    from django.db import transaction
    from django.core.urlresolvers import reverse

    qr = QuestionnaireResponse.objects.get(pk=questionnaire_response_id)
    patient_creator = PatientCreator(qr.registry, request.user)
    wrapper = DynamicDataWrapper(qr)
    questionnaire_data = wrapper.load_dynamic_data(qr.registry.code, "cdes")
    patient_id = None
    patient_blurb = None
    patient_link = None

    try:
        with transaction.atomic():
            created_patient = patient_creator.create_patient(None, qr, questionnaire_data)
            status = "success"
            message = "Patient created successfully"
            patient_blurb = "Patient %s created successfully" % created_patient
            patient_id = created_patient.pk
            default_context = created_patient.default_context(qr.registry)
            patient_link = reverse('patient_edit', args=[qr.registry.code, patient_id, default_context.pk])

    except PatientCreatorError, pce:
        message = "Error creating patient: %s.Patient not created" % pce
        status = "fail"

    except Exception, ex:
        message = "Unhandled error during patient creation: %s. Patient not created" % ex
        status = "fail"

    return {"status": status,
            "message": message,
            "patient_id": patient_id,
            "patient_name": "%s" % created_patient,
            "patient_link": patient_link,
            "patient_blurb" : patient_blurb}<commit_msg>RDR-1326 fixed edit link when patient created<commit_after>import logging
logger = logging.getLogger(__name__)

def rpc_visibility(request, element):
    user = request.user
    if user.can("see", element):
        return True

def rpc_check_notifications(request):
    from rdrf.models import Notification
    user = request.user
    results = []
    notifications = Notification.objects.filter(
        to_username=user.username, seen=False).order_by('-created')
    for notification in notifications:
        results.append({"message": notification.message,
                        "from_user": notification.from_username, "link": notification.link})
    return results

def rpc_dismiss_notification(request, notification_id):
    from rdrf.models import Notification
    status = False
    try:
        notification = Notification.objects.get(pk=int(notification_id))
        notification.seen = True
        notification.save()
        status = True
    except Exception as ex:
        logger.error("could not mark notification with id %s as seen: %s" %
                     (notification_id, ex))
    return status

def rpc_fh_patient_is_index(request, patient_id):
    from registry.patients.models import Patient
    patient = Patient.objects.get(pk=patient_id)
    if patient.in_registry("fh"):
        is_index = patient.get_form_value(
            "fh", "ClinicalData", "fhDateSection", "CDEIndexOrRelative") != "fh_is_relative"
        return is_index
    else:
        return False

# Molecular Data Validation
def rpc_validate_dna(request, field_value):
    from rdrf.genetic_validation import GeneticValidator, GeneticType
    validator = GeneticValidator()
    return validator.validate(field_value, GeneticType.DNA)

def rpc_validate_exon(request, field_value):
    from rdrf.genetic_validation import GeneticValidator, GeneticType
    validator = GeneticValidator()
    return validator.validate(field_value, GeneticType.EXON)

def rpc_validate_rna(request, field_value):
    from rdrf.genetic_validation import GeneticValidator, GeneticType
    validator = GeneticValidator()
    return validator.validate(field_value, GeneticType.RNA)

def rpc_validate_protein(request, field_value):
    from rdrf.genetic_validation import GeneticValidator, GeneticType
    validator = GeneticValidator()
    return validator.validate(field_value, GeneticType.PROTEIN)

def rpc_reporting_command(request, queryId, registry_id, command, arg):
    # 2 possible commands/invocations client side from report definition screen:
    # get_field_data: used to build all the checkboxes for client
    # get_projection: process the checked checkboxes and get json representation
    # of the selected mongo fields ( used to build temp table)
    from rdrf.reporting_table import MongoFieldSelector
    from rdrf.models import Registry
    from explorer.models import Query
    user = request.user
    if queryId == "new":
        query_model = None
    else:
        query_model = Query.objects.get(pk=int(queryId))

    registry_model = Registry.objects.get(pk=int(registry_id))
    if command == "get_projection":
        checkbox_ids = arg["checkbox_ids"]
        longitudinal_ids = arg['longitudinal_ids']
        field_selector = MongoFieldSelector(user, registry_model, query_model, checkbox_ids, longitudinal_ids)
        result = field_selector.projections_json
        logger.debug("projections_json = %s" % result)
        return result
    elif command == "get_field_data":
        field_selector = MongoFieldSelector(user, registry_model, query_model)
        return field_selector.field_data
    else:
        raise Exception("unknown command: %s" % command)

# questionnaire handling

def rpc_load_matched_patient_data(request, patient_id, questionnaire_response_id):
    """
    Try to return any existing data for a patient corresponding the filled in values
    of a questionnaire filled out by on the questionnaire interface
    NB. The curator is responsible for matching an existing patient to the incoming
    questionnaire data.
    See RDR-1229 for a description of the use case.

    The existing data returned is the existing questionnaire values for this matched patient ( not the data
    provided in the questionnaire response itself - which potentially may overwrite the matched data if
    the curator indicates in the approval GUI.
    """
    from registry.patients.models import Patient
    from rdrf.models import QuestionnaireResponse
    from rdrf.questionnaires import Questionnaire

    questionnaire_response_model = QuestionnaireResponse.objects.get(pk=questionnaire_response_id)
    patient_model = Patient.objects.get(pk=patient_id)
    registry_model = questionnaire_response_model.registry
    questionnaire = Questionnaire(registry_model, questionnaire_response_model)
    existing_data = questionnaire.existing_data(patient_model)

    return { "link": existing_data.link,
             "name": existing_data.name,
             "questions": existing_data.questions}

def rpc_update_selected_cdes_from_questionnaire(request, patient_id, questionnaire_response_id, questionnaire_checked_ids):
    from registry.patients.models import Patient
    from rdrf.models import QuestionnaireResponse
    from rdrf.questionnaires import Questionnaire
    from django.db import transaction
    user = request.user
    questionnaire_response_model = QuestionnaireResponse.objects.get(pk=questionnaire_response_id)
    patient_model = Patient.objects.get(pk=patient_id)
    registry_model = questionnaire_response_model.registry
    questionnaire = Questionnaire(registry_model, questionnaire_response_model)
    mongo_data_before_update = patient_model.get_dynamic_data(registry_model)

    should_revert = False

    data_to_update = [question for question in questionnaire.questions if question.src_id in questionnaire_checked_ids]
    try:
        with transaction.atomic():
            errors = questionnaire.update_patient(patient_model, data_to_update)
            if len(errors) > 0:
                raise Exception("Errors occurred during update: %s" % ",".join(errors))
    except Exception, ex:
        should_revert = True
        logger.error("Update patient failed: rolled back: %s" % ex)

    if not should_revert:
        questionnaire_response_model.processed = True
        questionnaire_response_model.patient_id = patient_model.pk
        questionnaire_response_model.save()

        return {"status": "success", "message": "Patient updated successfully"}
    else:
        logger.info("Reverting to original mongo record for patient %s" % patient_id)
        patient_model.update_dynamic_data(registry_model, mongo_data_before_update)

        return {"status": "fail", "message": ",".join(errors)}

def rpc_create_patient_from_questionnaire(request, questionnaire_response_id):
    from rdrf.models import QuestionnaireResponse, Registry
    from rdrf.questionnaires import PatientCreator, PatientCreatorError
    from rdrf.dynamic_data import DynamicDataWrapper
    from django.db import transaction
    from django.core.urlresolvers import reverse

    qr = QuestionnaireResponse.objects.get(pk=questionnaire_response_id)
    patient_creator = PatientCreator(qr.registry, request.user)
    wrapper = DynamicDataWrapper(qr)
    questionnaire_data = wrapper.load_dynamic_data(qr.registry.code, "cdes")
    patient_id = None
    patient_blurb = None
    patient_link = None

    try:
        with transaction.atomic():
            created_patient = patient_creator.create_patient(None, qr, questionnaire_data)
            status = "success"
            message = "Patient created successfully"
            patient_blurb = "Patient %s created successfully" % created_patient
            patient_id = created_patient.pk
            patient_link = reverse('patient_edit', args=[qr.registry.code, patient_id])

    except PatientCreatorError, pce:
        message = "Error creating patient: %s.Patient not created" % pce
        status = "fail"

    except Exception, ex:
        message = "Unhandled error during patient creation: %s. Patient not created" % ex
        status = "fail"

    return {"status": status,
            "message": message,
            "patient_id": patient_id,
            "patient_name": "%s" % created_patient,
            "patient_link": patient_link,
            "patient_blurb" : patient_blurb}
<|endoftext|>


<commit_before>import asyncio
import discord
from   discord.ext import commands

def name(member : discord.Member):
    # A helper function to return the member's display name
    if member.nick:
        return member.nick
    elif member.name:
        return member.name
    else:
        return None

def memberForID(id, server):
    for member in server.members:
        if member.id == id:
            return member
    return None

def roleForID(id, server):
    for role in server.roles:
        if role.id == id:
            return role
    return None

def roleForName(name, server):
    for role in server.roles:
        if role.name == name:
            return role
    return None<commit_msg>Add serverNick<commit_after>import asyncio
import discord
from   discord.ext import commands

def name(member : discord.Member):
    # A helper function to return the member's display name
    if member.nick:
        return member.nick
    elif member.name:
        return member.name
    else:
        return None

def memberForID(id, server):
    for member in server.members:
        if member.id == id:
            return member
    return None

def roleForID(id, server):
    for role in server.roles:
        if role.id == id:
            return role
    return None

def roleForName(name, server):
    for role in server.roles:
        if role.name == name:
            return role
    return None

def serverNick(user, server):
    for member in server.members:
        if member.id == user.id:
            return self.name(member)
    return None
<|endoftext|>


<commit_before>#   run as ctest .
#
# or
#
#   cd ../test ; env PYTHONPATH=../build python3 tests/realign.py ; cd ../build

import unittest
from pyvcflib import *
import json

class RealignTest(unittest.TestCase):

    # Returns True or False.
    def test1(self):
        vcf = VariantCallFile()
        vcf.openFile("../samples/10158243.vcf")
        rec = Variant(vcf)
        vcf.getNextVariant(rec)
        self.assertEqual(rec.name,"grch38#chr4")
        self.assertEqual(rec.ref,'ACCCCCACCCCCACC')
        self.assertEqual(rec.alt,['ACC', 'AC', 'ACCCCCACCCCCAC', 'ACCCCCACC', 'ACA'])
        sw = rec.legacy_parsedAlternates(False,False,False,10.0,-9.0,15.0,6.66,0.0,"","",False,False)

    def test_sw_wf_compare(self):
        vcf = VariantCallFile()
        vcf.openFile("../samples/10134514.vcf")
        rec = Variant(vcf)
        vcf.getNextVariant(rec)
        self.assertEqual(rec.name,"grch38#chr4_10083863-10181258.vcf:grch38#chr4")
        self.assertEqual(rec.ref,'GGAGAATCCCAATTGATGG')
        self.assertEqual(rec.alt,['GTAGCATCCCAAGTGATGT', 'GTAGAATCCCAATTGATGT', 'GGAGCATCCCAATTGATGG', 'GG'])
        sw = rec.legacy_parsedAlternates(False,False,False,10.0,-9.0,15.0,6.66,0.0,"","",False,False)
        # for key, value in sw.items():
        #     print(f'SW allele key: {key}: ')
        #     for a in value:
        #         print(f'               {a.position}/{a.ref}/{a.alt} ')
        # note wf ignores paramaters
        wf = rec.legacy_parsedAlternates(False,False,False,10.0,-9.0,15.0,6.66,0.0,"","",True,False)
        # for key, value in wf.items():
        #     print(f'WF allele key: {key}: ')
        #     for a in value:
        #        print(f'               {a.position}/{a.ref}/{a.alt} ')
        self.assertEqual(len(wf),5)
        self.assertEqual(len(wf),len(sw))
        self.assertEqual(sw['GGAGAATCCCAATTGATGG'][0].alt,wf['GGAGAATCCCAATTGATGG'][0].alt)

    def test_wfbug2(self):
        vcf = VariantCallFile()
        vcf.openFile("../samples/10134514.vcf")
        rec = Variant(vcf)
        vcf.getNextVariant(rec)
        self.assertEqual(rec.name,"grch38#chr4_10083863-10181258.vcf:grch38#chr4")
        self.assertEqual(rec.ref,'GGAGAATCCCAATTGATGG')
        self.assertEqual(rec.alt,['GTAGCATCCCAAGTGATGT', 'GTAGAATCCCAATTGATGT', 'GGAGCATCCCAATTGATGG', 'GG'])
        wfa_params = wavefront_aligner_attr_default
        # string paramString = "0,19,39,3,81,1";
        wfa_params.distance_metric = distance_meric_t.gap_affine_2p
        wfa_params.affine2p_penalties.match = 0
        wfa_params.affine2p_penalties.mismatch = 19
        wfa_params.affine2p_penalties.gap_opening1 = 39
        wfa_params.affine2p_penalties.gap_extension1 = 3
        wfa_params.affine2p_penalties.gap_opening2 = 81
        wfa_params.affine2p_penalties.gap_extension2 = 1
        wfa_params.alignment_scope = alignment_scope_t.compute_alignment;
        # A dict is returned of alleles with variants and is_reversed
        wf = rec.parsedAlternates(True,True,False,"","",wfa_params,True,64,True)
        print(f'ref={rec.ref}')
        print(rec.info)
        for key1, value1 in wf.items():
            print(f'WF2 allele key: {key1}: ')
            for a in value1[0]:
                print(f'               {a.position}:{a.ref}/{a.alt} ')
        # Run a few tests
        self.assertEqual(len(wf),5)
        gg0 = wf['GG'][0][0]
        gg1 = wf['GG'][0][1]
        self.assertEqual(gg0.alt,"GG")
        self.assertEqual(gg1.alt,"G")
        self.assertEqual(wf['GGAGAATCCCAATTGATGG'][0][0].alt,"GGAGAATCCCAATTGATGG")
        # Collect unique alleles
        info = rec.info
        self.assertEqual(info['AC'],['11', '7', '1', '3'])
        unique = {}
        a = None
        for alt0, wfvalue in wf.items(): # wfvalue is a compound of bool is_rev and alleles
            is_rev = wfvalue[1]
            for wfmatch in wfvalue[0]:
                ref = wfmatch.ref
                aligned = wfmatch.alt
                wfpos = wfmatch.position
                wftag = f'{alt0}:{wfpos}:{ref}/{aligned}'
                if rec.ref == aligned:
                    alt_index = -1
                    AC = None
                    AF = None
                    AN = None
                else:
                    alt_index = rec.alt.index(alt0) # Raises a ValueError if there is no such item
                    AC = int(info['AC'][alt_index])
                    AF = float(info['AF'][alt_index])
                    AN = int(info['AN'][0])
                relpos = wfpos - rec.pos
                unique[wftag] = {
                    'pos0': rec.pos,
                    'ref0': rec.ref,
                    'alt0': alt0,
                    'ref1': ref,
                    'algn': aligned,
                    'pos1': wfpos,
                    'altidx': alt_index, # zero based
                    'relpos': relpos,
                    'AC': AC,
                    'AF': AF,
                    'AN': AN,
                    'is_rev': is_rev}
        # Did we get all?
        self.assertEqual(len(unique.items()),18)
        # Display
        uniqsorted = sorted(unique.items(),key = lambda r: r[1]['pos1'])
        print(json.dumps(uniqsorted,indent=4))
        # Check if all alleles were used by counting 'altidx'
        idxs = set(map(lambda k: unique[k]['altidx'],unique.keys()))
        self.assertEqual(len(idxs),len(rec.alt)+1)

        # Collect sample genotypes
        samples = rec.samples
        gts = []
        for sname in rec.sampleNames:
            # print(name,samples[name])
            gt = (samples[sname]['GT'])[0].split("|")
            # print(gt)
            gts.append(list(map(lambda item: int(item) if item.isdigit() else None,gt)))
        print(gts)
        # for each variant translate genotypes
        for tag,aln in uniqsorted:
            # print(tag)
            idx1 = aln['altidx']+1
            # print(idx1)
            genotypes = []
            for gt in gts:
                # print(gt)
                genotypes.append(list(map(lambda item: item,gt)))
            # print(list(genotypes))
            # Now we neet to plug in the new indices
            for gt in genotypes:
                for i,g in enumerate(gt):
                    if g == idx1:
                        gt[i] = 1 # only one genotype in play
                    else:
                        if g != None:
                            gt[i] = 0
            # print(list(genotypes))
            aln['samples'] = genotypes
        gts = None
        print(uniqsorted[10])
        self.assertEqual(uniqsorted[10][1]['samples'],[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, None], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 1], [0, 0], [0, 0], [0, 0], [0]])

        # We are now going to merge records using a dict. From
        #
        # 10134515:G/T AC=7
        # 10134515:G/T AC=11
        # 10134516:AGAATCCCAATTGATGG/ AC=3
        # 10134518:A/C AC=1
        # 10134518:A/C AC=11
        # 10134526:T/G AC=11
        # 10134532:G/T AC=7
        # 10134532:G/T AC=11
        #   into
        # 10134515:G/T AC=18
        # 10134516:AGAATCCCAATTGATGG/ AC=3
        # 10134518:A/C AC=12
        # 10134526:T/G AC=11
        # 10134532:G/T AC=18
        variants = {} # store new hash
        k = None
        v = None
        for k,v in uniqsorted:
            ref = v['ref1']
            aligned = v['algn']
            if ref != aligned:
                ntag = f"{v['pos1']}:{ref}/{aligned}"
                print(f"{ntag} AC={v['AC']}")
                if ntag in variants:
                    variants[ntag]['AC'] += v['AC']
                    # Check AN number is equal so we can compute AF by addition
                    self.assertEqual(variants[ntag]['AN'],v['AN'])
                    variants[ntag]['AF'] += v['AF']
                    # Merge genotypes if they come from different alleles
                    if v['altidx'] != variants[ntag]['altidx']:
                        for i,samplesi in enumerate(variants[ntag]['samples']):
                            result = samplesi.copy()
                            g2 = v['samples'][i]
                            for j,samplej in enumerate(g2):
                                if g2[j] and g2[j]>0:
                                    result[j] = g2[j]
                            # print(i,samplesi,v['samples'][i],result)
                else:
                    variants[ntag] = v

        print("into")
        for key,v in variants.items():
            print(f"{key} AC={v['AC']}")

        self.assertEqual(len(variants),5)
        self.assertEqual(variants['10134532:G/T']['AC'],18)
        key = None
        v = None
        for key,v in variants.items():
            ref_len = len(v['ref1'])
            aln_len = len(v['algn'])
            type = None
            if aln_len < ref_len:
                type = 'del'
            elif aln_len > ref_len:
                type = 'del'
            elif aln_len == ref_len:
                if ref_len == 1:
                    type = 'snp'
                else:
                    type = 'mnp'
            variants[key]['type'] = type
            # Set origin
            print(v)
            variants[key]['origin'] = f'{rec.name}:{rec.pos}'
        # print(json.dumps(variants,indent=4))

if __name__ == '__main__':
    unittest.main()
<commit_msg>Ooops<commit_after>#   run as ctest .
#
# or
#
#   cd ../test ; env PYTHONPATH=../build python3 tests/realign.py ; cd ../build

import unittest
from pyvcflib import *
import json

class RealignTest(unittest.TestCase):

    # Returns True or False.
    def test1(self):
        vcf = VariantCallFile()
        vcf.openFile("../samples/10158243.vcf")
        rec = Variant(vcf)
        vcf.getNextVariant(rec)
        self.assertEqual(rec.name,"grch38#chr4")
        self.assertEqual(rec.ref,'ACCCCCACCCCCACC')
        self.assertEqual(rec.alt,['ACC', 'AC', 'ACCCCCACCCCCAC', 'ACCCCCACC', 'ACA'])
        sw = rec.legacy_parsedAlternates(False,False,False,10.0,-9.0,15.0,6.66,0.0,"","",False,False)

    def test_sw_wf_compare(self):
        vcf = VariantCallFile()
        vcf.openFile("../samples/10134514.vcf")
        rec = Variant(vcf)
        vcf.getNextVariant(rec)
        self.assertEqual(rec.name,"grch38#chr4_10083863-10181258.vcf:grch38#chr4")
        self.assertEqual(rec.ref,'GGAGAATCCCAATTGATGG')
        self.assertEqual(rec.alt,['GTAGCATCCCAAGTGATGT', 'GTAGAATCCCAATTGATGT', 'GGAGCATCCCAATTGATGG', 'GG'])
        sw = rec.legacy_parsedAlternates(False,False,False,10.0,-9.0,15.0,6.66,0.0,"","",False,False)
        # for key, value in sw.items():
        #     print(f'SW allele key: {key}: ')
        #     for a in value:
        #         print(f'               {a.position}/{a.ref}/{a.alt} ')
        # note wf ignores paramaters
        wf = rec.legacy_parsedAlternates(False,False,False,10.0,-9.0,15.0,6.66,0.0,"","",True,False)
        # for key, value in wf.items():
        #     print(f'WF allele key: {key}: ')
        #     for a in value:
        #        print(f'               {a.position}/{a.ref}/{a.alt} ')
        self.assertEqual(len(wf),5)
        self.assertEqual(len(wf),len(sw))
        self.assertEqual(sw['GGAGAATCCCAATTGATGG'][0].alt,wf['GGAGAATCCCAATTGATGG'][0].alt)

    def test_wfbug2(self):
        vcf = VariantCallFile()
        vcf.openFile("../samples/10134514.vcf")
        rec = Variant(vcf)
        vcf.getNextVariant(rec)
        self.assertEqual(rec.name,"grch38#chr4_10083863-10181258.vcf:grch38#chr4")
        self.assertEqual(rec.ref,'GGAGAATCCCAATTGATGG')
        self.assertEqual(rec.alt,['GTAGCATCCCAAGTGATGT', 'GTAGAATCCCAATTGATGT', 'GGAGCATCCCAATTGATGG', 'GG'])
        wfa_params = wavefront_aligner_attr_default
        # string paramString = "0,19,39,3,81,1";
        wfa_params.distance_metric = distance_meric_t.gap_affine_2p
        wfa_params.affine2p_penalties.match = 0
        wfa_params.affine2p_penalties.mismatch = 19
        wfa_params.affine2p_penalties.gap_opening1 = 39
        wfa_params.affine2p_penalties.gap_extension1 = 3
        wfa_params.affine2p_penalties.gap_opening2 = 81
        wfa_params.affine2p_penalties.gap_extension2 = 1
        wfa_params.alignment_scope = alignment_scope_t.compute_alignment;
        # A dict is returned of alleles with variants and is_reversed
        wf = rec.parsedAlternates(True,True,False,"","",wfa_params,True,64,True)
        print(f'ref={rec.ref}')
        print(rec.info)
        for key1, value1 in wf.items():
            print(f'WF2 allele key: {key1}: ')
            for a in value1[0]:
                print(f'               {a.position}:{a.ref}/{a.alt} ')
        # Run a few tests
        self.assertEqual(len(wf),5)
        gg0 = wf['GG'][0][0]
        gg1 = wf['GG'][0][1]
        self.assertEqual(gg0.alt,"GG")
        self.assertEqual(gg1.alt,"G")
        self.assertEqual(wf['GGAGAATCCCAATTGATGG'][0][0].alt,"GGAGAATCCCAATTGATGG")
        # Collect unique alleles
        info = rec.info
        self.assertEqual(info['AC'],['11', '7', '1', '3'])
        unique = {}
        a = None
        for alt0, wfvalue in wf.items(): # wfvalue is a compound of bool is_rev and alleles
            is_rev = wfvalue[1]
            for wfmatch in wfvalue[0]:
                ref = wfmatch.ref
                aligned = wfmatch.alt
                wfpos = wfmatch.position
                wftag = f'{alt0}:{wfpos}:{ref}/{aligned}'
                if rec.ref == aligned:
                    alt_index = -1
                    AC = None
                    AF = None
                    AN = None
                else:
                    alt_index = rec.alt.index(alt0) # Raises a ValueError if there is no such item
                    AC = int(info['AC'][alt_index])
                    AF = float(info['AF'][alt_index])
                    AN = int(info['AN'][0])
                relpos = wfpos - rec.pos
                unique[wftag] = {
                    'pos0': rec.pos,
                    'ref0': rec.ref,
                    'alt0': alt0,
                    'ref1': ref,
                    'algn': aligned,
                    'pos1': wfpos,
                    'altidx': alt_index, # zero based
                    'relpos': relpos,
                    'AC': AC,
                    'AF': AF,
                    'AN': AN,
                    'is_rev': is_rev}
        # Did we get all?
        self.assertEqual(len(unique.items()),18)
        # Display
        uniqsorted = sorted(unique.items(),key = lambda r: r[1]['pos1'])
        print(json.dumps(uniqsorted,indent=4))
        # Check if all alleles were used by counting 'altidx'
        idxs = set(map(lambda k: unique[k]['altidx'],unique.keys()))
        self.assertEqual(len(idxs),len(rec.alt)+1)

        # Collect sample genotypes
        samples = rec.samples
        gts = []
        for sname in rec.sampleNames:
            # print(name,samples[name])
            gt = (samples[sname]['GT'])[0].split("|")
            # print(gt)
            gts.append(list(map(lambda item: int(item) if item.isdigit() else None,gt)))
        print(gts)
        # for each variant translate genotypes
        for tag,aln in uniqsorted:
            # print(tag)
            idx1 = aln['altidx']+1
            # print(idx1)
            genotypes = []
            for gt in gts:
                # print(gt)
                genotypes.append(list(map(lambda item: item,gt)))
            # print(list(genotypes))
            # Now we neet to plug in the new indices
            for gt in genotypes:
                for i,g in enumerate(gt):
                    if g == idx1:
                        gt[i] = 1 # only one genotype in play
                    else:
                        if g != None:
                            gt[i] = 0
            # print(list(genotypes))
            aln['samples'] = genotypes
        gts = None
        print(uniqsorted[10])
        self.assertEqual(uniqsorted[10][1]['samples'],[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, None], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 1], [0, 0], [0, 0], [0, 0], [0]])

        # We are now going to merge records using a dict. From
        #
        # 10134515:G/T AC=7
        # 10134515:G/T AC=11
        # 10134516:AGAATCCCAATTGATGG/ AC=3
        # 10134518:A/C AC=1
        # 10134518:A/C AC=11
        # 10134526:T/G AC=11
        # 10134532:G/T AC=7
        # 10134532:G/T AC=11
        #   into
        # 10134515:G/T AC=18
        # 10134516:AGAATCCCAATTGATGG/ AC=3
        # 10134518:A/C AC=12
        # 10134526:T/G AC=11
        # 10134532:G/T AC=18
        variants = {} # store new hash
        k = None
        v = None
        for k,v in uniqsorted:
            ref = v['ref1']
            aligned = v['algn']
            if ref != aligned:
                ntag = f"{v['pos1']}:{ref}/{aligned}"
                print(f"{ntag} AC={v['AC']}")
                if ntag in variants:
                    variants[ntag]['AC'] += v['AC']
                    # Check AN number is equal so we can compute AF by addition
                    self.assertEqual(variants[ntag]['AN'],v['AN'])
                    variants[ntag]['AF'] += v['AF']
                    # Merge genotypes if they come from different alleles
                    if v['altidx'] != variants[ntag]['altidx']:
                        for i,samplesi in enumerate(variants[ntag]['samples']):
                            result = samplesi.copy()
                            g2 = v['samples'][i]
                            for j,samplej in enumerate(g2):
                                if g2[j] and g2[j]>0:
                                    result[j] = g2[j]
                            # print(i,samplesi,v['samples'][i],result)
                else:
                    variants[ntag] = v

        print("into")
        for key,v in variants.items():
            print(f"{key} AC={v['AC']}")

        self.assertEqual(len(variants),5)
        self.assertEqual(variants['10134532:G/T']['AC'],18)
        key = None
        v = None
        for key,v in variants.items():
            ref_len = len(v['ref1'])
            aln_len = len(v['algn'])
            type = None
            if aln_len < ref_len:
                type = 'del'
            elif aln_len > ref_len:
                type = 'ins'
            elif aln_len == ref_len:
                if ref_len == 1:
                    type = 'snp'
                else:
                    type = 'mnp'
            variants[key]['type'] = type
            # Set origin
            print(v)
            variants[key]['origin'] = f'{rec.name}:{rec.pos}'
        # print(json.dumps(variants,indent=4))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># Copyright 2016 Curtis Sand
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.

"""A library of Audio Signal Generators for making digital noises."""

import math
import numpy
from scipy import fftpack

from .common import defaults
from .waveform import Waveform
from .effects import normalize

class Generator(object):
    """A Basic Signal Generator.

    Each signal produced is generated independently of previously generated
    signals.
    """
    def __init__(self, length=None, framerate=None, verbose=False,
                 fade_percentage=None):
        self.length = length
        if not length:
            self.length = defaults.length
        self.framerate = framerate
        if not framerate:
            self.framerate = defaults.framerate
        self.verbose = verbose
        self.fade_percentage = fade_percentage if fade_percentage else 0.02

    def _init(self, length=None, framerate=None, verbose=None, **kwargs):
        if length:
            self.length = length
        if framerate:
            self.framerate = framerate
        if verbose:
            self.verbose = verbose

        # framecount = frames / sec * sec
        self.framecount = int(self.framerate * self.length)
        # rectify length to actual framecount
        self.length = float(self.framecount) / self.framerate
        self.dprint('generating %s frames' % self.framecount)
        self.wavedata = numpy.zeros(self.framecount)
        if 'phase' in kwargs:
            self.phase = kwargs['phase']
        else:
            self.phase = numpy.random.random() * 2 * math.pi

    @property
    def waveform(self):
        return Waveform(self.wavedata, self.framerate)

    def dprint(self, msg):
        """Conditionally print a debugging message."""
        if self.verbose:
            print(msg)

    def whitenoise(self, *args, **kwargs):
        """Random Gaussian White Noise."""
        self._init(*args, **kwargs)
        self.wavedata = numpy.random.randn(self.framecount)
        return self.wavedata

    def _sinusoid_value(self, frame, frequency):
        """Calculate the value of a sinusoid wave at a given frequency."""
        return math.sin(
            self.phase +
            frame / ((self.framerate / frequency) / math.pi))

    def sin_constant(self, frequency, *args, **kwargs):
        """Sinusoid wave of constant frequency."""
        self._init(*args, **kwargs)
        frequency = float(frequency)
        fade_frames = self.fade_percentage * self.framecount
        fade_point = self.framecount - fade_frames
        for frame in range(self.framecount):
            value = self._sinusoid_value(frame, frequency)
            if frame > fade_point:  # fade the end of the note
                _old = value
                value *= 1 - (frame - fade_point) / fade_frames
                if frame % 50 == 0:
                    self.dprint('fade from %s to %s' % (_old, value))
            if frame < fade_frames:
                _old = value
                value *= frame / fade_frames
            self.wavedata[frame] = value
        return self.waveform

    def sin_linear(self, start_freq, end_freq, *args, **kwargs):
        """Sinusoid wave of linearly changing frequency."""
        self._init(*args, **kwargs)
        for frame in range(len(self.wavedata)):
            frequency = start_freq + frame * (
                float(end_freq - start_freq) / self.framecount)
            value = self._sinusoid_value(frame, frequency)
            self.wavedata[frame] = value
        return self.waveform

class FFTGenerator(Generator):
    '''Use an Inverse Fourier Transform to create a multifrequency sinusoid.

    The generated sinusoid is a single waveform comprised of multiple
    frequencies that were not generated as their own fundamental waveforms
    first.
    '''
    def __init__(self, length=None, framerate=None, verbose=False):
        self.approx_desired_precision = 10  # Hz
        self.length = length
        if not length:
            self.length = defaults.length
        self.framerate = framerate
        if not framerate:
            self.framerate = defaults.framerate
        self.verbose = verbose

    @property
    def window_size(self):
        """Length of a window size as determined by the desired precision."""
        return int(self.framerate / 2 / self.approx_desired_precision)

    @property
    def new_window(self):
        """Create a new empty array for a window."""
        return numpy.zeros(self.window_size)

    @property
    def frequencies(self):
        """The frequencies mapped to bins in the frequency domain."""
        return fftpack.fftfreq(self.window_size, 1.0 / self.framerate)

    def _get_frequency_bin(self, requested_freq):
        """Find the FFT bin corresponding closest to requested frequency."""
        diff = 20000  # Need large value, 20KHz is high enough
        closest_index = None
        for index, freq in enumerate(self.frequencies):
            if freq < 0:
                continue
            tmp_diff = abs(requested_freq - freq)
            if tmp_diff < diff:
                diff = tmp_diff
                closest_index = index
        self.dprint('using bin %s for freq %s' %
                    (closest_index, requested_freq))
        return closest_index

    def generate(self, frequencies, **kwargs):
        """Generate the requested waveform."""
        super(FFTGenerator, self)._init(**kwargs)
        wavedata = Waveform(numpy.zeros(int(self.framerate * self.length)))
        freq_domain_stub = self.new_window
        for frequency in frequencies:
            ifft_bin = self._get_frequency_bin(frequency)
            freq_domain_stub[ifft_bin] = self.framerate / len(frequencies)
        window = Waveform(normalize(numpy.real(
            fftpack.ifft(freq_domain_stub))))
        for count, frame in enumerate(range(0, self.framecount,
                                            self.window_size)):
            wavedata = wavedata.insert(frame, window)
        self.dprint('{} generated {}, {} length windows'.format(
            self.__class__.__name__, count, self.window_size))
        self.wavedata = wavedata
        return wavedata
<commit_msg>signal_generator: add first attempt at a continuous sinusoid generator<commit_after># Copyright 2016 Curtis Sand
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.

"""A library of Audio Signal Generators for making digital noises."""

import math
import numpy
from scipy import fftpack

from .common import defaults
from .waveform import Waveform
from .effects import normalize

class Generator(object):
    """A Basic Signal Generator.

    Each signal produced is generated independently of previously generated
    signals.
    """
    def __init__(self, length=None, framerate=None, verbose=False,
                 fade_percentage=None):
        self.length = length
        if not length:
            self.length = defaults.length
        self.framerate = framerate
        if not framerate:
            self.framerate = defaults.framerate
        self.verbose = verbose
        self.fade_percentage = fade_percentage if fade_percentage else 0.02

    def _init(self, length=None, framerate=None, verbose=None, **kwargs):
        if length:
            self.length = length
        if framerate:
            self.framerate = framerate
        if verbose:
            self.verbose = verbose

        # framecount = frames / sec * sec
        self.framecount = int(self.framerate * self.length)
        # rectify length to actual framecount
        self.length = float(self.framecount) / self.framerate
        self.dprint('generating %s frames' % self.framecount)
        self.wavedata = numpy.zeros(self.framecount)
        if 'phase' in kwargs:
            self.phase = kwargs['phase']
        else:
            self.phase = numpy.random.random() * 2 * math.pi

    @property
    def waveform(self):
        return Waveform(self.wavedata, self.framerate)

    def dprint(self, msg):
        """Conditionally print a debugging message."""
        if self.verbose:
            print(msg)

    def whitenoise(self, *args, **kwargs):
        """Random Gaussian White Noise."""
        self._init(*args, **kwargs)
        self.wavedata = numpy.random.randn(self.framecount)
        return self.wavedata

    def _sinusoid_value(self, frame, frequency):
        """Calculate the value of a sinusoid wave at a given frequency."""
        return math.sin(
            self.phase +
            frame / ((self.framerate / frequency) / math.pi))

    def sin_constant(self, frequency, *args, **kwargs):
        """Sinusoid wave of constant frequency."""
        self._init(*args, **kwargs)
        frequency = float(frequency)
        fade_frames = self.fade_percentage * self.framecount
        fade_point = self.framecount - fade_frames
        for frame in range(self.framecount):
            value = self._sinusoid_value(frame, frequency)
            if frame > fade_point:  # fade the end of the note
                _old = value
                value *= 1 - (frame - fade_point) / fade_frames
                if frame % 50 == 0:
                    self.dprint('fade from %s to %s' % (_old, value))
            if frame < fade_frames:
                _old = value
                value *= frame / fade_frames
            self.wavedata[frame] = value
        return self.waveform

    def sin_linear(self, start_freq, end_freq, *args, **kwargs):
        """Sinusoid wave of linearly changing frequency."""
        self._init(*args, **kwargs)
        for frame in range(len(self.wavedata)):
            frequency = start_freq + frame * (
                float(end_freq - start_freq) / self.framecount)
            value = self._sinusoid_value(frame, frequency)
            self.wavedata[frame] = value
        return self.waveform

class FFTGenerator(Generator):
    '''Use an Inverse Fourier Transform to create a multifrequency sinusoid.

    The generated sinusoid is a single waveform comprised of multiple
    frequencies that were not generated as their own fundamental waveforms
    first.
    '''
    def __init__(self, length=None, framerate=None, verbose=False):
        self.approx_desired_precision = 10  # Hz
        self.length = length
        if not length:
            self.length = defaults.length
        self.framerate = framerate
        if not framerate:
            self.framerate = defaults.framerate
        self.verbose = verbose

    @property
    def window_size(self):
        """Length of a window size as determined by the desired precision."""
        return int(self.framerate / 2 / self.approx_desired_precision)

    @property
    def new_window(self):
        """Create a new empty array for a window."""
        return numpy.zeros(self.window_size)

    @property
    def frequencies(self):
        """The frequencies mapped to bins in the frequency domain."""
        return fftpack.fftfreq(self.window_size, 1.0 / self.framerate)

    def _get_frequency_bin(self, requested_freq):
        """Find the FFT bin corresponding closest to requested frequency."""
        diff = 20000  # Need large value, 20KHz is high enough
        closest_index = None
        for index, freq in enumerate(self.frequencies):
            if freq < 0:
                continue
            tmp_diff = abs(requested_freq - freq)
            if tmp_diff < diff:
                diff = tmp_diff
                closest_index = index
        self.dprint('using bin %s for freq %s' %
                    (closest_index, requested_freq))
        return closest_index

    def generate(self, frequencies, **kwargs):
        """Generate the requested waveform."""
        super(FFTGenerator, self)._init(**kwargs)
        wavedata = Waveform(numpy.zeros(int(self.framerate * self.length)))
        freq_domain_stub = self.new_window
        for frequency in frequencies:
            ifft_bin = self._get_frequency_bin(frequency)
            freq_domain_stub[ifft_bin] = self.framerate / len(frequencies)
        window = Waveform(normalize(numpy.real(
            fftpack.ifft(freq_domain_stub))))
        for count, frame in enumerate(range(0, self.framecount,
                                            self.window_size)):
            wavedata = wavedata.insert(frame, window)
        self.dprint('{} generated {}, {} length windows'.format(
            self.__class__.__name__, count, self.window_size))
        self.wavedata = wavedata
        return wavedata

class ContinuousGenerator(Generator):
    """Generate and accumulate a continuous signal accross multiple notes.

    For the description below,
        LengthA="lenth_of_note - (1/2 * transition_length)"
    and
        LengthB="length_of_note - transition_length".

    For the first note generate a constant note of length LengthA.

    For subsequent notes generate a linear signal from the freq. of the first
    note to that of the second of the set transition length, followed by a
    constant note of length LengthB.

    If the keyword "end=True" is used then the constant portion of the length
    should only be LengthA. ::

        |         |            |            |
        {------}\ | LengthB    | LengthA    |
        |        \|            |            |
        | LengthA |\           |/{----------}
        |         | \{-------}/|            |

    Note: Due to the fact that this generator currently ignores the phase of
    the sinusoid being generated, when the frequency is modulated during a
    transition period there are some unfortunate problems. When the frequency
    of the waveform shifts we need to compensate by adding a phase shift so
    that we are using the same phase angle of a sinusoid of a new frequency.
    Since we ignore phase though, both frequency and phase shift and we end up
    seeing an antialiased waveform of an apparent frequency much higher than
    either of the two frequencies being transitioned between.

    TODO: See if I can figure out the appropriate phase calculation to anchor
    the phase angle of the sinusoid as the frequency shifts during the
    transition period.

    """
    def __init__(self, length=None, framerate=None, verbose=False):
        super(ContinuousGenerator, self).__init__(length, framerate, verbose)
        self.phase = 0  # don't do any random phase shifting
        self.frequency = 0.001  # avoid divide by zero
        self.end = False
        self.start = True
        self.last_frame = 0
        self.wavedata = numpy.zeros(1)
        self.transition_length = int(self.framerate * 0.1)
        if self.transition_length % 2 != 0:  # need even length transition
            self.transition_length += 1

    @property
    def _constant_length(self):
        adjustment = self.transition_length
        if len(self.wavedata) <= 1 or self.end:
            adjustment /= 2
        return int(self.framecount - adjustment)

    def _init(self, frequency=None, length=None, verbose=None, end=None,
              **kwargs):
        if frequency:
            self.last_frequency = self.frequency
            self.frequency = frequency
        if length:
            self.length = length
        if verbose:
            self.verbose = verbose
        if end:
            self.end = end

        # framecount = frames / sec * sec
        self.framecount = int(self.framerate * self.length)
        # rectify length to actual framecount
        self.length = float(self.framecount) / self.framerate

    def _prep_wavedata(self, transition=False):
        # save the frame where the next note starts
        adjustment = self.transition_length
        if not transition:
            adjustment = self._constant_length
        new_block = numpy.zeros(adjustment)
        self.wavedata = numpy.concatenate((self.wavedata, new_block))

    def _constant(self):
        """Append sinusoid wave of constant frequency to wavedata."""
        self.dprint('constant freq from frame %s to %s' %
                    (self.last_frame,
                     self.last_frame + self._constant_length))
        frequency = float(self.frequency)
        for frame in range(self.last_frame,
                           self.last_frame + self._constant_length):
            value = self._sinusoid_value(frame, frequency)
            print('const frame %s at value %s at %s' % (
                frame, value, frequency))
            self.wavedata[frame] = value
        self.last_frame = frame

    def _transition(self):
        """Append sinusoid wave of linearly changing frequency to wavedata."""
        self.dprint('transition from frame %s to %s' %
                    (self.last_frame,
                     self.last_frame + self.transition_length))
        for frame in range(self.last_frame,
                           self.last_frame + self.transition_length):
            modifier = ((frame - self.last_frame) *
                        float(self.frequency - self.last_frequency) /
                        self.transition_length)
            frequency = (self.last_frequency + modifier)
            value = self._sinusoid_value(frame, frequency)
            print('transition frame %s at %s value %s: mod %s' % (
                frame, frequency, value, modifier))
            self.wavedata[frame] = value
        self.last_frame = frame

    def generate(self, frequency, length, end=False, *args, **kwargs):
        self._init(frequency=frequency, length=length, end=end,
                   *args, **kwargs)
        self.dprint('generating %s new frames at %s' % (self.framecount,
                                                        frequency))
        if self.start:
            self.dprint('Starting initial frequency in the signal...')
            self._prep_wavedata()
            self._constant()
            self.dprint('signal is now %s long' % len(self.wavedata))
            self.start = False
        else:
            self.dprint('Adding transition... %s to %s' % (
                self.last_frequency, self.frequency))
            self._prep_wavedata(transition=True)
            self._transition()
            self.dprint('  transition now %s long' % len(self.wavedata))
            self.dprint('Adding the %s note...' % self.frequency)
            self._prep_wavedata()
            self._constant()
            self.dprint('signal is now %s long' % len(self.wavedata))
<|endoftext|>


<commit_before>import glob
import socket
import sys
import threading
import logging

import serial
from PySide2 import QtCore, QtGui, QtWidgets

from Codecs.AdcpCodec import AdcpCodec
from Comm.AdcpSerialPortServer import AdcpSerialPortServer

import configparser
settings = configparser.ConfigParser()
settings._interpolation = configparser.ExtendedInterpolation()
settings.read('../settings.ini')

logger = logging.getLogger("Ensemble File Report")
logger.setLevel(logging.DEBUG)
FORMAT = '[%(asctime)-15s][%(levelname)s][%(funcName)s] %(message)s'
logging.basicConfig(format=FORMAT)

class view_serial(QtWidgets.QWidget):
    """
    Create the QT display
    """
    def __init__(self, parent=None):
        QtWidgets.QWidget.__init__(self, parent)

        # Global variables

        self.serial_server = None
        self.raw_serial_socket = None
        self.raw_serial_alive = False
        self.ensemble_reader_thread = None
        self.adcp_writer_thread = None
        self.serial_buffer = ""

        raw_serial_tcp_port = settings.get('SerialServerSection', 'RawSerialTcpPort')
        ens_udp_port = settings.get('SerialServerSection', 'JsonEnsUdpPort')

        # QT GUI

        self.tcp_port_combobox = QtWidgets.QComboBox()
        self.tcp_port_combobox.setEditable(True)
        self.tcp_port_combobox.addItems([raw_serial_tcp_port, ""])

        self.ens_udp_port_combobox = QtWidgets.QComboBox()
        self.ens_udp_port_combobox.setEditable(True)
        self.ens_udp_port_combobox.addItems([ens_udp_port, ""])

        self.comm_port_combobox = QtWidgets.QComboBox()
        self.comm_port_combobox.addItems(self.serial_ports())

        self.comm_baud_combobox = QtWidgets.QComboBox()
        self.comm_baud_combobox.setEditable(True)
        self.comm_baud_combobox.addItems(["921600", "115200", "19200", ""])
        self.comm_baud_combobox.setCurrentIndex(1)

        connect = QtWidgets.QPushButton("Connect")
        connect.setFont(QtGui.QFont("Times", 18, QtGui.QFont.Bold))
        connect.clicked.connect(self.start_adcp_server)

        disconnect = QtWidgets.QPushButton("Reconnect")
        disconnect.setFont(QtGui.QFont("Times", 18, QtGui.QFont.Bold))
        disconnect.clicked.connect(self.reconnect_adcp_server)

        self.command_txtbox = QtWidgets.QLineEdit()

        send_cmd_btn = QtWidgets.QPushButton("Send")
        #send_cmd_btn.setFont(QtGui.QFont("Times", 18, QtGui.QFont.Bold))
        send_cmd_btn.clicked.connect(self.send_cmd_adcp_server)

        send_break_btn = QtWidgets.QPushButton("BREAK")
        #send_break_btn.setFont(QtGui.QFont("Times", 18, QtGui.QFont.Bold))
        send_break_btn.clicked.connect(self.send_break_adcp_server)

        #quit = QtWidgets.QPushButton("Quit")
        #quit.setFont(QtGui.QFont("Times", 18, QtGui.QFont.Bold))
        #self.connect(quit, QtCore.SIGNAL("clicked()"),
        #             QtWidgets.qApp, QtCore.SLOT("quit()"))

        self.serial_txtbox = QtWidgets.QTextEdit()
        self.serial_txtbox.setMinimumHeight(500)

        clear_btn = QtWidgets.QPushButton("Clear")
        #send_break_btn.setFont(QtGui.QFont("Times", 18, QtGui.QFont.Bold))
        clear_btn.clicked.connect(self.clear_serial)

        # Add Widgets
        gridLayout = QtWidgets.QGridLayout()
        gridLayout.addWidget(self.tcp_port_combobox, 0, 0)
        gridLayout.addWidget(self.ens_udp_port_combobox, 0, 1)
        gridLayout.addWidget(self.comm_port_combobox, 1, 0)
        gridLayout.addWidget(self.comm_baud_combobox, 1, 1)
        gridLayout.addWidget(connect, 2, 0)
        gridLayout.addWidget(disconnect, 2, 1)
        gridLayout.addWidget(self.command_txtbox, 3, 0)
        gridLayout.addWidget(send_cmd_btn, 3, 1)
        gridLayout.addWidget(send_break_btn, 3, 2)
        gridLayout.addWidget(self.serial_txtbox, 4, 0, 2, 3)    # row, cl, rs, cs
        gridLayout.addWidget(clear_btn, 5, 2)

        # Set Spacing
        #gridLayout.setColumnStretch(1, 10)
        gridLayout.setRowStretch(4, 20)

        # Add Menu
        #self.createMenu()
        #gridLayout.setMenuBar(self.createMenu())
        gridLayout.addWidget(self.createMenu())

        # Set Window attributes
        self.setLayout(gridLayout)
        self.setWindowTitle("Echo - RoweTech Inc.")

    def createMenu(self):
        menuBar = QtWidgets.QMenuBar()

        fileMenu = QtWidgets.QMenu("File")
        exitAction = fileMenu.addAction("E&xit")
        menuBar.addMenu(fileMenu)

        #exitAction.triggered.connect(self.exit)

        return menuBar

    def closeEvent(self, event):
        """
        Override the close event for the QWidget.
        If a serial port is open, close the connection.
        """
        logger.debug('Closing serial')
        self.stop_adcp_server()

        # Close window
        event.accept()

    def start_adcp_server(self):
        """
        Start the ADCP Serial TCP server
        """
        self.serial_server = AdcpSerialPortServer(self.get_tcp_port(),
                                                  self.comm_port_combobox.currentText(),
                                                  self.get_baud())

        # Create an ensemble reader
        #self.ensemble_reader_thread = threading.Thread(name='EnsembleReader', target=EnsembleReader.EnsembleReader(self.get_tcp_port())).start()

        # Connect to serial server to send commands
        self.adcp_writer_thread = threading.Thread(name='AdcpWriter', target=self.create_raw_serial_socket(self.get_tcp_port())).start()

        logger.debug("start server")

    def create_raw_serial_socket(self, port):
        """
        Connect to the ADCP serial server.
        """
        try:
            self.raw_serial_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self.raw_serial_socket.connect(('localhost', int(port)))
            self.raw_serial_socket.settimeout(1)    # Set timeout to stop thread if terminated
        except ConnectionRefusedError as err:
            logger.error("Serial Send Socket: ", err)
        except Exception as err:
            logger.error('Serial Send Socket: ", Error Opening socket', err)

        # Start a thread to read the raw data
        self.ensemble_reader_thread = ReadRawSerialThread(self.raw_serial_socket, int(self.ens_udp_port_combobox.currentText()))
        self.ensemble_reader_thread.raw_data.connect(self.on_raw_read)
        self.ensemble_reader_thread.start()

    @QtCore.Slot(object)
    def on_raw_read(self, data):
        """
        Buffer up the data received from the serial port from
        the TCP server.  Then set it to the text.
        :param data: Data received from the TCP server which came from the
                     serial port.
        """
        try:
            self.serial_buffer += bytes(data).decode()
        except Exception as err:
            #print("error decoding", err)
            # Decoding does not work for the binary data, only commands
            self.serial_buffer += str(data).strip()

        # Chop off the top portion of the message
        slicer = slice(-1130, None)
        self.serial_buffer = self.serial_buffer[slicer]

        # Set the text to the textbox
        self.serial_txtbox.setText(self.serial_buffer)

    def stop_adcp_server(self):
        """
        Stop the ADCP Serial TCP server
        """
        if self.serial_server is not None:
            self.serial_server.close()
            logger.debug("serial server stopped")
        else:
            logger.debug('No serial connection')

        # Close the socket
        self.raw_serial_socket.close()

        if self.adcp_writer_thread is not None:
            self.adcp_writer_thread.join()

        if self.ensemble_reader_thread is not None:
            #self.ensemble_reader_thread.terminate()
            #self.ensemble_reader_thread.setTerminationEnabled(True)
            self.ensemble_reader_thread.stop()

    def send_cmd_adcp_server(self):
        """
        Send the command to the socket
        """
        # Get the command from the txtbox
        cmd = self.command_txtbox.text().strip()

        # Encode the data to byte array and send to socket
        self.raw_serial_socket.send((cmd.strip()).encode())

    def send_break_adcp_server(self):
        """
        Send the BREAK command to the socket
        """
        cmd = "BREAK"

        # Encode the data to byte array and send to socket
        self.raw_serial_socket.send((cmd.strip()).encode())

    def reconnect_adcp_server(self):
        """
        Reconnect the serial port connection with the new
        settings.
        """
        print("reconnect")

    def get_baud(self):
        """
        Convert the baud rate from string to int.
        If an error, set the baud rate to 115200

        :returns: baud rate
        """
        try:
            baud = int(self.comm_baud_combobox.currentText())
        except Exception as err:
            logger.error("Error setting baud rate: ", err)
            self.comm_baud_combobox.setCurrentIndex(1)
            return "115200"

        return baud

    def get_tcp_port(self):
        """
        Convert the TCP port from string to int.
        If an error, set the default port to 55056

        :returns: TCP Port
        """
        try:
            port = int(self.tcp_port_combobox.currentText())
        except Exception as err:
            logger.error("Error setting TCP Port: ", err)
            self.tcp_port_combobox.setCurrentIndex(0)
            return "55056"

        return str(port)

    def serial_ports(self):
        """ Lists serial port names

        :raises EnvironmentError:
            On unsupported or unknown platforms
        :returns:
            A list of the serial ports available on the system
        """

        if sys.platform.startswith('win'):
            ports = ['COM%s' % (i + 1) for i in range(256)]
        elif sys.platform.startswith('linux') or sys.platform.startswith('cygwin'):
            # this excludes your current terminal "/dev/tty"
            ports = glob.glob('/dev/tty[A-Za-z]*')
        elif sys.platform.startswith('darwin'):
            ports = glob.glob('/dev/tty.*')
        else:
            raise EnvironmentError('Unsupported platform')

        result = []
        for port in ports:
            try:
                s = serial.Serial(port)
                s.close()
                result.append(port)
            except OSError as err:
                logger.error(err)
                pass
            except serial.SerialException as err:
                logger.error(err)
                pass
        return result

    def clear_serial(self):
        """
        Clear the serial port display.
        """
        # Set the text to the textbox
        self.serial_txtbox.setText("")

    def find_port(self):
        """
        Finds available port for Tornado / Flask
        :return: Available port
        :rtype: int
        """

        port_attempts = 0
        while port_attempts < 1000:
            try:
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                sock.bind(('localhost', 0))
                app_port = sock.getsockname()[1]
                sock.close()
                logger.debug("PORT: " + str(app_port))
                return app_port
            except Exception as err:
                port_attempts += 1
                logger.error(err)

        logger.error("FAILED AFTER 1000 PORT ATTEMPTS")
        sys.exit(1)

class ReadRawSerialThread(QtCore.QThread):
    """
    Create a Read raw serial data from TCP port thread.
    """

    raw_data = QtCore.Signal(object)

    def __init__(self, tcp_socket, ens_port, parent=None):
        QtCore.QThread.__init__(self, parent)
        self.socket = tcp_socket
        self.isAlive = True
        logger.debug("Read Socket thread started")

        # Initialize the ADCP Codec
        self.codec = AdcpCodec(ens_port)

        # Setup Waves
        ens_in_burst = settings.get('WavesProjectSection', 'EnsemblesInBurst')
        file_path = settings.get('WavesProjectSection', 'CaptureFilePath')
        lat = settings.get('WavesProjectSection', 'Lat')
        lon = settings.get('WavesProjectSection', 'Lon')
        bin1 = settings.get('WavesProjectSection', 'Bin1')
        bin2 = settings.get('WavesProjectSection', 'Bin2')
        bin3 = settings.get('WavesProjectSection', 'Bin3')
        self.codec.enable_waveforce_codec(int(ens_in_burst), file_path, float(lat), float(lon), int(bin1), int(bin2), int(bin3))

    def stop(self):
        """
        Stop the thread by setting the isAlive flag.
        """
        self.isAlive = False

    def run(self):
        """
        Run the loop that views the data from the serial port.
        :return:
        """
        self.exec()

    def exec(self):
        """
        Run the loop to view data from the serial port.
        Emit the data so the view can view the data.

        """
        while self.isAlive:
            try:
                # Read data from socket
                data = self.socket.recv(4096)

                # If data exist process
                if len(data) > 0:
                    self.raw_data.emit(data)

                    # Pass data to the decoder
                    self.codec.add(data)
            except socket.timeout:
                # Just a socket timeout, continue on
                pass

        logger.debug("Read Thread turned off")

<commit_msg>Fixed UI reconnect<commit_after>import glob
import socket
import sys
import threading
import logging

import serial
from PySide2 import QtCore, QtGui, QtWidgets

from Codecs.AdcpCodec import AdcpCodec
from Comm.AdcpSerialPortServer import AdcpSerialPortServer

import configparser
settings = configparser.ConfigParser()
settings._interpolation = configparser.ExtendedInterpolation()
settings.read('../settings.ini')

logger = logging.getLogger("Ensemble File Report")
logger.setLevel(logging.DEBUG)
FORMAT = '[%(asctime)-15s][%(levelname)s][%(funcName)s] %(message)s'
logging.basicConfig(format=FORMAT)

class view_serial(QtWidgets.QWidget):
    """
    Create the QT display
    """
    def __init__(self, parent=None):
        QtWidgets.QWidget.__init__(self, parent)

        # Global variables

        self.serial_server = None
        self.raw_serial_socket = None
        self.raw_serial_alive = False
        self.ensemble_reader_thread = None
        self.adcp_writer_thread = None
        self.serial_buffer = ""

        raw_serial_tcp_port = settings.get('SerialServerSection', 'RawSerialTcpPort')
        ens_udp_port = settings.get('SerialServerSection', 'JsonEnsUdpPort')

        # QT GUI

        self.tcp_port_combobox = QtWidgets.QComboBox()
        self.tcp_port_combobox.setEditable(True)
        self.tcp_port_combobox.addItems([raw_serial_tcp_port, ""])

        self.ens_udp_port_combobox = QtWidgets.QComboBox()
        self.ens_udp_port_combobox.setEditable(True)
        self.ens_udp_port_combobox.addItems([ens_udp_port, ""])

        self.comm_port_combobox = QtWidgets.QComboBox()
        self.comm_port_combobox.addItems(self.serial_ports())

        self.comm_baud_combobox = QtWidgets.QComboBox()
        self.comm_baud_combobox.setEditable(True)
        self.comm_baud_combobox.addItems(["921600", "115200", "19200", ""])
        self.comm_baud_combobox.setCurrentIndex(1)

        connect = QtWidgets.QPushButton("Connect")
        connect.setFont(QtGui.QFont("Times", 18, QtGui.QFont.Bold))
        connect.clicked.connect(self.start_adcp_server)

        disconnect = QtWidgets.QPushButton("Reconnect")
        disconnect.setFont(QtGui.QFont("Times", 18, QtGui.QFont.Bold))
        disconnect.clicked.connect(self.reconnect_adcp_server)

        self.command_txtbox = QtWidgets.QLineEdit()

        send_cmd_btn = QtWidgets.QPushButton("Send")
        #send_cmd_btn.setFont(QtGui.QFont("Times", 18, QtGui.QFont.Bold))
        send_cmd_btn.clicked.connect(self.send_cmd_adcp_server)

        send_break_btn = QtWidgets.QPushButton("BREAK")
        #send_break_btn.setFont(QtGui.QFont("Times", 18, QtGui.QFont.Bold))
        send_break_btn.clicked.connect(self.send_break_adcp_server)

        #quit = QtWidgets.QPushButton("Quit")
        #quit.setFont(QtGui.QFont("Times", 18, QtGui.QFont.Bold))
        #self.connect(quit, QtCore.SIGNAL("clicked()"),
        #             QtWidgets.qApp, QtCore.SLOT("quit()"))

        self.serial_txtbox = QtWidgets.QTextEdit()
        self.serial_txtbox.setMinimumHeight(500)

        clear_btn = QtWidgets.QPushButton("Clear")
        #send_break_btn.setFont(QtGui.QFont("Times", 18, QtGui.QFont.Bold))
        clear_btn.clicked.connect(self.clear_serial)

        # Add Widgets
        gridLayout = QtWidgets.QGridLayout()
        gridLayout.addWidget(self.tcp_port_combobox, 0, 0)
        gridLayout.addWidget(self.ens_udp_port_combobox, 0, 1)
        gridLayout.addWidget(self.comm_port_combobox, 1, 0)
        gridLayout.addWidget(self.comm_baud_combobox, 1, 1)
        gridLayout.addWidget(connect, 2, 0)
        gridLayout.addWidget(disconnect, 2, 1)
        gridLayout.addWidget(self.command_txtbox, 3, 0)
        gridLayout.addWidget(send_cmd_btn, 3, 1)
        gridLayout.addWidget(send_break_btn, 3, 2)
        gridLayout.addWidget(self.serial_txtbox, 4, 0, 2, 3)    # row, cl, rs, cs
        gridLayout.addWidget(clear_btn, 5, 2)

        # Set Spacing
        #gridLayout.setColumnStretch(1, 10)
        gridLayout.setRowStretch(4, 20)

        # Add Menu
        #self.createMenu()
        #gridLayout.setMenuBar(self.createMenu())
        gridLayout.addWidget(self.createMenu())

        # Set Window attributes
        self.setLayout(gridLayout)
        self.setWindowTitle("Echo - RoweTech Inc.")

    def createMenu(self):
        menuBar = QtWidgets.QMenuBar()

        fileMenu = QtWidgets.QMenu("File")
        exitAction = fileMenu.addAction("E&xit")
        menuBar.addMenu(fileMenu)

        #exitAction.triggered.connect(self.exit)

        return menuBar

    def closeEvent(self, event):
        """
        Override the close event for the QWidget.
        If a serial port is open, close the connection.
        """
        logger.debug('Closing serial')
        self.stop_adcp_server()

        # Close window
        event.accept()

    def start_adcp_server(self):
        """
        Start the ADCP Serial TCP server
        """
        self.serial_server = AdcpSerialPortServer(self.get_tcp_port(),
                                                  self.comm_port_combobox.currentText(),
                                                  self.get_baud())

        # Create an ensemble reader
        #self.ensemble_reader_thread = threading.Thread(name='EnsembleReader', target=EnsembleReader.EnsembleReader(self.get_tcp_port())).start()

        # Connect to serial server to send commands
        self.adcp_writer_thread = threading.Thread(name='AdcpWriter', target=self.create_raw_serial_socket(self.get_tcp_port())).start()

        logger.debug("start server")

    def create_raw_serial_socket(self, port):
        """
        Connect to the ADCP serial server.
        """
        try:
            self.raw_serial_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self.raw_serial_socket.connect(('localhost', int(port)))
            self.raw_serial_socket.settimeout(1)    # Set timeout to stop thread if terminated
        except ConnectionRefusedError as err:
            logger.error("Serial Send Socket: ", err)
        except Exception as err:
            logger.error('Serial Send Socket: ", Error Opening socket', err)

        # Start a thread to read the raw data
        self.ensemble_reader_thread = ReadRawSerialThread(self.raw_serial_socket, int(self.ens_udp_port_combobox.currentText()))
        self.ensemble_reader_thread.raw_data.connect(self.on_raw_read)
        self.ensemble_reader_thread.start()

    @QtCore.Slot(object)
    def on_raw_read(self, data):
        """
        Buffer up the data received from the serial port from
        the TCP server.  Then set it to the text.
        :param data: Data received from the TCP server which came from the
                     serial port.
        """
        try:
            self.serial_buffer += bytes(data).decode()
        except Exception as err:
            #print("error decoding", err)
            # Decoding does not work for the binary data, only commands
            self.serial_buffer += str(data).strip()

        # Chop off the top portion of the message
        slicer = slice(-1130, None)
        self.serial_buffer = self.serial_buffer[slicer]

        # Set the text to the textbox
        self.serial_txtbox.setText(self.serial_buffer)

    def stop_adcp_server(self):
        """
        Stop the ADCP Serial TCP server
        """
        if self.serial_server is not None:
            self.serial_server.close()
            logger.debug("serial server stopped")
        else:
            logger.debug('No serial connection')

        # Close the socket
        self.raw_serial_socket.close()

        if self.adcp_writer_thread is not None:
            self.adcp_writer_thread.join()

        if self.ensemble_reader_thread is not None:
            #self.ensemble_reader_thread.terminate()
            #self.ensemble_reader_thread.setTerminationEnabled(True)
            self.ensemble_reader_thread.stop()

    def send_cmd_adcp_server(self):
        """
        Send the command to the socket
        """
        # Get the command from the txtbox
        cmd = self.command_txtbox.text().strip()

        # Encode the data to byte array and send to socket
        self.raw_serial_socket.send((cmd.strip()).encode())

    def send_break_adcp_server(self):
        """
        Send the BREAK command to the socket
        """
        cmd = "BREAK"

        # Encode the data to byte array and send to socket
        self.raw_serial_socket.send((cmd.strip()).encode("utf-16"))

    def reconnect_adcp_server(self):
        """
        Reconnect the serial port connection with the new
        settings.
        """
        logger.debug("reconnect")
        cmd = "RECONNECT, " + self.comm_port_combobox.currentText() + "," + self.comm_baud_combobox.currentText()

        # Encode the data to byte array and send to socket
        self.raw_serial_socket.send((cmd.strip()).encode("utf-16"))

    def get_baud(self):
        """
        Convert the baud rate from string to int.
        If an error, set the baud rate to 115200

        :returns: baud rate
        """
        try:
            baud = int(self.comm_baud_combobox.currentText())
        except Exception as err:
            logger.error("Error setting baud rate: ", err)
            self.comm_baud_combobox.setCurrentIndex(1)
            return "115200"

        return baud

    def get_tcp_port(self):
        """
        Convert the TCP port from string to int.
        If an error, set the default port to 55056

        :returns: TCP Port
        """
        try:
            port = int(self.tcp_port_combobox.currentText())
        except Exception as err:
            logger.error("Error setting TCP Port: ", err)
            self.tcp_port_combobox.setCurrentIndex(0)
            return "55056"

        return str(port)

    def serial_ports(self):
        """ Lists serial port names

        :raises EnvironmentError:
            On unsupported or unknown platforms
        :returns:
            A list of the serial ports available on the system
        """

        if sys.platform.startswith('win'):
            ports = ['COM%s' % (i + 1) for i in range(256)]
        elif sys.platform.startswith('linux') or sys.platform.startswith('cygwin'):
            # this excludes your current terminal "/dev/tty"
            ports = glob.glob('/dev/tty[A-Za-z]*')
        elif sys.platform.startswith('darwin'):
            ports = glob.glob('/dev/tty.*')
        else:
            raise EnvironmentError('Unsupported platform')

        result = []
        for port in ports:
            try:
                s = serial.Serial(port)
                s.close()
                result.append(port)
            except OSError as err:
                logger.error(err)
                pass
            except serial.SerialException as err:
                logger.error(err)
                pass
        return result

    def clear_serial(self):
        """
        Clear the serial port display.
        """
        # Set the text to the textbox
        self.serial_txtbox.setText("")

    def find_port(self):
        """
        Finds available port for Tornado / Flask
        :return: Available port
        :rtype: int
        """

        port_attempts = 0
        while port_attempts < 1000:
            try:
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                sock.bind(('localhost', 0))
                app_port = sock.getsockname()[1]
                sock.close()
                logger.debug("PORT: " + str(app_port))
                return app_port
            except Exception as err:
                port_attempts += 1
                logger.error(err)

        logger.error("FAILED AFTER 1000 PORT ATTEMPTS")
        sys.exit(1)

class ReadRawSerialThread(QtCore.QThread):
    """
    Create a Read raw serial data from TCP port thread.
    """

    raw_data = QtCore.Signal(object)

    def __init__(self, tcp_socket, ens_port, parent=None):
        QtCore.QThread.__init__(self, parent)
        self.socket = tcp_socket
        self.isAlive = True
        logger.debug("Read Socket thread started")

        # Initialize the ADCP Codec
        self.codec = AdcpCodec(ens_port)

        # Setup Waves
        ens_in_burst = settings.get('WavesProjectSection', 'EnsemblesInBurst')
        file_path = settings.get('WavesProjectSection', 'CaptureFilePath')
        lat = settings.get('WavesProjectSection', 'Lat')
        lon = settings.get('WavesProjectSection', 'Lon')
        bin1 = settings.get('WavesProjectSection', 'Bin1')
        bin2 = settings.get('WavesProjectSection', 'Bin2')
        bin3 = settings.get('WavesProjectSection', 'Bin3')
        self.codec.enable_waveforce_codec(int(ens_in_burst), file_path, float(lat), float(lon), int(bin1), int(bin2), int(bin3))

    def stop(self):
        """
        Stop the thread by setting the isAlive flag.
        """
        self.isAlive = False

    def run(self):
        """
        Run the loop that views the data from the serial port.
        :return:
        """
        self.exec()

    def exec(self):
        """
        Run the loop to view data from the serial port.
        Emit the data so the view can view the data.

        """
        while self.isAlive:
            try:
                # Read data from socket
                data = self.socket.recv(4096)

                # If data exist process
                if len(data) > 0:
                    self.raw_data.emit(data)

                    # Pass data to the decoder
                    self.codec.add(data)
            except socket.timeout:
                # Just a socket timeout, continue on
                pass

        logger.debug("Read Thread turned off")

<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
The 'polyjit' experiment.

This experiment uses likwid to measure the performance of all binaries
when running with polyjit support enabled.
"""
from pprof.experiments.compilestats import get_compilestats
from pprof.experiment import step, substep, RuntimeExperiment
from pprof.settings import config
from pprof.utils.schema import CompileStat

from plumbum import local
from os import path

class PolyJIT(RuntimeExperiment):

    """The polyjit experiment."""

    def run_step_compilestats(self, p):
        """ Compile the project and track the compilestats. """
        with step("Track Compilestats @ -O3"):
            p.clean()
            p.prepare()
            p.download()
            with substep("Configure Project"):
                def track_compilestats(cc, **kwargs):
                    from pprof.utils import run as r
                    from pprof.utils.db import persist_compilestats
                    from pprof.utils.run import handle_stdin

                    new_cc = handle_stdin(cc["-mllvm", "-stats"], kwargs)

                    run, session = r.begin(new_cc, p.name, self.name,
                                           p.run_uuid)
                    retcode, stdout, stderr = r.guarded_exec(new_cc)

                    if retcode == 0:
                        stats = []
                        for stat in get_compilestats(stderr):
                            c = CompileStat()
                            c.name = stat["desc"].rstrip()
                            c.component = stat["component"].rstrip()
                            c.value = stat["value"]
                            stats.append(c)
                        persist_compilestats(run, session, stats)
                    r.end(run, session, stdout, stderr)

                p.compiler_extension = track_compilestats
                p.configure()

            with substep("Build Project"):
                p.build()

    def run_step_jit(self, p):
        """Run the experiment without likwid."""
        with step("JIT, no instrumentation"):
            p.clean()
            p.prepare()
            p.download()
            with substep("Build"):
                p.configure()
                p.build()
            with substep("Execute {}".format(p.name)):
                def run_with_time(run_f, args, **kwargs):
                    from pprof.utils import run as r
                    from pprof.utils.db import persist_time
                    from plumbum.cmd import time

                    project_name = kwargs.get("project_name", p.name)
                    timing_tag = "PPROF-JIT: "

                    run_cmd = time["-f", timing_tag + "%U-%S-%e", run_f]
                    run_cmd = r.handle_stdin(run_cmd[args], kwargs)

                    run, session = r.begin(str(run_cmd), project_name,
                                           self.name, p.run_uuid)

                    retcode, stdout, stderr = r.guarded_exec(run_cmd)
                    timings = r.fetch_time_output(
                        timing_tag, timing_tag + "{:g}-{:g}-{:g}",
                        stderr.split("\n"))
                    if len(timings) == 0:
                        return

                    persist_time(run, session, timings)
                    r.end(run, session, stdout, stderr)
                p.run(run_with_time)

    def run_step_likwid(self, p):
        """Run the experiment with likwid."""
        with step("JIT, likwid"):
            p.clean()
            p.prepare()
            p.download()
            p.cflags = ["-DLIKWID_PERFMON"] + p.cflags

            with substep("Build"):
                p.configure()
                p.build()
            with substep("Execute {}".format(p.name)):
                from pprof.settings import config

                def run_with_likwid(run_f, args, **kwargs):
                    from pprof.utils import run as r
                    from pprof.utils.db import persist_likwid
                    from pprof.likwid import get_likwid_perfctr
                    from plumbum.cmd import rm
                    from uuid import uuid4

                    project_name = kwargs.get("project_name", p.name)
                    likwid_f = p.name + ".txt"

                    for group in ["CLOCK"]:
                        likwid_path = path.join(config["likwiddir"], "bin")
                        likwid_perfctr = local[
                            path.join(likwid_path, "likwid-perfctr")]
                        for i in range(int(config["jobs"])):
                            run_cmd = \
                                likwid_perfctr["-O", "-o", likwid_f, "-m",
                                               "-C", "0-{:d}".format(i),
                                               "-g", group, run_f]

                            run_cmd = r.handle_stdin(run_cmd[args], kwargs)
                            run, session = r.begin(run_cmd, project_name,
                                                   self.name, p.run_uuid)

                            retcode, stdout, stderr = r.guarded_exec(run_cmd)

                            likwid_measurement = get_likwid_perfctr(likwid_f)
                            """ Use the project_name from the binary, because we
                                might encounter dynamically generated projects.
                            """
                            persist_likwid(run, session, likwid_measurement)
                            r.end(run, session, stdout, stderr)
                            rm("-f", likwid_f)
                            p.run_uuid = uuid4()
                p.run(run_with_likwid)

    def run_project(self, p):
        """
        Execute the pprof experiment.

        We perform this experiment in 2 steps:
            1. with likwid disabled.
            2. with likwid enabled.
        """
        p.ldflags = ["-lpjit", "-lgomp"]

        ld_lib_path = filter(None, config["ld_library_path"].split(":"))
        p.ldflags = ["-L" + el for el in ld_lib_path] + p.ldflags
        p.cflags = ["-Rpass=\"polyjit*\"",
                    "-Xclang", "-load",
                    "-Xclang", "LLVMPolyJIT.so",
                    "-O3",
                    "-mllvm", "-jitable",
                    "-mllvm", "-polly-delinearize=false",
                    "-mllvm", "-polly-detect-keep-going",
                    "-mllvm", "-polli"]
        with local.env(PPROF_ENABLE=0):
            self.run_step_likwid(p)
            self.run_step_jit(p)
            self.run_step_compilestats(p)
<commit_msg>polyjit: new run_group per core<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
The 'polyjit' experiment.

This experiment uses likwid to measure the performance of all binaries
when running with polyjit support enabled.
"""
from pprof.experiments.compilestats import get_compilestats
from pprof.experiment import step, substep, RuntimeExperiment
from pprof.settings import config
from pprof.utils.schema import CompileStat

from plumbum import local
from os import path

class PolyJIT(RuntimeExperiment):

    """The polyjit experiment."""

    def run_step_compilestats(self, p):
        """ Compile the project and track the compilestats. """
        with step("Track Compilestats @ -O3"):
            p.clean()
            p.prepare()
            p.download()
            with substep("Configure Project"):
                def track_compilestats(cc, **kwargs):
                    from pprof.utils import run as r
                    from pprof.utils.db import persist_compilestats
                    from pprof.utils.run import handle_stdin

                    new_cc = handle_stdin(cc["-mllvm", "-stats"], kwargs)

                    run, session = r.begin(new_cc, p.name, self.name,
                                           p.run_uuid)
                    retcode, stdout, stderr = r.guarded_exec(new_cc)

                    if retcode == 0:
                        stats = []
                        for stat in get_compilestats(stderr):
                            c = CompileStat()
                            c.name = stat["desc"].rstrip()
                            c.component = stat["component"].rstrip()
                            c.value = stat["value"]
                            stats.append(c)
                        persist_compilestats(run, session, stats)
                    r.end(run, session, stdout, stderr)

                p.compiler_extension = track_compilestats
                p.configure()

            with substep("Build Project"):
                p.build()

    def run_step_jit(self, p):
        """Run the experiment without likwid."""
        with step("JIT, no instrumentation"):
            p.clean()
            p.prepare()
            p.download()
            with substep("Build"):
                p.configure()
                p.build()
            with substep("Execute {}".format(p.name)):
                def run_with_time(run_f, args, **kwargs):
                    from pprof.utils import run as r
                    from pprof.utils.db import persist_time
                    from plumbum.cmd import time

                    project_name = kwargs.get("project_name", p.name)
                    timing_tag = "PPROF-JIT: "

                    run_cmd = time["-f", timing_tag + "%U-%S-%e", run_f]
                    run_cmd = r.handle_stdin(run_cmd[args], kwargs)

                    run, session = r.begin(str(run_cmd), project_name,
                                           self.name, p.run_uuid)

                    retcode, stdout, stderr = r.guarded_exec(run_cmd)
                    timings = r.fetch_time_output(
                        timing_tag, timing_tag + "{:g}-{:g}-{:g}",
                        stderr.split("\n"))
                    if len(timings) == 0:
                        return

                    persist_time(run, session, timings)
                    r.end(run, session, stdout, stderr)
                p.run(run_with_time)

    def run_step_likwid(self, p):
        """Run the experiment with likwid."""
        from pprof.settings import config
        def run_with_likwid(run_f, args, **kwargs):
            from pprof.utils import run as r
            from pprof.utils.db import persist_likwid
            from pprof.likwid import get_likwid_perfctr
            from plumbum.cmd import rm
            from uuid import uuid4

            project_name = kwargs.get("project_name", p.name)
            likwid_f = p.name + ".txt"

            for group in ["CLOCK"]:
                likwid_path = path.join(config["likwiddir"], "bin")
                likwid_perfctr = local[
                    path.join(likwid_path, "likwid-perfctr")]
                for i in range(int(config["jobs"])):
                    with substep("{} cores & uuid {}".format(i+1, p.run_uuid)):
                        run_cmd = \
                            likwid_perfctr["-O", "-o", likwid_f, "-m",
                                           "-C", "0-{:d}".format(i),
                                           "-g", group, run_f]
                        run_cmd = r.handle_stdin(run_cmd[args], kwargs)

                        run, session = r.begin(run_cmd, project_name,
                                               self.name, p.run_uuid)

                        retcode, stdout, stderr = r.guarded_exec(run_cmd)

                        likwid_measurement = get_likwid_perfctr(likwid_f)
                        """ Use the project_name from the binary, because we
                            might encounter dynamically generated projects.
                        """
                        persist_likwid(run, session, likwid_measurement)
                        r.end(run, session, stdout, stderr)
                        rm("-f", likwid_f)
                        p.run_uuid = uuid4()

        with step("JIT, likwid"):
            p.clean()
            p.prepare()
            p.download()
            p.cflags = ["-DLIKWID_PERFMON"] + p.cflags

            with substep("Build"):
                p.configure()
                p.build()
            with substep("Execute {}".format(p.name)):
                p.run(run_with_likwid)

    def run_project(self, p):
        """
        Execute the pprof experiment.

        We perform this experiment in 2 steps:
            1. with likwid disabled.
            2. with likwid enabled.
        """
        p.ldflags = ["-lpjit", "-lgomp"]

        ld_lib_path = filter(None, config["ld_library_path"].split(":"))
        p.ldflags = ["-L" + el for el in ld_lib_path] + p.ldflags
        p.cflags = ["-Rpass=\"polyjit*\"",
                    "-Xclang", "-load",
                    "-Xclang", "LLVMPolyJIT.so",
                    "-O3",
                    "-mllvm", "-jitable",
                    "-mllvm", "-polly-delinearize=false",
                    "-mllvm", "-polly-detect-keep-going",
                    "-mllvm", "-polli"]
        with local.env(PPROF_ENABLE=0):
            self.run_step_likwid(p)
            self.run_step_jit(p)
            self.run_step_compilestats(p)
<|endoftext|>