<commit_before>from collections import namedtuple
import copy
import datetime
from decimal import Decimal
import re
import rfc822
import time

import phpserialize

from ubersmith.api import METHODS, get_default_request_handler
from ubersmith.exceptions import ValidationError

__all__ = [
    # abstract call classes
    'BaseCall',
    'GroupCall',
    'FileCall',
    # generate generic calls
    'generate_generic_calls',
    # concrete call classes
    'client',
    'device',
    'order',
    'sales',
    'support',
    'uber',
]

_CLEANERS = {
    'bool': bool,
    'int': int,
    'decimal': lambda x: Decimal(x.replace(',', '')),
    'float': float,
    'timestamp': lambda x: datetime.datetime.fromtimestamp(float(x)),
    'date': lambda x: datetime.date(*time.strptime(x, '%b/%d/%Y')[:3]),
    'php_serialized': phpserialize.loads,
}

class BaseCall(object):
    """Abstract class to implement a call with validation/cleaning/etc."""

    method = ''  # ubersmith method name, should be defined on child classes
    required_fields = []  # field names that should be present in request_data
    rename_fields = {}  # fields to rename
    bool_fields = []  # fields to convert to ints
    int_fields = []  # fields to convert to ints
    decimal_fields = []  # fields to convert to decimals
    float_fields = []   # fields to convert to floats
    timestamp_fields = []  # fields to convert from timestamps to datetime
    date_fields = []  # fields to convert to datetime.date
    php_serialized_fields = []  # fields to convert from php serialized format

    def __init__(self, request_data=None, request_handler=None):
        """Setup call with provided request data and handler."""
        self.request_data = request_data or {}  # data for the request
        self.request_handler = request_handler or \
            get_default_request_handler()  # handler to fullfil the request
        self.response_data = None  # response data is stored here
        self.cleaned = None  # cleaned response data is stored here

    def render(self):
        """Validate, process, clean and return the result of the call."""
        if not self.validate():
            raise ValidationError

        self.process_request()
        self.clean()

        return self.cleaned

    def validate(self):
        """Validate request data before sending it out. Return True/False."""
        # check if required_fields aren't present
        for field in set(self.required_fields) - set(self.request_data):
            if not isinstance(field, basestring):
                # field was a collection, iterate over it and check by OR
                return bool(set(field) & set(self.request_data))
            return False
        return True

    def process_request(self):
        """Processing the call and set response_data."""
        self.response_data = self.request_handler.process_request(self.method,
                                                            self.request_data)

    def clean(self):
        """Clean response data."""
        cleaned = copy.deepcopy(self.response_data)
        _clean_fields(self, cleaned)
        self.cleaned = cleaned

class GroupCall(BaseCall):
    """Abstract class to implement a call that returns a group of results."""

    def clean(self):
        cleaned = copy.deepcopy(self.response_data)

        # convert top level keys to ints
        for key in self.response_data.iterkeys():
            _rename_key(cleaned, key, int(key))

        # clean fields on each member of the group
        for member in self.cleaned.itervalues():
            _clean_fields(self, member)

        self.cleaned = cleaned

class FileCall(BaseCall):
    """Abstract class to implement a call that returns a file."""
    _UbersmithFile = namedtuple('UbersmithFile', ['filename', 'type',
                                                  'modified', 'data'])

    def process_request(self):
        """Processing the call and set response_data."""
        self.response_data = self.request_handler.process_request(self.method,
                                                            self.request_data,
                                                            raw=True)

    def clean(self):
        fname = None
        disposition = self.response_data[0].get('content-disposition')
        if disposition:
            fname = re.search(r'filename="(.+?)"', disposition, re.I).group(1)
            fname = re.sub(r'[^a-z0-9-_\. ]', '-', fname, 0, re.I).lstrip('.')

        self.filename = fname
        self.type = self.response_data[0].get('content-type')
        last_modified = self.response_data[0].get('last-modified')
        if last_modified:
            self.modified = datetime.datetime(
                                      *rfc822.parsedate_tz(last_modified)[:7])
        else:
            self.modified = datetime.datetime.now()
        self.data = buffer(self.response_data[1])

        self.cleaned = self._UbersmithFile(self.filename, self.type,
                                           self.modified, self.data)

def _rename_key(d, old, new):
    """Rename a key on d from old to new."""
    if old in d and new not in d:
        d[new] = d[old]
        del d[old]

def _clean_fields(call, d):
    """Rename and clean fields on d using info on call."""
    # rename fields
    for old, new in call.rename_fields.iteritems():
        _rename_key(d, old, unicode(new))

    # clean fields
    for name, func in _CLEANERS.iteritems():
        for field in getattr(call, '{0}_fields'.format(name), []):
            if field in d and isinstance(d[field], basestring):
                d[field] = func(d[field])

def _get_call_class(method):
    """Find the call class for method if it exists else create one."""
    call_base, call_name = method.split('.', 1)
    # import the call class's module
    mod = __import__('ubersmith.calls.{0}'.format(call_base), fromlist=[''])
    # grab all the public members of the module
    gen = (getattr(mod, x) for x in dir(mod) if not x.startswith('_'))
    # filter them down to subclasses of BaseCall
    gen = (x for x in gen if type(x) is type and issubclass(x, BaseCall))
    # return first one that matches our method
    for call_class in gen:
        if call_class.method == method:
            return call_class
    else:
        class GenericCall(BaseCall):
            method = '.'.join((call_base, call_name))
        return GenericCall

def _make_generic_call(call_class):
    """Create a call function that is lexically bound to use call_class."""
    def generic_call(request_handler=None, **kwargs):
        return call_class(kwargs, request_handler).render()
    return generic_call

def generate_generic_calls(base, ns):
    # get all valid methods with base
    methods = (m for m in METHODS if m.split('.', 1)[0] == base)
    for method in methods:
        call_name = method.split('.', 1)[1]
        if call_name not in ns:
            # find the appropriate class
            call_class = _get_call_class(method)
            # create a call function and stick it in the namespace
            generic_call = _make_generic_call(call_class)
            generic_call.__name__ = str(call_name)
            generic_call.__doc__ = METHODS[method]
            # TODO this may or may not be a good idea, see: http://stackoverflow.com/questions/10113892/semantics-of-module
            # generic_call.__module__ = 'ubersmith.{0}'.format(base)
            ns[call_name] = generic_call
            # add call to __all__ if needed
            if '__all__' in ns and call_name not in ns['__all__']:
                ns['__all__'].append(call_name)
        else:
            if not ns[call_name].__doc__:
                print "{} missing __doc__".format(method)
                ns[call_name].__doc__ = METHODS[method]
<commit_msg>forgot to remove debug statement<commit_after>from collections import namedtuple
import copy
import datetime
from decimal import Decimal
import re
import rfc822
import time

import phpserialize

from ubersmith.api import METHODS, get_default_request_handler
from ubersmith.exceptions import ValidationError

__all__ = [
    # abstract call classes
    'BaseCall',
    'GroupCall',
    'FileCall',
    # generate generic calls
    'generate_generic_calls',
    # concrete call classes
    'client',
    'device',
    'order',
    'sales',
    'support',
    'uber',
]

_CLEANERS = {
    'bool': bool,
    'int': int,
    'decimal': lambda x: Decimal(x.replace(',', '')),
    'float': float,
    'timestamp': lambda x: datetime.datetime.fromtimestamp(float(x)),
    'date': lambda x: datetime.date(*time.strptime(x, '%b/%d/%Y')[:3]),
    'php_serialized': phpserialize.loads,
}

class BaseCall(object):
    """Abstract class to implement a call with validation/cleaning/etc."""

    method = ''  # ubersmith method name, should be defined on child classes
    required_fields = []  # field names that should be present in request_data
    rename_fields = {}  # fields to rename
    bool_fields = []  # fields to convert to ints
    int_fields = []  # fields to convert to ints
    decimal_fields = []  # fields to convert to decimals
    float_fields = []   # fields to convert to floats
    timestamp_fields = []  # fields to convert from timestamps to datetime
    date_fields = []  # fields to convert to datetime.date
    php_serialized_fields = []  # fields to convert from php serialized format

    def __init__(self, request_data=None, request_handler=None):
        """Setup call with provided request data and handler."""
        self.request_data = request_data or {}  # data for the request
        self.request_handler = request_handler or \
            get_default_request_handler()  # handler to fullfil the request
        self.response_data = None  # response data is stored here
        self.cleaned = None  # cleaned response data is stored here

    def render(self):
        """Validate, process, clean and return the result of the call."""
        if not self.validate():
            raise ValidationError

        self.process_request()
        self.clean()

        return self.cleaned

    def validate(self):
        """Validate request data before sending it out. Return True/False."""
        # check if required_fields aren't present
        for field in set(self.required_fields) - set(self.request_data):
            if not isinstance(field, basestring):
                # field was a collection, iterate over it and check by OR
                return bool(set(field) & set(self.request_data))
            return False
        return True

    def process_request(self):
        """Processing the call and set response_data."""
        self.response_data = self.request_handler.process_request(self.method,
                                                            self.request_data)

    def clean(self):
        """Clean response data."""
        cleaned = copy.deepcopy(self.response_data)
        _clean_fields(self, cleaned)
        self.cleaned = cleaned

class GroupCall(BaseCall):
    """Abstract class to implement a call that returns a group of results."""

    def clean(self):
        cleaned = copy.deepcopy(self.response_data)

        # convert top level keys to ints
        for key in self.response_data.iterkeys():
            _rename_key(cleaned, key, int(key))

        # clean fields on each member of the group
        for member in self.cleaned.itervalues():
            _clean_fields(self, member)

        self.cleaned = cleaned

class FileCall(BaseCall):
    """Abstract class to implement a call that returns a file."""
    _UbersmithFile = namedtuple('UbersmithFile', ['filename', 'type',
                                                  'modified', 'data'])

    def process_request(self):
        """Processing the call and set response_data."""
        self.response_data = self.request_handler.process_request(self.method,
                                                            self.request_data,
                                                            raw=True)

    def clean(self):
        fname = None
        disposition = self.response_data[0].get('content-disposition')
        if disposition:
            fname = re.search(r'filename="(.+?)"', disposition, re.I).group(1)
            fname = re.sub(r'[^a-z0-9-_\. ]', '-', fname, 0, re.I).lstrip('.')

        self.filename = fname
        self.type = self.response_data[0].get('content-type')
        last_modified = self.response_data[0].get('last-modified')
        if last_modified:
            self.modified = datetime.datetime(
                                      *rfc822.parsedate_tz(last_modified)[:7])
        else:
            self.modified = datetime.datetime.now()
        self.data = buffer(self.response_data[1])

        self.cleaned = self._UbersmithFile(self.filename, self.type,
                                           self.modified, self.data)

def _rename_key(d, old, new):
    """Rename a key on d from old to new."""
    if old in d and new not in d:
        d[new] = d[old]
        del d[old]

def _clean_fields(call, d):
    """Rename and clean fields on d using info on call."""
    # rename fields
    for old, new in call.rename_fields.iteritems():
        _rename_key(d, old, unicode(new))

    # clean fields
    for name, func in _CLEANERS.iteritems():
        for field in getattr(call, '{0}_fields'.format(name), []):
            if field in d and isinstance(d[field], basestring):
                d[field] = func(d[field])

def _get_call_class(method):
    """Find the call class for method if it exists else create one."""
    call_base, call_name = method.split('.', 1)
    # import the call class's module
    mod = __import__('ubersmith.calls.{0}'.format(call_base), fromlist=[''])
    # grab all the public members of the module
    gen = (getattr(mod, x) for x in dir(mod) if not x.startswith('_'))
    # filter them down to subclasses of BaseCall
    gen = (x for x in gen if type(x) is type and issubclass(x, BaseCall))
    # return first one that matches our method
    for call_class in gen:
        if call_class.method == method:
            return call_class
    else:
        class GenericCall(BaseCall):
            method = '.'.join((call_base, call_name))
        return GenericCall

def _make_generic_call(call_class):
    """Create a call function that is lexically bound to use call_class."""
    def generic_call(request_handler=None, **kwargs):
        return call_class(kwargs, request_handler).render()
    return generic_call

def generate_generic_calls(base, ns):
    # get all valid methods with base
    methods = (m for m in METHODS if m.split('.', 1)[0] == base)
    for method in methods:
        call_name = method.split('.', 1)[1]
        if call_name not in ns:
            # find the appropriate class
            call_class = _get_call_class(method)
            # create a call function and stick it in the namespace
            generic_call = _make_generic_call(call_class)
            generic_call.__name__ = str(call_name)
            generic_call.__doc__ = METHODS[method]
            # TODO this may or may not be a good idea, see: http://stackoverflow.com/questions/10113892/semantics-of-module
            # generic_call.__module__ = 'ubersmith.{0}'.format(base)
            ns[call_name] = generic_call
            # add call to __all__ if needed
            if '__all__' in ns and call_name not in ns['__all__']:
                ns['__all__'].append(call_name)
        else:
            if not ns[call_name].__doc__:
                ns[call_name].__doc__ = METHODS[method]
<|endoftext|>


<commit_before>import abc
import re

def c_type(type_decl):
    return _SimpleCType(type_decl)

class _CType(metaclass=abc.ABCMeta):
    """
    An ABC representing a type in C. Subclasses need to be able to
    declare and instantiate variables.
    """
    @abc.abstractmethod
    def declare(self, name):
        pass

    @abc.abstractmethod
    def initialize(self, name, value):
        pass

    @abc.abstractmethod
    def __repr__(self):
        pass

type_formatters = {
    'int': str,
    'char': "'{}'".format,
}

class _SimpleCType(_CType):
    """
    Represents a non-composite type in C.
    >>> t = _SimpleCType("int")
    >>> t
    int
    >>> t.declare("value")
    'int value'
    >>> t.initialize("value", '1')
    'int value = 1'
    >>> _SimpleCType('char').initialize("value", 'a')
    "char value = 'a'"
    """
    type_spec = re.compile(r'(\w+)\s*(\**)\s*((\[\d+\])*)')

    def __init__(self, type_decl):
        self.base_type, self.stars, self.arrays, _ = self.type_spec.match(
            type_decl).groups()

    def declare(self, name):
        return '{} {}'.format(
            self.base_type,
            name,
        )

    def initialize(self, name, value):
        return '{} {}{}{} = {}'.format(
            self.base_type,
            self.stars,
            name,
            self.arrays,
            type_formatters[self.base_type](value),
        )

    def __repr__(self):
        return '{}{}{}'.format(
            self.base_type,
            self.stars,
            self.arrays,
        )

class _ArrayCType(_CType):
    """
    Represents an array in C.
    >>> t = _ArrayCType("int", 3)
    >>> t.declare('array')
    'int array[3]'
    """

    def __init__(self, base_type, length):
        self.base_type = c_type(base_type)
        self.length = length

    def declare(self, name):
        return '{}[{}]'.format(self.base_type.declare(name), self.length)

    def initialize(self, name, value):
        raise NotImplementedError()

    def __repr__(self):
        raise NotImplementedError()
<commit_msg>Added ArrayCType.initialize test + imelpmentation<commit_after>import abc
import re

def c_type(type_decl):
    return _SimpleCType(type_decl)

class _CType(metaclass=abc.ABCMeta):
    """
    An ABC representing a type in C. Subclasses need to be able to
    declare and instantiate variables.
    """
    @abc.abstractmethod
    def declare(self, name):
        pass

    @abc.abstractmethod
    def initialize(self, name, value):
        pass

    @abc.abstractmethod
    def __repr__(self):
        pass

type_formatters = {
    'int': str,
    'char': "'{}'".format,
}

class _SimpleCType(_CType):
    """
    Represents a non-composite type in C.
    >>> t = _SimpleCType("int")
    >>> t
    int
    >>> t.declare("value")
    'int value'
    >>> t.initialize("value", '1')
    'int value = 1'
    >>> _SimpleCType('char').initialize("value", 'a')
    "char value = 'a'"
    """
    type_spec = re.compile(r'(\w+)\s*(\**)\s*((\[\d+\])*)')

    def __init__(self, type_decl):
        self.base_type, self.stars, self.arrays, _ = self.type_spec.match(
            type_decl).groups()

    def declare(self, name):
        return '{} {}'.format(
            self.base_type,
            name,
        )

    def initialize(self, name, value):
        return '{} {}{}{} = {}'.format(
            self.base_type,
            self.stars,
            name,
            self.arrays,
            type_formatters[self.base_type](value),
        )

    def __repr__(self):
        return '{}{}{}'.format(
            self.base_type,
            self.stars,
            self.arrays,
        )

class _ArrayCType(_CType):
    """
    Represents an array in C.
    >>> t = _ArrayCType("int", 3)
    >>> t.declare('array')
    'int array[3]'
    >>> t.initialize('array', [1, 2, 3])
    'int array[3] = {1, 2, 3}'
    """

    def __init__(self, base_type, length):
        self.base_type = c_type(base_type)
        self.length = length

    def declare(self, name):
        return '{}[{}]'.format(self.base_type.declare(name), self.length)

    def initialize(self, name, value):
        return '{}[{}] = {{{}}}'.format(
            self.base_type.declare(name),
            self.length,
            ', '.join(
                self.base_type.literal(v) for v in value
            )
        )

    def __repr__(self):
        raise NotImplementedError()
<|endoftext|>


<commit_before>"""
fizzbuzz.py
Author: <your name here>
Credit: <list sources used, if any>

Assignment:

Write a program that prints the numbers from 1 to 100. But for
multiples of three print “Fizz” instead of the number and for
the multiples of five print “Buzz”. For numbers which are multiples
of both three and five print “FizzBuzz”.

We will use a variation of this test in which the last number of
the series isn't necessarily 100, and the two numbers being tested
for multiples aren't necessarily three and five. For example, your
program should behave just like this:

How many numbers shall we print? 25
For multiples of what number shall we print 'Fizz'? 3
For multiples of what number shall we print 'Buzz'? 5
1
2
Fizz
4
Buzz
Fizz
7
8
Fizz
Buzz
11
Fizz
13
14
FizzBuzz
16
17
Fizz
19
Buzz
Fizz
22
23
Fizz
Buzz
"""
<commit_msg>Updated from Brython Server: September 18, 2015 2:25:36 PM EDT<commit_after>"""
fizzbuzz.py
Author: Jeff
Credit: None

Assignment:

Write a program that prints the numbers from 1 to 100. But for
multiples of three print “Fizz” instead of the number and for
the multiples of five print “Buzz”. For numbers which are multiples
of both three and five print “FizzBuzz”.

We will use a variation of this test in which the last number of
the series isn't necessarily 100, and the two numbers being tested
for multiples aren't necessarily three and five. For example, your
program should behave just like this:

How many numbers shall we print? 25
For multiples of what number shall we print 'Fizz'? 3
For multiples of what number shall we print 'Buzz'? 5
1
2
Fizz
4
Buzz
Fizz
7
8
Fizz
Buzz
11
Fizz
13
14
FizzBuzz
16
17
Fizz
19
Buzz
Fizz
22
23
Fizz
Buzz
"""
quan = int(input("How many numbers shall we print? "))
a = int(input("For multiples of what number shall we print 'Fizz'? "))
b = int(input("For multiples of what number shall we print 'Buzz'? "))
num = 0
while quan > 0:
    num += 1
    if num%a == 0 and num%b == 0:
        print("FizzBuzz")
    else:
        if num%a == 0:
            print("Fizz")
        elif num%b == 0:
            print("Buzz")
        else:
            print(num)
    quan -= 1

<|endoftext|>


<commit_before>from django.shortcuts import render
from django.conf import settings
from django.core.files import File
from protein.models import ProteinFamily, ProteinAlias, ProteinSet, Protein, ProteinSegment
from common.views import AbsTargetSelection
from common.views import AbsSegmentSelection
from common.views import AbsMiscSelection
from common.selection import SelectionItem
from mutation.models import *
import math
import os, shutil, subprocess, signal
import uuid
from phylogenetic_trees.PrepareTree import *
from collections import OrderedDict

Alignment = getattr(__import__('common.alignment_' + settings.SITE_NAME, fromlist=['Alignment']), 'Alignment')

def kill_phylo(): #FIXME, needs better way of handling this!
    p = subprocess.Popen(['ps', '-A'], stdout=subprocess.PIPE)
    out, err = p.communicate()
    for line in out.splitlines():
        if 'protdist' in str(line):
            pid = int(line.split(None, 1)[0])
            os.kill(pid, signal.SIGKILL)

class TargetSelection(AbsTargetSelection):
    step = 1
    number_of_steps = 3
    docs = 'sequences.html#phylogeneric-trees'
    selection_boxes = OrderedDict([
        ('targets', True),
        ('segments', False),
    ])
    buttons = {
        'continue': {
            'label': 'Continue to next step',
            'url': '/phylogenetic_trees/segmentselection',
            'color': 'success',
        },
    }

class SegmentSelection(AbsSegmentSelection):
    step = 2
    number_of_steps = 3
    docs = 'sequences.html#phylogeneric-trees'
    selection_boxes = OrderedDict([
        ('targets', True),
        ('segments', True),
    ])
    buttons = {
        'continue': {
            'label': 'Continue to next step',
            'url': '/phylogenetic_trees/treesettings',
            'color': 'success',
        },
    }

class TreeSettings(AbsMiscSelection):
    step = 3
    number_of_steps = 3
    docs = 'sequences.html#phylogeneric-trees'
    title = 'SELECT TREE OPTIONS'
    description = 'Select options for tree generation in the middle column.\nOnce you have selected your' \
        + ' settings, click the green button.'
    docs = '/documentation/similarities'
    selection_boxes = OrderedDict([
        ('targets', True),
        ('segments', True),
    ])
    buttons = {
        'continue': {
            'label': 'Draw tree',
            'url': '/phylogenetic_trees/render',
            'color': 'success',
        },
    }
    tree_settings = True

class Treeclass:
    family = {}

    def __init__(self):
        self.Additional_info={"crystal": {"include":"False", "order":6, "colours":{"crystal_true":"#6dcde1","crystal_false":"#EEE"}, "color_type":"single", "proteins":[], "parent":None, "child": None, "name":"Crystals"},
                "class": {"include":"True", "order":0, "colours":{}, "proteins":[], "color_type":"grayscale", "parent":[], "child": ["family,ligand"], "name":"Class"},
                "family": {"include":"True", "order":1, "colours":{}, "proteins":[], "color_type":"spectrum", "parent":[], "child": ["ligand"], "name":"Ligand type"},
                "ligand": {"include":"True", "order":2, "colours":{}, "proteins":[], "color_type":"spectrum", "parent":["family","class"], "child": [], "name":"Receptor type"},
                "mutant": {"include":"False", "order":3, "colours":{"mutant_true":"#6dcde1","mutant_false":"#EEE"}, "color_type":"single", "proteins":[], "parent":[], "child": ["mutant_plus","mutant_minus"], "name":"Mutated proteins"},
                "mutant_plus": {"include":"False", "order":4, "colours":{"mutant_plus_true":"#6dcde1","mutant_plus_false":"#EEE"}, "color_type":"single", "proteins":[], "parent":"mutant", "child": [], "name":"Positive affinity mutants"},
                "mutant_minus": {"include":"False", "order":5, "colours":{"mutant_minus_true":"#6dcde1","mutant_minus_false":"#EEE"}, "color_type":"single", "proteins":[], "parent":"mutant", "child": [], "name":"Negative affinity mutants"}
                }
        self.buttons = [(x[1]['order'],x[1]['name']) for x in sorted(self.Additional_info.items(), key= lambda x: x[1]['order']) if x[1]['include']=='True']
        self.family = {}
        self.phylip = None
        self.outtree = None
        self.dir = ''

    def Prepare_file(self, request,build=False):
        self.Tree = PrepareTree(build)
        a=Alignment()

        sets = ProteinSet.objects.all()
        #### Get additional data ####
        crysts=[]
        for n in sets:
            if n.id==1:
                for prot in n.proteins.all():
                    crysts.append(prot.entry_name)

        #############################
        # get the user selection from session
        if build != False:
        ################################## FOR BUILDING STATISTICS ONLY##########################
            build_proteins=[]
            if build == '001':
                cons_prots = []
                for prot in Protein.objects.filter(sequence_type__slug='consensus', species_id=1):
                    if prot.family.slug.startswith('001') and len(prot.family.slug.split('_'))==3:
                        build_proteins.append(prot)
                for set in sets:
                    if set.id==1:
                        for prot in set.proteins.all():
                            if prot.family.slug.startswith('001_') and prot.species.latin_name=='Homo sapiens':
                                build_proteins.append(prot)
            else:
                for prot in Protein.objects.filter(sequence_type__slug='wt', species_id=1):
                    if prot.family.slug.startswith(build):
                        build_proteins.append(prot)
            a.load_proteins(build_proteins)
            segments = ProteinSegment.objects.all()
            a.load_segments(segments)
            self.bootstrap,self.UPGMA,self.branches,self.ttype=[0,1,1,0]
        ##################################################################
        else:
            simple_selection=request.session.get('selection', False)
            a.load_proteins_from_selection(simple_selection)
            a.load_segments_from_selection(simple_selection)
            self.bootstrap,self.UPGMA,self.branches,self.ttype = map(int,simple_selection.tree_settings)
        if self.bootstrap!=0:
            self.bootstrap=pow(10,self.bootstrap)
        #### Create an alignment object
        a.build_alignment()
        a.calculate_statistics()
        a.calculate_similarity()
        self.total = len(a.proteins)
        total_length = 0
        for chain in a.proteins[0].alignment:
            total_length += len(a.proteins[0].alignment[chain])
        families = ProteinFamily.objects.all()
        self.famdict = {}
        for n in families:
            self.famdict[self.Tree.trans_0_2_A(n.slug)]=n.name
        dirname = unique_filename = uuid.uuid4()
        os.mkdir('/tmp/%s' %dirname)
        infile = open('/tmp/%s/infile' %dirname,'w')
        infile.write('    '+str(self.total)+'    '+str(total_length)+'\n')
        if len(a.proteins) < 3:
            return 'More_prots',None, None, None, None,None,None,None
        ####Get additional protein information
        accesions = {}
        for n in a.proteins:
            fam = self.Tree.trans_0_2_A(n.protein.family.slug)
            if n.protein.sequence_type.slug == 'consensus':
                fam+='_CON'
            entry_name = n.protein.entry_name
            name = n.protein.name.replace('','').replace('','').replace('','').replace('','')
            if '&' in name and ';' in name:
                name = name.replace('&','').replace(';',' ')
            acc = n.protein.accession
            if acc:
                acc = acc.replace('-','_')
            else:
                acc = link.replace('-','_')[:6]
            spec = str(n.protein.species)
            fam += '_'+n.protein.species.common_name.replace(' ','_').upper()
            desc = name
            if entry_name in crysts:
                if not fam in self.Additional_info['crystal']['proteins']:
                    self.Additional_info['crystal']['proteins'].append(fam)
            if len(name)>25:
                name=name[:25]+'...'
            self.family[entry_name] = {'name':name,'family':fam,'description':desc,'species':spec,'class':'','accession':acc,'ligand':'','type':'','link': entry_name}
            accesions[acc]=entry_name
            ####Write PHYLIP input
            sequence = ''
            for chain in n.alignment:
                for residue in n.alignment[chain]:
                    sequence += residue[2].replace('_','-')
            infile.write(acc+' '*9+sequence+'\n')
        infile.close()

        ####Run bootstrap
        if self.bootstrap:
        ### Write phylip input options
            inp = open('/tmp/%s/temp' %dirname,'w')
            inp.write('\n'.join(['r',str(self.bootstrap),'y','77','y'])+'\n')
            inp.close()
        ###
            try:
                subprocess.check_output(['phylip seqboot<temp'], shell=True, cwd = '/tmp/%s' %dirname, timeout=60)
                os.rename('/tmp/%s/outfile' %dirname, '/tmp/%s/infile' %dirname)
            except:
                kill_phylo() #FIXME, needs better way of handling this!
                return "too big","too big","too big","too big","too big","too big","too big","too big"

        ### Write phylip input options
        inp = open('/tmp/%s/temp' %dirname,'w')
        if self.bootstrap:
            inp.write('\n'.join(['m','d',str(self.bootstrap),'y'])+'\n')
        else:
            inp.write('y\n')
        inp.close()
        ###
        try:
            subprocess.check_output(['phylip protdist<temp>>log'], shell=True, cwd = '/tmp/%s' %dirname, timeout=60)
        except:
            kill_phylo() #FIXME, needs better way of handling this!
            return "too big","too big","too big","too big","too big","too big","too big","too big"
        os.rename('/tmp/%s/infile' %dirname, '/tmp/%s/dupa' %dirname)
        os.rename('/tmp/%s/outfile' %dirname, '/tmp/%s/infile' %dirname)
        inp = open('/tmp/%s/temp' %dirname,'w')
        if self.bootstrap:
        ### Write phylip input options
            if self.UPGMA:
                inp.write('\n'.join(['N','m',str(self.bootstrap),'111','y'])+'\n')
            else:
                inp.write('\n'.join(['m',str(self.bootstrap),'111','y'])+'\n')
        else:
            if self.UPGMA:
                inp.write('N\ny\n')
            else:
                inp.write('y\n')
        inp.close()
        ###
        try:
            subprocess.check_output(['phylip neighbor<temp'], shell=True, cwd = '/tmp/%s' %dirname, timeout=60)
        except:
            kill_phylo() #FIXME, needs better way of handling this!
            return "too big","too big","too big","too big","too big","too big","too big","too big"
        if self.bootstrap:
            os.rename('/tmp/%s/outfile' %dirname, '/tmp/%s/infile' %dirname)
            os.rename('/tmp/%s/outtree' %dirname, '/tmp/%s/intree' %dirname)
           ### Write phylip input options
            inp = open('/tmp/%s/temp' %dirname,'w')
            inp.write('y\n')
            inp.close()
        ###
        try:
            subprocess.check_output(['phylip consense<temp'], shell=True, cwd = '/tmp/%s' %dirname, timeout=60)
        except:
            kill_phylo() #FIXME, needs better way of handling this!
            return "too big","too big","too big","too big","too big","too big","too big","too big"
        self.phylip = open('/tmp/%s/outtree' %dirname).read()
        for acc in accesions.keys():
            self.phylip=self.phylip.replace(acc,accesions[acc])
#        self.phylogeny_output = self.phylip
        self.outtree = open('/tmp/%s/outfile' %dirname).read().lstrip()
        phylogeny_input = self.get_phylogeny('/tmp/%s/' %dirname)
        shutil.rmtree('/tmp/%s' %dirname)

        if build != False:
            open('static/home/images/'+build+'_legend.svg','w').write(str(self.Tree.legend))
            open('static/home/images/'+build+'_tree.xml','w').write(phylogeny_input)
        else:
            return phylogeny_input, self.branches, self.ttype, self.total, str(self.Tree.legend), self.Tree.box, self.Additional_info, self.buttons

    def get_phylogeny(self, dirname):

        self.Tree.treeDo(dirname, self.phylip,self.branches,self.family,self.Additional_info, self.famdict)
        phylogeny_input = open('%s/out.xml' %dirname,'r').read().replace('\n','')
        return phylogeny_input

    def get_data(self):
        return self.branches, self.ttype, self.total, str(self.Tree.legend), self.Tree.box, self.Additional_info, self.buttons

def get_buttons(request):
    Tree_class=request.session['Tree']
    buttons = [(x[1]['order'],x[1]['name']) for x in sorted(Tree_class.Additional_info.items(), key= lambda x: x[1]['order']) if x[1]['include']=='True']
    return render(request, 'phylogenetic_trees/ring_buttons.html', {'but':buttons })

def modify_tree(request):
    try:
        shutil.rmtree('/tmp/modify')
    except:
        pass
    arg = request.GET.getlist('arg[]')
    value = request.GET.getlist('value[]')
    Tree_class=request.session['Tree']
    for n in range(len(arg)):
        Tree_class.Additional_info[arg[n].replace('_btn','')]['include']=value[n]
    request.session['Tree']=Tree_class
    os.mkdir('/tmp/modify')
    phylogeny_input = Tree_class.get_phylogeny('/tmp/modify')
    branches, ttype, total, legend, box, Additional_info, buttons=Tree_class.get_data()
    shutil.rmtree('/tmp/modify')
    if ttype == '1':
        float(total)/4*100
    else:
        count = 1900 - 1400/math.sqrt(float(total))
    print(count)
    return render(request, 'phylogenetic_trees/main.html', {'phylo': phylogeny_input, 'branch':branches, 'ttype': ttype, 'count':count, 'leg':legend, 'b':box, 'add':Additional_info, 'but':buttons, 'phylip':Tree_class.phylip, 'outtree':Tree_class.outtree})

def render_tree(request):
    Tree_class=Treeclass()
    phylogeny_input, branches, ttype, total, legend, box, Additional_info, buttons=Tree_class.Prepare_file(request)
    if phylogeny_input == 'too big':
        return render(request, 'phylogenetic_trees/too_big.html')

    if phylogeny_input == 'More_prots':
        return render(request, 'phylogenetic_trees/warning.html')

    if ttype == '1':
        float(total)/4*100
    else:
        count = 1900 - 1400/math.sqrt(float(total))

    request.session['Tree']=Tree_class
    return render(request, 'phylogenetic_trees/alignment.html', {'phylo': phylogeny_input, 'branch':branches, 'ttype': ttype, 'count':count, 'leg':legend, 'b':box, 'add':Additional_info, 'but':buttons, 'phylip':Tree_class.phylip, 'outtree':Tree_class.outtree })

<commit_msg>fix phylo<commit_after>from django.shortcuts import render
from django.conf import settings
from django.core.files import File
from protein.models import ProteinFamily, ProteinAlias, ProteinSet, Protein, ProteinSegment
from common.views import AbsTargetSelection
from common.views import AbsSegmentSelection
from common.views import AbsMiscSelection
from common.selection import SelectionItem
from mutation.models import *
import math
import os, shutil, subprocess, signal
import uuid
from phylogenetic_trees.PrepareTree import *
from collections import OrderedDict

Alignment = getattr(__import__('common.alignment_' + settings.SITE_NAME, fromlist=['Alignment']), 'Alignment')

def kill_phylo(): #FIXME, needs better way of handling this!
    p = subprocess.Popen(['ps', '-A'], stdout=subprocess.PIPE)
    out, err = p.communicate()
    for line in out.splitlines():
        if 'protdist' in str(line):
            pid = int(line.split(None, 1)[0])
            os.kill(pid, signal.SIGKILL)

class TargetSelection(AbsTargetSelection):
    step = 1
    number_of_steps = 3
    docs = 'sequences.html#phylogeneric-trees'
    selection_boxes = OrderedDict([
        ('targets', True),
        ('segments', False),
    ])
    buttons = {
        'continue': {
            'label': 'Continue to next step',
            'url': '/phylogenetic_trees/segmentselection',
            'color': 'success',
        },
    }

class SegmentSelection(AbsSegmentSelection):
    step = 2
    number_of_steps = 3
    docs = 'sequences.html#phylogeneric-trees'
    selection_boxes = OrderedDict([
        ('targets', True),
        ('segments', True),
    ])
    buttons = {
        'continue': {
            'label': 'Continue to next step',
            'url': '/phylogenetic_trees/treesettings',
            'color': 'success',
        },
    }

class TreeSettings(AbsMiscSelection):
    step = 3
    number_of_steps = 3
    docs = 'sequences.html#phylogeneric-trees'
    title = 'SELECT TREE OPTIONS'
    description = 'Select options for tree generation in the middle column.\nOnce you have selected your' \
        + ' settings, click the green button.'
    docs = '/documentation/similarities'
    selection_boxes = OrderedDict([
        ('targets', True),
        ('segments', True),
    ])
    buttons = {
        'continue': {
            'label': 'Draw tree',
            'url': '/phylogenetic_trees/render',
            'color': 'success',
        },
    }
    tree_settings = True

class Treeclass:
    family = {}

    def __init__(self):
        self.Additional_info={"crystal": {"include":"False", "order":6, "colours":{"crystal_true":"#6dcde1","crystal_false":"#EEE"}, "color_type":"single", "proteins":[], "parent":None, "child": None, "name":"Crystals"},
                "class": {"include":"True", "order":0, "colours":{}, "proteins":[], "color_type":"grayscale", "parent":[], "child": ["family,ligand"], "name":"Class"},
                "family": {"include":"True", "order":1, "colours":{}, "proteins":[], "color_type":"spectrum", "parent":[], "child": ["ligand"], "name":"Ligand type"},
                "ligand": {"include":"True", "order":2, "colours":{}, "proteins":[], "color_type":"spectrum", "parent":["family","class"], "child": [], "name":"Receptor type"},
                "mutant": {"include":"False", "order":3, "colours":{"mutant_true":"#6dcde1","mutant_false":"#EEE"}, "color_type":"single", "proteins":[], "parent":[], "child": ["mutant_plus","mutant_minus"], "name":"Mutated proteins"},
                "mutant_plus": {"include":"False", "order":4, "colours":{"mutant_plus_true":"#6dcde1","mutant_plus_false":"#EEE"}, "color_type":"single", "proteins":[], "parent":"mutant", "child": [], "name":"Positive affinity mutants"},
                "mutant_minus": {"include":"False", "order":5, "colours":{"mutant_minus_true":"#6dcde1","mutant_minus_false":"#EEE"}, "color_type":"single", "proteins":[], "parent":"mutant", "child": [], "name":"Negative affinity mutants"}
                }
        self.buttons = [(x[1]['order'],x[1]['name']) for x in sorted(self.Additional_info.items(), key= lambda x: x[1]['order']) if x[1]['include']=='True']
        self.family = {}
        self.phylip = None
        self.outtree = None
        self.dir = ''

    def Prepare_file(self, request,build=False):
        self.Tree = PrepareTree(build)
        a=Alignment()

        sets = ProteinSet.objects.all()
        #### Get additional data ####
        crysts=[]
        for n in sets:
            if n.id==1:
                for prot in n.proteins.all():
                    crysts.append(prot.entry_name)

        #############################
        # get the user selection from session
        if build != False:
        ################################## FOR BUILDING STATISTICS ONLY##########################
            build_proteins=[]
            if build == '001':
                cons_prots = []
                for prot in Protein.objects.filter(sequence_type__slug='consensus', species_id=1):
                    if prot.family.slug.startswith('001') and len(prot.family.slug.split('_'))==3:
                        build_proteins.append(prot)
                for set in sets:
                    if set.id==1:
                        for prot in set.proteins.all():
                            if prot.family.slug.startswith('001_') and prot.species.latin_name=='Homo sapiens':
                                build_proteins.append(prot)
            else:
                for prot in Protein.objects.filter(sequence_type__slug='wt', species_id=1):
                    if prot.family.slug.startswith(build):
                        build_proteins.append(prot)
            a.load_proteins(build_proteins)
            segments = ProteinSegment.objects.all()
            a.load_segments(segments)
            self.bootstrap,self.UPGMA,self.branches,self.ttype=[0,1,1,0]
        ##################################################################
        else:
            simple_selection=request.session.get('selection', False)
            a.load_proteins_from_selection(simple_selection)
            a.load_segments_from_selection(simple_selection)
            self.bootstrap,self.UPGMA,self.branches,self.ttype = map(int,simple_selection.tree_settings)
        if self.bootstrap!=0:
            self.bootstrap=pow(10,self.bootstrap)
        #### Create an alignment object
        a.build_alignment()
        a.calculate_statistics()
        a.calculate_similarity()
        self.total = len(a.proteins)
        total_length = 0
        for chain in a.proteins[0].alignment:
            total_length += len(a.proteins[0].alignment[chain])
        families = ProteinFamily.objects.all()
        self.famdict = {}
        for n in families:
            self.famdict[self.Tree.trans_0_2_A(n.slug)]=n.name
        dirname = unique_filename = uuid.uuid4()
        os.mkdir('/tmp/%s' %dirname)
        infile = open('/tmp/%s/infile' %dirname,'w')
        infile.write('    '+str(self.total)+'    '+str(total_length)+'\n')
        if len(a.proteins) < 3:
            return 'More_prots',None, None, None, None,None,None,None
        ####Get additional protein information
        accesions = {}
        for n in a.proteins:
            fam = self.Tree.trans_0_2_A(n.protein.family.slug)
            if n.protein.sequence_type.slug == 'consensus':
                fam+='_CON'
            entry_name = n.protein.entry_name
            name = n.protein.name.replace('','').replace('','').replace('','').replace('','')
            if '&' in name and ';' in name:
                name = name.replace('&','').replace(';',' ')
            acc = n.protein.accession
            if acc:
                acc = acc.replace('-','_')
            else:
                acc = link.replace('-','_')[:6]
            spec = str(n.protein.species)
            fam += '_'+n.protein.species.common_name.replace(' ','_').upper()
            desc = name
            if entry_name in crysts:
                if not fam in self.Additional_info['crystal']['proteins']:
                    self.Additional_info['crystal']['proteins'].append(fam)
            if len(name)>25:
                name=name[:25]+'...'
            self.family[entry_name] = {'name':name,'family':fam,'description':desc,'species':spec,'class':'','accession':acc,'ligand':'','type':'','link': entry_name}
            accesions[acc]=entry_name
            ####Write PHYLIP input
            sequence = ''
            for chain in n.alignment:
                for residue in n.alignment[chain]:
                    sequence += residue[2].replace('_','-')
            infile.write(acc+' '*9+sequence+'\n')
        infile.close()

        ####Run bootstrap
        if self.bootstrap:
        ### Write phylip input options
            inp = open('/tmp/%s/temp' %dirname,'w')
            inp.write('\n'.join(['r',str(self.bootstrap),'y','77','y'])+'\n')
            inp.close()
        ###
            try:
                subprocess.check_output(['phylip seqboot<temp'], shell=True, cwd = '/tmp/%s' %dirname, timeout=60)
                os.rename('/tmp/%s/outfile' %dirname, '/tmp/%s/infile' %dirname)
            except:
                kill_phylo() #FIXME, needs better way of handling this!
                return "too big","too big","too big","too big","too big","too big","too big","too big"

        ### Write phylip input options
        inp = open('/tmp/%s/temp' %dirname,'w')
        if self.bootstrap:
            inp.write('\n'.join(['m','d',str(self.bootstrap),'y'])+'\n')
        else:
            inp.write('y\n')
        inp.close()
        ###
        try:
            subprocess.check_output(['phylip protdist<temp>>log'], shell=True, cwd = '/tmp/%s' %dirname, timeout=60)
        except:
            kill_phylo() #FIXME, needs better way of handling this!
            return "too big","too big","too big","too big","too big","too big","too big","too big"
        os.rename('/tmp/%s/infile' %dirname, '/tmp/%s/dupa' %dirname)
        os.rename('/tmp/%s/outfile' %dirname, '/tmp/%s/infile' %dirname)
        inp = open('/tmp/%s/temp' %dirname,'w')
        if self.bootstrap:
        ### Write phylip input options
            if self.UPGMA:
                inp.write('\n'.join(['N','m',str(self.bootstrap),'111','y'])+'\n')
            else:
                inp.write('\n'.join(['m',str(self.bootstrap),'111','y'])+'\n')
        else:
            if self.UPGMA:
                inp.write('N\ny\n')
            else:
                inp.write('y\n')
        inp.close()
        ###
        try:
            subprocess.check_output(['phylip neighbor<temp'], shell=True, cwd = '/tmp/%s' %dirname, timeout=60)
        except:
            kill_phylo() #FIXME, needs better way of handling this!
            return "too big","too big","too big","too big","too big","too big","too big","too big"
        if self.bootstrap:
            os.rename('/tmp/%s/outfile' %dirname, '/tmp/%s/infile' %dirname)
            os.rename('/tmp/%s/outtree' %dirname, '/tmp/%s/intree' %dirname)
           ### Write phylip input options
            inp = open('/tmp/%s/temp' %dirname,'w')
            inp.write('y\n')
            inp.close()
        ###
            try:
                subprocess.check_output(['phylip consense<temp'], shell=True, cwd = '/tmp/%s' %dirname, timeout=60)
            except:
                kill_phylo() #FIXME, needs better way of handling this!
                return "too big","too big","too big","too big","too big","too big","too big","too big"
        self.phylip = open('/tmp/%s/outtree' %dirname).read()
        for acc in accesions.keys():
            self.phylip=self.phylip.replace(acc,accesions[acc])
#        self.phylogeny_output = self.phylip
        self.outtree = open('/tmp/%s/outfile' %dirname).read().lstrip()
        phylogeny_input = self.get_phylogeny('/tmp/%s/' %dirname)
        shutil.rmtree('/tmp/%s' %dirname)

        if build != False:
            open('static/home/images/'+build+'_legend.svg','w').write(str(self.Tree.legend))
            open('static/home/images/'+build+'_tree.xml','w').write(phylogeny_input)
        else:
            return phylogeny_input, self.branches, self.ttype, self.total, str(self.Tree.legend), self.Tree.box, self.Additional_info, self.buttons

    def get_phylogeny(self, dirname):

        self.Tree.treeDo(dirname, self.phylip,self.branches,self.family,self.Additional_info, self.famdict)
        phylogeny_input = open('%s/out.xml' %dirname,'r').read().replace('\n','')
        return phylogeny_input

    def get_data(self):
        return self.branches, self.ttype, self.total, str(self.Tree.legend), self.Tree.box, self.Additional_info, self.buttons

def get_buttons(request):
    Tree_class=request.session['Tree']
    buttons = [(x[1]['order'],x[1]['name']) for x in sorted(Tree_class.Additional_info.items(), key= lambda x: x[1]['order']) if x[1]['include']=='True']
    return render(request, 'phylogenetic_trees/ring_buttons.html', {'but':buttons })

def modify_tree(request):
    try:
        shutil.rmtree('/tmp/modify')
    except:
        pass
    arg = request.GET.getlist('arg[]')
    value = request.GET.getlist('value[]')
    Tree_class=request.session['Tree']
    for n in range(len(arg)):
        Tree_class.Additional_info[arg[n].replace('_btn','')]['include']=value[n]
    request.session['Tree']=Tree_class
    os.mkdir('/tmp/modify')
    phylogeny_input = Tree_class.get_phylogeny('/tmp/modify')
    branches, ttype, total, legend, box, Additional_info, buttons=Tree_class.get_data()
    shutil.rmtree('/tmp/modify')
    if ttype == '1':
        float(total)/4*100
    else:
        count = 1900 - 1400/math.sqrt(float(total))
    print(count)
    return render(request, 'phylogenetic_trees/main.html', {'phylo': phylogeny_input, 'branch':branches, 'ttype': ttype, 'count':count, 'leg':legend, 'b':box, 'add':Additional_info, 'but':buttons, 'phylip':Tree_class.phylip, 'outtree':Tree_class.outtree})

def render_tree(request):
    Tree_class=Treeclass()
    phylogeny_input, branches, ttype, total, legend, box, Additional_info, buttons=Tree_class.Prepare_file(request)
    if phylogeny_input == 'too big':
        return render(request, 'phylogenetic_trees/too_big.html')

    if phylogeny_input == 'More_prots':
        return render(request, 'phylogenetic_trees/warning.html')

    if ttype == '1':
        float(total)/4*100
    else:
        count = 1900 - 1400/math.sqrt(float(total))

    request.session['Tree']=Tree_class
    return render(request, 'phylogenetic_trees/alignment.html', {'phylo': phylogeny_input, 'branch':branches, 'ttype': ttype, 'count':count, 'leg':legend, 'b':box, 'add':Additional_info, 'but':buttons, 'phylip':Tree_class.phylip, 'outtree':Tree_class.outtree })

<|endoftext|>


<commit_before># coding=utf-8

import time
import os
import logging

try:
    from selenium import webdriver
except ImportError:
    webdriver = None

from urllib.request import urlopen
from bs4 import BeautifulSoup

__author__ = "DefaltSimon"
__license__ = "MIT"

things = [
    "  ",
    "\n",
    "\t",
    "\r"
]

user_search = "https://osu.ppy.sh/u/"
beatmap_search = "https://osu.ppy.sh/p/beatmaplist?q="

url_encode = lambda a: str(a).replace(" ", "%20")

log = logging.getLogger(__name__)
log.setLevel(logging.INFO)

# Exception

class OsuParserError(Exception):
    pass

# Utilities and Handlers

def clean(text):
    for ch in things:
        text = str(text).replace(ch, "")
    return text.strip(" ").strip("\n")

def handle_exceptions(fn):
    def handle(*args, **kwargs):
        try:
            fn(*args, **kwargs)
        except:
            log.error("Something went wrong while executing function {}".format(fn.__name__))
            raise OsuParserError("could not finish {}".format(fn.__name__))

    return handle

def phantom_in_path(path):
    return os.path.isfile(path)

class User(object):
    """
    A user on osu.ppy.sh/u/
    """
    def __init__(self, **kwargs):
        """
        Initializes all data.
        Missing data is None (of selenium was set to False when fetching).
        :param kwargs: kwargs
        """
        self.username = kwargs.get("username")
        self.age = kwargs.get("age")

        self.occupation = kwargs.get("occupation")
        self.country = kwargs.get("country")

        self.date_joined = kwargs.get("date_joined")
        self.date_active = kwargs.get("date_active")

        self.rank_world = kwargs.get("rank_world")
        self.rank_country = kwargs.get("rank_country")

        self.performace = kwargs.get("performance")
        self.level = kwargs.get("level")

        self.hit_accuracy = kwargs.get("hit_accuracy")
        self.total_hits = kwargs.get("total_hits")

        self.play_count = kwargs.get("play_count")
        self.play_time = kwargs.get("play_time")

        self.ranked_score = kwargs.get("ranked_score")
        self.total_score = kwargs.get("total_score")

        self.max_combo = kwargs.get("max_combo")

class OsuParser:
    """
    Osu! python library
    This library is in no way affiliated with Osu! or peppy.
    """
    def __init__(self, selenium_path="phantomjs", max_cache_age=43200):  # 12 hours
        self.cache = {}
        self.ages = {}

        self.max_age = max_cache_age

        self.selenium_available = bool(webdriver)
        self.selenium_path = selenium_path

        log.info("Initialized. Selenium {}".format("installed. Path: {}".format(selenium_path) if webdriver else "not found, using minimal mode!"))

    # @handle_exceptions
    def get_user_by_name(self, username, selenium=True, allow_cache=True):
        """
        Gets a user profile by the username.
        :param username: string - username
        :param selenium: bool indicating if you want to use full browser Javascript support (making all features available but slowing it down)
        :param allow_cache: bool indicating if the cache is allowed to be used
        :return: User object
        """
        if not self.selenium_available:
            selenium = False

        # Fetch from cache if allowed to
        if allow_cache and (str(username) in self.cache):
            if int(time.time()) - int(self.ages.get(str(username))) < self.max_age:
                assert isinstance(self.cache.get(str(username)), User)

                log.info("Using cache for {}".format(username))
                return self.cache.get(str(username))

            # Ignore cache if it's older than max_age
            else:
                pass

        # Use if selenium and phantomJS are installed and enabled
        # HIGHLY RECOMMENDED to install and use selenium
        if selenium:
            a = webdriver.PhantomJS(executable_path=self.selenium_path)
            a.get(user_search + url_encode(username))
            u = a.page_source

        # Or just use urllib
        # WARNING! Not all functions are available when using this mode!
        else:
            log.info("Selenium not installed or disabled, using minimal mode")
            u = urlopen(user_search + url_encode(username))

        # BS Instance
        sp = BeautifulSoup(u, "html.parser")

        try:
            username = clean(sp.find("div", {"class": "profile-username"}).text)
        except AttributeError:
            username = None

        try:
            age = clean(sp.find("div", {"title": "Age"}).text).strip(" years")
        except AttributeError:
            age = None

        try:
            occupation = clean(sp.find("div", {"title": "Occupation"}).text)
        except AttributeError:
            occupation = None

        try:
            interests = clean(sp.find("div", {"title": "Interests"}).text)
        except AttributeError:
            interests = None

        try:
            country = sp.find("img", {"class": "flag"}).get("title")
        except AttributeError:
            country = None

        # UTC time
        date_joined = sp.find("div", {"title": "Arrived"}).find("time", {"class": "timeago"}).text
        date_active = sp.find("div", {"title": "Last Active"}).find("time", {"class": "timeago"}).text

        if selenium:
            mid = clean(sp.find("div", {"class": "profileStatLine"}).find("b").text)
            rank_world = mid.strip("Performance: ").split("(")[0]
            rank_country = clean(sp.find("div", {"class": "profileStatLine"}).find("span").text)
            performance = mid.strip("Performance: ").split("(")[1].strip(")")

            # print(rank_world, rank_country, performance, sep="\n")

            pages = sp.find("div", {"id": "general"}).find_all("div", {"class": "profileStatLine"})
            pages.pop(0)

            ranked_score = clean(pages.pop(0).text.strip("Ranked Score")).strip(": ")
            hit_accuracy = clean(pages.pop(0).text.strip("Hit Accuracy")).strip(": ")
            play_count = clean(pages.pop(0).text.strip("Play Count")).strip(": ")
            play_time = clean(pages.pop(0).text.strip("Play Time")).strip(": ")
            total_score = clean(pages.pop(0).text.strip("Total Score")).strip(": ")
            level = clean(pages.pop(0).text.strip("Current Level")).strip(": ")
            total_hits = clean(pages.pop(0).text.strip("Total Hits")).strip(": ")
            max_combo = clean(pages.pop(0).text.strip("Maximum Combo")).strip(": ")

        else:
            # Assign all variables as None if selenium is not installed or disabled
            rank_world = rank_country = performance = ranked_score = hit_accuracy = None
            play_time = play_count = total_score = level = total_hits = max_combo = None

        # Generates an object
        obj = User(
            username=username,
            age=age,
            occupation=occupation,
            interests=interests,
            country=country,
            date_joined=date_joined,
            date_active=date_active,
            rank_world=rank_world,
            rank_country=rank_country,
            performance=performance,
            ranked_score=ranked_score,
            hit_accuracy=hit_accuracy,
            play_count=play_count,
            play_time=play_time,
            total_score=total_score,
            level=level,
            total_hits=total_hits,
            max_combo=max_combo
        )

        self.cache[username] = obj
        self.ages[username] = int(time.time())
        return obj

# Tests
# logging.basicConfig(level=logging.INFO)
# o = OsuParser(selenium_path="phantomjs/phantomjs.exe")
# while True:
#     user = input("Name:")
#
#     t = time.time()
#     user = o.get_user_by_name(user, selenium=True)
#
#     print(user.username, user.max_combo)
#     print("Took {}".format(time.time() - t))
#<commit_msg>0.1b update<commit_after># coding=utf-8

import time
import os
import logging

try:
    from selenium import webdriver
except ImportError:
    webdriver = None

from urllib.request import urlopen
from bs4 import BeautifulSoup

__author__ = "DefaltSimon"
__license__ = "MIT"

things = [
    "  ",
    "\n",
    "\t",
    "\r"
]

user_search = "https://osu.ppy.sh/u/"
beatmap_search = "https://osu.ppy.sh/p/beatmaplist?q="

url_encode = lambda a: str(a).replace(" ", "%20")

log = logging.getLogger(__name__)
log.setLevel(logging.INFO)

# Exception

class OsuParserError(Exception):
    pass

# Utilities and Handlers

def clean(text):
    for ch in things:
        text = str(text).replace(ch, "")
    return text.strip(" ").strip("\n")

def handle_exceptions(fn):
    def handle(*args, **kwargs):
        try:
            fn(*args, **kwargs)
        except:
            log.error("Something went wrong while executing function {}".format(fn.__name__))
            raise OsuParserError("could not finish {}".format(fn.__name__))

    return handle

def phantom_in_path(path):
    return os.path.isfile(path)

class User(object):
    """
    A user on osu.ppy.sh/u/
    """
    def __init__(self, **kwargs):
        """
        Initializes all data.
        Missing data is None (of selenium was set to False when fetching).
        :param kwargs: kwargs
        """
        self.username = kwargs.get("username")
        self.age = kwargs.get("age")

        self.occupation = kwargs.get("occupation")
        self.country = kwargs.get("country")

        self.date_joined = kwargs.get("date_joined")
        self.date_active = kwargs.get("date_active")

        self.rank_world = kwargs.get("rank_world")
        self.rank_country = kwargs.get("rank_country")

        self.performance = kwargs.get("performance")
        self.level = kwargs.get("level")

        self.hit_accuracy = kwargs.get("hit_accuracy")
        self.total_hits = kwargs.get("total_hits")

        self.play_count = kwargs.get("play_count")
        self.play_time = kwargs.get("play_time")

        self.ranked_score = kwargs.get("ranked_score")
        self.total_score = kwargs.get("total_score")

        self.max_combo = kwargs.get("max_combo")

class OsuParser:
    """
    Osu! python library
    This library is in no way affiliated with Osu! or peppy.
    """
    def __init__(self, selenium_path="phantomjs", max_cache_age=43200):  # 12 hours
        self.cache = {}
        self.ages = {}

        self.max_age = max_cache_age

        self.selenium_available = bool(webdriver)
        self.selenium_path = selenium_path

        log.info("Initialized. Selenium {}".format("installed. Path: {}".format(selenium_path) if webdriver else "not found, using minimal mode!"))

    # @handle_exceptions
    def get_user_by_name(self, username, selenium=True, allow_cache=True):
        """
        Gets a user profile by the username.
        :param username: string - username
        :param selenium: bool indicating if you want to use full browser Javascript support (making all features available but slowing it down)
        :param allow_cache: bool indicating if the cache is allowed to be used
        :return: User object
        """
        if not self.selenium_available:
            selenium = False

        # Fetch from cache if allowed to
        if allow_cache and (str(username) in self.cache):
            if int(time.time()) - int(self.ages.get(str(username))) < self.max_age:
                assert isinstance(self.cache.get(str(username)), User)

                log.info("Using cache for {}".format(username))
                return self.cache.get(str(username))

            # Ignore cache if it's older than max_age
            else:
                pass

        # Use if selenium and phantomJS are installed and enabled
        # HIGHLY RECOMMENDED to install and use selenium
        if selenium:
            a = webdriver.PhantomJS(executable_path=self.selenium_path)
            a.get(user_search + url_encode(username))
            u = a.page_source

        # Or just use urllib
        # WARNING! Not all functions are available when using this mode!
        else:
            log.info("Selenium not installed or disabled, using minimal mode")
            u = urlopen(user_search + url_encode(username))

        # BS Instance
        sp = BeautifulSoup(u, "html.parser")

        try:
            username = clean(sp.find("div", {"class": "profile-username"}).text)
        except AttributeError:
            username = None

        try:
            age = clean(sp.find("div", {"title": "Age"}).text).strip(" years")
        except AttributeError:
            age = None

        try:
            occupation = clean(sp.find("div", {"title": "Occupation"}).text)
        except AttributeError:
            occupation = None

        try:
            interests = clean(sp.find("div", {"title": "Interests"}).text)
        except AttributeError:
            interests = None

        try:
            country = sp.find("img", {"class": "flag"}).get("title")
        except AttributeError:
            country = None

        # UTC time
        date_joined = sp.find("div", {"title": "Arrived"}).find("time", {"class": "timeago"}).text
        date_active = sp.find("div", {"title": "Last Active"}).find("time", {"class": "timeago"}).text

        if selenium:
            mid = clean(sp.find("div", {"class": "profileStatLine"}).find("b").text)
            rank_world = mid.strip("Performance: ").split("(")[0]
            rank_country = clean(sp.find("div", {"class": "profileStatLine"}).find("span").text)
            performance = mid.strip("Performance: ").split("(")[1].strip(")")

            # print(rank_world, rank_country, performance, sep="\n")

            pages = sp.find("div", {"id": "general"}).find_all("div", {"class": "profileStatLine"})
            pages.pop(0)

            ranked_score = clean(pages.pop(0).text.strip("Ranked Score")).strip(": ")
            hit_accuracy = clean(pages.pop(0).text.strip("Hit Accuracy")).strip(": ")
            play_count = clean(pages.pop(0).text.strip("Play Count")).strip(": ")
            play_time = clean(pages.pop(0).text.strip("Play Time")).strip(": ")
            total_score = clean(pages.pop(0).text.strip("Total Score")).strip(": ")
            level = clean(pages.pop(0).text.strip("Current Level")).strip(": ")
            total_hits = clean(pages.pop(0).text.strip("Total Hits")).strip(": ")
            max_combo = clean(pages.pop(0).text.strip("Maximum Combo")).strip(": ")

        else:
            # Assign all variables as None if selenium is not installed or disabled
            rank_world = rank_country = performance = ranked_score = hit_accuracy = None
            play_time = play_count = total_score = level = total_hits = max_combo = None

        # Generates an object
        obj = User(
            username=username,
            age=age,
            occupation=occupation,
            interests=interests,
            country=country,
            date_joined=date_joined,
            date_active=date_active,
            rank_world=rank_world,
            rank_country=rank_country,
            performance=performance,
            ranked_score=ranked_score,
            hit_accuracy=hit_accuracy,
            play_count=play_count,
            play_time=play_time,
            total_score=total_score,
            level=level,
            total_hits=total_hits,
            max_combo=max_combo
        )

        self.cache[username] = obj
        self.ages[username] = int(time.time())
        return obj

# Tests
# logging.basicConfig(level=logging.INFO)
# o = OsuParser(selenium_path="phantomjs/phantomjs.exe")
# while True:
#     user = input("Name:")
#
#     t = time.time()
#     user = o.get_user_by_name(user, selenium=True)
#
#     print(user.username, user.max_combo)
#     print("Took {}".format(time.time() - t))
#
<|endoftext|>


<commit_before>#!/usr/bin/env python

'''
===============================================================================
                   ______   ___  _____  ___    ___   ___  _____
                  / __/ /  / _ |/ ___/ |_  |  / _ | / _ |/ ___/
                 / _// /__/ __ / /__  / __/  / __ |/ __ / /__
                /_/ /____/_/ |_\___/ /____/ /_/ |_/_/ |_\___/

                                   FLAC 2 AAC
===============================================================================

Script:         flac2aac.py
Author:         Sebastian Weigand   www.sw-dd.com
Email:          sab@sw-dd.com
Current:        February 2014
Copyright:      2012-2014, Sebastian Weigand
License:        MIT
Description:    A script which converts FLAC files to MPEG4/AAC files and
                copies over song metadata, utilizing Apple's CoreAudio
                framework for better AAC support.
Version:        2.0
Requirements:
    OS:         Mac OS X, v10.5+    [afconvert]
    Platform:   Python 2.6+         [multiprocessing]
    Binaries:   flac                [decoding]
    Python Lib: mutagen             [metadata]

===============================================================================
'''

import os
import sys
from getpass import getuser
from shutil import rmtree

# =============================================================================
# Sanity Checking:
# =============================================================================

try:
    import mutagen
    import fnmatch
    import argparse

    from subprocess import call, Popen, PIPE
    from multiprocessing import Pool, cpu_count

except ImportError, e:
    exit('Error: Unable to import requisite modules: %s' % e)

def is_progam_valid(program):
    paths = os.environ["PATH"].split(os.pathsep)

    for path in paths:
        if os.access(os.path.join(path, program), os.X_OK):
            return True
    return False

for program in ['flac', 'afconvert']:
    if not is_progam_valid(program):
        exit('Error: Unable to execute/find "%s" from your PATH.' % program)

afconvert_help_formats = Popen(['afconvert', '-hf'], stderr=PIPE).communicate()[1]
data_formats = [
    format for format in ['aac', 'aace', 'aacf', 'aach', 'aacl', 'aacp'] if format in afconvert_help_formats]

def fix_path(path):
    return os.path.realpath(os.path.expanduser(path))

# =============================================================================
# Argument Parsing:
# =============================================================================

parser = argparse.ArgumentParser(
    description='Converts FLAC to MPEG4/AAC via CoreAudio and transfers metadata using Mutagen.',
    epilog='Note: Mac OS X v10.5+ is required for HE AAC (aach), and 10.7 is required for HE AAC v2 (aacp).')

parser.add_argument('location', default=fix_path(os.curdir),
                    type=str, help='the location to search for media files [.]')
parser.add_argument('-q', '--quality', type=int, default=75, help='VBR quality, in percent [75]')
parser.add_argument('-c', '--codec', choices=data_formats,
                    default='aac', help='codec to use, if available on your platform [aac]')
parser.add_argument('--abr', action="store_true", default=False,
                    help='use average bitrate, instead of variable [no]')
parser.add_argument('--bitrate', type=int, default=256, help='average bitrate, in KB/s [256]')
args = parser.parse_args()

# Fix up the arguments:
bitrate = args.bitrate * 1000
quality = str(int(args.quality / 100.0 * 127))
args.location = fix_path(args.location)

# Make sure we've got good paths:
if os.path.isdir(args.location):
    if not os.access(args.location, os.W_OK and os.R_OK):
        exit('Cannot read/write: %s' % args.location)
else:
    exit('Requested location does not exist: %s' % args.location)

# See `afconvert -h` to set the right mode:
if args.abr:
    vbr_mode = 0
else:
    vbr_mode = 3

# =============================================================================
# Transcoding:
# =============================================================================

# First create our output directory:
output_location = os.path.join(args.location, 'converted_audio')
tmp_location = '/tmp/intermediate_audio/'
file_count = 0

for loc in (output_location, tmp_location):
    if not os.path.exists(loc):
        os.mkdir(loc)

def get_flac_files(location):
    global file_count
    for path, dirs, files in os.walk(location):
        for f in files:
            if f.endswith('.flac') and not f.startswith('.'):
                file_count += 1
                yield os.path.join(path, f)

def convert_flac_to_aac(flac_file):
    try:
        metadata = mutagen.File(flac_file, easy=True)

        validKeys = [
            'title', 'album', 'artist', 'albumartist', 'date', 'comment', 'description', 'grouping', 'genre',
            'copyright', 'albumsort', 'albumartistsort', 'artistsort', 'titlesort', 'composersort', 'tracknumber', 'discnumber']

        # Sometimes extraneous data will be stored within FLAC's metadata, and that
        # confuses the direct key updating methods of the Easy-branch of Mutagen,
        # so clean it up here:

        for key in metadata.keys():
            if key not in validKeys:
                del metadata[key]

        # Keep these in /tmp:
        wav_file = os.path.join(tmp_location, os.path.basename(flac_file[:-5])) + '.wav'

        # Final location is back where we started:
        m4a_file = os.path.join(output_location, os.path.basename(flac_file[:-5])) + '.m4a'

        call(['flac', '-s', '-f', '-d', flac_file, '-o', wav_file])

        call(['afconvert', '-f', 'm4af', '-d', args.codec, '-b', str(bitrate), '--src-complexity',
              'bats', '-s', str(vbr_mode), '-u', 'vbrq', quality, '--soundcheck-generate', wav_file, m4a_file])

        m4aData = mutagen.File(m4a_file, easy=True)
        m4aData.update(metadata)
        m4aData.save()
        os.remove(wav_file)

        print 'Converted file:', m4a_file

    except KeyboardInterrupt:
        pass

# =============================================================================
# Main:
# =============================================================================

print 'Processing:', args.location

try:
    pool = Pool()
    p = pool.map_async(convert_flac_to_aac, get_flac_files(args.location))
    p.get(0xFFFF)
except KeyboardInterrupt:
    exit('Aborting.')

rmtree(tmp_location, ignore_errors=True)

print 'Converted %s files.\n' % file_count

# EOF
<commit_msg>FLAC2AAC: Now Supports Apple Lossless<commit_after>#!/usr/bin/env python

'''
===============================================================================
                   ______   ___  _____  ___    ___   ___  _____
                  / __/ /  / _ |/ ___/ |_  |  / _ | / _ |/ ___/
                 / _// /__/ __ / /__  / __/  / __ |/ __ / /__
                /_/ /____/_/ |_\___/ /____/ /_/ |_/_/ |_\___/

                                   FLAC 2 AAC
===============================================================================

Script:         flac2aac.py
Author:         Sebastian Weigand   www.sw-dd.com
Email:          sab@sw-dd.com
Current:        February 2014
Copyright:      2012-2014, Sebastian Weigand
License:        MIT
Description:    A script which converts FLAC files to MPEG4/AAC files and
                copies over song metadata, utilizing Apple's CoreAudio
                framework for better AAC support.
Version:        2.1 (Lossless Support)
Requirements:
    OS:         Mac OS X, v10.5+    [afconvert]
    Platform:   Python 2.6+         [multiprocessing]
    Binaries:   flac                [decoding]
    Python Lib: mutagen             [metadata]

===============================================================================
'''

import os
import sys
from getpass import getuser
from shutil import rmtree

# =============================================================================
# Sanity Checking:
# =============================================================================

try:
    import mutagen
    import fnmatch
    import argparse

    from subprocess import call, Popen, PIPE
    from multiprocessing import Pool, cpu_count

except ImportError, e:
    exit('Error: Unable to import requisite modules: %s' % e)

def is_progam_valid(program):
    paths = os.environ["PATH"].split(os.pathsep)

    for path in paths:
        if os.access(os.path.join(path, program), os.X_OK):
            return True
    return False

for program in ['flac', 'afconvert']:
    if not is_progam_valid(program):
        exit('Error: Unable to execute/find "%s" from your PATH.' % program)

afconvert_help_formats = Popen(['afconvert', '-hf'], stderr=PIPE).communicate()[1]
data_formats = [
    format for format in ['aac', 'aace', 'aacf', 'aach', 'aacl', 'aacp'] if format in afconvert_help_formats]

def fix_path(path):
    return os.path.realpath(os.path.expanduser(path))

# =============================================================================
# Argument Parsing:
# =============================================================================

parser = argparse.ArgumentParser(
    description='Converts FLAC to MPEG4/AAC via CoreAudio and transfers metadata using Mutagen.',
    epilog='Note: Mac OS X v10.5+ is required for HE AAC (aach), and 10.7 is required for HE AAC v2 (aacp).')

parser.add_argument('location', default=fix_path(os.curdir),
                    type=str, help='the location to search for media files [.]')
parser.add_argument('-l', '--lossless', action="store_true", default=False,
                    help='encode in Apple Lossless (ALAC), overrides codec options [no]')
parser.add_argument('-q', '--quality', type=int, default=75, help='VBR quality, in percent [75]')
parser.add_argument('-c', '--codec', choices=data_formats,
                    default='aac', help='codec to use, if available on your platform [aac]')
parser.add_argument('--abr', action="store_true", default=False,
                    help='use average bitrate, instead of variable [no]')
parser.add_argument('--bitrate', type=int, default=256, help='average bitrate, in KB/s [256]')
args = parser.parse_args()

# Fix up the arguments:
bitrate = args.bitrate * 1000
quality = str(int(args.quality / 100.0 * 127))
args.location = fix_path(args.location)

# Make sure we've got good paths:
if os.path.isdir(args.location):
    if not os.access(args.location, os.W_OK and os.R_OK):
        exit('Cannot read/write: %s' % args.location)
else:
    exit('Requested location does not exist: %s' % args.location)

# See `afconvert -h` to set the right mode:
if args.abr:
    vbr_mode = 0
else:
    vbr_mode = 3

# =============================================================================
# Transcoding:
# =============================================================================

# First create our output directory:
output_location = os.path.join(args.location, 'converted_audio')
tmp_location = '/tmp/intermediate_audio/'
file_count = 0

for loc in (output_location, tmp_location):
    if not os.path.exists(loc):
        os.mkdir(loc)

def get_flac_files(location):
    global file_count
    for path, dirs, files in os.walk(location):
        for f in files:
            if f.endswith('.flac') and not f.startswith('.'):
                file_count += 1
                yield os.path.join(path, f)

def convert_flac_to_aac(flac_file):
    try:
        metadata = mutagen.File(flac_file, easy=True)

        validKeys = [
            'title', 'album', 'artist', 'albumartist', 'date', 'comment', 'description', 'grouping', 'genre',
            'copyright', 'albumsort', 'albumartistsort', 'artistsort', 'titlesort', 'composersort', 'tracknumber', 'discnumber']

        # Sometimes extraneous data will be stored within FLAC's metadata, and that
        # confuses the direct key updating methods of the Easy-branch of Mutagen,
        # so clean it up here:

        for key in metadata.keys():
            if key not in validKeys:
                del metadata[key]

        # Keep these in /tmp:
        wav_file = os.path.join(tmp_location, os.path.basename(flac_file[:-5])) + '.wav'

        # Final location is back where we started:
        m4a_file = os.path.join(output_location, os.path.basename(flac_file[:-5])) + '.m4a'

        call(['flac', '-s', '-f', '-d', flac_file, '-o', wav_file])

        if args.lossless:
            call(['afconvert', '-f', 'm4af', '-d', 'alac',
                 '--soundcheck-generate', wav_file, m4a_file])
        else:
            call(
                ['afconvert', '-f', 'm4af', '-d', args.codec, '-b', str(bitrate), '--src-complexity',
                 'bats', '-s', str(vbr_mode), '-u', 'vbrq', quality, '--soundcheck-generate', wav_file, m4a_file])

        m4aData = mutagen.File(m4a_file, easy=True)
        m4aData.update(metadata)
        m4aData.save()
        os.remove(wav_file)

        print 'Converted file:', m4a_file

    except KeyboardInterrupt:
        pass

# =============================================================================
# Main:
# =============================================================================

print 'Processing:', args.location

try:
    pool = Pool()
    p = pool.map_async(convert_flac_to_aac, get_flac_files(args.location))
    p.get(0xFFFF)
except KeyboardInterrupt:
    exit('Aborting.')

rmtree(tmp_location, ignore_errors=True)

print 'Converted %s files.\n' % file_count

# EOF
<|endoftext|>


<commit_before>
import atexit
from queue import Queue
from subprocess import Popen, PIPE
import sys
from threading import Thread
from tkinter import BOTH, END, Frame, Text, Tk, ttk

import psutil

class HarmonbotGUI:

	def __init__(self, master):
		self.master = master
		master.title("Harmonbot")

		self.notebook = ttk.Notebook(master)
		for tab in ("overview", "discord", "discord_listener", "twitch", "telegram"):
			frame = Frame(self.notebook)
			setattr(self, f"{tab}_tab", frame)
			self.notebook.add(frame, text = tab.replace('_', ' ').title())
		self.notebook.pack()

		for overview_frame in ("discord", "discord_listener", "twitch", "telegram"):
			frame = Frame(self.overview_tab)
			setattr(self, f"overview_{overview_frame}_frame", frame)
			text = Text(frame)
			setattr(self, f"overview_{overview_frame}_text", text)
			text.pack()
		self.overview_discord_frame.grid(row = 1, column = 1)
		self.overview_discord_listener_frame.grid(row = 2, column = 1)
		self.overview_twitch_frame.grid(row = 1, column = 2)
		self.overview_telegram_frame.grid(row = 2, column = 2)

		for tab in ("discord", "discord_listener", "twitch", "telegram"):
			notebook_tab = getattr(self, f"{tab}_tab")
			frame = Frame(notebook_tab)
			setattr(self, f"{tab}_frame", frame)
			frame.pack(expand = True, fill = BOTH)
			text = Text(frame)
			setattr(self, f"{tab}_text", text)
			text.pack(expand = True, fill = BOTH)

if __name__ == "__main__":
	root = Tk()
	root.wm_state("zoomed")  # Maximized window
	harmonbot_gui = HarmonbotGUI(root)

	processes = {}
	process_kwargs = {"stdout": PIPE, "stderr": PIPE, "bufsize": 1, "universal_newlines": True}
	processes["discord"] = Popen([sys.executable, "-u", "Harmonbot.py"], cwd = "Discord", **process_kwargs)
	processes["discord_listener"] = Popen(["go", "run", "Harmonbot_Listener.go"], cwd = "Discord", shell = True, **process_kwargs)
	processes["twitch"] = Popen(["pyw", "-3.6", "-u", "Twitch_Harmonbot.py"], cwd = "Twitch", **process_kwargs)
	# TODO: Update to use Python 3.7 executable
	processes["telegram"] = Popen([sys.executable, "-u", "Telegram_Harmonbot.py"], cwd = "Telegram", **process_kwargs)

	def enqueue_output(out, queue):
		for line in iter(out.readline, ""):
			queue.put(line)
		out.close()

	stdout_queues = {}
	stdout_threads = {}
	stderr_queues = {}
	stderr_threads = {}
	for name, process in processes.items():
		# stdout
		stdout_queue = Queue()
		stdout_queues[name] = stdout_queue
		stdout_thread = Thread(target = enqueue_output, args = (process.stdout, stdout_queue))
		stdout_threads[name] = stdout_thread
		stdout_thread.daemon = True
		stdout_thread.start()
		# stderr
		stderr_queue = Queue()
		stderr_queues[name] = stderr_queue
		stderr_thread = Thread(target = enqueue_output, args = (process.stderr, stderr_queue))
		stderr_threads[name] = stderr_thread
		stderr_thread.daemon = True
		stderr_thread.start()

	def process_outputs():
		for name, stdout_queue in stdout_queues.items():
			while not stdout_queue.empty():
				line = stdout_queue.get_nowait()
				text = getattr(harmonbot_gui, f"overview_{name}_text")
				text.insert(END, line)
				text = getattr(harmonbot_gui, f"{name}_text")
				text.insert(END, line)
		root.after(1, process_outputs)

	def process_error_outputs():
		for name, stderr_queue in stderr_queues.items():
			while not stderr_queue.empty():
				line = stderr_queue.get_nowait()
				text = getattr(harmonbot_gui, f"overview_{name}_text")
				text.insert(END, line)
				text = getattr(harmonbot_gui, f"{name}_text")
				text.insert(END, line)
		root.after(1, process_error_outputs)
		# TODO: Check order with stdout

	def check_discord_process_ended():
		if processes["discord"].poll() is None:
			root.after(1, check_discord_process_ended)
		else:
			line = "Discord process ended"
			harmonbot_gui.overview_discord_text.insert(END, line)
			harmonbot_gui.discord_text.insert(END, line)

	def check_discord_listener_process_ended():
		if processes["discord_listener"].poll() is None:
			root.after(1, check_discord_listener_process_ended)
		else:
			line = "Discord listener process ended"
			harmonbot_gui.overview_discord_listener_text.insert(END, line)
			harmonbot_gui.discord_listener_text.insert(END, line)

	def check_twitch_process_ended():
		if processes["twitch"].poll() is None:
			root.after(1, check_twitch_process_ended)
		else:
			line = "Twitch process ended"
			harmonbot_gui.overview_twitch_text.insert(END, line)
			harmonbot_gui.twitch_text.insert(END, line)

	def check_telegram_process_ended():
		if processes["telegram"].poll() is None:
			root.after(1, check_telegram_process_ended)
		else:
			line = "Telegram process ended"
			harmonbot_gui.overview_telegram_text.insert(END, line)
			harmonbot_gui.telegram_text.insert(END, line)

	for function in (process_outputs, process_error_outputs, check_discord_process_ended,
						check_discord_listener_process_ended, check_twitch_process_ended,
						check_telegram_process_ended):
		root.after(0, function)

	def cleanup():
		processes["discord"].terminate()
		go_process = psutil.Process(processes["discord_listener"].pid)
		for process in go_process.children(recursive = True):
			process.terminate()
		go_process.terminate()
		processes["twitch"].terminate()
		processes["telegram"].terminate()
		## root.destroy()

	atexit.register(cleanup)

	## root.protocol("WM_DELETE_WINDOW", cleanup)
	root.mainloop()

<commit_msg>[GUI] Dynamically output stdout and stderr to each text object<commit_after>
import atexit
from queue import Queue
from subprocess import Popen, PIPE
import sys
from threading import Thread
from tkinter import BOTH, END, Frame, Text, Tk, ttk

import psutil

class HarmonbotGUI:

	def __init__(self, master):
		self.master = master
		master.title("Harmonbot")

		self.notebook = ttk.Notebook(master)
		for tab in ("overview", "discord", "discord_listener", "twitch", "telegram"):
			frame = Frame(self.notebook)
			setattr(self, f"{tab}_tab", frame)
			self.notebook.add(frame, text = tab.replace('_', ' ').title())
		self.notebook.pack()

		for overview_frame in ("discord", "discord_listener", "twitch", "telegram"):
			frame = Frame(self.overview_tab)
			setattr(self, f"overview_{overview_frame}_frame", frame)
			text = Text(frame)
			setattr(self, f"overview_{overview_frame}_text", text)
			text.pack()
		self.overview_discord_frame.grid(row = 1, column = 1)
		self.overview_discord_listener_frame.grid(row = 2, column = 1)
		self.overview_twitch_frame.grid(row = 1, column = 2)
		self.overview_telegram_frame.grid(row = 2, column = 2)

		for tab in ("discord", "discord_listener", "twitch", "telegram"):
			notebook_tab = getattr(self, f"{tab}_tab")
			frame = Frame(notebook_tab)
			setattr(self, f"{tab}_frame", frame)
			frame.pack(expand = True, fill = BOTH)
			text = Text(frame)
			setattr(self, f"{tab}_text", text)
			text.pack(expand = True, fill = BOTH)

if __name__ == "__main__":
	root = Tk()
	root.wm_state("zoomed")  # Maximized window
	harmonbot_gui = HarmonbotGUI(root)

	processes = {}
	process_kwargs = {"stdout": PIPE, "stderr": PIPE, "bufsize": 1, "universal_newlines": True}
	processes["discord"] = Popen([sys.executable, "-u", "Harmonbot.py"], cwd = "Discord", **process_kwargs)
	processes["discord_listener"] = Popen(["go", "run", "Harmonbot_Listener.go"], cwd = "Discord", shell = True, **process_kwargs)
	processes["twitch"] = Popen(["pyw", "-3.6", "-u", "Twitch_Harmonbot.py"], cwd = "Twitch", **process_kwargs)
	# TODO: Update to use Python 3.7 executable
	processes["telegram"] = Popen([sys.executable, "-u", "Telegram_Harmonbot.py"], cwd = "Telegram", **process_kwargs)

	def enqueue_output(out, queue):
		for line in iter(out.readline, ""):
			queue.put(line)
		out.close()

	stdout_queues = {}
	stdout_threads = {}
	stderr_queues = {}
	stderr_threads = {}
	for name, process in processes.items():
		# stdout
		stdout_queue = Queue()
		stdout_queues[name] = stdout_queue
		stdout_thread = Thread(target = enqueue_output, args = (process.stdout, stdout_queue))
		stdout_threads[name] = stdout_thread
		stdout_thread.daemon = True
		stdout_thread.start()
		# stderr
		stderr_queue = Queue()
		stderr_queues[name] = stderr_queue
		stderr_thread = Thread(target = enqueue_output, args = (process.stderr, stderr_queue))
		stderr_threads[name] = stderr_thread
		stderr_thread.daemon = True
		stderr_thread.start()

	def process_outputs():
		for name, stdout_queue in stdout_queues.items():
			while not stdout_queue.empty():
				line = stdout_queue.get_nowait()
				for text_name in (f"overview_{name}_text", f"{name}_text"):
					text = getattr(harmonbot_gui, text_name)
					text.insert(END, line)
		root.after(1, process_outputs)

	def process_error_outputs():
		for name, stderr_queue in stderr_queues.items():
			while not stderr_queue.empty():
				line = stderr_queue.get_nowait()
				for text_name in (f"overview_{name}_text", f"{name}_text"):
					text = getattr(harmonbot_gui, text_name)
					text.insert(END, line)
		root.after(1, process_error_outputs)
		# TODO: Check order with stdout

	def check_discord_process_ended():
		if processes["discord"].poll() is None:
			root.after(1, check_discord_process_ended)
		else:
			line = "Discord process ended"
			harmonbot_gui.overview_discord_text.insert(END, line)
			harmonbot_gui.discord_text.insert(END, line)

	def check_discord_listener_process_ended():
		if processes["discord_listener"].poll() is None:
			root.after(1, check_discord_listener_process_ended)
		else:
			line = "Discord listener process ended"
			harmonbot_gui.overview_discord_listener_text.insert(END, line)
			harmonbot_gui.discord_listener_text.insert(END, line)

	def check_twitch_process_ended():
		if processes["twitch"].poll() is None:
			root.after(1, check_twitch_process_ended)
		else:
			line = "Twitch process ended"
			harmonbot_gui.overview_twitch_text.insert(END, line)
			harmonbot_gui.twitch_text.insert(END, line)

	def check_telegram_process_ended():
		if processes["telegram"].poll() is None:
			root.after(1, check_telegram_process_ended)
		else:
			line = "Telegram process ended"
			harmonbot_gui.overview_telegram_text.insert(END, line)
			harmonbot_gui.telegram_text.insert(END, line)

	for function in (process_outputs, process_error_outputs, check_discord_process_ended,
						check_discord_listener_process_ended, check_twitch_process_ended,
						check_telegram_process_ended):
		root.after(0, function)

	def cleanup():
		processes["discord"].terminate()
		go_process = psutil.Process(processes["discord_listener"].pid)
		for process in go_process.children(recursive = True):
			process.terminate()
		go_process.terminate()
		processes["twitch"].terminate()
		processes["telegram"].terminate()
		## root.destroy()

	atexit.register(cleanup)

	## root.protocol("WM_DELETE_WINDOW", cleanup)
	root.mainloop()

<|endoftext|>