<commit_before>import os
import argparse
import sys

from ij import IJ
from loci.plugins import BF
from loci.plugins.in import ImporterOptions
#import WindowManager

def split_top(path):
    """Like `os.path.split`, but splitting from the topmost directory.

    Parameters
    ----------
    path : string
        The path to be split.

    Returns
    -------
    (head, tail) : string tuple
        The topmost directory in path (`head`) and the rest of the path
        (`tail`).

    Examples
    --------
    >>> path = '/path/to/some/files'
    >>> split_top(path)
    ('/path', 'to/some/files')
    """
    split = os.path.split
    tails = []
    head = path
    tail = ''
    while len(head) != 0 and head != '/':
        head, tail = split(head)
        tails.append(tail)
    if head == '/':
        tails.append(head)
    tails.reverse()
    head = tails[0]
    if len(tails[1:]) > 0:
        tail = os.path.join(*tails[1:])
    else:
        tail = ''
    return head, tail

def is_cellomics_image(fn):
    """Determine whether a file is a Cellomics image.

    Parameters
    ----------
    fn : string
        The filename of the file in question.

    Returns
    -------
    is_cellom : bool
        True if the filename points to a Cellomics image.
    """
    is_cellom = fn.endswith('.C01') or fn.endswith('.c01')
    return is_cellom

def is_cellomics_mask(fn):
    """Determine whether a file is a Cellomics mask image.

    Parameters
    ----------
    fn : string
        The filename.

    Returns
    -------
    is_mask : bool
        True if the filename points to a Cellomics mask image.
    """
    is_mask = fn.endswith('o1.C01') or fn.endswith('o1.c01')
    return is_mask

def convert_files(out_base, path, files, error_file=None, ignore_masks=False):
    """Convert cellomics .C01 files to TIFF files in a sibling directory.

    This function is designed to be used with `os.walk`.

    Parameters
    ----------
    out_base : string
        The name of the sibling directory in which to place converted files.
    path : string
        The path to the files to be converted.
    files : list of string
        The filenames of files, including non-.C01 files.
    error_file : string, optional
        A file to which to log "corrupted" images.
    ignore_masks : bool, optional
        Ignore files ending in "o1.C01".

    Returns
    -------
    None : Use for side effects only.

    Examples
    --------
    >>> in_dir = 'cellomics_files'
    >>> out_dir = 'tiff_files'
    >>> files = os.listdir(in_dir)
    >>> files
    ['image1.c01', 'image2.c01']
    >>> convert_files(out_dir, in_dir, files)
    >>> os.listdir(out_dir)
    ['image1.tif', 'image2.tif']
    """
    in_base, in_tail = split_top(path)
    out_path = os.path.join(out_base, in_tail)
    if not os.path.isdir(out_path):
        os.makedirs(out_path)
    if error_file is None:
        ferr = sys.stdout
    else:
        ferr = open(os.path.join(out_base, error_file), 'w')
    files = filter(is_cellomics_image, files)
    if ignore_masks:
        files = filter(lambda fn: not is_cellomics_mask(fn), files)
    files = sorted(files)
    for fn in files:
        fin = os.path.join(path, fn)
        print fin
        fout = os.path.join(out_path, fn)[:-4] + '.tif'
        opts = ImporterOptions()
        opts.setUngroupFiles(True)
        if not os.path.exists(fout):
            opts.setId(fin)
            try:
                imp = BF.openImagePlus(opts)[0]
            except:
                ferr.write(fin + '\n')
                ferr.flush()
            else:
                print "creating", fout
                IJ.saveAs(imp, 'Tiff', fout)
                imp.close()
        else:
            print fout, "exists"

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Convert a bunch of Cellomics .C01 files to TIFFs.')
    parser.add_argument('root_path', help='The path containing .C01 files')
    parser.add_argument('out_path', help='The path to output the TIFFs.')
    parser.add_argument('-E', '--error-file', metavar='FILENAME',
                        help='Log problem filenames to the given filename.')
    parser.add_argument('-m', '--ignore-masks', action='store_true',
                        help='Ignore files ending in "o1.C01".')

    print sys.argv
    args = parser.parse_args()
    paths = os.walk(args.root_path)
    for path, dirs, files in paths:
        convert_files(path.replace(args.root_path, args.out_path), path,
                      files, args.error_file, args.ignore_masks)
<commit_msg>Only replace first instance of root path<commit_after>import os
import argparse
import sys

from ij import IJ
from loci.plugins import BF
from loci.plugins.in import ImporterOptions
#import WindowManager

def split_top(path):
    """Like `os.path.split`, but splitting from the topmost directory.

    Parameters
    ----------
    path : string
        The path to be split.

    Returns
    -------
    (head, tail) : string tuple
        The topmost directory in path (`head`) and the rest of the path
        (`tail`).

    Examples
    --------
    >>> path = '/path/to/some/files'
    >>> split_top(path)
    ('/path', 'to/some/files')
    """
    split = os.path.split
    tails = []
    head = path
    tail = ''
    while len(head) != 0 and head != '/':
        head, tail = split(head)
        tails.append(tail)
    if head == '/':
        tails.append(head)
    tails.reverse()
    head = tails[0]
    if len(tails[1:]) > 0:
        tail = os.path.join(*tails[1:])
    else:
        tail = ''
    return head, tail

def is_cellomics_image(fn):
    """Determine whether a file is a Cellomics image.

    Parameters
    ----------
    fn : string
        The filename of the file in question.

    Returns
    -------
    is_cellom : bool
        True if the filename points to a Cellomics image.
    """
    is_cellom = fn.endswith('.C01') or fn.endswith('.c01')
    return is_cellom

def is_cellomics_mask(fn):
    """Determine whether a file is a Cellomics mask image.

    Parameters
    ----------
    fn : string
        The filename.

    Returns
    -------
    is_mask : bool
        True if the filename points to a Cellomics mask image.
    """
    is_mask = fn.endswith('o1.C01') or fn.endswith('o1.c01')
    return is_mask

def convert_files(out_base, path, files, error_file=None, ignore_masks=False):
    """Convert cellomics .C01 files to TIFF files in a sibling directory.

    This function is designed to be used with `os.walk`.

    Parameters
    ----------
    out_base : string
        The name of the sibling directory in which to place converted files.
    path : string
        The path to the files to be converted.
    files : list of string
        The filenames of files, including non-.C01 files.
    error_file : string, optional
        A file to which to log "corrupted" images.
    ignore_masks : bool, optional
        Ignore files ending in "o1.C01".

    Returns
    -------
    None : Use for side effects only.

    Examples
    --------
    >>> in_dir = 'cellomics_files'
    >>> out_dir = 'tiff_files'
    >>> files = os.listdir(in_dir)
    >>> files
    ['image1.c01', 'image2.c01']
    >>> convert_files(out_dir, in_dir, files)
    >>> os.listdir(out_dir)
    ['image1.tif', 'image2.tif']
    """
    in_base, in_tail = split_top(path)
    out_path = os.path.join(out_base, in_tail)
    if not os.path.isdir(out_path):
        os.makedirs(out_path)
    if error_file is None:
        ferr = sys.stdout
    else:
        ferr = open(os.path.join(out_base, error_file), 'w')
    files = filter(is_cellomics_image, files)
    if ignore_masks:
        files = filter(lambda fn: not is_cellomics_mask(fn), files)
    files = sorted(files)
    for fn in files:
        fin = os.path.join(path, fn)
        print fin
        fout = os.path.join(out_path, fn)[:-4] + '.tif'
        opts = ImporterOptions()
        opts.setUngroupFiles(True)
        if not os.path.exists(fout):
            opts.setId(fin)
            try:
                imp = BF.openImagePlus(opts)[0]
            except:
                ferr.write(fin + '\n')
                ferr.flush()
            else:
                print "creating", fout
                IJ.saveAs(imp, 'Tiff', fout)
                imp.close()
        else:
            print fout, "exists"

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Convert a bunch of Cellomics .C01 files to TIFFs.')
    parser.add_argument('root_path', help='The path containing .C01 files')
    parser.add_argument('out_path', help='The path to output the TIFFs.')
    parser.add_argument('-E', '--error-file', metavar='FILENAME',
                        help='Log problem filenames to the given filename.')
    parser.add_argument('-m', '--ignore-masks', action='store_true',
                        help='Ignore files ending in "o1.C01".')

    print sys.argv
    args = parser.parse_args()
    paths = os.walk(args.root_path)
    for path, dirs, files in paths:
        convert_files(path.replace(args.root_path, args.out_path, 1), path,
                      files, args.error_file, args.ignore_masks)
<|endoftext|>


<commit_before>import sys
import traceback

from django.core.management.base import BaseCommand

from ...models import DataSource

class Command(BaseCommand):
    help = 'Synchronize data sources'

    def handle(self, *args, **options):
        self.stdout.write('sync: Running syncdata')

        for data_source_cls in DataSource.__subclasses__():
            print 'sync: Synchronizing %s' % data_source_cls.__name__
            for data_source in data_source_cls.objects.all():
                try:
                    data_source.synchronize()
                except Exception:
                    print 'There was an exception while synchronizing %s' % (
                        data_source,)
                    traceback.print_exc(file=sys.stdout)
                    continue

        self.stdout.write('sync: Done running syncdata')
<commit_msg>more accurate name<commit_after>import sys
import traceback

from django.core.management.base import BaseCommand

from ...models import DataSource

class Command(BaseCommand):
    help = 'Synchronize data sources'

    def handle(self, *args, **options):
        self.stdout.write('sync: Running syncdata')

        for data_source_cls in DataSource.__subclasses__():
            for data_source in data_source_cls.objects.all():
                print 'sync: Synchronizing %s' % data_source.get_name_display()
                try:
                    data_source.synchronize()
                except Exception:
                    print 'There was an exception while synchronizing %s' % (
                        data_source,)
                    traceback.print_exc(file=sys.stdout)
                    continue

        self.stdout.write('sync: Done running syncdata')
<|endoftext|>


<commit_before>#!/usr/bin/env python

import rados
import rpm
# import subprocess
# import logging
# import argparse

# Refer
# http://permalink.gmane.org/gmane.comp.file-systems.ceph.user/23090
# https://github.com/ceph/calamari/blob/master/salt/srv/salt/_modules/ceph.py
# http://docs.ceph.com/docs/hammer/rados/api/python/

conf = "/etc/ceph/ceph.conf"
ad_key = "/etc/ceph/ceph.client.admin.keyring"

class CephCheck(object):

    def __init__(self, conf=conf, ad_key=ad_key):
        self.conf = conf
        self.ad_key = ad_key

    def verify(self):
        # 1. Check for ceph-deploy rpm
        print("\nceph-check :\n")
        print("Check #1 :")
        ts = rpm.TransactionSet()
        mi = ts.dbMatch()
        mi.pattern('name', rpm.RPMMIRE_GLOB, 'ceph*')
        print("Checking ceph package installations.\n")
        try:
            for h in mi:
                print("%s-%s-%s" % (h['name'], h['version'], h['release']))
        except:
            print("Not able to find `ceph packages`")
        # 2. Check for ceph.conf
        print("\nCheck #2 :")
        try:
            conf_file = open(self.conf)
            print(type(conf_file))
        except:
            print(self.conf, "doesn't exist!")
        #if not self.conf:
        #    print("/etc/ceph/ceph.conf not found")
        #    print("Is this node part of the cluster?")

    def connect(self):
        """Initialize the cluster connection"""
        try:
            cluster = rados.Rados(conffile='')
        except TypeError as e:
            raise e

        print("\nTrying to connect to the cluster..\n")

        try:
            cluster.connect()
        except Exception as e:
            print("Connection error : ", e)
            raise e
        finally:
            print("\nConnected to the cluster!\n")

    def help(self):
        """We help others"""
        print("\n")
        print("Usage : ")
        print("")

if __name__ == "__main__":
    ceph_check = CephCheck()
    ceph_check.verify()  # < add args, else print it'll take the default conf
<commit_msg>Check for redhat-release, need to clean up later<commit_after>#!/usr/bin/env python

import rados
import rpm
# import subprocess
# import logging
# import argparse

conf = "/etc/ceph/ceph.conf"
ad_key = "/etc/ceph/ceph.client.admin.keyring"

class CephCheck(object):

    def __init__(self, conf=conf, ad_key=ad_key):
        self.conf = conf
        self.ad_key = ad_key

    def verify(self):
        # 1. Check ceph packages
        print("\nceph-check :\n")
        print("Check #1 :")
        ts = rpm.TransactionSet()
        mi = ts.dbMatch()
        mi.pattern('name', rpm.RPMMIRE_GLOB, 'ceph*')
        print("Checking ceph package installations.\n")
        try:
            for h in mi:
                print("%s-%s-%s" % (h['name'], h['version'], h['release']))
        except:
            print("Not able to find `ceph packages`")

        # 2. RHEL check
        try:
            with open("/etc/redhat-release", 'r') as release:
                print(release.read())
        except IOError:
            print("Not able to find RHEL release\n")
        self.connect()

    def connect(self):
        """Initialize the cluster connection"""
        try:
            cluster = rados.Rados(conffile='')
        except TypeError as e:
            raise e

        print("\nTrying to connect to the cluster..\n")

        try:
            cluster.connect()
        except Exception as e:
            print("Connection error : ", e)
            raise e
        finally:
            print("\nConnected to the cluster!\n")

    def help(self):
        """We help others"""
        print("\n")
        print("Usage : ")
        print("")

if __name__ == "__main__":
    ceph_check = CephCheck()
    ceph_check.verify()  # < add args, else print it'll take the default conf
<|endoftext|>


<commit_before># (C) Datadog, Inc. 2010-2016
# All rights reserved
# Licensed under Simplified BSD License (see LICENSE)

# stdlib
import os
import re

# 3rd party
import simplejson as json

# project
from utils.subprocess_output import get_subprocess_output
from config import _is_affirmative
from checks import AgentCheck

EVENT_TYPE = SOURCE_TYPE_NAME = 'ceph'

class CephCheck(AgentCheck):
    """ Collect metrics and events from ceph """

    DEFAULT_CEPH_CMD = '/usr/bin/ceph'
    NAMESPACE = "ceph"

    def __init__(self, name, init_config, agentConfig, instances=None):
        AgentCheck.__init__(self, name, init_config, agentConfig, instances)

    def _collect_raw(self, ceph_cmd, instance):
        use_sudo = _is_affirmative(instance.get('use_sudo', False))
        ceph_args = []
        if use_sudo:
            test_sudo = os.system('setsid sudo -l < /dev/null')
            if test_sudo != 0:
                raise Exception('The dd-agent user does not have sudo access')
            ceph_args = ['sudo', ceph_cmd]
        else:
            ceph_args = [ceph_cmd]

        args = ceph_args + ['version']
        try:
            output,_,_ = get_subprocess_output(args, self.log)
        except Exception as e:
            raise Exception('Unable to run cmd=%s: %s' % (' '.join(args), str(e)))

        raw = {}
        for cmd in ('mon_status', 'status', 'df detail', 'osd pool stats', 'osd perf', 'health detail'):
            try:
                args = ceph_args + cmd.split() + ['-fjson']
                output,_,_ = get_subprocess_output(args, self.log)
                res = json.loads(output)
            except Exception as e:
                self.log.warning('Unable to parse data from cmd=%s: %s' % (cmd, str(e)))
                continue

            name = cmd.replace(' ', '_')
            raw[name] = res

        return raw

    def _extract_tags(self, raw, instance):
        tags = instance.get('tags', [])
        if 'mon_status' in raw:
            fsid = raw['mon_status']['monmap']['fsid']
            tags.append(self.NAMESPACE + '_fsid:%s' % fsid)
            tags.append(self.NAMESPACE + '_mon_state:%s' % raw['mon_status']['state'])

        return tags

    def _publish(self, raw, func, keyspec, tags):
        try:
            for k in keyspec:
                raw = raw[k]
            func(self.NAMESPACE + '.' + k, raw, tags)
        except KeyError:
            return

    def _extract_metrics(self, raw, tags):
        try:
            for osdperf in raw['osd_perf']['osd_perf_infos']:
                local_tags = tags + ['ceph_osd:osd%s' % osdperf['id']]
                self._publish(osdperf, self.gauge, ['perf_stats', 'apply_latency_ms'], local_tags)
                self._publish(osdperf, self.gauge, ['perf_stats', 'commit_latency_ms'], local_tags)
        except KeyError:
            self.log.debug('Error retrieving osdperf metrics')

        try:
            health = {'num_near_full_osds': 0, 'num_full_osds': 0}
            # Health summary will be empty if no bad news
            if raw['health_detail']['summary'] != []:
                for osdhealth in raw['health_detail']['detail']:
                    osd, pct = self._osd_pct_used(osdhealth)
                    if osd:
                        local_tags = tags + ['ceph_osd:%s' % osd.replace('.','')]

                        if 'near' in osdhealth:
                            health['num_near_full_osds'] += 1
                            local_health = {'osd.pct_used': pct}
                            self._publish(local_health, self.gauge, ['osd.pct_used'], local_tags)
                        else:
                            health['num_full_osds'] += 1
                            local_health = {'osd.pct_used': pct}
                            self._publish(local_health, self.gauge, ['osd.pct_used'], local_tags)

            self._publish(health, self.gauge, ['num_full_osds'], tags)
            self._publish(health, self.gauge, ['num_near_full_osds'], tags)
        except KeyError:
            self.log.debug('Error retrieving health metrics')

        try:
            for osdinfo in raw['osd_pool_stats']:
                name = osdinfo.get('pool_name')
                local_tags = tags + ['ceph_pool:%s' % name]
                ops = 0
                try:
                    self._publish(osdinfo, self.gauge, ['client_io_rate', 'read_op_per_sec'], local_tags)
                    ops += osdinfo['client_io_rate']['read_op_per_sec']
                except KeyError:
                    osdinfo['client_io_rate'].update({'read_op_per_sec' : 0})
                    self._publish(osdinfo, self.gauge, ['client_io_rate', 'read_op_per_sec'], local_tags)

                try:
                    self._publish(osdinfo, self.gauge, ['client_io_rate', 'write_op_per_sec'], local_tags)
                    ops += osdinfo['client_io_rate']['write_op_per_sec']
                except KeyError:
                    osdinfo['client_io_rate'].update({'write_op_per_sec' : 0})
                    self._publish(osdinfo, self.gauge, ['client_io_rate', 'write_op_per_sec'], local_tags)

                try:
                    osdinfo['client_io_rate']['op_per_sec']
                    self._publish(osdinfo, self.gauge, ['client_io_rate', 'op_per_sec'], local_tags)
                except KeyError:
                    osdinfo['client_io_rate'].update({'op_per_sec' : ops})
                    self._publish(osdinfo, self.gauge, ['client_io_rate', 'op_per_sec'], local_tags)

                try:
                    osdinfo['client_io_rate']['read_bytes_sec']
                    self._publish(osdinfo, self.gauge, ['client_io_rate', 'read_bytes_sec'], local_tags)
                except KeyError:
                    osdinfo['client_io_rate'].update({'read_bytes_sec' : 0})
                    self._publish(osdinfo, self.gauge, ['client_io_rate', 'read_bytes_sec'], local_tags)

                try:
                    osdinfo['client_io_rate']['write_bytes_sec']
                    self._publish(osdinfo, self.gauge, ['client_io_rate', 'write_bytes_sec'], local_tags)
                except KeyError:
                    osdinfo['client_io_rate'].update({'write_bytes_sec' : 0})
                    self._publish(osdinfo, self.gauge, ['client_io_rate', 'write_bytes_sec'], local_tags)
        except KeyError:
            self.log.debug('Error retrieving osd_pool_stats metrics')

        try:
            osdstatus = raw['status']['osdmap']['osdmap']
            self._publish(osdstatus, self.gauge, ['num_osds'], tags)
            self._publish(osdstatus, self.gauge, ['num_in_osds'], tags)
            self._publish(osdstatus, self.gauge, ['num_up_osds'], tags)

        except KeyError:
            self.log.debug('Error retrieving osdstatus metrics')

        try:
            pgstatus = raw['status']['pgmap']
            self._publish(pgstatus, self.gauge, ['num_pgs'], tags)
            for pgstate in pgstatus['pgs_by_state']:
                s_name = pgstate['state_name'].replace("+", "_")
                self.gauge(self.NAMESPACE + '.pgstate.' + s_name, pgstate['count'], tags)
        except KeyError:
            self.log.debug('Error retrieving pgstatus metrics')

        try:
            num_mons = len(raw['mon_status']['monmap']['mons'])
            self.gauge(self.NAMESPACE + '.num_mons', num_mons, tags)
        except KeyError:
            self.log.debug('Error retrieving mon_status metrics')

        try:
            stats = raw['df_detail']['stats']
            self._publish(stats, self.gauge, ['total_objects'], tags)
            used = float(stats['total_used_bytes'])
            total = float(stats['total_bytes'])
            if total > 0:
                self.gauge(self.NAMESPACE + '.aggregate_pct_used', 100.0*used/total, tags)

            l_pools = raw['df_detail']['pools']
            self.gauge(self.NAMESPACE + '.num_pools', len(l_pools), tags)
            for pdata in l_pools:
                local_tags = list(tags + [self.NAMESPACE + '_pool:%s' % pdata['name']])
                stats = pdata['stats']
                used = float(stats['bytes_used'])
                avail = float(stats['max_avail'])
                total = used+avail
                if total > 0:
                    self.gauge(self.NAMESPACE + '.pct_used', 100.0*used/total, local_tags)
                self.gauge(self.NAMESPACE + '.num_objects', stats['objects'], local_tags)
                self.rate(self.NAMESPACE + '.read_bytes', stats['rd_bytes'], local_tags)
                self.rate(self.NAMESPACE + '.write_bytes', stats['wr_bytes'], local_tags)

        except (KeyError, ValueError):
            self.log.debug('Error retrieving df_detail metrics')

    def _osd_pct_used(self, health):
            """Take a single health check string, return (OSD name, percentage used)"""
            # Full string looks like: osd.2 is full at 95%
            # Near full string: osd.1 is near full at 94%
            pct = re.compile('\d+%').findall(health)
            osd = re.compile('osd.\d+').findall(health)
            if len(pct) > 0 and len(osd) > 0:
                return osd[0], int(pct[0][:-1])
            else:
                return None, None

    def _perform_service_checks(self, raw, tags):
        if 'status' in raw:
            s_status = raw['status']['health']['overall_status']
            if s_status.find('_OK') != -1:
                status = AgentCheck.OK
            elif s_status.find('_WARN') != -1:
                status = AgentCheck.WARNING
            else:
                status = AgentCheck.CRITICAL
            self.service_check(self.NAMESPACE + '.overall_status', status)

    def check(self, instance):
        ceph_cmd = instance.get('ceph_cmd') or self.DEFAULT_CEPH_CMD
        raw = self._collect_raw(ceph_cmd, instance)
        tags = self._extract_tags(raw, instance)
        self._extract_metrics(raw, tags)
        self._perform_service_checks(raw, tags)
<commit_msg>updates ceph<commit_after># (C) Datadog, Inc. 2010-2016
# All rights reserved
# Licensed under Simplified BSD License (see LICENSE)

"""ceph check
Collects metrics from ceph clusters
"""

# stdlib
import os
import re

# project
from checks import AgentCheck
from utils.subprocess_output import get_subprocess_output
from config import _is_affirmative

EVENT_TYPE = SOURCE_TYPE_NAME = 'ceph'

# third party
import simplejson as json

class Ceph(AgentCheck):
    """ Collect metrics and events from ceph """

    DEFAULT_CEPH_CMD = '/usr/bin/ceph'
    NAMESPACE = "ceph"

    def _collect_raw(self, ceph_cmd, instance):
        use_sudo = _is_affirmative(instance.get('use_sudo', False))
        ceph_args = []
        if use_sudo:
            test_sudo = os.system('setsid sudo -l < /dev/null')
            if test_sudo != 0:
                raise Exception('The dd-agent user does not have sudo access')
            ceph_args = ['sudo', ceph_cmd]
        else:
            ceph_args = [ceph_cmd]

        args = ceph_args + ['version']
        try:
            output,_,_ = get_subprocess_output(args, self.log)
        except Exception as e:
            raise Exception('Unable to run cmd=%s: %s' % (' '.join(args), str(e)))

        raw = {}
        for cmd in ('mon_status', 'status', 'df detail', 'osd pool stats', 'osd perf', 'health detail'):
            try:
                args = ceph_args + cmd.split() + ['-fjson']
                output,_,_ = get_subprocess_output(args, self.log)
                res = json.loads(output)
            except Exception as e:
                self.log.warning('Unable to parse data from cmd=%s: %s' % (cmd, str(e)))
                continue

            name = cmd.replace(' ', '_')
            raw[name] = res

        return raw

    def _extract_tags(self, raw, instance):
        tags = instance.get('tags', [])
        if 'mon_status' in raw:
            fsid = raw['mon_status']['monmap']['fsid']
            tags.append(self.NAMESPACE + '_fsid:%s' % fsid)
            tags.append(self.NAMESPACE + '_mon_state:%s' % raw['mon_status']['state'])

        return tags

    def _publish(self, raw, func, keyspec, tags):
        try:
            for k in keyspec:
                raw = raw[k]
            func(self.NAMESPACE + '.' + k, raw, tags)
        except KeyError:
            return

    def _extract_metrics(self, raw, tags):
        try:
            for osdperf in raw['osd_perf']['osd_perf_infos']:
                local_tags = tags + ['ceph_osd:osd%s' % osdperf['id']]
                self._publish(osdperf, self.gauge, ['perf_stats', 'apply_latency_ms'], local_tags)
                self._publish(osdperf, self.gauge, ['perf_stats', 'commit_latency_ms'], local_tags)
        except KeyError:
            self.log.debug('Error retrieving osdperf metrics')

        try:
            health = {'num_near_full_osds': 0, 'num_full_osds': 0}
            # Health summary will be empty if no bad news
            if raw['health_detail']['summary'] != []:
                for osdhealth in raw['health_detail']['detail']:
                    osd, pct = self._osd_pct_used(osdhealth)
                    if osd:
                        local_tags = tags + ['ceph_osd:%s' % osd.replace('.','')]

                        if 'near' in osdhealth:
                            health['num_near_full_osds'] += 1
                            local_health = {'osd.pct_used': pct}
                            self._publish(local_health, self.gauge, ['osd.pct_used'], local_tags)
                        else:
                            health['num_full_osds'] += 1
                            local_health = {'osd.pct_used': pct}
                            self._publish(local_health, self.gauge, ['osd.pct_used'], local_tags)

            self._publish(health, self.gauge, ['num_full_osds'], tags)
            self._publish(health, self.gauge, ['num_near_full_osds'], tags)
        except KeyError:
            self.log.debug('Error retrieving health metrics')

        try:
            for osdinfo in raw['osd_pool_stats']:
                name = osdinfo.get('pool_name')
                local_tags = tags + ['ceph_pool:%s' % name]
                ops = 0
                try:
                    self._publish(osdinfo, self.gauge, ['client_io_rate', 'read_op_per_sec'], local_tags)
                    ops += osdinfo['client_io_rate']['read_op_per_sec']
                except KeyError:
                    osdinfo['client_io_rate'].update({'read_op_per_sec' : 0})
                    self._publish(osdinfo, self.gauge, ['client_io_rate', 'read_op_per_sec'], local_tags)

                try:
                    self._publish(osdinfo, self.gauge, ['client_io_rate', 'write_op_per_sec'], local_tags)
                    ops += osdinfo['client_io_rate']['write_op_per_sec']
                except KeyError:
                    osdinfo['client_io_rate'].update({'write_op_per_sec' : 0})
                    self._publish(osdinfo, self.gauge, ['client_io_rate', 'write_op_per_sec'], local_tags)

                try:
                    osdinfo['client_io_rate']['op_per_sec']
                    self._publish(osdinfo, self.gauge, ['client_io_rate', 'op_per_sec'], local_tags)
                except KeyError:
                    osdinfo['client_io_rate'].update({'op_per_sec' : ops})
                    self._publish(osdinfo, self.gauge, ['client_io_rate', 'op_per_sec'], local_tags)

                try:
                    osdinfo['client_io_rate']['read_bytes_sec']
                    self._publish(osdinfo, self.gauge, ['client_io_rate', 'read_bytes_sec'], local_tags)
                except KeyError:
                    osdinfo['client_io_rate'].update({'read_bytes_sec' : 0})
                    self._publish(osdinfo, self.gauge, ['client_io_rate', 'read_bytes_sec'], local_tags)

                try:
                    osdinfo['client_io_rate']['write_bytes_sec']
                    self._publish(osdinfo, self.gauge, ['client_io_rate', 'write_bytes_sec'], local_tags)
                except KeyError:
                    osdinfo['client_io_rate'].update({'write_bytes_sec' : 0})
                    self._publish(osdinfo, self.gauge, ['client_io_rate', 'write_bytes_sec'], local_tags)
        except KeyError:
            self.log.debug('Error retrieving osd_pool_stats metrics')

        try:
            osdstatus = raw['status']['osdmap']['osdmap']
            self._publish(osdstatus, self.gauge, ['num_osds'], tags)
            self._publish(osdstatus, self.gauge, ['num_in_osds'], tags)
            self._publish(osdstatus, self.gauge, ['num_up_osds'], tags)

        except KeyError:
            self.log.debug('Error retrieving osdstatus metrics')

        try:
            pgstatus = raw['status']['pgmap']
            self._publish(pgstatus, self.gauge, ['num_pgs'], tags)
            for pgstate in pgstatus['pgs_by_state']:
                s_name = pgstate['state_name'].replace("+", "_")
                self.gauge(self.NAMESPACE + '.pgstate.' + s_name, pgstate['count'], tags)
        except KeyError:
            self.log.debug('Error retrieving pgstatus metrics')

        try:
            num_mons = len(raw['mon_status']['monmap']['mons'])
            self.gauge(self.NAMESPACE + '.num_mons', num_mons, tags)
        except KeyError:
            self.log.debug('Error retrieving mon_status metrics')

        try:
            stats = raw['df_detail']['stats']
            self._publish(stats, self.gauge, ['total_objects'], tags)
            used = float(stats['total_used_bytes'])
            total = float(stats['total_bytes'])
            if total > 0:
                self.gauge(self.NAMESPACE + '.aggregate_pct_used', 100.0*used/total, tags)

            l_pools = raw['df_detail']['pools']
            self.gauge(self.NAMESPACE + '.num_pools', len(l_pools), tags)
            for pdata in l_pools:
                local_tags = list(tags + [self.NAMESPACE + '_pool:%s' % pdata['name']])
                stats = pdata['stats']
                used = float(stats['bytes_used'])
                avail = float(stats['max_avail'])
                total = used+avail
                if total > 0:
                    self.gauge(self.NAMESPACE + '.pct_used', 100.0*used/total, local_tags)
                self.gauge(self.NAMESPACE + '.num_objects', stats['objects'], local_tags)
                self.rate(self.NAMESPACE + '.read_bytes', stats['rd_bytes'], local_tags)
                self.rate(self.NAMESPACE + '.write_bytes', stats['wr_bytes'], local_tags)

        except (KeyError, ValueError):
            self.log.debug('Error retrieving df_detail metrics')

    def _osd_pct_used(self, health):
            """Take a single health check string, return (OSD name, percentage used)"""
            # Full string looks like: osd.2 is full at 95%
            # Near full string: osd.1 is near full at 94%
            pct = re.compile('\d+%').findall(health)
            osd = re.compile('osd.\d+').findall(health)
            if len(pct) > 0 and len(osd) > 0:
                return osd[0], int(pct[0][:-1])
            else:
                return None, None

    def _perform_service_checks(self, raw, tags):
        if 'status' in raw:
            s_status = raw['status']['health']['overall_status']
            if s_status.find('_OK') != -1:
                status = AgentCheck.OK
            elif s_status.find('_WARN') != -1:
                status = AgentCheck.WARNING
            else:
                status = AgentCheck.CRITICAL
            self.service_check(self.NAMESPACE + '.overall_status', status)

    def check(self, instance):
        ceph_cmd = instance.get('ceph_cmd') or self.DEFAULT_CEPH_CMD
        raw = self._collect_raw(ceph_cmd, instance)
        tags = self._extract_tags(raw, instance)
        self._extract_metrics(raw, tags)
        self._perform_service_checks(raw, tags)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (C) 2016 Red Hat, Inc., Vimal A.R <vimal@redhat.com>

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

# __author__ = "Vimal A.R"
# __version__ = "v0.1"
# __license__ = "GNU GPL v2.0"
# __email__ = "vimal@redhat.com"

from __future__ import print_function
import time
import subprocess
import json
import sys
import os
import getpass
import ConfigParser
import logging

CONF_FILE = "/etc/ceph/ceph.conf"
ADMIN_KEYRING = "/etc/ceph/ceph.client.admin.keyring"
CEPH_CHECK_LOG = os.path.expanduser("~") + "/ceph_check.log"

# `logging` MODULE CONFIG ###
# 1. Set the application name (override the default `root` logger)
cc_logger = logging.getLogger("ceph_check")
cc_logger.setLevel(logging.DEBUG)
# 2. Set a Log handler
handler = logging.FileHandler(CEPH_CHECK_LOG)
# 3. Set up a log format
log_format = logging.Formatter(
    '%(asctime)s: %(name)s: %(levelname)s: %(message)s')
# 4. Pass the log format to the log handler
handler.setFormatter(log_format)
# 5. Add the log handler to the logger object `cc_logger`
cc_logger.addHandler(handler)
# LOGGING MODULE CONFIG END ###

class CephCheck(object):
    """The main ceph-check class"""

    def __init__(self, conffile, keyring):
        self.conffile = conffile
        self.keyring = keyring
        self.cc_condition()

    def cc_condition(self):
        """
        Conditions the user has to meet for
        a successfull execution of `ceph_check`
        """
        cc_logger.info("##################START###################")
        cc_logger.info("Printing the ideal conditions to the user!")
        print("\nNOTE:\n")
        print("`ceph_check` has the following three requirements:")
        print(" * Ansible has to present on this node.")
        print(" * Passwordless SSH access to the cluster nodes")
        print(" * Read access to the admin keyring on this node")
        print("\nIf these conditions are not met, this program may error out!")
        cc_logger.info("Requesting user's agreement on the conditions [Y/N]")
        agreement = raw_input("\nCan we proceed? (Y/N) : ")
        if agreement in ["Y", "y", "yes", "Yes"]:
            cc_logger.info("User answered {0}".format(agreement))
            cc_logger.info("Calling keyring_check()")
            self.keyring_check()
        else:
            cc_logger.info("User answered {0}".format(agreement))
            print("Please re-run the program after meeting the criteria!")
            cc_logger.info("Exiting!")
            sys.exit(-1)

    def keyring_check(self):
        """
        Check if a custom keyring exists
        """
        config_file = ConfigParser.SafeConfigParser()
        cc_logger.info("Reading '{0}'".format(CONF_FILE))
        config_file.read(self.conffile)
        cc_logger.info("Checking keyring")
        try:
            keyring_custom = config_file.get('global', 'keyring')
            cc_logger.info(
                "Found custom keyring at {0}".format(keyring_custom))
            self.keyring_permission(keyring_custom)
        except ConfigParser.NoOptionError:
            cc_logger.info(
                "No custom admin keyring specified in %s" % self.conffile)
            # cc_logger.info("Falling back to %s" % self.keyring)
            cc_logger.info("Falling back to {0}".format(self.keyring))
            cc_logger.info("Calling keyring_permission()")
            self.keyring_permission(self.keyring)

    def keyring_permission(self, keyring):
        """
        Check the existence and permission of the keyring
        """
        if os.path.isfile(self.keyring):
            cc_logger.info("{0} exists".format(self.keyring))
            if os.access(self.keyring, os.R_OK):
                cc_logger.info("{0} has read permissions".format(self.keyring))
                cc_logger.info("Calling ceph_report()")
                self.ceph_report()
            else:
                cc_logger.info("User {0} does not have read permissions for {1}".format(
                    getpass.getuser(), self.keyring))
                print("User {0} does not have read permissions for {1}".format(
                    getpass.getuser(), self.keyring))
                cc_logger.info("Exiting!")
                print("\nExiting!\n")
                sys.exit(-1)
        else:
            cc_logger.info("Cannot find keyring at {0}".format(keyring))
            print("\nCannot find keyring at {0}".format(keyring))
            cc_logger.info("Exiting!")
            print("\nExiting!\n")
            sys.exit()

    def ceph_report(self):
        report = "/tmp/report-" + time.strftime("%d%m%Y-%H%M%S")
        cc_logger.info("Generating cluster report")
        try:
            with open(report, "w") as output:
                subprocess.call(["/usr/bin/ceph", "report"], stdout=output)
                cc_logger.info("Saved to %s" % report)
                cc_logger.info("Calling report_parse_summary()")
                self.report_parse_summary(report)
        except IOError:
            # It's very unlikely we'll hit this and exit here.
            print("Cannot write to /tmp, check permissions! Exiting!\n")
            cc_logger("Cannot create {0}".format(report))
            cc_logger.info("Exiting in ceph_report() - IOError")
            sys.exit(-1)

    def report_parse_summary(self, report):
        """Gets the MON/OSD node list for now
            Can add more parsers later
        """
        with open(report) as obj:
            json_obj = json.load(obj)
            cluster_status = json_obj['health']['overall_status']
            cc_logger.info("CLUSTER STATUS : {0}".format(cluster_status))
            print("REPORT")
            print("------\n")
            print("CLUSTER STATUS : {0}".format(cluster_status))
            # Get this printed in RED color :)
            if cluster_status != "HEALTH_OK":
                # Print a general cluster summary
                # We iterate over each object within the "summary" dict,
                # and print the 'value' for the 'summary' key
                cc_logger.info("Cluster **not** HEALTHY!!")
                print("Cluster summary:\n")
                for i in json_obj['health']['summary']:
                    print(i['summary'])
        self.cluster_status(report)

    def cluster_status(self, report):
        # Calling all helper functions irrespective of cluster status
        self.mon_status_check(report)
        self.osd_status_check(report)
        self.pool_info(report)
        self.pg_info(report)
        # Calling the generic system config checks
        self.ssh_check()

    def get_osd_and_mon(report):
        """Get the list of MONs and OSDs from the report"""
        pass
        # Note for self: Refer ceph_osd_meta.py from ceph report parse

    # 1. Cluster checks (MON, OSD, PG, Pool etc..)

    def mon_status_check(self, report):
        print("\n# MON status: \n")
        with open(report) as obj:
            json_obj = json.load(obj)
            print("Mon map epoch: %s\n" % str(json_obj['monmap']['epoch']))
            for mon in json_obj['monmap']['mons']:
                print("Monitor rank : %s" % str(mon['rank']))
                print("Host name    : %s" % str(mon['name']))
                print("IP Address   : %s\n" % str(mon['addr']))

    def osd_status_check(self, report):
        print("\n# OSD status: \n")

    def pool_info(self, report):
        print("\n# Pool status: \n")

    def pg_info(self, report):
        print("\n# Placement Group status: \n")

    # 2. System config checks (SSH access, hardware, RAID etc.. )
    def ssh_check(self):
        """Check the ssh access to the MON/OSD nodes, and print
            a) Hostname
            b) IP address
            c) ....
        """
        print("Checking SSH passwordless access to the Cluster nodes")
        print("Get a dict of hostname and IP address of MONs and OSD nodes")
        pass

if __name__ == "__main__":
    checker = CephCheck(CONF_FILE, ADMIN_KEYRING)
<commit_msg>* Fixed `ceph report` printing the report number to stdout  in subprocess module.<commit_after>#!/usr/bin/env python

# Copyright (C) 2016 Red Hat, Inc., Vimal A.R <vimal@redhat.com>

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

# __author__ = "Vimal A.R"
# __version__ = "v0.1"
# __license__ = "GNU GPL v2.0"
# __email__ = "vimal@redhat.com"

from __future__ import print_function
import time
import subprocess
import json
import sys
import os
import getpass
import ConfigParser
import logging

CONF_FILE = "/etc/ceph/ceph.conf"
ADMIN_KEYRING = "/etc/ceph/ceph.client.admin.keyring"
CEPH_CHECK_LOG = os.path.expanduser("~") + "/ceph_check.log"

# `logging` MODULE CONFIG ###
# 1. Set the application name (override the default `root` logger)
cc_logger = logging.getLogger("ceph_check")
cc_logger.setLevel(logging.DEBUG)
# 2. Set a Log handler
handler = logging.FileHandler(CEPH_CHECK_LOG)
# 3. Set up a log format
log_format = logging.Formatter(
    '%(asctime)s: %(name)s: %(levelname)s: %(message)s')
# 4. Pass the log format to the log handler
handler.setFormatter(log_format)
# 5. Add the log handler to the logger object `cc_logger`
cc_logger.addHandler(handler)
# LOGGING MODULE CONFIG END ###

class CephCheck(object):
    """The main ceph-check class"""

    def __init__(self, conffile, keyring):
        self.conffile = conffile
        self.keyring = keyring
        self.cc_condition()

    def cc_condition(self):
        """
        Conditions the user has to meet for
        a successfull execution of `ceph_check`
        """
        cc_logger.info("##################START###################")
        cc_logger.info("Printing the ideal conditions to the user!")
        print("\nNOTE:\n")
        print("`ceph_check` has the following three requirements:")
        print(" * Ansible has to present on this node.")
        print(" * Passwordless SSH access to the cluster nodes")
        print(" * Read access to the admin keyring on this node")
        print("\nIf these conditions are not met, this program may error out!")
        cc_logger.info("Requesting user's agreement on the conditions [Y/N]")
        agreement = raw_input("\nCan we proceed? (Y/N) : ")
        if agreement in ["Y", "y", "yes", "Yes"]:
            cc_logger.info("User answered {0}".format(agreement))
            cc_logger.info("Calling keyring_check()")
            self.keyring_check()
        else:
            cc_logger.info("User answered {0}".format(agreement))
            print("Please re-run the program after meeting the criteria!")
            cc_logger.info("Exiting!")
            sys.exit(-1)

    def keyring_check(self):
        """
        Check if a custom keyring exists
        """
        config_file = ConfigParser.SafeConfigParser()
        cc_logger.info("Reading '{0}'".format(CONF_FILE))
        config_file.read(self.conffile)
        cc_logger.info("Checking keyring")
        try:
            keyring_custom = config_file.get('global', 'keyring')
            cc_logger.info(
                "Found custom keyring at {0}".format(keyring_custom))
            self.keyring_permission(keyring_custom)
        except ConfigParser.NoOptionError:
            cc_logger.info(
                "No custom admin keyring specified in %s" % self.conffile)
            # cc_logger.info("Falling back to %s" % self.keyring)
            cc_logger.info("Falling back to {0}".format(self.keyring))
            cc_logger.info("Calling keyring_permission()")
            self.keyring_permission(self.keyring)

    def keyring_permission(self, keyring):
        """
        Check the existence and permission of the keyring
        """
        if os.path.isfile(self.keyring):
            cc_logger.info("{0} exists".format(self.keyring))
            if os.access(self.keyring, os.R_OK):
                cc_logger.info("{0} has read permissions".format(self.keyring))
                cc_logger.info("Calling ceph_report()")
                self.ceph_report()
            else:
                cc_logger.info("User {0} does not have read permissions for {1}".format(
                    getpass.getuser(), self.keyring))
                print("User {0} does not have read permissions for {1}".format(
                    getpass.getuser(), self.keyring))
                cc_logger.info("Exiting!")
                print("\nExiting!\n")
                sys.exit(-1)
        else:
            cc_logger.info("Cannot find keyring at {0}".format(keyring))
            print("\nCannot find keyring at {0}".format(keyring))
            cc_logger.info("Exiting!")
            print("\nExiting!\n")
            sys.exit()

    def ceph_report(self):
        report = "/tmp/report-" + time.strftime("%d%m%Y-%H%M%S")
        cc_logger.info("Generating cluster report")
        try:
            with open(report, "w") as output:
                subprocess.call(["/usr/bin/ceph", "report"],
                                stdout=output, stderr=subprocess.PIPE)
                cc_logger.info("Saved to %s" % report)
                cc_logger.info("Calling report_parse_summary()")
                self.report_parse_summary(report)
        except IOError:
            # It's very unlikely we'll hit this and exit here.
            print("Cannot write to /tmp, check permissions! Exiting!\n")
            cc_logger("Cannot create {0}".format(report))
            cc_logger.info("Exiting in ceph_report() - IOError")
            sys.exit(-1)

    def report_parse_summary(self, report):
        """Gets the MON/OSD node list for now
            Can add more parsers later
        """
        with open(report) as obj:
            json_obj = json.load(obj)
            cluster_status = json_obj['health']['overall_status']
            cc_logger.info("CLUSTER STATUS : {0}".format(cluster_status))
            print("\nceph_check REPORT:")
            print("-\n" * 20)
            print("CLUSTER STATUS : {0}".format(cluster_status))
            # Get this printed in RED color :)
            if cluster_status != "HEALTH_OK":
                # Print a general cluster summary
                # We iterate over each object within the "summary" dict,
                # and print the 'value' for the 'summary' key
                cc_logger.info("Cluster **not** HEALTHY!!")
                print("Cluster summary:\n")
                for i in json_obj['health']['summary']:
                    print(i['summary'])
        self.cluster_status(report)

    def cluster_status(self, report):
        # Calling all helper functions irrespective of cluster status
        self.mon_status_check(report)
        self.osd_status_check(report)
        self.pool_info(report)
        self.pg_info(report)
        # Calling the generic system config checks
        self.ssh_check()

    def get_osd_and_mon(report):
        """Get the list of MONs and OSDs from the report"""
        pass
        # Note for self: Refer ceph_osd_meta.py from ceph report parse

    # 1. Cluster checks (MON, OSD, PG, Pool etc..)

    def mon_status_check(self, report):
        print("\n# MON status: \n")
        with open(report) as obj:
            json_obj = json.load(obj)
            print("Mon map epoch: %s\n" % str(json_obj['monmap']['epoch']))
            for mon in json_obj['monmap']['mons']:
                print("Monitor rank : %s" % str(mon['rank']))
                print("Host name    : %s" % str(mon['name']))
                print("IP Address   : %s\n" % str(mon['addr']))

    def osd_status_check(self, report):
        print("\n# OSD status: \n")

    def pool_info(self, report):
        print("\n# Pool status: \n")

    def pg_info(self, report):
        print("\n# Placement Group status: \n")

    # 2. System config checks (SSH access, hardware, RAID etc.. )
    def ssh_check(self):
        """Check the ssh access to the MON/OSD nodes, and print
            a) Hostname
            b) IP address
            c) ....
        """
        print("Checking SSH passwordless access to the Cluster nodes")
        print("Get a dict of hostname and IP address of MONs and OSD nodes")
        pass

if __name__ == "__main__":
    checker = CephCheck(CONF_FILE, ADMIN_KEYRING)
<|endoftext|>


<commit_before># ***************************************************************************
# *   Copyright (c) 2017 Markus Hovorka <m.hovorka@live.de>                 *
# *   Copyright (c) 2018 Bernd Hahnebach <bernd@bimstatik.org>              *
# *                                                                         *
# *   This file is part of the FreeCAD CAx development system.              *
# *                                                                         *
# *   This program is free software; you can redistribute it and/or modify  *
# *   it under the terms of the GNU Lesser General Public License (LGPL)    *
# *   as published by the Free Software Foundation; either version 2 of     *
# *   the License, or (at your option) any later version.                   *
# *   for detail see the LICENCE text file.                                 *
# *                                                                         *
# *   This program is distributed in the hope that it will be useful,       *
# *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
# *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
# *   GNU Library General Public License for more details.                  *
# *                                                                         *
# *   You should have received a copy of the GNU Library General Public     *
# *   License along with this program; if not, write to the Free Software   *
# *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  *
# *   USA                                                                   *
# *                                                                         *
# ***************************************************************************
""" Collection of functions for the Fem module.

This module contains function for extracting relevant parts of geometry and
a few unrelated function useful at various places in the Fem module.
"""

__title__  = "FEM Utilities"
__author__ = "Markus Hovorka, Bernd Hahnebach"
__url__    = "https://www.freecadweb.org"

import os
import sys

import FreeCAD
if FreeCAD.GuiUp:
    import FreeCADGui
    from PySide import QtGui

# ************************************************************************************************
# document objects
def createObject(doc, name, proxy, viewProxy=None):
    """ Add python object to document using python type string.

    Add a document object suitable for the *proxy* and the *viewProxy* to *doc*
    and attach it to the *proxy* and the *viewProxy*. This function can only be
    used with python proxies that specify their C++ type via the BaseType class
    member (e.g. Cube.BaseType). If there already exists a object with *name* a
    suitable unique name is generated. To auto generate a name pass ``""``.

    :param doc:         document object to which the object is added
    :param name:        string of the name of new object in *doc*, use
                        ``""`` to generate a name
    :param proxy:       python proxy for new object
    :param viewProxy:   view proxy for new object

    :returns:           reference to new object
    """
    obj = doc.addObject(proxy.BaseType, name)
    proxy(obj)
    if FreeCAD.GuiUp and viewProxy is not None:
        viewProxy(obj.ViewObject)
    return obj

# typeID and object type defs
def type_of_obj(obj):
    """ Return type of *obj* honoring the special typesystem of Fem.

    Python objects of the Fem workbench define their type via a class member
    ``<Class>.Type``. Return this type if the property exists. If not return
    the conventional ``TypeId`` value.

    :para obj: a document object
    """
    if hasattr(obj, "Proxy") and hasattr(obj.Proxy, "Type"):
        return obj.Proxy.Type
    return obj.TypeId

def is_of_type(obj, ty):
    """ Compare type of *obj* with *ty* honoring Fems typesystem.

    See :py:func:`type_of_obj` for more info about the special typesystem of
    the Fem module.

    :returns:
     ``True`` if *obj* is of type *ty*, ``False`` otherwise. Type must match
     exactly: Derived objects are not considered to be of type of one of their
     super classes.
    """
    return type_of_obj(obj) == ty

def is_derived_from(obj, t):
    """ Check if *obj* is derived from *t* honoring Fems typesytem.

    Essentially just call ``obj.isDerivedFrom(t)`` and return it's value. For
    objects using Fems typesystem (see :py:func:`type_of_obj`) return always
    True if the Fem type is equal to *t*.

    :note:
     Inheritance of Fem types is not checked. If *obj* uses Fems typesystem the
     type is just checked for equality. If the type doesn't match
     ``obj.isDerivedFrom`` is called as usual. See
     https://forum.freecadweb.org/viewtopic.php?f=10&t=32625
    """
    if (hasattr(obj, "Proxy") and hasattr(obj.Proxy, "Type") and obj.Proxy.Type == t):
        return True
    return obj.isDerivedFrom(t)

# ************************************************************************************************
# working dir
def get_pref_working_dir(solver_obj):
    """ Return working directory for solver honoring user settings.

    :throws femtools.errors.MustSaveError:
     If user setting is set to BESIDE and the document isn't saved.

    :note:
     Not working correctly for most cases because this circumvents directory
     caching of the solver framework. For solver use getMachine from run.py
     instead.
    """
    from femsolver import settings
    dir_setting = settings.get_dir_setting()
    if dir_setting == settings.DirSetting.TEMPORARY:
        setting_working_dir = get_temp_dir(solver_obj)
    elif dir_setting == settings.DirSetting.BESIDE:
        setting_working_dir = get_beside_dir(solver_obj)
    elif dir_setting == settings.DirSetting.CUSTOM:
        setting_working_dir = get_custom_dir(solver_obj)
    else:
        setting_working_dir = ""
    return setting_working_dir

# these are a duplicate of the methods in src/Mod/Fem/femsolver/run.py
# see commit a9c19ca6d42c for more information
# the FEM preferences will be used by both
def get_temp_dir(obj=None):
    from tempfile import mkdtemp
    return mkdtemp(prefix="fcfem_")

def get_beside_dir(obj):
    base = get_beside_base(obj)
    specific_path = os.path.join(base, obj.Label)
    if not os.path.isdir(specific_path):
        make_dir(specific_path)
    return specific_path

def get_custom_dir(obj):
    base = get_custom_base(obj)
    specific_path = os.path.join(
        base, obj.Document.Name, obj.Label)
    if not os.path.isdir(specific_path):
        make_dir(specific_path)
    return specific_path

def get_beside_base(obj):
    fcstdPath = obj.Document.FileName
    if fcstdPath == "":
        new_path = get_temp_dir()
        error_message = (
            "Please save the file before executing a solver or creating a mesh. "
            "This must be done because the location of the working directory "
            "is set to \"Beside *.FCStd File\". For the moment the tmp dir {} is used."
            .format(new_path)
        )
        FreeCAD.Console.PrintError("{}\n".format(error_message))
        if FreeCAD.GuiUp:
            QtGui.QMessageBox.critical(
                FreeCADGui.getMainWindow(),
                "Can't start Solver or Mesh creation besides FC file.",
                error_message
            )

        # from .errors import MustSaveError
        # raise MustSaveError()
        return new_path
    else:
        return os.path.splitext(fcstdPath)[0]

def get_custom_base(solver):
    from femsolver.settings import get_custom_dir
    path = get_custom_dir()
    if not os.path.isdir(path):
        new_path = get_temp_dir()
        error_message = (
            "Selected working directory {} doesn't exist. "
            " For the moment the tmp dir {} is used."
            .format(path, new_path)
        )
        FreeCAD.Console.PrintError("{}\n".format(error_message))
        if FreeCAD.GuiUp:
            QtGui.QMessageBox.critical(
                FreeCADGui.getMainWindow(),
                "Can't start Solver or Mesh creation.",
                error_message
            )
        # from .errors import DirectoryDoesNotExistError
        # raise DirectoryDoesNotExistError("Invalid path")
        return new_path
    return path

def check_working_dir(wdir):
    # check if working_dir exist, if not use a tmp dir and inform the user
    # print(wdir)
    from os.path import isdir
    if isdir(wdir):
        return True
    else:
        return False

def make_dir(specific_path):
    try:
        os.makedirs(specific_path)
    except OSError:
        new_path = get_temp_dir()
        # it could fail for various reasons, full disk, etc
        # beside dir fails on installed FC examples from start wb
        error_message = (
            "Failed to create the directory {}. "
            " For the moment the tmp dir {} is used."
            .format(specific_path, new_path)
        )
        FreeCAD.Console.PrintError("{}\n".format(error_message))
        return new_path
    return specific_path

# ************************************************************************************************
# other
def get_part_to_mesh(mesh_obj):
    """
    gmsh mesh object: the Attribute is Part
    netgen mesh object: the Attribute is Shape
    other mesh objects: do not have a Attribute which holds the part to mesh
    """
    if is_derived_from(mesh_obj, "Fem::FemMeshGmsh"):
        return mesh_obj.Part
    elif is_derived_from(mesh_obj, "Fem::FemMeshShapeNetgenObject"):
        return mesh_obj.Shape
    else:
        return None
    # TODO: the Attributes should be named with the same name
    # should it be Shape or Part?
    # IMHO Part since the Attributes references the document object and not a Shape

def getBoundBoxOfAllDocumentShapes(doc):
    """ Calculate bounding box containing all objects inside *doc*.

    :returns:
     A bounding box containing all objects that have a *Shape* attribute (all
     Part and PartDesign objects). If the document contains no such objects or
     no objects at all return ``None``.
    """
    overalboundbox = None
    for o in doc.Objects:
        # netgen mesh obj has an attribute Shape which is an Document obj, which has no BB
        if hasattr(o, "Shape") and hasattr(o.Shape, "BoundBox"):
            try:
                bb = o.Shape.BoundBox
            except Exception:
                bb = None
            if bb.isValid():
                if not overalboundbox:
                    overalboundbox = bb
                overalboundbox.add(bb)
    return overalboundbox

def getSelectedFace(selectionex):
    """ Return selected face if exactly one face is selected.

    :returns:
     The selected face as a ``Part::TopoShape`` if exactly one face is selected.
     Otherwise return ``None``.

    :param selectionex:
     A list of selection object like the one Gui.Selection.getSelectionEx()
     returns.
    """
    aFace = None
    # print(selectionex)
    if len(selectionex) != 1:
        FreeCAD.Console.PrintMessage("No or more than one object selected.\n")
    else:
        sel = selectionex[0]
        if len(sel.SubObjects) != 1:
            FreeCAD.Console.PrintMessage("More than one element selected.\n")
        else:
            aFace = sel.SubObjects[0]
            if aFace.ShapeType != "Face":
                FreeCAD.Console.PrintMessage("Not a Face selected.\n")
            else:
                FreeCAD.Console.PrintMessage(":-)\n")
                return aFace
    return aFace

def get_refshape_type(fem_doc_object):
    """ Return shape type the constraints references.

    Determine single shape type of references of *fem_doc_object* which must be
    a constraint (=have a *References* property). All references must be of the
    same type which is than returned as a string. A type can be "Vertex",
    "Edge", "Face" or "Solid".

    :param fem_doc_object:
     A constraint object with a *References* property.

    :returns:
     A string representing the shape type ("Vertex", "Edge", "Face" or
     "Solid"). If *fem_doc_object* isn't a constraint ``""`` is returned.

    :note:
     Undefined behaviour if the type of the references of one object are
     not all the same.

    :note:
     Undefined behaviour if constraint contains no references (empty list).
    """
    from femtools.geomtools import get_element
    if hasattr(fem_doc_object, "References") and fem_doc_object.References:
        first_ref_obj = fem_doc_object.References[0]
        first_ref_shape = get_element(first_ref_obj[0], first_ref_obj[1][0])
        st = first_ref_shape.ShapeType
        FreeCAD.Console.PrintMessage(
            "References: {} in {}, {}\n". format(st, fem_doc_object.Name, fem_doc_object.Label)
        )
        return st
    else:
        FreeCAD.Console.PrintMessage(
            "References: empty in {}, {}\n". format(fem_doc_object.Name, fem_doc_object.Label)
        )
        return ""

def pydecode(bytestring):
    """ Return *bytestring* as a unicode string for python 2 and 3.

    For python 2 *bytestring* is converted to a string of type ``unicode``. For
    python 3 it is returned as is because it uses unicode for it's ``str`` type
    already.
    """
    if sys.version_info.major < 3:
        return bytestring
    else:
        return bytestring.decode("utf-8")
<commit_msg>FEM: utils, fix clipping FemMesh without Shape<commit_after># ***************************************************************************
# *   Copyright (c) 2017 Markus Hovorka <m.hovorka@live.de>                 *
# *   Copyright (c) 2018 Bernd Hahnebach <bernd@bimstatik.org>              *
# *                                                                         *
# *   This file is part of the FreeCAD CAx development system.              *
# *                                                                         *
# *   This program is free software; you can redistribute it and/or modify  *
# *   it under the terms of the GNU Lesser General Public License (LGPL)    *
# *   as published by the Free Software Foundation; either version 2 of     *
# *   the License, or (at your option) any later version.                   *
# *   for detail see the LICENCE text file.                                 *
# *                                                                         *
# *   This program is distributed in the hope that it will be useful,       *
# *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
# *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
# *   GNU Library General Public License for more details.                  *
# *                                                                         *
# *   You should have received a copy of the GNU Library General Public     *
# *   License along with this program; if not, write to the Free Software   *
# *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  *
# *   USA                                                                   *
# *                                                                         *
# ***************************************************************************
""" Collection of functions for the Fem module.

This module contains function for extracting relevant parts of geometry and
a few unrelated function useful at various places in the Fem module.
"""

__title__  = "FEM Utilities"
__author__ = "Markus Hovorka, Bernd Hahnebach"
__url__    = "https://www.freecadweb.org"

import os
import sys

import FreeCAD
if FreeCAD.GuiUp:
    import FreeCADGui
    from PySide import QtGui

# ************************************************************************************************
# document objects
def createObject(doc, name, proxy, viewProxy=None):
    """ Add python object to document using python type string.

    Add a document object suitable for the *proxy* and the *viewProxy* to *doc*
    and attach it to the *proxy* and the *viewProxy*. This function can only be
    used with python proxies that specify their C++ type via the BaseType class
    member (e.g. Cube.BaseType). If there already exists a object with *name* a
    suitable unique name is generated. To auto generate a name pass ``""``.

    :param doc:         document object to which the object is added
    :param name:        string of the name of new object in *doc*, use
                        ``""`` to generate a name
    :param proxy:       python proxy for new object
    :param viewProxy:   view proxy for new object

    :returns:           reference to new object
    """
    obj = doc.addObject(proxy.BaseType, name)
    proxy(obj)
    if FreeCAD.GuiUp and viewProxy is not None:
        viewProxy(obj.ViewObject)
    return obj

# typeID and object type defs
def type_of_obj(obj):
    """ Return type of *obj* honoring the special typesystem of Fem.

    Python objects of the Fem workbench define their type via a class member
    ``<Class>.Type``. Return this type if the property exists. If not return
    the conventional ``TypeId`` value.

    :para obj: a document object
    """
    if hasattr(obj, "Proxy") and hasattr(obj.Proxy, "Type"):
        return obj.Proxy.Type
    return obj.TypeId

def is_of_type(obj, ty):
    """ Compare type of *obj* with *ty* honoring Fems typesystem.

    See :py:func:`type_of_obj` for more info about the special typesystem of
    the Fem module.

    :returns:
     ``True`` if *obj* is of type *ty*, ``False`` otherwise. Type must match
     exactly: Derived objects are not considered to be of type of one of their
     super classes.
    """
    return type_of_obj(obj) == ty

def is_derived_from(obj, t):
    """ Check if *obj* is derived from *t* honoring Fems typesytem.

    Essentially just call ``obj.isDerivedFrom(t)`` and return it's value. For
    objects using Fems typesystem (see :py:func:`type_of_obj`) return always
    True if the Fem type is equal to *t*.

    :note:
     Inheritance of Fem types is not checked. If *obj* uses Fems typesystem the
     type is just checked for equality. If the type doesn't match
     ``obj.isDerivedFrom`` is called as usual. See
     https://forum.freecadweb.org/viewtopic.php?f=10&t=32625
    """
    if (hasattr(obj, "Proxy") and hasattr(obj.Proxy, "Type") and obj.Proxy.Type == t):
        return True
    return obj.isDerivedFrom(t)

# ************************************************************************************************
# working dir
def get_pref_working_dir(solver_obj):
    """ Return working directory for solver honoring user settings.

    :throws femtools.errors.MustSaveError:
     If user setting is set to BESIDE and the document isn't saved.

    :note:
     Not working correctly for most cases because this circumvents directory
     caching of the solver framework. For solver use getMachine from run.py
     instead.
    """
    from femsolver import settings
    dir_setting = settings.get_dir_setting()
    if dir_setting == settings.DirSetting.TEMPORARY:
        setting_working_dir = get_temp_dir(solver_obj)
    elif dir_setting == settings.DirSetting.BESIDE:
        setting_working_dir = get_beside_dir(solver_obj)
    elif dir_setting == settings.DirSetting.CUSTOM:
        setting_working_dir = get_custom_dir(solver_obj)
    else:
        setting_working_dir = ""
    return setting_working_dir

# these are a duplicate of the methods in src/Mod/Fem/femsolver/run.py
# see commit a9c19ca6d42c for more information
# the FEM preferences will be used by both
def get_temp_dir(obj=None):
    from tempfile import mkdtemp
    return mkdtemp(prefix="fcfem_")

def get_beside_dir(obj):
    base = get_beside_base(obj)
    specific_path = os.path.join(base, obj.Label)
    if not os.path.isdir(specific_path):
        make_dir(specific_path)
    return specific_path

def get_custom_dir(obj):
    base = get_custom_base(obj)
    specific_path = os.path.join(
        base, obj.Document.Name, obj.Label)
    if not os.path.isdir(specific_path):
        make_dir(specific_path)
    return specific_path

def get_beside_base(obj):
    fcstdPath = obj.Document.FileName
    if fcstdPath == "":
        new_path = get_temp_dir()
        error_message = (
            "Please save the file before executing a solver or creating a mesh. "
            "This must be done because the location of the working directory "
            "is set to \"Beside *.FCStd File\". For the moment the tmp dir {} is used."
            .format(new_path)
        )
        FreeCAD.Console.PrintError("{}\n".format(error_message))
        if FreeCAD.GuiUp:
            QtGui.QMessageBox.critical(
                FreeCADGui.getMainWindow(),
                "Can't start Solver or Mesh creation besides FC file.",
                error_message
            )

        # from .errors import MustSaveError
        # raise MustSaveError()
        return new_path
    else:
        return os.path.splitext(fcstdPath)[0]

def get_custom_base(solver):
    from femsolver.settings import get_custom_dir
    path = get_custom_dir()
    if not os.path.isdir(path):
        new_path = get_temp_dir()
        error_message = (
            "Selected working directory {} doesn't exist. "
            " For the moment the tmp dir {} is used."
            .format(path, new_path)
        )
        FreeCAD.Console.PrintError("{}\n".format(error_message))
        if FreeCAD.GuiUp:
            QtGui.QMessageBox.critical(
                FreeCADGui.getMainWindow(),
                "Can't start Solver or Mesh creation.",
                error_message
            )
        # from .errors import DirectoryDoesNotExistError
        # raise DirectoryDoesNotExistError("Invalid path")
        return new_path
    return path

def check_working_dir(wdir):
    # check if working_dir exist, if not use a tmp dir and inform the user
    # print(wdir)
    from os.path import isdir
    if isdir(wdir):
        return True
    else:
        return False

def make_dir(specific_path):
    try:
        os.makedirs(specific_path)
    except OSError:
        new_path = get_temp_dir()
        # it could fail for various reasons, full disk, etc
        # beside dir fails on installed FC examples from start wb
        error_message = (
            "Failed to create the directory {}. "
            " For the moment the tmp dir {} is used."
            .format(specific_path, new_path)
        )
        FreeCAD.Console.PrintError("{}\n".format(error_message))
        return new_path
    return specific_path

# ************************************************************************************************
# other
def get_part_to_mesh(mesh_obj):
    """
    gmsh mesh object: the Attribute is Part
    netgen mesh object: the Attribute is Shape
    other mesh objects: do not have a Attribute which holds the part to mesh
    """
    if is_derived_from(mesh_obj, "Fem::FemMeshGmsh"):
        return mesh_obj.Part
    elif is_derived_from(mesh_obj, "Fem::FemMeshShapeNetgenObject"):
        return mesh_obj.Shape
    else:
        return None
    # TODO: the Attributes should be named with the same name
    # should it be Shape or Part?
    # IMHO Part since the Attributes references the document object and not a Shape

def getBoundBoxOfAllDocumentShapes(doc):
    """ Calculate bounding box containing all objects inside *doc*.

    :returns:
     A bounding box containing all objects that have a *Shape* attribute (all
     Part and PartDesign objects). If the document contains no such objects or
     no objects at all return ``None``.
    """
    overalboundbox = None
    # netgen mesh obj has an attribute Shape which is an Document obj, which has no BB
    # a FemMesh without a Shape could be clipped too
    # https://forum.freecadweb.org/viewtopic.php?f=18&t=52920
    for o in doc.Objects:

        bb = None
        if hasattr(o, "Shape") and hasattr(o.Shape, "BoundBox"):
            try:
                bb = o.Shape.BoundBox
            except Exception:
                pass
        elif hasattr(o, "FemMesh") and hasattr(o.FemMesh, "BoundBox"):
            try:
                bb = o.FemMesh.BoundBox
            except Exception:
                pass
        if bb.isValid():
            if not overalboundbox:
                overalboundbox = bb
            else:
                overalboundbox.add(bb)
    return overalboundbox

def getSelectedFace(selectionex):
    """ Return selected face if exactly one face is selected.

    :returns:
     The selected face as a ``Part::TopoShape`` if exactly one face is selected.
     Otherwise return ``None``.

    :param selectionex:
     A list of selection object like the one Gui.Selection.getSelectionEx()
     returns.
    """
    aFace = None
    # print(selectionex)
    if len(selectionex) != 1:
        FreeCAD.Console.PrintMessage("No or more than one object selected.\n")
    else:
        sel = selectionex[0]
        if len(sel.SubObjects) != 1:
            FreeCAD.Console.PrintMessage("More than one element selected.\n")
        else:
            aFace = sel.SubObjects[0]
            if aFace.ShapeType != "Face":
                FreeCAD.Console.PrintMessage("Not a Face selected.\n")
            else:
                FreeCAD.Console.PrintMessage(":-)\n")
                return aFace
    return aFace

def get_refshape_type(fem_doc_object):
    """ Return shape type the constraints references.

    Determine single shape type of references of *fem_doc_object* which must be
    a constraint (=have a *References* property). All references must be of the
    same type which is than returned as a string. A type can be "Vertex",
    "Edge", "Face" or "Solid".

    :param fem_doc_object:
     A constraint object with a *References* property.

    :returns:
     A string representing the shape type ("Vertex", "Edge", "Face" or
     "Solid"). If *fem_doc_object* isn't a constraint ``""`` is returned.

    :note:
     Undefined behaviour if the type of the references of one object are
     not all the same.

    :note:
     Undefined behaviour if constraint contains no references (empty list).
    """
    from femtools.geomtools import get_element
    if hasattr(fem_doc_object, "References") and fem_doc_object.References:
        first_ref_obj = fem_doc_object.References[0]
        first_ref_shape = get_element(first_ref_obj[0], first_ref_obj[1][0])
        st = first_ref_shape.ShapeType
        FreeCAD.Console.PrintMessage(
            "References: {} in {}, {}\n". format(st, fem_doc_object.Name, fem_doc_object.Label)
        )
        return st
    else:
        FreeCAD.Console.PrintMessage(
            "References: empty in {}, {}\n". format(fem_doc_object.Name, fem_doc_object.Label)
        )
        return ""

def pydecode(bytestring):
    """ Return *bytestring* as a unicode string for python 2 and 3.

    For python 2 *bytestring* is converted to a string of type ``unicode``. For
    python 3 it is returned as is because it uses unicode for it's ``str`` type
    already.
    """
    if sys.version_info.major < 3:
        return bytestring
    else:
        return bytestring.decode("utf-8")
<|endoftext|>


<commit_before>from flask import Flask, render_template, request, redirect
import pandas as pd
import numpy as np
import sys
import os
app = Flask(__name__)
app.gui_cols = ['artist_name','track_name','tempo','uri']
app.training_set = pd.DataFrame()
app.suggest_set = pd.DataFrame()
app.seed_uris = []
app.new_playlist = True

PROJ_ROOT = os.path.join(os.getcwd(), os.pardir, os.pardir)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
pd.options.display.float_format = '{:,.3f}'.format

# load environment variables from .env file using dotenv.
from dotenv import load_dotenv
dotenv_path = os.path.join(PROJ_ROOT, '.env')
load_dotenv(dotenv_path)

# add the 'src' directory as one where we can import modules
src_dir = os.path.join(PROJ_ROOT, 'src')
sys.path.append(src_dir)

from data.spotipy_functions import *
from models.train_model import *

@app.route("/", methods=['GET'])
def home_page_get():
  return render_template("index.html")

@app.route("/display", methods=['GET','POST'])
def home_page_post():
  if request.method == 'POST':
    app.seed_uris = []
    for i in range(5):
      if request.form['uri{}'.format(i)] != '':
        app.seed_uris.append(request.form['uri{}'.format(i)][-22:])
    app.suggest_set = band_BPMs(get_new_recs_and_feats(app.seed_uris,30),80,170)
    if app.new_playlist:
      app.training_set = define_training_set(app.seed_uris)
      app.new_playlist = False
    else:
      if app.training_set.status.min() < 0:
        app.suggest_set = train_NB_model(app.suggest_set, app.training_set)
  display_suggest_set = app.suggest_set[app.gui_cols + ['P_accept']]
  display_suggest_set.loc[:,'Accept'] = list(map(lambda x: 'Accept'.format(x), np.array(display_suggest_set.index)))
  display_suggest_set.loc[:,'Reject'] = list(map(lambda x: 'Reject'.format(x), np.array(display_suggest_set.index)))
  display_suggest_set = display_suggest_set.sort_values(by=['P_accept'], ascending=False)
  display_training_set = app.training_set[app.training_set.status == 1][app.gui_cols]
  return render_template("output.html", seeds=app.seed_uris, training=display_training_set.to_html(escape=False), suggest=display_suggest_set.to_html(escape=False))

@app.route("/seed")
def seed_input():
  return render_template("seed_input.html")

@app.route("/accept/<suggest_id>")
def accept_track(suggest_id):
  suggest_id_int = int(suggest_id)
  app.suggest_set, app.training_set = process_track(app.suggest_set, app.training_set, suggest_id_int, 1)
  # only train the model when it has positive and negative examples
  if app.training_set.status.min() < 0:
    app.suggest_set = train_NB_model(app.suggest_set, app.training_set)
  return redirect('/display')

@app.route("/reject/<suggest_id>")
def reject_track(suggest_id):
  suggest_id_int = int(suggest_id)
  app.suggest_set, app.training_set = process_track(app.suggest_set, app.training_set, suggest_id_int, -1)
  app.suggest_set = train_NB_model(app.suggest_set, app.training_set)
  return redirect('/display')

if __name__ == "__main__":
  app.run(debug=True)<commit_msg>Update PROJ_ROOT<commit_after>from flask import Flask, render_template, request, redirect
import pandas as pd
import numpy as np
import sys
import os
app = Flask(__name__)
app.gui_cols = ['artist_name','track_name','tempo','uri']
app.training_set = pd.DataFrame()
app.suggest_set = pd.DataFrame()
app.seed_uris = []
app.new_playlist = True

#PROJ_ROOT = os.path.join(os.getcwd(), os.pardir, os.pardir)
PROJ_ROOT = os.getcwd()
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
pd.options.display.float_format = '{:,.3f}'.format

# load environment variables from .env file using dotenv.
from dotenv import load_dotenv
dotenv_path = os.path.join(PROJ_ROOT, '.env')
load_dotenv(dotenv_path)

# add the 'src' directory as one where we can import modules
src_dir = os.path.join(PROJ_ROOT, 'src')
sys.path.append(src_dir)

from data.spotipy_functions import *
from models.train_model import *

@app.route("/", methods=['GET'])
def home_page_get():
  return render_template("index.html")

@app.route("/display", methods=['GET','POST'])
def home_page_post():
  if request.method == 'POST':
    app.seed_uris = []
    for i in range(5):
      if request.form['uri{}'.format(i)] != '':
        app.seed_uris.append(request.form['uri{}'.format(i)][-22:])
    app.suggest_set = band_BPMs(get_new_recs_and_feats(app.seed_uris,30),80,170)
    if app.new_playlist:
      app.training_set = define_training_set(app.seed_uris)
      app.new_playlist = False
    else:
      if app.training_set.status.min() < 0:
        app.suggest_set = train_NB_model(app.suggest_set, app.training_set)
  display_suggest_set = app.suggest_set[app.gui_cols + ['P_accept']]
  display_suggest_set.loc[:,'Accept'] = list(map(lambda x: 'Accept'.format(x), np.array(display_suggest_set.index)))
  display_suggest_set.loc[:,'Reject'] = list(map(lambda x: 'Reject'.format(x), np.array(display_suggest_set.index)))
  display_suggest_set = display_suggest_set.sort_values(by=['P_accept'], ascending=False)
  display_training_set = app.training_set[app.training_set.status == 1][app.gui_cols]
  return render_template("output.html", seeds=app.seed_uris, training=display_training_set.to_html(escape=False), suggest=display_suggest_set.to_html(escape=False))

@app.route("/seed")
def seed_input():
  return render_template("seed_input.html")

@app.route("/accept/<suggest_id>")
def accept_track(suggest_id):
  suggest_id_int = int(suggest_id)
  app.suggest_set, app.training_set = process_track(app.suggest_set, app.training_set, suggest_id_int, 1)
  # only train the model when it has positive and negative examples
  if app.training_set.status.min() < 0:
    app.suggest_set = train_NB_model(app.suggest_set, app.training_set)
  return redirect('/display')

@app.route("/reject/<suggest_id>")
def reject_track(suggest_id):
  suggest_id_int = int(suggest_id)
  app.suggest_set, app.training_set = process_track(app.suggest_set, app.training_set, suggest_id_int, -1)
  app.suggest_set = train_NB_model(app.suggest_set, app.training_set)
  return redirect('/display')

if __name__ == "__main__":
  app.run(debug=True)<|endoftext|>