<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__version__ = "1.1"
'''
Python 2.7
Quick Program to log mqtt data (normally from mcThings modules, but could work with anything)

Nick Waterton 13th January 2017: V 1.0: Initial Release
Nick Waterton 15th January 2017: V 1.1: Added Nan Handling. No warning on +/-5s publising delays
'''

#from __future__ import print_function  #if you want python 3 print function

#import mosquitto
import paho.mqtt.client as paho
import json
import datetime
from collections import OrderedDict
import logging
import time
from logging.handlers import RotatingFileHandler

#----------- Start of Classes ------------

#----------- End of Classes ------------

#----------- Local Routines ------------

def decode_payload(payload):
    '''
    decode timestamp (if any), and format json for pretty printing
    '''
    indent = master_indent + 31 #number of spaces to indent json data
    try:
        #if it's json data, decode it (use OrderedDict to preserve keys order), else return as is...
        json_data = json.loads(payload.replace(":nan", ":NaN"), object_pairs_hook=OrderedDict)
        if not isinstance(json_data, dict): #if it's not a dictionary, probably just a number
            return json_data
        publish_delay_string = ''
        if "time" in json_data.keys():
            try:
                timestamp_string = str(json_data['time'])

                if "." in timestamp_string:
                    timestamp = datetime.datetime.fromtimestamp(float(timestamp_string))
                elif len(timestamp_string) <=10:
                    timestamp = datetime.datetime.fromtimestamp(int(timestamp_string))
                else:
                    timestamp = datetime.datetime.fromtimestamp(int(timestamp_string)/1000.0)
                current_time = datetime.datetime.now()
                publish_delay = (current_time - timestamp).total_seconds()

                if publish_delay > 5000:    #date/time set wrong on module
                    publish_delay_string = "Date/time not updated on module\n"
                elif publish_delay > 5 or publish_delay < -5:   #no warning on +/-5 seconds delay
                    publish_delay_string = "WARNING: Publishing Delay of : %.3f s\n" % publish_delay
                else:
                    publish_delay_string = "Publishing Delay of          : %.3f s\n" % publish_delay
            except ValueError as e:
                timestamp = e
        json_data_string = "\n".join((indent * " ") + i for i in (publish_delay_string+json.dumps(json_data, indent = 2)).splitlines())
        formatted_data = "Decoded timestamp: %s\n%s" % (timestamp, json_data_string)
    #except ValueError as e:
    except Exception as e:
        log.warn("Error Decoding json: %s" %e)
        formatted_data = payload

    return formatted_data

# Define event callbacks
def on_connect(mosq, obj, rc):
    global broker_connected
    broker_connected = True
    log.debug("connected to broker, rc: %s" % str(rc))
    log.info("subscribing to %s" % (sub_topic))
    mqttc.subscribe(sub_topic, 0)

def on_disconnect(mqttc, obj, rc):
    global broker_connected
    if rc != 0:
        broker_connected = False
        log.warn("Unexpected Disconnect From Broker - reconnecting")
    else:
        log.info("Disconnected From Broker")

    mqttc.reconnect()

def on_message(mosq, obj, msg):
    #log.debug("%s, %s, %s" % (msg.topic, str(msg.qos), str(msg.payload)))
    global master_indent
    if arg.indent == 0:
        master_indent = max(master_indent, len(msg.topic))
    if arg.raw:
        log_string = msg.payload
    else:
        log_string = decode_payload(msg.payload)
    log.info("%-{:d}s : %s".format(master_indent) % (msg.topic,log_string))

def on_publish(mosq, obj, mid):
    #log.debug("mid: " + str(mid))
    pass

def on_subscribe(mosq, obj, mid, granted_qos):
    log.debug("Subscribed: %s %s" % (str(mid), str(granted_qos)))

def on_log(mosq, obj, level, string):
    log.debug(string)

def setup_logger(logger_name, log_file, level=logging.DEBUG, console=False):
    l = logging.getLogger(logger_name)
    if logger_name == 'Main':
        formatter = logging.Formatter('[%(levelname)1.1s %(asctime)s] %(message)s')
    else:
        formatter = logging.Formatter('%(message)s')
    fileHandler = logging.handlers.RotatingFileHandler(log_file, mode='a', maxBytes=2000000, backupCount=5)
    fileHandler.setFormatter(formatter)
    if console == True:
      streamHandler = logging.StreamHandler()

    l.setLevel(level)
    l.addHandler(fileHandler)
    if console == True:
      streamHandler.setFormatter(formatter)
      l.addHandler(streamHandler)

if __name__ == '__main__':
    import argparse, sys
    #-------- Command Line -----------------
    parser = argparse.ArgumentParser(description='Log MQTT data for mcThings modules')
    parser.add_argument('-t','--topic', action='store',type=str, default="MCThings/#", help='MQTT Topic to subscribe to (can use wildcards # and +)')
    parser.add_argument('-b','--broker', action='store',type=str, default="192.168.100.119", help='ipaddress of MQTT broker (default: 192.168.100.119)')
    parser.add_argument('-p','--port', action='store',type=int, default=1883, help='MQTT broker port number (default: 1883)')
    parser.add_argument('-U','--user', action='store',type=str, default=None, help='MQTT broker user name (default: None)')
    parser.add_argument('-P','--password', action='store',type=str, default=None, help='MQTT broker password (default: None)')
    parser.add_argument('-i','--indent', action='store',type=int, default=0, help='Default indentation=auto')
    parser.add_argument('-l','--log', action='store',type=str, default="/home/nick/Scripts/mcThings.log", help='path/name of log file (default: /home/nick/Scripts/mcThings.log)')
    parser.add_argument('-e','--echo', action='store_true', help='Echo to Console (default: True)', default = True)
    parser.add_argument('-D','--debug', action='store_true', help='debug mode', default = False)
    parser.add_argument('-r','--raw', action='store_true', help='Output raw data, no decoding of json data', default = False)
    parser.add_argument('--version', action='version', version="%(prog)s ("+__version__+")")

    arg = parser.parse_args()

    #----------- Global Variables -----------
    broker_connected = False
    master_indent = arg.indent
    #-------------- Main --------------

    if arg.debug:
        log_level = logging.DEBUG
    else:
        log_level = logging.INFO

    #setup logging
    setup_logger('Main', arg.log,level=log_level,console=arg.echo)

    log = logging.getLogger('Main')

    #------------ Main ------------------

    log.info("*******************")
    log.info("* Program Started *")
    log.info("*******************")

    log.debug("-- DEBUG Mode ON -")

    log.info("Logging McThings modules MQTT data")
    if arg.raw:
        log.info("Showing RAW data")
    else:
        log.info("Showing DECODED data")

    if arg.indent == 0:
        log.info("Using auto indenting")

    #broker = "127.0.0.1"  # mosquitto broker is running on localhost
    broker = arg.broker
    port = arg.port
    sub_topic = arg.topic

    mqttc = paho.Client()
    # Assign event callbacks
    mqttc.on_message = on_message
    mqttc.on_connect = on_connect
    mqttc.on_disconnect = on_disconnect
    mqttc.on_publish = on_publish
    mqttc.on_subscribe = on_subscribe

    if arg.debug:
        # Uncomment To enable debug messages
        #mqttc.on_log = on_log
        pass

    #mqttc.reconnect_delay_set(10, 30, True)
    try:
        if arg.user != None:
            mqttc.username_pw_set(arg.user, arg.password)
        mqttc.connect(broker, port, 60) # Ping MQTT broker every 60 seconds if no data is published from this script.
    except socket.error:
        log.error("Unable to connect to MQTT Broker")

    try:
        print("<Cntl>C twice to Exit")

        mqttc.loop_forever()

    except (KeyboardInterrupt, SystemExit):
        log.info ("Exit Program")
        sys.exit()

    finally:
        mqttc.loop_stop()
        mqttc.disconnect()
        log.info ("Program Ended")
<commit_msg>Fixed Error handling<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__version__ = "1.1"
'''
Python 2.7
Quick Program to log mqtt data (normally from mcThings modules, but could work with anything)

Nick Waterton 13th January 2017: V 1.0: Initial Release
Nick Waterton 15th January 2017: V 1.1: Added Nan Handling. No warning on +/-5s publishing delays
'''

#from __future__ import print_function  #if you want python 3 print function

#import mosquitto
import paho.mqtt.client as paho
import json
import datetime
from collections import OrderedDict
import logging
import time
from logging.handlers import RotatingFileHandler

#----------- Start of Classes ------------

#----------- End of Classes ------------

#----------- Local Routines ------------

def decode_payload(payload):
    '''
    decode timestamp (if any), and format json for pretty printing
    '''
    indent = master_indent + 31 #number of spaces to indent json data
    try:
        #if it's json data, decode it (use OrderedDict to preserve keys order), else return as is...
        json_data = json.loads(payload.replace(":nan", ":NaN"), object_pairs_hook=OrderedDict)
        if not isinstance(json_data, dict): #if it's not a dictionary, probably just a number
            return json_data
        publish_delay_string = ''
        if "time" in json_data.keys():
            try:
                timestamp_string = str(json_data['time'])

                if "." in timestamp_string:
                    timestamp = datetime.datetime.fromtimestamp(float(timestamp_string))
                elif len(timestamp_string) <=10:
                    timestamp = datetime.datetime.fromtimestamp(int(timestamp_string))
                else:
                    timestamp = datetime.datetime.fromtimestamp(int(timestamp_string)/1000.0)
                current_time = datetime.datetime.now()
                publish_delay = (current_time - timestamp).total_seconds()

                if publish_delay > 5000:    #date/time set wrong on module
                    publish_delay_string = "Date/time not updated on module\n"
                elif publish_delay > 5 or publish_delay < -5:   #no warning on +/-5 seconds delay
                    publish_delay_string = "WARNING: Publishing Delay of : %.3f s\n" % publish_delay
                else:
                    publish_delay_string = "Publishing Delay of          : %.3f s\n" % publish_delay
            except ValueError as e:
                timestamp = e
        json_data_string = "\n".join((indent * " ") + i for i in (publish_delay_string+json.dumps(json_data, indent = 2)).splitlines())
        formatted_data = "Decoded timestamp: %s\n%s" % (timestamp, json_data_string)
    except ValueError as e:
        formatted_data = payload
    except Exception as e:
        log.warn("Error : %s" %e)
        formatted_data = payload

    return formatted_data

# Define event callbacks
def on_connect(mosq, obj, rc):
    global broker_connected
    broker_connected = True
    log.debug("connected to broker, rc: %s" % str(rc))
    log.info("subscribing to %s" % (sub_topic))
    mqttc.subscribe(sub_topic, 0)

def on_disconnect(mqttc, obj, rc):
    global broker_connected
    if rc != 0:
        broker_connected = False
        log.warn("Unexpected Disconnect From Broker - reconnecting")
    else:
        log.info("Disconnected From Broker")

    mqttc.reconnect()

def on_message(mosq, obj, msg):
    #log.debug("%s, %s, %s" % (msg.topic, str(msg.qos), str(msg.payload)))
    global master_indent
    if arg.indent == 0:
        master_indent = max(master_indent, len(msg.topic))
    if arg.raw:
        log_string = msg.payload
    else:
        log_string = decode_payload(msg.payload)
    log.info("%-{:d}s : %s".format(master_indent) % (msg.topic,log_string))

def on_publish(mosq, obj, mid):
    #log.debug("mid: " + str(mid))
    pass

def on_subscribe(mosq, obj, mid, granted_qos):
    log.debug("Subscribed: %s %s" % (str(mid), str(granted_qos)))

def on_log(mosq, obj, level, string):
    log.debug(string)

def setup_logger(logger_name, log_file, level=logging.DEBUG, console=False):
    l = logging.getLogger(logger_name)
    if logger_name == 'Main':
        formatter = logging.Formatter('[%(levelname)1.1s %(asctime)s] %(message)s')
    else:
        formatter = logging.Formatter('%(message)s')
    fileHandler = logging.handlers.RotatingFileHandler(log_file, mode='a', maxBytes=2000000, backupCount=5)
    fileHandler.setFormatter(formatter)
    if console == True:
      streamHandler = logging.StreamHandler()

    l.setLevel(level)
    l.addHandler(fileHandler)
    if console == True:
      streamHandler.setFormatter(formatter)
      l.addHandler(streamHandler)

if __name__ == '__main__':
    import argparse, sys
    #-------- Command Line -----------------
    parser = argparse.ArgumentParser(description='Log MQTT data for mcThings modules')
    parser.add_argument('-t','--topic', action='store',type=str, default="MCThings/#", help='MQTT Topic to subscribe to (can use wildcards # and +)')
    parser.add_argument('-b','--broker', action='store',type=str, default="192.168.100.119", help='ipaddress of MQTT broker (default: 192.168.100.119)')
    parser.add_argument('-p','--port', action='store',type=int, default=1883, help='MQTT broker port number (default: 1883)')
    parser.add_argument('-U','--user', action='store',type=str, default=None, help='MQTT broker user name (default: None)')
    parser.add_argument('-P','--password', action='store',type=str, default=None, help='MQTT broker password (default: None)')
    parser.add_argument('-i','--indent', action='store',type=int, default=0, help='Default indentation=auto')
    parser.add_argument('-l','--log', action='store',type=str, default="/home/nick/Scripts/mcThings.log", help='path/name of log file (default: /home/nick/Scripts/mcThings.log)')
    parser.add_argument('-e','--echo', action='store_true', help='Echo to Console (default: True)', default = True)
    parser.add_argument('-D','--debug', action='store_true', help='debug mode', default = False)
    parser.add_argument('-r','--raw', action='store_true', help='Output raw data, no decoding of json data', default = False)
    parser.add_argument('--version', action='version', version="%(prog)s ("+__version__+")")

    arg = parser.parse_args()

    #----------- Global Variables -----------
    broker_connected = False
    master_indent = arg.indent
    #-------------- Main --------------

    if arg.debug:
        log_level = logging.DEBUG
    else:
        log_level = logging.INFO

    #setup logging
    setup_logger('Main', arg.log,level=log_level,console=arg.echo)

    log = logging.getLogger('Main')

    #------------ Main ------------------

    log.info("*******************")
    log.info("* Program Started *")
    log.info("*******************")

    log.debug("-- DEBUG Mode ON -")

    log.info("Logging McThings modules MQTT data")
    if arg.raw:
        log.info("Showing RAW data")
    else:
        log.info("Showing DECODED data")

    if arg.indent == 0:
        log.info("Using auto indenting")

    #broker = "127.0.0.1"  # mosquitto broker is running on localhost
    broker = arg.broker
    port = arg.port
    sub_topic = arg.topic

    mqttc = paho.Client()
    # Assign event callbacks
    mqttc.on_message = on_message
    mqttc.on_connect = on_connect
    mqttc.on_disconnect = on_disconnect
    mqttc.on_publish = on_publish
    mqttc.on_subscribe = on_subscribe

    if arg.debug:
        # Uncomment To enable debug messages
        #mqttc.on_log = on_log
        pass

    #mqttc.reconnect_delay_set(10, 30, True)
    try:
        if arg.user != None:
            mqttc.username_pw_set(arg.user, arg.password)
        mqttc.connect(broker, port, 60) # Ping MQTT broker every 60 seconds if no data is published from this script.
    except socket.error:
        log.error("Unable to connect to MQTT Broker")

    try:
        print("<Cntl>C twice to Exit")

        mqttc.loop_forever()

    except (KeyboardInterrupt, SystemExit):
        log.info ("Exit Program")
        sys.exit()

    finally:
        mqttc.loop_stop()
        mqttc.disconnect()
        log.info ("Program Ended")
<|endoftext|>


<commit_before>#pylint: disable=missing-docstring
#* This file is part of the MOOSE framework
#* https://www.mooseframework.org
#*
#* All rights reserved, see COPYRIGHT for full restrictions
#* https://github.com/idaholab/moose/blob/master/COPYRIGHT
#*
#* Licensed under LGPL 2.1, please see LICENSE for details
#* https://www.gnu.org/licenses/lgpl-2.1.html

import os
import uuid
import mooseutils
from MooseDocs.base import components
from MooseDocs.tree import pages, tokens, html
from MooseDocs.extensions import core, command, heading

def make_extension(**kwargs):
    return ContentExtension(**kwargs)

Collapsible = tokens.newToken('Collapsible', summary=u'')
ContentToken = tokens.newToken('ContentToken', location=u'')
AtoZToken = tokens.newToken('AtoZToken', location=u'', level=None, buttons=bool)

class ContentExtension(command.CommandExtension):
    """
    Allows for the creation of markdown contents lists.
    """
    @staticmethod
    def defaultConfig():
        config = command.CommandExtension.defaultConfig()
        return config

    def extend(self, reader, renderer):
        self.requires(core, heading, command)
        self.addCommand(reader, ContentCommand())
        self.addCommand(reader, AtoZCommand())
        renderer.add('Collapsible', RenderCollapsible())
        renderer.add('AtoZToken', RenderAtoZ())

class ContentCommand(command.CommandComponent):
    COMMAND = 'contents' #TODO: Change this to content after format is working
    SUBCOMMAND = None

    @staticmethod
    def defaultSettings():
        settings = command.CommandComponent.defaultSettings()
        settings['location'] = (None, "The markdown content directory to build contents.")
        return settings

    def createToken(self, parent, info, page):

        tree = dict()
        tree[(u'',)] = core.UnorderedList(parent, browser_default=False)

        location = self.settings['location']

        func = lambda p: p.local.startswith(location) and isinstance(p, pages.Source)
        nodes = self.translator.findPages(func).sort(key=lambda n: n.local)

        for node in nodes:
            key = tuple(node.local.strip(os.sep).replace(location, '').split(os.sep))[:-1]
            for i in xrange(1, len(key) + 1):
                k = key[:i]
                if k not in tree:
                    col = Collapsible(tree[k[:-1]], summary=k[-1])
                    li = core.ListItem(col, class_='moose-source-item', tooltip=False)
                    tree[k] = core.UnorderedList(li, browser_default=False)

        for node in nodes:
            key = tuple(node.local.strip(os.sep).replace(location, '').split(os.sep))[:-1]
            loc = node.relativeDestination(page)
            li = core.ListItem(tree[key])
            core.Link(li, url=loc, string=node.name, class_='moose-source-item', tooltip=False)

        return parent

class AtoZCommand(command.CommandComponent):
    COMMAND = 'contents'
    SUBCOMMAND = 'a-to-z'

    @staticmethod
    def defaultSettings():
        settings = command.CommandComponent.defaultSettings()
        settings['level'] = (2, 'Heading level for A, B,... headings.')
        settings['buttons'] = (True, 'Display buttons linking to the A, B,... headings.')
        return settings

    def createToken(self, parent, info, page):
        return AtoZToken(parent, level=self.settings['level'], buttons=self.settings['buttons'])

class RenderCollapsible(components.RenderComponent):
    def createHTML(self, parent, token, page):

        details = html.Tag(parent, 'details')
        summary = html.Tag(details, 'summary')
        html.Tag(summary, 'span', class_='moose-section-icon')
        html.Tag(summary, 'span', string=token['summary'])
        return details

    def createLatex(self, parent, token, page):
        pass

class RenderAtoZ(components.RenderComponent):
    def createHTML(self, parent, token, page):
        pass

    def createMaterialize(self, parent, token, page):

        # Initialized alphabetized storage
        headings = dict()
        for letter in 'ABCDEFGHIJKLNMOPQRSTUVWXYZ':
            headings[letter] = dict()

        # Extract headings, default to filename if a heading is not found
        func = lambda n: n.local.startswith(token['location']) and isinstance(n, pages.Source)
        for node in self.translator.findPages(func):
            h_node = heading.find_heading(self.translator, node)
            if h_node is not None:
                r = html.Tag(None, 'span')
                self.renderer.render(r, h_node, page)
                key = r.text()
            else:
                r = None
                key = node.name

            letter = key[0].upper()
            headings[letter][key] = node.relativeDestination(page)

        # Buttons
        buttons = html.Tag(parent, 'div', class_='moose-a-to-z-buttons')
        if not token['buttons']:
            buttons.parent = None

        # Build lists
        for letter, items in headings.iteritems():
            id_ = uuid.uuid4()
            btn = html.Tag(buttons, 'a',
                           string=unicode(letter),
                           class_='btn moose-a-to-z-button',
                           href='#{}'.format(id_))

            if not items:
                btn.addClass('disabled')
                continue

            html.Tag(parent, 'h{}'.format(token['level']),
                     class_='moose-a-to-z',
                     id_=unicode(id_),
                     string=unicode(letter))

            row = html.Tag(parent, 'div', class_='row')

            links = [(text, href) for text, href in items.iteritems()]
            for chunk in mooseutils.make_chunks(links, 3):
                col = html.Tag(row, 'div', class_='col s12 m6 l4')
                ul = html.Tag(col, 'ul', class_='moose-a-to-z')
                for text, href in chunk:
                    li = html.Tag(ul, 'li')
                    html.Tag(li, 'a', href=href, string=unicode(text))

    def createLatex(self, parent, token, page):
        pass
<commit_msg>Improve source page rendering<commit_after>#pylint: disable=missing-docstring
#* This file is part of the MOOSE framework
#* https://www.mooseframework.org
#*
#* All rights reserved, see COPYRIGHT for full restrictions
#* https://github.com/idaholab/moose/blob/master/COPYRIGHT
#*
#* Licensed under LGPL 2.1, please see LICENSE for details
#* https://www.gnu.org/licenses/lgpl-2.1.html

import os
import uuid
import collections
import mooseutils
from MooseDocs.base import components
from MooseDocs.tree import pages, tokens, html
from MooseDocs.extensions import core, command, heading

def make_extension(**kwargs):
    return ContentExtension(**kwargs)

Collapsible = tokens.newToken('Collapsible', summary=u'')
ContentToken = tokens.newToken('ContentToken', location=u'', level=None)
AtoZToken = tokens.newToken('AtoZToken', location=u'', level=None, buttons=bool)

class ContentExtension(command.CommandExtension):
    """
    Allows for the creation of markdown contents lists.
    """
    @staticmethod
    def defaultConfig():
        config = command.CommandExtension.defaultConfig()
        return config

    def extend(self, reader, renderer):
        self.requires(core, heading, command)
        self.addCommand(reader, ContentCommand())
        self.addCommand(reader, AtoZCommand())
        renderer.add('Collapsible', RenderCollapsible())
        renderer.add('AtoZToken', RenderAtoZ())
        renderer.add('ContentToken', RenderContentToken())

class ContentCommand(command.CommandComponent):
    COMMAND = 'contents' #TODO: Change this to content after format is working
    SUBCOMMAND = None

    @staticmethod
    def defaultSettings():
        settings = command.CommandComponent.defaultSettings()
        settings['location'] = (None, "The markdown content directory to build contents.")
        settings['level'] = (2, 'Heading level for top-level headings.')
        return settings

    def createToken(self, parent, info, page):
        ContentToken(parent, location=self.settings['location'], level=self.settings['level'])
        return parent

class AtoZCommand(command.CommandComponent):
    COMMAND = 'contents'
    SUBCOMMAND = 'a-to-z'

    @staticmethod
    def defaultSettings():
        settings = command.CommandComponent.defaultSettings()
        settings['level'] = (2, 'Heading level for A, B,... headings.')
        settings['buttons'] = (True, 'Display buttons linking to the A, B,... headings.')
        return settings

    def createToken(self, parent, info, page):
        return AtoZToken(parent, level=self.settings['level'], buttons=self.settings['buttons'])

class RenderContentToken(components.RenderComponent):

    def createHTML(self, parent, token, page):
        pass

    def createMaterialize(self, parent, token, page):

        location = token['location']
        func = lambda p: p.local.startswith(location) and isinstance(p, pages.Source)
        nodes = self.translator.findPages(func)
        nodes.sort(key=lambda n: n.local)

        headings = collections.defaultdict(list)
        for node in nodes:
            key = tuple(node.local.replace(location, '').strip(os.sep).split(os.sep))
            head = key[0] if len(key) > 1 else u''
            headings[head].append((node.name, node.relativeDestination(page)))

        headings = [(h, items) for h, items in headings.iteritems()]
        headings.sort(key=lambda h: h[0])

        # Build lists
        for head, items in headings:

            if head:
                html.Tag(parent, 'h{}'.format(token['level']),
                         class_='moose-a-to-z',
                         string=unicode(head))

            row = html.Tag(parent, 'div', class_='row')

            for chunk in mooseutils.make_chunks(items, 3):
                col = html.Tag(row, 'div', class_='col s12 m6 l4')
                ul = html.Tag(col, 'ul', class_='moose-a-to-z')
                for text, href in chunk:
                    li = html.Tag(ul, 'li')
                    html.Tag(li, 'a', href=href, string=unicode(text.replace('.md', '')))

class RenderCollapsible(components.RenderComponent):
    def createHTML(self, parent, token, page):

        details = html.Tag(parent, 'details')
        summary = html.Tag(details, 'summary')
        html.Tag(summary, 'span', class_='moose-section-icon')
        html.Tag(summary, 'span', string=token['summary'])
        return details

    def createLatex(self, parent, token, page):
        pass

class RenderAtoZ(components.RenderComponent):
    def createHTML(self, parent, token, page):
        pass

    def createMaterialize(self, parent, token, page):

        # Initialized alphabetized storage
        headings = dict()
        for letter in 'ABCDEFGHIJKLNMOPQRSTUVWXYZ':
            headings[letter] = dict()

        # Extract headings, default to filename if a heading is not found
        func = lambda n: n.local.startswith(token['location']) and isinstance(n, pages.Source)
        for node in self.translator.findPages(func):
            h_node = heading.find_heading(self.translator, node)
            if h_node is not None:
                r = html.Tag(None, 'span')
                self.renderer.render(r, h_node, page)
                key = r.text()
            else:
                r = None
                key = node.name

            letter = key[0].upper()
            headings[letter][key] = node.relativeDestination(page)

        # Buttons
        buttons = html.Tag(parent, 'div', class_='moose-a-to-z-buttons')
        if not token['buttons']:
            buttons.parent = None

        # Build lists
        for letter, items in headings.iteritems():
            id_ = uuid.uuid4()
            btn = html.Tag(buttons, 'a',
                           string=unicode(letter),
                           class_='btn moose-a-to-z-button',
                           href='#{}'.format(id_))

            if not items:
                btn.addClass('disabled')
                continue

            html.Tag(parent, 'h{}'.format(token['level']),
                     class_='moose-a-to-z',
                     id_=unicode(id_),
                     string=unicode(letter))

            row = html.Tag(parent, 'div', class_='row')

            links = [(text, href) for text, href in items.iteritems()]
            for chunk in mooseutils.make_chunks(links, 3):
                col = html.Tag(row, 'div', class_='col s12 m6 l4')
                ul = html.Tag(col, 'ul', class_='moose-a-to-z')
                for text, href in chunk:
                    li = html.Tag(ul, 'li')
                    html.Tag(li, 'a', href=href, string=unicode(text))

    def createLatex(self, parent, token, page):
        pass
<|endoftext|>


<commit_before>def quicksort1(arr, start_idx=None, end_idx=None):
    """
    Sort array using Quicksort algorithm.
    Classical algorithm was introduced by C. A. R. Hoare.
    :param arr: Array to sort.
    :param start_idx: Start index. By default is zero.
    :param end_idx: End index. By default is the last index of array.
    :return: Sorted array.
    """
    if start_idx is None:
        start_idx = 0
    if end_idx is None:
        end_idx = len(arr) - 1

    if start_idx < end_idx:
        part_index = partition1(arr, start_idx, end_idx)
        quicksort1(arr, start_idx, part_index)
        quicksort1(arr, part_index + 1, end_idx)

def partition1(arr, start_idx, end_idx):
    pivot = arr[start_idx]
    i = start_idx - 1
    j = end_idx + 1
    while True:
        while True:
            i = i + 1
            if arr[i] >= pivot:
                break

        while True:
            j = j - 1
            if arr[j] <= pivot:
                break

        if i >= j:
            return j

        arr[i], arr[j] = arr[j], arr[i]

def quicksort2(arr, start_idx=None, end_idx=None):
    """
    Sort array using Quicksort algorithm.
    Algorithm was introduced by Nico Lomuto.
    :param arr: Array to sort.
    :param start_idx: Start index. By default is zero.
    :param end_idx: End index. By default is the last index of array.
    :return: Sorted array.
    """
    if start_idx is None:
        start_idx = 0
    if end_idx is None:
        end_idx = len(arr) - 1

    if start_idx < end_idx:
        part_index = partition2(arr, start_idx, end_idx)
        quicksort2(arr, start_idx, part_index - 1)
        quicksort2(arr, part_index + 1, end_idx)

def partition2(arr, start_idx, end_idx):
    pivot = arr[end_idx]
    i = start_idx - 1
    j = start_idx
    while j <= end_idx - 1:
        if arr[j] <= pivot:
            i += 1
            if i != j:
                arr[i], arr[j] = arr[j], arr[i]
        j += 1
    arr[i + 1], arr[end_idx] = arr[end_idx], arr[i + 1]
    return i + 1
<commit_msg>Adjusted Quicksort implementations.<commit_after>def quicksort1(arr, start_idx=None, end_idx=None):
    """
    Sort array using Quicksort algorithm.
    Classical algorithm was introduced by C. A. R. Hoare.
    :param arr: Array to sort.
    :param start_idx: Start index. By default is zero.
    :param end_idx: End index. By default is the last index of array.
    """
    if start_idx is None:
        start_idx = 0
    if end_idx is None:
        end_idx = len(arr) - 1

    if start_idx < end_idx:
        part_index = partition1(arr, start_idx, end_idx)
        quicksort1(arr, start_idx, part_index)
        quicksort1(arr, part_index + 1, end_idx)

def partition1(arr, start_idx, end_idx):
    pivot = arr[start_idx]
    i = start_idx - 1
    j = end_idx + 1
    while True:
        while True:
            i = i + 1
            if arr[i] >= pivot:
                break

        while True:
            j = j - 1
            if arr[j] <= pivot:
                break

        if i >= j:
            return j

        arr[i], arr[j] = arr[j], arr[i]

def quicksort2(arr, start_idx=None, end_idx=None):
    """
    Sort array using Quicksort algorithm.
    Algorithm was introduced by Nico Lomuto.
    :param arr: Array to sort.
    :param start_idx: Start index. By default is zero.
    :param end_idx: End index. By default is the last index of array.
    """
    if start_idx is None:
        start_idx = 0
    if end_idx is None:
        end_idx = len(arr) - 1

    if start_idx < end_idx:
        part_index = partition2(arr, start_idx, end_idx)
        quicksort2(arr, start_idx, part_index - 1)
        quicksort2(arr, part_index + 1, end_idx)

def partition2(arr, start_idx, end_idx):
    pivot = arr[end_idx]
    i = start_idx - 1
    j = start_idx
    while j <= end_idx - 1:
        if arr[j] <= pivot:
            i += 1
            if i != j:
                arr[i], arr[j] = arr[j], arr[i]
        j += 1
    arr[i + 1], arr[end_idx] = arr[end_idx], arr[i + 1]
    return i + 1
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
#
# test_maps.py
#
# Created by José Sánchez-Gallego on 22 Jun 2016.

from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

import copy

import pytest
import numpy as np
import astropy
import astropy.io.fits

import marvin
from marvin.tools.maps import Maps
import marvin.tools.spaxel
from marvin.core.exceptions import MarvinError
from marvin.utils.datamodel.dap.base import Property
from marvin.tests import marvin_test_if

def _assert_maps(maps, galaxy):
    """Basic checks for a Maps object."""

    assert maps is not None
    assert maps.plateifu == galaxy.plateifu
    assert maps.mangaid == galaxy.mangaid
    assert maps.wcs is not None
    assert maps.bintype == galaxy.bintype

    assert len(maps._shape) == len(galaxy.shape)
    for ii in range(len(maps._shape)):
        assert maps._shape[ii] == galaxy.shape[ii]

class TestMaps(object):

    def _get_maps_kwargs(self, galaxy, data_origin):

        if data_origin == 'file':
            maps_kwargs = dict(filename=galaxy.mapspath)
        else:
            maps_kwargs = dict(plateifu=galaxy.plateifu, release=galaxy.release,
                               bintype=galaxy.bintype, template=galaxy.template,
                               mode='local' if data_origin == 'db' else 'remote')

        return maps_kwargs

    def test_load(self, galaxy, exporigin):

        maps = Maps(**self._get_maps_kwargs(galaxy, exporigin))

        _assert_maps(maps, galaxy)

        assert maps.data_origin == exporigin

        if exporigin == 'file':
            assert isinstance(maps.data, astropy.io.fits.HDUList)
        elif exporigin == 'db':
            assert isinstance(maps.data, marvin.marvindb.dapdb.File)

        cube = maps.getCube()

        assert cube is not None
        assert cube.plateifu == galaxy.plateifu
        assert cube.mangaid == galaxy.mangaid

    @pytest.mark.parametrize('monkeyconfig', [('release', 'MPL-5')], indirect=True)
    def test_load_mpl4_global_mpl5(self, galaxy, monkeyconfig, data_origin):

        assert marvin.config.release == 'MPL-5'
        maps = Maps(**self._get_maps_kwargs(galaxy, data_origin))

        assert maps.release == galaxy.release
        assert maps._drpver == galaxy.drpver
        assert maps._dapver == galaxy.dapver

    def test_maps_redshift(self, maps, galaxy):
        redshift = maps.nsa.redshift \
            if maps.release == 'MPL-4' and maps.data_origin == 'file' else maps.nsa.z
        assert pytest.approx(redshift, galaxy.redshift)

    def test_release(self, galaxy):
        maps = Maps(plateifu=galaxy.plateifu)
        assert maps.release == galaxy.release

    def test_set_release_fails(self, galaxy):
        maps = Maps(plateifu=galaxy.plateifu)
        with pytest.raises(MarvinError) as ee:
            maps.release = 'a'
            assert 'the release cannot be changed' in str(ee.exception)

    def test_deepcopy(self, galaxy):
        maps1 = Maps(plateifu=galaxy.plateifu)
        maps2 = copy.deepcopy(maps1)

        for attr in vars(maps1):
            if not attr.startswith('_'):
                value = getattr(maps1, attr)
                value2 = getattr(maps2, attr)

                if isinstance(value, np.ndarray):
                    assert np.isclose(value, value2).all()

                elif isinstance(value, astropy.wcs.wcs.WCS):
                    for key in vars(value):
                        assert getattr(value, key) == getattr(value2, key)

                elif isinstance(value, marvin.tools.cube.Cube):
                    pass

                elif (isinstance(value, list) and len(value) > 0 and
                      isinstance(value[0], Property)):
                        for property1, property2 in zip(value, value2):
                            assert property1 == property2

                else:
                    assert value == value2, attr

class TestMaskbit(object):

    @marvin_test_if(mark='include', maps_release_only=dict(release=['MPL-4']))
    def test_quality_flag_mpl4(self, maps_release_only):
        ha = maps_release_only['emline_gflux_ha_6564']
        assert ha.quality_flag is None

    @marvin_test_if(mark='skip', maps_release_only=dict(release=['MPL-4']))
    def test_quality_flag(self, maps_release_only):
        ha = maps_release_only['emline_gflux_ha_6564']
        assert ha.quality_flag.mask == 0

    @pytest.mark.parametrize('flag',
                             ['manga_target1',
                              'manga_target2',
                              'manga_target3',
                              'target_flags'])
    def test_flag(self, flag, maps_release_only):
        ha = maps_release_only['emline_gflux_ha_6564']
        assert getattr(ha, flag, None) is not None
<commit_msg>Fixes #352. Tests that Marvin maps loaded from db and file match data read directly from file with astropy<commit_after>#!/usr/bin/env python
# encoding: utf-8
#
# test_maps.py
#
# Created by José Sánchez-Gallego on 22 Jun 2016.

from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

import copy
from os.path import join
import re

import pytest
import numpy as np
import astropy
import astropy.io.fits

import marvin
from marvin.tools.maps import Maps
import marvin.tools.spaxel
from marvin.core.exceptions import MarvinError
from marvin.utils.datamodel.dap.base import Property
from marvin.tests import marvin_test_if

def _assert_maps(maps, galaxy):
    """Basic checks for a Maps object."""

    assert maps is not None
    assert maps.plateifu == galaxy.plateifu
    assert maps.mangaid == galaxy.mangaid
    assert maps.wcs is not None
    assert maps.bintype == galaxy.bintype

    assert len(maps._shape) == len(galaxy.shape)
    for ii in range(len(maps._shape)):
        assert maps._shape[ii] == galaxy.shape[ii]

class TestMaps(object):

    def _get_maps_kwargs(self, galaxy, data_origin):

        if data_origin == 'file':
            maps_kwargs = dict(filename=galaxy.mapspath)
        else:
            maps_kwargs = dict(plateifu=galaxy.plateifu, release=galaxy.release,
                               bintype=galaxy.bintype, template=galaxy.template,
                               mode='local' if data_origin == 'db' else 'remote')

        return maps_kwargs

    def test_load(self, galaxy, exporigin):

        maps = Maps(**self._get_maps_kwargs(galaxy, exporigin))

        _assert_maps(maps, galaxy)

        assert maps.data_origin == exporigin

        if exporigin == 'file':
            assert isinstance(maps.data, astropy.io.fits.HDUList)
        elif exporigin == 'db':
            assert isinstance(maps.data, marvin.marvindb.dapdb.File)

        cube = maps.getCube()

        assert cube is not None
        assert cube.plateifu == galaxy.plateifu
        assert cube.mangaid == galaxy.mangaid

    @pytest.mark.slow
    def test_datamodel(self, galaxy, exporigin):

        maps = Maps(**self._get_maps_kwargs(galaxy, exporigin))

        fin = 'manga-{0}-{1}-MAPS-{2}.fits.gz'.format(galaxy.plate, galaxy.ifu, galaxy.bintemp)
        path = join(galaxy.mangaanalysis, galaxy.drpver, galaxy.dapver, galaxy.bintemp,
                    str(galaxy.plate), galaxy.ifu, fin)
        hdus = astropy.io.fits.open(path)

        for hdu in hdus[1:]:

            if ('IVAR' in hdu.name) or ('MASK' in hdu.name) or ('SIGMACORR' in hdu.name):
                continue

            name = hdu.name.lower()
            data = hdu.data
            header = hdu.header

            if len(data.shape) < 3:
                val = maps.getMap(name, exact=True).value
                assert val == pytest.approx(data, 0.0001), name

            else:
                for kk, vv in header.items():
                    channel_num = re.match('^[0-9]+$', kk[1:])

                    if (kk[0] == 'C') and (channel_num is not None):
                        channel = '_'.join(re.findall(r"[\w']+", vv)).lower()
                        fullname = '_'.join((name, channel))
                        val = maps.getMap(fullname, exact=True).value
                        assert val == pytest.approx(data[int(kk[1:]) - 1], 0.0001), name

    @pytest.mark.parametrize('monkeyconfig', [('release', 'MPL-5')], indirect=True)
    def test_load_mpl4_global_mpl5(self, galaxy, monkeyconfig, data_origin):

        assert marvin.config.release == 'MPL-5'
        maps = Maps(**self._get_maps_kwargs(galaxy, data_origin))

        assert maps.release == galaxy.release
        assert maps._drpver == galaxy.drpver
        assert maps._dapver == galaxy.dapver

    def test_maps_redshift(self, maps, galaxy):
        redshift = maps.nsa.redshift \
            if maps.release == 'MPL-4' and maps.data_origin == 'file' else maps.nsa.z
        assert pytest.approx(redshift, galaxy.redshift)

    def test_release(self, galaxy):
        maps = Maps(plateifu=galaxy.plateifu)
        assert maps.release == galaxy.release

    def test_set_release_fails(self, galaxy):
        maps = Maps(plateifu=galaxy.plateifu)
        with pytest.raises(MarvinError) as ee:
            maps.release = 'a'
            assert 'the release cannot be changed' in str(ee.exception)

    def test_deepcopy(self, galaxy):
        maps1 = Maps(plateifu=galaxy.plateifu)
        maps2 = copy.deepcopy(maps1)

        for attr in vars(maps1):
            if not attr.startswith('_'):
                value = getattr(maps1, attr)
                value2 = getattr(maps2, attr)

                if isinstance(value, np.ndarray):
                    assert np.isclose(value, value2).all()

                elif isinstance(value, astropy.wcs.wcs.WCS):
                    for key in vars(value):
                        assert getattr(value, key) == getattr(value2, key)

                elif isinstance(value, marvin.tools.cube.Cube):
                    pass

                elif (isinstance(value, list) and len(value) > 0 and
                      isinstance(value[0], Property)):
                        for property1, property2 in zip(value, value2):
                            assert property1 == property2

                else:
                    assert value == value2, attr

class TestMaskbit(object):

    @marvin_test_if(mark='include', maps_release_only=dict(release=['MPL-4']))
    def test_quality_flag_mpl4(self, maps_release_only):
        ha = maps_release_only['emline_gflux_ha_6564']
        assert ha.quality_flag is None

    @marvin_test_if(mark='skip', maps_release_only=dict(release=['MPL-4']))
    def test_quality_flag(self, maps_release_only):
        ha = maps_release_only['emline_gflux_ha_6564']
        assert ha.quality_flag.mask == 0

    @pytest.mark.parametrize('flag',
                             ['manga_target1',
                              'manga_target2',
                              'manga_target3',
                              'target_flags'])
    def test_flag(self, flag, maps_release_only):
        ha = maps_release_only['emline_gflux_ha_6564']
        assert getattr(ha, flag, None) is not None
<|endoftext|>


<commit_before>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from math import exp

import numpy
from numpy import array

from pyspark import RDD
from pyspark.mllib.common import callMLlibFunc
from pyspark.mllib.linalg import SparseVector, _convert_to_vector
from pyspark.mllib.regression import LabeledPoint, LinearModel, _regression_train_wrapper

__all__ = ['LogisticRegressionModel', 'LogisticRegressionWithSGD', 'LogisticRegressionWithLBFGS',
           'SVMModel', 'SVMWithSGD', 'NaiveBayesModel', 'NaiveBayes']

class LinearBinaryClassificationModel(LinearModel):
    """
    Represents a linear binary classification model that predicts to whether an
    example is positive (1.0) or negative (0.0).
    """
    def __init__(self, weights, intercept):
        super(LinearBinaryClassificationModel, self).__init__(weights, intercept)
        self._threshold = None

    def setThreshold(self, value):
        """
        .. note:: Experimental

        Sets the threshold that separates positive predictions from negative
        predictions. An example with prediction score greater than or equal
        to this threshold is identified as an positive, and negative otherwise.
        """
        self._threshold = value

    def clearThreshold(self):
        """
        .. note:: Experimental

        Clears the threshold so that `predict` will output raw prediction scores.
        """
        self._threshold = None

    def predict(self, test):
        """
        Predict values for a single data point or an RDD of points using
        the model trained.
        """
        raise NotImplementedError

class LogisticRegressionModel(LinearBinaryClassificationModel):

    """A linear binary classification model derived from logistic regression.

    >>> data = [
    ...     LabeledPoint(0.0, [0.0, 1.0]),
    ...     LabeledPoint(1.0, [1.0, 0.0]),
    ... ]
    >>> lrm = LogisticRegressionWithSGD.train(sc.parallelize(data))
    >>> lrm.predict([1.0, 0.0])
    1
    >>> lrm.predict([0.0, 1.0])
    0
    >>> lrm.predict(sc.parallelize([[1.0, 0.0], [0.0, 1.0]])).collect()
    [1, 0]
    >>> lrm.clearThreshold()
    >>> lrm.predict([0.0, 1.0])
    0.123...

    >>> sparse_data = [
    ...     LabeledPoint(0.0, SparseVector(2, {0: 0.0})),
    ...     LabeledPoint(1.0, SparseVector(2, {1: 1.0})),
    ...     LabeledPoint(0.0, SparseVector(2, {0: 1.0})),
    ...     LabeledPoint(1.0, SparseVector(2, {1: 2.0}))
    ... ]
    >>> lrm = LogisticRegressionWithSGD.train(sc.parallelize(sparse_data))
    >>> lrm.predict(array([0.0, 1.0]))
    1
    >>> lrm.predict(array([1.0, 0.0]))
    0
    >>> lrm.predict(SparseVector(2, {1: 1.0}))
    1
    >>> lrm.predict(SparseVector(2, {0: 1.0}))
    0
    """
    def __init__(self, weights, intercept):
        super(LogisticRegressionModel, self).__init__(weights, intercept)
        self._threshold = 0.5

    def predict(self, x):
        """
        Predict values for a single data point or an RDD of points using
        the model trained.
        """
        if isinstance(x, RDD):
            return x.map(lambda v: self.predict(v))

        x = _convert_to_vector(x)
        margin = self.weights.dot(x) + self._intercept
        if margin > 0:
            prob = 1 / (1 + exp(-margin))
        else:
            exp_margin = exp(margin)
            prob = exp_margin / (1 + exp_margin)
        if self._threshold is None:
            return prob
        else:
            return 1 if prob > self._threshold else 0

class LogisticRegressionWithSGD(object):

    @classmethod
    def train(cls, data, iterations=100, step=1.0, miniBatchFraction=1.0,
              initialWeights=None, regParam=0.01, regType="l2", intercept=False):
        """
        Train a logistic regression model on the given data.

        :param data:              The training data, an RDD of LabeledPoint.
        :param iterations:        The number of iterations (default: 100).
        :param step:              The step parameter used in SGD
                                  (default: 1.0).
        :param miniBatchFraction: Fraction of data to be used for each SGD
                                  iteration.
        :param initialWeights:    The initial weights (default: None).
        :param regParam:          The regularizer parameter (default: 0.01).
        :param regType:           The type of regularizer used for training
                                  our model.

                                  :Allowed values:
                                     - "l1" for using L1 regularization
                                     - "l2" for using L2 regularization
                                     - None for no regularization

                                     (default: "l2")

        :param intercept:         Boolean parameter which indicates the use
                                  or not of the augmented representation for
                                  training data (i.e. whether bias features
                                  are activated or not).
        """
        def train(rdd, i):
            return callMLlibFunc("trainLogisticRegressionModelWithSGD", rdd, int(iterations),
                                 float(step), float(miniBatchFraction), i, float(regParam), regType,
                                 bool(intercept))

        return _regression_train_wrapper(train, LogisticRegressionModel, data, initialWeights)

class LogisticRegressionWithLBFGS(object):

    @classmethod
    def train(cls, data, iterations=100, initialWeights=None, regParam=0.01, regType="l2",
              intercept=False, corrections=10, tolerance=1e-4):
        """
        Train a logistic regression model on the given data.

        :param data:           The training data, an RDD of LabeledPoint.
        :param iterations:     The number of iterations (default: 100).
        :param initialWeights: The initial weights (default: None).
        :param regParam:       The regularizer parameter (default: 0.01).
        :param regType:        The type of regularizer used for training
                               our model.

                               :Allowed values:
                                 - "l1" for using L1 regularization
                                 - "l2" for using L2 regularization
                                 - None for no regularization

                                 (default: "l2")

        :param intercept:      Boolean parameter which indicates the use
                               or not of the augmented representation for
                               training data (i.e. whether bias features
                               are activated or not).
        :param corrections:    The number of corrections used in the LBFGS
                               update (default: 10).
        :param tolerance:      The convergence tolerance of iterations for
                               L-BFGS (default: 1e-4).

        >>> data = [
        ...     LabeledPoint(0.0, [0.0, 1.0]),
        ...     LabeledPoint(1.0, [1.0, 0.0]),
        ... ]
        >>> lrm = LogisticRegressionWithLBFGS.train(sc.parallelize(data))
        >>> lrm.predict([1.0, 0.0])
        1
        >>> lrm.predict([0.0, 1.0])
        0
        """
        def train(rdd, i):
            return callMLlibFunc("trainLogisticRegressionModelWithLBFGS", rdd, int(iterations), i,
                                 float(regParam), str(regType), bool(intercept), int(corrections),
                                 float(tolerance))

        return _regression_train_wrapper(train, LogisticRegressionModel, data, initialWeights)

class SVMModel(LinearBinaryClassificationModel):

    """A support vector machine.

    >>> data = [
    ...     LabeledPoint(0.0, [0.0]),
    ...     LabeledPoint(1.0, [1.0]),
    ...     LabeledPoint(1.0, [2.0]),
    ...     LabeledPoint(1.0, [3.0])
    ... ]
    >>> svm = SVMWithSGD.train(sc.parallelize(data))
    >>> svm.predict([1.0])
    1
    >>> svm.predict(sc.parallelize([[1.0]])).collect()
    [1]
    >>> svm.clearThreshold()
    >>> svm.predict(array([1.0]))
    1.25...

    >>> sparse_data = [
    ...     LabeledPoint(0.0, SparseVector(2, {0: -1.0})),
    ...     LabeledPoint(1.0, SparseVector(2, {1: 1.0})),
    ...     LabeledPoint(0.0, SparseVector(2, {0: 0.0})),
    ...     LabeledPoint(1.0, SparseVector(2, {1: 2.0}))
    ... ]
    >>> svm = SVMWithSGD.train(sc.parallelize(sparse_data))
    >>> svm.predict(SparseVector(2, {1: 1.0}))
    1
    >>> svm.predict(SparseVector(2, {0: -1.0}))
    0
    """
    def __init__(self, weights, intercept):
        super(SVMModel, self).__init__(weights, intercept)
        self._threshold = 0.0

    def predict(self, x):
        """
        Predict values for a single data point or an RDD of points using
        the model trained.
        """
        if isinstance(x, RDD):
            return x.map(lambda v: self.predict(v))

        x = _convert_to_vector(x)
        margin = self.weights.dot(x) + self.intercept
        if self._threshold is None:
            return margin
        else:
            return 1 if margin > self._threshold else 0

class SVMWithSGD(object):

    @classmethod
    def train(cls, data, iterations=100, step=1.0, regParam=0.01,
              miniBatchFraction=1.0, initialWeights=None, regType="l2", intercept=False):
        """
        Train a support vector machine on the given data.

        :param data:              The training data, an RDD of LabeledPoint.
        :param iterations:        The number of iterations (default: 100).
        :param step:              The step parameter used in SGD
                                  (default: 1.0).
        :param regParam:          The regularizer parameter (default: 0.01).
        :param miniBatchFraction: Fraction of data to be used for each SGD
                                  iteration.
        :param initialWeights:    The initial weights (default: None).
        :param regType:           The type of regularizer used for training
                                  our model.

                                  :Allowed values:
                                     - "l1" for using L1 regularization
                                     - "l2" for using L2 regularization
                                     - None for no regularization

                                     (default: "l2")

        :param intercept:         Boolean parameter which indicates the use
                                  or not of the augmented representation for
                                  training data (i.e. whether bias features
                                  are activated or not).
        """
        def train(rdd, i):
            return callMLlibFunc("trainSVMModelWithSGD", rdd, int(iterations), float(step),
                                 float(regParam), float(miniBatchFraction), i, regType,
                                 bool(intercept))

        return _regression_train_wrapper(train, SVMModel, data, initialWeights)

class NaiveBayesModel(object):

    """
    Model for Naive Bayes classifiers.

    Contains two parameters:
    - pi: vector of logs of class priors (dimension C)
    - theta: matrix of logs of class conditional probabilities (CxD)

    >>> data = [
    ...     LabeledPoint(0.0, [0.0, 0.0]),
    ...     LabeledPoint(0.0, [0.0, 1.0]),
    ...     LabeledPoint(1.0, [1.0, 0.0]),
    ... ]
    >>> model = NaiveBayes.train(sc.parallelize(data))
    >>> model.predict(array([0.0, 1.0]))
    0.0
    >>> model.predict(array([1.0, 0.0]))
    1.0
    >>> model.predict(sc.parallelize([[1.0, 0.0]])).collect()
    [1.0]
    >>> sparse_data = [
    ...     LabeledPoint(0.0, SparseVector(2, {1: 0.0})),
    ...     LabeledPoint(0.0, SparseVector(2, {1: 1.0})),
    ...     LabeledPoint(1.0, SparseVector(2, {0: 1.0}))
    ... ]
    >>> model = NaiveBayes.train(sc.parallelize(sparse_data))
    >>> model.predict(SparseVector(2, {1: 1.0}))
    0.0
    >>> model.predict(SparseVector(2, {0: 1.0}))
    1.0
    """

    def __init__(self, labels, pi, theta):
        self.labels = labels
        self.pi = pi
        self.theta = theta

    def predict(self, x):
        """Return the most likely class for a data vector or an RDD of vectors"""
        if isinstance(x, RDD):
            return x.map(lambda v: self.predict(v))
        x = _convert_to_vector(x)
        return self.labels[numpy.argmax(self.pi + x.dot(self.theta.transpose()))]

class NaiveBayes(object):

    @classmethod
    def train(cls, data, lambda_=1.0):
        """
        Train a Naive Bayes model given an RDD of (label, features) vectors.

        This is the Multinomial NB (U{http://tinyurl.com/lsdw6p}) which can
        handle all kinds of discrete data.  For example, by converting
        documents into TF-IDF vectors, it can be used for document
        classification.  By making every vector a 0-1 vector, it can also be
        used as Bernoulli NB (U{http://tinyurl.com/p7c96j6}).

        :param data: RDD of LabeledPoint.
        :param lambda_: The smoothing parameter
        """
        first = data.first()
        if not isinstance(first, LabeledPoint):
            raise ValueError("`data` should be an RDD of LabeledPoint")
        labels, pi, theta = callMLlibFunc("trainNaiveBayes", data, lambda_)
        return NaiveBayesModel(labels.toArray(), pi.toArray(), numpy.array(theta))

def _test():
    import doctest
    from pyspark import SparkContext
    import pyspark.mllib.classification
    globs = pyspark.mllib.classification.__dict__.copy()
    globs['sc'] = SparkContext('local[4]', 'PythonTest', batchSize=2)
    (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
    globs['sc'].stop()
    if failure_count:
        exit(-1)

if __name__ == "__main__":
    _test()
<commit_msg>[SPARK-6080] [PySpark] correct LogisticRegressionWithLBFGS regType parameter for pyspark<commit_after>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from math import exp

import numpy
from numpy import array

from pyspark import RDD
from pyspark.mllib.common import callMLlibFunc
from pyspark.mllib.linalg import SparseVector, _convert_to_vector
from pyspark.mllib.regression import LabeledPoint, LinearModel, _regression_train_wrapper

__all__ = ['LogisticRegressionModel', 'LogisticRegressionWithSGD', 'LogisticRegressionWithLBFGS',
           'SVMModel', 'SVMWithSGD', 'NaiveBayesModel', 'NaiveBayes']

class LinearBinaryClassificationModel(LinearModel):
    """
    Represents a linear binary classification model that predicts to whether an
    example is positive (1.0) or negative (0.0).
    """
    def __init__(self, weights, intercept):
        super(LinearBinaryClassificationModel, self).__init__(weights, intercept)
        self._threshold = None

    def setThreshold(self, value):
        """
        .. note:: Experimental

        Sets the threshold that separates positive predictions from negative
        predictions. An example with prediction score greater than or equal
        to this threshold is identified as an positive, and negative otherwise.
        """
        self._threshold = value

    def clearThreshold(self):
        """
        .. note:: Experimental

        Clears the threshold so that `predict` will output raw prediction scores.
        """
        self._threshold = None

    def predict(self, test):
        """
        Predict values for a single data point or an RDD of points using
        the model trained.
        """
        raise NotImplementedError

class LogisticRegressionModel(LinearBinaryClassificationModel):

    """A linear binary classification model derived from logistic regression.

    >>> data = [
    ...     LabeledPoint(0.0, [0.0, 1.0]),
    ...     LabeledPoint(1.0, [1.0, 0.0]),
    ... ]
    >>> lrm = LogisticRegressionWithSGD.train(sc.parallelize(data))
    >>> lrm.predict([1.0, 0.0])
    1
    >>> lrm.predict([0.0, 1.0])
    0
    >>> lrm.predict(sc.parallelize([[1.0, 0.0], [0.0, 1.0]])).collect()
    [1, 0]
    >>> lrm.clearThreshold()
    >>> lrm.predict([0.0, 1.0])
    0.123...

    >>> sparse_data = [
    ...     LabeledPoint(0.0, SparseVector(2, {0: 0.0})),
    ...     LabeledPoint(1.0, SparseVector(2, {1: 1.0})),
    ...     LabeledPoint(0.0, SparseVector(2, {0: 1.0})),
    ...     LabeledPoint(1.0, SparseVector(2, {1: 2.0}))
    ... ]
    >>> lrm = LogisticRegressionWithSGD.train(sc.parallelize(sparse_data))
    >>> lrm.predict(array([0.0, 1.0]))
    1
    >>> lrm.predict(array([1.0, 0.0]))
    0
    >>> lrm.predict(SparseVector(2, {1: 1.0}))
    1
    >>> lrm.predict(SparseVector(2, {0: 1.0}))
    0
    """
    def __init__(self, weights, intercept):
        super(LogisticRegressionModel, self).__init__(weights, intercept)
        self._threshold = 0.5

    def predict(self, x):
        """
        Predict values for a single data point or an RDD of points using
        the model trained.
        """
        if isinstance(x, RDD):
            return x.map(lambda v: self.predict(v))

        x = _convert_to_vector(x)
        margin = self.weights.dot(x) + self._intercept
        if margin > 0:
            prob = 1 / (1 + exp(-margin))
        else:
            exp_margin = exp(margin)
            prob = exp_margin / (1 + exp_margin)
        if self._threshold is None:
            return prob
        else:
            return 1 if prob > self._threshold else 0

class LogisticRegressionWithSGD(object):

    @classmethod
    def train(cls, data, iterations=100, step=1.0, miniBatchFraction=1.0,
              initialWeights=None, regParam=0.01, regType="l2", intercept=False):
        """
        Train a logistic regression model on the given data.

        :param data:              The training data, an RDD of LabeledPoint.
        :param iterations:        The number of iterations (default: 100).
        :param step:              The step parameter used in SGD
                                  (default: 1.0).
        :param miniBatchFraction: Fraction of data to be used for each SGD
                                  iteration.
        :param initialWeights:    The initial weights (default: None).
        :param regParam:          The regularizer parameter (default: 0.01).
        :param regType:           The type of regularizer used for training
                                  our model.

                                  :Allowed values:
                                     - "l1" for using L1 regularization
                                     - "l2" for using L2 regularization
                                     - None for no regularization

                                     (default: "l2")

        :param intercept:         Boolean parameter which indicates the use
                                  or not of the augmented representation for
                                  training data (i.e. whether bias features
                                  are activated or not).
        """
        def train(rdd, i):
            return callMLlibFunc("trainLogisticRegressionModelWithSGD", rdd, int(iterations),
                                 float(step), float(miniBatchFraction), i, float(regParam), regType,
                                 bool(intercept))

        return _regression_train_wrapper(train, LogisticRegressionModel, data, initialWeights)

class LogisticRegressionWithLBFGS(object):

    @classmethod
    def train(cls, data, iterations=100, initialWeights=None, regParam=0.01, regType="l2",
              intercept=False, corrections=10, tolerance=1e-4):
        """
        Train a logistic regression model on the given data.

        :param data:           The training data, an RDD of LabeledPoint.
        :param iterations:     The number of iterations (default: 100).
        :param initialWeights: The initial weights (default: None).
        :param regParam:       The regularizer parameter (default: 0.01).
        :param regType:        The type of regularizer used for training
                               our model.

                               :Allowed values:
                                 - "l1" for using L1 regularization
                                 - "l2" for using L2 regularization
                                 - None for no regularization

                                 (default: "l2")

        :param intercept:      Boolean parameter which indicates the use
                               or not of the augmented representation for
                               training data (i.e. whether bias features
                               are activated or not).
        :param corrections:    The number of corrections used in the LBFGS
                               update (default: 10).
        :param tolerance:      The convergence tolerance of iterations for
                               L-BFGS (default: 1e-4).

        >>> data = [
        ...     LabeledPoint(0.0, [0.0, 1.0]),
        ...     LabeledPoint(1.0, [1.0, 0.0]),
        ... ]
        >>> lrm = LogisticRegressionWithLBFGS.train(sc.parallelize(data))
        >>> lrm.predict([1.0, 0.0])
        1
        >>> lrm.predict([0.0, 1.0])
        0
        """
        def train(rdd, i):
            return callMLlibFunc("trainLogisticRegressionModelWithLBFGS", rdd, int(iterations), i,
                                 float(regParam), regType, bool(intercept), int(corrections),
                                 float(tolerance))

        return _regression_train_wrapper(train, LogisticRegressionModel, data, initialWeights)

class SVMModel(LinearBinaryClassificationModel):

    """A support vector machine.

    >>> data = [
    ...     LabeledPoint(0.0, [0.0]),
    ...     LabeledPoint(1.0, [1.0]),
    ...     LabeledPoint(1.0, [2.0]),
    ...     LabeledPoint(1.0, [3.0])
    ... ]
    >>> svm = SVMWithSGD.train(sc.parallelize(data))
    >>> svm.predict([1.0])
    1
    >>> svm.predict(sc.parallelize([[1.0]])).collect()
    [1]
    >>> svm.clearThreshold()
    >>> svm.predict(array([1.0]))
    1.25...

    >>> sparse_data = [
    ...     LabeledPoint(0.0, SparseVector(2, {0: -1.0})),
    ...     LabeledPoint(1.0, SparseVector(2, {1: 1.0})),
    ...     LabeledPoint(0.0, SparseVector(2, {0: 0.0})),
    ...     LabeledPoint(1.0, SparseVector(2, {1: 2.0}))
    ... ]
    >>> svm = SVMWithSGD.train(sc.parallelize(sparse_data))
    >>> svm.predict(SparseVector(2, {1: 1.0}))
    1
    >>> svm.predict(SparseVector(2, {0: -1.0}))
    0
    """
    def __init__(self, weights, intercept):
        super(SVMModel, self).__init__(weights, intercept)
        self._threshold = 0.0

    def predict(self, x):
        """
        Predict values for a single data point or an RDD of points using
        the model trained.
        """
        if isinstance(x, RDD):
            return x.map(lambda v: self.predict(v))

        x = _convert_to_vector(x)
        margin = self.weights.dot(x) + self.intercept
        if self._threshold is None:
            return margin
        else:
            return 1 if margin > self._threshold else 0

class SVMWithSGD(object):

    @classmethod
    def train(cls, data, iterations=100, step=1.0, regParam=0.01,
              miniBatchFraction=1.0, initialWeights=None, regType="l2", intercept=False):
        """
        Train a support vector machine on the given data.

        :param data:              The training data, an RDD of LabeledPoint.
        :param iterations:        The number of iterations (default: 100).
        :param step:              The step parameter used in SGD
                                  (default: 1.0).
        :param regParam:          The regularizer parameter (default: 0.01).
        :param miniBatchFraction: Fraction of data to be used for each SGD
                                  iteration.
        :param initialWeights:    The initial weights (default: None).
        :param regType:           The type of regularizer used for training
                                  our model.

                                  :Allowed values:
                                     - "l1" for using L1 regularization
                                     - "l2" for using L2 regularization
                                     - None for no regularization

                                     (default: "l2")

        :param intercept:         Boolean parameter which indicates the use
                                  or not of the augmented representation for
                                  training data (i.e. whether bias features
                                  are activated or not).
        """
        def train(rdd, i):
            return callMLlibFunc("trainSVMModelWithSGD", rdd, int(iterations), float(step),
                                 float(regParam), float(miniBatchFraction), i, regType,
                                 bool(intercept))

        return _regression_train_wrapper(train, SVMModel, data, initialWeights)

class NaiveBayesModel(object):

    """
    Model for Naive Bayes classifiers.

    Contains two parameters:
    - pi: vector of logs of class priors (dimension C)
    - theta: matrix of logs of class conditional probabilities (CxD)

    >>> data = [
    ...     LabeledPoint(0.0, [0.0, 0.0]),
    ...     LabeledPoint(0.0, [0.0, 1.0]),
    ...     LabeledPoint(1.0, [1.0, 0.0]),
    ... ]
    >>> model = NaiveBayes.train(sc.parallelize(data))
    >>> model.predict(array([0.0, 1.0]))
    0.0
    >>> model.predict(array([1.0, 0.0]))
    1.0
    >>> model.predict(sc.parallelize([[1.0, 0.0]])).collect()
    [1.0]
    >>> sparse_data = [
    ...     LabeledPoint(0.0, SparseVector(2, {1: 0.0})),
    ...     LabeledPoint(0.0, SparseVector(2, {1: 1.0})),
    ...     LabeledPoint(1.0, SparseVector(2, {0: 1.0}))
    ... ]
    >>> model = NaiveBayes.train(sc.parallelize(sparse_data))
    >>> model.predict(SparseVector(2, {1: 1.0}))
    0.0
    >>> model.predict(SparseVector(2, {0: 1.0}))
    1.0
    """

    def __init__(self, labels, pi, theta):
        self.labels = labels
        self.pi = pi
        self.theta = theta

    def predict(self, x):
        """Return the most likely class for a data vector or an RDD of vectors"""
        if isinstance(x, RDD):
            return x.map(lambda v: self.predict(v))
        x = _convert_to_vector(x)
        return self.labels[numpy.argmax(self.pi + x.dot(self.theta.transpose()))]

class NaiveBayes(object):

    @classmethod
    def train(cls, data, lambda_=1.0):
        """
        Train a Naive Bayes model given an RDD of (label, features) vectors.

        This is the Multinomial NB (U{http://tinyurl.com/lsdw6p}) which can
        handle all kinds of discrete data.  For example, by converting
        documents into TF-IDF vectors, it can be used for document
        classification.  By making every vector a 0-1 vector, it can also be
        used as Bernoulli NB (U{http://tinyurl.com/p7c96j6}).

        :param data: RDD of LabeledPoint.
        :param lambda_: The smoothing parameter
        """
        first = data.first()
        if not isinstance(first, LabeledPoint):
            raise ValueError("`data` should be an RDD of LabeledPoint")
        labels, pi, theta = callMLlibFunc("trainNaiveBayes", data, lambda_)
        return NaiveBayesModel(labels.toArray(), pi.toArray(), numpy.array(theta))

def _test():
    import doctest
    from pyspark import SparkContext
    import pyspark.mllib.classification
    globs = pyspark.mllib.classification.__dict__.copy()
    globs['sc'] = SparkContext('local[4]', 'PythonTest', batchSize=2)
    (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
    globs['sc'].stop()
    if failure_count:
        exit(-1)

if __name__ == "__main__":
    _test()
<|endoftext|>


<commit_before>"""This test checks that AxSearch is functional.

It also checks that it is usable with a separate scheduler.
"""
import numpy as np

import ray
from ray.tune import run
from ray.tune.schedulers import AsyncHyperBandScheduler
from ray.tune.suggest.ax import AxSearch

def hartmann6(x):
    alpha = np.array([1.0, 1.2, 3.0, 3.2])
    A = np.array([
        [10, 3, 17, 3.5, 1.7, 8],
        [0.05, 10, 17, 0.1, 8, 14],
        [3, 3.5, 1.7, 10, 17, 8],
        [17, 8, 0.05, 10, 0.1, 14],
    ])
    P = 10**(-4) * np.array([
        [1312, 1696, 5569, 124, 8283, 5886],
        [2329, 4135, 8307, 3736, 1004, 9991],
        [2348, 1451, 3522, 2883, 3047, 6650],
        [4047, 8828, 8732, 5743, 1091, 381],
    ])
    y = 0.0
    for j, alpha_j in enumerate(alpha):
        t = 0
        for k in range(6):
            t += A[j, k] * ((x[k] - P[j, k])**2)
        y -= alpha_j * np.exp(-t)
    return y

def easy_objective(config, reporter):
    import time
    time.sleep(0.2)
    for i in range(config["iterations"]):
        x = np.array([config.get("x{}".format(i + 1)) for i in range(6)])
        reporter(
            timesteps_total=i,
            hartmann6=hartmann6(x),
            l2norm=np.sqrt((x**2).sum()))
        time.sleep(0.02)

if __name__ == "__main__":
    import argparse
    from ax.service.ax_client import AxClient

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--smoke-test", action="store_true", help="Finish quickly for testing")
    args, _ = parser.parse_known_args()

    ray.init()

    config = {
        "num_samples": 10 if args.smoke_test else 50,
        "config": {
            "iterations": 100,
        },
        "stop": {
            "timesteps_total": 100
        }
    }
    parameters = [
        {
            "name": "x1",
            "type": "range",
            "bounds": [0.0, 1.0],
            "value_type": "float",  # Optional, defaults to "bounds".
            "log_scale": False,  # Optional, defaults to False.
        },
        {
            "name": "x2",
            "type": "range",
            "bounds": [0.0, 1.0],
        },
        {
            "name": "x3",
            "type": "range",
            "bounds": [0.0, 1.0],
        },
        {
            "name": "x4",
            "type": "range",
            "bounds": [0.0, 1.0],
        },
        {
            "name": "x5",
            "type": "range",
            "bounds": [0.0, 1.0],
        },
        {
            "name": "x6",
            "type": "range",
            "bounds": [0.0, 1.0],
        },
    ]
    client = AxClient(enforce_sequential_optimization=False)
    client.create_experiment(
        parameters=parameters,
        objective_name="hartmann6",
        minimize=True,  # Optional, defaults to False.
        parameter_constraints=["x1 + x2 <= 2.0"],  # Optional.
        outcome_constraints=["l2norm <= 1.25"],  # Optional.
    )
    algo = AxSearch(client, max_concurrent=4)
    scheduler = AsyncHyperBandScheduler(metric="hartmann6", mode="max")
    run(easy_objective,
        name="ax",
        search_alg=algo,
        scheduler=scheduler,
        **config)
<commit_msg>[tune] Align scheduler mode with search algorithm in example of… (#7012)<commit_after>"""This test checks that AxSearch is functional.

It also checks that it is usable with a separate scheduler.
"""
import numpy as np

import ray
from ray.tune import run
from ray.tune.schedulers import AsyncHyperBandScheduler
from ray.tune.suggest.ax import AxSearch

def hartmann6(x):
    alpha = np.array([1.0, 1.2, 3.0, 3.2])
    A = np.array([
        [10, 3, 17, 3.5, 1.7, 8],
        [0.05, 10, 17, 0.1, 8, 14],
        [3, 3.5, 1.7, 10, 17, 8],
        [17, 8, 0.05, 10, 0.1, 14],
    ])
    P = 10**(-4) * np.array([
        [1312, 1696, 5569, 124, 8283, 5886],
        [2329, 4135, 8307, 3736, 1004, 9991],
        [2348, 1451, 3522, 2883, 3047, 6650],
        [4047, 8828, 8732, 5743, 1091, 381],
    ])
    y = 0.0
    for j, alpha_j in enumerate(alpha):
        t = 0
        for k in range(6):
            t += A[j, k] * ((x[k] - P[j, k])**2)
        y -= alpha_j * np.exp(-t)
    return y

def easy_objective(config, reporter):
    import time
    time.sleep(0.2)
    for i in range(config["iterations"]):
        x = np.array([config.get("x{}".format(i + 1)) for i in range(6)])
        reporter(
            timesteps_total=i,
            hartmann6=hartmann6(x),
            l2norm=np.sqrt((x**2).sum()))
        time.sleep(0.02)

if __name__ == "__main__":
    import argparse
    from ax.service.ax_client import AxClient

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--smoke-test", action="store_true", help="Finish quickly for testing")
    args, _ = parser.parse_known_args()

    ray.init()

    config = {
        "num_samples": 10 if args.smoke_test else 50,
        "config": {
            "iterations": 100,
        },
        "stop": {
            "timesteps_total": 100
        }
    }
    parameters = [
        {
            "name": "x1",
            "type": "range",
            "bounds": [0.0, 1.0],
            "value_type": "float",  # Optional, defaults to "bounds".
            "log_scale": False,  # Optional, defaults to False.
        },
        {
            "name": "x2",
            "type": "range",
            "bounds": [0.0, 1.0],
        },
        {
            "name": "x3",
            "type": "range",
            "bounds": [0.0, 1.0],
        },
        {
            "name": "x4",
            "type": "range",
            "bounds": [0.0, 1.0],
        },
        {
            "name": "x5",
            "type": "range",
            "bounds": [0.0, 1.0],
        },
        {
            "name": "x6",
            "type": "range",
            "bounds": [0.0, 1.0],
        },
    ]
    client = AxClient(enforce_sequential_optimization=False)
    client.create_experiment(
        parameters=parameters,
        objective_name="hartmann6",
        minimize=True,  # Optional, defaults to False.
        parameter_constraints=["x1 + x2 <= 2.0"],  # Optional.
        outcome_constraints=["l2norm <= 1.25"],  # Optional.
    )
    algo = AxSearch(client, max_concurrent=4)
    scheduler = AsyncHyperBandScheduler(metric="hartmann6", mode="min")
    run(easy_objective,
        name="ax",
        search_alg=algo,
        scheduler=scheduler,
        **config)
<|endoftext|>


<commit_before>def playerIcons(poi):
    if poi['id'] == 'Player':
        poi['icon'] = "http://overviewer.org/avatar/%s" % poi['EntityId']
        return "Last known location for %s" % poi['EntityId']

# Only signs with "-- RENDER --" on the last line will be shown
# Otherwise, people can't have secret bases and the render is too busy anyways.
def signFilter(poi):
    if poi['id'] == 'Sign':
        if poi['Text4'] == '-- RENDER --':
            return "\n".join([poi['Text1'], poi['Text2'], poi['Text3'], poi['Text4']])

worlds['minecraft'] = "/home/minecraft/server/world"
outputdir = "/home/minecraft/render/"

markers = [
    dict(name="Players", filterFunction=playerIcons),
    dict(name="Signs", filterFunction=signFilter)
]

renders["day"] = {
    'world': 'minecraft',
    'title': 'Day',
    'rendermode': 'smooth_lighting',
    "dimension": "overworld",
    'markers': markers
}

renders["night"] = {
    'world': 'minecraft',
    'title': 'Night',
    'rendermode': 'smooth_night',
    "dimension": "overworld",
    'markers': markers
}

renders["end"] = {
    "world": "minecraft",
    "title": "End",
    "rendermode": [Base(), EdgeLines(), SmoothLighting(strength=0.5)],
    "dimension": "end",
    'markers': markers
}
<commit_msg>Add nether render<commit_after>def playerIcons(poi):
    if poi['id'] == 'Player':
        poi['icon'] = "http://overviewer.org/avatar/%s" % poi['EntityId']
        return "Last known location for %s" % poi['EntityId']

# Only signs with "-- RENDER --" on the last line will be shown
# Otherwise, people can't have secret bases and the render is too busy anyways.
def signFilter(poi):
    if poi['id'] == 'Sign':
        if poi['Text4'] == '-- RENDER --':
            return "\n".join([poi['Text1'], poi['Text2'], poi['Text3'], poi['Text4']])

worlds['minecraft'] = "/home/minecraft/server/world"
outputdir = "/home/minecraft/render/"

markers = [
    dict(name="Players", filterFunction=playerIcons),
    dict(name="Signs", filterFunction=signFilter)
]

renders["day"] = {
    'world': 'minecraft',
    'title': 'Day',
    'rendermode': 'smooth_lighting',
    "dimension": "overworld",
    'markers': markers
}

renders["night"] = {
    'world': 'minecraft',
    'title': 'Night',
    'rendermode': 'smooth_night',
    "dimension": "overworld",
    'markers': markers
}

renders["nether"] = {
    "world": "minecraft",
    "title": "Nether",
    "rendermode": 'nether_smooth_lighting',
    "dimension": "nether",
    'markers': markers
}

renders["end"] = {
    "world": "minecraft",
    "title": "End",
    "rendermode": [Base(), EdgeLines(), SmoothLighting(strength=0.5)],
    "dimension": "end",
    'markers': markers
}
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os.path
#!/usr/bin/env python

'''
Config file
'''

import yaml

info_file = open(os.path.join(os.path.abspath('.'), 'info.yml'), 'r')
config_info = yaml.load(info_file)
info_file.close()
database_file = open(os.path.join(os.path.abspath('.'), 'database.yml'), 'r')
config_database = yaml.load(database_file)
database_file.close()
urls_file = open(os.path.join(os.path.abspath('.'), 'urls.yml'), 'r')
config_urls = yaml.load(urls_file)
urls_file.close()

class info(object):
    author = config_info['author']
    appname = config_info['appname']
    version = config_info['version']
    codename = config_info['codename']
    status = config_info['status']
    license = config_info['license']

class database(object):
    version = config_database['version']

class url(object):
    main_server = config_urls['main_server']
    repo_url = config_urls['repo_url']<commit_msg>updated config<commit_after>#!/usr/bin/env python

'''
Config file
'''

import yaml
import os.path
from lib.base import pather

PATHandler = pather.PATHandler(__file__)
pardir = PATHandler.parent_dir()

info_file = open(os.path.join(pardir, 'info.yml'), 'r')
config_info = yaml.load(info_file)
info_file.close()
database_file = open(os.path.join(pardir, 'database.yml'), 'r')
config_database = yaml.load(database_file)
database_file.close()
urls_file = open(os.path.join(pardir, 'urls.yml'), 'r')
config_urls = yaml.load(urls_file)
urls_file.close()

class info(object):
    author = config_info['author']
    appname = config_info['appname']
    version = config_info['version']
    codename = config_info['codename']
    status = config_info['status']
    license = config_info['license']

class database(object):
    version = config_database['version']

class url(object):
    main_server = config_urls['main_server']
    repo = config_urls['repo_url']<|endoftext|>


<commit_before>import ConfigParser

def get_config_object(filename :str):
    parsed_config = ConfigParser.ConfigParser().read(filename)
    return parsed_config<commit_msg>just some more code for congif parser<commit_after>import ConfigParser

class Config:

    def __init__(self, filename:str) -> dict:
        res = []
        raw_config_object = self.get_config_object(filename)
        res = self.validate(raw_config_object)
        return res

    def get_config_object(file :fp) -> dict:
        parsed_config = ConfigParser.ConfigParser().readfp(filename)
        return parsed_config

    '''
    need to raise different
    exceptions:
        warnings: (if emails is not specified)
        error: (if picture directory is not writable,
        or microphone details are not ints )
    '''
    def validate(self, raw_config_object):
        validated_object = []

        return validated_object
<|endoftext|>