<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
from frappe import _
from frappe.model.document import Document
from frappe.desk.doctype.notification_settings.notification_settings import (is_notifications_enabled,
	is_email_notifications_enabled, is_email_notifications_enabled_for_type, set_seen_value)

class NotificationLog(Document):
	def after_insert(self):
		frappe.publish_realtime('notification', after_commit=True, user=self.for_user)
		set_notifications_as_unseen(self.for_user)
		if is_email_notifications_enabled(self.for_user):
			send_notification_email(self)

def get_permission_query_conditions(for_user):
	if not for_user:
		for_user = frappe.session.user

	if for_user == 'Administrator':
		return

	return '''(`tabNotification Log`.for_user = '{user}')'''.format(user=for_user)

def enqueue_create_notification(users, doc):
	'''
	During installation of new site, enqueue_create_notification tries to connect to Redis.
	This breaks new site creation if Redis server is not running.
	We do not need any notifications in fresh installation
	'''

	if frappe.flags.in_install:
		return

	doc = frappe._dict(doc)

	if isinstance(users, frappe.string_types):
		users = [user.strip() for user in users.split(',') if user.strip()]

	frappe.enqueue(
		'frappe.desk.doctype.notification_log.notification_log.make_notification_logs',
		doc=doc,
		users=users,
		now=frappe.flags.in_test
	)

def make_notification_logs(doc, users):
	from frappe.social.doctype.energy_point_settings.energy_point_settings import is_energy_point_enabled
	for user in users:
		if frappe.db.exists('User', user):
			if is_notifications_enabled(user):
				if doc.type == 'Energy Point' and not is_energy_point_enabled():
					return

				_doc = frappe.new_doc('Notification Log')
				_doc.update(doc)
				_doc.for_user = user
				_doc.subject = _doc.subject.replace('
', '').replace('
', '')
				if _doc.for_user != _doc.from_user or doc.type == 'Energy Point':
					_doc.insert(ignore_permissions=True)

def send_notification_email(doc):
	is_type_enabled = is_email_notifications_enabled_for_type(doc.for_user, doc.type)
	if not is_type_enabled:
		return

	if doc.type == 'Energy Point' and doc.email_content is None:
		return

	from frappe.utils import get_url_to_form, strip_html

	doc_link = get_url_to_form(doc.document_type, doc.document_name)
	header = get_email_header(doc)
	email_subject = strip_html(doc.subject)

	frappe.sendmail(
		recipients = doc.for_user,
		subject = email_subject,
		template = "new_notification",
		args = {
			'body_content': doc.subject,
			'description': doc.email_content,
			'document_type': doc.document_type,
			'document_name': doc.document_name,
			'doc_link': doc_link
		},
		header = [header, 'orange'],
		now=frappe.flags.in_test
	)

def get_email_header(doc):
	return {
		'Default': _('New Notification'),
		'Mention': _('New Mention'),
		'Assignment': _('New Assignment'),
		'Share': _('New Document Shared'),
		'Energy Point': _('Energy Point Update'),
	}[doc.type or 'Default']

def get_title(doctype, docname, title_field=None):
	if not title_field:
		title_field = frappe.get_meta(doctype).get_title_field()
	title = docname if title_field == "name" else \
		frappe.db.get_value(doctype, docname, title_field)
	return title

def get_title_html(title):
	return '{0}'.format(title)

@frappe.whitelist()
def mark_all_as_read(docnames):
	docnames = frappe.parse_json(docnames)
	if docnames:
		filters = {'name': ['in', docnames]}
		frappe.db.set_value('Notification Log', filters, 'read', 1, update_modified=False)

@frappe.whitelist()
def mark_as_read(docname):
	if docname:
		frappe.db.set_value('Notification Log', docname, 'read', 1, update_modified=False)

@frappe.whitelist()
def trigger_indicator_hide():
	frappe.publish_realtime('indicator_hide', user=frappe.session.user)

def set_notifications_as_unseen(user):
	notification_settings_doc = frappe.get_doc('Notification Settings', user)
	if notification_settings_doc.seen == 1:
		set_seen_value(value = 0, user = user)
<commit_msg>fix: tests<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
from frappe import _
from frappe.model.document import Document
from frappe.desk.doctype.notification_settings.notification_settings import (is_notifications_enabled,
	is_email_notifications_enabled, is_email_notifications_enabled_for_type, set_seen_value)

class NotificationLog(Document):
	def after_insert(self):
		frappe.publish_realtime('notification', after_commit=True, user=self.for_user)
		set_notifications_as_unseen(self.for_user)
		if is_email_notifications_enabled(self.for_user):
			send_notification_email(self)

def get_permission_query_conditions(for_user):
	if not for_user:
		for_user = frappe.session.user

	if for_user == 'Administrator':
		return

	return '''(`tabNotification Log`.for_user = '{user}')'''.format(user=for_user)

def enqueue_create_notification(users, doc):
	'''
	During installation of new site, enqueue_create_notification tries to connect to Redis.
	This breaks new site creation if Redis server is not running.
	We do not need any notifications in fresh installation
	'''

	if frappe.flags.in_install:
		return

	doc = frappe._dict(doc)

	if isinstance(users, frappe.string_types):
		users = [user.strip() for user in users.split(',') if user.strip()]

	frappe.enqueue(
		'frappe.desk.doctype.notification_log.notification_log.make_notification_logs',
		doc=doc,
		users=users,
		now=frappe.flags.in_test
	)

def make_notification_logs(doc, users):
	from frappe.social.doctype.energy_point_settings.energy_point_settings import is_energy_point_enabled
	for user in users:
		if frappe.db.exists('User', user):
			if is_notifications_enabled(user):
				if doc.type == 'Energy Point' and not is_energy_point_enabled():
					return

				_doc = frappe.new_doc('Notification Log')
				_doc.update(doc)
				_doc.for_user = user
				_doc.subject = _doc.subject.replace('
', '').replace('
', '')
				if _doc.for_user != _doc.from_user or doc.type == 'Energy Point':
					_doc.insert(ignore_permissions=True)

def send_notification_email(doc):
	is_type_enabled = is_email_notifications_enabled_for_type(doc.for_user, doc.type)
	if not is_type_enabled:
		return

	if doc.type == 'Energy Point' and doc.email_content is None:
		return

	from frappe.utils import get_url_to_form, strip_html

	doc_link = get_url_to_form(doc.document_type, doc.document_name)
	header = get_email_header(doc)
	email_subject = strip_html(doc.subject)

	frappe.sendmail(
		recipients = doc.for_user,
		subject = email_subject,
		template = "new_notification",
		args = {
			'body_content': doc.subject,
			'description': doc.email_content,
			'document_type': doc.document_type,
			'document_name': doc.document_name,
			'doc_link': doc_link
		},
		header = [header, 'orange'],
		now=frappe.flags.in_test
	)

def get_email_header(doc):
	return {
		'Default': _('New Notification'),
		'Mention': _('New Mention'),
		'Assignment': _('New Assignment'),
		'Share': _('New Document Shared'),
		'Energy Point': _('Energy Point Update'),
	}[doc.type or 'Default']

def get_title(doctype, docname, title_field=None):
	if not title_field:
		title_field = frappe.get_meta(doctype).get_title_field()
	title = docname if title_field == "name" else \
		frappe.db.get_value(doctype, docname, title_field)
	return title

def get_title_html(title):
	return '{0}'.format(title)

@frappe.whitelist()
def mark_all_as_read(docnames):
	docnames = frappe.parse_json(docnames)
	if docnames:
		filters = {'name': ['in', docnames]}
		frappe.db.set_value('Notification Log', filters, 'read', 1, update_modified=False)

@frappe.whitelist()
def mark_as_read(docname):
	if docname:
		frappe.db.set_value('Notification Log', docname, 'read', 1, update_modified=False)

@frappe.whitelist()
def trigger_indicator_hide():
	frappe.publish_realtime('indicator_hide', user=frappe.session.user)

def set_notifications_as_unseen(user):
	try:
		notification_settings_doc = frappe.get_doc('Notification Settings', user)
		if notification_settings_doc.seen == 1:
			set_seen_value(value = 0, user = user)
	except frappe.DoesNotExistError:
		return
<|endoftext|>


<commit_before># Copyright (c) 2012 Azavea, Inc.
# See LICENSE in the project root for copying permission

from django.template import RequestContext
from django.shortcuts import render_to_response, get_object_or_404
from django.views.decorators.cache import cache_control
from django.db.models import Q
from django.conf import settings
from django.http import HttpResponseRedirect, Http404
from models import Location
import logging, hashlib
from faq.models import Topic, Question
from django.utils.translation import ugettext as _
from django.utils.translation import check_for_language
from django.utils import translation

logger = logging.getLogger(__name__)

# TODO: We probably don't need this function for the new version, I'm moving
# it in from the old version for now to avoid lots of refactoring.
def get_opts(selected_val='2'):
    """
    Gets option list for the distance dropdown (see base.html)
    selected_val: string representing the value of the dropdown that should be selected
    Default is '2'
    """
    # Options for distance dropdown
    # option value => (option text, enabled)
    distance_opts = { '-1': [_('Distance'), False],
                      '0.5': [_('< 0.5 mi'), False],
                      '1': [_('< 1 mi'), False],
                      '2': [_('< 2 mi'), False],
                      '5': [_('< 5 mi'), False],
                      '10': [_('< 10 mi'), False] }

    key = selected_val if selected_val in distance_opts else '2'
    distance_opts[key][1] = True
    result = [[k] + v for k, v in distance_opts.items()]
    return sorted(result, key=lambda a: float(a[0]))

def index(request):
    ctx = RequestContext(request, { })
    response = render_to_response('index.html', context_instance=ctx)
    return response

def about(request):
    ctx = RequestContext(request, {
        'options': get_opts(),
        'mapbarEnabled': True
    })
    return render_to_response('about.html', context_instance=ctx)

class TopicWrapper(object):
    topic = None
    questions = None

    def __init__(self, t, request):
        self.topic = t
        qs = Question.objects.filter(topic=t, status=Question.ACTIVE)
        if request.user.is_anonymous():
            qs = qs.exclude(protected=True)
        self.questions = list(qs)

def faq(request):
    tpl = 'faq-models.html'

    # get the language of the request
    lang = request.LANGUAGE_CODE

    # get the topics in this language
    topics = Topic.objects.filter(slug__startswith=lang + '-')
    if topics.count() == 0:
        topics = Topic.objects.filter(slug__startswith=settings.LANGUAGE_CODE[0:2] + '-')

    ctx = RequestContext(request, {
        'topics': [TopicWrapper(t, request) for t in topics],
        'options': get_opts(),
        'mapbarEnabled': True
    })
    return render_to_response(tpl, context_instance=ctx)

def setlang(request, language):
    nxt = '/'
    if 'next' in request.REQUEST:
        nxt = request.REQUEST['next']

    response = HttpResponseRedirect(nxt)
    if language and check_for_language(language):
        response.set_cookie(settings.LANGUAGE_COOKIE_NAME, language)

    return response
<commit_msg>Add functions to serve up single school page view for permalinks<commit_after># Copyright (c) 2012 Azavea, Inc.
# See LICENSE in the project root for copying permission

from django.template import RequestContext
from django.shortcuts import render_to_response, get_object_or_404
from django.views.decorators.cache import cache_control
from django.db.models import Q
from django.conf import settings
from django.http import HttpResponse, HttpResponseRedirect, Http404
from models import Location
import logging, hashlib
from faq.models import Topic, Question
from django.utils.translation import ugettext as _
from django.utils.translation import check_for_language
from django.utils import translation
import json

logger = logging.getLogger(__name__)

# TODO: We probably don't need this function for the new version, I'm moving
# it in from the old version for now to avoid lots of refactoring.
def get_opts(selected_val='2'):
    """
    Gets option list for the distance dropdown (see base.html)
    selected_val: string representing the value of the dropdown that should be selected
    Default is '2'
    """
    # Options for distance dropdown
    # option value => (option text, enabled)
    distance_opts = { '-1': [_('Distance'), False],
                      '0.5': [_('< 0.5 mi'), False],
                      '1': [_('< 1 mi'), False],
                      '2': [_('< 2 mi'), False],
                      '5': [_('< 5 mi'), False],
                      '10': [_('< 10 mi'), False] }

    key = selected_val if selected_val in distance_opts else '2'
    distance_opts[key][1] = True
    result = [[k] + v for k, v in distance_opts.items()]
    return sorted(result, key=lambda a: float(a[0]))

def index(request):
    ctx = RequestContext(request, { })
    response = render_to_response('index.html', context_instance=ctx)
    return response

def about(request):
    ctx = RequestContext(request, {
        'options': get_opts(),
        'mapbarEnabled': True
    })
    return render_to_response('about.html', context_instance=ctx)

class TopicWrapper(object):
    topic = None
    questions = None

    def __init__(self, t, request):
        self.topic = t
        qs = Question.objects.filter(topic=t, status=Question.ACTIVE)
        if request.user.is_anonymous():
            qs = qs.exclude(protected=True)
        self.questions = list(qs)

def faq(request):
    tpl = 'faq-models.html'

    # get the language of the request
    lang = request.LANGUAGE_CODE

    # get the topics in this language
    topics = Topic.objects.filter(slug__startswith=lang + '-')
    if topics.count() == 0:
        topics = Topic.objects.filter(slug__startswith=settings.LANGUAGE_CODE[0:2] + '-')

    ctx = RequestContext(request, {
        'topics': [TopicWrapper(t, request) for t in topics],
        'options': get_opts(),
        'mapbarEnabled': True
    })
    return render_to_response(tpl, context_instance=ctx)

def setlang(request, language):
    nxt = '/'
    if 'next' in request.REQUEST:
        nxt = request.REQUEST['next']

    response = HttpResponseRedirect(nxt)
    if language and check_for_language(language):
        response.set_cookie(settings.LANGUAGE_COOKIE_NAME, language)

    return response

# Location Stuff
def location_details(location_id):
    """
    Helper method that gets all the fields for a specific location.

    This is called by the detail page and the comparison page.
    """
    item = get_object_or_404(Location, id=location_id)
    return item.get_context_dict()

def location(request, location_id):
    """
    Render a detail page for a single location.
    """
    context = location_details(location_id)

    tpl = 'location.html'

    context = RequestContext(request, context)
    return render_to_response(tpl, context_instance=context)

def location_position(request, location_id):
    """
    Render a json response with the longitude and latitude of a single location.
    """
    loc = get_object_or_404(Location, id=location_id)
    results = [{'pk': location_id, 'lng':loc.geom[0], 'lat': loc.geom[1]}]
    return HttpResponse(json.dumps(results), content_type="application/json")

<|endoftext|>


<commit_before>from __future__ import print_function
import numpy as np
from astropy.io import fits
from astropy import wcs
from marvin import config
from marvin.api.api import Interaction

class Cube(object):

    def __init__(self, filename=None, plateifu=None, mangaid=None):
        args = [filename, plateifu, mangaid]
        errmsg = 'Enter filename, plateifu, or mangaid!'
        assert any(args), errmsg
        assert sum([bool(arg) for arg in args]) == 1, errmsg

        self.filename = filename
        self.mangaid = mangaid
        self.plateifu = plateifu

        self.useDB = None

        # convert from mangaid to plateifu
        # print warning if multiple plateifus correspond to one mangaid
        # FIX
        mangaid_to_plateifu = {'1-209232': '8485-1901'}
        if self.mangaid:
            self.plateifu = mangaid_to_plateifu[self.mangaid]

        if config.mode == 'local':
            if self.filename:
                self._openFile()
            else:
                try:
                    # self._getCubeFromPlateIFU()
                    self._getCubeFromMangaID()
                except RuntimeError:
                    # convert from plateifu to filename (sdss_access)
                    self.filename = plateifu_to_filename(self.plateifu)
                    try:
                        self._openFile()
                    except IOError as e:
                        if config.download:
                            # download file via sdsssync
                            # self._openFile()
                            pass
                        else:
                            raise Exception('File does not exist locally') from e
        elif config.mode == 'remote':
            if self.filename:
                raise FileNotFoundError('Cannot open file remotely.')
            else:
                # use API
                pass

    def getSpectrum(self, x=None, y=None, ra=None, dec=None, ext='flux'):
        """Returns the flux spectrum for an spaxel.

        Parameters
        ----------
        x, y : int
            The spaxel coordinates relative to the centre of the cube.

        ra, dec : float
            The coordinates of the spaxel to return. The closest spaxel to
            those coordinates will be returned.

        ext : str
            The extension of the cube to use, either `'flux'`, `'ivar'`, or
            `'mask'`.

        Returns
        -------
        result : np.array
            An array with the spectrum for the input coordinates.

        """

        # Checks that we have the correct set of inputs.
        if x or y:
            assert not ra and not dec, 'Either use (x, y) or (ra, dec)'
            assert x and y, 'Specify both x and y'
            inputMode = 'pix'
        elif ra or dec:
            assert not x and not y, 'Either use (x, y) or (ra, dec)'
            assert ra and dec, 'Specify both ra and dec'
            inputMode = 'sky'
        else:
            raise ValueError('You need to specify (x, y) or (ra, dec)')

        assert isinstance(ext, basestring)
        ext = ext.lower()
        assert ext in ['flux', 'ivar', 'mask'], \
            'ext needs to be either \'flux\', \'ivar\', or \'mask\''

        if not self.useDB:
            cubeExt = self._hdu[ext.upper()]
            if inputMode == 'sky':
                cubeWCS = wcs.WCS(cubeExt.header)
                x, y = cubeWCS.wcs_sky2pix([ra, dec], 1)

            cubeShape = cubeExt.shape
            # TODO: assert that pix coordinates are in cube
            yMid, xMid = cubeShape[1:] / 2.
            return cubeExt.data[:, yMid - y, xMid + x]

        else:

            raise NotImplementedError(
                'getSpectrum from DB not yet implemented')

        # ''' currently: x,y array indices
        # ideally: x,y in arcsecond relative to cube center '''
        # # spectrum = Spectrum(x,y)
        # if config.mode == 'file':
        #     ''' local (client has a file) '''
        #     shape = self.flux.shape
        #     assert len(shape) == 3, 'Dimensions of flux not = 3'
        #     assert x < shape[2] and y < shape[1], 'Input x,y coordinates greater than flux dimensions'
        #     return self.flux[:, y, x]
        # elif config.mode == 'db':
        #     ''' db means local (client) has db '''
        #     return self._cube.spaxels[0].flux
        # else:
        #     ''' local (client) has nothing '''
        #     route = 'cubes/{0}/spectra/x={1}/y={2}/'.format(self.mangaid, x, y)
        #     results = Interaction(route, request_type='get')
        #     return results.getData(astype=np.array)

    def _openFile(self):
<<<<<<< HEAD

        if not os.path.exists(self.filename):
            raise ValueError('filename {0} cannot be found'
                             .format(self.filename))

        self._hdu = fits.open(self.filename)
        self.useDB = False
=======
        try:
            self.hdu = fits.open(self.filename)
        except FileNotFoundError as e:
            raise Exception('{0} does not exist. Please provide full file path.'.format(self.filename)) from e
>>>>>>> 82d26d21c873b476e37be7ff65d61a256a835278

    @property
    def flux(self):
        if not self.useDB:
            return self._hdu['FLUX'].data
        else:
            return None

    # Switch to _getCubeFromPlateIFU
    def _getCubeFromMangaID(self):
        ''' server-side code '''
        from ..db.database import db
        import sdss.internal.database.utah.mangadb.DataModelClasses as datadb
        session = db.Session()
        self._cube = session.query(datadb.Cube).join(datadb.PipelineInfo, datadb.PipelineVersion).filter(datadb.PipelineVersion.version == 'trunk', datadb.Cube.mangaid == self.mangaid).first()
        print('cube', self._cube, len(self._cube.spaxels))
        self.ifu = self._cube.ifu.name
        self.ra = self._cube.ra
        self.dec = self._cube.dec
        self.plate = self._cube.plate
<commit_msg>fixed merge problem<commit_after>from __future__ import print_function
import numpy as np
from astropy.io import fits
from astropy import wcs
from marvin import config
from marvin.api.api import Interaction

class Cube(object):

    def __init__(self, filename=None, plateifu=None, mangaid=None):
        args = [filename, plateifu, mangaid]
        errmsg = 'Enter filename, plateifu, or mangaid!'
        assert any(args), errmsg
        assert sum([bool(arg) for arg in args]) == 1, errmsg

        self.filename = filename
        self.mangaid = mangaid
        self.plateifu = plateifu

        self.useDB = None

        # convert from mangaid to plateifu
        # print warning if multiple plateifus correspond to one mangaid
        # FIX
        mangaid_to_plateifu = {'1-209232': '8485-1901'}
        if self.mangaid:
            self.plateifu = mangaid_to_plateifu[self.mangaid]

        if config.mode == 'local':
            if self.filename:
                self._openFile()
            else:
                try:
                    # self._getCubeFromPlateIFU()
                    self._getCubeFromMangaID()
                except RuntimeError:
                    # convert from plateifu to filename (sdss_access)
                    self.filename = plateifu_to_filename(self.plateifu)
                    try:
                        self._openFile()
                    except IOError as e:
                        if config.download:
                            # download file via sdsssync
                            # self._openFile()
                            pass
                        else:
                            raise Exception('File does not exist locally') from e
        elif config.mode == 'remote':
            if self.filename:
                raise FileNotFoundError('Cannot open file remotely.')
            else:
                # use API
                pass

    def getSpectrum(self, x=None, y=None, ra=None, dec=None, ext='flux'):
        """Returns the flux spectrum for an spaxel.

        Parameters
        ----------
        x, y : int
            The spaxel coordinates relative to the centre of the cube.

        ra, dec : float
            The coordinates of the spaxel to return. The closest spaxel to
            those coordinates will be returned.

        ext : str
            The extension of the cube to use, either `'flux'`, `'ivar'`, or
            `'mask'`.

        Returns
        -------
        result : np.array
            An array with the spectrum for the input coordinates.

        """

        # Checks that we have the correct set of inputs.
        if x or y:
            assert not ra and not dec, 'Either use (x, y) or (ra, dec)'
            assert x and y, 'Specify both x and y'
            inputMode = 'pix'
        elif ra or dec:
            assert not x and not y, 'Either use (x, y) or (ra, dec)'
            assert ra and dec, 'Specify both ra and dec'
            inputMode = 'sky'
        else:
            raise ValueError('You need to specify (x, y) or (ra, dec)')

        assert isinstance(ext, basestring)
        ext = ext.lower()
        assert ext in ['flux', 'ivar', 'mask'], \
            'ext needs to be either \'flux\', \'ivar\', or \'mask\''

        if not self.useDB:
            cubeExt = self._hdu[ext.upper()]
            if inputMode == 'sky':
                cubeWCS = wcs.WCS(cubeExt.header)
                x, y = cubeWCS.wcs_sky2pix([ra, dec], 1)

            cubeShape = cubeExt.shape
            # TODO: assert that pix coordinates are in cube
            yMid, xMid = cubeShape[1:] / 2.
            return cubeExt.data[:, yMid - y, xMid + x]

        else:

            raise NotImplementedError(
                'getSpectrum from DB not yet implemented')

        # ''' currently: x,y array indices
        # ideally: x,y in arcsecond relative to cube center '''
        # # spectrum = Spectrum(x,y)
        # if config.mode == 'file':
        #     ''' local (client has a file) '''
        #     shape = self.flux.shape
        #     assert len(shape) == 3, 'Dimensions of flux not = 3'
        #     assert x < shape[2] and y < shape[1], 'Input x,y coordinates greater than flux dimensions'
        #     return self.flux[:, y, x]
        # elif config.mode == 'db':
        #     ''' db means local (client) has db '''
        #     return self._cube.spaxels[0].flux
        # else:
        #     ''' local (client) has nothing '''
        #     route = 'cubes/{0}/spectra/x={1}/y={2}/'.format(self.mangaid, x, y)
        #     results = Interaction(route, request_type='get')
        #     return results.getData(astype=np.array)

    def _openFile(self):

        if not os.path.exists(self.filename):
            raise ValueError('filename {0} cannot be found'
                             .format(self.filename))

        self._hdu = fits.open(self.filename)
        self.useDB = False

    @property
    def flux(self):
        if not self.useDB:
            return self._hdu['FLUX'].data
        else:
            return None

    # Switch to _getCubeFromPlateIFU
    def _getCubeFromMangaID(self):
        ''' server-side code '''
        from ..db.database import db
        import sdss.internal.database.utah.mangadb.DataModelClasses as datadb
        session = db.Session()
        self._cube = session.query(datadb.Cube).join(datadb.PipelineInfo, datadb.PipelineVersion).filter(datadb.PipelineVersion.version == 'trunk', datadb.Cube.mangaid == self.mangaid).first()
        print('cube', self._cube, len(self._cube.spaxels))
        self.ifu = self._cube.ifu.name
        self.ra = self._cube.ra
        self.dec = self._cube.dec
        self.plate = self._cube.plate
<|endoftext|>


<commit_before>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import itertools
import numpy as np

from pyspark import SparkContext
from pyspark import since, keyword_only
from pyspark.ml import Estimator, Model
from pyspark.ml.param import Params, Param, TypeConverters
from pyspark.ml.param.shared import HasSeed
from pyspark.ml.wrapper import JavaParams
from pyspark.sql.functions import rand
from pyspark.mllib.common import inherit_doc, _py2java

__all__ = ['ParamGridBuilder', 'CrossValidator', 'CrossValidatorModel', 'TrainValidationSplit',
           'TrainValidationSplitModel']

class ParamGridBuilder(object):
    r"""
    Builder for a param grid used in grid search-based model selection.

    >>> from pyspark.ml.classification import LogisticRegression
    >>> lr = LogisticRegression()
    >>> output = ParamGridBuilder() \
    ...     .baseOn({lr.labelCol: 'l'}) \
    ...     .baseOn([lr.predictionCol, 'p']) \
    ...     .addGrid(lr.regParam, [1.0, 2.0]) \
    ...     .addGrid(lr.maxIter, [1, 5]) \
    ...     .build()
    >>> expected = [
    ...     {lr.regParam: 1.0, lr.maxIter: 1, lr.labelCol: 'l', lr.predictionCol: 'p'},
    ...     {lr.regParam: 2.0, lr.maxIter: 1, lr.labelCol: 'l', lr.predictionCol: 'p'},
    ...     {lr.regParam: 1.0, lr.maxIter: 5, lr.labelCol: 'l', lr.predictionCol: 'p'},
    ...     {lr.regParam: 2.0, lr.maxIter: 5, lr.labelCol: 'l', lr.predictionCol: 'p'}]
    >>> len(output) == len(expected)
    True
    >>> all([m in expected for m in output])
    True

    .. versionadded:: 1.4.0
    """

    def __init__(self):
        self._param_grid = {}

    @since("1.4.0")
    def addGrid(self, param, values):
        """
        Sets the given parameters in this grid to fixed values.
        """
        self._param_grid[param] = values

        return self

    @since("1.4.0")
    def baseOn(self, *args):
        """
        Sets the given parameters in this grid to fixed values.
        Accepts either a parameter dictionary or a list of (parameter, value) pairs.
        """
        if isinstance(args[0], dict):
            self.baseOn(*args[0].items())
        else:
            for (param, value) in args:
                self.addGrid(param, [value])

        return self

    @since("1.4.0")
    def build(self):
        """
        Builds and returns all combinations of parameters specified
        by the param grid.
        """
        keys = self._param_grid.keys()
        grid_values = self._param_grid.values()
        return [dict(zip(keys, prod)) for prod in itertools.product(*grid_values)]

class ValidatorParams(HasSeed):
    """
    Common params for TrainValidationSplit and CrossValidator.
    """

    estimator = Param(Params._dummy(), "estimator", "estimator to be cross-validated")
    estimatorParamMaps = Param(Params._dummy(), "estimatorParamMaps", "estimator param maps")
    evaluator = Param(
        Params._dummy(), "evaluator",
        "evaluator used to select hyper-parameters that maximize the validator metric")

    def setEstimator(self, value):
        """
        Sets the value of :py:attr:`estimator`.
        """
        return self._set(estimator=value)

    def getEstimator(self):
        """
        Gets the value of estimator or its default value.
        """
        return self.getOrDefault(self.estimator)

    def setEstimatorParamMaps(self, value):
        """
        Sets the value of :py:attr:`estimatorParamMaps`.
        """
        return self._set(estimatorParamMaps=value)

    def getEstimatorParamMaps(self):
        """
        Gets the value of estimatorParamMaps or its default value.
        """
        return self.getOrDefault(self.estimatorParamMaps)

    def setEvaluator(self, value):
        """
        Sets the value of :py:attr:`evaluator`.
        """
        return self._set(evaluator=value)

    def getEvaluator(self):
        """
        Gets the value of evaluator or its default value.
        """
        return self.getOrDefault(self.evaluator)

class CrossValidator(Estimator, ValidatorParams):
    """
    K-fold cross validation.

    >>> from pyspark.ml.classification import LogisticRegression
    >>> from pyspark.ml.evaluation import BinaryClassificationEvaluator
    >>> from pyspark.mllib.linalg import Vectors
    >>> dataset = sqlContext.createDataFrame(
    ...     [(Vectors.dense([0.0]), 0.0),
    ...      (Vectors.dense([0.4]), 1.0),
    ...      (Vectors.dense([0.5]), 0.0),
    ...      (Vectors.dense([0.6]), 1.0),
    ...      (Vectors.dense([1.0]), 1.0)] * 10,
    ...     ["features", "label"])
    >>> lr = LogisticRegression()
    >>> grid = ParamGridBuilder().addGrid(lr.maxIter, [0, 1]).build()
    >>> evaluator = BinaryClassificationEvaluator()
    >>> cv = CrossValidator(estimator=lr, estimatorParamMaps=grid, evaluator=evaluator)
    >>> cvModel = cv.fit(dataset)
    >>> evaluator.evaluate(cvModel.transform(dataset))
    0.8333...

    .. versionadded:: 1.4.0
    """

    numFolds = Param(Params._dummy(), "numFolds", "number of folds for cross validation",
                     typeConverter=TypeConverters.toInt)

    @keyword_only
    def __init__(self, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,
                 seed=None):
        """
        __init__(self, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,\
                 seed=None)
        """
        super(CrossValidator, self).__init__()
        self._setDefault(numFolds=3)
        kwargs = self.__init__._input_kwargs
        self._set(**kwargs)

    @keyword_only
    @since("1.4.0")
    def setParams(self, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,
                  seed=None):
        """
        setParams(self, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,\
                  seed=None):
        Sets params for cross validator.
        """
        kwargs = self.setParams._input_kwargs
        return self._set(**kwargs)

    @since("1.4.0")
    def setNumFolds(self, value):
        """
        Sets the value of :py:attr:`numFolds`.
        """
        return self._set(numFolds=value)

    @since("1.4.0")
    def getNumFolds(self):
        """
        Gets the value of numFolds or its default value.
        """
        return self.getOrDefault(self.numFolds)

    def _fit(self, dataset):
        est = self.getOrDefault(self.estimator)
        epm = self.getOrDefault(self.estimatorParamMaps)
        numModels = len(epm)
        eva = self.getOrDefault(self.evaluator)
        nFolds = self.getOrDefault(self.numFolds)
        seed = self.getOrDefault(self.seed)
        h = 1.0 / nFolds
        randCol = self.uid + "_rand"
        df = dataset.select("*", rand(seed).alias(randCol))
        metrics = [0.0] * numModels
        for i in range(nFolds):
            validateLB = i * h
            validateUB = (i + 1) * h
            condition = (df[randCol] >= validateLB) & (df[randCol] < validateUB)
            validation = df.filter(condition)
            train = df.filter(~condition)
            for j in range(numModels):
                model = est.fit(train, epm[j])
                # TODO: duplicate evaluator to take extra params from input
                metric = eva.evaluate(model.transform(validation, epm[j]))
                metrics[j] += metric

        if eva.isLargerBetter():
            bestIndex = np.argmax(metrics)
        else:
            bestIndex = np.argmin(metrics)
        bestModel = est.fit(dataset, epm[bestIndex])
        return self._copyValues(CrossValidatorModel(bestModel, metrics))

    @since("1.4.0")
    def copy(self, extra=None):
        """
        Creates a copy of this instance with a randomly generated uid
        and some extra params. This copies creates a deep copy of
        the embedded paramMap, and copies the embedded and extra parameters over.

        :param extra: Extra parameters to copy to the new instance
        :return: Copy of this instance
        """
        if extra is None:
            extra = dict()
        newCV = Params.copy(self, extra)
        if self.isSet(self.estimator):
            newCV.setEstimator(self.getEstimator().copy(extra))
        # estimatorParamMaps remain the same
        if self.isSet(self.evaluator):
            newCV.setEvaluator(self.getEvaluator().copy(extra))
        return newCV

class CrossValidatorModel(Model, ValidatorParams):
    """
    Model from k-fold cross validation.

    .. versionadded:: 1.4.0
    """

    def __init__(self, bestModel, avgMetrics=[]):
        super(CrossValidatorModel, self).__init__()
        #: best model from cross validation
        self.bestModel = bestModel
        self.avgMetrics = avgMetrics

    def _transform(self, dataset):
        return self.bestModel.transform(dataset)

    @since("1.4.0")
    def copy(self, extra=None):
        """
        Creates a copy of this instance with a randomly generated uid
        and some extra params. This copies the underlying bestModel,
        creates a deep copy of the embedded paramMap, and
        copies the embedded and extra parameters over.

        :param extra: Extra parameters to copy to the new instance
        :return: Copy of this instance
        """
        if extra is None:
            extra = dict()
        bestModel = self.bestModel.copy(extra)
        avgMetrics = self.avgMetrics
        return CrossValidatorModel(bestModel, avgMetrics)

class TrainValidationSplit(Estimator, ValidatorParams):
    """
    Train-Validation-Split.

    >>> from pyspark.ml.classification import LogisticRegression
    >>> from pyspark.ml.evaluation import BinaryClassificationEvaluator
    >>> from pyspark.mllib.linalg import Vectors
    >>> dataset = sqlContext.createDataFrame(
    ...     [(Vectors.dense([0.0]), 0.0),
    ...      (Vectors.dense([0.4]), 1.0),
    ...      (Vectors.dense([0.5]), 0.0),
    ...      (Vectors.dense([0.6]), 1.0),
    ...      (Vectors.dense([1.0]), 1.0)] * 10,
    ...     ["features", "label"])
    >>> lr = LogisticRegression()
    >>> grid = ParamGridBuilder().addGrid(lr.maxIter, [0, 1]).build()
    >>> evaluator = BinaryClassificationEvaluator()
    >>> tvs = TrainValidationSplit(estimator=lr, estimatorParamMaps=grid, evaluator=evaluator)
    >>> tvsModel = tvs.fit(dataset)
    >>> evaluator.evaluate(tvsModel.transform(dataset))
    0.8333...

    .. versionadded:: 2.0.0
    """

    trainRatio = Param(Params._dummy(), "trainRatio", "Param for ratio between train and\
     validation data. Must be between 0 and 1.", typeConverter=TypeConverters.toFloat)

    @keyword_only
    def __init__(self, estimator=None, estimatorParamMaps=None, evaluator=None, trainRatio=0.75,
                 seed=None):
        """
        __init__(self, estimator=None, estimatorParamMaps=None, evaluator=None, trainRatio=0.75,\
                 seed=None)
        """
        super(TrainValidationSplit, self).__init__()
        self._setDefault(trainRatio=0.75)
        kwargs = self.__init__._input_kwargs
        self._set(**kwargs)

    @since("2.0.0")
    @keyword_only
    def setParams(self, estimator=None, estimatorParamMaps=None, evaluator=None, trainRatio=0.75,
                  seed=None):
        """
        setParams(self, estimator=None, estimatorParamMaps=None, evaluator=None, trainRatio=0.75,\
                  seed=None):
        Sets params for the train validation split.
        """
        kwargs = self.setParams._input_kwargs
        return self._set(**kwargs)

    @since("2.0.0")
    def setTrainRatio(self, value):
        """
        Sets the value of :py:attr:`trainRatio`.
        """
        return self._set(trainRatio=value)

    @since("2.0.0")
    def getTrainRatio(self):
        """
        Gets the value of trainRatio or its default value.
        """
        return self.getOrDefault(self.trainRatio)

    def _fit(self, dataset):
        est = self.getOrDefault(self.estimator)
        epm = self.getOrDefault(self.estimatorParamMaps)
        numModels = len(epm)
        eva = self.getOrDefault(self.evaluator)
        tRatio = self.getOrDefault(self.trainRatio)
        seed = self.getOrDefault(self.seed)
        randCol = self.uid + "_rand"
        df = dataset.select("*", rand(seed).alias(randCol))
        metrics = np.zeros(numModels)
        condition = (df[randCol] >= tRatio)
        validation = df.filter(condition)
        train = df.filter(~condition)
        for j in range(numModels):
            model = est.fit(train, epm[j])
            metric = eva.evaluate(model.transform(validation, epm[j]))
            metrics[j] += metric
        if eva.isLargerBetter():
            bestIndex = np.argmax(metrics)
        else:
            bestIndex = np.argmin(metrics)
        bestModel = est.fit(dataset, epm[bestIndex])
        return self._copyValues(TrainValidationSplitModel(bestModel))

    @since("2.0.0")
    def copy(self, extra=None):
        """
        Creates a copy of this instance with a randomly generated uid
        and some extra params. This copies creates a deep copy of
        the embedded paramMap, and copies the embedded and extra parameters over.

        :param extra: Extra parameters to copy to the new instance
        :return: Copy of this instance
        """
        if extra is None:
            extra = dict()
        newTVS = Params.copy(self, extra)
        if self.isSet(self.estimator):
            newTVS.setEstimator(self.getEstimator().copy(extra))
        # estimatorParamMaps remain the same
        if self.isSet(self.evaluator):
            newTVS.setEvaluator(self.getEvaluator().copy(extra))
        return newTVS

class TrainValidationSplitModel(Model, ValidatorParams):
    """
    Model from train validation split.

    .. versionadded:: 2.0.0
    """

    def __init__(self, bestModel):
        super(TrainValidationSplitModel, self).__init__()
        #: best model from cross validation
        self.bestModel = bestModel

    def _transform(self, dataset):
        return self.bestModel.transform(dataset)

    @since("2.0.0")
    def copy(self, extra=None):
        """
        Creates a copy of this instance with a randomly generated uid
        and some extra params. This copies the underlying bestModel,
        creates a deep copy of the embedded paramMap, and
        copies the embedded and extra parameters over.

        :param extra: Extra parameters to copy to the new instance
        :return: Copy of this instance
        """
        if extra is None:
            extra = dict()
        return TrainValidationSplitModel(self.bestModel.copy(extra))

if __name__ == "__main__":
    import doctest

    from pyspark.context import SparkContext
    from pyspark.sql import SQLContext
    globs = globals().copy()

    # The small batch size here ensures that we see multiple batches,
    # even in these small test examples:
    sc = SparkContext("local[2]", "ml.tuning tests")
    sqlContext = SQLContext(sc)
    globs['sc'] = sc
    globs['sqlContext'] = sqlContext
    (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
    sc.stop()
    if failure_count:
        exit(-1)
<commit_msg>[SPARK-15195][PYSPARK][DOCS] Update ml.tuning PyDocs<commit_after>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import itertools
import numpy as np

from pyspark import SparkContext
from pyspark import since, keyword_only
from pyspark.ml import Estimator, Model
from pyspark.ml.param import Params, Param, TypeConverters
from pyspark.ml.param.shared import HasSeed
from pyspark.ml.wrapper import JavaParams
from pyspark.sql.functions import rand
from pyspark.mllib.common import inherit_doc, _py2java

__all__ = ['ParamGridBuilder', 'CrossValidator', 'CrossValidatorModel', 'TrainValidationSplit',
           'TrainValidationSplitModel']

class ParamGridBuilder(object):
    r"""
    .. note:: Experimental

    Builder for a param grid used in grid search-based model selection.

    >>> from pyspark.ml.classification import LogisticRegression
    >>> lr = LogisticRegression()
    >>> output = ParamGridBuilder() \
    ...     .baseOn({lr.labelCol: 'l'}) \
    ...     .baseOn([lr.predictionCol, 'p']) \
    ...     .addGrid(lr.regParam, [1.0, 2.0]) \
    ...     .addGrid(lr.maxIter, [1, 5]) \
    ...     .build()
    >>> expected = [
    ...     {lr.regParam: 1.0, lr.maxIter: 1, lr.labelCol: 'l', lr.predictionCol: 'p'},
    ...     {lr.regParam: 2.0, lr.maxIter: 1, lr.labelCol: 'l', lr.predictionCol: 'p'},
    ...     {lr.regParam: 1.0, lr.maxIter: 5, lr.labelCol: 'l', lr.predictionCol: 'p'},
    ...     {lr.regParam: 2.0, lr.maxIter: 5, lr.labelCol: 'l', lr.predictionCol: 'p'}]
    >>> len(output) == len(expected)
    True
    >>> all([m in expected for m in output])
    True

    .. versionadded:: 1.4.0
    """

    def __init__(self):
        self._param_grid = {}

    @since("1.4.0")
    def addGrid(self, param, values):
        """
        Sets the given parameters in this grid to fixed values.
        """
        self._param_grid[param] = values

        return self

    @since("1.4.0")
    def baseOn(self, *args):
        """
        Sets the given parameters in this grid to fixed values.
        Accepts either a parameter dictionary or a list of (parameter, value) pairs.
        """
        if isinstance(args[0], dict):
            self.baseOn(*args[0].items())
        else:
            for (param, value) in args:
                self.addGrid(param, [value])

        return self

    @since("1.4.0")
    def build(self):
        """
        Builds and returns all combinations of parameters specified
        by the param grid.
        """
        keys = self._param_grid.keys()
        grid_values = self._param_grid.values()
        return [dict(zip(keys, prod)) for prod in itertools.product(*grid_values)]

class ValidatorParams(HasSeed):
    """
    Common params for TrainValidationSplit and CrossValidator.
    """

    estimator = Param(Params._dummy(), "estimator", "estimator to be cross-validated")
    estimatorParamMaps = Param(Params._dummy(), "estimatorParamMaps", "estimator param maps")
    evaluator = Param(
        Params._dummy(), "evaluator",
        "evaluator used to select hyper-parameters that maximize the validator metric")

    def setEstimator(self, value):
        """
        Sets the value of :py:attr:`estimator`.
        """
        return self._set(estimator=value)

    def getEstimator(self):
        """
        Gets the value of estimator or its default value.
        """
        return self.getOrDefault(self.estimator)

    def setEstimatorParamMaps(self, value):
        """
        Sets the value of :py:attr:`estimatorParamMaps`.
        """
        return self._set(estimatorParamMaps=value)

    def getEstimatorParamMaps(self):
        """
        Gets the value of estimatorParamMaps or its default value.
        """
        return self.getOrDefault(self.estimatorParamMaps)

    def setEvaluator(self, value):
        """
        Sets the value of :py:attr:`evaluator`.
        """
        return self._set(evaluator=value)

    def getEvaluator(self):
        """
        Gets the value of evaluator or its default value.
        """
        return self.getOrDefault(self.evaluator)

class CrossValidator(Estimator, ValidatorParams):
    """
    .. note:: Experimental

    K-fold cross validation.

    >>> from pyspark.ml.classification import LogisticRegression
    >>> from pyspark.ml.evaluation import BinaryClassificationEvaluator
    >>> from pyspark.mllib.linalg import Vectors
    >>> dataset = sqlContext.createDataFrame(
    ...     [(Vectors.dense([0.0]), 0.0),
    ...      (Vectors.dense([0.4]), 1.0),
    ...      (Vectors.dense([0.5]), 0.0),
    ...      (Vectors.dense([0.6]), 1.0),
    ...      (Vectors.dense([1.0]), 1.0)] * 10,
    ...     ["features", "label"])
    >>> lr = LogisticRegression()
    >>> grid = ParamGridBuilder().addGrid(lr.maxIter, [0, 1]).build()
    >>> evaluator = BinaryClassificationEvaluator()
    >>> cv = CrossValidator(estimator=lr, estimatorParamMaps=grid, evaluator=evaluator)
    >>> cvModel = cv.fit(dataset)
    >>> evaluator.evaluate(cvModel.transform(dataset))
    0.8333...

    .. versionadded:: 1.4.0
    """

    numFolds = Param(Params._dummy(), "numFolds", "number of folds for cross validation",
                     typeConverter=TypeConverters.toInt)

    @keyword_only
    def __init__(self, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,
                 seed=None):
        """
        __init__(self, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,\
                 seed=None)
        """
        super(CrossValidator, self).__init__()
        self._setDefault(numFolds=3)
        kwargs = self.__init__._input_kwargs
        self._set(**kwargs)

    @keyword_only
    @since("1.4.0")
    def setParams(self, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,
                  seed=None):
        """
        setParams(self, estimator=None, estimatorParamMaps=None, evaluator=None, numFolds=3,\
                  seed=None):
        Sets params for cross validator.
        """
        kwargs = self.setParams._input_kwargs
        return self._set(**kwargs)

    @since("1.4.0")
    def setNumFolds(self, value):
        """
        Sets the value of :py:attr:`numFolds`.
        """
        return self._set(numFolds=value)

    @since("1.4.0")
    def getNumFolds(self):
        """
        Gets the value of numFolds or its default value.
        """
        return self.getOrDefault(self.numFolds)

    def _fit(self, dataset):
        est = self.getOrDefault(self.estimator)
        epm = self.getOrDefault(self.estimatorParamMaps)
        numModels = len(epm)
        eva = self.getOrDefault(self.evaluator)
        nFolds = self.getOrDefault(self.numFolds)
        seed = self.getOrDefault(self.seed)
        h = 1.0 / nFolds
        randCol = self.uid + "_rand"
        df = dataset.select("*", rand(seed).alias(randCol))
        metrics = [0.0] * numModels
        for i in range(nFolds):
            validateLB = i * h
            validateUB = (i + 1) * h
            condition = (df[randCol] >= validateLB) & (df[randCol] < validateUB)
            validation = df.filter(condition)
            train = df.filter(~condition)
            for j in range(numModels):
                model = est.fit(train, epm[j])
                # TODO: duplicate evaluator to take extra params from input
                metric = eva.evaluate(model.transform(validation, epm[j]))
                metrics[j] += metric

        if eva.isLargerBetter():
            bestIndex = np.argmax(metrics)
        else:
            bestIndex = np.argmin(metrics)
        bestModel = est.fit(dataset, epm[bestIndex])
        return self._copyValues(CrossValidatorModel(bestModel, metrics))

    @since("1.4.0")
    def copy(self, extra=None):
        """
        Creates a copy of this instance with a randomly generated uid
        and some extra params. This copies creates a deep copy of
        the embedded paramMap, and copies the embedded and extra parameters over.

        :param extra: Extra parameters to copy to the new instance
        :return: Copy of this instance
        """
        if extra is None:
            extra = dict()
        newCV = Params.copy(self, extra)
        if self.isSet(self.estimator):
            newCV.setEstimator(self.getEstimator().copy(extra))
        # estimatorParamMaps remain the same
        if self.isSet(self.evaluator):
            newCV.setEvaluator(self.getEvaluator().copy(extra))
        return newCV

class CrossValidatorModel(Model, ValidatorParams):
    """
    .. note:: Experimental

    Model from k-fold cross validation.

    .. versionadded:: 1.4.0
    """

    def __init__(self, bestModel, avgMetrics=[]):
        super(CrossValidatorModel, self).__init__()
        #: best model from cross validation
        self.bestModel = bestModel
        #: Average cross-validation metrics for each paramMap in
        #: CrossValidator.estimatorParamMaps, in the corresponding order.
        self.avgMetrics = avgMetrics

    def _transform(self, dataset):
        return self.bestModel.transform(dataset)

    @since("1.4.0")
    def copy(self, extra=None):
        """
        Creates a copy of this instance with a randomly generated uid
        and some extra params. This copies the underlying bestModel,
        creates a deep copy of the embedded paramMap, and
        copies the embedded and extra parameters over.

        :param extra: Extra parameters to copy to the new instance
        :return: Copy of this instance
        """
        if extra is None:
            extra = dict()
        bestModel = self.bestModel.copy(extra)
        avgMetrics = self.avgMetrics
        return CrossValidatorModel(bestModel, avgMetrics)

class TrainValidationSplit(Estimator, ValidatorParams):
    """
    .. note:: Experimental

    Validation for hyper-parameter tuning. Randomly splits the input dataset into train and
    validation sets, and uses evaluation metric on the validation set to select the best model.
    Similar to :class:`CrossValidator`, but only splits the set once.

    >>> from pyspark.ml.classification import LogisticRegression
    >>> from pyspark.ml.evaluation import BinaryClassificationEvaluator
    >>> from pyspark.mllib.linalg import Vectors
    >>> dataset = sqlContext.createDataFrame(
    ...     [(Vectors.dense([0.0]), 0.0),
    ...      (Vectors.dense([0.4]), 1.0),
    ...      (Vectors.dense([0.5]), 0.0),
    ...      (Vectors.dense([0.6]), 1.0),
    ...      (Vectors.dense([1.0]), 1.0)] * 10,
    ...     ["features", "label"])
    >>> lr = LogisticRegression()
    >>> grid = ParamGridBuilder().addGrid(lr.maxIter, [0, 1]).build()
    >>> evaluator = BinaryClassificationEvaluator()
    >>> tvs = TrainValidationSplit(estimator=lr, estimatorParamMaps=grid, evaluator=evaluator)
    >>> tvsModel = tvs.fit(dataset)
    >>> evaluator.evaluate(tvsModel.transform(dataset))
    0.8333...

    .. versionadded:: 2.0.0
    """

    trainRatio = Param(Params._dummy(), "trainRatio", "Param for ratio between train and\
     validation data. Must be between 0 and 1.", typeConverter=TypeConverters.toFloat)

    @keyword_only
    def __init__(self, estimator=None, estimatorParamMaps=None, evaluator=None, trainRatio=0.75,
                 seed=None):
        """
        __init__(self, estimator=None, estimatorParamMaps=None, evaluator=None, trainRatio=0.75,\
                 seed=None)
        """
        super(TrainValidationSplit, self).__init__()
        self._setDefault(trainRatio=0.75)
        kwargs = self.__init__._input_kwargs
        self._set(**kwargs)

    @since("2.0.0")
    @keyword_only
    def setParams(self, estimator=None, estimatorParamMaps=None, evaluator=None, trainRatio=0.75,
                  seed=None):
        """
        setParams(self, estimator=None, estimatorParamMaps=None, evaluator=None, trainRatio=0.75,\
                  seed=None):
        Sets params for the train validation split.
        """
        kwargs = self.setParams._input_kwargs
        return self._set(**kwargs)

    @since("2.0.0")
    def setTrainRatio(self, value):
        """
        Sets the value of :py:attr:`trainRatio`.
        """
        return self._set(trainRatio=value)

    @since("2.0.0")
    def getTrainRatio(self):
        """
        Gets the value of trainRatio or its default value.
        """
        return self.getOrDefault(self.trainRatio)

    def _fit(self, dataset):
        est = self.getOrDefault(self.estimator)
        epm = self.getOrDefault(self.estimatorParamMaps)
        numModels = len(epm)
        eva = self.getOrDefault(self.evaluator)
        tRatio = self.getOrDefault(self.trainRatio)
        seed = self.getOrDefault(self.seed)
        randCol = self.uid + "_rand"
        df = dataset.select("*", rand(seed).alias(randCol))
        metrics = np.zeros(numModels)
        condition = (df[randCol] >= tRatio)
        validation = df.filter(condition)
        train = df.filter(~condition)
        for j in range(numModels):
            model = est.fit(train, epm[j])
            metric = eva.evaluate(model.transform(validation, epm[j]))
            metrics[j] += metric
        if eva.isLargerBetter():
            bestIndex = np.argmax(metrics)
        else:
            bestIndex = np.argmin(metrics)
        bestModel = est.fit(dataset, epm[bestIndex])
        return self._copyValues(TrainValidationSplitModel(bestModel))

    @since("2.0.0")
    def copy(self, extra=None):
        """
        Creates a copy of this instance with a randomly generated uid
        and some extra params. This copies creates a deep copy of
        the embedded paramMap, and copies the embedded and extra parameters over.

        :param extra: Extra parameters to copy to the new instance
        :return: Copy of this instance
        """
        if extra is None:
            extra = dict()
        newTVS = Params.copy(self, extra)
        if self.isSet(self.estimator):
            newTVS.setEstimator(self.getEstimator().copy(extra))
        # estimatorParamMaps remain the same
        if self.isSet(self.evaluator):
            newTVS.setEvaluator(self.getEvaluator().copy(extra))
        return newTVS

class TrainValidationSplitModel(Model, ValidatorParams):
    """
    .. note:: Experimental

    Model from train validation split.

    .. versionadded:: 2.0.0
    """

    def __init__(self, bestModel):
        super(TrainValidationSplitModel, self).__init__()
        #: best model from cross validation
        self.bestModel = bestModel

    def _transform(self, dataset):
        return self.bestModel.transform(dataset)

    @since("2.0.0")
    def copy(self, extra=None):
        """
        Creates a copy of this instance with a randomly generated uid
        and some extra params. This copies the underlying bestModel,
        creates a deep copy of the embedded paramMap, and
        copies the embedded and extra parameters over.

        :param extra: Extra parameters to copy to the new instance
        :return: Copy of this instance
        """
        if extra is None:
            extra = dict()
        return TrainValidationSplitModel(self.bestModel.copy(extra))

if __name__ == "__main__":
    import doctest

    from pyspark.context import SparkContext
    from pyspark.sql import SQLContext
    globs = globals().copy()

    # The small batch size here ensures that we see multiple batches,
    # even in these small test examples:
    sc = SparkContext("local[2]", "ml.tuning tests")
    sqlContext = SQLContext(sc)
    globs['sc'] = sc
    globs['sqlContext'] = sqlContext
    (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
    sc.stop()
    if failure_count:
        exit(-1)
<|endoftext|>


<commit_before>"""
Modulel loads our flat file data into the Postgres database. It will rebuild the
database of choice with sample or real data by first deleting any data in the
repository, and then re-load the data into the repository.

If you want to rebuild the data with actual data (i.e. what is in manifest.csv
instead of manifest.csv), run the same command without ‘sample’ at the end.

Notes:
 - manifest.csv has every flat file that needs to be loaded (i.e. CSV's we have
 downloaded).
 - other scripts can get data that is available from APIs, so manifest won't
 reflect all the data we are including.
 - meta.json provides meta information about our *SQL* tables. Note that
 because multiple CSV's can go into the same table (i.e. two different versions
 of the same data), there will be more rows in the CSV than there are 'tables'
 in the json.
"""

import sys
import os
import logging
import json
import csv

#Needed to make relative package imports when running this file as a script (i.e. for testing purposes).
#Read why here: https://www.blog.pythonlibrary.org/2016/03/01/python-101-all-about-imports/

python_filepath = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                               os.pardir))
sys.path.append(python_filepath)

# TODO: remove redundant imports
from housinginsights.ingestion.DataReader import DataReader
from housinginsights.tools import dbtools

from housinginsights.ingestion import DataReader, ManifestReader
from housinginsights.ingestion import CSVWriter, DataReader
from housinginsights.ingestion import HISql, TableWritingError
from housinginsights.ingestion import functions as ingestionfunctions

# configuration
# See /logs/example-logging.py for usage examples
logging_path = os.path.abspath(os.path.join(python_filepath, "logs"))
logging_filename = os.path.abspath(os.path.join(logging_path, "ingestion.log"))
logging.basicConfig(filename=logging_filename, level=logging.DEBUG)

# Pushes everything from the logger to the command line output as well.
logging.getLogger().addHandler(logging.StreamHandler())

#############################
# FUNCTIONS
#############################

# complete, tests not written
def drop_tables(database_choice):
    """
    Used to rebuild the database by first dropping all tables before running
    main()
    """
    engine = dbtools.get_database_engine(database_choice)
    db_conn = engine.connect()
    query_result = db_conn.execute("DROP SCHEMA public CASCADE;CREATE SCHEMA public;")

def main(database_choice, meta_path, manifest_path, keep_temp_files=True):
    """
    Main routine, calls all the other ones in this file as needed.

    Big picture:
        - Use manifest.csv to find out all the files we want to load
        - Compare current manifest.csv to the sql database manifest table, which
         reflects manifest.csv as of the last time this script was run, and
         tells us whether or not the file has already been loaded in this
         database.
        - Use meta.json to make sure the CSV file has all the fields we expect
        it to, and to decide the data type of each field.
        - Load the csv into the database

    If there is an error loading a file (either flagged error from fields not
    matching, or parsing error if data type isn't right):
    - skip loading this file,
    - report the error to SQL using update_sql_manifest(status="error")
    - use logging.warning() to write the specific error encountered to the
    log file
    - at the end of loading print an error message to the console
    """

    # load given meta.json and manifest.csv files into memory
    meta = ingestionfunctions.load_meta_data(meta_path)
    manifest = ManifestReader(manifest_path)

    # load given database choice and check/create sql manifest
    engine = dbtools.get_database_engine(database_choice)
    sql_manifest_exists = \
        ingestionfunctions.check_or_create_sql_manifest(engine=engine)
    logging.info("sql_manifest_exists: {}".format(sql_manifest_exists))

    #TODO should this be moved into the __init__ of ManifestReader? Do we ever want to use ManifestReader if it has duplicate rows?
    if not manifest.has_unique_ids():
        raise ValueError('Manifest has duplicate unique_data_id!')

    # Iterate through each row in the manifest then clean and validate. If it
    # passes validation, then write to temp psv file for loading into database.
    for manifest_row in manifest:
        #Incompletely filled out rows in the manifest can break the other code
        # TODO: figure out a way to flag this issue early in loading of manifest

        # only clean and validate data files flagged for use in database
        if manifest_row['include_flag'] == 'use':
            logging.info("{}: preparing to load row {} from the manifest".
                         format(manifest_row['unique_data_id'],len(manifest)))

            temp_filepath = os.path.abspath(os.path.join(logging_path,
                                                 'temp_{}.psv'.format(
                                                 manifest_row['unique_data_id'])
                                                    ))
            # prepare csv reader and writer objects
            csv_reader = DataReader(meta=meta, manifest_row=manifest_row,
                                    load_from="file")
            csv_writer = CSVWriter(meta=meta, manifest_row=manifest_row,
                                   filename=temp_filepath)

            # prepare objects for interfacing with database and then get
            # the equivalent manifest row from the database
            sql_interface = HISql(meta=meta, manifest_row=manifest_row,
                                  engine=engine, filename=temp_filepath)
            sql_manifest_row = sql_interface.get_sql_manifest_row()

            #Assign an appropriate testing cleaner
            tablename = manifest_row['destination_table']
            cleaner_class_name = meta[tablename]['cleaner']
            cleaner = ingestionfunctions.get_cleaner_from_name(
                                        meta=meta,
                                        manifest_row=manifest_row,
                                        name=cleaner_class_name)

            # Identify fields that exist in meta.json but not CSV
            # so we can add them to the row as it is cleaned and loaded.
            meta_only_fields = {}
            for field in meta[tablename]['fields']:
                if field['source_name'] not in csv_reader.keys:
                    # adds 'sql_name',None as key,value pairs in dict
                    meta_only_fields[field['sql_name']] = None

            #clean the file and save the output to a local pipe-delimited file
            if csv_reader.should_file_be_loaded(sql_manifest_row=sql_manifest_row):
                print("  Cleaning...")
                for idx, data_row in enumerate(csv_reader):
                    data_row.update(meta_only_fields) # insert other field dict
                    clean_data_row = cleaner.clean(data_row, idx)
                    if clean_data_row != None:
                        csv_writer.write(clean_data_row)

                csv_writer.close()
                print("  Loading...")

                #Decide whether to append or replace the table
                if meta[tablename]["replace_table"] == True:
                    logging.info("  replacing existing table")
                    sql_interface.drop_table()

                #Appends to table; if dropped, it recreates
                sql_interface.create_table_if_necessary()
                try:
                    sql_interface.write_file_to_sql()
                except TableWritingError:
                    #TODO tell user total count of errors.
                    #currently write_file_to_sql() just writes in log that file failed
                    pass
                if keep_temp_files == False:
                    csv_writer.remove_file()

if __name__ == '__main__':

    #local, real data is the default
    database_choice = 'local_database'
    scripts_path = os.path.abspath(os.path.join(python_filepath, 'scripts'))
    meta_path = os.path.abspath(os.path.join(scripts_path, 'meta.json'))
    manifest_path = os.path.abspath(os.path.join(scripts_path, 'manifest.csv'))

    if 'sample' in sys.argv:
        meta_path = os.path.abspath(os.path.join(scripts_path,
                                                 'meta_sample.json'))
        manifest_path = os.path.abspath(
            os.path.join(scripts_path, 'manifest_sample.csv'))

    if 'docker' in sys.argv:
        database_choice = 'docker_database'

    if 'remote' in sys.argv:
        database_choice = 'remote_database'
        #Don't want sample data in the remote database
        meta_path = os.path.abspath(os.path.join(scripts_path, 'meta.json'))
        manifest_path = os.path.abspath(
            os.path.join(scripts_path, 'manifest.csv'))

    if 'rebuild' in sys.argv:
        drop_tables(database_choice)

    main(database_choice, meta_path, manifest_path, keep_temp_files = True)
<commit_msg>status: class shell created - defined 'init' and 'drop_tables' methods. Cleaned up if run from main section to intialize the LoadData class and then drop table and load data accordingly. Need to complete load_data method and its dependent helper functions.<commit_after>"""
Modulel loads our flat file data into the Postgres database. It will rebuild the
database of choice with sample or real data by first deleting any data in the
repository, and then re-load the data into the repository.

If you want to rebuild the data with actual data (i.e. what is in manifest.csv
instead of manifest.csv), run the same command without ‘sample’ at the end.

Notes:
 - manifest.csv has every flat file that needs to be loaded (i.e. CSV's we have
 downloaded).
 - other scripts can get data that is available from APIs, so manifest won't
 reflect all the data we are including.
 - meta.json provides meta information about our *SQL* tables. Note that
 because multiple CSV's can go into the same table (i.e. two different versions
 of the same data), there will be more rows in the CSV than there are 'tables'
 in the json.
"""

import sys
import os
import logging

# Needed to make relative package imports when running this file as a script
# (i.e. for testing purposes).
# Read why here: https://www.blog.pythonlibrary.org/2016/03/01/python-101-all
# -about-imports/

python_filepath = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                               os.pardir))
sys.path.append(python_filepath)

from housinginsights.tools import dbtools

from housinginsights.ingestion import DataReader, ManifestReader
from housinginsights.ingestion import CSVWriter, DataReader
from housinginsights.ingestion import HISql, TableWritingError
from housinginsights.ingestion import functions as ingestionfunctions

# configuration: see /logs/example-logging.py for usage examples
logging_path = os.path.abspath(os.path.join(python_filepath, "logs"))
logging_filename = os.path.abspath(os.path.join(logging_path, "ingestion.log"))
logging.basicConfig(filename=logging_filename, level=logging.DEBUG)

# Pushes everything from the logger to the command line output as well.
logging.getLogger().addHandler(logging.StreamHandler())

# TODO: Class refactoring
class LoadData(object):

    def __init__(self, database_choice=None, meta_path=None,
                 manifest_path=None, keep_temp_files=True):
        """
        Initializes the class with optional arguments. The default behaviour
        is to load the local database with data tracked from meta.json
        and manifest.csv within the 'python/scripts' folder.

        :param database_choice: choice of 'local_database',
        'docker_database', and 'remote_database'
        :param meta_path: the path of the meta.json to be used
        :param manifest_path: the path of the manifest_path.csv to be used
        :param keep_temp_files: if True, temp clean pipe-delimited files will be
        archived in the 'python/logs' folder
        """

        # load defaults in no arguments passed
        scripts_path = os.path.abspath(os.path.join(python_filepath, 'scripts'))
        if database_choice is None:
            database_choice = 'local_database'
        if meta_path is None:
            meta_path = os.path.abspath(os.path.join(scripts_path, 'meta.json'))
        if manifest_path is None:
            manifest_path = os.path.abspath(os.path.join(scripts_path,
                                                      'manifest.csv'))
        self.keep_temp_files = keep_temp_files

        # load given meta.json and manifest.csv files into memory
        self.meta = ingestionfunctions.load_meta_data(meta_path)
        self.manifest = ManifestReader(manifest_path)

        # load given database choice and check/create sql manifest
        self.engine = dbtools.get_database_engine(database_choice)
        # sql_manifest_exists = \
        #     ingestionfunctions.check_or_create_sql_manifest(engine=engine)
        # logging.info("sql_manifest_exists: {}".format(sql_manifest_exists))

    def drop_tables(self):
        """
        Returns the outcome of dropping all the tables from the
        database_choice and then rebuilding.
        """
        db_conn = self.engine.connect()
        return db_conn.execute(
            "DROP SCHEMA public CASCADE;CREATE SCHEMA public;")

    def load_data(self):
        """
        Start the process for loading the data into the database.

        :return: None
        """

        # TODO: should this be moved into the __init__ of ManifestReader? Do we
        # TODO: ever want to use ManifestReader if it has duplicate rows?
        if not self.manifest.has_unique_ids():
            raise ValueError('Manifest has duplicate unique_data_id!')

    # - Use manifest.csv to find out all the files we want to load
    def _
    # - Compare current manifest.csv to the sql database manifest table, which
    #  reflects manifest.csv as of the last time this script was run, and
    #  tells us whether or not the file has already been loaded in this
    #  database.
    # - Use meta.json to make sure the CSV file has all the fields we expect
    # it to, and to decide the data type of each field.
    # - Load the csv into the database

#############################
# FUNCTIONS
#############################

# # complete, tests not written
# def drop_tables(database_choice):
#     """
#     Used to rebuild the database by first dropping all tables before running
#     main()
#     """
#     engine = dbtools.get_database_engine(database_choice)
#     db_conn = engine.connect()
#     query_result = db_conn.execute("DROP SCHEMA public CASCADE;CREATE SCHEMA public;")

def main(database_choice, meta_path, manifest_path, keep_temp_files=True):
    """
    Main routine, calls all the other ones in this file as needed.

    Big picture:
        - Use manifest.csv to find out all the files we want to load
        - Compare current manifest.csv to the sql database manifest table, which
         reflects manifest.csv as of the last time this script was run, and
         tells us whether or not the file has already been loaded in this
         database.
        - Use meta.json to make sure the CSV file has all the fields we expect
        it to, and to decide the data type of each field.
        - Load the csv into the database

    If there is an error loading a file (either flagged error from fields not
    matching, or parsing error if data type isn't right):
    - skip loading this file,
    - report the error to SQL using update_sql_manifest(status="error")
    - use logging.warning() to write the specific error encountered to the
    log file
    - at the end of loading print an error message to the console
    """

    # # load given meta.json and manifest.csv files into memory
    # meta = ingestionfunctions.load_meta_data(meta_path)
    # manifest = ManifestReader(manifest_path)
    #
    # # load given database choice and check/create sql manifest
    # engine = dbtools.get_database_engine(database_choice)
    sql_manifest_exists = \
        ingestionfunctions.check_or_create_sql_manifest(engine=engine)
    logging.info("sql_manifest_exists: {}".format(sql_manifest_exists))

    # TODO: should this be moved into the __init__ of ManifestReader? Do we
    # TODO: ever want to use ManifestReader if it has duplicate rows?
    if not manifest.has_unique_ids():
        raise ValueError('Manifest has duplicate unique_data_id!')

    # Iterate through each row in the manifest then clean and validate. If it
    # passes validation, then write to temp psv file for loading into database.
    for manifest_row in manifest:
        #Incompletely filled out rows in the manifest can break the other code
        # TODO: figure out a way to flag this issue early in loading of manifest

        # only clean and validate data files flagged for use in database
        if manifest_row['include_flag'] == 'use':
            logging.info("{}: preparing to load row {} from the manifest".
                         format(manifest_row['unique_data_id'],len(manifest)))

            temp_filepath = os.path.abspath(os.path.join(logging_path,
                                                 'temp_{}.psv'.format(
                                                 manifest_row['unique_data_id'])
                                                    ))
            # prepare csv reader and writer objects
            csv_reader = DataReader(meta=meta, manifest_row=manifest_row,
                                    load_from="file")
            csv_writer = CSVWriter(meta=meta, manifest_row=manifest_row,
                                   filename=temp_filepath)

            # prepare objects for interfacing with database and then get
            # the equivalent manifest row from the database
            sql_interface = HISql(meta=meta, manifest_row=manifest_row,
                                  engine=engine, filename=temp_filepath)
            sql_manifest_row = sql_interface.get_sql_manifest_row()

            #Assign an appropriate testing cleaner
            tablename = manifest_row['destination_table']
            cleaner_class_name = meta[tablename]['cleaner']
            cleaner = ingestionfunctions.get_cleaner_from_name(
                                        meta=meta,
                                        manifest_row=manifest_row,
                                        name=cleaner_class_name)

            # Identify fields that exist in meta.json but not CSV
            # so we can add them to the row as it is cleaned and loaded.
            meta_only_fields = {}
            for field in meta[tablename]['fields']:
                if field['source_name'] not in csv_reader.keys:
                    # adds 'sql_name',None as key,value pairs in dict
                    meta_only_fields[field['sql_name']] = None

            #clean the file and save the output to a local pipe-delimited file
            if csv_reader.should_file_be_loaded(sql_manifest_row=sql_manifest_row):
                print("  Cleaning...")
                for idx, data_row in enumerate(csv_reader):
                    data_row.update(meta_only_fields) # insert other field dict
                    clean_data_row = cleaner.clean(data_row, idx)
                    if clean_data_row != None:
                        csv_writer.write(clean_data_row)

                csv_writer.close()
                print("  Loading...")

                #Decide whether to append or replace the table
                if meta[tablename]["replace_table"] == True:
                    logging.info("  replacing existing table")
                    sql_interface.drop_table()

                #Appends to table; if dropped, it recreates
                sql_interface.create_table_if_necessary()
                try:
                    sql_interface.write_file_to_sql()
                except TableWritingError:
                    #TODO tell user total count of errors.
                    #currently write_file_to_sql() just writes in log that file failed
                    pass
                if keep_temp_files == False:
                    csv_writer.remove_file()

if __name__ == '__main__':

    # TODO: Refactor next 3 if statements if possible
    # local, real data is the default
    database_choice = 'local_database'
    scripts_path = os.path.abspath(os.path.join(python_filepath, 'scripts'))
    meta_path = os.path.abspath(os.path.join(scripts_path, 'meta.json'))
    manifest_path = os.path.abspath(os.path.join(scripts_path, 'manifest.csv'))

    if 'sample' in sys.argv:
        meta_path = os.path.abspath(os.path.join(scripts_path,
                                                 'meta_sample.json'))
        manifest_path = os.path.abspath(
            os.path.join(scripts_path, 'manifest_sample.csv'))

    if 'docker' in sys.argv:
        database_choice = 'docker_database'

    if 'remote' in sys.argv:
        database_choice = 'remote_database'
        # Don't want sample data in the remote database
        meta_path = os.path.abspath(os.path.join(scripts_path, 'meta.json'))
        manifest_path = os.path.abspath(
            os.path.join(scripts_path, 'manifest.csv'))

    loader = LoadData(database_choice=database_choice, meta_path=meta_path,
                      manifest_path=manifest_path, keep_temp_files=True)

    if 'rebuild' in sys.argv:
        loader.drop_tables()

    loader.load_data()
<|endoftext|>


<commit_before># coding: utf-8

"""
Sparklines in ascii
source: https://github.com/rory/ascii_sparks/blob/master/ascii_sparks.py

"""

parts = u' ▁▂▃▄▅▆▇▉'

def sparks(nums):
    fraction = max(nums) / float(len(parts) - 1)
    return ''.join(parts[int(round(x / fraction))] for x in nums)
<commit_msg>Rescaled sparks.<commit_after># coding: utf-8

"""
Sparklines in ascii
source: https://github.com/rory/ascii_sparks/blob/master/ascii_sparks.py

"""

parts = u' ▁▂▃▄▅▆▇▉'

def sparks(nums):
    fraction = 1. / float(len(parts) - 1)
    return ''.join(parts[int(round(x / fraction))] for x in nums)
<|endoftext|>


<commit_before>from bluebottle.bb_projects.models import ProjectPhase
from django.utils.text import slugify

from ..middleware import HASHBANG, ESCAPED_FRAGMENT
from onepercentclub.tests.factory_models.project_factories import OnePercentProjectFactory
from onepercentclub.tests.utils import OnePercentSeleniumTestCase

def escape_url(url):
    return url.replace(HASHBANG, '?%s=' % ESCAPED_FRAGMENT)

class CrawlableTests(OnePercentSeleniumTestCase):
    """
    Tests one of the most complex pages, project list, with and without escaped fragments.
    """
    def setUp(self):
        self.init_projects()

        self.projects = dict([(slugify(title), title) for title in [
            u'Women first', u'Mobile payments for everyone!', u'Schools for children '
        ]])

        for slug, title in self.projects.items():
            project = OnePercentProjectFactory.create(title=title, slug=slug)
            project.status = ProjectPhase.objects.get(slug="campaign")
            project.save()
            self.some_project = project

        self.project_list_url = '{0}/en/?_escaped_fragment_=/projects'.format(self.live_server_url)
        self.project_url = '{0}/en/?_escaped_fragment_=/projects/{1}'.format(self.live_server_url, self.some_project.slug)

        self.client = self.client_class(SERVER_NAME=self.server_thread.host, SERVER_PORT=self.server_thread.port)

    def tearDown(self):
        from ..middleware import web_cache

        if web_cache._web_driver:
            web_cache._web_driver.service.stop()

    def test_project_list_via_hashbang(self):
        print "loading " + self.project_list_url
        response = self.client.get(self.project_list_url)

        for slug, title in self.projects.items():
            self.assertContains(response, title)

    def test_project_via_hashbang(self):
        print "loading " + self.project_url
        response = self.client.get(self.project_url)
        self.assertContains(response, self.some_project.title)

    def test_project_via_hashbang_with_query_params(self):
        response = self.client.get(self.project_url + '?x=12&y=100')
        self.assertContains(response, self.some_project.title)

    def test_project_via_hashbang_with_camel_case(self):
        response = self.client.get(self.project_list_url + '/Schools-For-Children')
        self.assertRedirects(response, self.project_url, status_code=301)

    def test_non_existing_project(self):
        # Non existing project should redirect to project list
        response = self.client.get(self.project_list_url + '/this-is-not-a-project')
        self.assertRedirects(response, self.project_list_url, status_code=301)
<commit_msg>Don't tear down driver in func tests<commit_after>from bluebottle.bb_projects.models import ProjectPhase
from django.utils.text import slugify

from ..middleware import HASHBANG, ESCAPED_FRAGMENT
from onepercentclub.tests.factory_models.project_factories import OnePercentProjectFactory
from onepercentclub.tests.utils import OnePercentSeleniumTestCase

def escape_url(url):
    return url.replace(HASHBANG, '?%s=' % ESCAPED_FRAGMENT)

class CrawlableTests(OnePercentSeleniumTestCase):
    """
    Tests one of the most complex pages, project list, with and without escaped fragments.
    """
    def setUp(self):
        self.init_projects()

        self.projects = dict([(slugify(title), title) for title in [
            u'Women first', u'Mobile payments for everyone!', u'Schools for children '
        ]])

        for slug, title in self.projects.items():
            project = OnePercentProjectFactory.create(title=title, slug=slug)
            project.status = ProjectPhase.objects.get(slug="campaign")
            project.save()
            self.some_project = project

        self.project_list_url = '{0}/en/?_escaped_fragment_=/projects'.format(self.live_server_url)
        self.project_url = '{0}/en/?_escaped_fragment_=/projects/{1}'.format(self.live_server_url, self.some_project.slug)

        self.client = self.client_class(SERVER_NAME=self.server_thread.host, SERVER_PORT=self.server_thread.port)

    def test_project_list_via_hashbang(self):
        print "loading " + self.project_list_url
        response = self.client.get(self.project_list_url)

        for slug, title in self.projects.items():
            self.assertContains(response, title)

    def test_project_via_hashbang(self):
        print "loading " + self.project_url
        response = self.client.get(self.project_url)
        self.assertContains(response, self.some_project.title)

    def test_project_via_hashbang_with_query_params(self):
        response = self.client.get(self.project_url + '?x=12&y=100')
        self.assertContains(response, self.some_project.title)

    def test_project_via_hashbang_with_camel_case(self):
        response = self.client.get(self.project_list_url + '/Schools-For-Children')
        self.assertRedirects(response, self.project_url, status_code=301)

    def test_non_existing_project(self):
        # Non existing project should redirect to project list
        response = self.client.get(self.project_list_url + '/this-is-not-a-project')
        self.assertRedirects(response, self.project_list_url, status_code=301)
<|endoftext|>