<commit_before>#!/usr/bin/env python 3

import os
import sublime
import sublime_plugin
import re
import sys

__version__ = '0.1.0'
__authors__ = ['Ryan Grannell (@RyanGrannell)']

is_python3 = sys.version_info[0] > 2

# -- Snippet
#
# -- A wrapper for constructing sublime-texts XML snippets
# -- from a snippet body, tabtrigger, scope and description.

def Snippet (content, trigger = None, scope = None):

	if scope and scope is not '*':
		if trigger:

			template = '''
				<snippet>
					<content><![CDATA[{0}]]></content>
					<tabTrigger>{1}</tabTrigger>
					<scope>{2}</scope>
				</snippet>
				'''

			return template.format(content, trigger, scope)

		else:

			template = '''
				<snippet>
					<content><![CDATA[{0}]]></content>
					<scope>{1}</scope>
					<tabTrigger>$$</tabTrigger>
				</snippet>
				'''

			return template.format(content, scope)

	else:

		if trigger:

			template = '''
				<snippet>
					<content><![CDATA[{0}]]></content>
					<tabTrigger>{1}</tabTrigger>
				</snippet>
				'''

			return template.format(content, trigger)

		else:

			template = '''
				<snippet>
					<content><![CDATA[{0}]]></content>
					<tabTrigger>$$</tabTrigger>
				</snippet>
				'''

			return template.format(content)

# -- parse_args
#
# -- parse the snippets argument line.

def parse_args (line):
	# -- lex and parse expressions of the form
	# -- > (ps|ts) [scope] [tab-trigger]

	# -- combine the following into the following large regexp:
	#^>[ 	]+(t|p)[ 	]+([st]\.[^ 	]+)[ 	]+.+

	prompt          = '>'
	space_rexp      = '[ 	]+'
	storage_rexp    = '(t|p)'
	lang_rexp       = '[*]|([st]\.[^ 	]+)'
	trigger_rexp    = '.+'

	valid_args_rexp = re.compile(prompt + space_rexp + storage_rexp + \
		space_rexp + lang_rexp + space_rexp + trigger_rexp)

	if valid_args_rexp.search(line) is None:
		print("-- snipboard: could not match " + line + " as snippet arguments.")
	else:

		blocks = re.split(space_rexp, line, 3)

		return {
			'storage' : blocks[1],
			'language': blocks[2],
			'trigger' : blocks[3]
		}

# -- compile_args
#
# -- compile the arguments passed to a snipboard snippet to
# -- a form useable by normal sublime snippets.

def compile_args (args):

	language = args['language']
	storage  = args['storage']
	trigger  = args['trigger']

	# -- compile s.lang -> source.lang,
	# -- compile t.lang -> text   .lang

	language = re.sub('^s\.', 'source.', language)
	language = re.sub('^t\.', 'text.',   language)

	return {
		'storage' : storage,
		'language': language,
		'trigger' : trigger
	}

# -- constants
#
# -- shorthand replacements for existing sublime-text
# -- snippets.

constants = (
	['$s',  '$SELECTION'],
	['$l',  '$TM_CURRENT_LINE'],
	['$w',  '$TM_CURRENT_WORD'],
	['$fn', '$TM_FILENAME'],
	['$fp', '$TM_FILEPATH'],
	['$li', '$TM_LINE_INDEX'],
	['$ln', '$TM_LINE_NUMBER'],
	['$st', '$TM_SOFT_TABS'],
	['$ts', '$TM_TAB_SIZE'],

	['$text',     '$SELECTION'],
	['$line',     '$TM_CURRENT_LINE'],
	['$word',     '$TM_CURRENT_WORD'],
	['$filename', '$TM_FILENAME'],
	['$filepath', '$TM_FILEPATH'],
	['$lineind',  '$TM_LINE_INDEX'],
	['$linenum',  '$TM_LINE_NUMBER'],
	['$softtab',  '$TM_SOFT_TABS'],
	['$tabsize',  '$TM_TAB_SIZE'],
)

# -- compile_body
#
# -- compile .

def compile_body (body):
	return body

# -- compile_snippet
#
# -- compile a snipboard snippet into an XML sublime text snippet.

def compile_snippet (snippet):

	if snippet.startswith('>'):
		# -- snippet includes additional arguments on the first line.

		snippet_args = snippet.split('\n')   [0]
		snippet_body = snippet.split('\n', 1)[1]

		args = compile_args(parse_args(snippet_args))
		body = compile_body(snippet_body)

		return args, Snippet(body, args['trigger'], args['language'])

	else:
		# -- snippet does not include additional arguments; use defaults.

		snippet_body = snippet

		args         = {
			"storage":  't',
			"language": '*',
			"trigger":  '$$'
		}

		body         = compile_body(snippet_body)

		return args, Snippet(body)

# -- write_to_snipboard
#
# -- write the sublime text snippet to a file.

def write_to_snipboard (args, content):

	fpaths = {
		'linux': os.path.expanduser('~/.config/sublime-text-3/Packages/snipboard/snipboard.sublime-snippet')
	}

	platform_name = sys.platform

	try:
		file = open(fpaths[platform_name], "w")
	except IOError:
		raise IOError('-- snipboard: could not open ' + fpaths[platform_name])
	else:
		print('-- snipboard: writing to ' + fpaths[platform_name])

		file.write(content)
		file.close()

# -- SnipboardCommand
#
# -- The interface to snipboard for SublimeText.

class SnipboardCommand (sublime_plugin.WindowCommand):

	def run (self):

		print('-- snipboard: initialised.')

		window = self.window
		view   = window.active_view()
		sel    = view.sel()

		# -- get text from the first selection.
		select_text = view.substr(sel[0])

		if select_text:
			args, xml = compile_snippet(select_text)
			write_to_snipboard(args, xml)
		else:
			raise SyntaxError('-- snipboard: cannot create a snippet from no input.')
<commit_msg>improved snipboard writing.<commit_after>#!/usr/bin/env python 3

import os
import sublime
import sublime_plugin
import re
import sys

__version__ = '0.1.0'
__authors__ = ['Ryan Grannell (@RyanGrannell)']

is_python3 = sys.version_info[0] > 2

# -- Snippet
#
# -- A wrapper for constructing sublime-texts XML snippets
# -- from a snippet body, tabtrigger, scope and description.

def Snippet (content, trigger = None, scope = None):

	if scope and scope is not '*':
		if trigger:

			template = '''
				<snippet>
					<content><![CDATA[{0}]]></content>
					<tabTrigger>{1}</tabTrigger>
					<scope>{2}</scope>
				</snippet>
				'''

			return template.format(content, trigger, scope)

		else:

			template = '''
				<snippet>
					<content><![CDATA[{0}]]></content>
					<scope>{1}</scope>
					<tabTrigger>$$</tabTrigger>
				</snippet>
				'''

			return template.format(content, scope)

	else:

		if trigger:

			template = '''
				<snippet>
					<content><![CDATA[{0}]]></content>
					<tabTrigger>{1}</tabTrigger>
				</snippet>
				'''

			return template.format(content, trigger)

		else:

			template = '''
				<snippet>
					<content><![CDATA[{0}]]></content>
					<tabTrigger>$$</tabTrigger>
				</snippet>
				'''

			return template.format(content)

# -- parse_args
#
# -- parse the snippets argument line.

def parse_args (line):
	# -- lex and parse expressions of the form
	# -- > (ps|ts) [scope] [tab-trigger]

	# -- combine the following into the following large regexp:
	#^>[ 	]+(t|p)[ 	]+([st]\.[^ 	]+)[ 	]+.+

	prompt          = '>'
	space_rexp      = '[ 	]+'
	storage_rexp    = '(t|p)'
	lang_rexp       = '[*]|([st]\.[^ 	]+)'
	trigger_rexp    = '.+'

	valid_args_rexp = re.compile(prompt + space_rexp + storage_rexp + \
		space_rexp + lang_rexp + space_rexp + trigger_rexp)

	if valid_args_rexp.search(line) is None:
		print("-- snipboard: could not match " + line + " as snippet arguments.")
	else:

		blocks = re.split(space_rexp, line, 3)

		return {
			'storage' : blocks[1],
			'language': blocks[2],
			'trigger' : blocks[3]
		}

# -- compile_args
#
# -- compile the arguments passed to a snipboard snippet to
# -- a form useable by normal sublime snippets.

def compile_args (args):

	language = args['language']
	storage  = args['storage']
	trigger  = args['trigger']

	# -- compile s.lang -> source.lang,
	# -- compile t.lang -> text   .lang

	language = re.sub('^s\.', 'source.', language)
	language = re.sub('^t\.', 'text.',   language)

	return {
		'storage' : storage,
		'language': language,
		'trigger' : trigger
	}

# -- constants
#
# -- shorthand replacements for existing sublime-text
# -- snippets.

constants = (
	['$s',  '$SELECTION'],
	['$l',  '$TM_CURRENT_LINE'],
	['$w',  '$TM_CURRENT_WORD'],
	['$fn', '$TM_FILENAME'],
	['$fp', '$TM_FILEPATH'],
	['$li', '$TM_LINE_INDEX'],
	['$ln', '$TM_LINE_NUMBER'],
	['$st', '$TM_SOFT_TABS'],
	['$ts', '$TM_TAB_SIZE'],

	['$text',     '$SELECTION'],
	['$line',     '$TM_CURRENT_LINE'],
	['$word',     '$TM_CURRENT_WORD'],
	['$filename', '$TM_FILENAME'],
	['$filepath', '$TM_FILEPATH'],
	['$lineind',  '$TM_LINE_INDEX'],
	['$linenum',  '$TM_LINE_NUMBER'],
	['$softtab',  '$TM_SOFT_TABS'],
	['$tabsize',  '$TM_TAB_SIZE'],
)

# -- compile_body
#
# -- compile .

def compile_body (body):
	return body

# -- compile_snippet
#
# -- compile a snipboard snippet into an XML sublime text snippet.

def compile_snippet (snippet):

	if snippet.startswith('>'):
		# -- snippet includes additional arguments on the first line.

		snippet_args = snippet.split('\n')   [0]
		snippet_body = snippet.split('\n', 1)[1]

		args = compile_args(parse_args(snippet_args))
		body = compile_body(snippet_body)

		return args, Snippet(body, args['trigger'], args['language'])

	else:
		# -- snippet does not include additional arguments; use defaults.

		snippet_body = snippet

		args         = {
			"storage":  't',
			"language": '*',
			"trigger":  '$$'
		}

		body         = compile_body(snippet_body)

		return args, Snippet(body)

# -- write_to_snipboard
#
# -- write the sublime text snippet to a file.

def write_to_snipboard (args, content):

	platform_name = sys.platform

	if args['storage'] is 't':
		# -- store temporarily.

		fpaths = {
			'linux': os.path.expanduser('~/.config/sublime-text-3/Packages/snipboard/snipboard.sublime-snippet')
		}
		out_path      = fpaths[platform_name]

	elif args['storage'] is 'p':
		# -- store permanently.

		pass

	else:
		raise KeyError("-- snipboard: internal error in 'write_to_snipboard'")

	try:
		file = open(out_path, "w")
	except IOError:
		raise IOError('-- snipboard: could not open ' + out_path)
	else:
		print('-- snipboard: writing to ' + out_path)

		file.write(content)
		file.close()

# -- SnipboardCommand
#
# -- The interface to snipboard for SublimeText.

class SnipboardCommand (sublime_plugin.WindowCommand):

	def run (self):

		print('-- snipboard: initialised.')

		window = self.window
		view   = window.active_view()
		sel    = view.sel()

		# -- get text from the first selection.
		select_text = view.substr(sel[0])

		if select_text:
			args, xml = compile_snippet(select_text)
			write_to_snipboard(args, xml)
		else:
			raise SyntaxError('-- snipboard: cannot create a snippet from no input.')
<|endoftext|>


<commit_before>from forgetracker.model import Globals
from forgetracker.tests.unit import TestWithModel
from pylons import c
from pyforge.lib import helpers as h

from ming.orm.ormsession import ThreadLocalORMSession

class TestGlobalsModel(TestWithModel):
    def test_it_has_current_tracker_globals(self):
        bugs_globals = Globals.query.get(app_config_id=c.app.config._id)
        assert Globals.for_current_tracker() == bugs_globals
        c.project.install_app('Tickets', 'doc_bugs')
        ThreadLocalORMSession.flush_all()
        h.set_context('test', 'doc_bugs')
        assert Globals.for_current_tracker() != bugs_globals

    def test_next_ticket_number_increments(self):
        assert Globals.next_ticket_num() == 1
        assert Globals.next_ticket_num() == 2

    def test_ticket_numbers_are_independent(self):
        assert Globals.next_ticket_num() == 1
        c.project.install_app('Tickets', 'doc_bugs')
        ThreadLocalORMSession.flush_all()
        h.set_context('test', 'doc_bugs')
        assert Globals.next_ticket_num() == 1

class TestCustomFields(TestWithModel):
    def test_it_has_sortable_custom_fields(self):
        tracker_globals = globals_with_custom_fields(
            [dict(label='Iteration Number',
                  name='_iteration_number',
                  show_in_search=False),
             dict(label='Point Estimate',
                  name='_point_estimate',
                  show_in_search=True)])
        expected = [dict(sortable_name='_point_estimate_s',
                         label='Point Estimate')]
        assert tracker_globals.sortable_custom_fields_shown_in_search() == expected

def globals_with_custom_fields(custom_fields):
    tracker_globals = Globals.for_current_tracker()
    tracker_globals.custom_fields = custom_fields
    ThreadLocalORMSession.flush_all()
    tracker_globals = Globals.for_current_tracker()
    return tracker_globals

<commit_msg>Extract common setup in globals model unit test<commit_after>from forgetracker.model import Globals
from forgetracker.tests.unit import TestWithModel
from pylons import c
from pyforge.lib import helpers as h

from ming.orm.ormsession import ThreadLocalORMSession

class TestGlobalsModel(TestWithModel):
    def setUp(self):
        super(TestGlobalsModel, self).setUp()
        c.project.install_app('Tickets', 'doc_bugs')
        ThreadLocalORMSession.flush_all()

    def test_it_has_current_tracker_globals(self):
        bugs_globals = Globals.query.get(app_config_id=c.app.config._id)
        assert Globals.for_current_tracker() == bugs_globals
        h.set_context('test', 'doc_bugs')
        assert Globals.for_current_tracker() != bugs_globals

    def test_next_ticket_number_increments(self):
        assert Globals.next_ticket_num() == 1
        assert Globals.next_ticket_num() == 2

    def test_ticket_numbers_are_independent(self):
        assert Globals.next_ticket_num() == 1
        h.set_context('test', 'doc_bugs')
        assert Globals.next_ticket_num() == 1

class TestCustomFields(TestWithModel):
    def test_it_has_sortable_custom_fields(self):
        tracker_globals = globals_with_custom_fields(
            [dict(label='Iteration Number',
                  name='_iteration_number',
                  show_in_search=False),
             dict(label='Point Estimate',
                  name='_point_estimate',
                  show_in_search=True)])
        expected = [dict(sortable_name='_point_estimate_s',
                         label='Point Estimate')]
        assert tracker_globals.sortable_custom_fields_shown_in_search() == expected

def globals_with_custom_fields(custom_fields):
    tracker_globals = Globals.for_current_tracker()
    tracker_globals.custom_fields = custom_fields
    ThreadLocalORMSession.flush_all()
    tracker_globals = Globals.for_current_tracker()
    return tracker_globals

<|endoftext|>


<commit_before>import collectd
import urllib2
import json
import itertools

SOLR_CORE_URL = "http://localhost:8983/solr/news"
SOLR_HANDLERS = {"query": "/select", "suggest": "/suggest", "simillar": "/mlt"}

def fetch_data():
    global SOLR_CORE_URL, SOLR_HANDLERS
    stats_url = "%s/admin/mbeans?stats=true&wt=json" % (SOLR_CORE_URL, )
    stats = urllib2.urlopen(stats_url)
    solr_data = json.load(stats)

    # Searcher information
    solr_data = solr_data["solr-mbeans"]

    # Data is return in form of [ "TYPE", { DATA }, "TYPE", ... ] so pair them up
    solr_data_iter = iter(solr_data)
    solr_data = itertools.izip(solr_data_iter, solr_data_iter)

    data = { "cache": {}, "handler_stats": {}, "update_stats": {} }
    for module, module_data in solr_data:
        if module == "CORE":
            data["docs"] = module_data["searcher"]["stats"]["numDocs"]
        elif module == "CACHE":
            data["cache"]["size"] = module_data["fieldValueCache"]["stats"]["size"]
            data["cache"]["hitratio"] = module_data["fieldValueCache"]["stats"]["hitratio"]
            data["cache"]["evictions"] = module_data["fieldValueCache"]["stats"]["evictions"]
        elif module == "QUERYHANDLER":
            interesting_handlers = { endpoint: name for name, endpoint in SOLR_HANDLERS.iteritems() }
            for handler, handler_data in module_data.iteritems():
                if handler not in interesting_handlers:
                    continue

                handler_name = interesting_handlers[handler]
                data["handler_stats"][handler_name] = {}
                data["handler_stats"][handler_name]["requests"] = handler_data["stats"]["requests"]
                data["handler_stats"][handler_name]["errors"] = handler_data["stats"]["errors"]
                data["handler_stats"][handler_name]["timeouts"] = handler_data["stats"]["timeouts"]
                data["handler_stats"][handler_name]["time_per_request"] = handler_data["stats"]["avgTimePerRequest"]
                data["handler_stats"][handler_name]["requests_per_second"] = handler_data["stats"]["avgRequestsPerSecond"]
        elif module == "UPDATEHANDLER":
            data["update_stats"]["commits"] = module_data["updateHandler"]["stats"]["commits"]
            data["update_stats"]["autocommits"] = module_data["updateHandler"]["stats"]["autocommits"]
            data["update_stats"]["soft_autocommits"] = module_data["updateHandler"]["stats"]["soft autocommits"]
            data["update_stats"]["optimizes"] = module_data["updateHandler"]["stats"]["optimizes"]
            data["update_stats"]["rollbacks"] = module_data["updateHandler"]["stats"]["rollbacks"]
            data["update_stats"]["expunges"] = module_data["updateHandler"]["stats"]["expungeDeletes"]
            data["update_stats"]["pending_docs"] = module_data["updateHandler"]["stats"]["docsPending"]
            data["update_stats"]["adds"] = module_data["updateHandler"]["stats"]["adds"]
            data["update_stats"]["deletes_by_id"] = module_data["updateHandler"]["stats"]["deletesById"]
            data["update_stats"]["deletes_by_query"] = module_data["updateHandler"]["stats"]["deletesByQuery"]
            data["update_stats"]["errors"] = module_data["updateHandler"]["stats"]["errors"]
    return data

# Used only for testing, remove when done!
if __name__ == "__main__":
   data = fetch_data()
   print data
<commit_msg>Implemented proper data dispatch<commit_after>import collectd
import urllib2
import json
import itertools

SOLR_CORE_URL = "http://localhost:8080/solr/news"
SOLR_HANDLERS = {"query": "/select", "suggest": "/suggest", "simillar": "/mlt"}

def dispatch_value(value, value_name, value_type, type_instance=None):
    val = collectd.Values(plugin="solr_info")
    val.type = value_type

    if type_instance is not None:
       val.plugin_instance = value_name
       val.type_instance = type_instance
    else:
       val.type_instance = value_name
    val.values = [value]
    val.dispatch()

def fetch_data():
    global SOLR_CORE_URL, SOLR_HANDLERS
    stats_url = "%s/admin/mbeans?stats=true&wt=json" % (SOLR_CORE_URL, )
    stats = urllib2.urlopen(stats_url)
    solr_data = json.load(stats)

    # Searcher information
    solr_data = solr_data["solr-mbeans"]

    # Data is return in form of [ "TYPE", { DATA }, "TYPE", ... ] so pair them up
    solr_data_iter = iter(solr_data)
    solr_data = itertools.izip(solr_data_iter, solr_data_iter)

    data = { "cache": {}, "handler_stats": {}, "update_stats": {} }
    for module, module_data in solr_data:
        if module == "CORE":
            data["docs"] = module_data["searcher"]["stats"]["numDocs"]
        elif module == "CACHE":
            data["cache"]["size"] = module_data["fieldValueCache"]["stats"]["size"]
            data["cache"]["hitratio"] = module_data["fieldValueCache"]["stats"]["hitratio"]
            data["cache"]["evictions"] = module_data["fieldValueCache"]["stats"]["evictions"]
        elif module == "QUERYHANDLER":
            interesting_handlers = { endpoint: name for name, endpoint in SOLR_HANDLERS.iteritems() }
            for handler, handler_data in module_data.iteritems():
                if handler not in interesting_handlers:
                    continue

                handler_name = interesting_handlers[handler]
                data["handler_stats"][handler_name] = {}
                data["handler_stats"][handler_name]["requests"] = handler_data["stats"]["requests"]
                data["handler_stats"][handler_name]["errors"] = handler_data["stats"]["errors"]
                data["handler_stats"][handler_name]["timeouts"] = handler_data["stats"]["timeouts"]
                data["handler_stats"][handler_name]["time_per_request"] = handler_data["stats"]["avgTimePerRequest"]
                data["handler_stats"][handler_name]["requests_per_second"] = handler_data["stats"]["avgRequestsPerSecond"]
        elif module == "UPDATEHANDLER":
            data["update_stats"]["commits"] = module_data["updateHandler"]["stats"]["commits"]
            data["update_stats"]["autocommits"] = module_data["updateHandler"]["stats"]["autocommits"]
            data["update_stats"]["soft_autocommits"] = module_data["updateHandler"]["stats"]["soft autocommits"]
            data["update_stats"]["optimizes"] = module_data["updateHandler"]["stats"]["optimizes"]
            data["update_stats"]["rollbacks"] = module_data["updateHandler"]["stats"]["rollbacks"]
            data["update_stats"]["expunges"] = module_data["updateHandler"]["stats"]["expungeDeletes"]
            data["update_stats"]["pending_docs"] = module_data["updateHandler"]["stats"]["docsPending"]
            data["update_stats"]["adds"] = module_data["updateHandler"]["stats"]["adds"]
            data["update_stats"]["deletes_by_id"] = module_data["updateHandler"]["stats"]["deletesById"]
            data["update_stats"]["deletes_by_query"] = module_data["updateHandler"]["stats"]["deletesByQuery"]
            data["update_stats"]["errors"] = module_data["updateHandler"]["stats"]["errors"]
    return data

def read_callback():
    data = fetch_data()
    dispatch_value(data["docs"], "index", "gauge", "documents")
    dispatch_value(data["cache"]["size"], "cache", "gauge", "size")
    dispatch_value(data["cache"]["hitratio"], "cache_hitratio", "gauge", "hitratio")
    dispatch_value(data["cache"]["evictions"], "cache", "gauge", "evictions")

    for handler_name, handler_data in data["handler_stats"].iteritems():
        dispatch_value(handler_data["requests"], handler_name, "gauge", "requests")
        dispatch_value(handler_data["errors"], handler_name, "gauge", "errors")
        dispatch_value(handler_data["timeouts"], handler_name, "gauge", "timeouts")
        dispatch_value(handler_data["time_per_request"], "request_times", "gauge", handler_name)
        dispatch_value(handler_data["requests_per_second"], "requests_per_second", "gauge", handler_name)

    dispatch_value(data["update_stats"]["commits"], "update", "gauge", "commits")
    dispatch_value(data["update_stats"]["autocommits"], "update", "gauge", "autocommits")
    dispatch_value(data["update_stats"]["soft_autocommits"], "update", "gauge", "soft_autocommits")
    dispatch_value(data["update_stats"]["optimizes"], "update", "gauge", "optimizes")
    dispatch_value(data["update_stats"]["expunges"], "update", "gauge", "expunges")
    dispatch_value(data["update_stats"]["rollbacks"], "update", "gauge", "rollbacks")
    dispatch_value(data["update_stats"]["pending_docs"], "update", "gauge", "pending_docs")
    dispatch_value(data["update_stats"]["adds"], "update", "gauge", "adds")
    dispatch_value(data["update_stats"]["deletes_by_id"], "update", "gauge", "deletes_by_id")
    dispatch_value(data["update_stats"]["deletes_by_query"], "update", "gauge", "deletes_by_query")
    dispatch_value(data["update_stats"]["errors"], "update", "gauge", "errors")

collectd.register_read(read_callback)
<|endoftext|>


<commit_before>import argparse

def problem1():
    """
    If we list all the natural numbers below 10 that are multiples of 3 or 5,
    we get 3, 5, 6 and 9. The sum of these multiples is 23.

    Find the sum of all the multiples of 3 or 5 below 1000.
    """
    total = 0
    for n in xrange(1000):
        if (n % 3 == 0) or (n % 5 == 0):
            total += n
    return total

def problem2():
    """
    Each new term in the Fibonacci sequence is generated by adding the
    previous two terms.

    By considering the terms in the Fibonacci sequence whose values do not
    exceed four million, find the sum of the even-valued terms.
    """
    total = 0
    t0, t1, t2 = 0, 1, 0
    while t0 <= 4000000:
        if t0 % 2 == 0:
            total += t0
        t2, t1 = t1, t0
        t0 = t2 + t1
    return total

def problem3():
    """
    The prime factors of 13195 are 5, 7, 13 and 29.

    What is the largest prime factor of the number 600851475143 ?
    """
    from math import floor, ceil, sqrt
    number = 600851475143

    def factor(num):
        for n in xrange(2, int(ceil(sqrt(num)))):
            if num % n == 0:
                return n, num / n
        return None, num

    f1, f2 = 1, number
    while f1 is not None:
        f1, f2 = factor(f2)
    return f2

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--problem', '-p', type=int,
        help='problem number to evaluate, if completed')

    args = parser.parse_args()
    if args.problem:
        try:
            result = globals()['problem' + str(args.problem)]()
            print 'Problem ' + str(args.problem) + ': ' + str(result)
        except KeyError:
            print 'Problem ' + str(args.problem) + ': Function not found'
<commit_msg>problem 4 solution<commit_after>import argparse

def problem1():
    """
    If we list all the natural numbers below 10 that are multiples of 3 or 5,
    we get 3, 5, 6 and 9. The sum of these multiples is 23.

    Find the sum of all the multiples of 3 or 5 below 1000.
    """
    total = 0
    for n in xrange(1000):
        if (n % 3 == 0) or (n % 5 == 0):
            total += n
    return total

def problem2():
    """
    Each new term in the Fibonacci sequence is generated by adding the
    previous two terms.

    By considering the terms in the Fibonacci sequence whose values do not
    exceed four million, find the sum of the even-valued terms.
    """
    total = 0
    t0, t1, t2 = 0, 1, 0
    while t0 <= 4000000:
        if t0 % 2 == 0:
            total += t0
        t2, t1 = t1, t0
        t0 = t2 + t1
    return total

def problem3():
    """
    The prime factors of 13195 are 5, 7, 13 and 29.

    What is the largest prime factor of the number 600851475143 ?
    """
    from math import floor, ceil, sqrt
    number = 600851475143

    def factor(num):
        for n in xrange(2, int(ceil(sqrt(num)))):
            if num % n == 0:
                return n, num / n
        return None, num

    f1, f2 = 1, number
    while f1 is not None:
        f1, f2 = factor(f2)
    return f2

def problem4():
    """
    A palindromic number reads the same both ways. The largest palindrome
    made from the product of two 2-digit numbers is 9009 = 91 x 99.

    Find the largest palindrome made from the product of two 3-digit numbers.
    """
    def is_palindrome(num):
        num = str(num)
        digits = len(num)
        for i in xrange(int(digits / 2 + 1)):
            if num[i] != num[digits - i - 1]:
                return False
        else:
            return True

    best = 0
    for i in xrange(100, 1000):
        for j in xrange(i, 1000):
            if is_palindrome(i * j):
                best = max(best, i * j)
    return best

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--problem', '-p', type=int,
        help='problem number to evaluate, if completed')

    args = parser.parse_args()
    if args.problem:
        try:
            result = globals()['problem' + str(args.problem)]()
            print 'Problem ' + str(args.problem) + ': ' + str(result)
        except KeyError:
            print 'Problem ' + str(args.problem) + ': Function not found'
<|endoftext|>


<commit_before>import Utils, numpy as np, scipy.sparse as sp
from Tests import checkDerivative

class IdentityMap(object):
    """
    SimPEG Map

    """

    __metaclass__ = Utils.SimPEGMetaClass

    mesh = None      #: A SimPEG Mesh

    def __init__(self, mesh):
        self.mesh = mesh

    @property
    def nP(self):
        """
            :rtype: int
            :return: number of parameters in the model
        """
        return self.mesh.nC

    @property
    def shape(self):
        """
            The default shape is (mesh.nC, nP).

            :rtype: (int,int)
            :return: shape of the operator as a tuple
        """
        return (self.mesh.nC, self.nP)

    def _transform(self, m):
        """
            Changes the model into the physical property.

            .. note::

                This can be called by the __mul__ property against a numpy.ndarray.

            :param numpy.array m: model
            :rtype: numpy.array
            :return: transformed model

        """
        return m

    def inverse(self, D):
        """
            Changes the physical property into the model.

            .. note::

                The *transformInverse* may not be easy to create in general.

            :param numpy.array D: physical property
            :rtype: numpy.array
            :return: model

        """
        raise NotImplementedError('The transformInverse is not implemented.')

    def deriv(self, m):
        """
            The derivative of the transformation.

            :param numpy.array m: model
            :rtype: scipy.csr_matrix
            :return: derivative of transformed model

        """
        return sp.identity(self.nP)

    def test(self, m=None, **kwargs):
        """Test the derivative of the mapping.

            :param numpy.array m: model
            :param kwargs: key word arguments of :meth:`SimPEG.Tests.checkDerivative`
            :rtype: bool
            :return: passed the test?

        """
        print 'Testing %s' % str(self)
        if m is None:
            m = np.random.rand(self.nP)
        if 'plotIt' not in kwargs:
            kwargs['plotIt'] = False
        return checkDerivative(lambda m : [self * m, self.deriv(m)], m, **kwargs)

    def _assertMatchesPair(self, pair):
        assert (isinstance(self, pair) or
            isinstance(self, ComboMap) and isinstance(self.maps[0], pair)
            ), "Mapping object must be an instance of a %s class."%(pair.__name__)

    def __mul__(self, val):
        if isinstance(val, IdentityMap):
            if not self.shape[1] == val.shape[0]:
                raise ValueError('Dimension mismatch in %s and %s.' % (str(self), str(val)))
            return ComboMap([self, val])
        elif isinstance(val, np.ndarray):
            if not self.shape[1] == val.shape[0]:
                raise ValueError('Dimension mismatch in %s and np.ndarray%s.' % (str(self), str(val.shape)))
            return self._transform(val)
        raise Exception('Unrecognized data type to multiply. Try a map or a numpy.ndarray!')

    def __str__(self):
        return "%s(%d,%d)" % (self.__class__.__name__, self.shape[0], self.shape[1])

class ComboMap(IdentityMap):
    """Combination of various maps."""

    def __init__(self, maps, **kwargs):
        IdentityMap.__init__(self, None, **kwargs)

        self.maps = []
        for ii, m in enumerate(maps):
            assert isinstance(m, IdentityMap), 'Unrecognized data type, inherit from an IdentityMap or ComboMap!'
            if ii > 0 and not self.shape[1] == m.shape[0]:
                prev = self.maps[-1]
                errArgs = (prev.__name__, prev.shape[0], prev.shape[1], m.__name__, m.shape[0], m.shape[1])
                raise ValueError('Dimension mismatch in map[%s] (%i, %i) and map[%s] (%i, %i).' % errArgs)

            if isinstance(m, ComboMap):
                self.maps += m.maps
            elif isinstance(m, IdentityMap):
                self.maps += [m]

    @property
    def shape(self):
        return (self.maps[0].shape[0], self.maps[-1].shape[1])

    @property
    def nP(self):
        """Number of model properties.

           The number of cells in the
           last dimension of the mesh."""
        return self.maps[-1].nP

    def _transform(self, m):
        for map_i in reversed(self.maps):
            m = map_i * m
        return m

    def deriv(self, m):
        deriv = 1
        mi = m
        for map_i in reversed(self.maps):
            deriv = map_i.deriv(mi) * deriv
            mi = map_i * mi
        return deriv

    def __str__(self):
        return 'ComboMap[%s]%s' % (' * '.join([m.__str__() for m in self.maps]), str(self.shape))

class ExpMap(IdentityMap):
    """

        Changes the model into the physical property.

        A common example of this is to invert for electrical conductivity
        in log space. In this case, your model will be log(sigma) and to
        get back to sigma, you can take the exponential:

        .. math::

            m = \log{\sigma}

            \exp{m} = \exp{\log{\sigma}} = \sigma
    """

    def __init__(self, mesh, **kwargs):
        IdentityMap.__init__(self, mesh, **kwargs)

    def _transform(self, m):
        return np.exp(Utils.mkvc(m))

    def inverse(self, D):
        """
            :param numpy.array D: physical property
            :rtype: numpy.array
            :return: model

            The *transformInverse* changes the physical property into the model.

            .. math::

                m = \log{\sigma}

        """
        return np.log(Utils.mkvc(D))

    def deriv(self, m):
        """
            :param numpy.array m: model
            :rtype: scipy.csr_matrix
            :return: derivative of transformed model

            The *transform* changes the model into the physical property.
            The *transformDeriv* provides the derivative of the *transform*.

            If the model *transform* is:

            .. math::

                m = \log{\sigma}

                \exp{m} = \exp{\log{\sigma}} = \sigma

            Then the derivative is:

            .. math::

                \\frac{\partial \exp{m}}{\partial m} = \\text{sdiag}(\exp{m})
        """
        return Utils.sdiag(np.exp(Utils.mkvc(m)))

class Vertical1DMap(IdentityMap):
    """Vertical1DMap

        Given a 1D vector through the last dimension
        of the mesh, this will extend to the full
        model space.
    """

    def __init__(self, mesh, **kwargs):
        IdentityMap.__init__(self, mesh, **kwargs)

    @property
    def nP(self):
        """Number of model properties.

           The number of cells in the
           last dimension of the mesh."""
        return self.mesh.vnC[self.mesh.dim-1]

    def _transform(self, m):
        """
            :param numpy.array m: model
            :rtype: numpy.array
            :return: transformed model
        """
        repNum = self.mesh.vnC[:self.mesh.dim-1].prod()
        return Utils.mkvc(m).repeat(repNum)

    def deriv(self, m):
        """
            :param numpy.array m: model
            :rtype: scipy.csr_matrix
            :return: derivative of transformed model
        """
        repNum = self.mesh.vnC[:self.mesh.dim-1].prod()
        repVec = sp.csr_matrix(
                    (np.ones(repNum),
                    (range(repNum), np.zeros(repNum))
                    ), shape=(repNum, 1))
        return sp.kron(sp.identity(self.nP), repVec)

class Mesh2Mesh(IdentityMap):
    """
        Takes a model on one mesh are translates it to another mesh.

    """

    def __init__(self, meshes, **kwargs):
        Utils.setKwargs(self, **kwargs)

        assert type(meshes) is list, "meshes must be a list of two meshes"
        assert len(meshes) == 2, "meshes must be a list of two meshes"
        assert meshes[0].dim == meshes[1].dim, """The two meshes must be the same dimension"""

        self.mesh  = meshes[0]
        self.mesh2 = meshes[1]

        self.P = self.mesh2.getInterpolationMat(self.mesh.gridCC,'CC',zerosOutside=True)

    @property
    def shape(self):
        """Number of parameters in the model."""
        return (self.mesh.nC, self.mesh2.nC)

    @property
    def nP(self):
        """Number of parameters in the model."""
        return self.mesh2.nC
    def _transform(self, m):
        return self.P*m
    def deriv(self, m):
        return self.P

class ActiveCells(IdentityMap):
    """
        Active model parameters.

    """

    indActive   = None #: Active Cells
    valInactive = None #: Values of inactive Cells
    nC          = None #: Number of cells in the full model

    def __init__(self, mesh, indActive, valInactive, nC=None):
        self.mesh  = mesh

        self.nC = nC or mesh.nC

        if indActive.dtype is not bool:
            z = np.zeros(self.nC,dtype=bool)
            z[indActive] = True
            indActive = z
        self.indActive = indActive
        self.indInactive = np.logical_not(indActive)
        if Utils.isScalar(valInactive):
            valInactive = np.ones(self.nC)*float(valInactive)

        valInactive[self.indActive] = 0
        self.valInactive = valInactive

        inds = np.nonzero(self.indActive)[0]
        self.P = sp.csr_matrix((np.ones(inds.size),(inds, range(inds.size))), shape=(self.nC, self.nP))

    @property
    def shape(self):
        return (self.nC, self.nP)

    @property
    def nP(self):
        """Number of parameters in the model."""
        return self.indActive.sum()

    def _transform(self, m):
        return self.P*m + self.valInactive
    def deriv(self, m):
        return self.P

class ComplexMap(IdentityMap):
    """ComplexMap

        default nP is nC in the mesh times 2 [real, imag]

    """
    def __init__(self, mesh, nP=None):
        IdentityMap.__init__(self, mesh)
        if nP is not None:
            assert nP%2 == 0, 'nP must be even.'
        self._nP = nP or (self.mesh.nC * 2)

    @property
    def nP(self):
        return self._nP

    @property
    def shape(self):
        return (self.nP/2,self.nP)

    def _transform(self, m):
        nC = self.mesh.nC
        return m[:nC] + m[nC:]*1j

    def deriv(self, m):
        nC = self.nP/2
        shp = (nC, nC*2)
        def fwd(v):
            return v[:nC] + v[nC:]*1j
        def adj(v):
            return np.r_[v.real,v.imag]
        return Utils.SimPEGLinearOperator(shp,fwd,adj)

    inverse = deriv

<commit_msg>circle map<commit_after>import Utils, numpy as np, scipy.sparse as sp
from Tests import checkDerivative

class IdentityMap(object):
    """
    SimPEG Map

    """

    __metaclass__ = Utils.SimPEGMetaClass

    mesh = None      #: A SimPEG Mesh

    def __init__(self, mesh):
        self.mesh = mesh

    @property
    def nP(self):
        """
            :rtype: int
            :return: number of parameters in the model
        """
        return self.mesh.nC

    @property
    def shape(self):
        """
            The default shape is (mesh.nC, nP).

            :rtype: (int,int)
            :return: shape of the operator as a tuple
        """
        return (self.mesh.nC, self.nP)

    def _transform(self, m):
        """
            Changes the model into the physical property.

            .. note::

                This can be called by the __mul__ property against a numpy.ndarray.

            :param numpy.array m: model
            :rtype: numpy.array
            :return: transformed model

        """
        return m

    def inverse(self, D):
        """
            Changes the physical property into the model.

            .. note::

                The *transformInverse* may not be easy to create in general.

            :param numpy.array D: physical property
            :rtype: numpy.array
            :return: model

        """
        raise NotImplementedError('The transformInverse is not implemented.')

    def deriv(self, m):
        """
            The derivative of the transformation.

            :param numpy.array m: model
            :rtype: scipy.csr_matrix
            :return: derivative of transformed model

        """
        return sp.identity(self.nP)

    def test(self, m=None, **kwargs):
        """Test the derivative of the mapping.

            :param numpy.array m: model
            :param kwargs: key word arguments of :meth:`SimPEG.Tests.checkDerivative`
            :rtype: bool
            :return: passed the test?

        """
        print 'Testing %s' % str(self)
        if m is None:
            m = np.random.rand(self.nP)
        if 'plotIt' not in kwargs:
            kwargs['plotIt'] = False
        return checkDerivative(lambda m : [self * m, self.deriv(m)], m, **kwargs)

    def _assertMatchesPair(self, pair):
        assert (isinstance(self, pair) or
            isinstance(self, ComboMap) and isinstance(self.maps[0], pair)
            ), "Mapping object must be an instance of a %s class."%(pair.__name__)

    def __mul__(self, val):
        if isinstance(val, IdentityMap):
            if not self.shape[1] == val.shape[0]:
                raise ValueError('Dimension mismatch in %s and %s.' % (str(self), str(val)))
            return ComboMap([self, val])
        elif isinstance(val, np.ndarray):
            if not self.shape[1] == val.shape[0]:
                raise ValueError('Dimension mismatch in %s and np.ndarray%s.' % (str(self), str(val.shape)))
            return self._transform(val)
        raise Exception('Unrecognized data type to multiply. Try a map or a numpy.ndarray!')

    def __str__(self):
        return "%s(%d,%d)" % (self.__class__.__name__, self.shape[0], self.shape[1])

class ComboMap(IdentityMap):
    """Combination of various maps."""

    def __init__(self, maps, **kwargs):
        IdentityMap.__init__(self, None, **kwargs)

        self.maps = []
        for ii, m in enumerate(maps):
            assert isinstance(m, IdentityMap), 'Unrecognized data type, inherit from an IdentityMap or ComboMap!'
            if ii > 0 and not self.shape[1] == m.shape[0]:
                prev = self.maps[-1]
                errArgs = (prev.__name__, prev.shape[0], prev.shape[1], m.__name__, m.shape[0], m.shape[1])
                raise ValueError('Dimension mismatch in map[%s] (%i, %i) and map[%s] (%i, %i).' % errArgs)

            if isinstance(m, ComboMap):
                self.maps += m.maps
            elif isinstance(m, IdentityMap):
                self.maps += [m]

    @property
    def shape(self):
        return (self.maps[0].shape[0], self.maps[-1].shape[1])

    @property
    def nP(self):
        """Number of model properties.

           The number of cells in the
           last dimension of the mesh."""
        return self.maps[-1].nP

    def _transform(self, m):
        for map_i in reversed(self.maps):
            m = map_i * m
        return m

    def deriv(self, m):
        deriv = 1
        mi = m
        for map_i in reversed(self.maps):
            deriv = map_i.deriv(mi) * deriv
            mi = map_i * mi
        return deriv

    def __str__(self):
        return 'ComboMap[%s]%s' % (' * '.join([m.__str__() for m in self.maps]), str(self.shape))

class ExpMap(IdentityMap):
    """

        Changes the model into the physical property.

        A common example of this is to invert for electrical conductivity
        in log space. In this case, your model will be log(sigma) and to
        get back to sigma, you can take the exponential:

        .. math::

            m = \log{\sigma}

            \exp{m} = \exp{\log{\sigma}} = \sigma
    """

    def __init__(self, mesh, **kwargs):
        IdentityMap.__init__(self, mesh, **kwargs)

    def _transform(self, m):
        return np.exp(Utils.mkvc(m))

    def inverse(self, D):
        """
            :param numpy.array D: physical property
            :rtype: numpy.array
            :return: model

            The *transformInverse* changes the physical property into the model.

            .. math::

                m = \log{\sigma}

        """
        return np.log(Utils.mkvc(D))

    def deriv(self, m):
        """
            :param numpy.array m: model
            :rtype: scipy.csr_matrix
            :return: derivative of transformed model

            The *transform* changes the model into the physical property.
            The *transformDeriv* provides the derivative of the *transform*.

            If the model *transform* is:

            .. math::

                m = \log{\sigma}

                \exp{m} = \exp{\log{\sigma}} = \sigma

            Then the derivative is:

            .. math::

                \\frac{\partial \exp{m}}{\partial m} = \\text{sdiag}(\exp{m})
        """
        return Utils.sdiag(np.exp(Utils.mkvc(m)))

class Vertical1DMap(IdentityMap):
    """Vertical1DMap

        Given a 1D vector through the last dimension
        of the mesh, this will extend to the full
        model space.
    """

    def __init__(self, mesh, **kwargs):
        IdentityMap.__init__(self, mesh, **kwargs)

    @property
    def nP(self):
        """Number of model properties.

           The number of cells in the
           last dimension of the mesh."""
        return self.mesh.vnC[self.mesh.dim-1]

    def _transform(self, m):
        """
            :param numpy.array m: model
            :rtype: numpy.array
            :return: transformed model
        """
        repNum = self.mesh.vnC[:self.mesh.dim-1].prod()
        return Utils.mkvc(m).repeat(repNum)

    def deriv(self, m):
        """
            :param numpy.array m: model
            :rtype: scipy.csr_matrix
            :return: derivative of transformed model
        """
        repNum = self.mesh.vnC[:self.mesh.dim-1].prod()
        repVec = sp.csr_matrix(
                    (np.ones(repNum),
                    (range(repNum), np.zeros(repNum))
                    ), shape=(repNum, 1))
        return sp.kron(sp.identity(self.nP), repVec)

class Mesh2Mesh(IdentityMap):
    """
        Takes a model on one mesh are translates it to another mesh.

    """

    def __init__(self, meshes, **kwargs):
        Utils.setKwargs(self, **kwargs)

        assert type(meshes) is list, "meshes must be a list of two meshes"
        assert len(meshes) == 2, "meshes must be a list of two meshes"
        assert meshes[0].dim == meshes[1].dim, """The two meshes must be the same dimension"""

        self.mesh  = meshes[0]
        self.mesh2 = meshes[1]

        self.P = self.mesh2.getInterpolationMat(self.mesh.gridCC,'CC',zerosOutside=True)

    @property
    def shape(self):
        """Number of parameters in the model."""
        return (self.mesh.nC, self.mesh2.nC)

    @property
    def nP(self):
        """Number of parameters in the model."""
        return self.mesh2.nC
    def _transform(self, m):
        return self.P*m
    def deriv(self, m):
        return self.P

class ActiveCells(IdentityMap):
    """
        Active model parameters.

    """

    indActive   = None #: Active Cells
    valInactive = None #: Values of inactive Cells
    nC          = None #: Number of cells in the full model

    def __init__(self, mesh, indActive, valInactive, nC=None):
        self.mesh  = mesh

        self.nC = nC or mesh.nC

        if indActive.dtype is not bool:
            z = np.zeros(self.nC,dtype=bool)
            z[indActive] = True
            indActive = z
        self.indActive = indActive
        self.indInactive = np.logical_not(indActive)
        if Utils.isScalar(valInactive):
            valInactive = np.ones(self.nC)*float(valInactive)

        valInactive[self.indActive] = 0
        self.valInactive = valInactive

        inds = np.nonzero(self.indActive)[0]
        self.P = sp.csr_matrix((np.ones(inds.size),(inds, range(inds.size))), shape=(self.nC, self.nP))

    @property
    def shape(self):
        return (self.nC, self.nP)

    @property
    def nP(self):
        """Number of parameters in the model."""
        return self.indActive.sum()

    def _transform(self, m):
        return self.P*m + self.valInactive
    def deriv(self, m):
        return self.P

class ComplexMap(IdentityMap):
    """ComplexMap

        default nP is nC in the mesh times 2 [real, imag]

    """
    def __init__(self, mesh, nP=None):
        IdentityMap.__init__(self, mesh)
        if nP is not None:
            assert nP%2 == 0, 'nP must be even.'
        self._nP = nP or (self.mesh.nC * 2)

    @property
    def nP(self):
        return self._nP

    @property
    def shape(self):
        return (self.nP/2,self.nP)

    def _transform(self, m):
        nC = self.mesh.nC
        return m[:nC] + m[nC:]*1j

    def deriv(self, m):
        nC = self.nP/2
        shp = (nC, nC*2)
        def fwd(v):
            return v[:nC] + v[nC:]*1j
        def adj(v):
            return np.r_[v.real,v.imag]
        return Utils.SimPEGLinearOperator(shp,fwd,adj)

    inverse = deriv

class CircleMap(IdentityMap):
    """CircleMap

        Parameterize the model space using a circle in a wholespace.

        ..math::

            \sigma(m) = \sigma_1 + (\sigma_2 - \sigma_1)\left(\\arctan\left(100*\sqrt{(\\vec{x}-x_0)^2 + (\\vec{y}-y_0)}-r\\right) \pi^{-1} + 0.5\\right)

        Define the model as:

        ..math::

            m = [\sigma_1, \sigma_2, x_0, y_0, r]

    """
    def __init__(self, mesh, logSigma=True):
        assert mesh.dim == 2, "Working for a 2D mesh only right now. But it isn't that hard to change.. :)"
        IdentityMap.__init__(self, mesh)
        self.logSigma = logSigma

    slope = 1e-1

    @property
    def nP(self):
        return 5

    def _transform(self, m):
        a = self.slope
        sig1,sig2,x,y,r = m[0],m[1],m[2],m[3],m[4]
        if self.logSigma:
            sig1, sig2 = np.exp(sig1), np.exp(sig2)
        X = self.mesh.gridCC[:,0]
        Y = self.mesh.gridCC[:,1]
        return sig1 + (sig2 - sig1)*(np.arctan(a*(np.sqrt((X-x)**2 + (Y-y)**2) - r))/np.pi + 0.5)

    def deriv(self, m):
        a = self.slope
        sig1,sig2,x,y,r = m[0],m[1],m[2],m[3],m[4]
        if self.logSigma:
            sig1, sig2 = np.exp(sig1), np.exp(sig2)
        X = self.mesh.gridCC[:,0]
        Y = self.mesh.gridCC[:,1]
        if self.logSigma:
            g1 = -(np.arctan(a*(-r + np.sqrt((X - x)**2 + (Y - y)**2)))/np.pi + 0.5)*sig1 + sig1
            g2 = (np.arctan(a*(-r + np.sqrt((X - x)**2 + (Y - y)**2)))/np.pi + 0.5)*sig2
        else:
            g1 = -(np.arctan(a*(-r + np.sqrt((X - x)**2 + (Y - y)**2)))/np.pi + 0.5) + 1.0
            g2 = (np.arctan(a*(-r + np.sqrt((X - x)**2 + (Y - y)**2)))/np.pi + 0.5)
        g3 = a*(-X + x)*(-sig1 + sig2)/(np.pi*(a**2*(-r + np.sqrt((X - x)**2 + (Y - y)**2))**2 + 1)*np.sqrt((X - x)**2 + (Y - y)**2))
        g4 = a*(-Y + y)*(-sig1 + sig2)/(np.pi*(a**2*(-r + np.sqrt((X - x)**2 + (Y - y)**2))**2 + 1)*np.sqrt((X - x)**2 + (Y - y)**2))
        g5 = -a*(-sig1 + sig2)/(np.pi*(a**2*(-r + np.sqrt((X - x)**2 + (Y - y)**2))**2 + 1))
        return np.c_[g1,g2,g3,g4,g5]
<|endoftext|>


<commit_before>from thinc.api import add, layerize, chain, clone, concatenate, with_flatten
from thinc.neural import Model, Maxout, Softmax, Affine
from thinc.neural._classes.hash_embed import HashEmbed
from thinc.neural.ops import NumpyOps, CupyOps

from thinc.neural._classes.convolution import ExtractWindow
from thinc.neural._classes.static_vectors import StaticVectors
from thinc.neural._classes.batchnorm import BatchNorm
from thinc.neural._classes.resnet import Residual
from thinc import describe
from thinc.describe import Dimension, Synapses, Biases, Gradient
from thinc.neural._classes.affine import _set_dimensions_if_needed

from .attrs import ID, LOWER, PREFIX, SUFFIX, SHAPE, TAG, DEP
from .tokens.doc import Doc

import numpy

@describe.on_data(_set_dimensions_if_needed)
@describe.attributes(
    nI=Dimension("Input size"),
    nF=Dimension("Number of features"),
    nO=Dimension("Output size"),
    W=Synapses("Weights matrix",
        lambda obj: (obj.nO, obj.nF, obj.nI),
        lambda W, ops: ops.xavier_uniform_init(W)),
    b=Biases("Bias vector",
        lambda obj: (obj.nO,)),
    d_W=Gradient("W"),
    d_b=Gradient("b")
)
class PrecomputableAffine(Model):
    def __init__(self, nO=None, nI=None, nF=None, **kwargs):
        Model.__init__(self, **kwargs)
        self.nO = nO
        self.nI = nI
        self.nF = nF

    def begin_update(self, X, drop=0.):
        # X: (b, i)
        # Xf: (b, f, i)
        # dY: (b, o)
        # dYf: (b, f, o)
        #Yf = numpy.einsum('bi,ofi->bfo', X, self.W)
        Yf = self.ops.xp.tensordot(
                X, self.W, axes=[[1], [2]]).transpose((0, 2, 1))
        Yf += self.b
        def backward(dY_ids, sgd=None):
            dY, ids = dY_ids
            Xf = X[ids]

            #dW = numpy.einsum('bo,bfi->ofi', dY, Xf)
            dW = self.ops.xp.tensordot(dY, Xf, axes=[[0], [0]])
            db = dY.sum(axis=0)
            #dXf = numpy.einsum('bo,ofi->bfi', dY, self.W)
            dXf = self.ops.xp.tensordot(dY, self.W, axes=[[1], [0]])

            self.d_W += dW
            self.d_b += db

            if sgd is not None:
                sgd(self._mem.weights, self._mem.gradient, key=self.id)
            return dXf
        return Yf, backward

@describe.on_data(_set_dimensions_if_needed)
@describe.attributes(
    nI=Dimension("Input size"),
    nF=Dimension("Number of features"),
    nP=Dimension("Number of pieces"),
    nO=Dimension("Output size"),
    W=Synapses("Weights matrix",
        lambda obj: (obj.nF, obj.nO, obj.nP, obj.nI),
        lambda W, ops: ops.xavier_uniform_init(W)),
    b=Biases("Bias vector",
        lambda obj: (obj.nO, obj.nP)),
    d_W=Gradient("W"),
    d_b=Gradient("b")
)
class PrecomputableMaxouts(Model):
    def __init__(self, nO=None, nI=None, nF=None, pieces=3, **kwargs):
        Model.__init__(self, **kwargs)
        self.nO = nO
        self.nP = pieces
        self.nI = nI
        self.nF = nF

    def begin_update(self, X, drop=0.):
        # X: (b, i)
        # Yfp: (b, f, o, p)
        # Xf: (f, b, i)
        # dYp: (b, o, p)
        # W: (f, o, p, i)
        # b: (o, p)

        # bi,opfi->bfop
        # bop,fopi->bfi
        # bop,fbi->opfi : fopi

        tensordot = self.ops.xp.tensordot
        ascontiguous = self.ops.xp.ascontiguousarray

        Yfp = tensordot(X, self.W, axes=[[1], [3]])
        Yfp += self.b

        def backward(dYp_ids, sgd=None):
            dYp, ids = dYp_ids
            Xf = X[ids]

            dXf = tensordot(dYp, self.W, axes=[[1, 2], [1,2]])
            dW = tensordot(dYp, Xf, axes=[[0], [0]])

            self.d_W += dW.transpose((2, 0, 1, 3))
            self.d_b += dYp.sum(axis=0)

            if sgd is not None:
                sgd(self._mem.weights, self._mem.gradient, key=self.id)
            return dXf
        return Yfp, backward

def Tok2Vec(width, embed_size, preprocess=None):
    cols = [LOWER, PREFIX, SUFFIX, SHAPE]
    with Model.define_operators({'>>': chain, '|': concatenate, '**': clone, '+': add}):
        lower = get_col(cols.index(LOWER))   >> HashEmbed(width, embed_size)
        prefix = get_col(cols.index(PREFIX)) >> HashEmbed(width, embed_size//2)
        suffix = get_col(cols.index(SUFFIX)) >> HashEmbed(width, embed_size//2)
        shape = get_col(cols.index(SHAPE))   >> HashEmbed(width, embed_size//2)

        tok2vec = (
            flatten
            >> (lower | prefix | suffix | shape )
            >> Maxout(width, width*4, pieces=3)
            >> Residual(ExtractWindow(nW=1) >> Maxout(width, width*3))
            >> Residual(ExtractWindow(nW=1) >> Maxout(width, width*3))
            >> Residual(ExtractWindow(nW=1) >> Maxout(width, width*3))
            >> Residual(ExtractWindow(nW=1) >> Maxout(width, width*3))
        )
        if preprocess not in (False, None):
            tok2vec = preprocess >> tok2vec
        # Work around thinc API limitations :(. TODO: Revise in Thinc 7
        tok2vec.nO = width
    return tok2vec

def get_col(idx):
    def forward(X, drop=0.):
        if isinstance(X, numpy.ndarray):
            ops = NumpyOps()
        else:
            ops = CupyOps()
        output = ops.xp.ascontiguousarray(X[:, idx])
        def backward(y, sgd=None):
            dX = ops.allocate(X.shape)
            dX[:, idx] += y
            return dX
        return output, backward
    return layerize(forward)

def zero_init(model):
    def _hook(self, X, y=None):
        self.W.fill(0)
    model.on_data_hooks.append(_hook)
    return model

def doc2feats(cols=None):
    cols = [ID, LOWER, PREFIX, SUFFIX, SHAPE]
    def forward(docs, drop=0.):
        feats = []
        for doc in docs:
            if 'cached_feats' not in doc.user_data:
                doc.user_data['cached_feats'] = model.ops.asarray(
                                                doc.to_array(cols),
                                                dtype='uint64')
            feats.append(doc.user_data['cached_feats'])
        return feats, None
    model = layerize(forward)
    model.cols = cols
    return model

def print_shape(prefix):
    def forward(X, drop=0.):
        return X, lambda dX, **kwargs: dX
    return layerize(forward)

@layerize
def get_token_vectors(tokens_attrs_vectors, drop=0.):
    ops = Model.ops
    tokens, attrs, vectors = tokens_attrs_vectors
    def backward(d_output, sgd=None):
        return (tokens, d_output)
    return vectors, backward

@layerize
def flatten(seqs, drop=0.):
    if isinstance(seqs[0], numpy.ndarray):
        ops = NumpyOps()
    elif hasattr(CupyOps.xp, 'ndarray') and isinstance(seqs[0], CupyOps.xp.ndarray):
        ops = CupyOps()
    else:
        raise ValueError("Unable to flatten sequence of type %s" % type(seqs[0]))
    lengths = [len(seq) for seq in seqs]
    def finish_update(d_X, sgd=None):
        return ops.unflatten(d_X, lengths)
    X = ops.xp.vstack(seqs)
    return X, finish_update
<commit_msg>Fix significant bug in feature calculation -- off by 1<commit_after>from thinc.api import add, layerize, chain, clone, concatenate, with_flatten
from thinc.neural import Model, Maxout, Softmax, Affine
from thinc.neural._classes.hash_embed import HashEmbed
from thinc.neural.ops import NumpyOps, CupyOps

from thinc.neural._classes.convolution import ExtractWindow
from thinc.neural._classes.static_vectors import StaticVectors
from thinc.neural._classes.batchnorm import BatchNorm
from thinc.neural._classes.resnet import Residual
from thinc import describe
from thinc.describe import Dimension, Synapses, Biases, Gradient
from thinc.neural._classes.affine import _set_dimensions_if_needed

from .attrs import ID, LOWER, PREFIX, SUFFIX, SHAPE, TAG, DEP
from .tokens.doc import Doc

import numpy

@describe.on_data(_set_dimensions_if_needed)
@describe.attributes(
    nI=Dimension("Input size"),
    nF=Dimension("Number of features"),
    nO=Dimension("Output size"),
    W=Synapses("Weights matrix",
        lambda obj: (obj.nO, obj.nF, obj.nI),
        lambda W, ops: ops.xavier_uniform_init(W)),
    b=Biases("Bias vector",
        lambda obj: (obj.nO,)),
    d_W=Gradient("W"),
    d_b=Gradient("b")
)
class PrecomputableAffine(Model):
    def __init__(self, nO=None, nI=None, nF=None, **kwargs):
        Model.__init__(self, **kwargs)
        self.nO = nO
        self.nI = nI
        self.nF = nF

    def begin_update(self, X, drop=0.):
        # X: (b, i)
        # Xf: (b, f, i)
        # dY: (b, o)
        # dYf: (b, f, o)
        #Yf = numpy.einsum('bi,ofi->bfo', X, self.W)
        Yf = self.ops.xp.tensordot(
                X, self.W, axes=[[1], [2]]).transpose((0, 2, 1))
        Yf += self.b
        def backward(dY_ids, sgd=None):
            dY, ids = dY_ids
            Xf = X[ids]

            #dW = numpy.einsum('bo,bfi->ofi', dY, Xf)
            dW = self.ops.xp.tensordot(dY, Xf, axes=[[0], [0]])
            db = dY.sum(axis=0)
            #dXf = numpy.einsum('bo,ofi->bfi', dY, self.W)
            dXf = self.ops.xp.tensordot(dY, self.W, axes=[[1], [0]])

            self.d_W += dW
            self.d_b += db

            if sgd is not None:
                sgd(self._mem.weights, self._mem.gradient, key=self.id)
            return dXf
        return Yf, backward

@describe.on_data(_set_dimensions_if_needed)
@describe.attributes(
    nI=Dimension("Input size"),
    nF=Dimension("Number of features"),
    nP=Dimension("Number of pieces"),
    nO=Dimension("Output size"),
    W=Synapses("Weights matrix",
        lambda obj: (obj.nF, obj.nO, obj.nP, obj.nI),
        lambda W, ops: ops.xavier_uniform_init(W)),
    b=Biases("Bias vector",
        lambda obj: (obj.nO, obj.nP)),
    d_W=Gradient("W"),
    d_b=Gradient("b")
)
class PrecomputableMaxouts(Model):
    def __init__(self, nO=None, nI=None, nF=None, pieces=3, **kwargs):
        Model.__init__(self, **kwargs)
        self.nO = nO
        self.nP = pieces
        self.nI = nI
        self.nF = nF

    def begin_update(self, X, drop=0.):
        # X: (b, i)
        # Yfp: (b, f, o, p)
        # Xf: (f, b, i)
        # dYp: (b, o, p)
        # W: (f, o, p, i)
        # b: (o, p)

        # bi,opfi->bfop
        # bop,fopi->bfi
        # bop,fbi->opfi : fopi

        tensordot = self.ops.xp.tensordot
        ascontiguous = self.ops.xp.ascontiguousarray

        Yfp = tensordot(X, self.W, axes=[[1], [3]])
        Yfp += self.b

        def backward(dYp_ids, sgd=None):
            dYp, ids = dYp_ids
            Xf = X[ids]

            dXf = tensordot(dYp, self.W, axes=[[1, 2], [1,2]])
            dW = tensordot(dYp, Xf, axes=[[0], [0]])

            self.d_W += dW.transpose((2, 0, 1, 3))
            self.d_b += dYp.sum(axis=0)

            if sgd is not None:
                sgd(self._mem.weights, self._mem.gradient, key=self.id)
            return dXf
        return Yfp, backward

def Tok2Vec(width, embed_size, preprocess=None):
    cols = [ID, LOWER, PREFIX, SUFFIX, SHAPE]
    with Model.define_operators({'>>': chain, '|': concatenate, '**': clone, '+': add}):
        lower = get_col(cols.index(LOWER))   >> HashEmbed(width, embed_size)
        prefix = get_col(cols.index(PREFIX)) >> HashEmbed(width, embed_size//2)
        suffix = get_col(cols.index(SUFFIX)) >> HashEmbed(width, embed_size//2)
        shape = get_col(cols.index(SHAPE))   >> HashEmbed(width, embed_size//2)

        tok2vec = (
            flatten
            >> (lower | prefix | suffix | shape )
            >> Maxout(width, width*4, pieces=3)
            >> Residual(ExtractWindow(nW=1) >> Maxout(width, width*3))
            >> Residual(ExtractWindow(nW=1) >> Maxout(width, width*3))
            >> Residual(ExtractWindow(nW=1) >> Maxout(width, width*3))
            >> Residual(ExtractWindow(nW=1) >> Maxout(width, width*3))
        )
        if preprocess not in (False, None):
            tok2vec = preprocess >> tok2vec
        # Work around thinc API limitations :(. TODO: Revise in Thinc 7
        tok2vec.nO = width
    return tok2vec

def get_col(idx):
    def forward(X, drop=0.):
        if isinstance(X, numpy.ndarray):
            ops = NumpyOps()
        else:
            ops = CupyOps()
        output = ops.xp.ascontiguousarray(X[:, idx], dtype=X.dtype)
        def backward(y, sgd=None):
            dX = ops.allocate(X.shape)
            dX[:, idx] += y
            return dX
        return output, backward
    return layerize(forward)

def zero_init(model):
    def _hook(self, X, y=None):
        self.W.fill(0)
    model.on_data_hooks.append(_hook)
    return model

def doc2feats(cols=None):
    cols = [ID, LOWER, PREFIX, SUFFIX, SHAPE]
    def forward(docs, drop=0.):
        feats = []
        for doc in docs:
            if 'cached_feats' not in doc.user_data:
                doc.user_data['cached_feats'] = model.ops.asarray(
                                                    doc.to_array(cols),
                                                    dtype='uint64')
            feats.append(doc.user_data['cached_feats'])
            assert feats[-1].dtype == 'uint64'
        return feats, None
    model = layerize(forward)
    model.cols = cols
    return model

def print_shape(prefix):
    def forward(X, drop=0.):
        return X, lambda dX, **kwargs: dX
    return layerize(forward)

@layerize
def get_token_vectors(tokens_attrs_vectors, drop=0.):
    ops = Model.ops
    tokens, attrs, vectors = tokens_attrs_vectors
    def backward(d_output, sgd=None):
        return (tokens, d_output)
    return vectors, backward

@layerize
def flatten(seqs, drop=0.):
    if isinstance(seqs[0], numpy.ndarray):
        ops = NumpyOps()
    elif hasattr(CupyOps.xp, 'ndarray') and isinstance(seqs[0], CupyOps.xp.ndarray):
        ops = CupyOps()
    else:
        raise ValueError("Unable to flatten sequence of type %s" % type(seqs[0]))
    lengths = [len(seq) for seq in seqs]
    def finish_update(d_X, sgd=None):
        return ops.unflatten(d_X, lengths)
    X = ops.xp.vstack(seqs)
    return X, finish_update
<|endoftext|>


<commit_before>import click

@click.group()
def cli():
    """A simple tool for converting video files.

    By default filmalize operates on all files in the current directory. If
    necessary, you may specify an individual file or a different working
    directory. For more information please see filmalize.py COMMAND --help."""
    pass

@cli.command()
@click.option('-f', '--file', type=click.Path(exists=True), help='specify a file')
@click.option('-d', '--directory', type=click.Path(exists=True), help='specify a directory')
@click.option('-r', '--recursive', default=False, is_flag=True, help='operate recursively')
def display(file, directory, recursive):
    """Display information about video files"""
    click.echo('Display Info')

@cli.command()
@click.option('-f', '--file', type=click.Path(exists=True), help='specify a file')
@click.option('-d', '--directory', type=click.Path(exists=True), help='specify a directory')
@click.option('-r', '--recursive', default=False, is_flag=True, help='operate recursively')
def convert(file, directory, recursive):
    """Convert video files"""
    click.echo('Convert File(s)')

if __name__ == '__main__':
    cli()
<commit_msg>Deduplicated Options, Exclusivity, Prepare Files List<commit_after>import click
import os

def exclusive(ctx_params, exclusive_params, error_message):
    """Utility function for enforcing exclusivity between options.

    Call at the top of a click.group() or click.group.command() function.

    Args:
        ctx_params (dict): The context parameters to search.
        exclusive_params (list of strings): Mutually exclusive parameters.
        error_message (str): The error message to display.

    Raises:
        click.UsageError: If more than one exclusive parameter is present in
            the context parameters.

    Examples:
        exclusive(click.get_current_context().params, ['param1', 'param2'],
            'paramters param1 and param2 are mutually exclusive')

        exclusive({**ctx.params, **ctx.parent.params}, ['a', 'b'],
            'command option b may not be specified with application option a')

    """
    if sum([1 if ctx_params[p] else 0 for p in exclusive_params]) > 1:
        raise click.UsageError(error_message)

@click.group()
@click.option('-f', '--file', type=click.Path(exists=True),
    help='Specify a file.')
@click.option('-d', '--directory', type=click.Path(exists=True),
    help='Specify a directory.')
@click.option('-r', '--recursive', is_flag=True, help='Operate recursively.')
@click.pass_context
def cli(ctx, file, directory, recursive):
    """A simple tool for converting video files.

    By default filmalize operates on all files in the current directory. If
    desired, you may specify an individual file or a different working
    directory. Directory operation may be recursive. A command is required.

    """

    exclusive(ctx.params, ['file', 'directory'],
        'a file may not be specified with a directory')
    exclusive(ctx.params, ['file', 'recursive'],
        'a file may not be specified with the recursive flag')

    if file:
        ctx.obj['FILES'] = [file]
    else:
        directory = directory if directory else '.'
        if recursive:
            ctx.obj['FILES'] = [os.path.join(root, file) for root, dirs, files
                in os.walk(directory) for file in files]
        else:
            ctx.obj['FILES'] = [dir_entry.path for dir_entry in
                os.scandir(directory) if dir_entry.is_file()]

@cli.command()
@click.pass_context
def display(ctx):
    """Display information about video file(s)"""
    click.echo(ctx.obj['FILES'])

@cli.command()
@click.pass_context
def convert(ctx):
    """Convert video file(s)"""
    click.echo('Convert File(s)')

if __name__ == '__main__':
    cli(obj={})
<|endoftext|>


<commit_before>"""Find spark home, and initialize by adding pyspark to sys.path.

If SPARK_HOME is defined, it will be used to put pyspark on sys.path.
Otherwise, common locations for spark (currently only Homebrew's default) will be searched.
"""

from glob import glob
import os
import sys

__version__ = '0.0.5'

def find():
    """Find a local spark installation.

    Will first check the SPARK_HOME env variable, and otherwise
    search common installation locations, e.g. from homebrew
    """
    spark_home = os.environ.get('SPARK_HOME', None)

    if not spark_home:
        for path in [
            '/usr/local/opt/apache-spark/libexec', # OS X Homebrew
            # Any other common places to look?
        ]:
            if os.path.exists(path):
                spark_home = path
                break

    if not spark_home:
        raise ValueError("Couldn't find Spark, make sure SPARK_HOME env is set"
                         " or Spark is in an expected location (e.g. from homebrew installation).")

    return spark_home

def change_rc(spark_home, spark_python, py4j):
    """Persists changes to enviornment by changing shell config.

    Adds lines to .bashrc to set enviornment variables
    including the adding of dependencies to the system path. Will only
    edit this file if they already exist. Currently only works for bash.

    Parameters
    ----------
    spark_home : str
        Path to Spark installation.
    spark_python : str
        Path to python subdirectory of Spark installation.
    py4j : str
        Path to py4j library.
    """

    bashrc_location = os.path.expanduser("~/.bashrc")

    if os.path.isfile(bashrc_location):
        with open(bashrc_location, 'a') as bashrc:
            bashrc.write("\n# Added by findspark\n")
            bashrc.write("export SPARK_HOME=" + spark_home + "\n")
            bashrc.write("export PYTHONPATH=" + spark_python + ":" +
                         py4j + ":$PYTHONPATH\n\n")

def edit_ipython_profile(spark_home, spark_python, py4j):
    """Adds a startup file to the current IPython profile to import pyspark.

    The startup file sets the required enviornment variables and imports pyspark.

    Parameters
    ----------
    spark_home : str
        Path to Spark installation.
    spark_python : str
        Path to python subdirectory of Spark installation.
    py4j : str
        Path to py4j library.
    """
    from IPython import get_ipython
    ip = get_ipython()

    if ip:
        profile_dir = ip.profile_dir.location
    else:
        from IPython.utils.path import locate_profile
        profile_dir = locate_profile()

    startup_file_loc = os.path.join(profile_dir, "startup", "findspark.py")

    with open(startup_file_loc, 'w') as startup_file:
        #Lines of code to be run when IPython starts
        startup_file.write("import sys, os\n")
        startup_file.write("os.environ['SPARK_HOME'] = '" + spark_home + "'\n")
        startup_file.write("sys.path[:0] = " + str([spark_python, py4j]) + "\n")
        startup_file.write("import pyspark\n")

def init(spark_home=None, python_path=None, edit_rc=False, edit_profile=False):
    """Make pyspark importable.

    Sets environmental variables and adds dependencies to sys.path.
    If no Spark location is provided, will try to find an installation.

    Parameters
    ----------
    spark_home : str, optional, default = None
        Path to Spark installation, will try to find automatically
        if not provided.
    python_path : str, optional, default = None
        Path to Python for Spark workers (PYSPARK_PYTHON),
        will use the currently running Python if not provided.
    edit_rc : bool, optional, default = False
        Whether to attempt to persist changes by appending to shell
        config.
    edit_profile : bool, optional, default = False
        Whether to create an IPython startup file to automatically
        configure and import pyspark.
    """

    if not spark_home:
        spark_home = find()

    if not python_path:
        python_path = sys.executable

    # ensure SPARK_HOME is defined
    os.environ['SPARK_HOME'] = spark_home

    # ensure PYSPARK_PYTHON is defined
    os.environ['PYSPARK_PYTHON'] = python_path

    # add pyspark to sys.path
    spark_python = os.path.join(spark_home, 'python')
    py4j = glob(os.path.join(spark_python, 'lib', 'py4j-*.zip'))[0]
    sys.path[:0] = [spark_python, py4j]

    if edit_rc:
        change_rc(spark_home, spark_python, py4j)

    if edit_profile:
        edit_ipython_profile(spark_home, spark_python, py4j)
<commit_msg>docstring typos<commit_after>"""Find spark home, and initialize by adding pyspark to sys.path.

If SPARK_HOME is defined, it will be used to put pyspark on sys.path.
Otherwise, common locations for spark (currently only Homebrew's default) will be searched.
"""

from glob import glob
import os
import sys

__version__ = '0.0.5'

def find():
    """Find a local spark installation.

    Will first check the SPARK_HOME env variable, and otherwise
    search common installation locations, e.g. from homebrew
    """
    spark_home = os.environ.get('SPARK_HOME', None)

    if not spark_home:
        for path in [
            '/usr/local/opt/apache-spark/libexec', # OS X Homebrew
            # Any other common places to look?
        ]:
            if os.path.exists(path):
                spark_home = path
                break

    if not spark_home:
        raise ValueError("Couldn't find Spark, make sure SPARK_HOME env is set"
                         " or Spark is in an expected location (e.g. from homebrew installation).")

    return spark_home

def change_rc(spark_home, spark_python, py4j):
    """Persists changes to environment by changing shell config.

    Adds lines to .bashrc to set environment variables
    including the adding of dependencies to the system path. Will only
    edit this file if they already exist. Currently only works for bash.

    Parameters
    ----------
    spark_home : str
        Path to Spark installation.
    spark_python : str
        Path to python subdirectory of Spark installation.
    py4j : str
        Path to py4j library.
    """

    bashrc_location = os.path.expanduser("~/.bashrc")

    if os.path.isfile(bashrc_location):
        with open(bashrc_location, 'a') as bashrc:
            bashrc.write("\n# Added by findspark\n")
            bashrc.write("export SPARK_HOME=" + spark_home + "\n")
            bashrc.write("export PYTHONPATH=" + spark_python + ":" +
                         py4j + ":$PYTHONPATH\n\n")

def edit_ipython_profile(spark_home, spark_python, py4j):
    """Adds a startup file to the current IPython profile to import pyspark.

    The startup file sets the required environment variables and imports pyspark.

    Parameters
    ----------
    spark_home : str
        Path to Spark installation.
    spark_python : str
        Path to python subdirectory of Spark installation.
    py4j : str
        Path to py4j library.
    """
    from IPython import get_ipython
    ip = get_ipython()

    if ip:
        profile_dir = ip.profile_dir.location
    else:
        from IPython.utils.path import locate_profile
        profile_dir = locate_profile()

    startup_file_loc = os.path.join(profile_dir, "startup", "findspark.py")

    with open(startup_file_loc, 'w') as startup_file:
        #Lines of code to be run when IPython starts
        startup_file.write("import sys, os\n")
        startup_file.write("os.environ['SPARK_HOME'] = '" + spark_home + "'\n")
        startup_file.write("sys.path[:0] = " + str([spark_python, py4j]) + "\n")
        startup_file.write("import pyspark\n")

def init(spark_home=None, python_path=None, edit_rc=False, edit_profile=False):
    """Make pyspark importable.

    Sets environment variables and adds dependencies to sys.path.
    If no Spark location is provided, will try to find an installation.

    Parameters
    ----------
    spark_home : str, optional, default = None
        Path to Spark installation, will try to find automatically
        if not provided.
    python_path : str, optional, default = None
        Path to Python for Spark workers (PYSPARK_PYTHON),
        will use the currently running Python if not provided.
    edit_rc : bool, optional, default = False
        Whether to attempt to persist changes by appending to shell
        config.
    edit_profile : bool, optional, default = False
        Whether to create an IPython startup file to automatically
        configure and import pyspark.
    """

    if not spark_home:
        spark_home = find()

    if not python_path:
        python_path = sys.executable

    # ensure SPARK_HOME is defined
    os.environ['SPARK_HOME'] = spark_home

    # ensure PYSPARK_PYTHON is defined
    os.environ['PYSPARK_PYTHON'] = python_path

    # add pyspark to sys.path
    spark_python = os.path.join(spark_home, 'python')
    py4j = glob(os.path.join(spark_python, 'lib', 'py4j-*.zip'))[0]
    sys.path[:0] = [spark_python, py4j]

    if edit_rc:
        change_rc(spark_home, spark_python, py4j)

    if edit_profile:
        edit_ipython_profile(spark_home, spark_python, py4j)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from flask import Blueprint, render_template, g, request
from dataviva.apps.general.views import get_locale
from dataviva.api.hedu.models import Yc_hedu, Ybc_hedu
from dataviva.api.attrs.models import Bra, Course_hedu
from dataviva.api.hedu.services import Major, MajorUniversities, MajorMunicipalities
from dataviva import db
from sqlalchemy.sql.expression import func

mod = Blueprint('major', __name__,
                template_folder='templates',
                url_prefix='/<lang_code>/major',
                static_folder='static')

@mod.before_request
def before_request():
    g.page_type = 'category'

@mod.url_value_preprocessor
def pull_lang_code(endpoint, values):
    g.locale = values.pop('lang_code')

@mod.url_defaults
def add_language_code(endpoint, values):
    values.setdefault('lang_code', get_locale())

@mod.route('/<course_hedu_id>')
def index(course_hedu_id):
    bra_id = request.args.get('bra_id')

    max_year_query = db.session.query(
        func.max(Yc_hedu.year)).filter_by(course_hedu_id=course_hedu_id)

    if bra_id:
        major_service = Major(course_hedu_id, bra_id)
        universities_service = MajorUniversities(course_hedu_id, bra_id)
        municipalities_service = MajorMunicipalities(course_hedu_id, bra_id)

        rank_query = Ybc_hedu.query.filter(
            Ybc_hedu.year == max_year_query,
            Ybc_hedu.bra_id == bra_id,
            func.length(Ybc_hedu.course_hedu_id) == len(course_hedu_id))\
            .order_by(Ybc_hedu.enrolled.desc())
    else:
        major_service = Major(course_hedu_id, '')
        universities_service = MajorUniversities(course_hedu_id, '')
        municipalities_service = MajorMunicipalities(course_hedu_id, '')

        rank_query = Yc_hedu.query.filter(
            Yc_hedu.year == max_year_query,
            func.length(Yc_hedu.course_hedu_id) == len(course_hedu_id))\
            .order_by(Yc_hedu.enrolled.desc())

    rank = rank_query.all()

    if not bra_id:
        header = {
            'name': major_service.name(),
            'enrolled': major_service.enrolled(),
            'entrants': major_service.entrants(),
            'graduates': major_service.graduates(),
            'profile': major_service.profile(),
            'year': major_service.year(),
            'field_id': course_hedu_id[:2]
        }
    else:
        header = {
            'name': major_service.name(),
            'enrolled': major_service.enrolled(),
            'entrants': major_service.entrants(),
            'graduates': major_service.graduates(),
            'profile': major_service.profile(),
            'year': major_service.year(),
            'field_id': course_hedu_id[:2],
            'id': course_hedu_id,
            'bra_id': bra_id,
            'location_name': major_service.location_name()
        }

    body = {
        'university_with_more_enrolled': universities_service.university_with_more_enrolled(),
        'highest_enrolled_number_by_university': universities_service.highest_enrolled_number(),
        'municipality_with_more_enrolled': municipalities_service.municipality_with_more_enrolled(),
        'municipality_with_more_enrolled_state': municipalities_service.municipality_with_more_enrolled_state(),
        'highest_enrolled_number_by_municipality': municipalities_service.highest_enrolled_number(),
        'university_with_more_entrants': universities_service.university_with_more_entrants(),
        'highest_entrant_number_by_university': universities_service.highest_entrants_number(),
        'municipality_with_more_entrants': municipalities_service.municipality_with_more_entrants(),
        'municipality_with_more_entrants_state': municipalities_service.municipality_with_more_entrants_state(),
        'highest_entrant_number_by_municipality': municipalities_service.highest_entrants_number(),
        'university_with_more_graduates': universities_service.university_with_more_graduates(),
        'highest_graduate_number_by_university': universities_service.highest_graduates_number(),
        'municipality_with_more_graduates': municipalities_service.municipality_with_more_graduates(),
        'municipality_with_more_graduates_state': municipalities_service.municipality_with_more_graduates_state(),
        'highest_graduate_number_by_municipality': municipalities_service.highest_graduates_number()
    }

    for index, maj in enumerate(rank):
        if rank[index].course_hedu_id == course_hedu_id:
            header['rank'] = index + 1
            break

    location = Bra.query.filter(Bra.id == bra_id).first()

    major = Course_hedu.query.filter(Course_hedu.id == course_hedu_id).first()

    return render_template('major/index.html', header=header, body=body, location=location, major=major)

@mod.route('/<course_hedu_id>/graphs/<tab>', methods=['POST'])
def graphs(course_hedu_id, tab):
    bra = request.args.get('bra_id')
    major = Course_hedu.query.filter_by(id=course_hedu_id).first_or_404()
    location = Bra.query.filter_by(id=bra).first()
    return render_template('major/graphs-'+tab+'.html', major=major, location=location)
<commit_msg>Updates error treatment in major service.<commit_after># -*- coding: utf-8 -*-
from flask import Blueprint, render_template, g, request
from dataviva.apps.general.views import get_locale
from dataviva.api.hedu.models import Yc_hedu, Ybc_hedu
from dataviva.api.attrs.models import Bra, Course_hedu
from dataviva.api.hedu.services import Major, MajorUniversities, MajorMunicipalities
from dataviva import db
from sqlalchemy.sql.expression import func

mod = Blueprint('major', __name__,
                template_folder='templates',
                url_prefix='/<lang_code>/major',
                static_folder='static')

@mod.before_request
def before_request():
    g.page_type = 'category'

@mod.url_value_preprocessor
def pull_lang_code(endpoint, values):
    g.locale = values.pop('lang_code')

@mod.url_defaults
def add_language_code(endpoint, values):
    values.setdefault('lang_code', get_locale())

@mod.route('/<course_hedu_id>')
def index(course_hedu_id):
    bra_id = request.args.get('bra_id')

    max_year_query = db.session.query(
        func.max(Yc_hedu.year)).filter_by(course_hedu_id=course_hedu_id)

    if bra_id:
        major_service = Major(course_hedu_id, bra_id)
        universities_service = MajorUniversities(course_hedu_id, bra_id)
        municipalities_service = MajorMunicipalities(course_hedu_id, bra_id)

        rank_query = Ybc_hedu.query.filter(
            Ybc_hedu.year == max_year_query,
            Ybc_hedu.bra_id == bra_id,
            func.length(Ybc_hedu.course_hedu_id) == len(course_hedu_id))\
            .order_by(Ybc_hedu.enrolled.desc())
    else:
        major_service = Major(course_hedu_id, '')
        universities_service = MajorUniversities(course_hedu_id, '')
        municipalities_service = MajorMunicipalities(course_hedu_id, '')

        rank_query = Yc_hedu.query.filter(
            Yc_hedu.year == max_year_query,
            func.length(Yc_hedu.course_hedu_id) == len(course_hedu_id))\
            .order_by(Yc_hedu.enrolled.desc())

    rank = rank_query.all()
    hedu_max_year = db.session.query(func.max(Yc_hedu.year)).first()[0]

    if not bra_id:
        header = {
            'name': major_service.name(),
            'enrolled': major_service.enrolled(),
            'entrants': major_service.entrants(),
            'graduates': major_service.graduates(),
            'profile': major_service.profile(),
            'year': major_service.year(),
            'field_id': course_hedu_id[:2]
        }
    else:
        header = {
            'name': major_service.name(),
            'enrolled': major_service.enrolled(),
            'entrants': major_service.entrants(),
            'graduates': major_service.graduates(),
            'profile': major_service.profile(),
            'year': major_service.year(),
            'field_id': course_hedu_id[:2],
            'id': course_hedu_id,
            'bra_id': bra_id,
            'location_name': major_service.location_name()
        }

    body = {
        'university_with_more_enrolled': universities_service.university_with_more_enrolled(),
        'highest_enrolled_number_by_university': universities_service.highest_enrolled_number(),
        'municipality_with_more_enrolled': municipalities_service.municipality_with_more_enrolled(),
        'municipality_with_more_enrolled_state': municipalities_service.municipality_with_more_enrolled_state(),
        'highest_enrolled_number_by_municipality': municipalities_service.highest_enrolled_number(),
        'university_with_more_entrants': universities_service.university_with_more_entrants(),
        'highest_entrant_number_by_university': universities_service.highest_entrants_number(),
        'municipality_with_more_entrants': municipalities_service.municipality_with_more_entrants(),
        'municipality_with_more_entrants_state': municipalities_service.municipality_with_more_entrants_state(),
        'highest_entrant_number_by_municipality': municipalities_service.highest_entrants_number(),
        'university_with_more_graduates': universities_service.university_with_more_graduates(),
        'highest_graduate_number_by_university': universities_service.highest_graduates_number(),
        'municipality_with_more_graduates': municipalities_service.municipality_with_more_graduates(),
        'municipality_with_more_graduates_state': municipalities_service.municipality_with_more_graduates_state(),
        'highest_graduate_number_by_municipality': municipalities_service.highest_graduates_number()
    }

    for index, maj in enumerate(rank):
        if rank[index].course_hedu_id == course_hedu_id:
            header['rank'] = index + 1
            break

    location = Bra.query.filter(Bra.id == bra_id).first()

    major = Course_hedu.query.filter(Course_hedu.id == course_hedu_id).first()

    if header['enrolled'] is None or hedu_max_year != header['year']:
        abort(404)
    else:
        return render_template('major/index.html', header=header, body=body, location=location, major=major)

@mod.route('/<course_hedu_id>/graphs/<tab>', methods=['POST'])
def graphs(course_hedu_id, tab):
    bra = request.args.get('bra_id')
    major = Course_hedu.query.filter_by(id=course_hedu_id).first_or_404()
    location = Bra.query.filter_by(id=bra).first()
    return render_template('major/graphs-'+tab+'.html', major=major, location=location)
<|endoftext|>