<commit_before>from collections import defaultdict
import datetime
import sys
import threading
import tornado.ioloop
import tornado.web
import urllib
import uuid

from prisoner.workflow import PolicyProcessor, SocialObjectGateway

SERVER_URL = "http://localhost:8888"

class ExperimentBuilder(object):
	""" The ExperimentBuilder is the interface for bootstrapping an
	experiment with PRISONER. After instantiating an ExperimentBuilder,
	complete the following steps:

	- call provide_privacy_policy() with the path to your privacy policy XML
	  file
	- call provide_experimental_design() with the path to your experimental
	  design XML file
	- call authenticate_participant() with the id of the participant in this
	  session
	- call authenticate_providers() with a list of services which the
	  participant must authenticate with to participate
	- call build() to generate a pre-experiment flow, which allows
	  participants to review a human-readable version of your privacy
	  policy, and to authenticate themselves with providers as needed.
	"""
	def __init__(self):
		self.sog = SocialObjectGateway.SocialObjectsGateway()
		self.participant = None
		self.providers = None
		self.token = str(uuid.uuid4())
		self.session = defaultdict(dict)
		self.title = None
		self.contact = None
		self.connection_string = None
		self.last_touch = None

	def get_props(self, who_for=None):
		props = self.sog.persistence.props

		if not who_for:
			return props
		else:
			return props[who_for]

	def provide_db_string(self, db_string):
		self.connection_string = db_string

	def provide_title(self, title):
		""" The title of the experiment as presented to your
		participants.
		:param title: Friendly experiment title
		:type title: str """
		self.title = title

	def provide_contact(self, contact):
		""" How to contact someone in connection with this experiment,
		eg. an email address. This should be provided in a form that fits the following
		sentence construction:
			"Contact the researcher at <contact>."
		:param contact: Contact information
		:type contact: str
		"""
		self.contact = contact

	def provide_privacy_policy(self, policy):
		""" Provide the privacy policy for this experiment.

		:param policy: Path to privacy policy file
		:type policy: str
		"""
		self.sog.provide_privacy_policy(policy)

	def provide_experimental_design(self, exp_design):
		""" Provide the experimental design for this experiment.

		:param exp_design: Path to experimental design file
		:type exp_design: str
		"""
		self.sog.provide_experimental_design(exp_design,
		self.connection_string)

	def authenticate_participant(self, schema, participant_id):
		""" Provide the ID of the participant in this experiment. This
		participant must exist in the participant table for this
		experiment.

		:param participant_id: ID of participant
		:type participant_id: int
		"""
		participant = self.sog.persistence.get_participant(schema, participant_id)
		self.participant = participant
		self.sog.participant = participant
		return self.participant

	def authenticate_providers(self, providers):
		""" Provide a list of provider names this participant needs to
		be authenticated with to participate (eg. if they are only using
		a subset of providers all participants will be using, only include that subset
		in this list). When the experiment is built, each gateway will inject its own
		authentication logic.

		:param providers: List of providers to authenticate with
		:type providers: list[str]
		"""
		self.providers = providers

	def build_schema(self):
		""" Constructs the database schema (destroying whatever data
		might already exist). This places the database in a state in which participants
		may be registered, and experiments run, but does not return usable interfaces to
		the rest of the workflow (such as the SocialObjectGateway) """

		self.sog.persistence.do_build_schema(drop_first=True)

	def build(self, callback_url):
		""" Using the information provided by the participation client,
		instigate the experiment consent process. This does the
		following:

		- parse the experimental design and privacy policy and generate
		  a human-readable document, relevant to the participant, which
		  also lists which providers the participant will be asked to authenticate with
		- creates a temporary web server - the participation client must
		  access the returned URL using the cookie provided when the
		  ExperimentBuilder was instantiated
		- when the user consents to the policies, each service gateway
		  for which authentication is needed provides a URL to
		  authenticate with which the participant is asked to visit in
		turn (decorated by additional context from PRISONER for participants'
		confidence). Note, this URL must contain the entire
		authentication flow, so you may need to host this yourself, particularly if this
		involves two (or more) factor authentication as users are bounced between URLs
		(many authentication flows expect a URL callback). This flow
		must return a token to persist alongside the Participant.

		: param callback: A callable to be invoked only when consent is
		confirmed - ie. the entrypoint for the participation client
		:type callback: callable
		: returns: URL participant must visit to begin consent flow
		"""
		# does authenticated participant already have a meta row?
		# if so, consent already given, so skip consent and load auth,
		for provider in self.providers:
			provider_auth = self.sog.persistence.get_existing_provider_auth(self.sog.participant[0],
			provider)
			if provider_auth:
				can_auth = self.sog.restore_authentication(provider,
				provider_auth)
				if can_auth:
					self.providers.remove(provider)

		# no providers to authenticate
		if not self.providers:
			return callback_url

		# indicate server needs to do a consent flow
		self.exp_callback = callback_url
		return True

		# start server
		# DEPRECATED: No more ad-hoc server - this is handled
		# by the PRISONER WS server now!
		application = tornado.web.Application([
		(r"/", ConsentFlowHandler),
		(r"/confirm", ProviderAuthentHandler),
		(r"/complete", CompleteConsentHandler),
		(r".*",CallbackHandler),
		], builder=self)
		application.listen(8888)
		t = threading.Thread(target=tornado.ioloop.IOLoop.instance().start)
		t.start()
		return "%s/?pctoken=%s" % (SERVER_URL, self.token)
		# serve human readable policies

	def consent_confirmed(cookie):
		""" Called when user with given cookie accepts consent. If
		cookie is valid, continue the authentication flow for that participant.
		"""
		pass

class CompleteConsentHandler(tornado.web.RequestHandler):
	""" Called when the user has authenticated themselves with the last
	provider necessary. This completes the authentication flow and allows the
	experimental application to begin. """
	def get(self):
		builder = self.application.settings["builder"]

		callback_provider = self.get_argument("cbprovider")
		builder.sog.complete_authentication(callback_provider,
		self.request)

		# evoke callback
		if "http" in builder.exp_callback:
			self.redirect(builder.exp_callback)
		else:
			self.write(builder.exp_callback)

class ConsentFlowHandler(tornado.web.RequestHandler):
	""" This renders the human-readable representation of the privacy
	policy and ensures the participant understands the data requirements of the
	experimental application before providing consent. """
	def get(self):
		builder = self.application.settings["builder"]
		token = self.get_argument("pctoken")
		self.write("Stand back. We're doing science.
")
		if(token != builder.token):
			self.write("Token %s is not %s" % (token, self.token))
			return

		else:
			self.write("(human readable consent here)

" +\
			"Go here if you agree to " %builder.token +\
			"the invisible information here.")

class CallbackHandler(tornado.web.RequestHandler):
	""" Takes a parameter (callback), and calls the unescaped version of
	that URL (useful for baking nested params in a callback URL)
	"""
	def get(self):
		url = urllib.unquote(self.request.uri)
		url = url.replace("?token","&token") # this is an insane shim for a bug in LFM
		url = url.replace("?state","&state") # temp FB shim
		self.redirect(url)

class ProviderAuthentHandler(tornado.web.RequestHandler):
	""" Called during the authentication flow for each provider. Informs the
	participant about the service they are about to authenticate themselves with,
	then redirects to the appropriate URL for that service. """
	def get(self):
		builder = self.application.settings["builder"]
		token = self.get_argument("pctoken")
		providers = builder.providers
		try:
			current_provider = self.get_argument("provider")
			if current_provider not in providers:
				self.write("Invalid provider.")
				return
			providers.pop()
		except:
			current_provider = None
			self.write("For this experiment, we need you to login to some services.
")
			provider = providers[len(providers)-1]
			self.write("Login to"%(provider, token)+\
			 " %s" % provider)
			return

		if providers:
			callback = "%s/confirm?pctoken=%s&provider=%s&cbprovider=%s" % (SERVER_URL, token,
			providers[len(providers)-1], current_provider)
		else:
			callback = "%s/complete?pctoken=%s&cbprovider=%s" % (SERVER_URL, token,
			current_provider)

		try:
			callback_provider = self.get_argument("cbprovider")
			builder.sog.complete_authentication(callback_provider,
			self.request)
		except:
			pass

		url = builder.sog.request_authentication(current_provider,
		callback=urllib.quote(callback,safe=":/"))
		self.redirect(url)
<commit_msg>[ExperimentBuilder] docstrings updated<commit_after>from collections import defaultdict
import datetime
import sys
import threading
import tornado.ioloop
import tornado.web
import urllib
import uuid

from prisoner.workflow import PolicyProcessor, SocialObjectGateway

SERVER_URL = "http://localhost:8888"

class ExperimentBuilder(object):
	""" The ExperimentBuilder is the interface for bootstrapping an
	experiment with PRISONER. After instantiating an ExperimentBuilder,
	complete the following steps:

	- call provide_privacy_policy() with the path to your privacy policy XML
	  file
	- call provide_experimental_design() with the path to your experimental
	  design XML file
	- call authenticate_participant() with the id of the participant in this
	  session
	- call authenticate_providers() with a list of services which the
	  participant must authenticate with to participate
	- call build() to generate a pre-experiment flow, which allows
	  participants to review a human-readable version of your privacy
	  policy, and to authenticate themselves with providers as needed.
	"""
	def __init__(self):
		self.sog = SocialObjectGateway.SocialObjectsGateway()
		self.participant = None
		self.providers = None
		self.token = str(uuid.uuid4())
		self.session = defaultdict(dict)
		self.title = None
		self.contact = None
		self.connection_string = None
		self.last_touch = None

	def get_props(self, who_for=None):
		""" Retrieve the props for a given target (eg. PRISONER or a provider)

		:param who_for: the target to retrieve props for
		:type who_for: str
		"""

		props = self.sog.persistence.props

		if not who_for:
			return props
		else:
			return props[who_for]

	def provide_db_string(self, db_string):
		""" Set connection string for this experiment.

		:param db_string: conncetion string
		:type db_string: str
		"""

		self.connection_string = db_string

	def provide_title(self, title):
		""" The title of the experiment as presented to your
		participants.
		:param title: Friendly experiment title
		:type title: str """
		self.title = title

	def provide_contact(self, contact):
		""" How to contact someone in connection with this experiment,
		eg. an email address. This should be provided in a form that fits the following
		sentence construction:
			"Contact the researcher at <contact>."
		:param contact: Contact information
		:type contact: str
		"""
		self.contact = contact

	def provide_privacy_policy(self, policy):
		""" Provide the privacy policy for this experiment.

		:param policy: Path to privacy policy file
		:type policy: str
		"""
		self.sog.provide_privacy_policy(policy)

	def provide_experimental_design(self, exp_design):
		""" Provide the experimental design for this experiment.

		:param exp_design: Path to experimental design file
		:type exp_design: str
		"""
		self.sog.provide_experimental_design(exp_design,
		self.connection_string)

	def authenticate_participant(self, schema, participant_id):
		""" Provide the ID of the participant in this experiment. This
		participant must exist in the participant table for this
		experiment.

		:param participant_id: ID of participant
		:type participant_id: int
		"""
		participant = self.sog.persistence.get_participant(schema, participant_id)
		self.participant = participant
		self.sog.participant = participant
		return self.participant

	def authenticate_providers(self, providers):
		""" Provide a list of provider names this participant needs to
		be authenticated with to participate (eg. if they are only using
		a subset of providers all participants will be using, only include that subset
		in this list). When the experiment is built, each gateway will inject its own
		authentication logic.

		:param providers: List of providers to authenticate with
		:type providers: list[str]
		"""
		self.providers = providers

	def build_schema(self):
		""" Constructs the database schema (destroying whatever data
		might already exist). This places the database in a state in which participants
		may be registered, and experiments run, but does not return usable interfaces to
		the rest of the workflow (such as the SocialObjectGateway) """

		self.sog.persistence.do_build_schema(drop_first=True)

	def build(self, callback_url):
		""" Using the information provided by the participation client,
		instigate the experiment consent process. This does the
		following:

		- parse the experimental design and privacy policy and generate
		  a human-readable document, relevant to the participant, which
		  also lists which providers the participant will be asked to authenticate with
		- creates a temporary web server - the participation client must
		  access the returned URL using the cookie provided when the
		  ExperimentBuilder was instantiated
		- when the user consents to the policies, each service gateway
		  for which authentication is needed provides a URL to
		  authenticate with which the participant is asked to visit in
		turn (decorated by additional context from PRISONER for participants'
		confidence). Note, this URL must contain the entire
		authentication flow, so you may need to host this yourself, particularly if this
		involves two (or more) factor authentication as users are bounced between URLs
		(many authentication flows expect a URL callback). This flow
		must return a token to persist alongside the Participant.

		: param callback_url: A callable to be invoked only when consent is
		confirmed - ie. the entrypoint for the participation client
		:type callback_url: callable
		: returns: URL participant must visit to begin consent flow
		"""
		# does authenticated participant already have a meta row?
		# if so, consent already given, so skip consent and load auth,
		for provider in self.providers:
			provider_auth = self.sog.persistence.get_existing_provider_auth(self.sog.participant[0],
			provider)
			if provider_auth:
				can_auth = self.sog.restore_authentication(provider,
				provider_auth)
				if can_auth:
					self.providers.remove(provider)

		# no providers to authenticate
		if not self.providers:
			return callback_url

		# indicate server needs to do a consent flow
		self.exp_callback = callback_url
		return True

		# start server
		# DEPRECATED: No more ad-hoc server - this is handled
		# by the PRISONER WS server now!
		application = tornado.web.Application([
		(r"/", ConsentFlowHandler),
		(r"/confirm", ProviderAuthentHandler),
		(r"/complete", CompleteConsentHandler),
		(r".*",CallbackHandler),
		], builder=self)
		application.listen(8888)
		t = threading.Thread(target=tornado.ioloop.IOLoop.instance().start)
		t.start()
		return "%s/?pctoken=%s" % (SERVER_URL, self.token)
		# serve human readable policies

	def consent_confirmed(cookie):
		""" Called when user with given cookie accepts consent. If
		cookie is valid, continue the authentication flow for that participant.
		"""
		pass

class CompleteConsentHandler(tornado.web.RequestHandler):
	""" Called when the user has authenticated themselves with the last
	provider necessary. This completes the authentication flow and allows the
	experimental application to begin. """
	def get(self):
		builder = self.application.settings["builder"]

		callback_provider = self.get_argument("cbprovider")
		builder.sog.complete_authentication(callback_provider,
		self.request)

		# evoke callback
		if "http" in builder.exp_callback:
			self.redirect(builder.exp_callback)
		else:
			self.write(builder.exp_callback)

class ConsentFlowHandler(tornado.web.RequestHandler):
	""" This renders the human-readable representation of the privacy
	policy and ensures the participant understands the data requirements of the
	experimental application before providing consent. """
	def get(self):
		builder = self.application.settings["builder"]
		token = self.get_argument("pctoken")
		self.write("Stand back. We're doing science.
")
		if(token != builder.token):
			self.write("Token %s is not %s" % (token, self.token))
			return

		else:
			self.write("(human readable consent here)

" +\
			"Go here if you agree to " %builder.token +\
			"the invisible information here.")

class CallbackHandler(tornado.web.RequestHandler):
	""" Takes a parameter (callback), and calls the unescaped version of
	that URL (useful for baking nested params in a callback URL)
	"""
	def get(self):
		url = urllib.unquote(self.request.uri)
		url = url.replace("?token","&token") # this is an insane shim for a bug in LFM
		url = url.replace("?state","&state") # temp FB shim
		self.redirect(url)

class ProviderAuthentHandler(tornado.web.RequestHandler):
	""" Called during the authentication flow for each provider. Informs the
	participant about the service they are about to authenticate themselves with,
	then redirects to the appropriate URL for that service. """
	def get(self):
		builder = self.application.settings["builder"]
		token = self.get_argument("pctoken")
		providers = builder.providers
		try:
			current_provider = self.get_argument("provider")
			if current_provider not in providers:
				self.write("Invalid provider.")
				return
			providers.pop()
		except:
			current_provider = None
			self.write("For this experiment, we need you to login to some services.
")
			provider = providers[len(providers)-1]
			self.write("Login to"%(provider, token)+\
			 " %s" % provider)
			return

		if providers:
			callback = "%s/confirm?pctoken=%s&provider=%s&cbprovider=%s" % (SERVER_URL, token,
			providers[len(providers)-1], current_provider)
		else:
			callback = "%s/complete?pctoken=%s&cbprovider=%s" % (SERVER_URL, token,
			current_provider)

		try:
			callback_provider = self.get_argument("cbprovider")
			builder.sog.complete_authentication(callback_provider,
			self.request)
		except:
			pass

		url = builder.sog.request_authentication(current_provider,
		callback=urllib.quote(callback,safe=":/"))
		self.redirect(url)
<|endoftext|>


<commit_before>import cPickle as pickle

from map.models import CityBorder
from crime.models import CriminalRecord

def vectorize(grid_size=1000, period='monthly'):
    file = grid_size + 'grid_size_' + period + 'period_vector.p'
    try:
        vectors = pickle.load(open(file, "rb"))
    except (OSError, IOError) as e:
        city = CityBorder.objects.get(name='Chicago')
        grid = city.generateGrid(grid_size)
        vectors = []
        first_data = CriminalRecord.objects.first()
        last_data = CriminalRecord.objects.last()
        first_year = first_data.date.year
        first_month = first_data.date.month
        last_year = last_data.date.year
        last_month = last_data.date.month
        start = 12 * first_year + first_month - 1
        end = 12 * last_year + last_month
        for ym in range(start, end ):
            year, month = divmod( ym, 12 )
            month += 1
            print year, month
            vector = []
            for i in xrange(len(grid)):
                g = grid[i]
                crimes = CriminalRecord.objects.filter(
                    date__month=month, date__year=year,
                    location__intersects=g).count()
                has_crime = int(crimes > 0)
                vector.append(has_crime)
            vectors.append(vector)
        pickle.dump(vectors, open(file, "wb"))
    return vectors
<commit_msg>add a parameter 'new' to control if a new vector will be returned whether a vector file is present or not<commit_after>import cPickle as pickle

from map.models import CityBorder
from crime.models import CriminalRecord

def vectorize(grid_size=1000, period='monthly', new=False):
    file = grid_size + 'grid_size_' + period + 'period_vector.p'
    if new:
        try:
            vectors = pickle.load(open(file, "rb"))
        except (OSError, IOError) as e:
            city = CityBorder.objects.get(name='Chicago')
            grid = city.generateGrid(grid_size)
            vectors = []
            first_data = CriminalRecord.objects.first()
            last_data = CriminalRecord.objects.last()
            first_year = first_data.date.year
            first_month = first_data.date.month
            last_year = last_data.date.year
            last_month = last_data.date.month
            start = 12 * first_year + first_month - 1
            end = 12 * last_year + last_month
            for ym in range(start, end ):
                year, month = divmod( ym, 12 )
                month += 1
                print year, month
                vector = []
                for i in xrange(len(grid)):
                    g = grid[i]
                    crimes = CriminalRecord.objects.filter(
                        date__month=month, date__year=year,
                        location__intersects=g).count()
                    has_crime = int(crimes > 0)
                    vector.append(has_crime)
                vectors.append(vector)
            pickle.dump(vectors, open(file, "wb"))
        return vectors
    else:
        city = CityBorder.objects.get(name='Chicago')
        grid = city.generateGrid(grid_size)
        vectors = []
        first_data = CriminalRecord.objects.first()
        last_data = CriminalRecord.objects.last()
        first_year = first_data.date.year
        first_month = first_data.date.month
        last_year = last_data.date.year
        last_month = last_data.date.month
        start = 12 * first_year + first_month - 1
        end = 12 * last_year + last_month
        for ym in range(start, end ):
            year, month = divmod( ym, 12 )
            month += 1
            print year, month
            vector = []
            for i in xrange(len(grid)):
                g = grid[i]
                crimes = CriminalRecord.objects.filter(
                    date__month=month, date__year=year,
                    location__intersects=g).count()
                has_crime = int(crimes > 0)
                vector.append(has_crime)
            vectors.append(vector)
        pickle.dump(vectors, open(file, "wb"))

<|endoftext|>


<commit_before># test_sagews.py
# basic tests of sage worksheet using TCP protocol with sage_server
import socket
import conftest
import os
import re

from textwrap import dedent

class TestBasic:
    def test_connection_type(self, sagews):
        print("type %s"%type(sagews))
        assert isinstance(sagews, conftest.ConnectionJSON)
        return

    def test_set_file_env(self, exec2):
        code = "os.chdir(salvus.data[\'path\']);__file__=salvus.data[\'file\']"
        exec2(code)

    def test_assignment(self, exec2):
        code = "x = 42\nx\n"
        output = "42\n"
        exec2(code, output)

    def test_issue70(self, exec2):
        code = dedent(r"""
        for i in range(1):
            pass
        'x'
        """)
        output = dedent(r"""
        'x'
        """).lstrip()
        exec2(code, output)

    def test_issue819(self, exec2):
        code = dedent(r"""
        def never_called(a):
            print 'should not execute 1', a
            # comment
        # comment at indent 0
            print 'should not execute 2', a
        22
        """)
        output = "22\n"
        exec2(code, output)

    def test_search_doc(self, exec2):
        code = "search_doc('laurent')"
        html = "https://www.google.com/search\?q=site%3Adoc.sagemath.org\+laurent\&oq=site%3Adoc.sagemath.org"
        exec2(code, html_pattern = html)

    def test_show_doc(self, test_id, sagews):
        # issue 476
        code = "show?"
        patn = "import smc_sagews.graphics\nsmc_sagews.graphics.graph_to_d3_jsonable?"
        m = conftest.message.execute_code(code = code, id = test_id)
        sagews.send_json(m)
        typ, mesg = sagews.recv()
        assert typ == 'json'
        assert mesg['id'] == test_id
        assert 'code' in mesg
        assert 'source' in mesg['code']
        assert re.sub('\s+','',patn) in re.sub('\s+','',mesg['code']['source'])
        conftest.recv_til_done(sagews, test_id)

    # https://github.com/sagemathinc/smc/issues/1107
    def test_sage_underscore_1(self, exec2):
        exec2("2/5","2/5\n")
    def test_sage_underscore_2(self, exec2):
        exec2("_","2/5\n")

    # https://github.com/sagemathinc/smc/issues/978
    def test_mode_comments_1(self, exec2):
        exec2(dedent("""
        def f(s):
            print "s='%s'"%s"""))
    def test_mode_comments_2(self, exec2):
        exec2(dedent("""
        %f
        123
        # foo
        456"""), dedent("""
        s='123
        # foo
        456'
        """).lstrip())

    def test_block_parser(self, exec2):
        """
        .. NOTE::

            This function supplies a list of expected outputs to `exec2`.
        """
        exec2(dedent("""
        pi.n().round()
        [x for x in [1,2,3] if x<3]
        for z in ['a','b']:
            z
        else:
            z"""), ["3\n","[1, 2]\n","'a'\n'b'\n'b'\n"])

class TestIntrospect:
    # test names end with SMC issue number
    def test_sage_autocomplete_1188(self, execintrospect):
        execintrospect('2016.fa', ["ctor","ctorial"], "fa")
    def test_sage_autocomplete_295_setup(self, exec2):
        exec2("aaa=Rings()._super_categories_for_classes;len(aaa[0].axioms())","6\n")
    def test_sage_autocomplete_295a(self, execintrospect):
        execintrospect('for a in aa', ["a"], "aa")
    def test_sage_autocomplete_295b(self, execintrospect):
        execintrospect('3 * aa', ["a"], "aa")
    def test_sage_autocomplete_701_setup(self, exec2):
        exec2(dedent("""
        class Xyz:
            numerical_attribute = 42
        x1 = Xyz()
        x1.numerical_attribute.next_prime()"""),"43\n")
    def test_sage_autocomplete_701a(self, execintrospect):
        execintrospect('3 / x1.nu', ["merical_attribute"], "nu")
    def test_sage_autocomplete_701b(self, execintrospect):
        execintrospect('aa', ["a"], "aa")
    def test_sage_autocomplete_701c(self, execintrospect):
        execintrospect('[aa', ["a"], "aa")
    def test_sage_autocomplete_701d(self, execintrospect):
        execintrospect('( aa', ["a"], "aa")
    def test_sage_autocomplete_734a(self, execintrospect):
        f = '*_factors'
        execintrospect(f, ["cunningham_prime_factors", "prime_factors"], f)
    def test_sage_autocomplete_734b(self, execintrospect):
        f = '*le_pr*'
        execintrospect(f, ["next_probable_prime"], f)
    def test_sage_autocomplete_734c(self, execintrospect):
        execintrospect('list.re*e', ["remove", "reverse"], 're*e')

class TestAttach:
    def test_define_paf(self, exec2):
        exec2(dedent(r"""
        def paf():
            print("attached files: %d"%len(attached_files()))
            print("\n".join(attached_files()))
        paf()"""),"attached files: 0\n\n")
    def test_attach_sage_1(self, exec2):
        exec2("%attach a.sage\npaf()", pattern="attached files: 1\n.*/a.sage\n")
    def test_attach_sage_2(self, exec2):
        exec2("f1('foo')","f1 arg = 'foo'\ntest f1 1\n")
    def test_attach_py_1(self, exec2):
        exec2("%attach a.py\npaf()", pattern="attached files: 2\n.*/a.py\n.*/a.sage\n")
    def test_attach_py_2(self, exec2):
        exec2("f2('foo')","test f2 1\n")
    def test_attach_html_1(self, execblob):
        execblob("%attach a.html", want_html=False, want_javascript=True, file_type='html')
    def test_attach_html_2(self, exec2):
        exec2("paf()", pattern="attached files: 3\n.*/a.html\n.*/a.py\n.*/a.sage\n")
    def test_detach_1(self, exec2):
        exec2("detach(attached_files())")
    def test_detach_2(self, exec2):
        exec2("paf()","attached files: 0\n\n")

class TestSearchSrc:
    def test_search_src_simple(self, execinteract):
        execinteract('search_src("convolution")')

    def test_search_src_max_chars(self, execinteract):
        execinteract('search_src("full cremonadatabase", max_chars = 1000)')

class TestIdentifiers:
    """
    see SMC issue #63
    """
    def test_ident_set_file_env(self, exec2):
        """emulate initial code block sent from UI, needed for first show_identifiers"""
        code = "os.chdir(salvus.data[\'path\']);__file__=salvus.data[\'file\']"
        exec2(code)
    def test_show_identifiers_initial(self, exec2):
        exec2("show_identifiers()","[]\n")

    def test_show_identifiers_vars(self, exec2):
        code = dedent(r"""
        k = ['a','b','c']
        A = {'a':'foo','b':'bar','c':'baz'}
        z = 99
        sorted(show_identifiers())""")
        exec2(code, "['A', 'k', 'z']\n")

    def test_save_and_reset(self,exec2,data_path):
        code = dedent(r"""
        save_session('%s')
        reset()
        show_identifiers()""")%data_path.join('session').strpath
        exec2(code,"[]\n")
    def test_load_session1(self,exec2,data_path):
        code = dedent(r"""
        pretty_print = 8
        view = 9
        load_session('%s')
        sorted(show_identifiers())""")%data_path.join('session').strpath
        output = "['A', 'k', 'pretty_print', 'view', 'z']\n"
        exec2(code,output)
    def test_load_session2(self,exec2):
        exec2("pretty_print,view","(8, 9)\n")

    def test_redefine_sage(self,exec2):
        code = dedent(r"""
        reset()
        sage=1
        show_identifiers()""")
        exec2(code,"['sage']\n")
<commit_msg>tests for #1225<commit_after># test_sagews.py
# basic tests of sage worksheet using TCP protocol with sage_server
import socket
import conftest
import os
import re

from textwrap import dedent

class TestBasic:
    def test_connection_type(self, sagews):
        print("type %s"%type(sagews))
        assert isinstance(sagews, conftest.ConnectionJSON)
        return

    def test_set_file_env(self, exec2):
        code = "os.chdir(salvus.data[\'path\']);__file__=salvus.data[\'file\']"
        exec2(code)

    def test_assignment(self, exec2):
        code = "x = 42\nx\n"
        output = "42\n"
        exec2(code, output)

    def test_issue70(self, exec2):
        code = dedent(r"""
        for i in range(1):
            pass
        'x'
        """)
        output = dedent(r"""
        'x'
        """).lstrip()
        exec2(code, output)

    def test_issue819(self, exec2):
        code = dedent(r"""
        def never_called(a):
            print 'should not execute 1', a
            # comment
        # comment at indent 0
            print 'should not execute 2', a
        22
        """)
        output = "22\n"
        exec2(code, output)

    def test_search_doc(self, exec2):
        code = "search_doc('laurent')"
        html = "https://www.google.com/search\?q=site%3Adoc.sagemath.org\+laurent\&oq=site%3Adoc.sagemath.org"
        exec2(code, html_pattern = html)

    def test_show_doc(self, test_id, sagews):
        # issue 476
        code = "show?"
        patn = "import smc_sagews.graphics\nsmc_sagews.graphics.graph_to_d3_jsonable?"
        m = conftest.message.execute_code(code = code, id = test_id)
        sagews.send_json(m)
        typ, mesg = sagews.recv()
        assert typ == 'json'
        assert mesg['id'] == test_id
        assert 'code' in mesg
        assert 'source' in mesg['code']
        assert re.sub('\s+','',patn) in re.sub('\s+','',mesg['code']['source'])
        conftest.recv_til_done(sagews, test_id)

    # https://github.com/sagemathinc/smc/issues/1107
    def test_sage_underscore_1(self, exec2):
        exec2("2/5","2/5\n")
    def test_sage_underscore_2(self, exec2):
        exec2("_","2/5\n")

    # https://github.com/sagemathinc/smc/issues/978
    def test_mode_comments_1(self, exec2):
        exec2(dedent("""
        def f(s):
            print "s='%s'"%s"""))
    def test_mode_comments_2(self, exec2):
        exec2(dedent("""
        %f
        123
        # foo
        456"""), dedent("""
        s='123
        # foo
        456'
        """).lstrip())

    def test_block_parser(self, exec2):
        """
        .. NOTE::

            This function supplies a list of expected outputs to `exec2`.
        """
        exec2(dedent("""
        pi.n().round()
        [x for x in [1,2,3] if x<3]
        for z in ['a','b']:
            z
        else:
            z"""), ["3\n","[1, 2]\n","'a'\n'b'\n'b'\n"])

class TestIntrospect:
    # test names end with SMC issue number
    def test_sage_autocomplete_1188(self, execintrospect):
        execintrospect('2016.fa', ["ctor","ctorial"], "fa")
    def test_sage_autocomplete_295_setup(self, exec2):
        exec2("aaa=Rings()._super_categories_for_classes;len(aaa[0].axioms())","6\n")
    def test_sage_autocomplete_295a(self, execintrospect):
        execintrospect('for a in aa', ["a"], "aa")
    def test_sage_autocomplete_295b(self, execintrospect):
        execintrospect('3 * aa', ["a"], "aa")
    def test_sage_autocomplete_701_setup(self, exec2):
        exec2(dedent("""
        class Xyz:
            numerical_attribute = 42
        x1 = Xyz()
        x1.numerical_attribute.next_prime()"""),"43\n")
    def test_sage_autocomplete_701a(self, execintrospect):
        execintrospect('3 / x1.nu', ["merical_attribute"], "nu")
    def test_sage_autocomplete_701b(self, execintrospect):
        execintrospect('aa', ["a"], "aa")
    def test_sage_autocomplete_701c(self, execintrospect):
        execintrospect('[aa', ["a"], "aa")
    def test_sage_autocomplete_701d(self, execintrospect):
        execintrospect('( aa', ["a"], "aa")
    def test_sage_autocomplete_734a(self, execintrospect):
        f = '*_factors'
        execintrospect(f, ["cunningham_prime_factors", "prime_factors"], f)
    def test_sage_autocomplete_734b(self, execintrospect):
        f = '*le_pr*'
        execintrospect(f, ["next_probable_prime"], f)
    def test_sage_autocomplete_734c(self, execintrospect):
        execintrospect('list.re*e', ["remove", "reverse"], 're*e')
    def test_sage_autocomplete_1225a(self, execintrospect):
        execintrospect('z = 12.5 * units.len', ["gth"], 'len')
    def test_sage_autocomplete_1225b_setup(self, exec2):
        exec2(dedent("""
        class TC:
            def __init__(self, xval):
                self.x = xval
        y = TC(49)
        """))
    def test_sage_autocomplete_1225b(self, execintrospect):
        execintrospect('z = 12 * y.', ["x"], '')

class TestAttach:
    def test_define_paf(self, exec2):
        exec2(dedent(r"""
        def paf():
            print("attached files: %d"%len(attached_files()))
            print("\n".join(attached_files()))
        paf()"""),"attached files: 0\n\n")
    def test_attach_sage_1(self, exec2):
        exec2("%attach a.sage\npaf()", pattern="attached files: 1\n.*/a.sage\n")
    def test_attach_sage_2(self, exec2):
        exec2("f1('foo')","f1 arg = 'foo'\ntest f1 1\n")
    def test_attach_py_1(self, exec2):
        exec2("%attach a.py\npaf()", pattern="attached files: 2\n.*/a.py\n.*/a.sage\n")
    def test_attach_py_2(self, exec2):
        exec2("f2('foo')","test f2 1\n")
    def test_attach_html_1(self, execblob):
        execblob("%attach a.html", want_html=False, want_javascript=True, file_type='html')
    def test_attach_html_2(self, exec2):
        exec2("paf()", pattern="attached files: 3\n.*/a.html\n.*/a.py\n.*/a.sage\n")
    def test_detach_1(self, exec2):
        exec2("detach(attached_files())")
    def test_detach_2(self, exec2):
        exec2("paf()","attached files: 0\n\n")

class TestSearchSrc:
    def test_search_src_simple(self, execinteract):
        execinteract('search_src("convolution")')

    def test_search_src_max_chars(self, execinteract):
        execinteract('search_src("full cremonadatabase", max_chars = 1000)')

class TestIdentifiers:
    """
    see SMC issue #63
    """
    def test_ident_set_file_env(self, exec2):
        """emulate initial code block sent from UI, needed for first show_identifiers"""
        code = "os.chdir(salvus.data[\'path\']);__file__=salvus.data[\'file\']"
        exec2(code)
    def test_show_identifiers_initial(self, exec2):
        exec2("show_identifiers()","[]\n")

    def test_show_identifiers_vars(self, exec2):
        code = dedent(r"""
        k = ['a','b','c']
        A = {'a':'foo','b':'bar','c':'baz'}
        z = 99
        sorted(show_identifiers())""")
        exec2(code, "['A', 'k', 'z']\n")

    def test_save_and_reset(self,exec2,data_path):
        code = dedent(r"""
        save_session('%s')
        reset()
        show_identifiers()""")%data_path.join('session').strpath
        exec2(code,"[]\n")
    def test_load_session1(self,exec2,data_path):
        code = dedent(r"""
        pretty_print = 8
        view = 9
        load_session('%s')
        sorted(show_identifiers())""")%data_path.join('session').strpath
        output = "['A', 'k', 'pretty_print', 'view', 'z']\n"
        exec2(code,output)
    def test_load_session2(self,exec2):
        exec2("pretty_print,view","(8, 9)\n")

    def test_redefine_sage(self,exec2):
        code = dedent(r"""
        reset()
        sage=1
        show_identifiers()""")
        exec2(code,"['sage']\n")
<|endoftext|>


<commit_before>
## License for the Python wrapper
## ==============================

## Copyright (c) 2004 David M. Cooke <cookedm@physics.mcmaster.ca>

## Permission is hereby granted, free of charge, to any person obtaining a copy of
## this software and associated documentation files (the "Software"), to deal in
## the Software without restriction, including without limitation the rights to
## use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
## of the Software, and to permit persons to whom the Software is furnished to do
## so, subject to the following conditions:

## The above copyright notice and this permission notice shall be included in all
## copies or substantial portions of the Software.

## THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
## IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
## FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
## AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
## LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
## OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
## SOFTWARE.

## Modifications by Travis Oliphant and Enthought, Inc.  for inclusion in SciPy

from numpy import zeros, float64, array, int32
import _lbfgsb
import optimize
from numpy.compat import asbytes

__all__ = ['fmin_l_bfgs_b']

approx_fprime = optimize.approx_fprime

def fmin_l_bfgs_b(func, x0, fprime=None, args=(),
                  approx_grad=0,
                  bounds=None, m=10, factr=1e7, pgtol=1e-5,
                  epsilon=1e-8,
                  iprint=-1, maxfun=15000, disp=None):
    """
    Minimize a function func using the L-BFGS-B algorithm.

    Parameters
    ----------
    func : callable f(x,*args)
        Function to minimise.
    x0 : ndarray
        Initial guess.
    fprime : callable fprime(x,*args)
        The gradient of `func`.  If None, then `func` returns the function
        value and the gradient (``f, g = func(x, *args)``), unless
        `approx_grad` is True in which case `func` returns only ``f``.
    args : sequence
        Arguments to pass to `func` and `fprime`.
    approx_grad : bool
        Whether to approximate the gradient numerically (in which case
        `func` returns only the function value).
    bounds : list
        ``(min, max)`` pairs for each element in ``x``, defining
        the bounds on that parameter. Use None for one of ``min`` or
        ``max`` when there is no bound in that direction.
    m : int
        The maximum number of variable metric corrections
        used to define the limited memory matrix. (The limited memory BFGS
        method does not store the full hessian but uses this many terms in an
        approximation to it.)
    factr : float
        The iteration stops when
        ``(f^k - f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= factr * eps``,
        where ``eps`` is the machine precision, which is automatically
        generated by the code. Typical values for `factr` are: 1e12 for
        low accuracy; 1e7 for moderate accuracy; 10.0 for extremely
        high accuracy.
    pgtol : float
        The iteration will stop when
        ``max{|proj g_i | i = 1, ..., n} <= pgtol``
        where ``pg_i`` is the i-th component of the projected gradient.
    epsilon : float
        Step size used when `approx_grad` is True, for numerically
        calculating the gradient
    iprint : int
        Controls the frequency of output. ``iprint < 0`` means no output.
    disp : int, optional
        If zero, then no output.  If positive number, then this over-rides
        `iprint`.
    maxfun : int
        Maximum number of function evaluations.

    Returns
    -------
    x : ndarray
        Estimated position of the minimum.
    f : float
        Value of `func` at the minimum.
    d : dict
        Information dictionary.

        * d['warnflag'] is

          - 0 if converged,
          - 1 if too many function evaluations,
          - 2 if stopped for another reason, given in d['task']

        * d['grad'] is the gradient at the minimum (should be 0 ish)
        * d['funcalls'] is the number of function calls made.

    Notes
    -----
    License of L-BFGS-B (Fortran code):

    The version included here (in fortran code) is 2.1 (released in 1997).
    It was written by Ciyou Zhu, Richard Byrd, and Jorge Nocedal
    <nocedal@ece.nwu.edu>. It carries the following condition for use:

    This software is freely available, but we expect that all publications
    describing work using this software , or all commercial products using it,
    quote at least one of the references given below.

    References
    ----------
    * R. H. Byrd, P. Lu and J. Nocedal. A Limited Memory Algorithm for Bound
      Constrained Optimization, (1995), SIAM Journal on Scientific and
      Statistical Computing , 16, 5, pp. 1190-1208.
    * C. Zhu, R. H. Byrd and J. Nocedal. L-BFGS-B: Algorithm 778: L-BFGS-B,
      FORTRAN routines for large scale bound constrained optimization (1997),
      ACM Transactions on Mathematical Software, Vol 23, Num. 4, pp. 550 - 560.

    """
    n = len(x0)

    if bounds is None:
        bounds = [(None,None)] * n
    if len(bounds) != n:
        raise ValueError('length of x0 != length of bounds')

    if disp is not None:
        if disp == 0:
            iprint = -1
        else:
            iprint = disp

    if approx_grad:
        def func_and_grad(x):
            f = func(x, *args)
            g = approx_fprime(x, func, epsilon, *args)
            return f, g
    elif fprime is None:
        def func_and_grad(x):
            f, g = func(x, *args)
            return f, g
    else:
        def func_and_grad(x):
            f = func(x, *args)
            g = fprime(x, *args)
            return f, g

    nbd = zeros((n,), int32)
    low_bnd = zeros((n,), float64)
    upper_bnd = zeros((n,), float64)
    bounds_map = {(None, None): 0,
              (1, None) : 1,
              (1, 1) : 2,
              (None, 1) : 3}
    for i in range(0, n):
        l,u = bounds[i]
        if l is not None:
            low_bnd[i] = l
            l = 1
        if u is not None:
            upper_bnd[i] = u
            u = 1
        nbd[i] = bounds_map[l, u]

    x = array(x0, float64)
    f = array(0.0, float64)
    g = zeros((n,), float64)
    wa = zeros((2*m*n+4*n + 12*m**2 + 12*m,), float64)
    iwa = zeros((3*n,), int32)
    task = zeros(1, 'S60')
    csave = zeros(1,'S60')
    lsave = zeros((4,), int32)
    isave = zeros((44,), int32)
    dsave = zeros((29,), float64)

    task[:] = 'START'

    n_function_evals = 0
    while 1:
#        x, f, g, wa, iwa, task, csave, lsave, isave, dsave = \
        _lbfgsb.setulb(m, x, low_bnd, upper_bnd, nbd, f, g, factr,
                       pgtol, wa, iwa, task, iprint, csave, lsave,
                       isave, dsave)
        task_str = task.tostring()
        if task_str.startswith(asbytes('FG')):
            # minimization routine wants f and g at the current x
            n_function_evals += 1
            # Overwrite f and g:
            f, g = func_and_grad(x)
        elif task_str.startswith(asbytes('NEW_X')):
            # new iteration
            if n_function_evals > maxfun:
                task[:] = 'STOP: TOTAL NO. of f AND g EVALUATIONS EXCEEDS LIMIT'
        else:
            break

    task_str = task.tostring().strip(asbytes('\x00')).strip()
    if task_str.startswith(asbytes('CONV')):
        warnflag = 0
    elif n_function_evals > maxfun:
        warnflag = 1
    else:
        warnflag = 2

    d = {'grad' : g,
         'task' : task_str,
         'funcalls' : n_function_evals,
         'warnflag' : warnflag
        }
    return x, f, d

if __name__ == '__main__':
    def func(x):
        f = 0.25*(x[0]-1)**2
        for i in range(1, x.shape[0]):
            f += (x[i] - x[i-1]**2)**2
        f *= 4
        return f
    def grad(x):
        g = zeros(x.shape, float64)
        t1 = x[1] - x[0]**2
        g[0] = 2*(x[0]-1) - 16*x[0]*t1
        for i in range(1, g.shape[0]-1):
            t2 = t1
            t1 = x[i+1] - x[i]**2
            g[i] = 8*t2 - 16*x[i]*t1
        g[-1] = 8*t1
        return g

    factr = 1e7
    pgtol = 1e-5

    n=25
    m=10

    bounds = [(None,None)] * n
    for i in range(0, n, 2):
        bounds[i] = (1.0, 100)
    for i in range(1, n, 2):
        bounds[i] = (-100, 100)

    x0 = zeros((n,), float64)
    x0[:] = 3

    x, f, d = fmin_l_bfgs_b(func, x0, fprime=grad, m=m,
                            factr=factr, pgtol=pgtol)
    print x
    print f
    print d
    x, f, d = fmin_l_bfgs_b(func, x0, approx_grad=1,
                            m=m, factr=factr, pgtol=pgtol)
    print x
    print f
    print d
<commit_msg>ENH: array_like input to optimize.fmin_l_bfgs_b + some cosmetic changes.<commit_after>
## License for the Python wrapper
## ==============================

## Copyright (c) 2004 David M. Cooke <cookedm@physics.mcmaster.ca>

## Permission is hereby granted, free of charge, to any person obtaining a copy of
## this software and associated documentation files (the "Software"), to deal in
## the Software without restriction, including without limitation the rights to
## use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
## of the Software, and to permit persons to whom the Software is furnished to do
## so, subject to the following conditions:

## The above copyright notice and this permission notice shall be included in all
## copies or substantial portions of the Software.

## THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
## IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
## FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
## AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
## LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
## OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
## SOFTWARE.

## Modifications by Travis Oliphant and Enthought, Inc.  for inclusion in SciPy

from numpy import array, asarray, float64, int32, zeros
import _lbfgsb
from optimize import approx_fprime
from numpy.compat import asbytes

__all__ = ['fmin_l_bfgs_b']

def fmin_l_bfgs_b(func, x0, fprime=None, args=(),
                  approx_grad=0,
                  bounds=None, m=10, factr=1e7, pgtol=1e-5,
                  epsilon=1e-8,
                  iprint=-1, maxfun=15000, disp=None):
    """
    Minimize a function func using the L-BFGS-B algorithm.

    Parameters
    ----------
    func : callable f(x,*args)
        Function to minimise.
    x0 : ndarray
        Initial guess.
    fprime : callable fprime(x,*args)
        The gradient of `func`.  If None, then `func` returns the function
        value and the gradient (``f, g = func(x, *args)``), unless
        `approx_grad` is True in which case `func` returns only ``f``.
    args : sequence
        Arguments to pass to `func` and `fprime`.
    approx_grad : bool
        Whether to approximate the gradient numerically (in which case
        `func` returns only the function value).
    bounds : list
        ``(min, max)`` pairs for each element in ``x``, defining
        the bounds on that parameter. Use None for one of ``min`` or
        ``max`` when there is no bound in that direction.
    m : int
        The maximum number of variable metric corrections
        used to define the limited memory matrix. (The limited memory BFGS
        method does not store the full hessian but uses this many terms in an
        approximation to it.)
    factr : float
        The iteration stops when
        ``(f^k - f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= factr * eps``,
        where ``eps`` is the machine precision, which is automatically
        generated by the code. Typical values for `factr` are: 1e12 for
        low accuracy; 1e7 for moderate accuracy; 10.0 for extremely
        high accuracy.
    pgtol : float
        The iteration will stop when
        ``max{|proj g_i | i = 1, ..., n} <= pgtol``
        where ``pg_i`` is the i-th component of the projected gradient.
    epsilon : float
        Step size used when `approx_grad` is True, for numerically
        calculating the gradient
    iprint : int
        Controls the frequency of output. ``iprint < 0`` means no output.
    disp : int, optional
        If zero, then no output.  If positive number, then this over-rides
        `iprint`.
    maxfun : int
        Maximum number of function evaluations.

    Returns
    -------
    x : array_like
        Estimated position of the minimum.
    f : float
        Value of `func` at the minimum.
    d : dict
        Information dictionary.

        * d['warnflag'] is

          - 0 if converged,
          - 1 if too many function evaluations,
          - 2 if stopped for another reason, given in d['task']

        * d['grad'] is the gradient at the minimum (should be 0 ish)
        * d['funcalls'] is the number of function calls made.

    Notes
    -----
    License of L-BFGS-B (Fortran code):

    The version included here (in fortran code) is 2.1 (released in 1997).
    It was written by Ciyou Zhu, Richard Byrd, and Jorge Nocedal
    <nocedal@ece.nwu.edu>. It carries the following condition for use:

    This software is freely available, but we expect that all publications
    describing work using this software , or all commercial products using it,
    quote at least one of the references given below.

    References
    ----------
    * R. H. Byrd, P. Lu and J. Nocedal. A Limited Memory Algorithm for Bound
      Constrained Optimization, (1995), SIAM Journal on Scientific and
      Statistical Computing , 16, 5, pp. 1190-1208.
    * C. Zhu, R. H. Byrd and J. Nocedal. L-BFGS-B: Algorithm 778: L-BFGS-B,
      FORTRAN routines for large scale bound constrained optimization (1997),
      ACM Transactions on Mathematical Software, Vol 23, Num. 4, pp. 550 - 560.

    """
    x0 = asarray(x0).ravel()
    n, = x0.shape

    if bounds is None:
        bounds = [(None,None)] * n
    if len(bounds) != n:
        raise ValueError('length of x0 != length of bounds')

    if disp is not None:
        if disp == 0:
            iprint = -1
        else:
            iprint = disp

    if approx_grad:
        def func_and_grad(x):
            f = func(x, *args)
            g = approx_fprime(x, func, epsilon, *args)
            return f, g
    elif fprime is None:
        def func_and_grad(x):
            f, g = func(x, *args)
            return f, g
    else:
        def func_and_grad(x):
            f = func(x, *args)
            g = fprime(x, *args)
            return f, g

    nbd = zeros(n, int32)
    low_bnd = zeros(n, float64)
    upper_bnd = zeros(n, float64)
    bounds_map = {(None, None): 0,
              (1, None) : 1,
              (1, 1) : 2,
              (None, 1) : 3}
    for i in range(0, n):
        l,u = bounds[i]
        if l is not None:
            low_bnd[i] = l
            l = 1
        if u is not None:
            upper_bnd[i] = u
            u = 1
        nbd[i] = bounds_map[l, u]

    x = array(x0, float64)
    f = array(0.0, float64)
    g = zeros((n,), float64)
    wa = zeros(2*m*n+4*n + 12*m**2 + 12*m, float64)
    iwa = zeros(3*n, int32)
    task = zeros(1, 'S60')
    csave = zeros(1,'S60')
    lsave = zeros(4, int32)
    isave = zeros(44, int32)
    dsave = zeros(29, float64)

    task[:] = 'START'

    n_function_evals = 0
    while 1:
#        x, f, g, wa, iwa, task, csave, lsave, isave, dsave = \
        _lbfgsb.setulb(m, x, low_bnd, upper_bnd, nbd, f, g, factr,
                       pgtol, wa, iwa, task, iprint, csave, lsave,
                       isave, dsave)
        task_str = task.tostring()
        if task_str.startswith(asbytes('FG')):
            # minimization routine wants f and g at the current x
            n_function_evals += 1
            # Overwrite f and g:
            f, g = func_and_grad(x)
        elif task_str.startswith(asbytes('NEW_X')):
            # new iteration
            if n_function_evals > maxfun:
                task[:] = 'STOP: TOTAL NO. of f AND g EVALUATIONS EXCEEDS LIMIT'
        else:
            break

    task_str = task.tostring().strip(asbytes('\x00')).strip()
    if task_str.startswith(asbytes('CONV')):
        warnflag = 0
    elif n_function_evals > maxfun:
        warnflag = 1
    else:
        warnflag = 2

    d = {'grad' : g,
         'task' : task_str,
         'funcalls' : n_function_evals,
         'warnflag' : warnflag
        }
    return x, f, d

if __name__ == '__main__':
    def func(x):
        f = 0.25*(x[0]-1)**2
        for i in range(1, x.shape[0]):
            f += (x[i] - x[i-1]**2)**2
        f *= 4
        return f
    def grad(x):
        g = zeros(x.shape, float64)
        t1 = x[1] - x[0]**2
        g[0] = 2*(x[0]-1) - 16*x[0]*t1
        for i in range(1, g.shape[0]-1):
            t2 = t1
            t1 = x[i+1] - x[i]**2
            g[i] = 8*t2 - 16*x[i]*t1
        g[-1] = 8*t1
        return g

    factr = 1e7
    pgtol = 1e-5

    n=25
    m=10

    bounds = [(None,None)] * n
    for i in range(0, n, 2):
        bounds[i] = (1.0, 100)
    for i in range(1, n, 2):
        bounds[i] = (-100, 100)

    x0 = zeros((n,), float64)
    x0[:] = 3

    x, f, d = fmin_l_bfgs_b(func, x0, fprime=grad, m=m,
                            factr=factr, pgtol=pgtol)
    print x
    print f
    print d
    x, f, d = fmin_l_bfgs_b(func, x0, approx_grad=1,
                            m=m, factr=factr, pgtol=pgtol)
    print x
    print f
    print d
<|endoftext|>


<commit_before>import shutil
import json
import os.path
import time

from coverage.report import Reporter

DEFAULT_DIR = "clogs"
JSON_CLOGS_FILE = "clogs.json.js"
JSON_CLOGS_TEMPLATE = "var json_clogs = {clogs};"

class ClogReporter(Reporter):
    """
    A reporter that incrementally updates a JSON file, storing coverage data.

    """

    STATIC_DIR = "report/static"

    def __init__(self, coverage, clogger, ignore_errors=False):
        super(ClogReporter, self).__init__(coverage, ignore_errors)

        self.clogger = clogger
        self.directory = self.clogger.directory

        self.clogs = self._load_clogs(directory)
        self.branches = coverage.data.has_arcs()

    def coverage_info(self, cu):
        analysis = self.coverage._analyze(cu)

        info = {"numbers" : analysis.numbers,
                "name" : cu.name,
                "percent_coverage" : analysis.numbers.pc_covered_str,
                "statements" : analysis.numbers.n_statements,
                "missing" : analysis.numbers.n_missing,}

        if self.branches:
            info.update(branches=nums.n_branches,
                        missing_branches=nums.n_missing_branches)

        return info

    def report(self, morfs, config=None):
        self.find_code_units(morfs, config)
        units = [self.coverage_info(cu) for cu in self.code_units]

        # XXX : Use the time of coverage, not time of the report
        metadata = {"date" : time.time() * 1000,  # javascript needs milisecs
                    "coverage" : units,
                    "total" : len(units),}

    def _load_clogs(self):
        try:
            with open(os.path.join(self.directory, JSON_CLOGS_FILE)) as clogs:
                js_clogs = clogs.read()
        except OSError as e:
            if e.errno == errno.ENOENT:
                return []
            raise
        else:
            lsplit, clogs, rsplit = js_clogs.partition("{clogs}")
            self.clogs = json.loads(clogs.lsplit(lsplit).rsplit(rsplit))

    def _write_clogs(self):
        with open(os.path.join(self.directory, JSON_CLOGS_FILE), "w") as f:
            f.write(JSON_CLOGS_TEMPLATE.format(clogs=json.dumps(self.clogs))

    def write(self):
        static = os.path.join(self.directory, "static")
        shutil.copytree(self.STATIC_DIR, static)
        self._write_clogs()
<commit_msg>Typo.<commit_after>import shutil
import json
import os.path
import time

from coverage.report import Reporter

DEFAULT_DIR = "clogs"
JSON_CLOGS_FILE = "clogs.json.js"
JSON_CLOGS_TEMPLATE = "var json_clogs = {clogs};"

class ClogReporter(Reporter):
    """
    A reporter that incrementally updates a JSON file, storing coverage data.

    """

    STATIC_DIR = "report/static"

    def __init__(self, coverage, clogger, ignore_errors=False):
        super(ClogReporter, self).__init__(coverage, ignore_errors)

        self.clogger = clogger
        self.directory = self.clogger.directory

        self.clogs = self._load_clogs(directory)
        self.branches = coverage.data.has_arcs()

    def coverage_info(self, cu):
        analysis = self.coverage._analyze(cu)

        info = {"numbers" : analysis.numbers,
                "name" : cu.name,
                "percent_coverage" : analysis.numbers.pc_covered_str,
                "statements" : analysis.numbers.n_statements,
                "missing" : analysis.numbers.n_missing,}

        if self.branches:
            info.update(branches=nums.n_branches,
                        missing_branches=nums.n_missing_branches)

        return info

    def report(self, morfs, config=None):
        self.find_code_units(morfs, config)
        units = [self.coverage_info(cu) for cu in self.code_units]

        # XXX : Use the time of coverage, not time of the report
        metadata = {"date" : time.time() * 1000,  # javascript needs milisecs
                    "coverage" : units,
                    "total" : len(units),}

    def _load_clogs(self):
        try:
            with open(os.path.join(self.directory, JSON_CLOGS_FILE)) as clogs:
                js_clogs = clogs.read()
        except OSError as e:
            if e.errno == errno.ENOENT:
                return []
            raise
        else:
            lstrip, clogs, rstrip = js_clogs.partition("{clogs}")
            self.clogs = json.loads(clogs.lstrip(lstrip).rstrip(rstrip))

    def _write_clogs(self):
        with open(os.path.join(self.directory, JSON_CLOGS_FILE), "w") as f:
            f.write(JSON_CLOGS_TEMPLATE.format(clogs=json.dumps(self.clogs))

    def write(self):
        static = os.path.join(self.directory, "static")
        shutil.copytree(self.STATIC_DIR, static)
        self._write_clogs()
<|endoftext|>


<commit_before>"""
Tests for the Tumblr fetcher. Run with noveltorpedo/website/manage.py test
while in this directory.
"""

import fetch_tumblr

from datetime import datetime
from pytz import utc
from logging import INFO
from django.test import TestCase
from django.core.exceptions import ObjectDoesNotExist
from noveltorpedo import models

class MockBlog(object):
    """
    A fake Tumblr blog to be retrieved by the mock client.
    """
    def __init__(self, blog_name, blog_title, is_nsfw):
        """
        Create a new mock blog.

        Args:
            blog_name: The short name of the blog (string, no spaces).
            blog_title: The blog title (string, spaces OK).
            is_nsfw: Boolean.
        """
        super(MockBlog, self).__init__()
        self.name = blog_name
        self.title = blog_title
        self.is_nsfw = is_nsfw
        self.posts = []

    def add_post(self, title="", body="", timestamp=None):
        """
        Add a post to the mock blog.

        Args:
            title: A string, defaults to empty.
            body: A string, defaults to empty.
            timestamp: UNIX timestamp, defaults to now.
        """
        post = {}
        post["title"] = title
        post["body"] = body
        if timestamp is None:
            post["timestamp"] = (datetime.now() -
                                 datetime.fromtimestamp(0)).total_seconds()
        else:
            post["timestamp"] = timestamp
        self.posts.append(post)

class MockTumblrClient(object):
    """
    A mockup of the Tumblr API client provided by pytumblr, providing
    only the subset of its functionality used by fetch_tumblr.
    """
    def __init__(self):
        """
        Create a mock client and initialize the blog dict.
        """
        super(MockTumblrClient, self).__init__()
        self.blogs = {}

    def add_blog(self, blog):
        """
        Add a new mock blog so that it can be retrieved later.

        Args:
            blog: A MockBlog instance which may have posts in it.
        """
        self.blogs[blog.name] = blog

    def blog_info(self, blog_name):
        """
        Fetch general information about the specified blog. This mimics
        the pytumblr client's blog_info call. To make blogs available for
        finding, add them first with add_blog.

        Args:
            blog_name: The short name of a MockBlog.

        Returns:
            A dictionary with one key, "blog", whose value is another
            dictionary with keys including "title" and "body" (strings)
            and "is_nsfw" (boolean).

        Raises:
            KeyError: The requested MockBlog hasn't been added to the client.
        """
        return {"blog":self.blogs[blog_name].__dict__}

    def posts(self, blog_name, **kwargs):
        """
        Fetch the content of posts in the specified blog. This method assumes
        it's being called by fetch_tumblr.get_posts and has all the keyword
        arguments that function uses.

        Args:
            blog_name: The short_name or url of a MockBlog.
            offset: int, how many posts back (from most recent) to start.
            limit: int, how many posts to return.
            type: Ignored, should always be "text".
            filter: Ignored, should always be "text".

        Returns:
            A dictionary with one key, "posts", whose value is a list of
            dictionaries which each include the keys "title" and "body" (strings)
            and "timestamp" (a UNIX timestamp).

        Raises:
            KeyError: The requested MockBlog hasn't been added to the client.
        """
        # These are incidentally type assertions.
        assert(kwargs["type"] == "text")
        assert(kwargs["filter"] == "text")
        assert(kwargs["offset"] >= 0)
        assert(0 <= kwargs["limit"] <= fetch_tumblr.MAX_POSTS)
        # fetch_tumblr will pass blog urls, chop off the ".tumblr.com" part.
        blog_name = blog_name.split(".", 1)[0]
        all_posts = self.blogs[blog_name].posts
        # In production code I'd store them sorted instead, but in tests the
        # insert:retrieve ratio is so much higher that I think this is faster.
        all_posts.sort(key=lambda p: p["timestamp"], reverse=True)
        start = kwargs["offset"]
        end = kwargs["offset"] + kwargs["limit"]
        return {"posts":all_posts[start:end]}

class TumblrTests(TestCase):
    """
    Tests for fetch_tumblr.py.
    """
    def setUp(self):
        """
        Preparation to do before every test.
        """
        # We don't need to sleep between calls when just running tests.
        fetch_tumblr.host_attrs["wait"] = 0
        # Create the host entry for Tumblr in the database.
        fetch_tumblr.tumblr = models.Host.objects.get_or_create(**fetch_tumblr.host_attrs)[0]
        # Replace the pytumblr client with a mockup.
        fetch_tumblr.client = MockTumblrClient()
        # Don't spam us with debug messages.
        fetch_tumblr.logger.setLevel(INFO)

    def add_simple_blog(self):
        """
        Insert a fake blog with one post.
        """
        mock_blog = MockBlog("mock_blog", "My Mockup Blog", False)
        mock_blog.add_post("Title of the Post", "Contents of the post.")
        fetch_tumblr.client.add_blog(mock_blog)

    def add_long_blog(self, count=111):
        """
        Insert a fake blog with many posts that have sequential timestamps.
        """
        long_blog = MockBlog("long_blog", "A Really Long Blog", False)
        for i in xrange(1, count+1):
            long_blog.add_post("Post #{0}".format(i), "", i)
        fetch_tumblr.client.add_blog(long_blog)

    def add_empty_blog(self):
        """
        Insert a fake blog with no posts.
        """
        empty_blog = MockBlog("empty_blog", "Shhhh.", False)
        fetch_tumblr.client.add_blog(empty_blog)

    def add_nsfw_blog(self):
        """
        Insert a fake blog that's marked NSFW.
        """
        nsfw_blog = MockBlog("nsfw_blog", "Don't Read This at Work!", True)
        nsfw_blog.add_post("A Raunchy Post", "This is not work-appropriate.")
        fetch_tumblr.client.add_blog(nsfw_blog)

    def make_segments(self, blog_name):
        """
	Retrieve all story segments corresponding to a blog name from the
        database. Requires both that the blog has been added to the client
        with MockTumblrClient.add_blog(), and calls fetch_tumblr.update_story()
        to create the segments (if the blog has any posts in it).

        Args:
            blog_name: The short name of a MockBlog.

        Returns:
            A django QuerySet of StorySegment objects.
        """
        sh = fetch_tumblr.get_or_create_storyhost(blog_name)
        fetch_tumblr.update_story(sh)
        return models.StorySegment.objects.filter(story=sh.story)

    def test_get_create_storyhost(self):
        """
        Verify that get_or_create_storyhost does both of those things and
        returns the same StoryHost object for the same input.
        """
        self.add_simple_blog()
        created_sh = fetch_tumblr.get_or_create_storyhost("mock_blog")
        retrieved_sh = fetch_tumblr.get_or_create_storyhost("mock_blog")
        self.assertIsInstance(created_sh, models.StoryHost)
        self.assertEqual(created_sh, retrieved_sh)

    def test_no_such_storyhost(self):
        """
        Verify that the appropriate exception is raised when the requested
        blog doesn't exist.
        """
        # No blogs exist at all.
        self.assertRaises(fetch_tumblr.TumblrNotFound,
                          fetch_tumblr.get_or_create_storyhost,
                          "nonexistent_blog")
        self.add_simple_blog()
        # A blog exists, but not the one we're looking for.
        self.assertRaises(fetch_tumblr.TumblrNotFound,
                          fetch_tumblr.get_or_create_storyhost,
                          "nonexistent_blog")
        self.add_simple_blog()

    def test_correct_storyhost(self):
        """
        Verify that get_or_create_storyhost inserts accurate blog info.
        """
        # Add some blogs to our imaginary Tumblr.
        self.add_simple_blog()
        self.add_nsfw_blog()
        blogs = fetch_tumblr.client.blogs
        for blog_name in ("mock_blog", "nsfw_blog"):
            # Create a storyhost in the database.
            fetch_tumblr.get_or_create_storyhost(blog_name)
            # Retrieve the storyhost from the database.
            sh = models.StoryHost.objects.get(url=blog_name+".tumblr.com")
            story = sh.story
            author = models.Author.objects.get(name=blog_name)
            # Check the storyhost's attributes.
            self.assertEqual(sh.host, fetch_tumblr.tumblr)
            # Last scraped should be the oldest possible, because we haven't yet.
            self.assertEqual(sh.last_scraped, datetime.min.replace(tzinfo=utc))
            # Check the story attributes.
            self.assertEqual(story.title, blogs[blog_name].title)
            # Tumblrs have exactly one author.
            self.assertEqual(story.authors.count(), 1)
            self.assertIn(author, story.authors.all())
            if blog_name is "nsfw_blog":
                # The following verifies there's exactly one attribute.
                attr = models.StoryAttribute.objects.get(story=story)
                self.assertEqual(attr.key, "nsfw")
                self.assertEqual(attr.value, "TRUE")
            else:
                # Doing this instead of assertRaises because of import path
                # weirdness, c.f. http://stackoverflow.com/q/549677/7619818
                # (If you can figure out how to make the plain assertRaises
                # work, by all means change this!)
                try:
                    attr = models.StoryAttribute.objects.get(story=story)
                except ObjectDoesNotExist as e:
                    # This is the superclass for all DoesNotExist errors.
                    self.assertIsInstance(e, models.StoryAttribute.DoesNotExist)

    def test_get_posts(self):
        """
        Verify post retrieval from the client. This is honestly a test of the
        mock itself more than the fetcher, which is just a passthrough, but it
        does assert (in MockTumblrClient.posts()) that the right arguments are
        passed through.
        """
        self.add_long_blog(50)
        posts = fetch_tumblr.get_posts("long_blog")
        # No limit was specified, so we should've gotten the max.
        self.assertEqual(len(posts), fetch_tumblr.MAX_POSTS)
        # This is set in add_long_blog; what we're actually doing here
        # is verifying that the default offset was correct (0).
        self.assertEqual(posts[0]["timestamp"], 50)
        posts = fetch_tumblr.get_posts("long_blog", offset=1)
        self.assertEqual(posts[0]["timestamp"], 49)
        self.assertEqual(len(posts), 20)
        posts = fetch_tumblr.get_posts("long_blog", offset=100)
        self.assertEqual(len(posts), 0)
        posts = fetch_tumblr.get_posts("long_blog", offset=49)
        self.assertEqual(len(posts), 1)
        self.assertEqual(posts[0]["timestamp"], 1)
        posts = fetch_tumblr.get_posts("long_blog", limit=1)
        self.assertEqual(len(posts), 1)

    def test_update_finds_all(self):
        """
        Verify that update_story() will get all the posts, even if that takes
        many queries and doesn't end on an even multiple of the limit.
        """
        # A nice big prime number of posts.
        self.add_long_blog(111)
        segments = self.make_segments("long_blog")
        self.assertEqual(segments.count(), 111)
        oldest = segments.earliest("published")
        newest = segments.latest("published")
        self.assertEqual(oldest.title, "Post #1")
        self.assertEqual(newest.title, "Post #111")

    def test_update_gets_new(self):
        """
        Verify that update_story() adds new posts and only new posts.
        """
        self.add_simple_blog()
        segments = self.make_segments("mock_blog")
        self.assertEqual(segments.count(), 1)
        # Calling update again doesn't add any posts.
        segments = self.make_segments("mock_blog")
        self.assertEqual(segments.count(), 1)
        # Add another (empty) post, now there should be two.
        fetch_tumblr.client.blogs["mock_blog"].add_post()
        segments = self.make_segments("mock_blog")
        self.assertEqual(segments.count(), 2)

    def test_no_posts_no_problem(self):
        """
        Verify that update_story() handles empty blogs.
        """
        self.add_empty_blog()
        segments = self.make_segments("empty_blog")
        self.assertEqual(segments.count(), 0)
        # Really the assertion for this test is that no other error was raised.

    def test_segment_contents(self):
        """
        Verify that the inserted segment matches the post data.
        """
        self.add_empty_blog()
        post = {
            "title": "For a Good",
            "body": "Timestamp, call:",
            "timestamp": 8675309
            # (April 11, 1970, if you were wondering)
        }
        fetch_tumblr.client.blogs["empty_blog"].add_post(**post)
        segment = self.make_segments("empty_blog").first()
        self.assertEqual(segment.title, post["title"])
        self.assertEqual(segment.contents, post["body"])
        dt = datetime.fromtimestamp(post["timestamp"]).replace(tzinfo=utc)
        self.assertEqual(segment.published, dt)

    def test_last_scraped(self):
        """
        Verify that updating a story changes its last_scraped, whether or not
        new posts were found.
        """
        self.add_simple_blog()
        sh = fetch_tumblr.get_or_create_storyhost("mock_blog")
        old_scrape_date = sh.last_scraped
        fetch_tumblr.update_story(sh)
        self.assertNotEqual(old_scrape_date, sh.last_scraped)
        old_scrape_date = sh.last_scraped
        fetch_tumblr.client.blogs["mock_blog"].add_post()
        fetch_tumblr.update_story(sh)
        self.assertNotEqual(old_scrape_date, sh.last_scraped)
<commit_msg>PEP8 cleanup in tests.<commit_after>"""
Tests for the Tumblr fetcher. Run with noveltorpedo/website/manage.py test
while in this directory.
"""

import fetch_tumblr

from datetime import datetime
from pytz import utc
from logging import INFO
from django.test import TestCase
from django.core.exceptions import ObjectDoesNotExist
from noveltorpedo import models

class MockBlog(object):
    """
    A fake Tumblr blog to be retrieved by the mock client.
    """
    def __init__(self, blog_name, blog_title, is_nsfw):
        """
        Create a new mock blog.

        Args:
            blog_name: The short name of the blog (string, no spaces).
            blog_title: The blog title (string, spaces OK).
            is_nsfw: Boolean.
        """
        super(MockBlog, self).__init__()
        self.name = blog_name
        self.title = blog_title
        self.is_nsfw = is_nsfw
        self.posts = []

    def add_post(self, title="", body="", timestamp=None):
        """
        Add a post to the mock blog.

        Args:
            title: A string, defaults to empty.
            body: A string, defaults to empty.
            timestamp: UNIX timestamp, defaults to now.
        """
        post = {}
        post["title"] = title
        post["body"] = body
        if timestamp is None:
            post["timestamp"] = (datetime.now() -
                                 datetime.fromtimestamp(0)).total_seconds()
        else:
            post["timestamp"] = timestamp
        self.posts.append(post)

class MockTumblrClient(object):
    """
    A mockup of the Tumblr API client provided by pytumblr, providing
    only the subset of its functionality used by fetch_tumblr.
    """
    def __init__(self):
        """
        Create a mock client and initialize the blog dict.
        """
        super(MockTumblrClient, self).__init__()
        self.blogs = {}

    def add_blog(self, blog):
        """
        Add a new mock blog so that it can be retrieved later.

        Args:
            blog: A MockBlog instance which may have posts in it.
        """
        self.blogs[blog.name] = blog

    def blog_info(self, blog_name):
        """
        Fetch general information about the specified blog. This mimics
        the pytumblr client's blog_info call. To make blogs available for
        finding, add them first with add_blog.

        Args:
            blog_name: The short name of a MockBlog.

        Returns:
            A dictionary with one key, "blog", whose value is another
            dictionary with keys including "title" and "body" (strings)
            and "is_nsfw" (boolean).

        Raises:
            KeyError: The requested MockBlog hasn't been added to the client.
        """
        return {"blog": self.blogs[blog_name].__dict__}

    def posts(self, blog_name, **kwargs):
        """
        Fetch the content of posts in the specified blog. This method assumes
        it's being called by fetch_tumblr.get_posts and has all the keyword
        arguments that function uses.

        Args:
            blog_name: The short_name or url of a MockBlog.
            offset: int, how many posts back (from most recent) to start.
            limit: int, how many posts to return.
            type: Ignored, should always be "text".
            filter: Ignored, should always be "text".

        Returns:
            A dictionary with one key, "posts", whose value is a list of
            dictionaries which each include the keys "title" and "body"
            (strings) and "timestamp" (a UNIX timestamp).

        Raises:
            KeyError: The requested MockBlog hasn't been added to the client.
        """
        # These are incidentally type assertions.
        assert(kwargs["type"] == "text")
        assert(kwargs["filter"] == "text")
        assert(kwargs["offset"] >= 0)
        assert(0 <= kwargs["limit"] <= fetch_tumblr.MAX_POSTS)
        # fetch_tumblr will pass blog urls, chop off the ".tumblr.com" part.
        blog_name = blog_name.split(".", 1)[0]
        all_posts = self.blogs[blog_name].posts
        # In production code I'd store them sorted instead, but in tests the
        # insert:retrieve ratio is so much higher that I think this is faster.
        all_posts.sort(key=lambda p: p["timestamp"], reverse=True)
        start = kwargs["offset"]
        end = kwargs["offset"] + kwargs["limit"]
        return {"posts": all_posts[start:end]}

class TumblrTests(TestCase):
    """
    Tests for fetch_tumblr.py.
    """
    def setUp(self):
        """
        Preparation to do before every test.
        """
        # We don't need to sleep between calls when just running tests.
        fetch_tumblr.host_attrs["wait"] = 0
        # Create the host entry for Tumblr in the database.
        attrs = fetch_tumblr.host_attrs
        fetch_tumblr.tumblr = models.Host.objects.get_or_create(**attrs)[0]
        # Replace the pytumblr client with a mockup.
        fetch_tumblr.client = MockTumblrClient()
        # Don't spam us with debug messages.
        fetch_tumblr.logger.setLevel(INFO)

    def add_simple_blog(self):
        """
        Insert a fake blog with one post.
        """
        mock_blog = MockBlog("mock_blog", "My Mockup Blog", False)
        mock_blog.add_post("Title of the Post", "Contents of the post.")
        fetch_tumblr.client.add_blog(mock_blog)

    def add_long_blog(self, count=111):
        """
        Insert a fake blog with many posts that have sequential timestamps.
        """
        long_blog = MockBlog("long_blog", "A Really Long Blog", False)
        for i in xrange(1, count+1):
            long_blog.add_post("Post #{0}".format(i), "", i)
        fetch_tumblr.client.add_blog(long_blog)

    def add_empty_blog(self):
        """
        Insert a fake blog with no posts.
        """
        empty_blog = MockBlog("empty_blog", "Shhhh.", False)
        fetch_tumblr.client.add_blog(empty_blog)

    def add_nsfw_blog(self):
        """
        Insert a fake blog that's marked NSFW.
        """
        nsfw_blog = MockBlog("nsfw_blog", "Don't Read This at Work!", True)
        nsfw_blog.add_post("A Raunchy Post", "This is not work-appropriate.")
        fetch_tumblr.client.add_blog(nsfw_blog)

    def make_segments(self, blog_name):
        """
        Retrieve all story segments corresponding to a blog name from the
        database. Requires both that the blog has been added to the client
        with MockTumblrClient.add_blog(), and calls fetch_tumblr.update_story()
        to create the segments (if the blog has any posts in it).

        Args:
            blog_name: The short name of a MockBlog.

        Returns:
            A django QuerySet of StorySegment objects.
        """
        sh = fetch_tumblr.get_or_create_storyhost(blog_name)
        fetch_tumblr.update_story(sh)
        return models.StorySegment.objects.filter(story=sh.story)

    def test_get_create_storyhost(self):
        """
        Verify that get_or_create_storyhost does both of those things and
        returns the same StoryHost object for the same input.
        """
        self.add_simple_blog()
        created_sh = fetch_tumblr.get_or_create_storyhost("mock_blog")
        retrieved_sh = fetch_tumblr.get_or_create_storyhost("mock_blog")
        self.assertIsInstance(created_sh, models.StoryHost)
        self.assertEqual(created_sh, retrieved_sh)

    def test_no_such_storyhost(self):
        """
        Verify that the appropriate exception is raised when the requested
        blog doesn't exist.
        """
        # No blogs exist at all.
        self.assertRaises(fetch_tumblr.TumblrNotFound,
                          fetch_tumblr.get_or_create_storyhost,
                          "nonexistent_blog")
        self.add_simple_blog()
        # A blog exists, but not the one we're looking for.
        self.assertRaises(fetch_tumblr.TumblrNotFound,
                          fetch_tumblr.get_or_create_storyhost,
                          "nonexistent_blog")
        self.add_simple_blog()

    def test_correct_storyhost(self):
        """
        Verify that get_or_create_storyhost inserts accurate blog info.
        """
        # Add some blogs to our imaginary Tumblr.
        self.add_simple_blog()
        self.add_nsfw_blog()
        blogs = fetch_tumblr.client.blogs
        for blog_name in ("mock_blog", "nsfw_blog"):
            # Create a storyhost in the database.
            fetch_tumblr.get_or_create_storyhost(blog_name)
            # Retrieve the storyhost from the database.
            sh = models.StoryHost.objects.get(url=blog_name+".tumblr.com")
            story = sh.story
            author = models.Author.objects.get(name=blog_name)
            # Check the storyhost's attributes.
            self.assertEqual(sh.host, fetch_tumblr.tumblr)
            # Last scraped should be the oldest possible, because we haven't.
            self.assertEqual(sh.last_scraped,
                             datetime.min.replace(tzinfo=utc))
            # Check the story attributes.
            self.assertEqual(story.title, blogs[blog_name].title)
            # Tumblrs have exactly one author.
            self.assertEqual(story.authors.count(), 1)
            self.assertIn(author, story.authors.all())
            if blog_name is "nsfw_blog":
                # The following verifies there's exactly one attribute.
                attr = models.StoryAttribute.objects.get(story=story)
                self.assertEqual(attr.key, "nsfw")
                self.assertEqual(attr.value, "TRUE")
            else:
                # Doing this instead of assertRaises because of import path
                # weirdness, c.f. http://stackoverflow.com/q/549677/7619818
                # (If you can figure out how to make the plain assertRaises
                # work, by all means change this!)
                try:
                    attr = models.StoryAttribute.objects.get(story=story)
                except ObjectDoesNotExist as e:
                    # This is the superclass for all DoesNotExist errors.
                    self.assertIsInstance(e,
                                          models.StoryAttribute.DoesNotExist)

    def test_get_posts(self):
        """
        Verify post retrieval from the client. This is honestly a test of the
        mock itself more than the fetcher, which is just a passthrough, but it
        does assert (in MockTumblrClient.posts()) that the right arguments are
        passed through.
        """
        self.add_long_blog(50)
        posts = fetch_tumblr.get_posts("long_blog")
        # No limit was specified, so we should've gotten the max.
        self.assertEqual(len(posts), fetch_tumblr.MAX_POSTS)
        # This is set in add_long_blog; what we're actually doing here
        # is verifying that the default offset was correct (0).
        self.assertEqual(posts[0]["timestamp"], 50)
        posts = fetch_tumblr.get_posts("long_blog", offset=1)
        self.assertEqual(posts[0]["timestamp"], 49)
        self.assertEqual(len(posts), 20)
        posts = fetch_tumblr.get_posts("long_blog", offset=100)
        self.assertEqual(len(posts), 0)
        posts = fetch_tumblr.get_posts("long_blog", offset=49)
        self.assertEqual(len(posts), 1)
        self.assertEqual(posts[0]["timestamp"], 1)
        posts = fetch_tumblr.get_posts("long_blog", limit=1)
        self.assertEqual(len(posts), 1)

    def test_update_finds_all(self):
        """
        Verify that update_story() will get all the posts, even if that takes
        many queries and doesn't end on an even multiple of the limit.
        """
        # A nice big prime number of posts.
        self.add_long_blog(111)
        segments = self.make_segments("long_blog")
        self.assertEqual(segments.count(), 111)
        oldest = segments.earliest("published")
        newest = segments.latest("published")
        self.assertEqual(oldest.title, "Post #1")
        self.assertEqual(newest.title, "Post #111")

    def test_update_gets_new(self):
        """
        Verify that update_story() adds new posts and only new posts.
        """
        self.add_simple_blog()
        segments = self.make_segments("mock_blog")
        self.assertEqual(segments.count(), 1)
        # Calling update again doesn't add any posts.
        segments = self.make_segments("mock_blog")
        self.assertEqual(segments.count(), 1)
        # Add another (empty) post, now there should be two.
        fetch_tumblr.client.blogs["mock_blog"].add_post()
        segments = self.make_segments("mock_blog")
        self.assertEqual(segments.count(), 2)

    def test_no_posts_no_problem(self):
        """
        Verify that update_story() handles empty blogs.
        """
        self.add_empty_blog()
        segments = self.make_segments("empty_blog")
        self.assertEqual(segments.count(), 0)
        # Really the assertion for this test is that no other error was raised.

    def test_segment_contents(self):
        """
        Verify that the inserted segment matches the post data.
        """
        self.add_empty_blog()
        post = {
            "title": "For a Good",
            "body": "Timestamp, call:",
            "timestamp": 8675309
            # (April 11, 1970, if you were wondering)
        }
        fetch_tumblr.client.blogs["empty_blog"].add_post(**post)
        segment = self.make_segments("empty_blog").first()
        self.assertEqual(segment.title, post["title"])
        self.assertEqual(segment.contents, post["body"])
        dt = datetime.fromtimestamp(post["timestamp"]).replace(tzinfo=utc)
        self.assertEqual(segment.published, dt)

    def test_last_scraped(self):
        """
        Verify that updating a story changes its last_scraped, whether or not
        new posts were found.
        """
        self.add_simple_blog()
        sh = fetch_tumblr.get_or_create_storyhost("mock_blog")
        old_scrape_date = sh.last_scraped
        fetch_tumblr.update_story(sh)
        self.assertNotEqual(old_scrape_date, sh.last_scraped)
        old_scrape_date = sh.last_scraped
        fetch_tumblr.client.blogs["mock_blog"].add_post()
        fetch_tumblr.update_story(sh)
        self.assertNotEqual(old_scrape_date, sh.last_scraped)
<|endoftext|>


<commit_before>"""update lib_raid_errors table

Revision ID: 93ff199763ac
Revises: b1063869f198
Create Date: 2017-07-27 00:13:29.765073+00:00

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '93ff199763ac'
down_revision = 'b1063869f198'
branch_labels = None
depends_on = None

def upgrade():
    op.drop_constraint('lib_raid_errors_pkey', 'lib_raid_errors', type_='primary')

    op.add_column('lib_raid_errors', sa.Column('id', sa.BigInteger(),
                  autoincrement=True))
    op.create_primary_key("lib_raid_errors_pkey", "lib_raid_errors", ["id", ])
    # ### end Alembic commands ###

def downgrade():
    op.drop_constraint('lib_raid_errors_pkey', 'lib_raid_errors', type_='primary')
    op.drop_column('lib_raid_errors', 'id')

    op.create_primary_key("lib_raid_errors_pkey", "lib_raid_errors", ["time", "hostname", "disk"])

    # ### end Alembic commands ###
<commit_msg>update lib_raid_errors alembic revision to properly add autoincrement<commit_after>"""update lib_raid_errors table

Revision ID: 93ff199763ac
Revises: b1063869f198
Create Date: 2017-07-27 00:13:29.765073+00:00

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.schema import Sequence, CreateSequence

# revision identifiers, used by Alembic.
revision = '93ff199763ac'
down_revision = 'b1063869f198'
branch_labels = None
depends_on = None

def upgrade():
    op.drop_constraint('lib_raid_errors_pkey', 'lib_raid_errors', type_='primary')

    op.execute(CreateSequence(Sequence("lib_raid_errors_id_seq")))
    op.add_column('lib_raid_errors', sa.Column('id', sa.BigInteger(), nullable=False,
                  server_default=sa.text("nextval('lib_raid_errors_id_seq'::regclass)")))

    op.create_primary_key("lib_raid_errors_pkey", "lib_raid_errors", ["id", ])
    # ### end Alembic commands ###

def downgrade():
    op.drop_constraint('lib_raid_errors_pkey', 'lib_raid_errors', type_='primary')
    op.drop_column('lib_raid_errors', 'id')

    op.create_primary_key("lib_raid_errors_pkey", "lib_raid_errors", ["time", "hostname", "disk"])

    # ### end Alembic commands ###
<|endoftext|>