<commit_before>#!/usr/bin/python
# coding: utf8

import geocoder
import pytest
import unittest

address = '453 Booth Street, Ottawa'
location = 'Ottawa, Ontario'
ip = '74.125.226.99'
repeat = 3
ottawa = (45.4215296, -75.6971930)
toronto = (43.653226, -79.3831843)

def test_entry_points():
    geocoder.ip
    geocoder.osm
    geocoder.bing
    geocoder.nokia
    geocoder.yahoo
    geocoder.google
    geocoder.tomtom
    geocoder.arcgis
    geocoder.geonames
    geocoder.mapquest
    geocoder.timezone
    geocoder.maxmind
    geocoder.elevation
    geocoder.freegeoip
    geocoder.geolytica
    geocoder.timezone
    geocoder.opencage
    geocoder.elevation
    geocoder.canadapost

"""
Server Down

def test_freegeoip():
    g = geocoder.freegeoip(ip)
    assert g.ok
"""

def test_maxmind():
    g = geocoder.maxmind(ip)
    assert g.ok

def test_google():
    g = geocoder.google(location)
    assert g.ok

def test_google_reverse():
    g = geocoder.google(ottawa, method='reverse')
    assert g.ok

def test_google_timezone():
    g = geocoder.google(ottawa, method='timezone')
    assert g.ok

def test_google_elevation():
    g = geocoder.google(ottawa, method='elevation')
    assert g.ok

def test_bing():
    g = geocoder.bing(location)
    assert g.ok

def test_bing_reverse():
    g = geocoder.bing(ottawa, method='reverse')
    assert g.ok

def test_opencage():
    g = geocoder.opencage(location)
    assert g.ok

def test_opencage_reverse():
    g = geocoder.opencage(ottawa, method='reverse')
    assert g.ok

def test_yahoo():
    g = geocoder.yahoo(location)
    assert g.ok

def test_arcgis():
    g = geocoder.arcgis(location)
    assert g.ok

def test_geolytica():
    g = geocoder.geolytica(address)
    assert g.ok

def test_canadapost():
    g = geocoder.canadapost(address)
    assert g.ok

def test_nokia():
    g = geocoder.nokia(location)
    assert g.ok

def test_osm():
    g = geocoder.osm(location)
    assert g.ok

def test_tomtom():
    g = geocoder.tomtom(location)
    assert g.ok

def test_mapquest():
    g = geocoder.mapquest(location)
    assert g.ok

def test_geonames():
    g = geocoder.geonames(location)
    assert g.ok
    <commit_msg>Tests: assert city name (to detect bytes/str errors in Python 3)<commit_after>#!/usr/bin/python
# coding: utf8

import geocoder
import pytest
import unittest

address = '453 Booth Street, Ottawa'
location = 'Ottawa, Ontario'
city = 'Ottawa'
ip = '74.125.226.99'
repeat = 3
ottawa = (45.4215296, -75.6971930)
toronto = (43.653226, -79.3831843)

def test_entry_points():
    geocoder.ip
    geocoder.osm
    geocoder.bing
    geocoder.nokia
    geocoder.yahoo
    geocoder.google
    geocoder.tomtom
    geocoder.arcgis
    geocoder.geonames
    geocoder.mapquest
    geocoder.timezone
    geocoder.maxmind
    geocoder.elevation
    geocoder.freegeoip
    geocoder.geolytica
    geocoder.timezone
    geocoder.opencage
    geocoder.elevation
    geocoder.canadapost

"""
Server Down

def test_freegeoip():
    g = geocoder.freegeoip(ip)
    assert g.ok
"""

def test_maxmind():
    g = geocoder.maxmind(ip)
    assert g.ok

def test_google():
    g = geocoder.google(location)
    assert g.ok
    assert g.city == city

def test_google_reverse():
    g = geocoder.google(ottawa, method='reverse')
    assert g.ok

def test_google_timezone():
    g = geocoder.google(ottawa, method='timezone')
    assert g.ok

def test_google_elevation():
    g = geocoder.google(ottawa, method='elevation')
    assert g.ok

def test_bing():
    g = geocoder.bing(location)
    assert g.ok
    assert g.city == city

def test_bing_reverse():
    g = geocoder.bing(ottawa, method='reverse')
    assert g.ok

def test_opencage():
    g = geocoder.opencage(location)
    assert g.ok
    assert g.city == city

def test_opencage_reverse():
    g = geocoder.opencage(ottawa, method='reverse')
    assert g.ok

def test_yahoo():
    g = geocoder.yahoo(location)
    assert g.ok
    assert g.city == city

def test_arcgis():
    g = geocoder.arcgis(location)
    assert g.ok

def test_geolytica():
    g = geocoder.geolytica(address)
    assert g.ok

def test_canadapost():
    g = geocoder.canadapost(address)
    assert g.ok

def test_nokia():
    g = geocoder.nokia(location)
    assert g.ok
    assert g.city == city

def test_osm():
    g = geocoder.osm(location)
    assert g.ok
    assert g.city == city

def test_tomtom():
    g = geocoder.tomtom(location)
    assert g.ok
    assert g.city == city

def test_mapquest():
    g = geocoder.mapquest(location)
    assert g.ok
    assert g.city == city

def test_geonames():
    g = geocoder.geonames(location)
    assert g.ok
<|endoftext|>


<commit_before>import argparse
import mock
import subprocess
import unittest

import git_gate

class TestParseArgs(unittest.TestCase):

    def test_project_and_url(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project",
             "--project-url", "https://git.testing/project"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.project_url, "https://git.testing/project")

    def test_project_and_ref(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project", "--go-get-all",
             "--project-ref", "v1"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.project_ref, "v1")

    def test_merging_other(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project", "--go-get-all",
             "--merge-url", "https://git.testing/proposed"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.merge_url, "https://git.testing/proposed")
        self.assertEqual(args.merge_ref, "HEAD")

    def test_merging_other_ref(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project", "--go-get-all",
             "--merge-url", "https://git.testing/proposed",
             "--merge-ref", "feature"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.merge_url, "https://git.testing/proposed")
        self.assertEqual(args.merge_ref, "feature")

    def test_project_with_deps(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project",
             "--project-url", "https://git.testing/project",
             "--dependencies", "git.testing/a", "git.testing/b"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.project_url, "https://git.testing/project")
        self.assertEqual(args.dependencies, ["git.testing/a", "git.testing/b"])
        self.assertEqual(args.go_get_all, False)
        self.assertEqual(args.tsv_path, None)

    def test_project_with_go_deps(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project", "--go-get-all"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.dependencies, None)
        self.assertEqual(args.go_get_all, True)
        self.assertEqual(args.tsv_path, None)

    def test_project_with_tsv_path(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project",
             "--project-url", "https://git.testing/project",
             "--tsv-path", "/a/file.tsv"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.project_url, "https://git.testing/project")
        self.assertEqual(args.dependencies, None)
        self.assertEqual(args.go_get_all, False)
        self.assertEqual(args.tsv_path, "/a/file.tsv")

class TestSubcommandError(unittest.TestCase):

    def test_subcommand_error(self):
        proc_error = subprocess.CalledProcessError(1, ["git"])
        err = git_gate.SubcommandError("git", "clone", proc_error)
        self.assertEqual(str(err), "Subprocess git clone failed with code 1")

class TestGoTest(unittest.TestCase):

    maxDiff = None

    def setUp(self):
        # Patch out and record actions run as part of go_test()
        self.actions = []
        self.patch_action("git_gate.print_now", lambda s: ("print", s))
        self.patch_action("git_gate.SubcommandRunner.__call__",
                          lambda self, *args: (self.command,) + args)
        self.patch_action("os.chdir", lambda d: ("chdir", d))

        # Verify go commands run with GOPATH overridden
        real_runner = git_gate.SubcommandRunner

        def _check(command, environ=None):
            if command in ("go", "godeps"):
                self.assertIsInstance(environ, dict)
                self.assertEquals(environ.get("GOPATH"), "/tmp/fake")
            return real_runner(command, environ)

        patcher = mock.patch("git_gate.SubcommandRunner", side_effect=_check)
        patcher.start()
        self.addCleanup(patcher.stop)

    def patch_action(self, target, action):
        def _record(*args, **kwargs):
            self.actions.append(action(*args, **kwargs))
        patcher = mock.patch(target, _record)
        patcher.start()
        self.addCleanup(patcher.stop)

    @staticmethod
    def make_args(project, **kwargs):
        return argparse.Namespace(
            project=project,
            project_url=kwargs.get("project_url"),
            project_ref=kwargs.get("project_ref"),
            merge_url=kwargs.get("merge_url"),
            merge_ref=kwargs.get("merge_ref"),
            go_get_all=kwargs.get("go_get_all"),
            dependencies=kwargs.get("dependencies", []),
            tsv_path=kwargs.get("tsv_path"),
        )

    def test_get_test(self):
        args = self.make_args("git.testing/project", go_get_all=True)
        git_gate.go_test(args, "/tmp/fake")
        self.assertEqual(self.actions, [
            ('print', 'Getting git.testing/project and dependencies using go'),
            ('go', 'get', '-v', '-d', '-t', 'git.testing/project/...'),
            ('chdir', '/tmp/fake/src/git.testing/project'),
            ('go', 'build', 'git.testing/project/...'),
            ('go', 'test', 'git.testing/project/...')
        ])

    def test_get_merge_test(self):
        args = self.make_args("git.testing/project", go_get_all=True,
                              merge_url="https://git.testing/proposed",
                              merge_ref="HEAD")
        git_gate.go_test(args, "/tmp/fake")
        self.assertEqual(self.actions, [
            ('print', 'Getting git.testing/project and dependencies using go'),
            ('go', 'get', '-v', '-d', '-t', 'git.testing/project/...'),
            ('chdir', '/tmp/fake/src/git.testing/project'),
            ('print', 'Merging https://git.testing/proposed ref HEAD'),
            ('git', 'fetch', 'https://git.testing/proposed', 'HEAD'),
            ('git', 'merge', '--no-ff', '-m', 'Merged HEAD', 'FETCH_HEAD'),
            ('go', 'build', 'git.testing/project/...'),
            ('go', 'test', 'git.testing/project/...')
        ])

    def test_get_merge_other_test(self):
        args = self.make_args("git.testing/project", go_get_all=True,
                              project_url="https://git.testing/project",
                              project_ref="v1",
                              merge_url="https://git.testing/proposed",
                              merge_ref="feature")
        git_gate.go_test(args, "/tmp/fake")
        self.assertEqual(self.actions, [
            ('print', 'Cloning git.testing/project from'
             ' https://git.testing/project'),
            ('git', 'clone', 'https://git.testing/project',
             '/tmp/fake/src/git.testing/project'),
            ('print', 'Getting git.testing/project and dependencies using go'),
            ('go', 'get', '-v', '-d', '-t', 'git.testing/project/...'),
            ('chdir', '/tmp/fake/src/git.testing/project'),
            ('print', 'Switching repository to v1'),
            ('git', 'checkout', 'v1'),
            ('print', 'Merging https://git.testing/proposed ref feature'),
            ('git', 'fetch', 'https://git.testing/proposed', 'feature'),
            ('git', 'merge', '--no-ff', '-m', 'Merged feature', 'FETCH_HEAD'),
            ('go', 'build', 'git.testing/project/...'),
            ('go', 'test', 'git.testing/project/...')
        ])

    def test_deps_test(self):
        args = self.make_args("git.testing/project",
                              project_url="https://git.testing/project",
                              dependencies=["git.testing/a", "git.testing/b"])
        git_gate.go_test(args, "/tmp/fake")
        self.assertEqual(self.actions, [
            ('print', 'Cloning git.testing/project from'
             ' https://git.testing/project'),
            ('git', 'clone', 'https://git.testing/project',
             '/tmp/fake/src/git.testing/project'),
            ('chdir', '/tmp/fake/src/git.testing/project'),
            ('print', 'Getting git.testing/a and dependencies using go'),
            ('go', 'get', '-v', '-d', 'git.testing/a'),
            ('print', 'Getting git.testing/b and dependencies using go'),
            ('go', 'get', '-v', '-d', 'git.testing/b'),
            ('go', 'build', 'git.testing/project/...'),
            ('go', 'test', 'git.testing/project/...')
        ])

    def test_tsv_test(self):
        args = self.make_args("git.testing/project",
                              project_url="https://git.testing/project",
                              tsv_path="git.testing/project/dependencies.tsv")
        git_gate.go_test(args, "/tmp/fake")
        self.assertEqual(self.actions, [
            ('print', 'Getting and installing godeps'),
            ('go', 'get', '-v', '-d', 'launchpad.net/godeps/...'),
            ('go', 'install', 'launchpad.net/godeps/...'),
            ('print', 'Cloning git.testing/project from'
             ' https://git.testing/project'),
            ('git', 'clone', 'https://git.testing/project',
             '/tmp/fake/src/git.testing/project'),
            ('chdir', '/tmp/fake/src/git.testing/project'),
            ('print', 'Getting dependencies using godeps from'
             ' /tmp/fake/src/git.testing/project/dependencies.tsv'),
            ('/tmp/fake/bin/godeps', '-u',
             '/tmp/fake/src/git.testing/project/dependencies.tsv'),
            ('go', 'build', 'git.testing/project/...'),
            ('go', 'test', 'git.testing/project/...')
        ])
<commit_msg>Simplify make_args and add docs per review by abentley<commit_after>import argparse
import mock
import subprocess
import unittest

import git_gate

class TestParseArgs(unittest.TestCase):

    def test_project_and_url(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project",
             "--project-url", "https://git.testing/project"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.project_url, "https://git.testing/project")

    def test_project_and_ref(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project", "--go-get-all",
             "--project-ref", "v1"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.project_ref, "v1")

    def test_merging_other(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project", "--go-get-all",
             "--merge-url", "https://git.testing/proposed"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.merge_url, "https://git.testing/proposed")
        self.assertEqual(args.merge_ref, "HEAD")

    def test_merging_other_ref(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project", "--go-get-all",
             "--merge-url", "https://git.testing/proposed",
             "--merge-ref", "feature"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.merge_url, "https://git.testing/proposed")
        self.assertEqual(args.merge_ref, "feature")

    def test_project_with_deps(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project",
             "--project-url", "https://git.testing/project",
             "--dependencies", "git.testing/a", "git.testing/b"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.project_url, "https://git.testing/project")
        self.assertEqual(args.dependencies, ["git.testing/a", "git.testing/b"])
        self.assertEqual(args.go_get_all, False)
        self.assertEqual(args.tsv_path, None)

    def test_project_with_go_deps(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project", "--go-get-all"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.dependencies, None)
        self.assertEqual(args.go_get_all, True)
        self.assertEqual(args.tsv_path, None)

    def test_project_with_tsv_path(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project",
             "--project-url", "https://git.testing/project",
             "--tsv-path", "/a/file.tsv"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.project_url, "https://git.testing/project")
        self.assertEqual(args.dependencies, None)
        self.assertEqual(args.go_get_all, False)
        self.assertEqual(args.tsv_path, "/a/file.tsv")

class TestSubcommandError(unittest.TestCase):

    def test_subcommand_error(self):
        proc_error = subprocess.CalledProcessError(1, ["git"])
        err = git_gate.SubcommandError("git", "clone", proc_error)
        self.assertEqual(str(err), "Subprocess git clone failed with code 1")

class TestGoTest(unittest.TestCase):
    """
    Tests for go_test function.

    Class has setup that patches out each operation with relevent side effects,
    running a command, printing output, or changing directory. Those are
    recorded in order in the actions list, so each test can supply a set of
    arguments then just match the actions.
    """

    maxDiff = None

    def setUp(self):
        # Patch out and record actions run as part of go_test()
        self.actions = []
        self.patch_action("git_gate.print_now", lambda s: ("print", s))
        self.patch_action("git_gate.SubcommandRunner.__call__",
                          lambda self, *args: (self.command,) + args)
        self.patch_action("os.chdir", lambda d: ("chdir", d))

        # Verify go commands run with GOPATH overridden
        real_runner = git_gate.SubcommandRunner

        def _check(command, environ=None):
            if command in ("go", "godeps"):
                self.assertIsInstance(environ, dict)
                self.assertEquals(environ.get("GOPATH"), "/tmp/fake")
            return real_runner(command, environ)

        patcher = mock.patch("git_gate.SubcommandRunner", side_effect=_check)
        patcher.start()
        self.addCleanup(patcher.stop)

    def patch_action(self, target, func):
        """Patch target recording each call in actions as wrapped by func."""
        def _record(*args, **kwargs):
            self.actions.append(func(*args, **kwargs))
        patcher = mock.patch(target, _record)
        patcher.start()
        self.addCleanup(patcher.stop)

    args = frozenset("project project_url project_ref merge_url merge_ref"
                     " go_get_all dependencies tsv_path".split())

    @classmethod
    def make_args(cls, project, **kwargs):
        """Gives args like parse_args with all values defaulted to None."""
        if not cls.args.issuperset(kwargs):
            raise ValueError("Invalid arguments given: {!r}".format(kwargs))
        kwargs["project"] = project
        return argparse.Namespace(**dict((k, kwargs.get(k)) for k in cls.args))

    def test_get_test(self):
        args = self.make_args("git.testing/project", go_get_all=True)
        git_gate.go_test(args, "/tmp/fake")
        self.assertEqual(self.actions, [
            ('print', 'Getting git.testing/project and dependencies using go'),
            ('go', 'get', '-v', '-d', '-t', 'git.testing/project/...'),
            ('chdir', '/tmp/fake/src/git.testing/project'),
            ('go', 'build', 'git.testing/project/...'),
            ('go', 'test', 'git.testing/project/...')
        ])

    def test_get_merge_test(self):
        args = self.make_args("git.testing/project", go_get_all=True,
                              merge_url="https://git.testing/proposed",
                              merge_ref="HEAD")
        git_gate.go_test(args, "/tmp/fake")
        self.assertEqual(self.actions, [
            ('print', 'Getting git.testing/project and dependencies using go'),
            ('go', 'get', '-v', '-d', '-t', 'git.testing/project/...'),
            ('chdir', '/tmp/fake/src/git.testing/project'),
            ('print', 'Merging https://git.testing/proposed ref HEAD'),
            ('git', 'fetch', 'https://git.testing/proposed', 'HEAD'),
            ('git', 'merge', '--no-ff', '-m', 'Merged HEAD', 'FETCH_HEAD'),
            ('go', 'build', 'git.testing/project/...'),
            ('go', 'test', 'git.testing/project/...')
        ])

    def test_get_merge_other_test(self):
        args = self.make_args("git.testing/project", go_get_all=True,
                              project_url="https://git.testing/project",
                              project_ref="v1",
                              merge_url="https://git.testing/proposed",
                              merge_ref="feature")
        git_gate.go_test(args, "/tmp/fake")
        self.assertEqual(self.actions, [
            ('print', 'Cloning git.testing/project from'
             ' https://git.testing/project'),
            ('git', 'clone', 'https://git.testing/project',
             '/tmp/fake/src/git.testing/project'),
            ('print', 'Getting git.testing/project and dependencies using go'),
            ('go', 'get', '-v', '-d', '-t', 'git.testing/project/...'),
            ('chdir', '/tmp/fake/src/git.testing/project'),
            ('print', 'Switching repository to v1'),
            ('git', 'checkout', 'v1'),
            ('print', 'Merging https://git.testing/proposed ref feature'),
            ('git', 'fetch', 'https://git.testing/proposed', 'feature'),
            ('git', 'merge', '--no-ff', '-m', 'Merged feature', 'FETCH_HEAD'),
            ('go', 'build', 'git.testing/project/...'),
            ('go', 'test', 'git.testing/project/...')
        ])

    def test_deps_test(self):
        args = self.make_args("git.testing/project",
                              project_url="https://git.testing/project",
                              dependencies=["git.testing/a", "git.testing/b"])
        git_gate.go_test(args, "/tmp/fake")
        self.assertEqual(self.actions, [
            ('print', 'Cloning git.testing/project from'
             ' https://git.testing/project'),
            ('git', 'clone', 'https://git.testing/project',
             '/tmp/fake/src/git.testing/project'),
            ('chdir', '/tmp/fake/src/git.testing/project'),
            ('print', 'Getting git.testing/a and dependencies using go'),
            ('go', 'get', '-v', '-d', 'git.testing/a'),
            ('print', 'Getting git.testing/b and dependencies using go'),
            ('go', 'get', '-v', '-d', 'git.testing/b'),
            ('go', 'build', 'git.testing/project/...'),
            ('go', 'test', 'git.testing/project/...')
        ])

    def test_tsv_test(self):
        args = self.make_args("git.testing/project",
                              project_url="https://git.testing/project",
                              tsv_path="git.testing/project/dependencies.tsv")
        git_gate.go_test(args, "/tmp/fake")
        self.assertEqual(self.actions, [
            ('print', 'Getting and installing godeps'),
            ('go', 'get', '-v', '-d', 'launchpad.net/godeps/...'),
            ('go', 'install', 'launchpad.net/godeps/...'),
            ('print', 'Cloning git.testing/project from'
             ' https://git.testing/project'),
            ('git', 'clone', 'https://git.testing/project',
             '/tmp/fake/src/git.testing/project'),
            ('chdir', '/tmp/fake/src/git.testing/project'),
            ('print', 'Getting dependencies using godeps from'
             ' /tmp/fake/src/git.testing/project/dependencies.tsv'),
            ('/tmp/fake/bin/godeps', '-u',
             '/tmp/fake/src/git.testing/project/dependencies.tsv'),
            ('go', 'build', 'git.testing/project/...'),
            ('go', 'test', 'git.testing/project/...')
        ])
<|endoftext|>


<commit_before>import argparse
import mock
import subprocess
import unittest

import git_gate

class TestParseArgs(unittest.TestCase):

    def test_project_and_url(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project",
             "--project-url", "https://git.testing/project"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.project_url, "https://git.testing/project")

    def test_project_and_ref(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project", "--go-get-all",
             "--project-ref", "v1"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.project_ref, "v1")

    def test_merging_other(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project", "--go-get-all",
             "--merge-url", "https://git.testing/proposed"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.merge_url, "https://git.testing/proposed")
        self.assertEqual(args.merge_ref, "HEAD")

    def test_merging_other_ref(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project", "--go-get-all",
             "--merge-url", "https://git.testing/proposed",
             "--merge-ref", "feature"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.merge_url, "https://git.testing/proposed")
        self.assertEqual(args.merge_ref, "feature")

    def test_project_with_deps(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project",
             "--project-url", "https://git.testing/project",
             "--dependencies", "git.testing/a", "git.testing/b"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.project_url, "https://git.testing/project")
        self.assertEqual(args.dependencies, ["git.testing/a", "git.testing/b"])
        self.assertEqual(args.go_get_all, False)
        self.assertEqual(args.tsv_path, None)

    def test_project_with_go_deps(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project", "--go-get-all"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.dependencies, None)
        self.assertEqual(args.go_get_all, True)
        self.assertEqual(args.tsv_path, None)

    def test_project_with_tsv_path(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project",
             "--project-url", "https://git.testing/project",
             "--tsv-path", "/a/file.tsv"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.project_url, "https://git.testing/project")
        self.assertEqual(args.dependencies, None)
        self.assertEqual(args.go_get_all, False)
        self.assertEqual(args.tsv_path, "/a/file.tsv")

class TestSubcommandError(unittest.TestCase):

    def test_subcommand_error(self):
        proc_error = subprocess.CalledProcessError(1, ["git"])
        err = git_gate.SubcommandError("git", "clone", proc_error)
        self.assertEqual(str(err), "Subprocess git clone failed with code 1")

class TestGoTest(unittest.TestCase):

    maxDiff = None

    def setUp(self):
        # Patch out and record actions run as part of go_test()
        self.actions = []
        self.patch_action("git_gate.print_now", lambda s: ("print", s))
        self.patch_action("git_gate.SubcommandRunner.__call__",
                          lambda self, *args: (self.command,) + args)
        self.patch_action("os.chdir", lambda d: ("chdir", d))
        # Verify go commands run with GOPATH overridden
        real_runner = git_gate.SubcommandRunner
        def _check(command, environ=None):
            if command in ("go", "godeps"):
                self.assertIsInstance(environ, dict)
                self.assertEquals(environ.get("GOPATH"), "/tmp/fake")
            return real_runner(command, environ)
        patcher = mock.patch("git_gate.SubcommandRunner", side_effect=_check)
        patcher.start()
        self.addCleanup(patcher.stop)

    def patch_action(self, target, action):
        def _record(*args, **kwargs):
            self.actions.append(action(*args, **kwargs))
        patcher = mock.patch(target, _record)
        patcher.start()
        self.addCleanup(patcher.stop)

    @staticmethod
    def make_args(project, **kwargs):
        return argparse.Namespace(
            project=project,
            project_url=kwargs.get("project_url"),
            project_ref=kwargs.get("project_ref"),
            merge_url=kwargs.get("merge_url"),
            merge_ref=kwargs.get("merge_ref"),
            go_get_all=kwargs.get("go_get_all"),
            dependencies=kwargs.get("dependencies", []),
            tsv_path=kwargs.get("tsv_path"),
        )

    def test_get_test(self):
        args = self.make_args("git.testing/project", go_get_all=True)
        git_gate.go_test(args, "/tmp/fake")
        self.assertEqual(self.actions, [
            ('print', 'Getting git.testing/project and dependencies using go'),
            ('go', 'get', '-v', '-d', '-t', 'git.testing/project/...'),
            ('chdir', '/tmp/fake/src/git.testing/project'),
            ('go', 'build', 'git.testing/project/...'),
            ('go', 'test', 'git.testing/project/...')
        ])

    def test_get_merge_test(self):
        args = self.make_args("git.testing/project", go_get_all=True,
                              merge_url="https://git.testing/proposed",
                              merge_ref="HEAD")
        git_gate.go_test(args, "/tmp/fake")
        self.assertEqual(self.actions, [
            ('print', 'Getting git.testing/project and dependencies using go'),
            ('go', 'get', '-v', '-d', '-t', 'git.testing/project/...'),
            ('chdir', '/tmp/fake/src/git.testing/project'),
            ('print', 'Merging https://git.testing/proposed ref HEAD'),
            ('git', 'fetch', 'https://git.testing/proposed', 'HEAD'),
            ('git', 'merge', '--no-ff', '-m', 'Merged HEAD', 'FETCH_HEAD'),
            ('go', 'build', 'git.testing/project/...'),
            ('go', 'test', 'git.testing/project/...')
        ])

    def test_get_merge_other_test(self):
        args = self.make_args("git.testing/project", go_get_all=True,
                              project_url="https://git.testing/project",
                              project_ref="v1",
                              merge_url="https://git.testing/proposed",
                              merge_ref="feature")
        git_gate.go_test(args, "/tmp/fake")
        self.assertEqual(self.actions, [
            ('print', 'Cloning git.testing/project from'
             ' https://git.testing/project'),
            ('git', 'clone', 'https://git.testing/project',
             '/tmp/fake/src/git.testing/project'),
            ('print', 'Getting git.testing/project and dependencies using go'),
            ('go', 'get', '-v', '-d', '-t', 'git.testing/project/...'),
            ('chdir', '/tmp/fake/src/git.testing/project'),
            ('print', 'Switching repository to v1'),
            ('git', 'checkout', 'v1'),
            ('print', 'Merging https://git.testing/proposed ref feature'),
            ('git', 'fetch', 'https://git.testing/proposed', 'feature'),
            ('git', 'merge', '--no-ff', '-m', 'Merged feature', 'FETCH_HEAD'),
            ('go', 'build', 'git.testing/project/...'),
            ('go', 'test', 'git.testing/project/...')
        ])

    def test_deps_test(self):
        args = self.make_args("git.testing/project",
                              project_url="https://git.testing/project",
                              dependencies=["git.testing/a", "git.testing/b"])
        git_gate.go_test(args, "/tmp/fake")
        self.assertEqual(self.actions, [
            ('print', 'Cloning git.testing/project from'
             ' https://git.testing/project'),
            ('git', 'clone', 'https://git.testing/project',
             '/tmp/fake/src/git.testing/project'),
            ('chdir', '/tmp/fake/src/git.testing/project'),
            ('print', 'Getting git.testing/a and dependencies using go'),
            ('go', 'get', '-v', '-d', 'git.testing/a'),
            ('print', 'Getting git.testing/b and dependencies using go'),
            ('go', 'get', '-v', '-d', 'git.testing/b'),
            ('go', 'build', 'git.testing/project/...'),
            ('go', 'test', 'git.testing/project/...')
        ])

    def test_tsv_test(self):
        args = self.make_args("git.testing/project",
                              project_url="https://git.testing/project",
                              tsv_path="git.testing/project/dependencies.tsv")
        git_gate.go_test(args, "/tmp/fake")
        self.assertEqual(self.actions, [
            ('print', 'Getting and installing godeps'),
            ('go', 'get', '-v', '-d', 'launchpad.net/godeps/...'),
            ('go', 'install', 'launchpad.net/godeps/...'),
            ('print', 'Cloning git.testing/project from'
             ' https://git.testing/project'),
            ('git', 'clone', 'https://git.testing/project',
             '/tmp/fake/src/git.testing/project'),
            ('chdir', '/tmp/fake/src/git.testing/project'),
            ('print', 'Getting dependencies using godeps from'
             ' /tmp/fake/src/git.testing/project/dependencies.tsv'),
            ('/tmp/fake/bin/godeps', '-u',
             '/tmp/fake/src/git.testing/project/dependencies.tsv'),
            ('go', 'build', 'git.testing/project/...'),
            ('go', 'test', 'git.testing/project/...')
        ])
<commit_msg>Fix lint<commit_after>import argparse
import mock
import subprocess
import unittest

import git_gate

class TestParseArgs(unittest.TestCase):

    def test_project_and_url(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project",
             "--project-url", "https://git.testing/project"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.project_url, "https://git.testing/project")

    def test_project_and_ref(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project", "--go-get-all",
             "--project-ref", "v1"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.project_ref, "v1")

    def test_merging_other(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project", "--go-get-all",
             "--merge-url", "https://git.testing/proposed"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.merge_url, "https://git.testing/proposed")
        self.assertEqual(args.merge_ref, "HEAD")

    def test_merging_other_ref(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project", "--go-get-all",
             "--merge-url", "https://git.testing/proposed",
             "--merge-ref", "feature"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.merge_url, "https://git.testing/proposed")
        self.assertEqual(args.merge_ref, "feature")

    def test_project_with_deps(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project",
             "--project-url", "https://git.testing/project",
             "--dependencies", "git.testing/a", "git.testing/b"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.project_url, "https://git.testing/project")
        self.assertEqual(args.dependencies, ["git.testing/a", "git.testing/b"])
        self.assertEqual(args.go_get_all, False)
        self.assertEqual(args.tsv_path, None)

    def test_project_with_go_deps(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project", "--go-get-all"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.dependencies, None)
        self.assertEqual(args.go_get_all, True)
        self.assertEqual(args.tsv_path, None)

    def test_project_with_tsv_path(self):
        args = git_gate.parse_args(
            ["--project", "git.testing/project",
             "--project-url", "https://git.testing/project",
             "--tsv-path", "/a/file.tsv"])
        self.assertEqual(args.project, "git.testing/project")
        self.assertEqual(args.project_url, "https://git.testing/project")
        self.assertEqual(args.dependencies, None)
        self.assertEqual(args.go_get_all, False)
        self.assertEqual(args.tsv_path, "/a/file.tsv")

class TestSubcommandError(unittest.TestCase):

    def test_subcommand_error(self):
        proc_error = subprocess.CalledProcessError(1, ["git"])
        err = git_gate.SubcommandError("git", "clone", proc_error)
        self.assertEqual(str(err), "Subprocess git clone failed with code 1")

class TestGoTest(unittest.TestCase):

    maxDiff = None

    def setUp(self):
        # Patch out and record actions run as part of go_test()
        self.actions = []
        self.patch_action("git_gate.print_now", lambda s: ("print", s))
        self.patch_action("git_gate.SubcommandRunner.__call__",
                          lambda self, *args: (self.command,) + args)
        self.patch_action("os.chdir", lambda d: ("chdir", d))

        # Verify go commands run with GOPATH overridden
        real_runner = git_gate.SubcommandRunner

        def _check(command, environ=None):
            if command in ("go", "godeps"):
                self.assertIsInstance(environ, dict)
                self.assertEquals(environ.get("GOPATH"), "/tmp/fake")
            return real_runner(command, environ)

        patcher = mock.patch("git_gate.SubcommandRunner", side_effect=_check)
        patcher.start()
        self.addCleanup(patcher.stop)

    def patch_action(self, target, action):
        def _record(*args, **kwargs):
            self.actions.append(action(*args, **kwargs))
        patcher = mock.patch(target, _record)
        patcher.start()
        self.addCleanup(patcher.stop)

    @staticmethod
    def make_args(project, **kwargs):
        return argparse.Namespace(
            project=project,
            project_url=kwargs.get("project_url"),
            project_ref=kwargs.get("project_ref"),
            merge_url=kwargs.get("merge_url"),
            merge_ref=kwargs.get("merge_ref"),
            go_get_all=kwargs.get("go_get_all"),
            dependencies=kwargs.get("dependencies", []),
            tsv_path=kwargs.get("tsv_path"),
        )

    def test_get_test(self):
        args = self.make_args("git.testing/project", go_get_all=True)
        git_gate.go_test(args, "/tmp/fake")
        self.assertEqual(self.actions, [
            ('print', 'Getting git.testing/project and dependencies using go'),
            ('go', 'get', '-v', '-d', '-t', 'git.testing/project/...'),
            ('chdir', '/tmp/fake/src/git.testing/project'),
            ('go', 'build', 'git.testing/project/...'),
            ('go', 'test', 'git.testing/project/...')
        ])

    def test_get_merge_test(self):
        args = self.make_args("git.testing/project", go_get_all=True,
                              merge_url="https://git.testing/proposed",
                              merge_ref="HEAD")
        git_gate.go_test(args, "/tmp/fake")
        self.assertEqual(self.actions, [
            ('print', 'Getting git.testing/project and dependencies using go'),
            ('go', 'get', '-v', '-d', '-t', 'git.testing/project/...'),
            ('chdir', '/tmp/fake/src/git.testing/project'),
            ('print', 'Merging https://git.testing/proposed ref HEAD'),
            ('git', 'fetch', 'https://git.testing/proposed', 'HEAD'),
            ('git', 'merge', '--no-ff', '-m', 'Merged HEAD', 'FETCH_HEAD'),
            ('go', 'build', 'git.testing/project/...'),
            ('go', 'test', 'git.testing/project/...')
        ])

    def test_get_merge_other_test(self):
        args = self.make_args("git.testing/project", go_get_all=True,
                              project_url="https://git.testing/project",
                              project_ref="v1",
                              merge_url="https://git.testing/proposed",
                              merge_ref="feature")
        git_gate.go_test(args, "/tmp/fake")
        self.assertEqual(self.actions, [
            ('print', 'Cloning git.testing/project from'
             ' https://git.testing/project'),
            ('git', 'clone', 'https://git.testing/project',
             '/tmp/fake/src/git.testing/project'),
            ('print', 'Getting git.testing/project and dependencies using go'),
            ('go', 'get', '-v', '-d', '-t', 'git.testing/project/...'),
            ('chdir', '/tmp/fake/src/git.testing/project'),
            ('print', 'Switching repository to v1'),
            ('git', 'checkout', 'v1'),
            ('print', 'Merging https://git.testing/proposed ref feature'),
            ('git', 'fetch', 'https://git.testing/proposed', 'feature'),
            ('git', 'merge', '--no-ff', '-m', 'Merged feature', 'FETCH_HEAD'),
            ('go', 'build', 'git.testing/project/...'),
            ('go', 'test', 'git.testing/project/...')
        ])

    def test_deps_test(self):
        args = self.make_args("git.testing/project",
                              project_url="https://git.testing/project",
                              dependencies=["git.testing/a", "git.testing/b"])
        git_gate.go_test(args, "/tmp/fake")
        self.assertEqual(self.actions, [
            ('print', 'Cloning git.testing/project from'
             ' https://git.testing/project'),
            ('git', 'clone', 'https://git.testing/project',
             '/tmp/fake/src/git.testing/project'),
            ('chdir', '/tmp/fake/src/git.testing/project'),
            ('print', 'Getting git.testing/a and dependencies using go'),
            ('go', 'get', '-v', '-d', 'git.testing/a'),
            ('print', 'Getting git.testing/b and dependencies using go'),
            ('go', 'get', '-v', '-d', 'git.testing/b'),
            ('go', 'build', 'git.testing/project/...'),
            ('go', 'test', 'git.testing/project/...')
        ])

    def test_tsv_test(self):
        args = self.make_args("git.testing/project",
                              project_url="https://git.testing/project",
                              tsv_path="git.testing/project/dependencies.tsv")
        git_gate.go_test(args, "/tmp/fake")
        self.assertEqual(self.actions, [
            ('print', 'Getting and installing godeps'),
            ('go', 'get', '-v', '-d', 'launchpad.net/godeps/...'),
            ('go', 'install', 'launchpad.net/godeps/...'),
            ('print', 'Cloning git.testing/project from'
             ' https://git.testing/project'),
            ('git', 'clone', 'https://git.testing/project',
             '/tmp/fake/src/git.testing/project'),
            ('chdir', '/tmp/fake/src/git.testing/project'),
            ('print', 'Getting dependencies using godeps from'
             ' /tmp/fake/src/git.testing/project/dependencies.tsv'),
            ('/tmp/fake/bin/godeps', '-u',
             '/tmp/fake/src/git.testing/project/dependencies.tsv'),
            ('go', 'build', 'git.testing/project/...'),
            ('go', 'test', 'git.testing/project/...')
        ])
<|endoftext|>


<commit_before>def serve(app_name, uuid):
    return True

def unserve(app_name, uuid):
    return True
<commit_msg>first release<commit_after>from empressx.retinue.tasks import web

def serve(app_name, uuid):
    web.serve.apply_async(args=(app_name, uuid))
    return True

def unserve(app_name, uuid):
    web.unserve.apply_async(args=(app_name, uuid))
    return True
<|endoftext|>


<commit_before>"""
Filename: test_logitdyn.py
Author: Tomohiro Kusano

Tests for logitdyn.py

"""
from __future__ import division

import numpy as np
from numpy.testing import assert_array_equal, assert_array_almost_equal_nulp
from nose.tools import eq_, ok_, raises

from logitdyn import LogitDynamics
from game_tools import NormalFormGame

class TestLogitDynamics:
    '''Test the methods of LogitDynamics'''

    def setUp(self):
        '''Setup a LogitDynamics instance'''
        # symmetric 2x2 coordination game
        payoff_matrix = [[4, 0],
                         [3, 2]]
        beta = 4.0
        g = NormalFormGame(payoff_matrix)
        self.ld = LogitDynamics(g, beta=beta)

    def test_set_init_actions_with_given_init_actions(self):
        init_actions = (0, 1)
        self.ld.set_init_actions(init_actions)
        assert_array_equal(self.ld.current_actions, init_actions)

    def test_set_init_actions_when_init_action_dist_None(self):
        self.ld.set_init_actions()  # Action dist randomly chosen
        init_actions = self.ld.current_actions

        ok_(all(
            action in list(range(2)) for action in self.ld.current_actions)
            )

    def test_simulate_seed(self):
        np.random.seed(291)
        seq = self.ld.simulate(ts_length=10, init_actions=(0, 0))
        assert_array_equal(
            seq,
            [[0, 0],
             [0, 0],
             [0, 0],
             [0, 1],
             [1, 1],
             [1, 1],
             [1, 1],
             [1, 1],
             [1, 1],
             [1, 1]]
        )

    def test_simulate_lln(self):
        n = 100
        T = 1000
        seq = self.ld.replicate(T=T, num_reps=n)
        count = 0
        for i in range(n):
            if all(seq[i, :] == [1, 1]):
                count += 1
        frequency = count / n

        # 0.981367209 = prob that the stationary distribution assigns to [1, 1]
        ok_(np.abs(frequency-0.981367209) < 0.05)

def test_set_choice_probs_with_asymmetric_payoff_matrix():
    bimatrix = np.array([[(4, 4), (1, 1), (0, 3)],
                             [(3, 0), (1, 1), (2, 2)]])
    beta = 1.0
    g = NormalFormGame(bimatrix)
    ld = LogitDynamics(g, beta=beta)

    # (Normalized) CDFs of logit choice
    cdfs = np.ones((bimatrix.shape[1], bimatrix.shape[0]))
    cdfs[:, 0] = 1 / (1 + np.exp(beta*(bimatrix[1, :, 0]-bimatrix[0, :, 0])))

    # self.ld.players[0].logit_choice_cdfs: unnormalized
    cdfs_computed = ld.players[0].logit_choice_cdfs
    cdfs_computed = cdfs_computed / cdfs_computed[..., [-1]]  # Normalized

    assert_array_almost_equal_nulp(cdfs_computed, cdfs)

if __name__ == '__main__':
    import sys
    import nose

    argv = sys.argv[:]
    argv.append('--verbose')
    argv.append('--nocapture')
    nose.main(argv=argv, defaultTest=__file__)
<commit_msg>Minor edit<commit_after>"""
Filename: test_logitdyn.py
Author: Tomohiro Kusano

Tests for logitdyn.py

"""
from __future__ import division

import numpy as np
from numpy.testing import assert_array_equal, assert_array_almost_equal_nulp
from nose.tools import eq_, ok_, raises

from logitdyn import LogitDynamics
from game_tools import NormalFormGame

class TestLogitDynamics:
    '''Test the methods of LogitDynamics'''

    def setUp(self):
        '''Setup a LogitDynamics instance'''
        # symmetric 2x2 coordination game
        payoff_matrix = [[4, 0],
                         [3, 2]]
        beta = 4.0
        g = NormalFormGame(payoff_matrix)
        self.ld = LogitDynamics(g, beta=beta)

    def test_set_init_actions_with_given_init_actions(self):
        init_actions = (0, 1)
        self.ld.set_init_actions(init_actions)
        assert_array_equal(self.ld.current_actions, init_actions)

    def test_set_init_actions_when_init_action_dist_None(self):
        self.ld.set_init_actions()  # Action dist randomly chosen
        init_actions = self.ld.current_actions

        ok_(all(
            action in list(range(2)) for action in self.ld.current_actions)
            )

    def test_simulate_seed(self):
        np.random.seed(291)
        seq = self.ld.simulate(ts_length=10, init_actions=(0, 0))
        assert_array_equal(
            seq,
            [[0, 0],
             [0, 0],
             [0, 0],
             [0, 1],
             [1, 1],
             [1, 1],
             [1, 1],
             [1, 1],
             [1, 1],
             [1, 1]]
        )

    def test_simulate_lln(self):
        n = 100
        T = 1000
        seq = self.ld.replicate(T=T, num_reps=n)
        count = 0
        for i in range(n):
            if all(seq[i, :] == [1, 1]):
                count += 1
        frequency = count / n

        # 0.981367209 = prob that the stationary distribution assigns to [1, 1]
        ok_(np.abs(frequency-0.981367209) < 0.05)

def test_set_choice_probs_with_asymmetric_payoff_matrix():
    bimatrix = np.array([[(4, 4), (1, 1), (0, 3)],
                         [(3, 0), (1, 1), (2, 2)]])
    beta = 1.0
    g = NormalFormGame(bimatrix)
    ld = LogitDynamics(g, beta=beta)

    # (Normalized) CDFs of logit choice
    cdfs = np.ones((bimatrix.shape[1], bimatrix.shape[0]))
    cdfs[:, 0] = 1 / (1 + np.exp(beta*(bimatrix[1, :, 0]-bimatrix[0, :, 0])))

    # self.ld.players[0].logit_choice_cdfs: unnormalized
    cdfs_computed = ld.players[0].logit_choice_cdfs
    cdfs_computed = cdfs_computed / cdfs_computed[..., [-1]]  # Normalized

    assert_array_almost_equal_nulp(cdfs_computed, cdfs)

if __name__ == '__main__':
    import sys
    import nose

    argv = sys.argv[:]
    argv.append('--verbose')
    argv.append('--nocapture')
    nose.main(argv=argv, defaultTest=__file__)
<|endoftext|>


<commit_before>"""
Classes and functions for isotope image validation
"""
import numpy as np
import pandas as pd
from operator import mul, add

from pyIMS.image_measures import measure_of_chaos, isotope_image_correlation, isotope_pattern_match

class ImgMeasures(object):
    """ Container for isotope image metrics

    Args
    ----------
    chaos : float
        measure of chaos, pyIMS.image_measures.measure_of_chaos
    image_corr : float
        isotope image correlation, pyIMS.image_measures.isotope_image_correlation
    pattern_match : float
        theoretical pattern match, pyIMS.image_measures.isotope_pattern_match
    """

    def __init__(self, chaos, image_corr, pattern_match):
        self.chaos = chaos
        self.image_corr = image_corr
        self.pattern_match = pattern_match

    @staticmethod
    def _replace_nan(v, new_v=0):
        if not v or np.isinf(v) or np.isnan(v):
            return new_v
        else:
            return v

    def to_tuple(self, replace_nan=True):
        """ Convert metrics to a tuple

        Args
        ------------
        replace_nan : bool
            replace invalid metric values with the default one
        Returns
        ------------
        : tuple
            tuple of metrics
        """
        if replace_nan:
            return (self._replace_nan(self.chaos),
                    self._replace_nan(self.image_corr),
                    self._replace_nan(self.pattern_match))
        else:
            return self.chaos, self.image_corr, self.pattern_match

def get_compute_img_metrics(empty_matrix, img_gen_conf):
    """ Returns a function for computing isotope image metrics

    Args
    ------------
    empty_matrix : ndarray
        empty matrix of the same shape as isotope images
    img_gen_conf : dict
        isotope_generation section of the dataset config
    Returns
    ------------
    : function
        function that returns tuples of metrics for every list of isotope images
    """
    def compute(iso_images_sparse, sf_ints):
        diff = len(sf_ints) - len(iso_images_sparse)
        iso_imgs = [empty_matrix if img is None else img.toarray()
                    for img in iso_images_sparse + [None] * diff]
        iso_imgs_flat = [img.flat[:] for img in iso_imgs]

        measures = ImgMeasures(0, 0, 0)
        if len(iso_imgs) > 0:
            measures.pattern_match = isotope_pattern_match(iso_imgs_flat, sf_ints)
            measures.image_corr = isotope_image_correlation(iso_imgs_flat, weights=sf_ints[1:])
            measures.chaos = measure_of_chaos(iso_imgs[0], img_gen_conf['nlevels'], overwrite=False)
        return measures.to_tuple()

    return compute

def _calculate_msm(sf_metrics_df):
    return sf_metrics_df.chaos * sf_metrics_df.spatial * sf_metrics_df.spectral

def sf_image_metrics(sf_images, sc, formulas, ds, ds_config):
    """ Compute isotope image metrics for each formula

    Args
    ------------
    sc : pyspark.SparkContext
    ds_config : dict
        dataset configuration
    ds : engine.dataset.Dataset
    formulas : engine.formulas.Formulas
    sf_images : pyspark.rdd.RDD
        RDD of (formula, list[images]) pairs
    Returns
    ------------
    : pandas.DataFrame
    """
    nrows, ncols = ds.get_dims()
    empty_matrix = np.zeros((nrows, ncols))
    compute_metrics = get_compute_img_metrics(empty_matrix, ds_config['image_generation'])
    sf_add_ints_map_brcast = sc.broadcast(formulas.get_sf_peak_ints())
    # sf_peak_ints_brcast = sc.broadcast(formulas.get_sf_peak_ints())

    sf_metrics = (sf_images
                  .map(lambda ((sf, adduct), imgs):
                      (sf, adduct) + compute_metrics(imgs, sf_add_ints_map_brcast.value[(sf, adduct)]))
                  ).collect()
    sf_metrics_df = (pd.DataFrame(sf_metrics, columns=['sf_id', 'adduct', 'chaos', 'spatial', 'spectral'])
                     .set_index(['sf_id', 'adduct']))
    sf_metrics_df['msm'] = _calculate_msm(sf_metrics_df)
    return sf_metrics_df

def sf_image_metrics_est_fdr(sf_metrics_df, formulas, fdr):
    sf_msm_df = formulas.get_sf_adduct_sorted_df()
    sf_msm_df = sf_msm_df.join(sf_metrics_df.msm).fillna(0)

    sf_adduct_fdr = fdr.estimate_fdr(sf_msm_df)
    return sf_metrics_df.join(sf_adduct_fdr, how='inner')[['chaos', 'spatial', 'spectral', 'msm', 'fdr']]

def filter_sf_images(sf_images, sf_metrics_df):
    # TODO: copying all images to the driver is really slow
    return sf_images.filter(lambda (sf_i, _): sf_i in sf_metrics_df.index).collectAsMap()

<commit_msg>pyIMS.measure_of_chaos -> cpyImagingMSpec.measure_of_chaos<commit_after>"""
Classes and functions for isotope image validation
"""
import numpy as np
import pandas as pd
from operator import mul, add

from pyIMS.image_measures import isotope_image_correlation, isotope_pattern_match
from cpyImagingMSpec import measure_of_chaos

class ImgMeasures(object):
    """ Container for isotope image metrics

    Args
    ----------
    chaos : float
        measure of chaos, pyIMS.image_measures.measure_of_chaos
    image_corr : float
        isotope image correlation, pyIMS.image_measures.isotope_image_correlation
    pattern_match : float
        theoretical pattern match, pyIMS.image_measures.isotope_pattern_match
    """

    def __init__(self, chaos, image_corr, pattern_match):
        self.chaos = chaos
        self.image_corr = image_corr
        self.pattern_match = pattern_match

    @staticmethod
    def _replace_nan(v, new_v=0):
        if not v or np.isinf(v) or np.isnan(v):
            return new_v
        else:
            return v

    def to_tuple(self, replace_nan=True):
        """ Convert metrics to a tuple

        Args
        ------------
        replace_nan : bool
            replace invalid metric values with the default one
        Returns
        ------------
        : tuple
            tuple of metrics
        """
        if replace_nan:
            return (self._replace_nan(self.chaos),
                    self._replace_nan(self.image_corr),
                    self._replace_nan(self.pattern_match))
        else:
            return self.chaos, self.image_corr, self.pattern_match

def get_compute_img_metrics(empty_matrix, img_gen_conf):
    """ Returns a function for computing isotope image metrics

    Args
    ------------
    empty_matrix : ndarray
        empty matrix of the same shape as isotope images
    img_gen_conf : dict
        isotope_generation section of the dataset config
    Returns
    ------------
    : function
        function that returns tuples of metrics for every list of isotope images
    """
    def compute(iso_images_sparse, sf_ints):
        diff = len(sf_ints) - len(iso_images_sparse)
        iso_imgs = [empty_matrix if img is None else img.toarray()
                    for img in iso_images_sparse + [None] * diff]
        iso_imgs_flat = [img.flat[:] for img in iso_imgs]

        measures = ImgMeasures(0, 0, 0)
        if len(iso_imgs) > 0:
            measures.pattern_match = isotope_pattern_match(iso_imgs_flat, sf_ints)
            measures.image_corr = isotope_image_correlation(iso_imgs_flat, weights=sf_ints[1:])
            measures.chaos = measure_of_chaos(iso_imgs[0], img_gen_conf['nlevels'])
        return measures.to_tuple()

    return compute

def _calculate_msm(sf_metrics_df):
    return sf_metrics_df.chaos * sf_metrics_df.spatial * sf_metrics_df.spectral

def sf_image_metrics(sf_images, sc, formulas, ds, ds_config):
    """ Compute isotope image metrics for each formula

    Args
    ------------
    sc : pyspark.SparkContext
    ds_config : dict
        dataset configuration
    ds : engine.dataset.Dataset
    formulas : engine.formulas.Formulas
    sf_images : pyspark.rdd.RDD
        RDD of (formula, list[images]) pairs
    Returns
    ------------
    : pandas.DataFrame
    """
    nrows, ncols = ds.get_dims()
    empty_matrix = np.zeros((nrows, ncols))
    compute_metrics = get_compute_img_metrics(empty_matrix, ds_config['image_generation'])
    sf_add_ints_map_brcast = sc.broadcast(formulas.get_sf_peak_ints())
    # sf_peak_ints_brcast = sc.broadcast(formulas.get_sf_peak_ints())

    sf_metrics = (sf_images
                  .map(lambda ((sf, adduct), imgs):
                      (sf, adduct) + compute_metrics(imgs, sf_add_ints_map_brcast.value[(sf, adduct)]))
                  ).collect()
    sf_metrics_df = (pd.DataFrame(sf_metrics, columns=['sf_id', 'adduct', 'chaos', 'spatial', 'spectral'])
                     .set_index(['sf_id', 'adduct']))
    sf_metrics_df['msm'] = _calculate_msm(sf_metrics_df)
    return sf_metrics_df

def sf_image_metrics_est_fdr(sf_metrics_df, formulas, fdr):
    sf_msm_df = formulas.get_sf_adduct_sorted_df()
    sf_msm_df = sf_msm_df.join(sf_metrics_df.msm).fillna(0)

    sf_adduct_fdr = fdr.estimate_fdr(sf_msm_df)
    return sf_metrics_df.join(sf_adduct_fdr, how='inner')[['chaos', 'spatial', 'spectral', 'msm', 'fdr']]

def filter_sf_images(sf_images, sf_metrics_df):
    # TODO: copying all images to the driver is really slow
    return sf_images.filter(lambda (sf_i, _): sf_i in sf_metrics_df.index).collectAsMap()

<|endoftext|>


<commit_before>import os
import stat
import sys
import jinja2
import yaml
import click
import markdown
import shutil
import datetime
from collections import OrderedDict

def copytree(src, dst, symlinks = False, ignore = None):
    """
    Copy a tree of files and dirs and merge into existing dir if needed
    Source: http://stackoverflow.com/a/22331852
    """
    if not os.path.exists(dst):
        os.makedirs(dst)
        shutil.copystat(src, dst)
    lst = os.listdir(src)
    if ignore:
        excl = ignore(src, lst)
        lst = [x for x in lst if x not in excl]
    for item in lst:
        s = os.path.join(src, item)
        d = os.path.join(dst, item)
        if symlinks and os.path.islink(s):
            if os.path.lexists(d):
                os.remove(d)
            os.symlink(os.readlink(s), d)
            try:
                st = os.lstat(s)
                mode = stat.S_IMODE(st.st_mode)
                os.lchmod(d, mode)
            except:
                pass # lchmod not available
        elif os.path.isdir(s):
            copytree(s, d, symlinks, ignore)
        else:
            shutil.copy2(s, d)

def ordered_load(stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict):
    class OrderedLoader(Loader):
        pass
    def construct_mapping(loader, node):
        loader.flatten_mapping(node)
        return object_pairs_hook(loader.construct_pairs(node))
    OrderedLoader.add_constructor(
        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
        construct_mapping)
    return yaml.load(stream, OrderedLoader)

def include_type_exists(key):
    """
    Check whether the include type (plugin) is valid/exists.
    Needs a file of the format plugins/<key>.py
    """
    script_dir = os.path.dirname(os.path.abspath(__file__))
    return os.path.isfile(os.path.join(script_dir, 'plugins', key + '.py'))

def render_include(site, environment, key, params):
    """
    Render paragoo include. Typically looks like:
    @@@key=param@@@
    @@@key=param1:param2:param3@@@
    """
    # Load the relevant plugin
    plugin = __import__('plugins.' + key, globals(), locals(), [key], -1)
    return plugin.render(site, environment, params)

def paragoo_includes(site, environment, body, token='@@@'):
    """
    Filter that looks for blocks surrounded by `token` and include that content type in
    the rendered html
    """
    result = ''
    if body:
        body_parts = body.split(token)
        is_content = True
        for part in body_parts:
            if is_content:
                result += part
                is_content = False
            else:
                include_parts = part.split('=')
                include_params = include_parts[1].split(':')
                print '  ' + str(include_parts)
                if include_type_exists(include_parts[0]):
                    result += render_include(site, environment, include_parts[0], include_params)
                else:
                    print 'E Plugin not found for include with key "' + include_parts[0] + '"'
                is_content = True
    return result

CONTENT_TYPES = {
    'markdown': 'md',
    'html': 'html',
    'gallery': 'gallery',
}

def generate_navbar(structure, pathprefix):
    navbar = []
    for section in structure['sections']:
        navbar_section = []
        section_data = structure['sections'][section]
        section_url = os.path.join('/', pathprefix, section + '/')
        section_title = section_data['title']
        if 'pages' in section_data:
            for page in section_data['pages']:
                url = os.path.join('/', pathprefix, section, page)
                title = structure['sections'][section]['pages'][page]['title']
                if title:
                    # Only add a page to the navigation if it has a title, otherwise it's hidden
                    navbar_section.append((url, page, title))
        if section_title:
            navbar.append((section_url, section, section_title, navbar_section))
    return navbar

def ensure_dir(f):
    d = os.path.dirname(f)
    if not os.path.exists(d):
        os.makedirs(d)

def get_file_contents(filename):
    data = None
    try:
        with open(filename) as pf:
            data = pf.read()
    except IOError:
        #print 'File not found: ' + filename
        pass
    return data

def load_page_source(source_uses_subdirs, section_dir, page, page_data):
    if page:
        if source_uses_subdirs:
            filename = os.path.join(section_dir, page)
        else:
            filename = section_dir + '_' + page
    else:
        # Section without pages, only one source file
        filename = section_dir

    try:
        content_type = page_data['type']
    except KeyError:
        content_type = 'unknown'
    try:
        filename = page_data['file']
    except KeyError:
        # No filename was defined explicitely (normal behaviour)
        pass
    if content_type == 'unknown':
        # Try to find the file
        for ct in CONTENT_TYPES:
            try_filename = filename + '.' + ct
            data = get_file_contents(try_filename)
            if data:
                break
    else:
        filename += '.' + CONTENT_TYPES[content_type]
        data = get_file_contents(filename)
    if data and content_type == 'markdown':
        data = markdown.markdown(data, output_format='html5')
    return data

def template_replace(content, replacements):
    """
    Replace text in rendered page with their replacements, for example to ensure
    absolute paths, or replace links of the type:
    href="page/section1/page1/"
    with
    href="/page/section1/page1/"
    when 'page' is pathprefix
    """
    #for key, value in replacements:
    for needle in replacements:
        content = content.replace(needle, replacements[needle])
    return content

## Main program
@click.group()
def cli():
    """
    Paragoo website generator
    """
    pass

@cli.command()
@click.option('-s', '--site', prompt='Site path')
def check_config(site):
    """
    Check site config (site.yaml) for correctness
    """
    click.secho('Needs implementing', fg='red')

#@cli.command('run_disruptions')
@cli.command()
@click.option('-s', '--site', help='Site path', prompt='Site path')
@click.option('-t', '--template', help='Template path', prompt='Template path')
@click.option('-o', '--output_dir', help='Output path for generated content', prompt='Output path for generated content')
@click.option('-p', '--pathprefix', help='Prepend navigation paths with this url-part', default='')
@click.option('--makerooturi/--nomakerooturi', help='Replace relative paths starting with pathprefix to start with a /', default=False)
@click.option('--clean/--noclean', help='Clean the output_dir first or not', default=False)
def generate_site(site, template, output_dir, pathprefix, makerooturi, clean):
    """
    Generate the website specified in the config
    """
    # Change default encoding to UTF-8
    # We need to reload sys module first, because setdefaultencoding is available
    # only at startup time
    reload(sys)
    sys.setdefaultencoding('utf-8')

    print '> start'
    try:
        f = open(os.path.join(site, 'site.yaml'))

        print('r Reading structure from ' + os.path.join(site, 'site.yaml'))

        structure = ordered_load(f, yaml.SafeLoader)
        f.close()
    except IOError as e:
        print e
        sys.exit(1)

    # Templates can live anywhere, define them on the command line
    template_dir = template
    #loader = jinja2.FileSystemLoader(template_dir)
    loader = jinja2.FileSystemLoader(
            [template_dir,
             os.path.join(os.path.dirname(__file__), 'templates/includes'),
             os.path.join(os.path.dirname(__file__), 'templates')])
    environment = jinja2.Environment(loader=loader, trim_blocks=True, lstrip_blocks=True)

    #environment.filters['paragoo_includes'] = paragoo_includes

    template = environment.get_template('base.html')

    if clean:
        print 'd Cleaning up output_dir ' + output_dir
        if os.path.exists(output_dir):
            current_time = datetime.datetime.now()
            dt_format = '%Y-%m-%dT%H:%M:%S%z'
            timestamp = current_time.strftime(dt_format)
            #dst = os.path.join(os.path.dirname(output_dir), timestamp)
            dst = output_dir + '_' + timestamp
            shutil.move(output_dir, dst)
    else:
        print '! Not cleaning up, overwrite existing, keeping others'

    ensure_dir(output_dir)

    # Site-global texts
    site_fields = ['title', 'author', 'description', 'logo', 'copyright', 'footer', 'about_title', 'about', 'linkblocks']
    site_data = {}
    for field in site_fields:
        if field in structure:
            site_data[field] = structure[field]
        else:
            print '! site field "' + field + '" not found'

    source_uses_subdirs = True
    try:
        source_uses_subdirs = structure['subdirs']
    except KeyError:
        print 'I Defaulting to searching sub directories for source files'

    if pathprefix != '' and pathprefix[0] == '/':
        # Stip leading / from the prefix
        pathprefix = pathprefix[1:]

    try:
        template_replacements = dict(structure['replacements'])
    except KeyError:
        template_replacements = {}

    print template_replacements

    if makerooturi:
        template_replacements['href="' + pathprefix] = 'href="/' + pathprefix

    # Create navbar datastructure
    navbar = generate_navbar(structure, pathprefix)

    very_first_page = True # Homepage of website, root
    for section in structure['sections']:
        # Iterate over the sections
        section_data = structure['sections'][section]
        section_filename = os.path.join(output_dir, pathprefix, section)
        source_section_filename = os.path.join(site, section)
        if not source_uses_subdirs:
            source_section_filename = os.path.join(site, 'pages', section)
        first_page = True # Homepage of section
        if not 'pages' in section_data:
            print '- section ' + section + ' does not have pages'
            page_data = structure['sections'][section]
            htmlbody = load_page_source(source_uses_subdirs, source_section_filename, None, {})
            #data = load_page_source(source_uses_subdirs, os.path.dirname(source_section_filename), section, {})
            if htmlbody:
                # Template variables
                data = site_data.copy()
                data['site'] = site_data
                try:
                    data['author'] = page_data['author']
                except KeyError:
                    data['author'] = site_data['author']
                data['page'] = section_data
                if 'description' in section_data:
                    data['description'] = section_data['description']
                data['htmlbody'] = paragoo_includes(site, environment, htmlbody)
                data['navbar'] = navbar
                data['active_section'] = section
                data['active_page'] = section
                data['structure'] = structure
                # Render the page
                output = template.render(data)
                output = template_replace(output, template_replacements)
                filename = os.path.join(section_filename, 'index.html')
                ensure_dir(filename)
                with open(filename, 'w') as pf:
                    pf.write(output)
                if first_page:
                    # Also save an index file for the section (first page in section is section homepage)
                    first_page = False
                    filename = os.path.join(section_filename, 'index.html')
                    with open(filename, 'w') as pf:
                        pf.write(output)
                if very_first_page:
                    # Also save an index file for the homepage, root of site
                    very_first_page = False
                    filename = os.path.join(output_dir, 'index.html')
                    with open(filename, 'w') as pf:
                        pf.write(output)
            else:
                print 'E hm, also no section page found'
        else:
            for page in section_data['pages']:
                # Loop over its pages
                page_data = structure['sections'][section]['pages'][page]
                htmlbody = load_page_source(source_uses_subdirs, source_section_filename, page, page_data)
                # Template variables
                data = site_data.copy()
                data['site'] = site_data
                try:
                    data['author'] = page_data['author']
                except KeyError:
                    data['author'] = site_data['author']
                data['page'] = page_data
                data['htmlbody'] = paragoo_includes(site, environment, htmlbody)
                data['navbar'] = navbar
                data['active_section'] = section
                data['active_page'] = page
                data['structure'] = structure
                # Render the page
                output = template.render(data)
                output = template_replace(output, template_replacements)
                # Save to output_dir
                filename = os.path.join(section_filename, page, 'index.html')
                ensure_dir(filename)
                with open(filename, 'w') as pf:
                    pf.write(output)
                if first_page:
                    # Also save an index file for the section (first page in section is section homepage)
                    first_page = False
                    filename = os.path.join(section_filename, 'index.html')
                    with open(filename, 'w') as pf:
                        pf.write(output)
                if very_first_page:
                    # Also save an index file for the homepage, root of site
                    very_first_page = False
                    filename = os.path.join(output_dir, 'index.html')
                    with open(filename, 'w') as pf:
                        pf.write(output)
    # Generate static pages to be used with the httpd's error directives
    error_pages = {'404': 'Not found', '403': 'Forbidden'}
    for page in error_pages:
        data['page'] = {'title': page + ' ' + error_pages[page]}
        try:
            data['htmlbody'] = paragoo_includes(site, environment, structure['errorpage'])
        except KeyError:
            data['htmlbody'] = 'An error occurred. Use the navigation to find something else on the website or use history back to go back to where you came from.'
        output = template.render(data)
        filename = os.path.join(output_dir, page + '.html')
        with open(filename, 'w') as pf:
            pf.write(output)
    # Copy the directories with static assets
    static_dirs = ['images', 'styles', 'scripts', 'css', 'font', 'js', 'static']
    src_dirs = [template_dir, os.path.dirname(site)]
    for dirname in static_dirs:
        for src_dir in src_dirs:
            src = os.path.join(src_dir, dirname)
            print '- copying directory "' + src + '"'
            dst = os.path.join(output_dir, dirname)
            if not os.path.exists(src):
                print 'E Source directory not found, skipping'
            else:
                copytree(src, dst)
                #try:
                #    shutil.copytree(src, dst, symlinks=False, ignore=None)
                #except OSError:
                #    print 'E Directory already exists, skipping'
    print '> done'

if __name__ == '__main__':
    """
    Paragoo is ran standalone
    """
    cli()
<commit_msg>Support for toggling automatic title per page<commit_after>import os
import stat
import sys
import jinja2
import yaml
import click
import markdown
import shutil
import datetime
from collections import OrderedDict

def copytree(src, dst, symlinks = False, ignore = None):
    """
    Copy a tree of files and dirs and merge into existing dir if needed
    Source: http://stackoverflow.com/a/22331852
    """
    if not os.path.exists(dst):
        os.makedirs(dst)
        shutil.copystat(src, dst)
    lst = os.listdir(src)
    if ignore:
        excl = ignore(src, lst)
        lst = [x for x in lst if x not in excl]
    for item in lst:
        s = os.path.join(src, item)
        d = os.path.join(dst, item)
        if symlinks and os.path.islink(s):
            if os.path.lexists(d):
                os.remove(d)
            os.symlink(os.readlink(s), d)
            try:
                st = os.lstat(s)
                mode = stat.S_IMODE(st.st_mode)
                os.lchmod(d, mode)
            except:
                pass # lchmod not available
        elif os.path.isdir(s):
            copytree(s, d, symlinks, ignore)
        else:
            shutil.copy2(s, d)

def ordered_load(stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict):
    class OrderedLoader(Loader):
        pass
    def construct_mapping(loader, node):
        loader.flatten_mapping(node)
        return object_pairs_hook(loader.construct_pairs(node))
    OrderedLoader.add_constructor(
        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
        construct_mapping)
    return yaml.load(stream, OrderedLoader)

def include_type_exists(key):
    """
    Check whether the include type (plugin) is valid/exists.
    Needs a file of the format plugins/<key>.py
    """
    script_dir = os.path.dirname(os.path.abspath(__file__))
    return os.path.isfile(os.path.join(script_dir, 'plugins', key + '.py'))

def render_include(site, environment, key, params):
    """
    Render paragoo include. Typically looks like:
    @@@key=param@@@
    @@@key=param1:param2:param3@@@
    """
    # Load the relevant plugin
    plugin = __import__('plugins.' + key, globals(), locals(), [key], -1)
    return plugin.render(site, environment, params)

def paragoo_includes(site, environment, body, token='@@@'):
    """
    Filter that looks for blocks surrounded by `token` and include that content type in
    the rendered html
    """
    result = ''
    if body:
        body_parts = body.split(token)
        is_content = True
        for part in body_parts:
            if is_content:
                result += part
                is_content = False
            else:
                include_parts = part.split('=')
                include_params = include_parts[1].split(':')
                print '  ' + str(include_parts)
                if include_type_exists(include_parts[0]):
                    result += render_include(site, environment, include_parts[0], include_params)
                else:
                    print 'E Plugin not found for include with key "' + include_parts[0] + '"'
                is_content = True
    return result

CONTENT_TYPES = {
    'markdown': 'md',
    'html': 'html',
    'gallery': 'gallery',
}

def generate_navbar(structure, pathprefix):
    navbar = []
    for section in structure['sections']:
        navbar_section = []
        section_data = structure['sections'][section]
        section_url = os.path.join('/', pathprefix, section + '/')
        section_title = section_data['title']
        if 'pages' in section_data:
            for page in section_data['pages']:
                url = os.path.join('/', pathprefix, section, page)
                title = structure['sections'][section]['pages'][page]['title']
                if title:
                    # Only add a page to the navigation if it has a title, otherwise it's hidden
                    navbar_section.append((url, page, title))
        if section_title:
            navbar.append((section_url, section, section_title, navbar_section))
    return navbar

def ensure_dir(f):
    d = os.path.dirname(f)
    if not os.path.exists(d):
        os.makedirs(d)

def get_file_contents(filename):
    data = None
    try:
        with open(filename) as pf:
            data = pf.read()
    except IOError:
        #print 'File not found: ' + filename
        pass
    return data

def load_page_source(source_uses_subdirs, section_dir, page, page_data):
    if page:
        if source_uses_subdirs:
            filename = os.path.join(section_dir, page)
        else:
            filename = section_dir + '_' + page
    else:
        # Section without pages, only one source file
        filename = section_dir

    try:
        content_type = page_data['type']
    except KeyError:
        content_type = 'unknown'
    try:
        filename = page_data['file']
    except KeyError:
        # No filename was defined explicitely (normal behaviour)
        pass
    if content_type == 'unknown':
        # Try to find the file
        for ct in CONTENT_TYPES:
            try_filename = filename + '.' + ct
            data = get_file_contents(try_filename)
            if data:
                break
    else:
        filename += '.' + CONTENT_TYPES[content_type]
        data = get_file_contents(filename)
    if data and content_type == 'markdown':
        data = markdown.markdown(data, output_format='html5')
    return data

def template_replace(content, replacements):
    """
    Replace text in rendered page with their replacements, for example to ensure
    absolute paths, or replace links of the type:
    href="page/section1/page1/"
    with
    href="/page/section1/page1/"
    when 'page' is pathprefix
    """
    #for key, value in replacements:
    for needle in replacements:
        content = content.replace(needle, replacements[needle])
    return content

## Main program
@click.group()
def cli():
    """
    Paragoo website generator
    """
    pass

@cli.command()
@click.option('-s', '--site', prompt='Site path')
def check_config(site):
    """
    Check site config (site.yaml) for correctness
    """
    click.secho('Needs implementing', fg='red')

#@cli.command('run_disruptions')
@cli.command()
@click.option('-s', '--site', help='Site path', prompt='Site path')
@click.option('-t', '--template', help='Template path', prompt='Template path')
@click.option('-o', '--output_dir', help='Output path for generated content', prompt='Output path for generated content')
@click.option('-p', '--pathprefix', help='Prepend navigation paths with this url-part', default='')
@click.option('--makerooturi/--nomakerooturi', help='Replace relative paths starting with pathprefix to start with a /', default=False)
@click.option('--clean/--noclean', help='Clean the output_dir first or not', default=False)
def generate_site(site, template, output_dir, pathprefix, makerooturi, clean):
    """
    Generate the website specified in the config
    """
    # Change default encoding to UTF-8
    # We need to reload sys module first, because setdefaultencoding is available
    # only at startup time
    reload(sys)
    sys.setdefaultencoding('utf-8')

    print '> start'
    try:
        f = open(os.path.join(site, 'site.yaml'))

        print('r Reading structure from ' + os.path.join(site, 'site.yaml'))

        structure = ordered_load(f, yaml.SafeLoader)
        f.close()
    except IOError as e:
        print e
        sys.exit(1)

    # Templates can live anywhere, define them on the command line
    template_dir = template
    #loader = jinja2.FileSystemLoader(template_dir)
    loader = jinja2.FileSystemLoader(
            [template_dir,
             os.path.join(os.path.dirname(__file__), 'templates/includes'),
             os.path.join(os.path.dirname(__file__), 'templates')])
    environment = jinja2.Environment(loader=loader, trim_blocks=True, lstrip_blocks=True)

    #environment.filters['paragoo_includes'] = paragoo_includes

    template = environment.get_template('base.html')

    if clean:
        print 'd Cleaning up output_dir ' + output_dir
        if os.path.exists(output_dir):
            current_time = datetime.datetime.now()
            dt_format = '%Y-%m-%dT%H:%M:%S%z'
            timestamp = current_time.strftime(dt_format)
            #dst = os.path.join(os.path.dirname(output_dir), timestamp)
            dst = output_dir + '_' + timestamp
            shutil.move(output_dir, dst)
    else:
        print '! Not cleaning up, overwrite existing, keeping others'

    ensure_dir(output_dir)

    # Site-global texts
    site_fields = ['title', 'author', 'description', 'logo', 'copyright', 'footer', 'about_title', 'about', 'linkblocks']
    site_data = {}
    for field in site_fields:
        if field in structure:
            site_data[field] = structure[field]
        else:
            print '! site field "' + field + '" not found'

    source_uses_subdirs = True
    try:
        source_uses_subdirs = structure['subdirs']
    except KeyError:
        print 'I Defaulting to searching sub directories for source files'

    if pathprefix != '' and pathprefix[0] == '/':
        # Stip leading / from the prefix
        pathprefix = pathprefix[1:]

    try:
        template_replacements = dict(structure['replacements'])
    except KeyError:
        template_replacements = {}

    print template_replacements

    if makerooturi:
        template_replacements['href="' + pathprefix] = 'href="/' + pathprefix

    # Create navbar datastructure
    navbar = generate_navbar(structure, pathprefix)

    very_first_page = True # Homepage of website, root
    for section in structure['sections']:
        # Iterate over the sections
        section_data = structure['sections'][section]
        section_filename = os.path.join(output_dir, pathprefix, section)
        source_section_filename = os.path.join(site, section)
        if not source_uses_subdirs:
            source_section_filename = os.path.join(site, 'pages', section)
        first_page = True # Homepage of section
        if not 'pages' in section_data:
            print '- section ' + section + ' does not have pages'
            page_data = structure['sections'][section]
            htmlbody = load_page_source(source_uses_subdirs, source_section_filename, None, {})
            #data = load_page_source(source_uses_subdirs, os.path.dirname(source_section_filename), section, {})
            if htmlbody:
                # Template variables
                data = site_data.copy()
                data['site'] = site_data
                try:
                    data['author'] = page_data['author']
                except KeyError:
                    data['author'] = site_data['author']
                data['page'] = section_data
                if 'description' in section_data:
                    data['description'] = section_data['description']
                if 'show_title' in section_data:
                    data['show_title'] = section_data['show_title']
                else:
                    # Default to showing the page's title
                    data['show_title'] = True
                data['htmlbody'] = paragoo_includes(site, environment, htmlbody)
                data['navbar'] = navbar
                data['active_section'] = section
                data['active_page'] = section
                data['structure'] = structure
                # Render the page
                output = template.render(data)
                output = template_replace(output, template_replacements)
                filename = os.path.join(section_filename, 'index.html')
                ensure_dir(filename)
                with open(filename, 'w') as pf:
                    pf.write(output)
                if first_page:
                    # Also save an index file for the section (first page in section is section homepage)
                    first_page = False
                    filename = os.path.join(section_filename, 'index.html')
                    with open(filename, 'w') as pf:
                        pf.write(output)
                if very_first_page:
                    # Also save an index file for the homepage, root of site
                    very_first_page = False
                    filename = os.path.join(output_dir, 'index.html')
                    with open(filename, 'w') as pf:
                        pf.write(output)
            else:
                print 'E hm, also no section page found'
        else:
            for page in section_data['pages']:
                # Loop over its pages
                page_data = structure['sections'][section]['pages'][page]
                htmlbody = load_page_source(source_uses_subdirs, source_section_filename, page, page_data)
                # Template variables
                data = site_data.copy()
                data['site'] = site_data
                try:
                    data['author'] = page_data['author']
                except KeyError:
                    data['author'] = site_data['author']
                if 'show_title' in page_data:
                    data['show_title'] = page_data['show_title']
                else:
                    # Default to showing the page's title
                    data['show_title'] = True
                data['page'] = page_data
                data['htmlbody'] = paragoo_includes(site, environment, htmlbody)
                data['navbar'] = navbar
                data['active_section'] = section
                data['active_page'] = page
                data['structure'] = structure
                # Render the page
                output = template.render(data)
                output = template_replace(output, template_replacements)
                # Save to output_dir
                filename = os.path.join(section_filename, page, 'index.html')
                ensure_dir(filename)
                with open(filename, 'w') as pf:
                    pf.write(output)
                if first_page:
                    # Also save an index file for the section (first page in section is section homepage)
                    first_page = False
                    filename = os.path.join(section_filename, 'index.html')
                    with open(filename, 'w') as pf:
                        pf.write(output)
                if very_first_page:
                    # Also save an index file for the homepage, root of site
                    very_first_page = False
                    filename = os.path.join(output_dir, 'index.html')
                    with open(filename, 'w') as pf:
                        pf.write(output)
    # Generate static pages to be used with the httpd's error directives
    error_pages = {'404': 'Not found', '403': 'Forbidden'}
    for page in error_pages:
        data['page'] = {'title': page + ' ' + error_pages[page]}
        try:
            data['htmlbody'] = paragoo_includes(site, environment, structure['errorpage'])
        except KeyError:
            data['htmlbody'] = 'An error occurred. Use the navigation to find something else on the website or use history back to go back to where you came from.'
        output = template.render(data)
        filename = os.path.join(output_dir, page + '.html')
        with open(filename, 'w') as pf:
            pf.write(output)
    # Copy the directories with static assets
    static_dirs = ['images', 'styles', 'scripts', 'css', 'font', 'js', 'static']
    src_dirs = [template_dir, os.path.dirname(site)]
    for dirname in static_dirs:
        for src_dir in src_dirs:
            src = os.path.join(src_dir, dirname)
            print '- copying directory "' + src + '"'
            dst = os.path.join(output_dir, dirname)
            if not os.path.exists(src):
                print 'E Source directory not found, skipping'
            else:
                copytree(src, dst)
                #try:
                #    shutil.copytree(src, dst, symlinks=False, ignore=None)
                #except OSError:
                #    print 'E Directory already exists, skipping'
    print '> done'

if __name__ == '__main__':
    """
    Paragoo is ran standalone
    """
    cli()
<|endoftext|>


<commit_before>import math
import unittest
import numpy as np
import register

class TestRegister(unittest.TestCase):

    def test_basic_construction(self):
        """Constructing a valid register with no superposition, works"""
        state = np.array([1, 0], dtype=np.complex_)
        r = register.Register(state)

    def test_superposition_construction(self):
        """Constructing a valid register with superposition, works"""
        state = np.array([1/math.sqrt(2), 1/math.sqrt(2)], dtype=np.complex_)
        r = register.Register(state)

    # TODO
    def test_invalid_size_constraint(self):
        """Constructing register with invalid size, fails"""

    def test_invalid_state_constraint(self):
        """Constructing register with invalid state, fails"""
        state = np.array([1, 1, 1, 1], dtype=np.complex_)
        self.assertRaises(Exception, register.Register, state)

if __name__ == '__main__':
    unittest.main()<commit_msg>add (failing) invalid register size test<commit_after>import math
import unittest
import numpy as np
import register

class TestRegister(unittest.TestCase):

    def test_basic_construction(self):
        """Constructing a valid register with no superposition, works"""
        state = np.array([1, 0], dtype=np.complex_)
        r = register.Register(state)

    def test_superposition_construction(self):
        """Constructing a valid register with superposition, works"""
        state = np.array([1/math.sqrt(2), 1/math.sqrt(2)], dtype=np.complex_)
        r = register.Register(state)

    def test_invalid_size_constraint(self):
        """Constructing register with invalid size, fails"""
        state = np.array([1/math.sqrt(2), 1/math.sqrt(2), 0], dtype=np.complex_)
        self.assertRaises(Exception, register.Register, state)

    def test_invalid_state_constraint(self):
        """Constructing register with invalid state, fails"""
        state = np.array([1, 1, 1, 1], dtype=np.complex_)
        self.assertRaises(Exception, register.Register, state)

if __name__ == '__main__':
    unittest.main()<|endoftext|>