<commit_before>def check(process_output, judge_output, **kwargs):
    return judge_output == process_output
<commit_msg>Add presentation error feedback to identical checker (#342)<commit_after>from dmoj.checkers._checker import standard
from dmoj.result import CheckerResult
from dmoj.utils.unicode import utf8bytes

def check(process_output, judge_output, pe_allowed=True, **kwargs):
    if judge_output == process_output:
        return True
    feedback = None
    if pe_allowed and standard(utf8bytes(judge_output), utf8bytes(process_output)):
        # in the event the standard checker would have passed the problem, raise a presentation error
        feedback = "Presentation Error"
    return CheckerResult(False, 0, feedback=feedback)
<|endoftext|>


<commit_before><commit_msg>More simplification of performance graph plotting.<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# ----------------------------------------------------------------------------
# Copyright © 2016, Continuum Analytics, Inc. All rights reserved.
#
# The full license is in the file LICENSE.txt, distributed with this software.
# ----------------------------------------------------------------------------
from __future__ import absolute_import, print_function

from copy import deepcopy
import errno
import platform
import os

from conda_kapsel.commands.main import _parse_args_and_run_subcommand
from conda_kapsel.commands.run import run_command, main
from conda_kapsel.commands.prepare_with_mode import UI_MODE_TEXT_ASSUME_YES_DEVELOPMENT
from conda_kapsel.internal.test.tmpfile_utils import with_directory_contents
from conda_kapsel.project_file import DEFAULT_PROJECT_FILENAME

from conda_kapsel.test.project_utils import project_dir_disable_dedicated_env

class Args(object):
    def __init__(self, **kwargs):
        self.directory = "."
        self.env_spec = None
        self.mode = UI_MODE_TEXT_ASSUME_YES_DEVELOPMENT
        self.command = None
        self.extra_args_for_command = None
        for key in kwargs:
            setattr(self, key, kwargs[key])

python_exe = "python"
if platform.system() == 'Windows':
    python_exe = "python.exe"

def test_run_command(monkeypatch):

    executed = {}

    def mock_execvpe(file, args, env):
        executed['file'] = file
        executed['args'] = args
        executed['env'] = env

    mock_environ = deepcopy(os.environ)
    mock_environ['FOO'] = 'bar'

    monkeypatch.setattr('os.environ', mock_environ)
    monkeypatch.setattr('os.execvpe', mock_execvpe)

    def check_run(dirname):
        project_dir_disable_dedicated_env(dirname)

        result = run_command(dirname,
                             UI_MODE_TEXT_ASSUME_YES_DEVELOPMENT,
                             conda_environment=None,
                             command_name=None,
                             extra_command_args=None)
        assert result is None
        assert 'file' in executed
        assert 'args' in executed
        assert 'env' in executed
        assert executed['file'].endswith(python_exe)
        assert executed['args'][0].endswith(python_exe)
        assert '--version' == executed['args'][1]
        assert 'bar' == executed['env']['FOO']

    with_directory_contents(
        {DEFAULT_PROJECT_FILENAME: """
variables:
  FOO: {}

commands:
  default:
    conda_app_entry: python --version

"""}, check_run)

def test_run_command_no_app_entry(capsys):
    def check_run_no_app_entry(dirname):
        project_dir_disable_dedicated_env(dirname)
        result = run_command(dirname,
                             UI_MODE_TEXT_ASSUME_YES_DEVELOPMENT,
                             conda_environment=None,
                             command_name=None,
                             extra_command_args=None)
        assert result is None

    with_directory_contents({DEFAULT_PROJECT_FILENAME: """

"""}, check_run_no_app_entry)

    out, err = capsys.readouterr()
    assert out == ""
    assert 'No known run command' in err

def test_run_command_nonexistent_project(capsys):
    def check_run_nonexistent(dirname):
        project_dir = os.path.join(dirname, "nope")
        result = _parse_args_and_run_subcommand(['conda-kapsel', 'run', '--directory', project_dir])

        assert 1 == result

        out, err = capsys.readouterr()
        assert out == ""
        assert ("Project directory '%s' does not exist." % project_dir) in err

    with_directory_contents(dict(), check_run_nonexistent)

def test_run_command_failed_prepare(capsys):
    def check_run_failed_prepare(dirname):
        project_dir_disable_dedicated_env(dirname)
        result = run_command(dirname,
                             UI_MODE_TEXT_ASSUME_YES_DEVELOPMENT,
                             conda_environment=None,
                             command_name=None,
                             extra_command_args=None)
        assert result is None

    with_directory_contents({DEFAULT_PROJECT_FILENAME: """
variables:
  - WILL_NOT_BE_SET
"""}, check_run_failed_prepare)

    out, err = capsys.readouterr()
    assert out == ""
    assert 'Environment variable WILL_NOT_BE_SET is not set' in err

def test_main(monkeypatch, capsys):
    def mock_conda_create(prefix, pkgs, channels):
        raise RuntimeError("this test should not create an environment in %s with pkgs %r" % (prefix, pkgs))

    monkeypatch.setattr('conda_kapsel.internal.conda_api.create', mock_conda_create)

    executed = {}

    def mock_execvpe(file, args, env):
        executed['file'] = file
        executed['args'] = args
        executed['env'] = env

    monkeypatch.setattr('os.execvpe', mock_execvpe)

    def check_run_main(dirname):
        project_dir_disable_dedicated_env(dirname)
        result = main(Args(directory=dirname))

        assert 1 == result
        assert 'file' in executed
        assert 'args' in executed
        assert 'env' in executed
        assert executed['file'].endswith(python_exe)
        assert executed['args'][0].endswith(python_exe)
        assert '--version' == executed['args'][1]

    with_directory_contents(
        {DEFAULT_PROJECT_FILENAME: """
commands:
  default:
    conda_app_entry: python --version

"""}, check_run_main)

    out, err = capsys.readouterr()
    assert "" == out
    assert "" == err

def test_main_failed_exec(monkeypatch, capsys):
    def mock_execvpe(file, args, env):
        raise OSError(errno.ENOMEM, "It did not work, Michael")

    monkeypatch.setattr('os.execvpe', mock_execvpe)

    def check_run_main(dirname):
        project_dir_disable_dedicated_env(dirname)
        result = main(Args(directory=dirname))

        assert 1 == result

    with_directory_contents(
        {DEFAULT_PROJECT_FILENAME: """
commands:
  default:
    conda_app_entry: python --version

"""}, check_run_main)

    out, err = capsys.readouterr()
    assert "" == out
    assert 'Failed to execute' in err
    assert 'It did not work, Michael' in err

def test_main_dirname_not_provided_use_pwd(monkeypatch, capsys):
    executed = {}

    def mock_execvpe(file, args, env):
        executed['file'] = file
        executed['args'] = args
        executed['env'] = env

    monkeypatch.setattr('os.execvpe', mock_execvpe)

    def check_run_main(dirname):
        from os.path import abspath as real_abspath

        def mock_abspath(path):
            if path == ".":
                return dirname
            else:
                return real_abspath(path)

        monkeypatch.setattr('os.path.abspath', mock_abspath)

        project_dir_disable_dedicated_env(dirname)
        result = _parse_args_and_run_subcommand(['conda-kapsel', 'run'])

        assert 1 == result
        assert 'file' in executed
        assert 'args' in executed
        assert 'env' in executed
        assert executed['file'].endswith(python_exe)
        assert executed['args'][0].endswith(python_exe)
        assert '--version' == executed['args'][1]

    with_directory_contents(
        {DEFAULT_PROJECT_FILENAME: """
commands:
  default:
    conda_app_entry: python --version

"""}, check_run_main)

    out, err = capsys.readouterr()
    assert "" == out
    assert "" == err

def test_run_command_extra_args(monkeypatch, capsys):
    executed = {}

    def mock_execvpe(file, args, env):
        executed['file'] = file
        executed['args'] = args
        executed['env'] = env

    monkeypatch.setattr('os.execvpe', mock_execvpe)

    def check_run_main(dirname):
        from os.path import abspath as real_abspath

        def mock_abspath(path):
            if path == ".":
                return dirname
            else:
                return real_abspath(path)

        monkeypatch.setattr('os.path.abspath', mock_abspath)

        project_dir_disable_dedicated_env(dirname)
        result = _parse_args_and_run_subcommand(['conda-kapsel', 'run', '--directory', dirname, 'default', 'foo',
                                                 '$PATH', '--something'])

        assert 1 == result
        assert 'file' in executed
        assert 'args' in executed
        assert 'env' in executed
        assert executed['file'].endswith(python_exe)
        assert executed['args'][0].endswith(python_exe)
        assert len(executed['args']) == 5
        assert '--version' == executed['args'][1]
        assert 'foo' == executed['args'][2]
        assert '$PATH' == executed['args'][3]
        assert '--something' == executed['args'][4]

    with_directory_contents(
        {DEFAULT_PROJECT_FILENAME: """
commands:
  default:
    conda_app_entry: python --version

"""}, check_run_main)

    out, err = capsys.readouterr()
    assert "" == out
    assert "" == err

def test_run_command_extra_args_with_double_hyphen(monkeypatch, capsys):
    executed = {}

    def mock_execvpe(file, args, env):
        executed['file'] = file
        executed['args'] = args
        executed['env'] = env

    monkeypatch.setattr('os.execvpe', mock_execvpe)

    def check_run_main(dirname):
        from os.path import abspath as real_abspath

        def mock_abspath(path):
            if path == ".":
                return dirname
            else:
                return real_abspath(path)

        monkeypatch.setattr('os.path.abspath', mock_abspath)

        project_dir_disable_dedicated_env(dirname)
        # double hyphen lets us specify "--foo" as a command name
        result = _parse_args_and_run_subcommand(['conda-kapsel', 'run', '--directory', dirname, '--', '--foo', '--bar'])

        assert 1 == result
        assert 'file' in executed
        assert 'args' in executed
        assert 'env' in executed
        assert executed['file'].endswith(python_exe)
        assert executed['args'][0].endswith(python_exe)
        assert len(executed['args']) == 3
        assert '--version' == executed['args'][1]
        assert '--bar' == executed['args'][2]

    with_directory_contents(
        {DEFAULT_PROJECT_FILENAME: """
commands:
  "--foo":
    conda_app_entry: python --version
"""}, check_run_main)

    out, err = capsys.readouterr()
    assert "" == out
    assert "" == err

def _is_python_exe(path):
    assert path.endswith(python_exe)

def _test_run_command_foo(command_line, monkeypatch, capsys, file_assertion=_is_python_exe):
    executed = {}

    def mock_execvpe(file, args, env):
        executed['file'] = file
        executed['args'] = args
        executed['env'] = env

    monkeypatch.setattr('os.execvpe', mock_execvpe)

    def check_run_main(dirname):
        from os.path import abspath as real_abspath

        def mock_abspath(path):
            if path == ".":
                return dirname
            else:
                return real_abspath(path)

        monkeypatch.setattr('os.path.abspath', mock_abspath)

        project_dir_disable_dedicated_env(dirname)

        for n, i in enumerate(command_line):
            if i == '<DIRNAME>':
                command_line[n] = dirname

        result = _parse_args_and_run_subcommand(command_line)

        assert 1 == result
        assert 'file' in executed
        assert 'args' in executed
        assert 'env' in executed
        file_assertion(executed['file'])

        out, err = capsys.readouterr()
        assert "" == out
        assert "" == err

        return executed['args'][1:]

    return with_directory_contents(
        {DEFAULT_PROJECT_FILENAME: """
commands:
  default:
    conda_app_entry: python --version def
  foo:
    conda_app_entry: python --version foo
  bar:
    conda_app_entry: python --version bar
"""}, check_run_main)

def test_run_command_specify_name_after_options(monkeypatch, capsys):
    args = _test_run_command_foo(['conda-kapsel', 'run', '--directory', '<DIRNAME>', 'foo'], monkeypatch, capsys)

    assert args == ['--version', 'foo']

def test_run_command_specify_name_before_options(monkeypatch, capsys):
    args = _test_run_command_foo(['conda-kapsel', 'run', 'foo', '--directory', '<DIRNAME>'], monkeypatch, capsys)
    assert args[:-1] == ['--version', 'foo', '--directory']

def test_run_command_omit_name_use_default(monkeypatch, capsys):
    args = _test_run_command_foo(['conda-kapsel', 'run', '--directory', '<DIRNAME>'], monkeypatch, capsys)
    assert args == ['--version', 'def']

# can't put an assert in a lambda so this makes us a "lambda" with
# an assert in it
def _func_asserting_contains(what):
    def f(s):
        assert what in s

    return f

def test_run_command_executable_not_in_config(monkeypatch, capsys):
    args = _test_run_command_foo(
        ['conda-kapsel', 'run', '--directory', '<DIRNAME>', 'something1',
         'something2'], monkeypatch, capsys, _func_asserting_contains('something1'))
    assert args == ['something2']

def test_run_notebook_not_in_config(monkeypatch, capsys):
    args = _test_run_command_foo(
        ['conda-kapsel', 'run', '--directory', '<DIRNAME>',
         'something.ipynb'], monkeypatch, capsys, _func_asserting_contains('jupyter-notebook'))
    assert len(args) == 1
    assert args[0].endswith('something.ipynb')

def test_run_command_nonexistent_name(monkeypatch, capsys):
    def check_run_main(dirname):
        from os.path import abspath as real_abspath

        def mock_abspath(path):
            if path == ".":
                return dirname
            else:
                return real_abspath(path)

        monkeypatch.setattr('os.path.abspath', mock_abspath)

        project_dir_disable_dedicated_env(dirname)
        result = _parse_args_and_run_subcommand(['conda-kapsel', 'run', '--directory', dirname, 'nope'])

        assert 1 == result

        out, err = capsys.readouterr()
        assert "" == out
        assert "Failed to execute 'nope'" in err

    with_directory_contents(
        {DEFAULT_PROJECT_FILENAME: """
commands:
  default:
    conda_app_entry: python --version
  foo:
    conda_app_entry: python --version foo
  bar:
    conda_app_entry: python --version bar
"""}, check_run_main)
<commit_msg>Mock execvpe in test_run.py test_run_command_nonexistent_name<commit_after># -*- coding: utf-8 -*-
# ----------------------------------------------------------------------------
# Copyright © 2016, Continuum Analytics, Inc. All rights reserved.
#
# The full license is in the file LICENSE.txt, distributed with this software.
# ----------------------------------------------------------------------------
from __future__ import absolute_import, print_function

from copy import deepcopy
import errno
import platform
import os

from conda_kapsel.commands.main import _parse_args_and_run_subcommand
from conda_kapsel.commands.run import run_command, main
from conda_kapsel.commands.prepare_with_mode import UI_MODE_TEXT_ASSUME_YES_DEVELOPMENT
from conda_kapsel.internal.test.tmpfile_utils import with_directory_contents
from conda_kapsel.project_file import DEFAULT_PROJECT_FILENAME

from conda_kapsel.test.project_utils import project_dir_disable_dedicated_env

class Args(object):
    def __init__(self, **kwargs):
        self.directory = "."
        self.env_spec = None
        self.mode = UI_MODE_TEXT_ASSUME_YES_DEVELOPMENT
        self.command = None
        self.extra_args_for_command = None
        for key in kwargs:
            setattr(self, key, kwargs[key])

python_exe = "python"
if platform.system() == 'Windows':
    python_exe = "python.exe"

def test_run_command(monkeypatch):

    executed = {}

    def mock_execvpe(file, args, env):
        executed['file'] = file
        executed['args'] = args
        executed['env'] = env

    mock_environ = deepcopy(os.environ)
    mock_environ['FOO'] = 'bar'

    monkeypatch.setattr('os.environ', mock_environ)
    monkeypatch.setattr('os.execvpe', mock_execvpe)

    def check_run(dirname):
        project_dir_disable_dedicated_env(dirname)

        result = run_command(dirname,
                             UI_MODE_TEXT_ASSUME_YES_DEVELOPMENT,
                             conda_environment=None,
                             command_name=None,
                             extra_command_args=None)
        assert result is None
        assert 'file' in executed
        assert 'args' in executed
        assert 'env' in executed
        assert executed['file'].endswith(python_exe)
        assert executed['args'][0].endswith(python_exe)
        assert '--version' == executed['args'][1]
        assert 'bar' == executed['env']['FOO']

    with_directory_contents(
        {DEFAULT_PROJECT_FILENAME: """
variables:
  FOO: {}

commands:
  default:
    conda_app_entry: python --version

"""}, check_run)

def test_run_command_no_app_entry(capsys):
    def check_run_no_app_entry(dirname):
        project_dir_disable_dedicated_env(dirname)
        result = run_command(dirname,
                             UI_MODE_TEXT_ASSUME_YES_DEVELOPMENT,
                             conda_environment=None,
                             command_name=None,
                             extra_command_args=None)
        assert result is None

    with_directory_contents({DEFAULT_PROJECT_FILENAME: """

"""}, check_run_no_app_entry)

    out, err = capsys.readouterr()
    assert out == ""
    assert 'No known run command' in err

def test_run_command_nonexistent_project(capsys):
    def check_run_nonexistent(dirname):
        project_dir = os.path.join(dirname, "nope")
        result = _parse_args_and_run_subcommand(['conda-kapsel', 'run', '--directory', project_dir])

        assert 1 == result

        out, err = capsys.readouterr()
        assert out == ""
        assert ("Project directory '%s' does not exist." % project_dir) in err

    with_directory_contents(dict(), check_run_nonexistent)

def test_run_command_failed_prepare(capsys):
    def check_run_failed_prepare(dirname):
        project_dir_disable_dedicated_env(dirname)
        result = run_command(dirname,
                             UI_MODE_TEXT_ASSUME_YES_DEVELOPMENT,
                             conda_environment=None,
                             command_name=None,
                             extra_command_args=None)
        assert result is None

    with_directory_contents({DEFAULT_PROJECT_FILENAME: """
variables:
  - WILL_NOT_BE_SET
"""}, check_run_failed_prepare)

    out, err = capsys.readouterr()
    assert out == ""
    assert 'Environment variable WILL_NOT_BE_SET is not set' in err

def test_main(monkeypatch, capsys):
    def mock_conda_create(prefix, pkgs, channels):
        raise RuntimeError("this test should not create an environment in %s with pkgs %r" % (prefix, pkgs))

    monkeypatch.setattr('conda_kapsel.internal.conda_api.create', mock_conda_create)

    executed = {}

    def mock_execvpe(file, args, env):
        executed['file'] = file
        executed['args'] = args
        executed['env'] = env

    monkeypatch.setattr('os.execvpe', mock_execvpe)

    def check_run_main(dirname):
        project_dir_disable_dedicated_env(dirname)
        result = main(Args(directory=dirname))

        assert 1 == result
        assert 'file' in executed
        assert 'args' in executed
        assert 'env' in executed
        assert executed['file'].endswith(python_exe)
        assert executed['args'][0].endswith(python_exe)
        assert '--version' == executed['args'][1]

    with_directory_contents(
        {DEFAULT_PROJECT_FILENAME: """
commands:
  default:
    conda_app_entry: python --version

"""}, check_run_main)

    out, err = capsys.readouterr()
    assert "" == out
    assert "" == err

def test_main_failed_exec(monkeypatch, capsys):
    def mock_execvpe(file, args, env):
        raise OSError(errno.ENOMEM, "It did not work, Michael")

    monkeypatch.setattr('os.execvpe', mock_execvpe)

    def check_run_main(dirname):
        project_dir_disable_dedicated_env(dirname)
        result = main(Args(directory=dirname))

        assert 1 == result

    with_directory_contents(
        {DEFAULT_PROJECT_FILENAME: """
commands:
  default:
    conda_app_entry: python --version

"""}, check_run_main)

    out, err = capsys.readouterr()
    assert "" == out
    assert 'Failed to execute' in err
    assert 'It did not work, Michael' in err

def test_main_dirname_not_provided_use_pwd(monkeypatch, capsys):
    executed = {}

    def mock_execvpe(file, args, env):
        executed['file'] = file
        executed['args'] = args
        executed['env'] = env

    monkeypatch.setattr('os.execvpe', mock_execvpe)

    def check_run_main(dirname):
        from os.path import abspath as real_abspath

        def mock_abspath(path):
            if path == ".":
                return dirname
            else:
                return real_abspath(path)

        monkeypatch.setattr('os.path.abspath', mock_abspath)

        project_dir_disable_dedicated_env(dirname)
        result = _parse_args_and_run_subcommand(['conda-kapsel', 'run'])

        assert 1 == result
        assert 'file' in executed
        assert 'args' in executed
        assert 'env' in executed
        assert executed['file'].endswith(python_exe)
        assert executed['args'][0].endswith(python_exe)
        assert '--version' == executed['args'][1]

    with_directory_contents(
        {DEFAULT_PROJECT_FILENAME: """
commands:
  default:
    conda_app_entry: python --version

"""}, check_run_main)

    out, err = capsys.readouterr()
    assert "" == out
    assert "" == err

def test_run_command_extra_args(monkeypatch, capsys):
    executed = {}

    def mock_execvpe(file, args, env):
        executed['file'] = file
        executed['args'] = args
        executed['env'] = env

    monkeypatch.setattr('os.execvpe', mock_execvpe)

    def check_run_main(dirname):
        from os.path import abspath as real_abspath

        def mock_abspath(path):
            if path == ".":
                return dirname
            else:
                return real_abspath(path)

        monkeypatch.setattr('os.path.abspath', mock_abspath)

        project_dir_disable_dedicated_env(dirname)
        result = _parse_args_and_run_subcommand(['conda-kapsel', 'run', '--directory', dirname, 'default', 'foo',
                                                 '$PATH', '--something'])

        assert 1 == result
        assert 'file' in executed
        assert 'args' in executed
        assert 'env' in executed
        assert executed['file'].endswith(python_exe)
        assert executed['args'][0].endswith(python_exe)
        assert len(executed['args']) == 5
        assert '--version' == executed['args'][1]
        assert 'foo' == executed['args'][2]
        assert '$PATH' == executed['args'][3]
        assert '--something' == executed['args'][4]

    with_directory_contents(
        {DEFAULT_PROJECT_FILENAME: """
commands:
  default:
    conda_app_entry: python --version

"""}, check_run_main)

    out, err = capsys.readouterr()
    assert "" == out
    assert "" == err

def test_run_command_extra_args_with_double_hyphen(monkeypatch, capsys):
    executed = {}

    def mock_execvpe(file, args, env):
        executed['file'] = file
        executed['args'] = args
        executed['env'] = env

    monkeypatch.setattr('os.execvpe', mock_execvpe)

    def check_run_main(dirname):
        from os.path import abspath as real_abspath

        def mock_abspath(path):
            if path == ".":
                return dirname
            else:
                return real_abspath(path)

        monkeypatch.setattr('os.path.abspath', mock_abspath)

        project_dir_disable_dedicated_env(dirname)
        # double hyphen lets us specify "--foo" as a command name
        result = _parse_args_and_run_subcommand(['conda-kapsel', 'run', '--directory', dirname, '--', '--foo', '--bar'])

        assert 1 == result
        assert 'file' in executed
        assert 'args' in executed
        assert 'env' in executed
        assert executed['file'].endswith(python_exe)
        assert executed['args'][0].endswith(python_exe)
        assert len(executed['args']) == 3
        assert '--version' == executed['args'][1]
        assert '--bar' == executed['args'][2]

    with_directory_contents(
        {DEFAULT_PROJECT_FILENAME: """
commands:
  "--foo":
    conda_app_entry: python --version
"""}, check_run_main)

    out, err = capsys.readouterr()
    assert "" == out
    assert "" == err

def _is_python_exe(path):
    assert path.endswith(python_exe)

def _test_run_command_foo(command_line, monkeypatch, capsys, file_assertion=_is_python_exe):
    executed = {}

    def mock_execvpe(file, args, env):
        executed['file'] = file
        executed['args'] = args
        executed['env'] = env

    monkeypatch.setattr('os.execvpe', mock_execvpe)

    def check_run_main(dirname):
        from os.path import abspath as real_abspath

        def mock_abspath(path):
            if path == ".":
                return dirname
            else:
                return real_abspath(path)

        monkeypatch.setattr('os.path.abspath', mock_abspath)

        project_dir_disable_dedicated_env(dirname)

        for n, i in enumerate(command_line):
            if i == '<DIRNAME>':
                command_line[n] = dirname

        result = _parse_args_and_run_subcommand(command_line)

        assert 1 == result
        assert 'file' in executed
        assert 'args' in executed
        assert 'env' in executed
        file_assertion(executed['file'])

        out, err = capsys.readouterr()
        assert "" == out
        assert "" == err

        return executed['args'][1:]

    return with_directory_contents(
        {DEFAULT_PROJECT_FILENAME: """
commands:
  default:
    conda_app_entry: python --version def
  foo:
    conda_app_entry: python --version foo
  bar:
    conda_app_entry: python --version bar
"""}, check_run_main)

def test_run_command_specify_name_after_options(monkeypatch, capsys):
    args = _test_run_command_foo(['conda-kapsel', 'run', '--directory', '<DIRNAME>', 'foo'], monkeypatch, capsys)

    assert args == ['--version', 'foo']

def test_run_command_specify_name_before_options(monkeypatch, capsys):
    args = _test_run_command_foo(['conda-kapsel', 'run', 'foo', '--directory', '<DIRNAME>'], monkeypatch, capsys)
    assert args[:-1] == ['--version', 'foo', '--directory']

def test_run_command_omit_name_use_default(monkeypatch, capsys):
    args = _test_run_command_foo(['conda-kapsel', 'run', '--directory', '<DIRNAME>'], monkeypatch, capsys)
    assert args == ['--version', 'def']

# can't put an assert in a lambda so this makes us a "lambda" with
# an assert in it
def _func_asserting_contains(what):
    def f(s):
        assert what in s

    return f

def test_run_command_executable_not_in_config(monkeypatch, capsys):
    args = _test_run_command_foo(
        ['conda-kapsel', 'run', '--directory', '<DIRNAME>', 'something1',
         'something2'], monkeypatch, capsys, _func_asserting_contains('something1'))
    assert args == ['something2']

def test_run_notebook_not_in_config(monkeypatch, capsys):
    args = _test_run_command_foo(
        ['conda-kapsel', 'run', '--directory', '<DIRNAME>',
         'something.ipynb'], monkeypatch, capsys, _func_asserting_contains('jupyter-notebook'))
    assert len(args) == 1
    assert args[0].endswith('something.ipynb')

def test_run_command_nonexistent_name(monkeypatch, capsys):
    def check_run_main(dirname):
        from os.path import abspath as real_abspath

        def mock_abspath(path):
            if path == ".":
                return dirname
            else:
                return real_abspath(path)

        monkeypatch.setattr('os.path.abspath', mock_abspath)

        def mock_execvpe(file, args, env):
            assert file == 'nope'
            assert args == ['nope']
            raise OSError("no such file nope")

        monkeypatch.setattr('os.execvpe', mock_execvpe)

        project_dir_disable_dedicated_env(dirname)
        result = _parse_args_and_run_subcommand(['conda-kapsel', 'run', '--directory', dirname, 'nope'])

        assert 1 == result

        out, err = capsys.readouterr()
        assert "" == out
        assert "Failed to execute 'nope'" in err

    with_directory_contents(
        {DEFAULT_PROJECT_FILENAME: """
commands:
  default:
    conda_app_entry: python --version
  foo:
    conda_app_entry: python --version foo
  bar:
    conda_app_entry: python --version bar
"""}, check_run_main)
<|endoftext|>


<commit_before>from django.core.exceptions import ValidationError
from django.db import models

class EdgarSearch(models.Model):
    class Meta:
        verbose_name_plural = 'Edgar Requests'

    description = models.TextField(help_text='Description of the request.')
    request = models.TextField()
    response = models.TextField()
    date_accessed = models.DateTimeField(auto_now_add=True)

class EdgarCompanyInfo(models.Model):
    class Meta:
        verbose_name_plural = 'Companies'

    cik = models.CharField(max_length=200, blank=True)
    conformed_name = models.CharField(max_length=200, blank=True)
    ticker_symbol = models.CharField(max_length=10, blank=True)

    # Industry code
    sic_code = models.CharField(max_length=200, blank=True)
    sic_description = models.CharField(max_length=200, blank=True)

    state_location = models.CharField(max_length=10, blank=True)

    def clean(self):
        if not self.cik or not self.ticker_symbol:
            raise ValidationError('Must have CIK or Ticker Symbol at least')

    def __str__(self):
        return self.conformed_name

class EdgarSDFiling(models.Model):
    class Meta:
        verbose_name = 'SD Filing'
        verbose_name_plural = 'SD Filings'

    company = models.ForeignKey(EdgarCompanyInfo, on_delete=models.CASCADE, blank=True, null=True)
    date = models.DateField(blank=True, null=True, help_text='Filing Date')
    accepted = models.DateTimeField(blank=True, null=True, help_text='Accepted Date')
    sec_accession_number = models.CharField(max_length=200)
    link = models.TextField(blank=True, null=True)

    @classmethod
    def get_or_create_from_feed_entry(cls, entry, company):
        """
        Note the typo in the feed keys 'accession-nunber'
        """
        assert entry.get('filing-type') == 'SD'
        obj, _ = cls.objects.get_or_create(sec_accession_number=entry.get('accession-nunber'))
        if obj.company:
            # Test something hasn't gone wierd
            assert obj.company == company
        else:
            obj.company = company
        obj.date = entry.get('filing-date')
        obj.accepted = entry.get('updated')
        obj.link = entry.get('link')
        return obj.save()

class EdgarSDFilingDocument(models.Model):
    class Meta:
        verbose_name_plural = 'SD Filing Documents'

    filing = models.ForeignKey(EdgarSDFiling, on_delete=models.CASCADE)
    seq = models.PositiveIntegerField(blank=True, null=True)
    description = models.CharField(max_length=200)
    doc_type = models.CharField(max_length=50, blank=True)
    doc_size = models.PositiveIntegerField()
    doc_name = models.CharField(max_length=200)
    doc_url = models.TextField()
    doc_format = models.CharField(max_length=10)

class EdgarDocumentContent(models.Model):
    class Meta:
        verbose_name_plural = 'Document Content'

    document = models.ForeignKey(EdgarSDFilingDocument, on_delete=models.CASCADE)
    binary = models.BinaryField(blank=True, null=True)
    text = models.TextField(blank=True)

    def clean(self):
        if self.binary and self.text:
            raise ValidationError('Can only store one type of content')
        if not self.binary and not self.text:
            raise ValidationError('Must store some content')

    @property
    def content(self):
        if self.binary:
            return binary
        elif self.text:
            return text<commit_msg>Don't fail so hard if not SD<commit_after>from django.core.exceptions import ValidationError
from django.db import models

class EdgarSearch(models.Model):
    class Meta:
        verbose_name_plural = 'Edgar Requests'

    description = models.TextField(help_text='Description of the request.')
    request = models.TextField()
    response = models.TextField()
    date_accessed = models.DateTimeField(auto_now_add=True)

class EdgarCompanyInfo(models.Model):
    class Meta:
        verbose_name_plural = 'Companies'

    cik = models.CharField(max_length=200, blank=True)
    conformed_name = models.CharField(max_length=200, blank=True)
    ticker_symbol = models.CharField(max_length=10, blank=True)

    # Industry code
    sic_code = models.CharField(max_length=200, blank=True)
    sic_description = models.CharField(max_length=200, blank=True)

    state_location = models.CharField(max_length=10, blank=True)

    def clean(self):
        if not self.cik or not self.ticker_symbol:
            raise ValidationError('Must have CIK or Ticker Symbol at least')

    def __str__(self):
        return self.conformed_name

class EdgarSDFiling(models.Model):
    class Meta:
        verbose_name = 'SD Filing'
        verbose_name_plural = 'SD Filings'

    company = models.ForeignKey(EdgarCompanyInfo, on_delete=models.CASCADE, blank=True, null=True)
    date = models.DateField(blank=True, null=True, help_text='Filing Date')
    accepted = models.DateTimeField(blank=True, null=True, help_text='Accepted Date')
    sec_accession_number = models.CharField(max_length=200)
    link = models.TextField(blank=True, null=True)

    @classmethod
    def get_or_create_from_feed_entry(cls, entry, company):
        """
        Note the typo in the feed keys 'accession-nunber'
        """
        if entry.get('filing-type') != 'SD':
            return
        obj, _ = cls.objects.get_or_create(sec_accession_number=entry.get('accession-nunber'))
        if obj.company:
            # Test something hasn't gone wierd
            assert obj.company == company
        else:
            obj.company = company
        obj.date = entry.get('filing-date')
        obj.accepted = entry.get('updated')
        obj.link = entry.get('link')
        return obj.save()

class EdgarSDFilingDocument(models.Model):
    class Meta:
        verbose_name_plural = 'SD Filing Documents'

    filing = models.ForeignKey(EdgarSDFiling, on_delete=models.CASCADE)
    seq = models.PositiveIntegerField(blank=True, null=True)
    description = models.CharField(max_length=200)
    doc_type = models.CharField(max_length=50, blank=True)
    doc_size = models.PositiveIntegerField()
    doc_name = models.CharField(max_length=200)
    doc_url = models.TextField()
    doc_format = models.CharField(max_length=10)

class EdgarDocumentContent(models.Model):
    class Meta:
        verbose_name_plural = 'Document Content'

    document = models.ForeignKey(EdgarSDFilingDocument, on_delete=models.CASCADE)
    binary = models.BinaryField(blank=True, null=True)
    text = models.TextField(blank=True)

    def clean(self):
        if self.binary and self.text:
            raise ValidationError('Can only store one type of content')
        if not self.binary and not self.text:
            raise ValidationError('Must store some content')

    @property
    def content(self):
        if self.binary:
            return binary
        elif self.text:
            return text<|endoftext|>


<commit_before>from sklearn import cross_validation
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
import pandas as pd
from pandas.tools.plotting import scatter_matrix
import numpy as np
import matplotlib.pyplot as plt
import os
import MySQLdb
import db

class MachineLearning:

  DB_SETTINGS = db.DatabaseInterface().return_options()

  def __init__(self,
               data_frame = None,
               mysql = None,
               group_names = None,
               group_column_name = None,
               final_variables_number = None,
               start_with_pca = False,
               training_set_size = None):
    self.df = data_frame
    self.mysql = mysql
    self.gn = group_names
    self.gcn = group_column_name
    self.fvn = final_variables_number
    self.swp = start_with_pca
    self.tss = training_set_size
    self.coefficients = False
    self.prepare_df()

  def load_test(self):
    self.coefficients = self.generate_first_coefficients(self.variables)
    self.mysql = MySQLdb.connect(user = self.DB_SETTINGS['user'], db = self.DB_SETTINGS['name'], host = self.DB_SETTINGS['host'])
    self.df = pd.read_sql_query("SELECT * FROM tracks", con = self.mysql, index_col = ["spotify_id"]).drop("created_at", 1).drop("training", 1).drop("id", 1)
    self.gn = ["sad","happy","energy"]
    self.gcn = "mood"
    self.fvn = 4
    self.swp = False
    self.tss = 0.2
    self.prepare_df()
    return self

  def generate_first_coefficients(self, variables):
    coefficients = {}
    for variable in variables:
      coefficient = np.random.choice([1,-1]) * np.random.random()
      coefficients[variable] = coefficient
    return coefficients

  def prepare_df(self):
    if self.df is None:
      self.variables = []
      self.training_df = None
    else:
      self.variables = self.df.drop([self.gcn], axis=1).columns
      dropped_amount = int(self.tss * self.df.shape[0])
      dropped_indices = np.random.choice(self.df.index, dropped_amount, replace=False)
      self.testing_df = self.df[self.df.index.isin(dropped_indices)]
      self.training_df = self.df.drop(dropped_indices)
      self.dummy_testing_df = pd.get_dummies(self.testing_df)
      self.dummy_training_df = pd.get_dummies(self.training_df)

## PLOTTING METHODS

  def draw_corr_matrix(self):
    plt.imshow(self.dummy_df.corr(), cmap='hot', interpolation='nearest')
    plt.savefig('./plots/cor_matrix.png', bbox_inches='tight')
    plt.close()

  def draw_scatter_matrix(self):
    pd.tools.plotting.scatter_matrix(self.df)
    plt.savefig('./plots/scatter_matrix.png', bbox_inches='tight')
    plt.close()

## TUTORIAL METHODS

  def validation_dataset(self):
    array = self.df.values
    X = array[:1,11]
    Y = array[:0]
    validation_size = self.tss
    seed = 8
    self.X_train, self.X_validation, self.Y_train, self.Y_validation = cross_validation.train_test_split(X, Y, test_size=validation_size, random_state=seed)

test = MachineLearning().load_test()
<commit_msg>whatthewhat"<commit_after>from sklearn import model_selection
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
import pandas as pd
from pandas.tools.plotting import scatter_matrix
import numpy as np
import matplotlib.pyplot as plt
import os
import MySQLdb
import db

class MachineLearning:

  DB_SETTINGS = db.DatabaseInterface().return_options()

  def __init__(self,
               data_frame = None,
               mysql = None,
               group_names = None,
               group_column_name = None,
               final_variables_number = None,
               start_with_pca = False,
               training_set_size = None):
    self.df = data_frame
    self.mysql = mysql
    self.gn = group_names
    self.gcn = group_column_name
    self.fvn = final_variables_number
    self.swp = start_with_pca
    self.tss = training_set_size
    self.coefficients = False
    self.prepare_df()

  def load_test(self):
    self.coefficients = self.generate_first_coefficients(self.variables)
    self.mysql = MySQLdb.connect(user = self.DB_SETTINGS['user'], db = self.DB_SETTINGS['name'], host = self.DB_SETTINGS['host'])
    self.df = pd.read_sql_query("SELECT * FROM tracks", con = self.mysql, index_col = ["spotify_id"]).drop("created_at", 1).drop("training", 1).drop("id", 1)
    self.gn = ["sad","happy","energy"]
    self.gcn = "mood"
    self.fvn = 4
    self.swp = False
    self.tss = 0.2
    self.prepare_df()
    self.seed = 8
    self.num_folds = 10
    self.scoring = 'accuracy'
    return self

  def generate_first_coefficients(self, variables):
    coefficients = {}
    for variable in variables:
      coefficient = np.random.choice([1,-1]) * np.random.random()
      coefficients[variable] = coefficient
    return coefficients

  def prepare_df(self):
    if self.df is None:
      self.variables = []
      self.training_df = None
    else:
      self.variables = self.df.drop([self.gcn], axis=1).columns
      dropped_amount = int(self.tss * self.df.shape[0])
      dropped_indices = np.random.choice(self.df.index, dropped_amount, replace=False)
      self.testing_df = self.df[self.df.index.isin(dropped_indices)]
      self.training_df = self.df.drop(dropped_indices)
      self.dummy_testing_df = pd.get_dummies(self.testing_df)
      self.dummy_training_df = pd.get_dummies(self.training_df)

## PLOTTING METHODS

  def draw_corr_matrix(self):
    plt.imshow(self.dummy_df.corr(), cmap='hot', interpolation='nearest')
    plt.savefig('./plots/cor_matrix.png', bbox_inches='tight')
    plt.close()

  def draw_scatter_matrix(self):
    pd.tools.plotting.scatter_matrix(self.df)
    plt.savefig('./plots/scatter_matrix.png', bbox_inches='tight')
    plt.close()

## TUTORIAL METHODS

  def evaluate_models(self):
    array = self.df.values
    X = array[:,1:10]
    Y = array[:,0]
    self.X_train, self.X_validation, self.Y_train, self.Y_validation = model_selection.train_test_split(X, Y, test_size=self.tss, random_state=self.seed)
    num_instances = len(self.X_train)
    self.models = []
    self.models.append(('LR', LogisticRegression()))
    self.models.append(('LDA', LinearDiscriminantAnalysis()))
    self.models.append(('KNN', KNeighborsClassifier()))
    self.models.append(('CART', DecisionTreeClassifier()))
    self.models.append(('NB', GaussianNB()))
    self.models.append(('SVM', SVC()))
    # evaluate each model in turn
    self.results = []
    self.names = []
    for name, model in self.models:
      kfold = model_selection.KFold(n_splits=self.num_folds, random_state=self.seed)
      cv_results = model_selection.cross_val_score(model, self.X_train, self.Y_train, cv=kfold, scoring=self.scoring)
      self.results.append(cv_results)
      self.names.append(name)
      msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std())
      print(msg)

  def simply_the_best(self):
    lda = LinearDiscriminantAnalysis()
    lda.fit(self.X_train, self.Y_train)
    self.predictions = lda.predict(self.X_validation)
    print(accuracy_score(self.Y_validation, self.predictions))
    print(confusion_matrix(self.Y_validation, self.predictions))
    print(classification_report(self.Y_validation, self.predictions))

test = MachineLearning().load_test()
<|endoftext|>


<commit_before>#!/usr/bin/python

import atexit
import os
import json
import urllib
import sys
import collections
import dateutil.parser
import requests
import subprocess
import httplib
import sqlite3
import datetime
import scraperwiki
import httplib
import random
import codecs

from secrets import *

logf = open(os.path.expanduser("~/all.log"), 'a', buffering=1)

def log(message):
    """
    Write message to the log file with a timestamp and a pid and
    a final newline.
    """

    timestamp = datetime.datetime.now().isoformat()
    pid = os.getpid()

    logf.write("{} {} {}\n".format(timestamp, pid, message))

def on_exit():
    log("exiting (via atexit)")

atexit.register(on_exit)

log("started with arguments: {!r}".format(sys.argv))
log("started with environ: ONETIME={!r}, MODE={!r}".format(
  os.environ.get("ONETIME"),
  os.environ.get("MODE")))

# Horrendous hack to work around some Twitter / Python incompatibility
# http://bobrochel.blogspot.co.nz/2010/11/bad-servers-chunked-encoding-and.html
def patch_http_response_read(func):
    def inner(*args):
        try:
            return func(*args)
        except httplib.IncompleteRead, e:
            return e.partial

    return inner
httplib.HTTPResponse.read = patch_http_response_read(httplib.HTTPResponse.read)

# Make sure you install this version of "twitter":
# http://pypi.python.org/pypi/twitter
# http://mike.verdone.ca/twitter/
# https://github.com/sixohsix/twitter
import twitter

#########################################################################
# Authentication to Twitter

# This is designed to, when good, be submitted as a patch to add to twitter.oauth_dance (which
# currently only has a function for PIN authentication, not redirect)
from twitter.api import Twitter
from twitter.oauth import OAuth, write_token_file, read_token_file
from twitter.oauth_dance import parse_oauth_tokens
def oauth_url_dance(consumer_key, consumer_secret, callback_url, oauth_verifier, pre_verify_token_filename, verified_token_filename):
    # Verification happens in two stages...

    # 1) If we haven't done a pre-verification yet... Then we get credentials from Twitter
    # that will be used to sign our redirect to them, find the redirect, and instruct the Javascript
    # that called us to do the redirect.
    if not os.path.exists(CREDS_PRE_VERIFIY):
        twitter = Twitter(auth=OAuth('', '', consumer_key, consumer_secret), format='', api_version=None)
        oauth_token, oauth_token_secret = parse_oauth_tokens(twitter.oauth.request_token(oauth_callback = callback_url))
        write_token_file(pre_verify_token_filename, oauth_token, oauth_token_secret)

        oauth_url = 'https://api.twitter.com/oauth/authorize?' + urllib.urlencode({ 'oauth_token': oauth_token })
        return oauth_url

    # 2) We've done pre-verification, hopefully the user has authed us in Twitter
    # and we've been redirected to. Check we are and ask for the permanent tokens.
    oauth_token, oauth_token_secret = read_token_file(CREDS_PRE_VERIFIY)
    twitter = Twitter(auth=OAuth( oauth_token, oauth_token_secret, consumer_key, consumer_secret), format='', api_version=None)
    oauth_token, oauth_token_secret = parse_oauth_tokens(twitter.oauth.access_token(oauth_verifier=oauth_verifier))
    write_token_file(verified_token_filename, oauth_token, oauth_token_secret)
    return oauth_token, oauth_token_secret

def do_tool_oauth():
    if not os.path.exists(CREDS_VERIFIED):
        if len(sys.argv) < 3:
            result = "need-oauth"
        else:
            (callback_url, oauth_verifier) = (sys.argv[1], sys.argv[2])
            result = oauth_url_dance(CONSUMER_KEY, CONSUMER_SECRET, callback_url, oauth_verifier, CREDS_PRE_VERIFIY, CREDS_VERIFIED)
        # a string means a URL for a redirect (otherwise we get a tuple back with auth tokens in)
        if type(result) == str:
            set_status_and_exit('auth-redirect', 'error', 'Permission needed from Twitter', { 'url': result } )

    oauth_token, oauth_token_secret = read_token_file(CREDS_VERIFIED)
    tw = twitter.Twitter(auth=twitter.OAuth( oauth_token, oauth_token_secret, CONSUMER_KEY, CONSUMER_SECRET))
    return tw

# Afer detecting an auth failed error mid work, call this
def clear_auth_and_restart():
    # remove auth files and respawn
    try:
        os.remove(CREDS_PRE_VERIFIY)
        os.remove(CREDS_VERIFIED)
    except OSError:
        # don't worry if the files aren't there
        pass
    subprocess.call(sys.argv)
    sys.exit()

#########################################################################
# Helper functions

# Signal back to the calling Javascript, to the database, and custard's status API, our status
def set_status_and_exit(status, typ, message, extra = {}):
    global mode

    extra['status'] = status
    print json.dumps(extra)

    requests.post("https://scraperwiki.com/api/status", data={'type':typ, 'message':message})

    data = { 'id': 'tweets', 'mode': mode, 'current_status': status }
    scraperwiki.sql.save(['id'], data, table_name='__status')

    log("{} mode={!r}, status={!r}, type={!r}, message={!r}".format(
      "set_status_and_exit", mode, status, typ, message))
    sys.exit()

def process_results(results, query_terms):
    datas = []
    for tweet in results['statuses']:
        data = collections.OrderedDict()

        data['id_str'] = str(tweet['id_str'])
        data['tweet_url'] = "https://twitter.com/" + tweet['user']['screen_name'] + "/status/" + str(tweet['id_str'])
        data['created_at'] = dateutil.parser.parse(tweet['created_at'])

        data['text'] = tweet['text']
        data['lang'] = tweet['lang']

        data['retweet_count'] = tweet['retweet_count']
        data['favorite_count'] = tweet['favorite_count']
        # conversation thread length?

        data['screen_name'] = tweet['user']['screen_name']
        data['in_reply_to_screen_name'] = tweet['in_reply_to_screen_name']
        data['in_reply_to_status_id'] = tweet['in_reply_to_status_id']

        try:
            data['lat'] = tweet['geo']['coordinates'][0]
            data['lng'] = tweet['geo']['coordinates'][1]
        except:
            pass

        entities = tweet.get('entities', {})

        urls = entities.get('urls')
        data['url'] = urls[0].get(u'expanded_url') if urls else None

        media = entities.get('media')
        data['media'] = media[0].get(u'media_url_https', '') if media else None

        users = entities.get('user_mentions')
        data['user_mention' ] = users[0].get(u'screen_name','') if users else None

        hashtags = entities.get('hashtags')
        data['hashtags'] = hashtags[0].get(u'text','') if hashtags else None

        data['query'] = query_terms

        datas.append(data)

    if datas:
        min_id = min(x['id_str'] for x in datas)
        max_id = max(x['id_str'] for x in datas)
        log("about to save; min_id {}, max_id {}".format(min_id, max_id))
    else:
        log("no datas")
    scraperwiki.sql.save(['id_str'], datas, table_name="tweets")
    return len(results['statuses'])

#########################################################################
# Main code

pages_got = 0
onetime = 'ONETIME' in os.environ

if 'MODE' in os.environ:
    mode = os.environ['MODE']
else:
    try:
        mode = scraperwiki.sql.select('mode from __status')[0]['mode']
    except sqlite3.OperationalError:
        # happens when '__status' table doesn't exist
        mode = 'clearing-backlog'
log("mode = {!r}".format(mode))

try:
    # Parameters to this command vary:
    #   a. None: try and scrape Twitter followers
    #   b. callback_url oauth_verifier: have just come back from Twitter with these oauth tokens
    #   c. "clean-slate": wipe database and start again
    if len(sys.argv) > 1 and sys.argv[1] == 'clean-slate':
        scraperwiki.sql.execute("drop table if exists tweets")
        scraperwiki.sql.execute("drop table if exists __status")
        os.system("crontab -r >/dev/null 2>&1")
        scraperwiki.sql.dt.create_table({'id_str': '1'}, 'tweets')
        mode = 'clearing-backlog'
        set_status_and_exit('clean-slate', 'error', 'No query set')
        sys.exit()

    # Make the tweets table *first* with dumb data, calling DumpTruck directly,
    # so it appears before the status one in the list
    scraperwiki.sql.dt.create_table({'id_str': '1'}, 'tweets')

    # Get query we're working on from file we store it in
    query_terms = codecs.open("query.txt", "r", "utf-8").read().strip()

    # Connect to Twitter
    tw = do_tool_oauth()

    # Called for diagnostic information only
    if len(sys.argv) > 1 and sys.argv[1] == 'diagnostics':
        diagnostics = {}
        diagnostics['_rate_limit_status'] = tw.application.rate_limit_status()
        diagnostics['limit'] = diagnostics['_rate_limit_status']['resources']['search']['/search/tweets']['limit']
        diagnostics['remaining'] = diagnostics['_rate_limit_status']['resources']['search']['/search/tweets']['remaining']
        diagnostics['reset'] = diagnostics['_rate_limit_status']['resources']['search']['/search/tweets']['reset']
        diagnostics['_account_settings'] = tw.account.settings()
        diagnostics['user'] = diagnostics['_account_settings']['screen_name']

        statuses = scraperwiki.sql.select('* from __status')[0]
        diagnostics['mode'] = statuses['mode']
        diagnostics['status'] = statuses['current_status']

        crontab = subprocess.check_output("crontab -l | grep twsearch.py; true", stderr=subprocess.STDOUT, shell=True)
        diagnostics['crontab'] = crontab

        print json.dumps(diagnostics)
        sys.exit()

    # Mode changes
    assert mode in ['clearing-backlog', 'backlog-cleared', 'monitoring'] # should never happen
    if mode == 'backlog-cleared':
        # we shouldn't run, because we've cleared the backlog already
        set_status_and_exit("ok-updating", 'ok', '')
    else:
        if 'MODE' in os.environ or not os.path.isfile("crontab"):
            # frontend has defined a new mode, so we should make a new crontab
            crontab = open("tool/crontab.template").read()
            # run at a random minute to distribute load (platform should really do this for us!)
            crontab = crontab.replace("RANDOM", str(random.randint(0, 59)))
            open("crontab", "w").write(crontab)
        # implement whatever crontab has been written to the crontab text file
        # (this may or may not be different to the existing crontab)
        os.system("crontab crontab")

    # Get tweets older than what we've already got. There are two slightly
    # subtle features of this code: bootstrapping, and "got > 1" (the loop
    # termination condition).
    #
    # Bootstrap: at the very beginning when there are no tweets (no rows in
    # tweets table) this code still works and gets the first batch of tweets.
    # This is because of some rather subtle behaviour of SQL and scraperwiki.sql:
    # When the table is empty the query "min(id_str) from tweets" will return
    # a row with None as the value associated with the key "min(id_str)", so
    # min_id will be set to None, and tw.search.tweets Does The Right Thing when
    # None is used as the max_id parameter.
    #
    # Loop termination: Note that we search with max_id set to the id of some
    # tweet that we have already saved, which means we'll get that tweet in our
    # results, which means that we only have _new_ tweets if the number that we
    # got is bigger than 1.

    got = 2
    while got > 1:
        min_id = scraperwiki.sql.select("min(id_str) from tweets")[0]["min(id_str)"]
        log("min_id {}".format(min_id))
        results = tw.search.tweets(q=query_terms, max_id = min_id)
        got = process_results(results, query_terms)
        log("got {}".format(got))
        pages_got += 1
        if onetime:
            break

    # we've reached as far back as we'll ever get, so we're done forever
    if not onetime and mode == 'clearing-backlog':
        mode = 'backlog-cleared'
        os.system("crontab -r >/dev/null 2>&1")
        set_status_and_exit("ok-updating", 'ok', '')

    # Get tweets more recent than what we've already got
    if mode == 'monitoring':
        got = 2
        while got > 1:
            max_id = scraperwiki.sql.select("max(id_str) from tweets")[0]["max(id_str)"]
            log("max_id {}".format(max_id))
            results = tw.search.tweets(q=query_terms, since_id = max_id)
            got = process_results(results, query_terms)
            log("got {}".format(got))
            pages_got += 1
            if onetime:
                break

except twitter.api.TwitterHTTPError, e:
    if "Twitter sent status 401 for URL" in str(e):
        clear_auth_and_restart()

    # https://dev.twitter.com/docs/error-codes-responses
    obj = json.loads(e.response_data)
    code = obj['errors'][0]['code']
    # authentication failure
    if (code in [32, 89]):
        clear_auth_and_restart()
    # rate limit exceeded
    if code == 34:
        set_status_and_exit('not-there', 'error', 'User not on Twitter')
    if code == 195:
        set_status_and_exit('invalid-query', 'error', "That isn't a valid Twitter search")
    if code == 44:
        set_status_and_exit('near-not-supported', 'error', "Twitter's API doesn't support NEAR")
    if code == 88:
        # provided we got at least one page, rate limit isn't an error but expected
        if pages_got == 0:
            set_status_and_exit('rate-limit', 'error', 'Twitter has asked us to slow down')
    else:
        # anything else is an unexpected error - if ones occur a lot, add the above instead
        raise
except httplib.IncompleteRead, e:
    # I think this is effectively a rate limit error - so only count if it was first error
    if pages_got == 0:
        set_status_and_exit('rate-limit', 'error', 'Twitter broke the connection')

# Save progress message
set_status_and_exit("ok-updating", 'ok', '')

<commit_msg>Explicitly set result_type to the apparent default now, 'recent'<commit_after>#!/usr/bin/python

import atexit
import os
import json
import urllib
import sys
import collections
import dateutil.parser
import requests
import subprocess
import httplib
import sqlite3
import datetime
import scraperwiki
import httplib
import random
import codecs

from secrets import *

logf = open(os.path.expanduser("~/all.log"), 'a', buffering=1)

def log(message):
    """
    Write message to the log file with a timestamp and a pid and
    a final newline.
    """

    timestamp = datetime.datetime.now().isoformat()
    pid = os.getpid()

    logf.write("{} {} {}\n".format(timestamp, pid, message))

def on_exit():
    log("exiting (via atexit)")

atexit.register(on_exit)

log("started with arguments: {!r}".format(sys.argv))
log("started with environ: ONETIME={!r}, MODE={!r}".format(
  os.environ.get("ONETIME"),
  os.environ.get("MODE")))

# Horrendous hack to work around some Twitter / Python incompatibility
# http://bobrochel.blogspot.co.nz/2010/11/bad-servers-chunked-encoding-and.html
def patch_http_response_read(func):
    def inner(*args):
        try:
            return func(*args)
        except httplib.IncompleteRead, e:
            return e.partial

    return inner
httplib.HTTPResponse.read = patch_http_response_read(httplib.HTTPResponse.read)

# Make sure you install this version of "twitter":
# http://pypi.python.org/pypi/twitter
# http://mike.verdone.ca/twitter/
# https://github.com/sixohsix/twitter
import twitter

#########################################################################
# Authentication to Twitter

# This is designed to, when good, be submitted as a patch to add to twitter.oauth_dance (which
# currently only has a function for PIN authentication, not redirect)
from twitter.api import Twitter
from twitter.oauth import OAuth, write_token_file, read_token_file
from twitter.oauth_dance import parse_oauth_tokens
def oauth_url_dance(consumer_key, consumer_secret, callback_url, oauth_verifier, pre_verify_token_filename, verified_token_filename):
    # Verification happens in two stages...

    # 1) If we haven't done a pre-verification yet... Then we get credentials from Twitter
    # that will be used to sign our redirect to them, find the redirect, and instruct the Javascript
    # that called us to do the redirect.
    if not os.path.exists(CREDS_PRE_VERIFIY):
        twitter = Twitter(auth=OAuth('', '', consumer_key, consumer_secret), format='', api_version=None)
        oauth_token, oauth_token_secret = parse_oauth_tokens(twitter.oauth.request_token(oauth_callback = callback_url))
        write_token_file(pre_verify_token_filename, oauth_token, oauth_token_secret)

        oauth_url = 'https://api.twitter.com/oauth/authorize?' + urllib.urlencode({ 'oauth_token': oauth_token })
        return oauth_url

    # 2) We've done pre-verification, hopefully the user has authed us in Twitter
    # and we've been redirected to. Check we are and ask for the permanent tokens.
    oauth_token, oauth_token_secret = read_token_file(CREDS_PRE_VERIFIY)
    twitter = Twitter(auth=OAuth( oauth_token, oauth_token_secret, consumer_key, consumer_secret), format='', api_version=None)
    oauth_token, oauth_token_secret = parse_oauth_tokens(twitter.oauth.access_token(oauth_verifier=oauth_verifier))
    write_token_file(verified_token_filename, oauth_token, oauth_token_secret)
    return oauth_token, oauth_token_secret

def do_tool_oauth():
    if not os.path.exists(CREDS_VERIFIED):
        if len(sys.argv) < 3:
            result = "need-oauth"
        else:
            (callback_url, oauth_verifier) = (sys.argv[1], sys.argv[2])
            result = oauth_url_dance(CONSUMER_KEY, CONSUMER_SECRET, callback_url, oauth_verifier, CREDS_PRE_VERIFIY, CREDS_VERIFIED)
        # a string means a URL for a redirect (otherwise we get a tuple back with auth tokens in)
        if type(result) == str:
            set_status_and_exit('auth-redirect', 'error', 'Permission needed from Twitter', { 'url': result } )

    oauth_token, oauth_token_secret = read_token_file(CREDS_VERIFIED)
    tw = twitter.Twitter(auth=twitter.OAuth( oauth_token, oauth_token_secret, CONSUMER_KEY, CONSUMER_SECRET))
    return tw

# Afer detecting an auth failed error mid work, call this
def clear_auth_and_restart():
    # remove auth files and respawn
    try:
        os.remove(CREDS_PRE_VERIFIY)
        os.remove(CREDS_VERIFIED)
    except OSError:
        # don't worry if the files aren't there
        pass
    subprocess.call(sys.argv)
    sys.exit()

#########################################################################
# Helper functions

# Signal back to the calling Javascript, to the database, and custard's status API, our status
def set_status_and_exit(status, typ, message, extra = {}):
    global mode

    extra['status'] = status
    print json.dumps(extra)

    requests.post("https://scraperwiki.com/api/status", data={'type':typ, 'message':message})

    data = { 'id': 'tweets', 'mode': mode, 'current_status': status }
    scraperwiki.sql.save(['id'], data, table_name='__status')

    log("{} mode={!r}, status={!r}, type={!r}, message={!r}".format(
      "set_status_and_exit", mode, status, typ, message))
    sys.exit()

def process_results(results, query_terms):
    datas = []
    for tweet in results['statuses']:
        data = collections.OrderedDict()

        data['id_str'] = str(tweet['id_str'])
        data['tweet_url'] = "https://twitter.com/" + tweet['user']['screen_name'] + "/status/" + str(tweet['id_str'])
        data['created_at'] = dateutil.parser.parse(tweet['created_at'])

        data['text'] = tweet['text']
        data['lang'] = tweet['lang']

        data['retweet_count'] = tweet['retweet_count']
        data['favorite_count'] = tweet['favorite_count']
        # conversation thread length?

        data['screen_name'] = tweet['user']['screen_name']
        data['in_reply_to_screen_name'] = tweet['in_reply_to_screen_name']
        data['in_reply_to_status_id'] = tweet['in_reply_to_status_id']

        try:
            data['lat'] = tweet['geo']['coordinates'][0]
            data['lng'] = tweet['geo']['coordinates'][1]
        except:
            pass

        entities = tweet.get('entities', {})

        urls = entities.get('urls')
        data['url'] = urls[0].get(u'expanded_url') if urls else None

        media = entities.get('media')
        data['media'] = media[0].get(u'media_url_https', '') if media else None

        users = entities.get('user_mentions')
        data['user_mention' ] = users[0].get(u'screen_name','') if users else None

        hashtags = entities.get('hashtags')
        data['hashtags'] = hashtags[0].get(u'text','') if hashtags else None

        data['query'] = query_terms

        datas.append(data)

    if datas:
        min_id = min(x['id_str'] for x in datas)
        max_id = max(x['id_str'] for x in datas)
        log("about to save; min_id {}, max_id {}".format(min_id, max_id))
    else:
        log("no datas")
    scraperwiki.sql.save(['id_str'], datas, table_name="tweets")
    return len(results['statuses'])

#########################################################################
# Main code

pages_got = 0
onetime = 'ONETIME' in os.environ

if 'MODE' in os.environ:
    mode = os.environ['MODE']
else:
    try:
        mode = scraperwiki.sql.select('mode from __status')[0]['mode']
    except sqlite3.OperationalError:
        # happens when '__status' table doesn't exist
        mode = 'clearing-backlog'
log("mode = {!r}".format(mode))

try:
    # Parameters to this command vary:
    #   a. None: try and scrape Twitter followers
    #   b. callback_url oauth_verifier: have just come back from Twitter with these oauth tokens
    #   c. "clean-slate": wipe database and start again
    if len(sys.argv) > 1 and sys.argv[1] == 'clean-slate':
        scraperwiki.sql.execute("drop table if exists tweets")
        scraperwiki.sql.execute("drop table if exists __status")
        os.system("crontab -r >/dev/null 2>&1")
        scraperwiki.sql.dt.create_table({'id_str': '1'}, 'tweets')
        mode = 'clearing-backlog'
        set_status_and_exit('clean-slate', 'error', 'No query set')
        sys.exit()

    # Make the tweets table *first* with dumb data, calling DumpTruck directly,
    # so it appears before the status one in the list
    scraperwiki.sql.dt.create_table({'id_str': '1'}, 'tweets')

    # Get query we're working on from file we store it in
    query_terms = codecs.open("query.txt", "r", "utf-8").read().strip()

    # Connect to Twitter
    tw = do_tool_oauth()

    # Called for diagnostic information only
    if len(sys.argv) > 1 and sys.argv[1] == 'diagnostics':
        diagnostics = {}
        diagnostics['_rate_limit_status'] = tw.application.rate_limit_status()
        diagnostics['limit'] = diagnostics['_rate_limit_status']['resources']['search']['/search/tweets']['limit']
        diagnostics['remaining'] = diagnostics['_rate_limit_status']['resources']['search']['/search/tweets']['remaining']
        diagnostics['reset'] = diagnostics['_rate_limit_status']['resources']['search']['/search/tweets']['reset']
        diagnostics['_account_settings'] = tw.account.settings()
        diagnostics['user'] = diagnostics['_account_settings']['screen_name']

        statuses = scraperwiki.sql.select('* from __status')[0]
        diagnostics['mode'] = statuses['mode']
        diagnostics['status'] = statuses['current_status']

        crontab = subprocess.check_output("crontab -l | grep twsearch.py; true", stderr=subprocess.STDOUT, shell=True)
        diagnostics['crontab'] = crontab

        print json.dumps(diagnostics)
        sys.exit()

    # Mode changes
    assert mode in ['clearing-backlog', 'backlog-cleared', 'monitoring'] # should never happen
    if mode == 'backlog-cleared':
        # we shouldn't run, because we've cleared the backlog already
        set_status_and_exit("ok-updating", 'ok', '')
    else:
        if 'MODE' in os.environ or not os.path.isfile("crontab"):
            # frontend has defined a new mode, so we should make a new crontab
            crontab = open("tool/crontab.template").read()
            # run at a random minute to distribute load (platform should really do this for us!)
            crontab = crontab.replace("RANDOM", str(random.randint(0, 59)))
            open("crontab", "w").write(crontab)
        # implement whatever crontab has been written to the crontab text file
        # (this may or may not be different to the existing crontab)
        os.system("crontab crontab")

    # Get tweets older than what we've already got. There are two slightly
    # subtle features of this code: bootstrapping, and "got > 1" (the loop
    # termination condition).
    #
    # Bootstrap: at the very beginning when there are no tweets (no rows in
    # tweets table) this code still works and gets the first batch of tweets.
    # This is because of some rather subtle behaviour of SQL and scraperwiki.sql:
    # When the table is empty the query "min(id_str) from tweets" will return
    # a row with None as the value associated with the key "min(id_str)", so
    # min_id will be set to None, and tw.search.tweets Does The Right Thing when
    # None is used as the max_id parameter.
    #
    # Loop termination: Note that we search with max_id set to the id of some
    # tweet that we have already saved, which means we'll get that tweet in our
    # results, which means that we only have _new_ tweets if the number that we
    # got is bigger than 1.

    got = 2
    while got > 1:
        min_id = scraperwiki.sql.select("min(id_str) from tweets")[0]["min(id_str)"]
        log("min_id {}".format(min_id))
        results = tw.search.tweets(q=query_terms, result_type = 'recent', max_id = min_id)
        got = process_results(results, query_terms)
        log("got {}".format(got))
        pages_got += 1
        if onetime:
            break

    # we've reached as far back as we'll ever get, so we're done forever
    if not onetime and mode == 'clearing-backlog':
        mode = 'backlog-cleared'
        os.system("crontab -r >/dev/null 2>&1")
        set_status_and_exit("ok-updating", 'ok', '')

    # Get tweets more recent than what we've already got
    if mode == 'monitoring':
        got = 2
        while got > 1:
            max_id = scraperwiki.sql.select("max(id_str) from tweets")[0]["max(id_str)"]
            log("max_id {}".format(max_id))
            results = tw.search.tweets(q=query_terms, result_type = 'recent', since_id = max_id)
            got = process_results(results, query_terms)
            log("got {}".format(got))
            pages_got += 1
            if onetime:
                break

except twitter.api.TwitterHTTPError, e:
    if "Twitter sent status 401 for URL" in str(e):
        clear_auth_and_restart()

    # https://dev.twitter.com/docs/error-codes-responses
    obj = json.loads(e.response_data)
    code = obj['errors'][0]['code']
    # authentication failure
    if (code in [32, 89]):
        clear_auth_and_restart()
    # rate limit exceeded
    if code == 34:
        set_status_and_exit('not-there', 'error', 'User not on Twitter')
    if code == 195:
        set_status_and_exit('invalid-query', 'error', "That isn't a valid Twitter search")
    if code == 44:
        set_status_and_exit('near-not-supported', 'error', "Twitter's API doesn't support NEAR")
    if code == 88:
        # provided we got at least one page, rate limit isn't an error but expected
        if pages_got == 0:
            set_status_and_exit('rate-limit', 'error', 'Twitter has asked us to slow down')
    else:
        # anything else is an unexpected error - if ones occur a lot, add the above instead
        raise
except httplib.IncompleteRead, e:
    # I think this is effectively a rate limit error - so only count if it was first error
    if pages_got == 0:
        set_status_and_exit('rate-limit', 'error', 'Twitter broke the connection')

# Save progress message
set_status_and_exit("ok-updating", 'ok', '')

<|endoftext|>


<commit_before>from twisted.python.failure import Failure
from twisted.internet.defer import maybeDeferred, Deferred, DeferredList
from twisted.internet.defer import succeed, fail

from .interfaces import IFailedPromise

from twisted.internet.defer import inlineCallbacks as future_generator
from twisted.internet.defer import returnValue  # XXX how to do in asyncio?

class FailedPromise(IFailedPromise):
    """
    XXX provide an abstract-base-class for IFailedPromise or similar,
    probably. For consistency between asyncio/twisted.

    ...i.e. to be explicit about what methods and variables may be
    used. Currently:

    .type
    .value
    .traceback

    .printTraceback
    """
    pass

FailedPromise.register(Failure)

# API methods for txaio

def create_future():
    return Deferred()

def create_future_success(result):
    return succeed(result)

def create_future_error(error=None):
    if error is None:
        error = create_failure()
    else:
        assert isinstance(error, Failure)
    return fail(error)

def as_future(fun, *args, **kwargs):
    return maybeDeferred(fun, *args, **kwargs)

def resolve_future(future, result=None):
    future.callback(result)

def reject_future(future, error=None):
    if error is None:
        error = create_failure()
    elif isinstance(error, Exception):
        print("FIXME: passing Exception to reject_future")
        error = Failure(error)
    else:
        assert isinstance(error, IFailedPromise)
    future.errback(error)

def create_failure(exception=None):
    """
    Create a Failure instance.

    if ``exception`` is None (the default), we MUST be inside an
    "except" block. This encapsulates the exception into an object
    that implements IFailedPromise
    """
    if exception:
        return Failure(exception)
    return Failure()

def add_future_callbacks(future, callback, errback):
    """
    callback or errback may be None, but at least one must be
    non-None.
    """
    assert future is not None
    if callback is None:
        assert errback is not None
        future.addErrback(errback)
    else:
        # Twisted allows errback to be None here
        future.addCallbacks(callback, errback)
    return future

def gather_futures(futures, consume_exceptions=True,
                   first_result=False, first_exception=False):
    def completed(res):
        if first_result:
            return res[0]  # XXX really? what if third one fired first?
        rtn = []
        for (ok, value) in res:
            rtn.append(value)
            if not ok and not consume_exceptions:
                value.raiseException()
        return rtn

    def failed(f):
        # This only gets called if fireOnOneErrback=True *and*
        # consume_exceptions=Ture and in that case, the "failure"
        # always contains a FirstError
        assert not consume_exceptions
        assert first_exception
        return f.value.subFailure

    dl = DeferredList(
        list(futures),
        consumeErrors=consume_exceptions,
        fireOnOneCallback=first_result,
        fireOnOneErrback=first_exception
    )
    add_future_callbacks(dl, completed, failed)
    return dl
<commit_msg>nicer error message<commit_after>from twisted.python.failure import Failure
from twisted.internet.defer import maybeDeferred, Deferred, DeferredList
from twisted.internet.defer import succeed, fail

from .interfaces import IFailedPromise

from twisted.internet.defer import inlineCallbacks as future_generator
from twisted.internet.defer import returnValue  # XXX how to do in asyncio?

class FailedPromise(IFailedPromise):
    """
    XXX provide an abstract-base-class for IFailedPromise or similar,
    probably. For consistency between asyncio/twisted.

    ...i.e. to be explicit about what methods and variables may be
    used. Currently:

    .type
    .value
    .traceback

    .printTraceback
    """
    pass

FailedPromise.register(Failure)

# API methods for txaio

def create_future():
    return Deferred()

def create_future_success(result):
    return succeed(result)

def create_future_error(error=None):
    if error is None:
        error = create_failure()
    else:
        assert isinstance(error, Failure)
    return fail(error)

def as_future(fun, *args, **kwargs):
    return maybeDeferred(fun, *args, **kwargs)

def resolve_future(future, result=None):
    future.callback(result)

def reject_future(future, error=None):
    if error is None:
        error = create_failure()
    elif isinstance(error, Exception):
        print("FIXME: passing Exception to reject_future")
        error = Failure(error)
    else:
        if not isinstance(error, IFailedPromise):
            raise RuntimeError("reject_future requires an IFailedPromise")
    future.errback(error)

def create_failure(exception=None):
    """
    Create a Failure instance.

    if ``exception`` is None (the default), we MUST be inside an
    "except" block. This encapsulates the exception into an object
    that implements IFailedPromise
    """
    if exception:
        return Failure(exception)
    return Failure()

def add_future_callbacks(future, callback, errback):
    """
    callback or errback may be None, but at least one must be
    non-None.
    """
    assert future is not None
    if callback is None:
        assert errback is not None
        future.addErrback(errback)
    else:
        # Twisted allows errback to be None here
        future.addCallbacks(callback, errback)
    return future

def gather_futures(futures, consume_exceptions=True,
                   first_result=False, first_exception=False):
    def completed(res):
        if first_result:
            return res[0]  # XXX really? what if third one fired first?
        rtn = []
        for (ok, value) in res:
            rtn.append(value)
            if not ok and not consume_exceptions:
                value.raiseException()
        return rtn

    def failed(f):
        # This only gets called if fireOnOneErrback=True *and*
        # consume_exceptions=Ture and in that case, the "failure"
        # always contains a FirstError
        assert not consume_exceptions
        assert first_exception
        return f.value.subFailure

    dl = DeferredList(
        list(futures),
        consumeErrors=consume_exceptions,
        fireOnOneCallback=first_result,
        fireOnOneErrback=first_exception
    )
    add_future_callbacks(dl, completed, failed)
    return dl
<|endoftext|>


<commit_before>header_tmpl = """<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
 <title>$$TITLE/$</title>
    <link href="{{page.node.full_url}}/atom.xml" rel="self"/>
    <link href="{{site.full_url}}"/>
 <updated>{{now|xmldatetime}}</updated>
 <id>{{site.full_url}}</id>
"""

post_tmpl = """
        <entry>
        <title>{{node_page.title}}</title>
        <author><name>{{site.author}}</name></author>
        <link href="{{node_page.full_url}}"/>
        <updated>{{node_page.updated|default:node_page.created|xmldatetime}}</updated>
        <published>{{node_page.created|xmldatetime}}</published>
        <id>{{node_page.full_url}}</id>
        {%block entry_extra%}{%endblock%}
        <content type="html">
            {%filter force_escape%}{% render_article node_page %}{%endfilter%}
        </content>
        </entry>
"""

footer_tmpl = """
</feed>"""

def generate(load_path, offset, output_path):
    # Scan for number of items
    import os, fnmatch
    all_files = []
    files = os.listdir(load_path)
    for f in fnmatch.filter(files, '*.html'):
        all_files.append(f)

    final = header_tmpl
    module_count = len(all_files)
    curr = offset
    while curr >= 0 and curr > offset - 10:
        chunk_id = curr
        fname = "section_%d.html" % (chunk_id + 1)
        fh = open(os.path.join(load_path, fname))

        section = ''.join(fh.readlines())
        fh.close()
        final += section
        curr -= 1

    final += footer_tmpl

    fh = open(output_path, 'w')
    fh.write(final)
    fh.close()

if __name__ == '__main__':
    # Generate the last 10 posts
    generate('12night', 2, 'atom.xml')

<commit_msg>Simple templating. First "working" XML output.<commit_after>header_tmpl = """<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
 <title>The Daily Bard</title>
 <id>{{site.full_url}}</id>
"""

post_tmpl = """
        <entry>
        <title>{{title}}</title>
        <author><name>{{site.author}}</name></author>
        <link href="{{full_url}}"/>
        <published>{{created|xmldatetime}}</published>
        <updated>{{created|xmldatetime}}</updated>
        <id>{{full_url}}</id>
        <content type="html">
<![CDATA[
{{content}}
]]>
        </content>
        </entry>
"""

footer_tmpl = """
</feed>"""

def expand_template(tmpl_text, values):
    import re
    search_re = re.compile("{{(.*)}}")
    def replace_func(matchobj):
        key = matchobj.group(1)
        if values.has_key(key):
            return values[key]
        return "MISSING"
    output = re.sub(search_re, replace_func, tmpl_text)
    return output

def generate(load_path, offset, output_path):
    # Scan for number of items
    import os, fnmatch
    all_files = []
    files = os.listdir(load_path)
    for f in fnmatch.filter(files, '*.html'):
        all_files.append(f)

    final = header_tmpl
    module_count = len(all_files)
    curr = offset
    while curr >= 0 and curr > offset - 10:
        chunk_id = curr
        fname = "section_%d.html" % (chunk_id + 1)

        fh = open(os.path.join(load_path, fname))
        section = '
'.join(fh.readlines())
        fh.close()

        values = { "content" : section,
                   "site.author" : "Shakespeare, William",
                   "title" : "Section %d" % chunk_id,
                   "full_url" : "http://clarets.org",
                   "created|xmldatetime" : "2014-02-15T09:00:00Z"   }
        final += expand_template(post_tmpl, values)
        curr -= 1

    final += footer_tmpl

    fh = open(output_path, 'w')
    fh.write(final)
    fh.close()

if __name__ == '__main__':
    # Generate the last 10 posts
    generate('sections/kinglear', 2, 'atom_kinglear.xml')

<|endoftext|>


<commit_before>"""
Reads input from OpenShift v3
"""
import json
import os

from dock.plugin import InputPlugin

class OSv3InputPlugin(InputPlugin):
    key = "osv3"

    def __init__(self, **kwargs):
        """
        constructor
        """
        # call parent constructor
        super(OSv3InputPlugin, self).__init__(**kwargs)

    def run(self):
        """
        each plugin has to implement this method -- it is used to run the plugin actually

        response from plugin is kept and used in json result response
        """
        build_json_str = os.environ['BUILD']
        build_json = json.loads(build_json_str)
        git_url = os.environ['SOURCE_URI']
        git_ref = os.environ.get('SOURCE_REF', None)
        image = os.environ['OUTPUT_IMAGE']
        target_registry = os.environ.get('OUTPUT_REGISTRY', None)
        plugins_json = os.environ.get('DOCK_PLUGINS', '{}')
        plugins_json = json.loads(plugins_json)

        input_json = {
            'git_url': git_url,
            'image': image,
            'git_commit': git_ref,
            'target_registries': [target_registry] if target_registry is not None else None,
            'prebuild_plugins': plugins_json.get('prebuild_plugins', None),
            'postbuild_plugins': plugins_json.get('postbuild_plugins', None),
            'target_registries_insecure': True,  # FIXME: create plugin for this
        }

        self.log.debug("build json: %s", input_json)

        return input_json
<commit_msg>osv3: load parent registry in input plugin<commit_after>"""
Reads input from OpenShift v3
"""
import json
import os

from dock.plugin import InputPlugin

class OSv3InputPlugin(InputPlugin):
    key = "osv3"

    def __init__(self, **kwargs):
        """
        constructor
        """
        # call parent constructor
        super(OSv3InputPlugin, self).__init__(**kwargs)

    def run(self):
        """
        each plugin has to implement this method -- it is used to run the plugin actually

        response from plugin is kept and used in json result response
        """
        build_json_str = os.environ['BUILD']
        build_json = json.loads(build_json_str)
        git_url = os.environ['SOURCE_URI']
        git_ref = os.environ.get('SOURCE_REF', None)
        image = os.environ['OUTPUT_IMAGE']
        target_registry = os.environ.get('OUTPUT_REGISTRY', None)
        plugins_json = os.environ.get('DOCK_PLUGINS', '{}')
        plugins_json = json.loads(plugins_json)

        source_registry = None
        source_registry_insecure = None
        try:
            match = [x for x in plugins_json['prebuild_plugins'] if x.get('name', None) == 'change_source_registry']
            source_registry = match[0]['args']['registry_uri']
            source_registry_insecure = match[0]['args'].get('insecure_registry', False)
        except (IndexError, KeyError) as ex:
            self.log.error("source registry is not configured: '%s'", repr(ex))

        input_json = {
            'git_url': git_url,
            'image': image,
            'git_commit': git_ref,
            'target_registries': [target_registry] if target_registry is not None else None,
            'prebuild_plugins': plugins_json.get('prebuild_plugins', None),
            'postbuild_plugins': plugins_json.get('postbuild_plugins', None),
            'target_registries_insecure': True,  # FIXME: create plugin for this
            'parent_registry': source_registry,
            'parent_registry_insecure': source_registry_insecure,
        }

        self.log.debug("build json: %s", input_json)

        return input_json
<|endoftext|>