<commit_before>import os
import subprocess as sp
import pytest
from textwrap import dedent
import yaml
import tempfile
import requests
from bioconda_utils import utils
from bioconda_utils import pkg_test
from bioconda_utils import docker_utils
from bioconda_utils import cli
from bioconda_utils import build
from helpers import ensure_missing, Recipes, tmp_env_matrix
from conda_build import api
from conda_build.metadata import MetaData

def test_get_deps():
    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: one
              version: 0.1
        two:
          meta.yaml: |
            package:
              name: two
              version: 0.1
            requirements:
              build:
                - one
        three:
          meta.yaml: |
            package:
              name: three
              version: 0.1
            requirements:
              build:
                - one
              run:
                - two
    """), from_string=True)
    r.write_recipes()
    assert list(utils.get_deps(r.recipe_dirs['two'])) == ['one']
    assert list(utils.get_deps(r.recipe_dirs['three'], build=True)) == ['one']
    assert list(utils.get_deps(r.recipe_dirs['three'], build=False)) == ['two']

def _single_build(docker_builder=None):
    """
    Tests the building of a single configured recipe, with or without docker
    """
    r = Recipes('test_case.yaml')
    r.write_recipes()
    env_matrix = list(utils.EnvMatrix(tmp_env_matrix()))[0]
    conda_bld = docker_utils.get_host_conda_bld()
    built_package = os.path.join(conda_bld, 'linux-64', 'one-0.1-0.tar.bz2')
    ensure_missing(built_package)
    build.build(
        recipe=r.recipe_dirs['one'],
        recipe_folder='.',
        docker_builder=docker_builder,
        env=env_matrix,
    )
    assert os.path.exists(built_package)
    return built_package

def test_single_build1():
    built_package = _single_build(docker_builder=None)

def test_single_build_docker_with_post_test():
    docker_builder = docker_utils.RecipeBuilder(verbose=True)
    r = Recipes('test_case.yaml')
    r.write_recipes()
    env_matrix = list(utils.EnvMatrix(tmp_env_matrix()))[0]
    conda_bld = docker_utils.get_host_conda_bld()
    built_package = os.path.join(conda_bld, 'linux-64', 'one-0.1-0.tar.bz2')
    ensure_missing(built_package)
    build.build(
        recipe=r.recipe_dirs['one'],
        recipe_folder='.',
        docker_builder=docker_builder,
        env=env_matrix)
    assert os.path.exists(built_package)
    pkg_test.test_package(built_package)
    ensure_missing(built_package)

def test_single_build_docker():
    docker_builder = docker_utils.RecipeBuilder(verbose=True)
    built_package = _single_build(docker_builder=docker_builder)
    ensure_missing(built_package)

def test_docker_builder_build():
    """
    Tests just the build_recipe method of a RecipeBuilder object.

    Makes sure the built recipe shows up on the host machine.
    """
    conda_bld = docker_utils.get_host_conda_bld()
    built_package = os.path.join(conda_bld, 'linux-64', 'one-0.1-0.tar.bz2')
    ensure_missing(built_package)
    docker_builder = docker_utils.RecipeBuilder(verbose=True)
    r = Recipes('test_case.yaml')
    r.write_recipes()
    recipe_dir = r.recipe_dirs['one']
    docker_builder.build_recipe(recipe_dir, build_args='')
    assert os.path.exists(built_package)
    os.unlink(built_package)
    assert not os.path.exists(built_package)

def test_docker_build_image_fails():
    template = dedent(
        """
        FROM {self.image}
        RUN nonexistent command
        """)
    with pytest.raises(sp.CalledProcessError):
        docker_builder = docker_utils.RecipeBuilder(
            verbose=True, dockerfile_template=template)

def test_conda_purge_cleans_up():
    def tmp_dir_exists(d):
        contents = os.listdir(d)
        for i in contents:
            if i.startswith('tmp') and '_' in i:
                return True

    bld = docker_utils.get_host_conda_bld(purge=False)
    assert tmp_dir_exists(bld)
    bld = docker_utils.get_host_conda_bld(purge=True)
    assert not tmp_dir_exists(bld)

def test_local_channel():
    r = Recipes('test_case.yaml')
    r.write_recipes()
    with pytest.raises(SystemExit) as e:
        cli.build(r.basedir,
                  config={},
                  packages="*",
                  testonly=False,
                  force=False,
                  docker=True,
                  loglevel="debug",
                  )
        assert e.code == 0
    conda_bld = docker_utils.get_host_conda_bld()
    print(os.listdir(os.path.join(conda_bld, 'linux-64')))

def test_env_matrix():
    contents = {
        'CONDA_PY': [27, 35],
        'CONDA_BOOST': '1.60'
    }

    with open(tempfile.NamedTemporaryFile().name, 'w') as fout:
        fout.write(yaml.dump(contents, default_flow_style=False))

    e1 = utils.EnvMatrix(contents)
    e2 = utils.EnvMatrix(fout.name)
    assert e1.env == e2.env
    assert sorted(
        [sorted(i) for i in e1]) == sorted([sorted(i) for i in e2]) == [
        [
            ('CONDA_BOOST', '1.60'),
            ('CONDA_PY', 27),
        ],
        [
            ('CONDA_BOOST', '1.60'),
            ('CONDA_PY', 35),
        ]
    ]

def test_filter_recipes_no_skipping():
    """
    No recipes have skip so make sure none are filtered out.
    """
    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: one
              version: "0.1"
        """), from_string=True)
    r.write_recipes()
    env_matrix = {
        'CONDA_PY': [27, 35],
        'CONDA_BOOST': '1.60'
    }
    recipes = list(r.recipe_dirs.values())
    assert len(recipes) == 1
    filtered = list(
        utils.filter_recipes(recipes, env_matrix, channels=['bioconda']))
    assert len(filtered) == 1

def test_filter_recipes_skip_is_true():
    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: one
              version: "0.1"
            build:
              skip: true
        """), from_string=True)
    r.write_recipes()
    env_matrix = {}
    recipes = list(r.recipe_dirs.values())
    filtered = list(
        utils.filter_recipes(recipes, env_matrix))
    assert len(filtered) == 0

def test_filter_recipes_skip_py27():
    """
    When we add build/skip = True # [py27] to recipe, it should not be
    filtered out. This is because python version is not encoded in the output
    package name, and so one-0.1-0.tar.bz2 will still be created for py35.
    """
    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: one
              version: "0.1"
            build:
              skip: true  # [py27]
        """), from_string=True)
    r.write_recipes()
    env_matrix = {
        'CONDA_PY': [27, 35],
        'CONDA_BOOST': '1.60'
    }
    recipes = list(r.recipe_dirs.values())
    filtered = list(
        utils.filter_recipes(recipes, env_matrix, channels=['bioconda']))
    assert len(filtered) == 1

def test_filter_recipes_skip_py27_in_build_string():
    """
    When CONDA_PY is in the build string, py27 should be skipped
    """
    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: one
              version: "0.1"
            build:
              string: {{CONDA_PY}}_{{PKG_BUILDNUM}}
        """), from_string=True)
    r.write_recipes()
    env_matrix = {
        'CONDA_PY': [27, 35],
    }
    recipes = list(r.recipe_dirs.values())
    filtered = list(
        utils.filter_recipes(recipes, env_matrix, channels=['bioconda']))

    # one recipe, two targets
    assert len(filtered) == 1
    assert len(filtered[0][1]) == 2

    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: one
              version: "0.1"
            build:
              string: {{CONDA_PY}}_{{PKG_BUILDNUM}}
              skip: True # [py27]
        """), from_string=True)
    r.write_recipes()
    env_matrix = {
        'CONDA_PY': [27, 35],
    }
    recipes = list(r.recipe_dirs.values())
    filtered = list(
        utils.filter_recipes(recipes, env_matrix, channels=['bioconda']))

    # one recipe, one target
    assert len(filtered) == 1
    assert len(filtered[0][1]) == 1

def test_filter_recipes_extra_in_build_string():
    """
    If CONDA_EXTRA is in os.environ, the pkg name should still be identifiable.

    This helps test env vars that don't have other defaults like CONDA_PY does
    (e.g., CONDA_BOOST in bioconda)
    """
    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: one
              version: "0.1"
            build:
              number: 0
              string: {{CONDA_EXTRA}}_{{PKG_BUILDNUM}}
        """), from_string=True)
    r.write_recipes()
    os.environ['CONDA_EXTRA'] = 'asdf'
    pkg = utils.built_package_path(r.recipe_dirs['one'])
    assert os.path.basename(pkg) == 'one-0.1-asdf_0.tar.bz2'

def test_filter_recipes_existing_package():

    # use a known-to-exist package in bioconda
    #
    # note that we need python as a run requirement in order to get the "pyXY"
    # in the build string that matches the existing bioconda built package.
    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: gffutils
              version: "0.8.7.1"
            requirements:
              run:
                - python
        """), from_string=True)
    r.write_recipes()
    recipes = list(r.recipe_dirs.values())
    env_matrix = {
        'CONDA_PY': [27, 35],
    }
    pkgs = utils.get_channel_packages('bioconda')
    pth = utils.built_package_path(recipes[0])
    filtered = list(
        utils.filter_recipes(recipes, env_matrix, channels=['bioconda']))
    assert len(filtered) == 0

def test_filter_recipes_force_existing_package():
    # same as above, but this time force the recipe
    # TODO: refactor as py.test fixture
    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: gffutils
              version: "0.8.7.1"
            requirements:
              run:
                - python
        """), from_string=True)
    r.write_recipes()
    recipes = list(r.recipe_dirs.values())
    env_matrix = {
        'CONDA_PY': [27, 35],
    }
    pkgs = utils.get_channel_packages('bioconda')
    pth = utils.built_package_path(recipes[0])
    filtered = list(
        utils.filter_recipes(
            recipes, env_matrix, channels=['bioconda'], force=True))
    assert len(filtered) == 1

def test_get_channel_packages():
    with pytest.raises(requests.HTTPError):
        utils.get_channel_packages('bioconda_xyz_nonexistent_channel')
    utils.get_channel_packages('bioconda')

def test_built_package_path():
    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: one
              version: "0.1"
            requirements:
              run:
                - python

        two:
          meta.yaml: |
            package:
              name: two
              version: "0.1"
            build:
              number: 0
              string: ncurses{{ CONDA_NCURSES }}_{{ PKG_BUILDNUM }}
        """), from_string=True)
    r.write_recipes()

    # assumes we're running on py35
    assert os.path.basename(
        utils.built_package_path(r.recipe_dirs['one'])
    ) == 'one-0.1-py35_0.tar.bz2'

    # resetting with a different CONDA_PY passed as env dict
    assert os.path.basename(
        utils.built_package_path(r.recipe_dirs['one'], env=dict(CONDA_PY=27))
    ) == 'one-0.1-py27_0.tar.bz2'

    # resetting CONDA_PY using os.environ
    existing_env = dict(os.environ)
    try:
        os.environ['CONDA_PY'] = '27'
        assert os.path.basename(
            utils.built_package_path(r.recipe_dirs['one'])
        ) == 'one-0.1-py27_0.tar.bz2'
        os.environ = existing_env
    except:
        os.environ = existing_env
        raise

    existing_env = dict(os.environ)
    os.environ['CONDA_NCURSES'] = '9.0'
    assert os.path.basename(
        utils.built_package_path(r.recipe_dirs['two'], env=os.environ)
    ) == 'two-0.1-ncurses9.0_0.tar.bz2'

def test_string_or_float_to_integer_python():
    f = utils._string_or_float_to_integer_python
    assert f(27) == f('27') == f(2.7) == f('2.7') == 27
<commit_msg>add tests that are marked as failing, see https://github.com/conda/conda-build/issues/1464<commit_after>import os
import subprocess as sp
import pytest
from textwrap import dedent
import yaml
import tempfile
import requests
from bioconda_utils import utils
from bioconda_utils import pkg_test
from bioconda_utils import docker_utils
from bioconda_utils import cli
from bioconda_utils import build
from helpers import ensure_missing, Recipes, tmp_env_matrix
from conda_build import api
from conda_build.metadata import MetaData

def test_get_deps():
    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: one
              version: 0.1
        two:
          meta.yaml: |
            package:
              name: two
              version: 0.1
            requirements:
              build:
                - one
        three:
          meta.yaml: |
            package:
              name: three
              version: 0.1
            requirements:
              build:
                - one
              run:
                - two
    """), from_string=True)
    r.write_recipes()
    assert list(utils.get_deps(r.recipe_dirs['two'])) == ['one']
    assert list(utils.get_deps(r.recipe_dirs['three'], build=True)) == ['one']
    assert list(utils.get_deps(r.recipe_dirs['three'], build=False)) == ['two']

def _single_build(docker_builder=None):
    """
    Tests the building of a single configured recipe, with or without docker
    """
    r = Recipes('test_case.yaml')
    r.write_recipes()
    env_matrix = list(utils.EnvMatrix(tmp_env_matrix()))[0]
    conda_bld = docker_utils.get_host_conda_bld()
    built_package = os.path.join(conda_bld, 'linux-64', 'one-0.1-0.tar.bz2')
    ensure_missing(built_package)
    build.build(
        recipe=r.recipe_dirs['one'],
        recipe_folder='.',
        docker_builder=docker_builder,
        env=env_matrix,
    )
    assert os.path.exists(built_package)
    return built_package

def test_single_build1():
    built_package = _single_build(docker_builder=None)

def test_single_build_docker_with_post_test():
    docker_builder = docker_utils.RecipeBuilder(verbose=True)
    r = Recipes('test_case.yaml')
    r.write_recipes()
    env_matrix = list(utils.EnvMatrix(tmp_env_matrix()))[0]
    conda_bld = docker_utils.get_host_conda_bld()
    built_package = os.path.join(conda_bld, 'linux-64', 'one-0.1-0.tar.bz2')
    ensure_missing(built_package)
    build.build(
        recipe=r.recipe_dirs['one'],
        recipe_folder='.',
        docker_builder=docker_builder,
        env=env_matrix)
    assert os.path.exists(built_package)
    pkg_test.test_package(built_package)
    ensure_missing(built_package)

def test_single_build_docker():
    docker_builder = docker_utils.RecipeBuilder(verbose=True)
    built_package = _single_build(docker_builder=docker_builder)
    ensure_missing(built_package)

def test_docker_builder_build():
    """
    Tests just the build_recipe method of a RecipeBuilder object.

    Makes sure the built recipe shows up on the host machine.
    """
    conda_bld = docker_utils.get_host_conda_bld()
    built_package = os.path.join(conda_bld, 'linux-64', 'one-0.1-0.tar.bz2')
    ensure_missing(built_package)
    docker_builder = docker_utils.RecipeBuilder(verbose=True)
    r = Recipes('test_case.yaml')
    r.write_recipes()
    recipe_dir = r.recipe_dirs['one']
    docker_builder.build_recipe(recipe_dir, build_args='')
    assert os.path.exists(built_package)
    os.unlink(built_package)
    assert not os.path.exists(built_package)

def test_docker_build_image_fails():
    template = dedent(
        """
        FROM {self.image}
        RUN nonexistent command
        """)
    with pytest.raises(sp.CalledProcessError):
        docker_builder = docker_utils.RecipeBuilder(
            verbose=True, dockerfile_template=template)

def test_conda_purge_cleans_up():
    def tmp_dir_exists(d):
        contents = os.listdir(d)
        for i in contents:
            if i.startswith('tmp') and '_' in i:
                return True

    bld = docker_utils.get_host_conda_bld(purge=False)
    assert tmp_dir_exists(bld)
    bld = docker_utils.get_host_conda_bld(purge=True)
    assert not tmp_dir_exists(bld)

def test_local_channel():
    r = Recipes('test_case.yaml')
    r.write_recipes()
    with pytest.raises(SystemExit) as e:
        cli.build(r.basedir,
                  config={},
                  packages="*",
                  testonly=False,
                  force=False,
                  docker=True,
                  loglevel="debug",
                  )
        assert e.code == 0
    conda_bld = docker_utils.get_host_conda_bld()
    print(os.listdir(os.path.join(conda_bld, 'linux-64')))

def test_env_matrix():
    contents = {
        'CONDA_PY': [27, 35],
        'CONDA_BOOST': '1.60'
    }

    with open(tempfile.NamedTemporaryFile().name, 'w') as fout:
        fout.write(yaml.dump(contents, default_flow_style=False))

    e1 = utils.EnvMatrix(contents)
    e2 = utils.EnvMatrix(fout.name)
    assert e1.env == e2.env
    assert sorted(
        [sorted(i) for i in e1]) == sorted([sorted(i) for i in e2]) == [
        [
            ('CONDA_BOOST', '1.60'),
            ('CONDA_PY', 27),
        ],
        [
            ('CONDA_BOOST', '1.60'),
            ('CONDA_PY', 35),
        ]
    ]

def test_filter_recipes_no_skipping():
    """
    No recipes have skip so make sure none are filtered out.
    """
    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: one
              version: "0.1"
        """), from_string=True)
    r.write_recipes()
    env_matrix = {
        'CONDA_PY': [27, 35],
        'CONDA_BOOST': '1.60'
    }
    recipes = list(r.recipe_dirs.values())
    assert len(recipes) == 1
    filtered = list(
        utils.filter_recipes(recipes, env_matrix, channels=['bioconda']))
    assert len(filtered) == 1

def test_filter_recipes_skip_is_true():
    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: one
              version: "0.1"
            build:
              skip: true
        """), from_string=True)
    r.write_recipes()
    env_matrix = {}
    recipes = list(r.recipe_dirs.values())
    filtered = list(
        utils.filter_recipes(recipes, env_matrix))
    assert len(filtered) == 0

def test_filter_recipes_skip_py27():
    """
    When we add build/skip = True # [py27] to recipe, it should not be
    filtered out. This is because python version is not encoded in the output
    package name, and so one-0.1-0.tar.bz2 will still be created for py35.
    """
    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: one
              version: "0.1"
            build:
              skip: true  # [py27]
        """), from_string=True)
    r.write_recipes()
    env_matrix = {
        'CONDA_PY': [27, 35],
        'CONDA_BOOST': '1.60'
    }
    recipes = list(r.recipe_dirs.values())
    filtered = list(
        utils.filter_recipes(recipes, env_matrix, channels=['bioconda']))
    assert len(filtered) == 1

def test_filter_recipes_skip_py27_in_build_string():
    """
    When CONDA_PY is in the build string, py27 should be skipped
    """
    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: one
              version: "0.1"
            build:
              string: {{CONDA_PY}}_{{PKG_BUILDNUM}}
        """), from_string=True)
    r.write_recipes()
    env_matrix = {
        'CONDA_PY': [27, 35],
    }
    recipes = list(r.recipe_dirs.values())
    filtered = list(
        utils.filter_recipes(recipes, env_matrix, channels=['bioconda']))

    # one recipe, two targets
    assert len(filtered) == 1
    assert len(filtered[0][1]) == 2

    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: one
              version: "0.1"
            build:
              string: {{CONDA_PY}}_{{PKG_BUILDNUM}}
              skip: True # [py27]
        """), from_string=True)
    r.write_recipes()
    env_matrix = {
        'CONDA_PY': [27, 35],
    }
    recipes = list(r.recipe_dirs.values())
    filtered = list(
        utils.filter_recipes(recipes, env_matrix, channels=['bioconda']))

    # one recipe, one target
    assert len(filtered) == 1
    assert len(filtered[0][1]) == 1

# See https://github.com/conda/conda-build/issues/1464
@pytest.mark.xfail
def test_filter_recipes_extra_in_build_string():
    """
    If CONDA_EXTRA is in os.environ, the pkg name should still be identifiable.

    This helps test env vars that don't have other defaults like CONDA_PY does
    (e.g., CONDA_BOOST in bioconda)
    """
    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: one
              version: "0.1"
            build:
              number: 1
              string: {{CONDA_EXTRA}}_{{PKG_BUILDNUM}}_{{PKG_NAME}}
        """), from_string=True)
    r.write_recipes()
    recipe = r.recipe_dirs['one']

    from conda_build.render import bldpkg_path

    metadata = MetaData(recipe, api.Config(**dict(CONDA_EXTRA='asdf')))
    print(bldpkg_path(metadata, metadata.config))

    os.environ['CONDA_EXTRA'] = 'asdf'
    pkg = utils.built_package_path(recipe)
    assert os.path.basename(pkg) == 'one-0.1-asdf_0.tar.bz2'

def test_filter_recipes_existing_package():
    # use a known-to-exist package in bioconda
    #
    # note that we need python as a run requirement in order to get the "pyXY"
    # in the build string that matches the existing bioconda built package.
    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: gffutils
              version: "0.8.7.1"
            requirements:
              run:
                - python
        """), from_string=True)
    r.write_recipes()
    recipes = list(r.recipe_dirs.values())
    env_matrix = {
        'CONDA_PY': [27, 35],
    }
    pkgs = utils.get_channel_packages('bioconda')
    pth = utils.built_package_path(recipes[0])
    filtered = list(
        utils.filter_recipes(recipes, env_matrix, channels=['bioconda']))
    assert len(filtered) == 0

def test_filter_recipes_force_existing_package():
    # same as above, but this time force the recipe
    # TODO: refactor as py.test fixture
    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: gffutils
              version: "0.8.7.1"
            requirements:
              run:
                - python
        """), from_string=True)
    r.write_recipes()
    recipes = list(r.recipe_dirs.values())
    env_matrix = {
        'CONDA_PY': [27, 35],
    }
    pkgs = utils.get_channel_packages('bioconda')
    pth = utils.built_package_path(recipes[0])
    filtered = list(
        utils.filter_recipes(
            recipes, env_matrix, channels=['bioconda'], force=True))
    assert len(filtered) == 1

def test_get_channel_packages():
    with pytest.raises(requests.HTTPError):
        utils.get_channel_packages('bioconda_xyz_nonexistent_channel')
    utils.get_channel_packages('bioconda')

def test_built_package_path():
    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: one
              version: "0.1"
            requirements:
              run:
                - python

        two:
          meta.yaml: |
            package:
              name: two
              version: "0.1"
            build:
              number: 0
              string: ncurses{{ CONDA_NCURSES }}_{{ PKG_BUILDNUM }}
        """), from_string=True)
    r.write_recipes()

    # assumes we're running on py35
    assert os.path.basename(
        utils.built_package_path(r.recipe_dirs['one'])
    ) == 'one-0.1-py35_0.tar.bz2'

    # resetting with a different CONDA_PY passed as env dict
    assert os.path.basename(
        utils.built_package_path(r.recipe_dirs['one'], env=dict(CONDA_PY=27))
    ) == 'one-0.1-py27_0.tar.bz2'

    # resetting CONDA_PY using os.environ
    existing_env = dict(os.environ)
    try:
        os.environ['CONDA_PY'] = '27'
        assert os.path.basename(
            utils.built_package_path(r.recipe_dirs['one'])
        ) == 'one-0.1-py27_0.tar.bz2'
        os.environ = existing_env
    except:
        os.environ = existing_env
        raise

# See https://github.com/conda/conda-build/issues/1464
@pytest.mark.xfail
def test_built_package_path2():
    r = Recipes(dedent(
        """
        one:
          meta.yaml: |
            package:
              name: one
              version: "0.1"
            requirements:
              run:
                - python

        two:
          meta.yaml: |
            package:
              name: two
              version: "0.1"
            build:
              number: 0
              string: ncurses{{ CONDA_NCURSES }}_{{ PKG_BUILDNUM }}
        """), from_string=True)
    r.write_recipes()

    existing_env = dict(os.environ)
    os.environ['CONDA_NCURSES'] = '9.0'
    assert os.path.basename(
        utils.built_package_path(r.recipe_dirs['two'], env=os.environ)
    ) == 'two-0.1-ncurses9.0_0.tar.bz2'

def test_string_or_float_to_integer_python():
    f = utils._string_or_float_to_integer_python
    assert f(27) == f('27') == f(2.7) == f('2.7') == 27
<|endoftext|>


<commit_before>"""
config rendering and verification

Copyright (c) 2010 Mika Eloranta
See LICENSE for details.

"""

import sys
import itertools
import datetime
import logging
import difflib
from path import path
from . import errors
from . import util
from . import colors

import Cheetah.Template
from Cheetah.Template import Template as CheetahTemplate

try:
    import genshi
    import genshi.template
except ImportError:
    genshi = None

class Manager:
    def __init__(self, confman):
        self.log = logging.getLogger("manager")
        self.files = []
        self.error_count = 0
        self.confman = confman
        self.buckets = {}
        self.audit_format = "%8s %s: %s"

    def get_bucket(self, name):
        return self.buckets.setdefault(name, [])

    def emit_error(self, node, source_path, error):
        self.log.warning("node %s: %s: %s: %s", node.name, source_path,
                         error.__class__.__name__, error)
        self.error_count += 1

    def copy_tree(self, entry, remote, path_prefix=""):
        def progress(copied, total):
            sys.stderr.write("\r%s/%s bytes copied" % (copied, total))

        dest_dir = path(path_prefix + entry["dest_path"])
        try:
            remote.stat(dest_dir)
        except errors.RemoteError:
            remote.makedirs(dest_dir)

        for file_path in path(entry["source_path"]).files():
            dest_path = dest_dir / file_path.basename()
            lstat = file_path.stat()
            try:
                rstat = remote.stat(dest_path)
                # copy if mtime or size differs
                copy = ((lstat.st_size != rstat.st_size)
                        or (lstat.st_mtime != rstat.st_mtime))
            except errors.RemoteError:
                copy = True

            if copy:
                self.log.info("copying: %s", dest_path)
                remote.put_file(file_path, dest_path, callback=progress)
                remote.utime(dest_path, (int(lstat.st_mtime),
                                         int(lstat.st_mtime)))
                sys.stderr.write("\n")
            else:
                self.log.info("already copied: %s", dest_path)

    def verify(self, show=False, deploy=False, audit=False, show_diff=False,
               verbose=False, callback=None, path_prefix="", raw=False,
               access_method=None, color_mode="auto"):
        self.log.debug("verify: %s", dict(show=show, deploy=deploy,
                                          audit=audit, show_diff=show_diff,
                                          verbose=verbose, callback=callback))
        files = [f for f in self.files if not f.get("report")]
        reports = [f for f in self.files if f.get("report")]
        if path_prefix and not path_prefix.endswith("/"):
            path_prefix += "/"

        color = colors.Output(sys.stdout, color=color_mode).color
        error_count = 0
        for entry in itertools.chain(files, reports):
            if not entry["node"].verify_enabled():
                self.log.debug("filtered: verify disabled: %r", entry)
                continue

            filtered_out = False
            if callback and not callback(entry):
                self.log.debug("filtered: callback: %r", entry)
                filtered_out = True

            self.log.debug("verify: %r", entry)
            render = entry["render"]
            failed = False
            node_name = entry["node"].name

            if entry["type"] == "dir":
                if filtered_out:
                    # ignore
                    pass
                elif deploy:
                    # copy a directory recursively
                    remote = entry["node"].get_remote(override=access_method)
                    self.copy_tree(entry, remote, path_prefix=path_prefix)
                else:
                    # verify
                    try:
                        dir_stats = util.dir_stats(entry["source_path"])
                    except (OSError, IOError), error:
                        raise errors.VerifyError(
                            "cannot copy files from '%s': %s: %s"% (
                                error.__class__.__name__, error))

                    if dir_stats["file_count"] == 0:
                        self.log.warning("source directory '%s' is empty" % (
                                entry["source_path"]))
                    elif verbose:
                        self.log.info(
                            "[OK] copy source directory '%(path)s' has "
                            "%(file_count)s files, "
                            "%(total_bytes)s bytes" % dir_stats)

                # dir handled, next!
                continue

            source_path = entry["config"].path / entry["source_path"]
            try:
                dest_path = entry["dest_path"]
                if dest_path[-1:] == "/":
                    # dest path ending in slash: use source filename
                    dest_path = path(dest_path) / source_path.basename()

                if raw:
                    dest_path, output = dest_path, source_path.bytes()
                else:
                    dest_path, output = render(source_path, dest_path)

                dest_path = path(path_prefix + dest_path).normpath()
                if (not audit and not deploy) and verbose:
                    # plain verify mode
                    self.log.info("OK: %s file: %s", node_name, dest_path)
            except (IOError, errors.Error), error:
                self.emit_error(entry["node"], source_path, error)
                output = util.format_error(error)
                failed = True
                error_count += 1

            if show and not filtered_out:
                if show_diff:
                    diff = difflib.unified_diff(
                        source_path.bytes().splitlines(True),
                        output.splitlines(True),
                        "template", "rendered",
                        "", "",
                        lineterm="\n")

                    show_output = diff

                else:
                    show_output = output

                identity = "%s%s%s" % (color(node_name, "node"),
                                       color(": path=", "header"),
                                       color(dest_path, "path"))
                sys.stdout.write("%s %s %s\n" % (color("--- BEGIN", "header"),
                                               identity,
                                               color("---", "header")))

                if isinstance(show_output, (str, unicode)):
                    print show_output
                else:
                    diff_colors = {"+": "lgreen", "@": "white", "-": "lred"}
                    for line in show_output:
                        sys.stdout.write(
                            color(line, diff_colors.get(line[:1], "reset")))

                sys.stdout.write("%s %s %s\n\n" % (color("--- END", "header"),
                                                   identity,
                                                   color("---", "header")))
                sys.stdout.flush()

            remote = None

            if (audit or deploy) and dest_path and (not failed) and (not filtered_out):
                # read existing file
                try:
                    remote = entry["node"].get_remote(override=access_method)
                    active_text = remote.read_file(dest_path)
                    stat = remote.stat(dest_path)
                    if stat:
                        active_time = datetime.datetime.fromtimestamp(
                            stat.st_mtime)
                    else:
                        active_time = ""
                except errors.RemoteError, error:
                    if audit:
                        self.log.error("%s: %s: %s: %s", node_name, dest_path,
                                       error.__class__.__name__, error)
                        error_count += 1

                    active_text = None
            else:
                active_text = None

            if active_text and audit:
                self.audit_output(entry, dest_path, active_text, active_time,
                                  output, show_diff=show_diff,
                                  color_mode=color_mode, verbose=verbose)

            if deploy and dest_path and (not failed) and (not filtered_out):
                remote = entry["node"].get_remote(override=access_method)
                try:
                    self.deploy_file(remote, entry, dest_path, output,
                                     active_text, verbose=verbose,
                                     mode=entry.get("mode"))
                except errors.RemoteError, error:
                    error_count += 1
                    self.log.error("%s: %s: %s", node_name, dest_path, error)
                    # NOTE: continuing

        if error_count:
            raise errors.VerifyError("failed: there were %s errors" % (
                    error_count))

    def deploy_file(self, remote, entry, dest_path, output, active_text,
                    verbose=False, mode=None):
        if output == active_text:
            # nothing to do
            if verbose:
                self.log.info(self.audit_format, "OK",
                              entry["node"].name, dest_path)

            return

        dest_dir = dest_path.dirname()
        try:
            remote.stat(dest_dir)
        except errors.RemoteError:
            remote.makedirs(dest_dir)

        remote.write_file(dest_path, output, mode=mode)
        post_process = entry.get("post_process")
        if post_process:
            # TODO: remote support
            post_process(dest_path)

        self.log.info(self.audit_format, "WROTE",
                      entry["node"].name, dest_path)

    def audit_output(self, entry, dest_path, active_text, active_time,
                     output, show_diff=False, color_mode="auto",
                     verbose=False):
        if (active_text is not None) and (active_text != output):
            self.log.warning(self.audit_format, "DIFFERS",
                             entry["node"].name, dest_path)
            if show_diff:
                color = colors.Output(sys.stdout, color=color_mode).color
                diff = difflib.unified_diff(
                    output.splitlines(True),
                    active_text.splitlines(True),
                    "config", "active",
                    "", active_time, # TODO: mtime for config?
                    lineterm="\n")

                diff_colors = {"+": "lgreen", "@": "white", "-": "lred"}
                for line in diff:
                    sys.stdout.write(
                        color(line, diff_colors.get(line[:1], "reset")))

                sys.stdout.flush()
        elif active_text and verbose:
            self.log.info(self.audit_format, "OK", entry["node"].name,
                          dest_path)

    def add_file(self, **kw):
        self.log.debug("add_file: %s", kw)
        self.files.append(kw)

class PlugIn:
    def __init__(self, manager, config, node, top_config):
        self.log = logging.getLogger("plugin")
        self.manager = manager
        self.config = config
        self.top_config = top_config
        self.node = node

    def add_file(self, source_path, dest_path=None, source_text=None,
                 render=None, report=False, post_process=None, mode=None):
        render = render or self.render_cheetah
        return self.manager.add_file(node=self.node, config=self.config,
                                     type="file", dest_path=dest_path,
                                     source_path=source_path,
                                     source_text=source_text,
                                     render=render, report=report,
                                     post_process=post_process,
                                     mode=mode)

    def add_dir(self, source_path, dest_path, render=None):
        render = render or self.render_cheetah
        return self.manager.add_file(type="dir", node=self.node,
                                     config=self.config, dest_path=dest_path,
                                     source_path=source_path, render=render)

    def get_one(self, name, nodes=True, systems=False):
        hits = list(self.manager.confman.find(name, nodes=nodes,
                                              systems=systems,
                                              full_match=True))
        names = (h.name for h in hits)
        if len(hits) > 1:
            raise errors.VerifyError("found more than one (%d) %r: %s" % (
                    len(hits), name, ", ".join(names)))
        elif len(hits) == 0:
            raise errors.VerifyError("did not find %r: %s" % (
                    name, ", ".join(names)))

        return hits[0]

    def get_system(self, name):
        return self.get_one(name, nodes=False, systems=True)

    def render_text(self, source_path, dest_path):
        try:
            return dest_path, file(source_path, "rb").read()
        except (IOError, OSError), error:
            raise errors.VerifyError(source_path, error)

    def add_edge(self, bucket_name, dest_node, dest_config, **kwargs):
        self.manager.get_bucket(bucket_name).append(
            dict(source_node=self.node, source_config=self.top_config,
                 dest_node=dest_node, dest_config=dest_config, **kwargs))

    def get_names(self):
        names = dict(node=self.node,
                     s=self.top_config.settings,
                     settings=self.top_config.settings,
                     system=self.node.system,
                     find=self.manager.confman.find,
                     find_config=self.manager.confman.find_config,
                     get_node=self.get_one,
                     get_system=self.get_system,
                     get_config=self.manager.confman.get_config,
                     config=self.top_config,
                     bucket=self.manager.get_bucket,
                     edge=self.add_edge,
                     plugin=self)
        return names

    def render_cheetah(self, source_path, dest_path):
        names = self.get_names()
        # TODO: template caching
        try:
            text = str(CheetahTemplate(file=source_path, searchList=[names]))
            if dest_path:
                dest_path = str(CheetahTemplate(dest_path, searchList=[names]))

            return dest_path, text
        except (Cheetah.Template.Error, SyntaxError,
                Cheetah.NameMapper.NotFound), error:
            raise errors.VerifyError("%s: %s: %s" % (
                source_path, error.__class__.__name__, error))

    def render_genshi_xml(self, source_path, dest_path):
        assert genshi, "Genshi is not installed"

        names = self.get_names()
        if dest_path:
            dest_path = str(CheetahTemplate(dest_path, searchList=[names]))

        try:
            tmpl = genshi.template.MarkupTemplate(file(source_path),
                                                  filepath=source_path)
            stream = tmpl.generate(**names)
            output = stream.render('xml')
            return dest_path, output
        except (errors.Error,
                genshi.template.TemplateError,
                IOError), error:
            raise errors.VerifyError(source_path, error)
<commit_msg>using deploy/audit --path-prefix also adds system hierarchy<commit_after>"""
config rendering and verification

Copyright (c) 2010 Mika Eloranta
See LICENSE for details.

"""

import sys
import itertools
import datetime
import logging
import difflib
from path import path
from . import errors
from . import util
from . import colors

import Cheetah.Template
from Cheetah.Template import Template as CheetahTemplate

try:
    import genshi
    import genshi.template
except ImportError:
    genshi = None

class Manager:
    def __init__(self, confman):
        self.log = logging.getLogger("manager")
        self.files = []
        self.error_count = 0
        self.confman = confman
        self.buckets = {}
        self.audit_format = "%8s %s: %s"

    def get_bucket(self, name):
        return self.buckets.setdefault(name, [])

    def emit_error(self, node, source_path, error):
        self.log.warning("node %s: %s: %s: %s", node.name, source_path,
                         error.__class__.__name__, error)
        self.error_count += 1

    def copy_tree(self, entry, remote, path_prefix=""):
        def progress(copied, total):
            sys.stderr.write("\r%s/%s bytes copied" % (copied, total))

        dest_dir = path(path_prefix + entry["dest_path"])
        try:
            remote.stat(dest_dir)
        except errors.RemoteError:
            remote.makedirs(dest_dir)

        for file_path in path(entry["source_path"]).files():
            dest_path = dest_dir / file_path.basename()
            lstat = file_path.stat()
            try:
                rstat = remote.stat(dest_path)
                # copy if mtime or size differs
                copy = ((lstat.st_size != rstat.st_size)
                        or (lstat.st_mtime != rstat.st_mtime))
            except errors.RemoteError:
                copy = True

            if copy:
                self.log.info("copying: %s", dest_path)
                remote.put_file(file_path, dest_path, callback=progress)
                remote.utime(dest_path, (int(lstat.st_mtime),
                                         int(lstat.st_mtime)))
                sys.stderr.write("\n")
            else:
                self.log.info("already copied: %s", dest_path)

    def verify(self, show=False, deploy=False, audit=False, show_diff=False,
               verbose=False, callback=None, path_prefix="", raw=False,
               access_method=None, color_mode="auto"):
        self.log.debug("verify: %s", dict(show=show, deploy=deploy,
                                          audit=audit, show_diff=show_diff,
                                          verbose=verbose, callback=callback))
        files = [f for f in self.files if not f.get("report")]
        reports = [f for f in self.files if f.get("report")]

        color = colors.Output(sys.stdout, color=color_mode).color
        error_count = 0
        for entry in itertools.chain(files, reports):
            if not entry["node"].verify_enabled():
                self.log.debug("filtered: verify disabled: %r", entry)
                continue

            filtered_out = False
            if callback and not callback(entry):
                self.log.debug("filtered: callback: %r", entry)
                filtered_out = True

            if path_prefix:
                item_path_prefix = "%s/%s/" % (path_prefix, entry["node"].name)
            else:
                item_path_prefix = ""

            self.log.debug("verify: %r", entry)
            render = entry["render"]
            failed = False
            node_name = entry["node"].name

            if entry["type"] == "dir":
                if filtered_out:
                    # ignore
                    pass
                elif deploy:
                    # copy a directory recursively
                    remote = entry["node"].get_remote(override=access_method)
                    self.copy_tree(entry, remote, path_prefix=item_path_prefix)
                else:
                    # verify
                    try:
                        dir_stats = util.dir_stats(entry["source_path"])
                    except (OSError, IOError), error:
                        raise errors.VerifyError(
                            "cannot copy files from '%s': %s: %s"% (
                                error.__class__.__name__, error))

                    if dir_stats["file_count"] == 0:
                        self.log.warning("source directory '%s' is empty" % (
                                entry["source_path"]))
                    elif verbose:
                        self.log.info(
                            "[OK] copy source directory '%(path)s' has "
                            "%(file_count)s files, "
                            "%(total_bytes)s bytes" % dir_stats)

                # dir handled, next!
                continue

            source_path = entry["config"].path / entry["source_path"]
            try:
                dest_path = entry["dest_path"]
                if dest_path[-1:] == "/":
                    # dest path ending in slash: use source filename
                    dest_path = path(dest_path) / source_path.basename()

                if raw:
                    dest_path, output = dest_path, source_path.bytes()
                else:
                    dest_path, output = render(source_path, dest_path)

                dest_path = path(item_path_prefix + dest_path).normpath()
                if (not audit and not deploy) and verbose:
                    # plain verify mode
                    self.log.info("OK: %s file: %s", node_name, dest_path)
            except (IOError, errors.Error), error:
                self.emit_error(entry["node"], source_path, error)
                output = util.format_error(error)
                failed = True
                error_count += 1

            if show and not filtered_out:
                if show_diff:
                    diff = difflib.unified_diff(
                        source_path.bytes().splitlines(True),
                        output.splitlines(True),
                        "template", "rendered",
                        "", "",
                        lineterm="\n")

                    show_output = diff

                else:
                    show_output = output

                identity = "%s%s%s" % (color(node_name, "node"),
                                       color(": path=", "header"),
                                       color(dest_path, "path"))
                sys.stdout.write("%s %s %s\n" % (color("--- BEGIN", "header"),
                                               identity,
                                               color("---", "header")))

                if isinstance(show_output, (str, unicode)):
                    print show_output
                else:
                    diff_colors = {"+": "lgreen", "@": "white", "-": "lred"}
                    for line in show_output:
                        sys.stdout.write(
                            color(line, diff_colors.get(line[:1], "reset")))

                sys.stdout.write("%s %s %s\n\n" % (color("--- END", "header"),
                                                   identity,
                                                   color("---", "header")))
                sys.stdout.flush()

            remote = None

            if (audit or deploy) and dest_path and (not failed) and (not filtered_out):
                # read existing file
                try:
                    remote = entry["node"].get_remote(override=access_method)
                    active_text = remote.read_file(dest_path)
                    stat = remote.stat(dest_path)
                    if stat:
                        active_time = datetime.datetime.fromtimestamp(
                            stat.st_mtime)
                    else:
                        active_time = ""
                except errors.RemoteError, error:
                    if audit:
                        self.log.error("%s: %s: %s: %s", node_name, dest_path,
                                       error.__class__.__name__, error)
                        error_count += 1

                    active_text = None
            else:
                active_text = None

            if active_text and audit:
                self.audit_output(entry, dest_path, active_text, active_time,
                                  output, show_diff=show_diff,
                                  color_mode=color_mode, verbose=verbose)

            if deploy and dest_path and (not failed) and (not filtered_out):
                remote = entry["node"].get_remote(override=access_method)
                try:
                    self.deploy_file(remote, entry, dest_path, output,
                                     active_text, verbose=verbose,
                                     mode=entry.get("mode"))
                except errors.RemoteError, error:
                    error_count += 1
                    self.log.error("%s: %s: %s", node_name, dest_path, error)
                    # NOTE: continuing

        if error_count:
            raise errors.VerifyError("failed: there were %s errors" % (
                    error_count))

    def deploy_file(self, remote, entry, dest_path, output, active_text,
                    verbose=False, mode=None):
        if output == active_text:
            # nothing to do
            if verbose:
                self.log.info(self.audit_format, "OK",
                              entry["node"].name, dest_path)

            return

        dest_dir = dest_path.dirname()
        try:
            remote.stat(dest_dir)
        except errors.RemoteError:
            remote.makedirs(dest_dir)

        remote.write_file(dest_path, output, mode=mode)
        post_process = entry.get("post_process")
        if post_process:
            # TODO: remote support
            post_process(dest_path)

        self.log.info(self.audit_format, "WROTE",
                      entry["node"].name, dest_path)

    def audit_output(self, entry, dest_path, active_text, active_time,
                     output, show_diff=False, color_mode="auto",
                     verbose=False):
        if (active_text is not None) and (active_text != output):
            self.log.warning(self.audit_format, "DIFFERS",
                             entry["node"].name, dest_path)
            if show_diff:
                color = colors.Output(sys.stdout, color=color_mode).color
                diff = difflib.unified_diff(
                    output.splitlines(True),
                    active_text.splitlines(True),
                    "config", "active",
                    "", active_time, # TODO: mtime for config?
                    lineterm="\n")

                diff_colors = {"+": "lgreen", "@": "white", "-": "lred"}
                for line in diff:
                    sys.stdout.write(
                        color(line, diff_colors.get(line[:1], "reset")))

                sys.stdout.flush()
        elif active_text and verbose:
            self.log.info(self.audit_format, "OK", entry["node"].name,
                          dest_path)

    def add_file(self, **kw):
        self.log.debug("add_file: %s", kw)
        self.files.append(kw)

class PlugIn:
    def __init__(self, manager, config, node, top_config):
        self.log = logging.getLogger("plugin")
        self.manager = manager
        self.config = config
        self.top_config = top_config
        self.node = node

    def add_file(self, source_path, dest_path=None, source_text=None,
                 render=None, report=False, post_process=None, mode=None):
        render = render or self.render_cheetah
        return self.manager.add_file(node=self.node, config=self.config,
                                     type="file", dest_path=dest_path,
                                     source_path=source_path,
                                     source_text=source_text,
                                     render=render, report=report,
                                     post_process=post_process,
                                     mode=mode)

    def add_dir(self, source_path, dest_path, render=None):
        render = render or self.render_cheetah
        return self.manager.add_file(type="dir", node=self.node,
                                     config=self.config, dest_path=dest_path,
                                     source_path=source_path, render=render)

    def get_one(self, name, nodes=True, systems=False):
        hits = list(self.manager.confman.find(name, nodes=nodes,
                                              systems=systems,
                                              full_match=True))
        names = (h.name for h in hits)
        if len(hits) > 1:
            raise errors.VerifyError("found more than one (%d) %r: %s" % (
                    len(hits), name, ", ".join(names)))
        elif len(hits) == 0:
            raise errors.VerifyError("did not find %r: %s" % (
                    name, ", ".join(names)))

        return hits[0]

    def get_system(self, name):
        return self.get_one(name, nodes=False, systems=True)

    def render_text(self, source_path, dest_path):
        try:
            return dest_path, file(source_path, "rb").read()
        except (IOError, OSError), error:
            raise errors.VerifyError(source_path, error)

    def add_edge(self, bucket_name, dest_node, dest_config, **kwargs):
        self.manager.get_bucket(bucket_name).append(
            dict(source_node=self.node, source_config=self.top_config,
                 dest_node=dest_node, dest_config=dest_config, **kwargs))

    def get_names(self):
        names = dict(node=self.node,
                     s=self.top_config.settings,
                     settings=self.top_config.settings,
                     system=self.node.system,
                     find=self.manager.confman.find,
                     find_config=self.manager.confman.find_config,
                     get_node=self.get_one,
                     get_system=self.get_system,
                     get_config=self.manager.confman.get_config,
                     config=self.top_config,
                     bucket=self.manager.get_bucket,
                     edge=self.add_edge,
                     plugin=self)
        return names

    def render_cheetah(self, source_path, dest_path):
        names = self.get_names()
        # TODO: template caching
        try:
            text = str(CheetahTemplate(file=source_path, searchList=[names]))
            if dest_path:
                dest_path = str(CheetahTemplate(dest_path, searchList=[names]))

            return dest_path, text
        except (Cheetah.Template.Error, SyntaxError,
                Cheetah.NameMapper.NotFound), error:
            raise errors.VerifyError("%s: %s: %s" % (
                source_path, error.__class__.__name__, error))

    def render_genshi_xml(self, source_path, dest_path):
        assert genshi, "Genshi is not installed"

        names = self.get_names()
        if dest_path:
            dest_path = str(CheetahTemplate(dest_path, searchList=[names]))

        try:
            tmpl = genshi.template.MarkupTemplate(file(source_path),
                                                  filepath=source_path)
            stream = tmpl.generate(**names)
            output = stream.render('xml')
            return dest_path, output
        except (errors.Error,
                genshi.template.TemplateError,
                IOError), error:
            raise errors.VerifyError(source_path, error)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2003-2009 Edgewall Software
# Copyright (C) 2003-2004 Jonas Borgström <jonas@edgewall.com>
# Copyright (C) 2005 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.
#
# Author: Jonas Borgström <jonas@edgewall.com>
#         Christopher Lenz <cmlenz@gmx.de>

from urllib import quote, urlencode
from trac.util.text import unicode_quote, unicode_urlencode

class Href(object):
    """Implements a callable that constructs URLs with the given base. The
    function can be called with any number of positional and keyword
    arguments which than are used to assemble the URL.

    Positional arguments are appended as individual segments to
    the path of the URL:

    >>> href = Href('/trac')
    >>> href('ticket', 540)
    '/trac/ticket/540'
    >>> href('ticket', 540, 'attachment', 'bugfix.patch')
    '/trac/ticket/540/attachment/bugfix.patch'
    >>> href('ticket', '540/attachment/bugfix.patch')
    '/trac/ticket/540/attachment/bugfix.patch'

    If a positional parameter evaluates to None, it will be skipped:

    >>> href('ticket', 540, 'attachment', None)
    '/trac/ticket/540/attachment'

    The first path segment can also be specified by calling an attribute
    of the instance, as follows:

    >>> href.ticket(540)
    '/trac/ticket/540'
    >>> href.changeset(42, format='diff')
    '/trac/changeset/42?format=diff'

    Simply calling the Href object with no arguments will return the base URL:

    >>> href()
    '/trac'

    Keyword arguments are added to the query string, unless the value is None:

    >>> href = Href('/trac')
    >>> href('timeline', format='rss')
    '/trac/timeline?format=rss'
    >>> href('timeline', format=None)
    '/trac/timeline'
    >>> href('search', q='foo bar')
    '/trac/search?q=foo+bar'

    Multiple values for one parameter are specified using a sequence (a list or
    tuple) for the parameter:

    >>> href('timeline', show=['ticket', 'wiki', 'changeset'])
    '/trac/timeline?show=ticket&show=wiki&show=changeset'

    Alternatively, query string parameters can be added by passing a dict or
    list as last positional argument:

    >>> href('timeline', {'from': '02/24/05', 'daysback': 30})
    '/trac/timeline?daysback=30&from=02%2F24%2F05'

    The usual way of quoting arguments that would otherwise be interpreted
    as Python keywords is supported too:

    >>> href('timeline', from_='02/24/05', daysback=30)
    '/trac/timeline?from=02%2F24%2F05&daysback=30'

    If the order of query string parameters should be preserved, you may also
    pass a sequence of (name, value) tuples as last positional argument:

    >>> href('query', (('group', 'component'), ('groupdesc', 1)))
    '/trac/query?group=component&groupdesc=1'

    >>> params = []
    >>> params.append(('group', 'component'))
    >>> params.append(('groupdesc', 1))
    >>> href('query', params)
    '/trac/query?group=component&groupdesc=1'

    By specifying an absolute base, the function returned will also generate
    absolute URLs:

    >>> href = Href('http://trac.edgewall.org')
    >>> href('ticket', 540)
    'http://trac.edgewall.org/ticket/540'

    >>> href = Href('https://trac.edgewall.org')
    >>> href('ticket', 540)
    'https://trac.edgewall.org/ticket/540'

    In common usage, it may improve readability to use the function-calling
    ability for the first component of the URL as mentioned earlier:

    >>> href = Href('/trac')
    >>> href.ticket(540)
    '/trac/ticket/540'
    >>> href.browser('/trunk/README.txt', format='txt')
    '/trac/browser/trunk/README.txt?format=txt'
    """

    def __init__(self, base):
        self.base = base
        self._derived = {}

    def __call__(self, *args, **kw):
        href = self.base
        if href and href[-1] == '/':
            href = href[:-1]
        params = []

        def add_param(name, value):
            if type(value) in (list, tuple):
                for i in [i for i in value if i != None]:
                    params.append((name, i))
            elif value != None:
                params.append((name, value))

        if args:
            lastp = args[-1]
            if lastp and type(lastp) is dict:
                for k,v in lastp.items():
                    add_param(k, v)
                args = args[:-1]
            elif lastp and type(lastp) in (list, tuple):
                for k,v in lastp:
                    add_param(k, v)
                args = args[:-1]

        # build the path
        path = '/'.join([unicode_quote(unicode(arg).strip('/')) for arg in args
                         if arg != None])
        if path:
            href += '/' + path

        # assemble the query string
        for k,v in kw.items():
            add_param(k.endswith('_') and k[:-1] or k, v)

        if params:
            href += '?' + unicode_urlencode(params)

        return href

    def __getattr__(self, name):
        if not self._derived.has_key(name):
            self._derived[name] = lambda *args, **kw: self(name, *args, **kw)
        return self._derived[name]

if __name__ == '__main__':
    import doctest, sys
    doctest.testmod(sys.modules[__name__])
<commit_msg>Href can be given a dictionary, a list or a tuple of parameters in its last positional argument, but this should also work if the collection is empty.<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2003-2009 Edgewall Software
# Copyright (C) 2003-2004 Jonas Borgström <jonas@edgewall.com>
# Copyright (C) 2005 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.
#
# Author: Jonas Borgström <jonas@edgewall.com>
#         Christopher Lenz <cmlenz@gmx.de>

from urllib import quote, urlencode
from trac.util.text import unicode_quote, unicode_urlencode

class Href(object):
    """Implements a callable that constructs URLs with the given base. The
    function can be called with any number of positional and keyword
    arguments which than are used to assemble the URL.

    Positional arguments are appended as individual segments to
    the path of the URL:

    >>> href = Href('/trac')
    >>> href('ticket', 540)
    '/trac/ticket/540'
    >>> href('ticket', 540, 'attachment', 'bugfix.patch')
    '/trac/ticket/540/attachment/bugfix.patch'
    >>> href('ticket', '540/attachment/bugfix.patch')
    '/trac/ticket/540/attachment/bugfix.patch'

    If a positional parameter evaluates to None, it will be skipped:

    >>> href('ticket', 540, 'attachment', None)
    '/trac/ticket/540/attachment'

    The first path segment can also be specified by calling an attribute
    of the instance, as follows:

    >>> href.ticket(540)
    '/trac/ticket/540'
    >>> href.changeset(42, format='diff')
    '/trac/changeset/42?format=diff'

    Simply calling the Href object with no arguments will return the base URL:

    >>> href()
    '/trac'

    Keyword arguments are added to the query string, unless the value is None:

    >>> href = Href('/trac')
    >>> href('timeline', format='rss')
    '/trac/timeline?format=rss'
    >>> href('timeline', format=None)
    '/trac/timeline'
    >>> href('search', q='foo bar')
    '/trac/search?q=foo+bar'

    Multiple values for one parameter are specified using a sequence (a list or
    tuple) for the parameter:

    >>> href('timeline', show=['ticket', 'wiki', 'changeset'])
    '/trac/timeline?show=ticket&show=wiki&show=changeset'

    Alternatively, query string parameters can be added by passing a dict or
    list as last positional argument:

    >>> href('timeline', {'from': '02/24/05', 'daysback': 30})
    '/trac/timeline?daysback=30&from=02%2F24%2F05'
    >>> href('timeline', {})
    '/trac/timeline'
    >>> href('timeline', [('from', '02/24/05')])
    '/trac/timeline?from=02%2F24%2F05'
    >>> href('timeline', ()) == href('timeline', []) == href('timeline', {})
    True

    The usual way of quoting arguments that would otherwise be interpreted
    as Python keywords is supported too:

    >>> href('timeline', from_='02/24/05', daysback=30)
    '/trac/timeline?from=02%2F24%2F05&daysback=30'

    If the order of query string parameters should be preserved, you may also
    pass a sequence of (name, value) tuples as last positional argument:

    >>> href('query', (('group', 'component'), ('groupdesc', 1)))
    '/trac/query?group=component&groupdesc=1'

    >>> params = []
    >>> params.append(('group', 'component'))
    >>> params.append(('groupdesc', 1))
    >>> href('query', params)
    '/trac/query?group=component&groupdesc=1'

    By specifying an absolute base, the function returned will also generate
    absolute URLs:

    >>> href = Href('http://trac.edgewall.org')
    >>> href('ticket', 540)
    'http://trac.edgewall.org/ticket/540'

    >>> href = Href('https://trac.edgewall.org')
    >>> href('ticket', 540)
    'https://trac.edgewall.org/ticket/540'

    In common usage, it may improve readability to use the function-calling
    ability for the first component of the URL as mentioned earlier:

    >>> href = Href('/trac')
    >>> href.ticket(540)
    '/trac/ticket/540'
    >>> href.browser('/trunk/README.txt', format='txt')
    '/trac/browser/trunk/README.txt?format=txt'
    """

    def __init__(self, base):
        self.base = base
        self._derived = {}

    def __call__(self, *args, **kw):
        href = self.base
        if href and href[-1] == '/':
            href = href[:-1]
        params = []

        def add_param(name, value):
            if type(value) in (list, tuple):
                for i in [i for i in value if i != None]:
                    params.append((name, i))
            elif value != None:
                params.append((name, value))

        if args:
            lastp = args[-1]
            if type(lastp) is dict:
                for k,v in lastp.items():
                    add_param(k, v)
                args = args[:-1]
            elif type(lastp) in (list, tuple):
                for k,v in lastp:
                    add_param(k, v)
                args = args[:-1]

        # build the path
        path = '/'.join([unicode_quote(unicode(arg).strip('/')) for arg in args
                         if arg != None])
        if path:
            href += '/' + path

        # assemble the query string
        for k,v in kw.items():
            add_param(k.endswith('_') and k[:-1] or k, v)

        if params:
            href += '?' + unicode_urlencode(params)

        return href

    def __getattr__(self, name):
        if not self._derived.has_key(name):
            self._derived[name] = lambda *args, **kw: self(name, *args, **kw)
        return self._derived[name]

if __name__ == '__main__':
    import doctest, sys
    doctest.testmod(sys.modules[__name__])
<|endoftext|>


<commit_before># -*- coding: utf8 -*-
# This file is part of PyBossa-links.
#
# Copyright (C) 2013 Daniel Lombraña González
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
import json
from base import Test, db
from links import utils, model

class TestUtils(Test):
    def test_valid_link_url(self):
        """Test valid_link_url method"""
        urls = [('http:', False),
                ('htt', False),
                ('ftp://example.com/img.jpg', False),
                ('http://', False),
                ('http://example.com/', True),
                ('http://example.com/img.jpg', True),
                ('data://', False)]
        err_msg = "URL not validated correctly"
        for u in urls:
            assert utils.valid_link_url(u[0]) == u[1], err_msg

    def test_allow_post(self):
        """Test allow_post method"""
        ip = "127.0.0.1"
        # Test first MAX_HITS
        for i in range(0, 5):
            err_msg = "%s st POST should be allowed" % i
            assert utils.allow_post(db, ip, hour=10, max_hits=5) is True, err_msg
        for i in range(0, 5):
            err_msg = "%s st POST should NOT be allowed" % i
            assert utils.allow_post(db, ip, hour=10, max_hits=5) is False, err_msg

        # Test if reset works
        err_msg = "Hits should be reset"
        assert utils.allow_post(db, ip, hour=0, max_hits=5) is True, err_msg
        # And it should be possible to post again
        for i in range(0, 4):
            err_msg = "%s st POST should be allowed" % i
            assert utils.allow_post(db, ip, hour=10, max_hits=5) is True, err_msg
        for i in range(0, 5):
            err_msg = "%s st POST should NOT be allowed" % i
            assert utils.allow_post(db, ip, hour=10, max_hits=5) is False, err_msg

    def test_handle_error(self):
        """Test handle_error method"""
        errors = [('invalid_url', 'Invalid URL', 415),
                  ('url_missing', 'url arg is missing', 415),
                  ('too_many_args', 'Too many arguments. url and project_slug are the only allowed arguments', 415),
                  ('rate_limit', 'Rate limit reached', 415),
                  ('project_slug_missing', 'Project slug arg is missing', 415),
                  ('project_not_found', 'Project not found', 404),
                  ('server_error', 'Server Error', 500),
                  ('unknown', 'Server Error', 500)]
        for e in errors:
            res = utils.handle_error(e[0])
            err = json.loads(res.response[0])
            assert err['status'] == 'failed', 'Member status != failed'
            assert err['error'] == e[1], "Wrong error msg for %s: %s" % (e[0],
                                                                         err['error'])

    def test_save_url(self):
        """Test save_url method"""
        self.project_fixtures()
        project = db.session.query(model.Project).first()
        pybossa = dict(endpoint='http://pybossa.com', api_key='tester')
        form = dict(url='http://daniellombrana.es', project_slug=project.slug)

        # Save without taking care of Throttling
        res = utils.save_url("127.0.0.1", form, pybossa, hour=0, max_hits=2)
        output = json.loads(res.response[0])
        err_msg = "URL should be saved"
        assert output['status'] == 'saved', err_msg
        assert output['new'] is True, err_msg

        # The same URL should be not saved, but reported
        res = utils.save_url("127.0.0.1", form, pybossa, hour=1000, max_hits=2)
        output = json.loads(res.response[0])
        err_msg = "URL.new should be False"
        assert output['status'] == 'saved', err_msg
        assert output['new'] is False, err_msg

        # With Throttling
        res = utils.save_url("127.0.0.1", form, pybossa, hour=1000, max_hits=2)
        output = json.loads(res.response[0])
        print output
        err_msg = "It should not allow to save the link"
        assert output['status'] == 'failed', err_msg
<commit_msg>Unit test for validate_post_args<commit_after># -*- coding: utf8 -*-
# This file is part of PyBossa-links.
#
# Copyright (C) 2013 Daniel Lombraña González
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
import json
from base import Test, db
from links import utils, model

class TestUtils(Test):
    def test_valid_link_url(self):
        """Test valid_link_url method"""
        urls = [('http:', False),
                ('htt', False),
                ('ftp://example.com/img.jpg', False),
                ('http://', False),
                ('http://example.com/', True),
                ('http://example.com/img.jpg', True),
                ('data://', False)]
        err_msg = "URL not validated correctly"
        for u in urls:
            assert utils.valid_link_url(u[0]) == u[1], err_msg

    def test_allow_post(self):
        """Test allow_post method"""
        ip = "127.0.0.1"
        # Test first MAX_HITS
        for i in range(0, 5):
            err_msg = "%s st POST should be allowed" % i
            assert utils.allow_post(db, ip, hour=10, max_hits=5) is True, err_msg
        for i in range(0, 5):
            err_msg = "%s st POST should NOT be allowed" % i
            assert utils.allow_post(db, ip, hour=10, max_hits=5) is False, err_msg

        # Test if reset works
        err_msg = "Hits should be reset"
        assert utils.allow_post(db, ip, hour=0, max_hits=5) is True, err_msg
        # And it should be possible to post again
        for i in range(0, 4):
            err_msg = "%s st POST should be allowed" % i
            assert utils.allow_post(db, ip, hour=10, max_hits=5) is True, err_msg
        for i in range(0, 5):
            err_msg = "%s st POST should NOT be allowed" % i
            assert utils.allow_post(db, ip, hour=10, max_hits=5) is False, err_msg

    def test_handle_error(self):
        """Test handle_error method"""
        errors = [('invalid_url', 'Invalid URL', 415),
                  ('url_missing', 'url arg is missing', 415),
                  ('too_many_args', 'Too many arguments. url and project_slug are the only allowed arguments', 415),
                  ('rate_limit', 'Rate limit reached', 415),
                  ('project_slug_missing', 'project_slug arg is missing', 415),
                  ('project_not_found', 'Project not found', 404),
                  ('server_error', 'Server Error', 500),
                  ('unknown', 'Server Error', 500)]
        for e in errors:
            res = utils.handle_error(e[0])
            err = json.loads(res.response[0])
            assert err['status'] == 'failed', 'Member status != failed'
            assert err['error'] == e[1], "Wrong error msg for %s: %s" % (e[0],
                                                                         err['error'])

    def test_validate_post_args(self):
        """Test validate_post_args method"""
        form = dict(url='http://algo', project_slug='slug')
        assert utils.validate_post_args(form) is None, "There should not be an error"

        form = dict(project_slug='slug')
        res = utils.validate_post_args(form)
        output = json.loads(res.response[0])
        assert output['error'] == 'url arg is missing', output

        form = dict(url='http://algo')
        res = utils.validate_post_args(form)
        output = json.loads(res.response[0])
        assert output['error'] == 'project_slug arg is missing', output

    def test_save_url(self):
        """Test save_url method"""
        self.project_fixtures()
        project = db.session.query(model.Project).first()
        pybossa = dict(endpoint='http://pybossa.com', api_key='tester')
        form = dict(url='http://daniellombrana.es', project_slug=project.slug)

        # Save without taking care of Throttling
        res = utils.save_url("127.0.0.1", form, pybossa, hour=0, max_hits=2)
        output = json.loads(res.response[0])
        err_msg = "URL should be saved"
        assert output['status'] == 'saved', err_msg
        assert output['new'] is True, err_msg

        # The same URL should be not saved, but reported
        res = utils.save_url("127.0.0.1", form, pybossa, hour=1000, max_hits=2)
        output = json.loads(res.response[0])
        err_msg = "URL.new should be False"
        assert output['status'] == 'saved', err_msg
        assert output['new'] is False, err_msg

        # With Throttling
        res = utils.save_url("127.0.0.1", form, pybossa, hour=1000, max_hits=2)
        output = json.loads(res.response[0])
        print output
        err_msg = "It should not allow to save the link"
        assert output['status'] == 'failed', err_msg
<|endoftext|>