<commit_before>"""
DVC
----
Make your data science projects reproducible and shareable.
"""
import os

VERSION_BASE = '0.15.3'
__version__ = VERSION_BASE

PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
HOMEPATH = os.path.dirname(PACKAGEPATH)
VERSIONPATH = os.path.join(PACKAGEPATH, 'version.py')

if os.path.exists(os.path.join(HOMEPATH, 'setup.py')):
    # dvc is run directly from source without installation or
    # __version__ is called from setup.py
    if os.getenv('APPVEYOR_REPO_TAG', '').lower() != 'true' \
       and os.getenv('TRAVIS_TAG', '') == '':
        # Dynamically update version
        try:
            import git
            repo = git.Repo(HOMEPATH)
            sha = repo.head.object.hexsha
            short_sha = repo.git.rev_parse(sha, short=6)
            dirty = '.mod' if repo.is_dirty() else ''
            __version__ = '{}+{}{}'.format(__version__, short_sha, dirty)

            # Write a helper file, that will be installed with the package
            # and will provide a true version of the installed dvc
            with open(VERSIONPATH, 'w+') as fd:
                fd.write('# AUTOGENERATED by dvc/__init__.py\n')
                fd.write('version = "{}"\n'.format(__version__))
        except Exception:
            pass
    else:
        # Remove version.py so that it doesn't get into the release
        if os.path.exists(VERSIONPATH):
            os.unlink(VERSIONPATH)
else:
    # dvc was installed with pip or something. Hopefully we have our
    # auto-generated version.py to help us provide a true version
    try:
        from dvc.version import version
        __version__ = version
    except Exception:
        pass

VERSION = __version__
<commit_msg>dvc: bump to 0.16.0<commit_after>"""
DVC
----
Make your data science projects reproducible and shareable.
"""
import os

VERSION_BASE = '0.16.0'
__version__ = VERSION_BASE

PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
HOMEPATH = os.path.dirname(PACKAGEPATH)
VERSIONPATH = os.path.join(PACKAGEPATH, 'version.py')

if os.path.exists(os.path.join(HOMEPATH, 'setup.py')):
    # dvc is run directly from source without installation or
    # __version__ is called from setup.py
    if os.getenv('APPVEYOR_REPO_TAG', '').lower() != 'true' \
       and os.getenv('TRAVIS_TAG', '') == '':
        # Dynamically update version
        try:
            import git
            repo = git.Repo(HOMEPATH)
            sha = repo.head.object.hexsha
            short_sha = repo.git.rev_parse(sha, short=6)
            dirty = '.mod' if repo.is_dirty() else ''
            __version__ = '{}+{}{}'.format(__version__, short_sha, dirty)

            # Write a helper file, that will be installed with the package
            # and will provide a true version of the installed dvc
            with open(VERSIONPATH, 'w+') as fd:
                fd.write('# AUTOGENERATED by dvc/__init__.py\n')
                fd.write('version = "{}"\n'.format(__version__))
        except Exception:
            pass
    else:
        # Remove version.py so that it doesn't get into the release
        if os.path.exists(VERSIONPATH):
            os.unlink(VERSIONPATH)
else:
    # dvc was installed with pip or something. Hopefully we have our
    # auto-generated version.py to help us provide a true version
    try:
        from dvc.version import version
        __version__ = version
    except Exception:
        pass

VERSION = __version__
<|endoftext|>


<commit_before># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Top-level presubmit script for buildbot.

See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts for
details on the presubmit API built into gcl.
"""

import sys

def CommonChecks(input_api, output_api):
  def join(*args):
    return input_api.os_path.join(input_api.PresubmitLocalPath(), *args)

  black_list = list(input_api.DEFAULT_BLACK_LIST) + [
      r'.*slave/.*/build.*/.*',
      r'.*depot_tools/.*',
      r'.*scripts/release/.*',
      r'.*scripts/slave/recipe_modules/.*',
      r'.*scripts/gsd_generate_index/.*',
      r'.*masters/.*/templates/.*\.html$',
      r'.*masters/.*/templates/.*\.css$',
      r'.*masters/.*/public_html/.*\.html$',
      r'.*masters/.*/public_html/.*\.css$',
  ]
  tests = []
  sys_path_backup = sys.path
  try:
    sys.path = [
        join('third_party'),
        join('third_party', 'buildbot_8_4p1'),
        join('third_party', 'buildbot_slave_8_4'),
        join('third_party', 'coverage-3.7.1'),
        join('third_party', 'decorator_3_3_1'),
        join('third_party', 'jinja2'),
        join('third_party', 'mock-1.0.1'),
        join('third_party', 'sqlalchemy_0_7_1'),
        join('third_party', 'sqlalchemy_migrate_0_7_1'),
        join('third_party', 'tempita_0_5'),
        join('third_party', 'twisted_10_2'),
        join('scripts'),
        # Initially, a separate run was done for unit tests but now that
        # pylint is fetched in memory with setuptools, it seems it caches
        # sys.path so modifications to sys.path aren't kept.
        join('scripts', 'master', 'unittests'),
        join('scripts', 'slave', 'unittests'),
        join('scripts', 'tools', 'deps2git'),
        join('site_config'),
        join('tests'),
    ] + sys.path

    disabled_warnings = [
      'C0301',  # Line too long (NN/80)
      'C0321',  # More than one statement on a single line
      'W0613',  # Unused argument
    ]
    tests.extend(input_api.canned_checks.GetPylint(
        input_api,
        output_api,
        black_list=black_list,
        disabled_warnings=disabled_warnings))
  finally:
    sys.path = sys_path_backup

  whitelist = [r'.+_test\.py$']
  tests.extend(input_api.canned_checks.GetUnitTestsInDirectory(
      input_api, output_api, 'tests', whitelist=whitelist))
  tests.extend(input_api.canned_checks.GetUnitTestsInDirectory(
      input_api,
      output_api,
      input_api.os_path.join('scripts', 'master', 'unittests'),
      whitelist))
  tests.extend(input_api.canned_checks.GetUnitTestsInDirectory(
      input_api,
      output_api,
      input_api.os_path.join('scripts', 'slave', 'unittests'),
      whitelist))
  tests.extend(input_api.canned_checks.GetUnitTestsInDirectory(
      input_api,
      output_api,
      input_api.os_path.join('scripts', 'common', 'unittests'),
      whitelist))
  tests.extend(input_api.canned_checks.GetUnitTestsInDirectory(
      input_api,
      output_api,
      input_api.os_path.join('scripts', 'tools', 'unittests'),
      whitelist))
  tests.extend(input_api.canned_checks.GetUnitTestsInDirectory(
      input_api,
      output_api,
      input_api.os_path.join('scripts', 'tools', 'blink_roller'),
      whitelist))

  try:
    sys.path = [join('scripts', 'common')] + sys.path
    import master_cfg_utils  # pylint: disable=F0401
    # Run the tests.
    with master_cfg_utils.TemporaryMasterPasswords():
      output = input_api.RunTests(tests)

    output.extend(input_api.canned_checks.PanProjectChecks(
      input_api, output_api, excluded_paths=black_list))
    return output
  finally:
    sys.path = sys_path_backup

def BuildInternalCheck(output, input_api, output_api):
  if output:
    b_i = input_api.os_path.join(input_api.PresubmitLocalPath(), '..',
                                 'build_internal')
    if input_api.os_path.exists(b_i):
      return [output_api.PresubmitNotifyResult(
          'You have a build_internal checkout. '
          'Updating it may resolve some issues.')]
  return []

def CheckChangeOnUpload(input_api, output_api):
  output = CommonChecks(input_api, output_api)
  output.extend(BuildInternalCheck(output, input_api, output_api))
  return output

def CheckChangeOnCommit(input_api, output_api):
  return CheckChangeOnUpload(input_api, output_api)
<commit_msg>PRESUBMIT: Exclude isolate folders from pylint check.<commit_after># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Top-level presubmit script for buildbot.

See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts for
details on the presubmit API built into gcl.
"""

import sys

def CommonChecks(input_api, output_api):
  def join(*args):
    return input_api.os_path.join(input_api.PresubmitLocalPath(), *args)

  black_list = list(input_api.DEFAULT_BLACK_LIST) + [
      r'.*slave/.*/build.*/.*',
      r'.*slave/.*/isolate.*/.*',
      r'.*depot_tools/.*',
      r'.*scripts/release/.*',
      r'.*scripts/slave/recipe_modules/.*',
      r'.*scripts/gsd_generate_index/.*',
      r'.*masters/.*/templates/.*\.html$',
      r'.*masters/.*/templates/.*\.css$',
      r'.*masters/.*/public_html/.*\.html$',
      r'.*masters/.*/public_html/.*\.css$',
  ]
  tests = []
  sys_path_backup = sys.path
  try:
    sys.path = [
        join('third_party'),
        join('third_party', 'buildbot_8_4p1'),
        join('third_party', 'buildbot_slave_8_4'),
        join('third_party', 'coverage-3.7.1'),
        join('third_party', 'decorator_3_3_1'),
        join('third_party', 'jinja2'),
        join('third_party', 'mock-1.0.1'),
        join('third_party', 'sqlalchemy_0_7_1'),
        join('third_party', 'sqlalchemy_migrate_0_7_1'),
        join('third_party', 'tempita_0_5'),
        join('third_party', 'twisted_10_2'),
        join('scripts'),
        # Initially, a separate run was done for unit tests but now that
        # pylint is fetched in memory with setuptools, it seems it caches
        # sys.path so modifications to sys.path aren't kept.
        join('scripts', 'master', 'unittests'),
        join('scripts', 'slave', 'unittests'),
        join('scripts', 'tools', 'deps2git'),
        join('site_config'),
        join('tests'),
    ] + sys.path

    disabled_warnings = [
      'C0301',  # Line too long (NN/80)
      'C0321',  # More than one statement on a single line
      'W0613',  # Unused argument
    ]
    tests.extend(input_api.canned_checks.GetPylint(
        input_api,
        output_api,
        black_list=black_list,
        disabled_warnings=disabled_warnings))
  finally:
    sys.path = sys_path_backup

  whitelist = [r'.+_test\.py$']
  tests.extend(input_api.canned_checks.GetUnitTestsInDirectory(
      input_api, output_api, 'tests', whitelist=whitelist))
  tests.extend(input_api.canned_checks.GetUnitTestsInDirectory(
      input_api,
      output_api,
      input_api.os_path.join('scripts', 'master', 'unittests'),
      whitelist))
  tests.extend(input_api.canned_checks.GetUnitTestsInDirectory(
      input_api,
      output_api,
      input_api.os_path.join('scripts', 'slave', 'unittests'),
      whitelist))
  tests.extend(input_api.canned_checks.GetUnitTestsInDirectory(
      input_api,
      output_api,
      input_api.os_path.join('scripts', 'common', 'unittests'),
      whitelist))
  tests.extend(input_api.canned_checks.GetUnitTestsInDirectory(
      input_api,
      output_api,
      input_api.os_path.join('scripts', 'tools', 'unittests'),
      whitelist))
  tests.extend(input_api.canned_checks.GetUnitTestsInDirectory(
      input_api,
      output_api,
      input_api.os_path.join('scripts', 'tools', 'blink_roller'),
      whitelist))

  try:
    sys.path = [join('scripts', 'common')] + sys.path
    import master_cfg_utils  # pylint: disable=F0401
    # Run the tests.
    with master_cfg_utils.TemporaryMasterPasswords():
      output = input_api.RunTests(tests)

    output.extend(input_api.canned_checks.PanProjectChecks(
      input_api, output_api, excluded_paths=black_list))
    return output
  finally:
    sys.path = sys_path_backup

def BuildInternalCheck(output, input_api, output_api):
  if output:
    b_i = input_api.os_path.join(input_api.PresubmitLocalPath(), '..',
                                 'build_internal')
    if input_api.os_path.exists(b_i):
      return [output_api.PresubmitNotifyResult(
          'You have a build_internal checkout. '
          'Updating it may resolve some issues.')]
  return []

def CheckChangeOnUpload(input_api, output_api):
  output = CommonChecks(input_api, output_api)
  output.extend(BuildInternalCheck(output, input_api, output_api))
  return output

def CheckChangeOnCommit(input_api, output_api):
  return CheckChangeOnUpload(input_api, output_api)
<|endoftext|>


<commit_before>"""Manages progress bars for dvc repo."""
from __future__ import print_function
import logging
from tqdm import tqdm
from copy import deepcopy
from concurrent.futures import ThreadPoolExecutor

class TqdmThreadPoolExecutor(ThreadPoolExecutor):
    """
    Ensure worker progressbars are cleared away properly.
    """

    def __enter__(self):
        """
        Creates a blank initial dummy progress bar if needed so that workers
        are forced to create "nested" bars.
        """
        blank_bar = Tqdm(bar_format="Multi-Threaded:", leave=False)
        if blank_bar.pos > 0:
            # already nested - don't need a placeholder bar
            blank_bar.close()
        self.bar = blank_bar
        return super(TqdmThreadPoolExecutor, self).__enter__()

    def __exit__(self, *a, **k):
        super(TqdmThreadPoolExecutor, self).__exit__(*a, **k)
        self.bar.close()

class Tqdm(tqdm):
    """
    maximum-compatibility tqdm-based progressbars
    """

    def __init__(
        self,
        iterable=None,
        disable=None,
        bytes=False,  # pylint: disable=W0622
        desc_truncate=None,
        leave=None,
        bar_format=(
            "{percentage:3.0f}%|{bar:10}|{desc} {bar:-10b}{n}/{total}"
            " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
        ),
        **kwargs
    ):
        """
        bytes   : shortcut for
            `unit='B', unit_scale=True, unit_divisor=1024, miniters=1`
        desc_truncate  : like `desc` but will truncate to 10 chars
        kwargs  : anything accepted by `tqdm.tqdm()`
        """
        kwargs = deepcopy(kwargs)
        kwargs.setdefault("unit_scale", True)
        if bytes:
            for k, v in dict(
                unit="B", unit_scale=True, unit_divisor=1024, miniters=1
            ).items():
                kwargs.setdefault(k, v)
        if desc_truncate is not None:
            kwargs.setdefault("desc", self.truncate(desc_truncate))
        if disable is None:
            disable = (
                logging.getLogger(__name__).getEffectiveLevel()
                >= logging.CRITICAL
            )
        super(Tqdm, self).__init__(
            iterable=iterable,
            disable=disable,
            leave=leave,
            bar_format=bar_format,
            **kwargs
        )

    def update_desc(self, desc, n=1, truncate=True):
        """
        Calls `set_description_str(truncate(desc))` and `update(n)`
        """
        self.set_description_str(
            self.truncate(desc) if truncate else desc, refresh=False
        )
        self.update(n)

    def update_to(self, current, total=None):
        if total:
            self.total = total  # pylint: disable=W0613,W0201
        self.update(current - self.n)

    @classmethod
    def truncate(cls, s, max_len=25, end=True, fill="..."):
        """
        Guarantee len(output) < max_lenself.
        >>> truncate("hello", 4)
        '...o'
        """
        if len(s) <= max_len:
            return s
        if len(fill) > max_len:
            return fill[-max_len:] if end else fill[:max_len]
        i = max_len - len(fill)
        return (fill + s[-i:]) if end else (s[:i] + fill)
<commit_msg>fix unknown total progress<commit_after>"""Manages progress bars for dvc repo."""
from __future__ import print_function
import logging
from tqdm import tqdm
from copy import deepcopy
from concurrent.futures import ThreadPoolExecutor

class TqdmThreadPoolExecutor(ThreadPoolExecutor):
    """
    Ensure worker progressbars are cleared away properly.
    """

    def __enter__(self):
        """
        Creates a blank initial dummy progress bar if needed so that workers
        are forced to create "nested" bars.
        """
        blank_bar = Tqdm(bar_format="Multi-Threaded:", leave=False)
        if blank_bar.pos > 0:
            # already nested - don't need a placeholder bar
            blank_bar.close()
        self.bar = blank_bar
        return super(TqdmThreadPoolExecutor, self).__enter__()

    def __exit__(self, *a, **k):
        super(TqdmThreadPoolExecutor, self).__exit__(*a, **k)
        self.bar.close()

class Tqdm(tqdm):
    """
    maximum-compatibility tqdm-based progressbars
    """
    BAR_FMT_DEFAULT = (
        "{percentage:3.0f}%|{bar:10}|{desc} {bar:-10b}{n}/{total}"
        " [{elapsed}<{remaining}, {rate_fmt:>11}{postfix}]"
    )
    BAR_FMT_NOTOTAL = "{desc} {bar:b}{n} [{elapsed}<??:??, {rate_fmt:>11}{postfix}]"

    def __init__(
        self,
        iterable=None,
        disable=None,
        bytes=False,  # pylint: disable=W0622
        desc_truncate=None,
        leave=None,
        bar_format=None,
        **kwargs
    ):
        """
        bytes   : shortcut for
            `unit='B', unit_scale=True, unit_divisor=1024, miniters=1`
        desc_truncate  : like `desc` but will truncate to 10 chars
        kwargs  : anything accepted by `tqdm.tqdm()`
        """
        kwargs = deepcopy(kwargs)
        kwargs.setdefault("unit_scale", True)
        if bytes:
            for k, v in dict(
                unit="B", unit_scale=True, unit_divisor=1024, miniters=1
            ).items():
                kwargs.setdefault(k, v)
        if desc_truncate is not None:
            kwargs.setdefault("desc", self.truncate(desc_truncate))
        if disable is None:
            disable = (
                logging.getLogger(__name__).getEffectiveLevel()
                >= logging.CRITICAL
            )
        if bar_format is None:
            if kwargs.get('total', getattr(iterable, '__len__', None)) is None:
                bar_format = self.BAR_FMT_NOTOTAL
            else:
                bar_format = self.BAR_FMT_DEFAULT
        super(Tqdm, self).__init__(
            iterable=iterable,
            disable=disable,
            leave=leave,
            bar_format=bar_format,
            **kwargs
        )

    def update_desc(self, desc, n=1, truncate=True):
        """
        Calls `set_description_str(truncate(desc))` and `update(n)`
        """
        self.set_description_str(
            self.truncate(desc) if truncate else desc, refresh=False
        )
        self.update(n)

    def update_to(self, current, total=None):
        if total:
            self.total = total  # pylint: disable=W0613,W0201
        self.update(current - self.n)

    @classmethod
    def truncate(cls, s, max_len=25, end=True, fill="..."):
        """
        Guarantee len(output) < max_lenself.
        >>> truncate("hello", 4)
        '...o'
        """
        if len(s) <= max_len:
            return s
        if len(fill) > max_len:
            return fill[-max_len:] if end else fill[:max_len]
        i = max_len - len(fill)
        return (fill + s[-i:]) if end else (s[:i] + fill)
<|endoftext|>


<commit_before>import logging
import os

import colorama

from . import locked
from dvc.exceptions import RecursiveAddingWhileUsingFilename
from dvc.output.base import OutputDoesNotExistError
from dvc.progress import Tqdm
from dvc.repo.scm_context import scm_context
from dvc.stage import Stage
from dvc.utils import LARGE_DIR_SIZE

logger = logging.getLogger(__name__)

@locked
@scm_context
def add(repo, targets, recursive=False, no_commit=False, fname=None):
    if recursive and fname:
        raise RecursiveAddingWhileUsingFilename()

    if isinstance(targets, str):
        targets = [targets]

    stages_list = []
    num_targets = len(targets)
    with Tqdm(total=num_targets, desc="Add", unit="file", leave=True) as pbar:
        if num_targets == 1:
            # clear unneeded top-level progress bar for single target
            pbar.bar_format = "Adding..."
            pbar.refresh()
        for target in targets:
            sub_targets = _find_all_targets(repo, target, recursive)
            pbar.total += len(sub_targets) - 1

            if os.path.isdir(target) and len(sub_targets) > LARGE_DIR_SIZE:
                logger.warning(
                    "You are adding a large directory '{target}' recursively,"
                    " consider tracking it as a whole instead.\n"
                    "{purple}HINT:{nc} Remove the generated DVC-file and then"
                    " run `{cyan}dvc add {target}{nc}`".format(
                        purple=colorama.Fore.MAGENTA,
                        cyan=colorama.Fore.CYAN,
                        nc=colorama.Style.RESET_ALL,
                        target=target,
                    )
                )

            stages = _create_stages(repo, sub_targets, fname, pbar=pbar)

            repo.check_modified_graph(stages)

            with Tqdm(
                total=len(stages),
                desc="Processing",
                unit="file",
                disable=True if len(stages) == 1 else None,
            ) as pbar_stages:
                for stage in stages:
                    try:
                        stage.save()
                    except OutputDoesNotExistError:
                        pbar.n -= 1
                        raise

                    if not no_commit:
                        stage.commit()

                    stage.dump()
                    pbar_stages.update()

            stages_list += stages

        if num_targets == 1:  # restore bar format for stats
            pbar.bar_format = pbar.BAR_FMT_DEFAULT

    return stages_list

def _find_all_targets(repo, target, recursive):
    if os.path.isdir(target) and recursive:
        return [
            fname
            for fname in Tqdm(
                repo.tree.walk_files(target),
                desc="Recursing " + target,
                bar_format=Tqdm.BAR_FMT_NOTOTAL,
                unit="file",
            )
            if not repo.is_dvc_internal(fname)
            if not Stage.is_stage_file(fname)
            if not repo.scm.belongs_to_scm(fname)
            if not repo.scm.is_tracked(fname)
        ]
    return [target]

def _create_stages(repo, targets, fname, pbar=None):
    stages = []

    for out in Tqdm(
        targets,
        desc="Creating stages",
        disable=True if len(targets) < LARGE_DIR_SIZE else None,
        unit="file",
    ):
        stage = Stage.create(repo, outs=[out], add=True, fname=fname)

        if not stage:
            if pbar is not None:
                pbar.total -= 1
            continue

        stages.append(stage)
        if pbar is not None:
            pbar.update_desc(out)

    return stages
<commit_msg>minor import tidy<commit_after>import logging
import os

import colorama

from . import locked
from ..exceptions import RecursiveAddingWhileUsingFilename
from ..output.base import OutputDoesNotExistError
from ..progress import Tqdm
from ..repo.scm_context import scm_context
from ..stage import Stage
from ..utils import LARGE_DIR_SIZE

logger = logging.getLogger(__name__)

@locked
@scm_context
def add(repo, targets, recursive=False, no_commit=False, fname=None):
    if recursive and fname:
        raise RecursiveAddingWhileUsingFilename()

    if isinstance(targets, str):
        targets = [targets]

    stages_list = []
    num_targets = len(targets)
    with Tqdm(total=num_targets, desc="Add", unit="file", leave=True) as pbar:
        if num_targets == 1:
            # clear unneeded top-level progress bar for single target
            pbar.bar_format = "Adding..."
            pbar.refresh()
        for target in targets:
            sub_targets = _find_all_targets(repo, target, recursive)
            pbar.total += len(sub_targets) - 1

            if os.path.isdir(target) and len(sub_targets) > LARGE_DIR_SIZE:
                logger.warning(
                    "You are adding a large directory '{target}' recursively,"
                    " consider tracking it as a whole instead.\n"
                    "{purple}HINT:{nc} Remove the generated DVC-file and then"
                    " run `{cyan}dvc add {target}{nc}`".format(
                        purple=colorama.Fore.MAGENTA,
                        cyan=colorama.Fore.CYAN,
                        nc=colorama.Style.RESET_ALL,
                        target=target,
                    )
                )

            stages = _create_stages(repo, sub_targets, fname, pbar=pbar)

            repo.check_modified_graph(stages)

            with Tqdm(
                total=len(stages),
                desc="Processing",
                unit="file",
                disable=True if len(stages) == 1 else None,
            ) as pbar_stages:
                for stage in stages:
                    try:
                        stage.save()
                    except OutputDoesNotExistError:
                        pbar.n -= 1
                        raise

                    if not no_commit:
                        stage.commit()

                    stage.dump()
                    pbar_stages.update()

            stages_list += stages

        if num_targets == 1:  # restore bar format for stats
            pbar.bar_format = pbar.BAR_FMT_DEFAULT

    return stages_list

def _find_all_targets(repo, target, recursive):
    if os.path.isdir(target) and recursive:
        return [
            fname
            for fname in Tqdm(
                repo.tree.walk_files(target),
                desc="Recursing " + target,
                bar_format=Tqdm.BAR_FMT_NOTOTAL,
                unit="file",
            )
            if not repo.is_dvc_internal(fname)
            if not Stage.is_stage_file(fname)
            if not repo.scm.belongs_to_scm(fname)
            if not repo.scm.is_tracked(fname)
        ]
    return [target]

def _create_stages(repo, targets, fname, pbar=None):
    stages = []

    for out in Tqdm(
        targets,
        desc="Creating stages",
        disable=True if len(targets) < LARGE_DIR_SIZE else None,
        unit="file",
    ):
        stage = Stage.create(repo, outs=[out], add=True, fname=fname)

        if not stage:
            if pbar is not None:
                pbar.total -= 1
            continue

        stages.append(stage)
        if pbar is not None:
            pbar.update_desc(out)

    return stages
<|endoftext|>


<commit_before>import os
import re
import requests

from nbformat import current as nbformat
import nbconvert.nbconvert as nbconvert

from flask import Flask , request, render_template
from flask import redirect, abort, Response

from statistics import Stats
from sqlalchemy import create_engine

from flask.ext.sqlalchemy import SQLAlchemy

app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = os.environ['DATABASE_URL']

#engine = create_engine('sqlite:///foo.db', echo=False)
engine = create_engine(app.config['SQLALCHEMY_DATABASE_URI'], echo=False)
stats = Stats(engine)

try :
    import pylibmc
    mc = pylibmc.Client(["127.0.0.1"], binary=True,
                    behaviors={"tcp_nodelay": True,
                                "ketama": True})

    def cachedfirstparam(function):

        def wrapper(*args, **kw):
            if len(args)+len(kw) != 1:
               return function(*args, **kw)
            else :
                key = kw.values()[0] if kw else args[0]
                skey = str(key)+str(function.__name__)
                mcv = mc.get(skey)
                #mcv = None
                if mcv :
                    return mcv
                else :
                    value = function(key)
                    mc.set(skey, value, time=600)
                    return value
        return wrapper

except :
    def cachedfirstparam(fun):
        return fun

@cachedfirstparam
def static(strng) :
    return open('static/'+strng).read()

@app.route('/')
def hello():
    nvisit = int(request.cookies.get('rendered_urls',0))
    betauser = (True if nvisit > 30 else False)

    print('user has rendered {n} urls'.format(n=nvisit))
    return render_template('index.html', betauser=betauser)

@app.errorhandler(500)
def internal_error(error):
    return render_template('500.html'), 500

@app.errorhandler(404)
def page_not_found(error):
    return render_template('404.html'), 404

@app.route('/popular')
def popular():
    entries = [{'url':y.url,'count':x} for x,y in stats.most_accessed(count=20)]
    return render_template('popular.html', entries=entries)

@app.route('/404')
def four_o_foru():
    abort(404)

@app.route('/500')
def fiveoo():
    abort(500)

@app.route('/create/', methods=['POST'])
def create(v=None):
    value = request.form['gistnorurl']

    response = None
    increasegen = False
    if v and not value:
        value = v
    gist = re.search(r'^https?://gist.github.com/([0-9]+)$', value)
    if re.match('^[0-9]+$', value):
        response = redirect('/'+value)
    elif gist:
        response = redirect('/'+gist.group(1))

    elif value.startswith('https://') and value.endswith('.ipynb'):
        response = redirect('/urls/'+value[8:])

    elif value.startswith('http://') and value.endswith('.ipynb'):
        response = redirect('/url/'+value[7:])
    else :
        response = render_template('unknown_filetype.html')

    response = app.make_response(response)
    nvisit = int(request.cookies.get('rendered_urls',0))
    response.set_cookie('rendered_urls',value=nvisit+1)
    return response

#https !
@cachedfirstparam
def cachedget(url):
    try :
        r = requests.get(url)
        if r.status_code is not 200 :
            abort(404)
        return r.content
    except Exception :
        abort(404)

@cachedfirstparam
@app.route('/urls/<path:url>')
def render_urls(url):
    try :
        stats.get('urls/'+url).access()
    except :
        print 'oups crashed'
    content = cachedget('https://'+url)
    return render_content(content)

#http !
@cachedfirstparam
@app.route('/url/<path:url>')
def render_url(url):
    stats.get('url/'+url).access()
    content = cachedget('http://'+url)
    return render_content(content)

@cachedfirstparam
def render_request(r=None):
    try:
        if r.status_code != 200:
            abort(404)
        return render_content(r.content)
    except  :
        abort(404)

def render_content(content):
    converter = nbconvert.ConverterHTML()
    converter.nb = nbformat.reads_json(content)
    return converter.convert()

@cachedfirstparam
@app.route('/<int:id>/')
def fetch_and_render(id=None):
    """Fetch and render a post from the Github API"""
    if id is None :
        return redirect('/')
    stats.get(id).access()
    r = requests.get('https://api.github.com/gists/{}'.format(id))

    if r.status_code != 200:
        abort(404)
    try :
        decoded = r.json.copy()
        files = decoded['files'].values()
        if len(files) == 1 :
            jsonipynb = files[0]['content']
            converter = nbconvert.ConverterHTML()
            converter.nb = nbformat.reads_json(jsonipynb)
            result = converter.convert()
        else :
            entries = []
            for file in files :
                entry = {}
                entry['path'] = file['filename']
                entry['url'] = '/%s/%s' %( id,file['filename'])
                entries.append(entry)
            return render_template('gistlist.html', entries=entries)
    except ValueError as e :
        abort(404)

    return result

@app.route('/<int:id>/<subfile>')
def gistsubfile(id, subfile):
    """Fetch and render a post from the Github API"""

    r = requests.get('https://api.github.com/gists/{}'.format(id))

    if r.status_code != 200:
        abort(404)
    try :
        decoded = r.json.copy()
        files = decoded['files'].values()
        thefile = [f for f in files if f['filename'] == subfile]
        jsonipynb = thefile[0]['content']
        if subfile.endswith('.ipynb'):
            return render_content(jsonipynb)
        else :
            return Response(jsonipynb, mimetype='text/plain')
    except ValueError as e :
        abort(404)

if __name__ == '__main__':
    # Bind to PORT if defined, otherwise default to 5000.
    port = int(os.environ.get('PORT', 5000))
    debug = os.path.exists('.debug')
    if debug :
        print 'DEBUG MODE IS ACTIVATED !!!'
    else :
        print 'debug is not activated'
    app.run(host='0.0.0.0', port=port, debug=debug)
<commit_msg>test, push<commit_after>import os
import re
import requests

from nbformat import current as nbformat
import nbconvert.nbconvert as nbconvert

from flask import Flask , request, render_template
from flask import redirect, abort, Response

from statistics import Stats
from sqlalchemy import create_engine

from flask.ext.sqlalchemy import SQLAlchemy

app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = os.environ['DATABASE_URL']

#engine = create_engine('sqlite:///foo.db', echo=False)
engine = create_engine(app.config['SQLALCHEMY_DATABASE_URI'], echo=False)
stats = Stats(engine)

try :
    import pylibmc
    mc = pylibmc.Client(["127.0.0.1"], binary=True,
                    behaviors={"tcp_nodelay": True,
                                "ketama": True})

    def cachedfirstparam(function):

        def wrapper(*args, **kw):
            if len(args)+len(kw) != 1:
               return function(*args, **kw)
            else :
                key = kw.values()[0] if kw else args[0]
                skey = str(key)+str(function.__name__)
                mcv = mc.get(skey)
                #mcv = None
                if mcv :
                    return mcv
                else :
                    value = function(key)
                    mc.set(skey, value, time=600)
                    return value
        return wrapper

except :
    def cachedfirstparam(fun):
        return fun

@cachedfirstparam
def static(strng) :
    return open('static/'+strng).read()

@app.route('/')
def hello():
    nvisit = int(request.cookies.get('rendered_urls',0))
    betauser = (True if nvisit > 30 else False)
    return render_template('index.html', betauser=betauser)

@app.errorhandler(500)
def internal_error(error):
    return render_template('500.html'), 500

@app.errorhandler(404)
def page_not_found(error):
    return render_template('404.html'), 404

@app.route('/popular')
def popular():
    entries = [{'url':y.url,'count':x} for x,y in stats.most_accessed(count=20)]
    return render_template('popular.html', entries=entries)

@app.route('/404')
def four_o_foru():
    abort(404)

@app.route('/500')
def fiveoo():
    abort(500)

@app.route('/create/', methods=['POST'])
def create(v=None):
    value = request.form['gistnorurl']

    response = None
    increasegen = False
    if v and not value:
        value = v
    gist = re.search(r'^https?://gist.github.com/([0-9]+)$', value)
    if re.match('^[0-9]+$', value):
        response = redirect('/'+value)
    elif gist:
        response = redirect('/'+gist.group(1))

    elif value.startswith('https://') and value.endswith('.ipynb'):
        response = redirect('/urls/'+value[8:])

    elif value.startswith('http://') and value.endswith('.ipynb'):
        response = redirect('/url/'+value[7:])
    else :
        response = render_template('unknown_filetype.html')

    response = app.make_response(response)
    nvisit = int(request.cookies.get('rendered_urls',0))
    response.set_cookie('rendered_urls',value=nvisit+1)
    return response

#https !
@cachedfirstparam
def cachedget(url):
    try :
        r = requests.get(url)
        if r.status_code is not 200 :
            abort(404)
        return r.content
    except Exception :
        abort(404)

@cachedfirstparam
@app.route('/urls/<path:url>')
def render_urls(url):
    try :
        stats.get('urls/'+url).access()
    except :
        print 'oups crashed'
    content = cachedget('https://'+url)
    return render_content(content)

#http !
@cachedfirstparam
@app.route('/url/<path:url>')
def render_url(url):
    stats.get('url/'+url).access()
    content = cachedget('http://'+url)
    return render_content(content)

@cachedfirstparam
def render_request(r=None):
    try:
        if r.status_code != 200:
            abort(404)
        return render_content(r.content)
    except  :
        abort(404)

def render_content(content):
    converter = nbconvert.ConverterHTML()
    converter.nb = nbformat.reads_json(content)
    return converter.convert()

@cachedfirstparam
@app.route('/<int:id>/')
def fetch_and_render(id=None):
    """Fetch and render a post from the Github API"""
    if id is None :
        return redirect('/')
    try :
        stats.get(str(id)).access()
    except :
        print 'oups ', id, 'crashed'
    r = requests.get('https://api.github.com/gists/{}'.format(id))

    if r.status_code != 200:
        abort(404)
    try :
        decoded = r.json.copy()
        files = decoded['files'].values()
        if len(files) == 1 :
            jsonipynb = files[0]['content']
            converter = nbconvert.ConverterHTML()
            converter.nb = nbformat.reads_json(jsonipynb)
            result = converter.convert()
        else :
            entries = []
            for file in files :
                entry = {}
                entry['path'] = file['filename']
                entry['url'] = '/%s/%s' %( id,file['filename'])
                entries.append(entry)
            return render_template('gistlist.html', entries=entries)
    except ValueError as e :
        abort(404)

    return result

@app.route('/<int:id>/<subfile>')
def gistsubfile(id, subfile):
    """Fetch and render a post from the Github API"""

    r = requests.get('https://api.github.com/gists/{}'.format(id))

    if r.status_code != 200:
        abort(404)
    try :
        decoded = r.json.copy()
        files = decoded['files'].values()
        thefile = [f for f in files if f['filename'] == subfile]
        jsonipynb = thefile[0]['content']
        if subfile.endswith('.ipynb'):
            return render_content(jsonipynb)
        else :
            return Response(jsonipynb, mimetype='text/plain')
    except ValueError as e :
        abort(404)

if __name__ == '__main__':
    # Bind to PORT if defined, otherwise default to 5000.
    port = int(os.environ.get('PORT', 5000))
    debug = os.path.exists('.debug')
    if debug :
        print 'DEBUG MODE IS ACTIVATED !!!'
    else :
        print 'debug is not activated'
    app.run(host='0.0.0.0', port=port, debug=debug)
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright (c) 2009 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Top-level presubmit script for Chromium.

See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
for details on the presubmit API built into gcl.
"""

EXCLUDED_PATHS = (
    r"breakpad[\\\/].*",
    r"skia[\\\/].*",
    r"v8[\\\/].*",
)

def CheckChangeOnUpload(input_api, output_api):
  results = []
  # What does this code do?
  # It loads the default black list (e.g. third_party, experimental, etc) and
  # add our black list (breakpad, skia and v8 are still not following
  # google style and are not really living this repository).
  # See presubmit_support.py InputApi.FilterSourceFile for the (simple) usage.
  black_list = input_api.DEFAULT_BLACK_LIST + EXCLUDED_PATHS
  sources = lambda x: input_api.FilterSourceFile(x, black_list=black_list)
  results.extend(input_api.canned_checks.CheckLongLines(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasNoTabs(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasBugField(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckChangeHasTestField(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckChangeSvnEolStyle(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckSvnForCommonMimeTypes(
      input_api, output_api))
  return results

def CheckChangeOnCommit(input_api, output_api):
  results = []
  black_list = input_api.DEFAULT_BLACK_LIST + EXCLUDED_PATHS
  sources = lambda x: input_api.FilterSourceFile(x, black_list=black_list)
  results.extend(input_api.canned_checks.CheckLongLines(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasNoTabs(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasBugField(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckChangeHasTestField(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckChangeSvnEolStyle(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckSvnForCommonMimeTypes(
      input_api, output_api))
  # TODO(thestig) temporarily disabled, doesn't work in third_party/
  #results.extend(input_api.canned_checks.CheckSvnModifiedDirectories(
  #    input_api, output_api, sources))
  # Make sure the tree is 'open'.
  # TODO(maruel): Run it in a separate thread to parallelize checks?
  results.extend(CheckTreeIsOpen(
      input_api,
      output_api,
      'http://chromium-status.appspot.com/status',
      '0',
      'http://chromium-status.appspot.com/current?format=raw'))
  results.extend(CheckTryJobExecution(input_api, output_api))
  return results

def CheckTryJobExecution(input_api, output_api):
  outputs = []
  if not input_api.change.issue or not input_api.change.patchset:
    return outputs
  url = "http://codereview.chromium.org/%d/get_build_results/%d" % (
            input_api.change.issue, input_api.change.patchset)
  PLATFORMS = ('win', 'linux', 'mac')
  try:
    connection = input_api.urllib2.urlopen(url)
    # platform|status|url
    values = [item.split('|', 2) for item in connection.read().splitlines()]
    connection.close()
    if not values:
      # It returned an empty list. Probably a private review.
      return outputs
    # Reformat as an dict of platform: [status, url]
    values = dict([[v[0], [v[1], v[2]]] for v in values])
    for platform in PLATFORMS:
      values.setdefault(platform, ['not started', ''])
    message = None
    non_success = [k.upper() for k,v in values.iteritems() if v[0] != 'success']
    if 'failure' in [v[0] for v in values.itervalues()]:
      message = 'Try job failures on %s!\n' % ', '.join(non_success)
    elif non_success:
      message = ('Unfinished (or not even started) try jobs on '
                 '%s.\n') % ', '.join(non_success)
    if message:
      message += (
          'Is try server wrong or broken? Please notify maruel@chromium.org. '
          'Thanks.\n')
      outputs.append(output_api.PresubmitPromptWarning(message=message))
  except input_api.urllib2.HTTPError, e:
    if e.code == 404:
      # Fallback to no try job.
      # TODO(maruel): Change to a PresubmitPromptWarning once the try server is
      # stable enough and it seems to work fine.
      outputs.append(output_api.PresubmitNotifyResult(
          'You should try the patch first.'))
    else:
      # Another HTTP error happened, warn the user.
      # TODO(maruel): Change to a PresubmitPromptWarning once it deemed to work
      # fine.
      outputs.append(output_api.PresubmitNotifyResult(
          'Got %s while looking for try job status.' % str(e)))
  return outputs

def CheckTreeIsOpen(input_api, output_api, url, closed, url_text):
  """Similar to the one in presubmit_canned_checks except it shows an helpful
  status text instead.
  """
  assert(input_api.is_committing)
  try:
    connection = input_api.urllib2.urlopen(url)
    status = connection.read()
    connection.close()
    if input_api.re.match(closed, status):
      long_text = status + '\n' + url
      try:
        connection = input_api.urllib2.urlopen(url_text)
        long_text = connection.read().strip()
        connection.close()
      except IOError:
        pass
      return [output_api.PresubmitError("The tree is closed.",
                                        long_text=long_text)]
  except IOError:
    pass
  return []

def GetPreferredTrySlaves():
  return ['win', 'linux', 'mac']
<commit_msg>Dummy checkin to trigger the builbot<commit_after>#!/usr/bin/python
# Copyright (c) 2009 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Top-level presubmit script for Chromium.

See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
for more details on the presubmit API built into gcl.
"""

EXCLUDED_PATHS = (
    r"breakpad[\\\/].*",
    r"skia[\\\/].*",
    r"v8[\\\/].*",
)

def CheckChangeOnUpload(input_api, output_api):
  results = []
  # What does this code do?
  # It loads the default black list (e.g. third_party, experimental, etc) and
  # add our black list (breakpad, skia and v8 are still not following
  # google style and are not really living this repository).
  # See presubmit_support.py InputApi.FilterSourceFile for the (simple) usage.
  black_list = input_api.DEFAULT_BLACK_LIST + EXCLUDED_PATHS
  sources = lambda x: input_api.FilterSourceFile(x, black_list=black_list)
  results.extend(input_api.canned_checks.CheckLongLines(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasNoTabs(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasBugField(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckChangeHasTestField(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckChangeSvnEolStyle(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckSvnForCommonMimeTypes(
      input_api, output_api))
  return results

def CheckChangeOnCommit(input_api, output_api):
  results = []
  black_list = input_api.DEFAULT_BLACK_LIST + EXCLUDED_PATHS
  sources = lambda x: input_api.FilterSourceFile(x, black_list=black_list)
  results.extend(input_api.canned_checks.CheckLongLines(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasNoTabs(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckChangeHasBugField(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckChangeHasTestField(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckChangeSvnEolStyle(
      input_api, output_api, sources))
  results.extend(input_api.canned_checks.CheckSvnForCommonMimeTypes(
      input_api, output_api))
  # TODO(thestig) temporarily disabled, doesn't work in third_party/
  #results.extend(input_api.canned_checks.CheckSvnModifiedDirectories(
  #    input_api, output_api, sources))
  # Make sure the tree is 'open'.
  # TODO(maruel): Run it in a separate thread to parallelize checks?
  results.extend(CheckTreeIsOpen(
      input_api,
      output_api,
      'http://chromium-status.appspot.com/status',
      '0',
      'http://chromium-status.appspot.com/current?format=raw'))
  results.extend(CheckTryJobExecution(input_api, output_api))
  return results

def CheckTryJobExecution(input_api, output_api):
  outputs = []
  if not input_api.change.issue or not input_api.change.patchset:
    return outputs
  url = "http://codereview.chromium.org/%d/get_build_results/%d" % (
            input_api.change.issue, input_api.change.patchset)
  PLATFORMS = ('win', 'linux', 'mac')
  try:
    connection = input_api.urllib2.urlopen(url)
    # platform|status|url
    values = [item.split('|', 2) for item in connection.read().splitlines()]
    connection.close()
    if not values:
      # It returned an empty list. Probably a private review.
      return outputs
    # Reformat as an dict of platform: [status, url]
    values = dict([[v[0], [v[1], v[2]]] for v in values])
    for platform in PLATFORMS:
      values.setdefault(platform, ['not started', ''])
    message = None
    non_success = [k.upper() for k,v in values.iteritems() if v[0] != 'success']
    if 'failure' in [v[0] for v in values.itervalues()]:
      message = 'Try job failures on %s!\n' % ', '.join(non_success)
    elif non_success:
      message = ('Unfinished (or not even started) try jobs on '
                 '%s.\n') % ', '.join(non_success)
    if message:
      message += (
          'Is try server wrong or broken? Please notify maruel@chromium.org. '
          'Thanks.\n')
      outputs.append(output_api.PresubmitPromptWarning(message=message))
  except input_api.urllib2.HTTPError, e:
    if e.code == 404:
      # Fallback to no try job.
      # TODO(maruel): Change to a PresubmitPromptWarning once the try server is
      # stable enough and it seems to work fine.
      outputs.append(output_api.PresubmitNotifyResult(
          'You should try the patch first.'))
    else:
      # Another HTTP error happened, warn the user.
      # TODO(maruel): Change to a PresubmitPromptWarning once it deemed to work
      # fine.
      outputs.append(output_api.PresubmitNotifyResult(
          'Got %s while looking for try job status.' % str(e)))
  return outputs

def CheckTreeIsOpen(input_api, output_api, url, closed, url_text):
  """Similar to the one in presubmit_canned_checks except it shows an helpful
  status text instead.
  """
  assert(input_api.is_committing)
  try:
    connection = input_api.urllib2.urlopen(url)
    status = connection.read()
    connection.close()
    if input_api.re.match(closed, status):
      long_text = status + '\n' + url
      try:
        connection = input_api.urllib2.urlopen(url_text)
        long_text = connection.read().strip()
        connection.close()
      except IOError:
        pass
      return [output_api.PresubmitError("The tree is closed.",
                                        long_text=long_text)]
  except IOError:
    pass
  return []

def GetPreferredTrySlaves():
  return ['win', 'linux', 'mac']
<|endoftext|>


<commit_before>import logging
import os
import shutil

import shortuuid

from dvc.exceptions import GetDVCFileError
from dvc.exceptions import NotDvcRepoError
from dvc.exceptions import OutputNotFoundError
from dvc.exceptions import UrlNotDvcRepoError
from dvc.external_repo import external_repo
from dvc.path_info import PathInfo
from dvc.stage import Stage
from dvc.state import StateNoop
from dvc.utils import resolve_output
from dvc.utils.fs import remove
from dvc.utils.fs import path_isin

logger = logging.getLogger(__name__)

def _copy_git_file(repo, src, dst):
    src_full_path = os.path.join(repo.root_dir, src)
    dst_full_path = os.path.abspath(dst)

    if os.path.isdir(src_full_path):
        shutil.copytree(src_full_path, dst_full_path)
    else:
        shutil.copy2(src_full_path, dst_full_path)

@staticmethod
def get(url, path, out=None, rev=None):
    out = resolve_output(path, out)

    if Stage.is_valid_filename(out):
        raise GetDVCFileError()

    # Creating a directory right beside the output to make sure that they
    # are on the same filesystem, so we could take the advantage of
    # reflink and/or hardlink. Not using tempfile.TemporaryDirectory
    # because it will create a symlink to tmpfs, which defeats the purpose
    # and won't work with reflink/hardlink.
    dpath = os.path.dirname(os.path.abspath(out))
    tmp_dir = os.path.join(dpath, "." + str(shortuuid.uuid()))
    try:
        with external_repo(cache_dir=tmp_dir, url=url, rev=rev) as repo:
            full_path = os.path.join(repo.root_dir, path)
            path_in_repo = path_isin(full_path, repo.root_dir)
            if os.path.exists(full_path) and path_in_repo:
                _copy_git_file(repo, path, out)
            else:
                # Note: we need to replace state, because in case of getting
                # DVC dependency on CIFS or NFS filesystems, sqlite-based state
                # will be unable to obtain lock
                repo.state = StateNoop()

                # Try any links possible to avoid data duplication.
                #
                # Not using symlink, because we need to remove cache after we
                # are done, and to make that work we would have to copy data
                # over anyway before removing the cache, so we might just copy
                # it right away.
                #
                # Also, we can't use theoretical "move" link type here, because
                # the same cache file might be used a few times in a directory.
                repo.cache.local.cache_types = ["reflink", "hardlink", "copy"]

                o = repo.find_out_by_relpath(path)
                with repo.state:
                    repo.cloud.pull(o.get_used_cache())
                o.path_info = PathInfo(os.path.abspath(out))
                with o.repo.state:
                    o.checkout()

    except NotDvcRepoError:
        raise UrlNotDvcRepoError(url)
    except OutputNotFoundError:
        raise OutputNotFoundError(path)
    finally:
        remove(tmp_dir)
<commit_msg>return early<commit_after>import logging
import os
import shutil

import shortuuid

from dvc.exceptions import GetDVCFileError
from dvc.exceptions import NotDvcRepoError
from dvc.exceptions import OutputNotFoundError
from dvc.exceptions import UrlNotDvcRepoError
from dvc.external_repo import external_repo
from dvc.path_info import PathInfo
from dvc.stage import Stage
from dvc.state import StateNoop
from dvc.utils import resolve_output
from dvc.utils.fs import remove
from dvc.utils.fs import path_isin

logger = logging.getLogger(__name__)

def _copy_git_file(repo, src, dst):
    src_full_path = os.path.join(repo.root_dir, src)
    dst_full_path = os.path.abspath(dst)

    if os.path.isdir(src_full_path):
        shutil.copytree(src_full_path, dst_full_path)
    else:
        shutil.copy2(src_full_path, dst_full_path)

@staticmethod
def get(url, path, out=None, rev=None):
    out = resolve_output(path, out)

    if Stage.is_valid_filename(out):
        raise GetDVCFileError()

    # Creating a directory right beside the output to make sure that they
    # are on the same filesystem, so we could take the advantage of
    # reflink and/or hardlink. Not using tempfile.TemporaryDirectory
    # because it will create a symlink to tmpfs, which defeats the purpose
    # and won't work with reflink/hardlink.
    dpath = os.path.dirname(os.path.abspath(out))
    tmp_dir = os.path.join(dpath, "." + str(shortuuid.uuid()))
    try:
        with external_repo(cache_dir=tmp_dir, url=url, rev=rev) as repo:
            full_path = os.path.join(repo.root_dir, path)
            path_in_repo = path_isin(full_path, repo.root_dir)
            if os.path.exists(full_path) and path_in_repo:
                _copy_git_file(repo, path, out)
                return

            # Note: we need to replace state, because in case of getting DVC
            # dependency on CIFS or NFS filesystems, sqlite-based state
            # will be unable to obtain lock
            repo.state = StateNoop()

            # Try any links possible to avoid data duplication.
            #
            # Not using symlink, because we need to remove cache after we are
            # done, and to make that work we would have to copy data over
            # anyway before removing the cache, so we might just copy it
            # right away.
            #
            # Also, we can't use theoretical "move" link type here, because
            # the same cache file might be used a few times in a directory.
            repo.cache.local.cache_types = ["reflink", "hardlink", "copy"]

            o = repo.find_out_by_relpath(path)
            with repo.state:
                repo.cloud.pull(o.get_used_cache())
            o.path_info = PathInfo(os.path.abspath(out))
            with o.repo.state:
                o.checkout()

    except NotDvcRepoError:
        raise UrlNotDvcRepoError(url)
    except OutputNotFoundError:
        raise OutputNotFoundError(path)
    finally:
        remove(tmp_dir)
<|endoftext|>


<commit_before># Django settings for expensify project.
import os
PROJECT_DIR = os.path.dirname("__file__")
DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = ''

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = '&9tv)j53cu!d5fklvy&q)1*m^b60$12g77%=y9=8v$2#f+bd_t'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'expensify.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'expensify.wsgi.application'

TEMPLATE_DIRS = (

    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    # Uncomment the next line to enable the admin:
    'django.contrib.admin',
    'exapp',
    'south',
    'django_openid_auth',
    # Uncomment the next line to enable admin documentation:
    # 'django.contrib.admindocs',
)

AUTHENTICATION_BACKENDS = (
    'django_openid_auth.auth.OpenIDBackend',
    'django.contrib.auth.backends.ModelBackend',
)

# Should users be created when new OpenIDs are used to log in?
OPENID_CREATE_USERS = True

# When logging in again, should we overwrite user details based on
# data received via Simple Registration?
OPENID_UPDATE_DETAILS_FROM_SREG = True

# If set, always use this as the identity URL rather than asking the
# user.  This only makes sense if it is a server URL.
OPENID_SSO_SERVER_URL = 'https://google.com/accounts/o8/site-xrds?hd=agiliq.com'

# Tell django.contrib.auth to use the OpenID signin URLs.
LOGIN_URL = '/openid/login/'
LOGIN_REDIRECT_URL = '/'

# Should django_auth_openid be used to sign into the admin interface?
OPENID_USE_AS_ADMIN_LOGIN = False

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

from local_settings import *
<commit_msg>Added static_root, template_debug from localsettings variable 'DEBUG', fixed typo: __file__<commit_after># Django settings for expensify project.
import os
PROJECT_DIR = os.path.dirname(__file__)

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = PROJECT_DIR+'/static'

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = '&9tv)j53cu!d5fklvy&q)1*m^b60$12g77%=y9=8v$2#f+bd_t'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'expensify.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'expensify.wsgi.application'

TEMPLATE_DIRS = (

    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    # Uncomment the next line to enable the admin:
    'django.contrib.admin',
    'exapp',
    'south',
    'django_openid_auth',
    # Uncomment the next line to enable admin documentation:
    # 'django.contrib.admindocs',
)

AUTHENTICATION_BACKENDS = (
    'django_openid_auth.auth.OpenIDBackend',
    'django.contrib.auth.backends.ModelBackend',
)

# Should users be created when new OpenIDs are used to log in?
OPENID_CREATE_USERS = True

# When logging in again, should we overwrite user details based on
# data received via Simple Registration?
OPENID_UPDATE_DETAILS_FROM_SREG = True

# If set, always use this as the identity URL rather than asking the
# user.  This only makes sense if it is a server URL.
#OPENID_SSO_SERVER_URL = 'https://google.com/accounts/o8/site-xrds?hd=agiliq.com'
OPENID_SSO_SERVER_URL = 'https://google.com/accounts/o8/id'

# Tell django.contrib.auth to use the OpenID signin URLs.
LOGIN_URL = '/openid/login/'
LOGIN_REDIRECT_URL = '/'

# Should django_auth_openid be used to sign into the admin interface?
OPENID_USE_AS_ADMIN_LOGIN = False

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

from local_settings import *
TEMPLATE_DEBUG = DEBUG
<|endoftext|>