<commit_before>import os
import warnings
from functools import lru_cache
from typing import Any
from typing import Dict
from typing import List
from typing import Set
from typing import Tuple
from typing import Union

import py

import _pytest._code
from _pytest.compat import getfslineno
from _pytest.mark.structures import Mark
from _pytest.mark.structures import MarkDecorator
from _pytest.mark.structures import NodeKeywords
from _pytest.outcomes import fail

if False:  # TYPE_CHECKING
    # Imported here due to circular import.
    from _pytest.fixtures import FixtureDef

SEP = "/"

tracebackcutdir = py.path.local(_pytest.__file__).dirpath()

@lru_cache(maxsize=None)
def _splitnode(nodeid):
    """Split a nodeid into constituent 'parts'.

    Node IDs are strings, and can be things like:
        ''
        'testing/code'
        'testing/code/test_excinfo.py'
        'testing/code/test_excinfo.py::TestFormattedExcinfo'

    Return values are lists e.g.
        []
        ['testing', 'code']
        ['testing', 'code', 'test_excinfo.py']
        ['testing', 'code', 'test_excinfo.py', 'TestFormattedExcinfo', '()']
    """
    if nodeid == "":
        # If there is no root node at all, return an empty list so the caller's logic can remain sane
        return ()
    parts = nodeid.split(SEP)
    # Replace single last element 'test_foo.py::Bar' with multiple elements 'test_foo.py', 'Bar'
    parts[-1:] = parts[-1].split("::")
    # Convert parts into a tuple to avoid possible errors with caching of a mutable type
    return tuple(parts)

def ischildnode(baseid, nodeid):
    """Return True if the nodeid is a child node of the baseid.

    E.g. 'foo/bar::Baz' is a child of 'foo', 'foo/bar' and 'foo/bar::Baz', but not of 'foo/blorp'
    """
    base_parts = _splitnode(baseid)
    node_parts = _splitnode(nodeid)
    if len(node_parts) < len(base_parts):
        return False
    return node_parts[: len(base_parts)] == base_parts

class Node:
    """ base class for Collector and Item the test collection tree.
    Collector subclasses have children, Items are terminal nodes."""

    def __init__(
        self, name, parent=None, config=None, session=None, fspath=None, nodeid=None
    ):
        #: a unique name within the scope of the parent node
        self.name = name

        #: the parent collector node.
        self.parent = parent

        #: the pytest config object
        self.config = config or parent.config

        #: the session this node is part of
        self.session = session or parent.session

        #: filesystem path where this node was collected from (can be None)
        self.fspath = fspath or getattr(parent, "fspath", None)

        #: keywords/markers collected from all scopes
        self.keywords = NodeKeywords(self)

        #: the marker objects belonging to this node
        self.own_markers = []  # type: List[Mark]

        #: allow adding of extra keywords to use for matching
        self.extra_keyword_matches = set()  # type: Set[str]

        # used for storing artificial fixturedefs for direct parametrization
        self._name2pseudofixturedef = {}  # type: Dict[str, FixtureDef]

        if nodeid is not None:
            assert "::()" not in nodeid
            self._nodeid = nodeid
        else:
            self._nodeid = self.parent.nodeid
            if self.name != "()":
                self._nodeid += "::" + self.name

    @property
    def ihook(self):
        """ fspath sensitive hook proxy used to call pytest hooks"""
        return self.session.gethookproxy(self.fspath)

    def __repr__(self):
        return "<{} {}>".format(self.__class__.__name__, getattr(self, "name", None))

    def warn(self, warning):
        """Issue a warning for this item.

        Warnings will be displayed after the test session, unless explicitly suppressed

        :param Warning warning: the warning instance to issue. Must be a subclass of PytestWarning.

        :raise ValueError: if ``warning`` instance is not a subclass of PytestWarning.

        Example usage:

        .. code-block:: python

            node.warn(PytestWarning("some message"))

        """
        from _pytest.warning_types import PytestWarning

        if not isinstance(warning, PytestWarning):
            raise ValueError(
                "warning must be an instance of PytestWarning or subclass, got {!r}".format(
                    warning
                )
            )
        path, lineno = get_fslocation_from_item(self)
        warnings.warn_explicit(
            warning,
            category=None,
            filename=str(path),
            lineno=lineno + 1 if lineno is not None else None,
        )

    # methods for ordering nodes
    @property
    def nodeid(self):
        """ a ::-separated string denoting its collection tree address. """
        return self._nodeid

    def __hash__(self):
        return hash(self.nodeid)

    def setup(self):
        pass

    def teardown(self):
        pass

    def listchain(self):
        """ return list of all parent collectors up to self,
            starting from root of collection tree. """
        chain = []
        item = self
        while item is not None:
            chain.append(item)
            item = item.parent
        chain.reverse()
        return chain

    def add_marker(
        self, marker: Union[str, MarkDecorator], append: bool = True
    ) -> None:
        """dynamically add a marker object to the node.

        :type marker: ``str`` or ``pytest.mark.*``  object
        :param marker:
            ``append=True`` whether to append the marker,
            if ``False`` insert at position ``0``.
        """
        from _pytest.mark import MARK_GEN

        if isinstance(marker, MarkDecorator):
            marker_ = marker
        elif isinstance(marker, str):
            marker_ = getattr(MARK_GEN, marker)
        else:
            raise ValueError("is not a string or pytest.mark.* Marker")
        self.keywords[marker_.name] = marker
        if append:
            self.own_markers.append(marker_.mark)
        else:
            self.own_markers.insert(0, marker_.mark)

    def iter_markers(self, name=None):
        """
        :param name: if given, filter the results by the name attribute

        iterate over all markers of the node
        """
        return (x[1] for x in self.iter_markers_with_node(name=name))

    def iter_markers_with_node(self, name=None):
        """
        :param name: if given, filter the results by the name attribute

        iterate over all markers of the node
        returns sequence of tuples (node, mark)
        """
        for node in reversed(self.listchain()):
            for mark in node.own_markers:
                if name is None or getattr(mark, "name", None) == name:
                    yield node, mark

    def get_closest_marker(self, name, default=None):
        """return the first marker matching the name, from closest (for example function) to farther level (for example
        module level).

        :param default: fallback return value of no marker was found
        :param name: name to filter by
        """
        return next(self.iter_markers(name=name), default)

    def listextrakeywords(self):
        """ Return a set of all extra keywords in self and any parents."""
        extra_keywords = set()  # type: Set[str]
        for item in self.listchain():
            extra_keywords.update(item.extra_keyword_matches)
        return extra_keywords

    def listnames(self):
        return [x.name for x in self.listchain()]

    def addfinalizer(self, fin):
        """ register a function to be called when this node is finalized.

        This method can only be called when this node is active
        in a setup chain, for example during self.setup().
        """
        self.session._setupstate.addfinalizer(fin, self)

    def getparent(self, cls):
        """ get the next parent node (including ourself)
        which is an instance of the given class"""
        current = self
        while current and not isinstance(current, cls):
            current = current.parent
        return current

    def _prunetraceback(self, excinfo):
        pass

    def _repr_failure_py(self, excinfo, style=None):
        # Type ignored: see comment where fail.Exception is defined.
        if excinfo.errisinstance(fail.Exception):  # type: ignore
            if not excinfo.value.pytrace:
                return str(excinfo.value)
        fm = self.session._fixturemanager
        if excinfo.errisinstance(fm.FixtureLookupError):
            return excinfo.value.formatrepr()
        if self.config.getoption("fulltrace", False):
            style = "long"
        else:
            tb = _pytest._code.Traceback([excinfo.traceback[-1]])
            self._prunetraceback(excinfo)
            if len(excinfo.traceback) == 0:
                excinfo.traceback = tb
            if style == "auto":
                style = "long"
        # XXX should excinfo.getrepr record all data and toterminal() process it?
        if style is None:
            if self.config.getoption("tbstyle", "auto") == "short":
                style = "short"
            else:
                style = "long"

        if self.config.getoption("verbose", 0) > 1:
            truncate_locals = False
        else:
            truncate_locals = True

        try:
            os.getcwd()
            abspath = False
        except OSError:
            abspath = True

        return excinfo.getrepr(
            funcargs=True,
            abspath=abspath,
            showlocals=self.config.getoption("showlocals", False),
            style=style,
            tbfilter=False,  # pruned already, or in --fulltrace mode.
            truncate_locals=truncate_locals,
        )

    def repr_failure(self, excinfo, style=None):
        return self._repr_failure_py(excinfo, style)

def get_fslocation_from_item(item):
    """Tries to extract the actual location from an item, depending on available attributes:

    * "fslocation": a pair (path, lineno)
    * "obj": a Python object that the item wraps.
    * "fspath": just a path

    :rtype: a tuple of (str|LocalPath, int) with filename and line number.
    """
    result = getattr(item, "location", None)
    if result is not None:
        return result[:2]
    obj = getattr(item, "obj", None)
    if obj is not None:
        return getfslineno(obj)
    return getattr(item, "fspath", "unknown location"), -1

class Collector(Node):
    """ Collector instances create children through collect()
        and thus iteratively build a tree.
    """

    class CollectError(Exception):
        """ an error during collection, contains a custom message. """

    def collect(self):
        """ returns a list of children (items and collectors)
            for this collection node.
        """
        raise NotImplementedError("abstract")

    def repr_failure(self, excinfo):
        """ represent a collection failure. """
        if excinfo.errisinstance(self.CollectError):
            exc = excinfo.value
            return str(exc.args[0])

        # Respect explicit tbstyle option, but default to "short"
        # (None._repr_failure_py defaults to "long" without "fulltrace" option).
        tbstyle = self.config.getoption("tbstyle", "auto")
        if tbstyle == "auto":
            tbstyle = "short"

        return self._repr_failure_py(excinfo, style=tbstyle)

    def _prunetraceback(self, excinfo):
        if hasattr(self, "fspath"):
            traceback = excinfo.traceback
            ntraceback = traceback.cut(path=self.fspath)
            if ntraceback == traceback:
                ntraceback = ntraceback.cut(excludepath=tracebackcutdir)
            excinfo.traceback = ntraceback.filter()

def _check_initialpaths_for_relpath(session, fspath):
    for initial_path in session._initialpaths:
        if fspath.common(initial_path) == initial_path:
            return fspath.relto(initial_path)

class FSCollector(Collector):
    def __init__(self, fspath, parent=None, config=None, session=None, nodeid=None):
        fspath = py.path.local(fspath)  # xxx only for test_resultlog.py?
        name = fspath.basename
        if parent is not None:
            rel = fspath.relto(parent.fspath)
            if rel:
                name = rel
            name = name.replace(os.sep, SEP)
        self.fspath = fspath

        session = session or parent.session

        if nodeid is None:
            nodeid = self.fspath.relto(session.config.rootdir)

            if not nodeid:
                nodeid = _check_initialpaths_for_relpath(session, fspath)
            if nodeid and os.sep != SEP:
                nodeid = nodeid.replace(os.sep, SEP)

        super().__init__(name, parent, config, session, nodeid=nodeid, fspath=fspath)

class File(FSCollector):
    """ base class for collecting tests from a file. """

class Item(Node):
    """ a basic test invocation item. Note that for a single function
    there might be multiple test invocation items.
    """

    nextitem = None

    def __init__(self, name, parent=None, config=None, session=None, nodeid=None):
        super().__init__(name, parent, config, session, nodeid=nodeid)
        self._report_sections = []  # type: List[Tuple[str, str, str]]

        #: user properties is a list of tuples (name, value) that holds user
        #: defined properties for this test.
        self.user_properties = []  # type: List[Tuple[str, Any]]

    def add_report_section(self, when: str, key: str, content: str) -> None:
        """
        Adds a new report section, similar to what's done internally to add stdout and
        stderr captured output::

            item.add_report_section("call", "stdout", "report section contents")

        :param str when:
            One of the possible capture states, ``"setup"``, ``"call"``, ``"teardown"``.
        :param str key:
            Name of the section, can be customized at will. Pytest uses ``"stdout"`` and
            ``"stderr"`` internally.

        :param str content:
            The full contents as a string.
        """
        if content:
            self._report_sections.append((when, key, content))

    def reportinfo(self):
        return self.fspath, None, ""

    @property
    def location(self):
        try:
            return self._location
        except AttributeError:
            location = self.reportinfo()
            fspath = self.session._node_location_to_relpath(location[0])
            location = (fspath, location[1], str(location[2]))
            self._location = location
            return location
<commit_msg>FSCollector: keep/use given fspath<commit_after>import os
import warnings
from functools import lru_cache
from typing import Any
from typing import Dict
from typing import List
from typing import Set
from typing import Tuple
from typing import Union

import py

import _pytest._code
from _pytest.compat import getfslineno
from _pytest.mark.structures import Mark
from _pytest.mark.structures import MarkDecorator
from _pytest.mark.structures import NodeKeywords
from _pytest.outcomes import fail

if False:  # TYPE_CHECKING
    # Imported here due to circular import.
    from _pytest.fixtures import FixtureDef

SEP = "/"

tracebackcutdir = py.path.local(_pytest.__file__).dirpath()

@lru_cache(maxsize=None)
def _splitnode(nodeid):
    """Split a nodeid into constituent 'parts'.

    Node IDs are strings, and can be things like:
        ''
        'testing/code'
        'testing/code/test_excinfo.py'
        'testing/code/test_excinfo.py::TestFormattedExcinfo'

    Return values are lists e.g.
        []
        ['testing', 'code']
        ['testing', 'code', 'test_excinfo.py']
        ['testing', 'code', 'test_excinfo.py', 'TestFormattedExcinfo', '()']
    """
    if nodeid == "":
        # If there is no root node at all, return an empty list so the caller's logic can remain sane
        return ()
    parts = nodeid.split(SEP)
    # Replace single last element 'test_foo.py::Bar' with multiple elements 'test_foo.py', 'Bar'
    parts[-1:] = parts[-1].split("::")
    # Convert parts into a tuple to avoid possible errors with caching of a mutable type
    return tuple(parts)

def ischildnode(baseid, nodeid):
    """Return True if the nodeid is a child node of the baseid.

    E.g. 'foo/bar::Baz' is a child of 'foo', 'foo/bar' and 'foo/bar::Baz', but not of 'foo/blorp'
    """
    base_parts = _splitnode(baseid)
    node_parts = _splitnode(nodeid)
    if len(node_parts) < len(base_parts):
        return False
    return node_parts[: len(base_parts)] == base_parts

class Node:
    """ base class for Collector and Item the test collection tree.
    Collector subclasses have children, Items are terminal nodes."""

    def __init__(
        self, name, parent=None, config=None, session=None, fspath=None, nodeid=None
    ):
        #: a unique name within the scope of the parent node
        self.name = name

        #: the parent collector node.
        self.parent = parent

        #: the pytest config object
        self.config = config or parent.config

        #: the session this node is part of
        self.session = session or parent.session

        #: filesystem path where this node was collected from (can be None)
        self.fspath = fspath or getattr(parent, "fspath", None)

        #: keywords/markers collected from all scopes
        self.keywords = NodeKeywords(self)

        #: the marker objects belonging to this node
        self.own_markers = []  # type: List[Mark]

        #: allow adding of extra keywords to use for matching
        self.extra_keyword_matches = set()  # type: Set[str]

        # used for storing artificial fixturedefs for direct parametrization
        self._name2pseudofixturedef = {}  # type: Dict[str, FixtureDef]

        if nodeid is not None:
            assert "::()" not in nodeid
            self._nodeid = nodeid
        else:
            self._nodeid = self.parent.nodeid
            if self.name != "()":
                self._nodeid += "::" + self.name

    @property
    def ihook(self):
        """ fspath sensitive hook proxy used to call pytest hooks"""
        return self.session.gethookproxy(self.fspath)

    def __repr__(self):
        return "<{} {}>".format(self.__class__.__name__, getattr(self, "name", None))

    def warn(self, warning):
        """Issue a warning for this item.

        Warnings will be displayed after the test session, unless explicitly suppressed

        :param Warning warning: the warning instance to issue. Must be a subclass of PytestWarning.

        :raise ValueError: if ``warning`` instance is not a subclass of PytestWarning.

        Example usage:

        .. code-block:: python

            node.warn(PytestWarning("some message"))

        """
        from _pytest.warning_types import PytestWarning

        if not isinstance(warning, PytestWarning):
            raise ValueError(
                "warning must be an instance of PytestWarning or subclass, got {!r}".format(
                    warning
                )
            )
        path, lineno = get_fslocation_from_item(self)
        warnings.warn_explicit(
            warning,
            category=None,
            filename=str(path),
            lineno=lineno + 1 if lineno is not None else None,
        )

    # methods for ordering nodes
    @property
    def nodeid(self):
        """ a ::-separated string denoting its collection tree address. """
        return self._nodeid

    def __hash__(self):
        return hash(self.nodeid)

    def setup(self):
        pass

    def teardown(self):
        pass

    def listchain(self):
        """ return list of all parent collectors up to self,
            starting from root of collection tree. """
        chain = []
        item = self
        while item is not None:
            chain.append(item)
            item = item.parent
        chain.reverse()
        return chain

    def add_marker(
        self, marker: Union[str, MarkDecorator], append: bool = True
    ) -> None:
        """dynamically add a marker object to the node.

        :type marker: ``str`` or ``pytest.mark.*``  object
        :param marker:
            ``append=True`` whether to append the marker,
            if ``False`` insert at position ``0``.
        """
        from _pytest.mark import MARK_GEN

        if isinstance(marker, MarkDecorator):
            marker_ = marker
        elif isinstance(marker, str):
            marker_ = getattr(MARK_GEN, marker)
        else:
            raise ValueError("is not a string or pytest.mark.* Marker")
        self.keywords[marker_.name] = marker
        if append:
            self.own_markers.append(marker_.mark)
        else:
            self.own_markers.insert(0, marker_.mark)

    def iter_markers(self, name=None):
        """
        :param name: if given, filter the results by the name attribute

        iterate over all markers of the node
        """
        return (x[1] for x in self.iter_markers_with_node(name=name))

    def iter_markers_with_node(self, name=None):
        """
        :param name: if given, filter the results by the name attribute

        iterate over all markers of the node
        returns sequence of tuples (node, mark)
        """
        for node in reversed(self.listchain()):
            for mark in node.own_markers:
                if name is None or getattr(mark, "name", None) == name:
                    yield node, mark

    def get_closest_marker(self, name, default=None):
        """return the first marker matching the name, from closest (for example function) to farther level (for example
        module level).

        :param default: fallback return value of no marker was found
        :param name: name to filter by
        """
        return next(self.iter_markers(name=name), default)

    def listextrakeywords(self):
        """ Return a set of all extra keywords in self and any parents."""
        extra_keywords = set()  # type: Set[str]
        for item in self.listchain():
            extra_keywords.update(item.extra_keyword_matches)
        return extra_keywords

    def listnames(self):
        return [x.name for x in self.listchain()]

    def addfinalizer(self, fin):
        """ register a function to be called when this node is finalized.

        This method can only be called when this node is active
        in a setup chain, for example during self.setup().
        """
        self.session._setupstate.addfinalizer(fin, self)

    def getparent(self, cls):
        """ get the next parent node (including ourself)
        which is an instance of the given class"""
        current = self
        while current and not isinstance(current, cls):
            current = current.parent
        return current

    def _prunetraceback(self, excinfo):
        pass

    def _repr_failure_py(self, excinfo, style=None):
        # Type ignored: see comment where fail.Exception is defined.
        if excinfo.errisinstance(fail.Exception):  # type: ignore
            if not excinfo.value.pytrace:
                return str(excinfo.value)
        fm = self.session._fixturemanager
        if excinfo.errisinstance(fm.FixtureLookupError):
            return excinfo.value.formatrepr()
        if self.config.getoption("fulltrace", False):
            style = "long"
        else:
            tb = _pytest._code.Traceback([excinfo.traceback[-1]])
            self._prunetraceback(excinfo)
            if len(excinfo.traceback) == 0:
                excinfo.traceback = tb
            if style == "auto":
                style = "long"
        # XXX should excinfo.getrepr record all data and toterminal() process it?
        if style is None:
            if self.config.getoption("tbstyle", "auto") == "short":
                style = "short"
            else:
                style = "long"

        if self.config.getoption("verbose", 0) > 1:
            truncate_locals = False
        else:
            truncate_locals = True

        try:
            os.getcwd()
            abspath = False
        except OSError:
            abspath = True

        return excinfo.getrepr(
            funcargs=True,
            abspath=abspath,
            showlocals=self.config.getoption("showlocals", False),
            style=style,
            tbfilter=False,  # pruned already, or in --fulltrace mode.
            truncate_locals=truncate_locals,
        )

    def repr_failure(self, excinfo, style=None):
        return self._repr_failure_py(excinfo, style)

def get_fslocation_from_item(item):
    """Tries to extract the actual location from an item, depending on available attributes:

    * "fslocation": a pair (path, lineno)
    * "obj": a Python object that the item wraps.
    * "fspath": just a path

    :rtype: a tuple of (str|LocalPath, int) with filename and line number.
    """
    result = getattr(item, "location", None)
    if result is not None:
        return result[:2]
    obj = getattr(item, "obj", None)
    if obj is not None:
        return getfslineno(obj)
    return getattr(item, "fspath", "unknown location"), -1

class Collector(Node):
    """ Collector instances create children through collect()
        and thus iteratively build a tree.
    """

    class CollectError(Exception):
        """ an error during collection, contains a custom message. """

    def collect(self):
        """ returns a list of children (items and collectors)
            for this collection node.
        """
        raise NotImplementedError("abstract")

    def repr_failure(self, excinfo):
        """ represent a collection failure. """
        if excinfo.errisinstance(self.CollectError):
            exc = excinfo.value
            return str(exc.args[0])

        # Respect explicit tbstyle option, but default to "short"
        # (None._repr_failure_py defaults to "long" without "fulltrace" option).
        tbstyle = self.config.getoption("tbstyle", "auto")
        if tbstyle == "auto":
            tbstyle = "short"

        return self._repr_failure_py(excinfo, style=tbstyle)

    def _prunetraceback(self, excinfo):
        if hasattr(self, "fspath"):
            traceback = excinfo.traceback
            ntraceback = traceback.cut(path=self.fspath)
            if ntraceback == traceback:
                ntraceback = ntraceback.cut(excludepath=tracebackcutdir)
            excinfo.traceback = ntraceback.filter()

def _check_initialpaths_for_relpath(session, fspath):
    for initial_path in session._initialpaths:
        if fspath.common(initial_path) == initial_path:
            return fspath.relto(initial_path)

class FSCollector(Collector):
    def __init__(
        self, fspath: py.path.local, parent=None, config=None, session=None, nodeid=None
    ) -> None:
        name = fspath.basename
        if parent is not None:
            rel = fspath.relto(parent.fspath)
            if rel:
                name = rel
            name = name.replace(os.sep, SEP)
        self.fspath = fspath

        session = session or parent.session

        if nodeid is None:
            nodeid = self.fspath.relto(session.config.rootdir)

            if not nodeid:
                nodeid = _check_initialpaths_for_relpath(session, fspath)
            if nodeid and os.sep != SEP:
                nodeid = nodeid.replace(os.sep, SEP)

        super().__init__(name, parent, config, session, nodeid=nodeid, fspath=fspath)

class File(FSCollector):
    """ base class for collecting tests from a file. """

class Item(Node):
    """ a basic test invocation item. Note that for a single function
    there might be multiple test invocation items.
    """

    nextitem = None

    def __init__(self, name, parent=None, config=None, session=None, nodeid=None):
        super().__init__(name, parent, config, session, nodeid=nodeid)
        self._report_sections = []  # type: List[Tuple[str, str, str]]

        #: user properties is a list of tuples (name, value) that holds user
        #: defined properties for this test.
        self.user_properties = []  # type: List[Tuple[str, Any]]

    def add_report_section(self, when: str, key: str, content: str) -> None:
        """
        Adds a new report section, similar to what's done internally to add stdout and
        stderr captured output::

            item.add_report_section("call", "stdout", "report section contents")

        :param str when:
            One of the possible capture states, ``"setup"``, ``"call"``, ``"teardown"``.
        :param str key:
            Name of the section, can be customized at will. Pytest uses ``"stdout"`` and
            ``"stderr"`` internally.

        :param str content:
            The full contents as a string.
        """
        if content:
            self._report_sections.append((when, key, content))

    def reportinfo(self):
        return self.fspath, None, ""

    @property
    def location(self):
        try:
            return self._location
        except AttributeError:
            location = self.reportinfo()
            fspath = self.session._node_location_to_relpath(location[0])
            location = (fspath, location[1], str(location[2]))
            self._location = location
            return location
<|endoftext|>


<commit_before>"""Filename matching with shell patterns.

fnmatch(FILENAME, PATTERN) matches according to the local convention.
fnmatchcase(FILENAME, PATTERN) always takes case in account.

The functions operate by translating the pattern into a regular
expression.  They cache the compiled regular expressions for speed.

The function translate(PATTERN) returns a regular expression
corresponding to PATTERN.  (It does not compile it.)

Based on code from fnmatch.py file distributed with Python 2.6.

Licensed under PSF License (see LICENSE.txt file).

Changes to original fnmatch module:
- translate function supports ``*`` and ``**`` similarly to fnmatch C library
"""

import os
import re

__all__ = ["filter", "fnmatch","fnmatchcase","translate"]

_cache = {}

def fnmatch(name, pat):
    """Test whether FILENAME matches PATTERN.

    Patterns are Unix shell style:

    - ``*``       matches everything except path separator
    - ``**``      matches everything
    - ``?``       matches any single character
    - ``[seq]``   matches any character in seq
    - ``[!seq]``  matches any char not in seq

    An initial period in FILENAME is not special.
    Both FILENAME and PATTERN are first case-normalized
    if the operating system requires it.
    If you don't want this, use fnmatchcase(FILENAME, PATTERN).
    """

    name = os.path.normcase(name).replace(os.sep, "/")
    pat = pat
    return fnmatchcase(name, pat)

def filter(names, pat):
    """Return the subset of the list NAMES that match PAT"""
    import posixpath
    result=[]
    pat=os.path.normcase(pat)
    if not pat in _cache:
        res = translate(pat)
        _cache[pat] = re.compile(res)
    match=_cache[pat].match
    if os.path is posixpath:
        # normcase on posix is NOP. Optimize it away from the loop.
        for name in names:
            if match(name):
                result.append(name)
    else:
        for name in names:
            if match(os.path.normcase(name)):
                result.append(name)
    return result

def fnmatchcase(name, pat):
    """Test whether FILENAME matches PATTERN, including case.

    This is a version of fnmatch() which doesn't case-normalize
    its arguments.
    """

    if not pat in _cache:
        res = translate(pat)
        _cache[pat] = re.compile(res)
    return _cache[pat].match(name) is not None

def translate(pat):
    """Translate a shell PATTERN to a regular expression.

    There is no way to quote meta-characters.
    """

    i, n = 0, len(pat)
    res = ''
    while i < n:
        c = pat[i]
        i = i+1
        if c == '*':
            j = i
            if j < n and pat[j] == '*':
                res = res + '.*'
            else:
                res = res + '[^/]*'
        elif c == '?':
            res = res + '.'
        elif c == '[':
            j = i
            if j < n and pat[j] == '!':
                j = j+1
            if j < n and pat[j] == ']':
                j = j+1
            while j < n and pat[j] != ']':
                j = j+1
            if j >= n:
                res = res + '\\['
            else:
                stuff = pat[i:j].replace('\\','\\\\')
                i = j+1
                if stuff[0] == '!':
                    stuff = '^' + stuff[1:]
                elif stuff[0] == '^':
                    stuff = '\\' + stuff
                res = '%s[%s]' % (res, stuff)
        else:
            res = res + re.escape(c)
    return res + '\Z(?ms)'
<commit_msg>Remove unused filter function<commit_after>"""Filename matching with shell patterns.

fnmatch(FILENAME, PATTERN) matches according to the local convention.
fnmatchcase(FILENAME, PATTERN) always takes case in account.

The functions operate by translating the pattern into a regular
expression.  They cache the compiled regular expressions for speed.

The function translate(PATTERN) returns a regular expression
corresponding to PATTERN.  (It does not compile it.)

Based on code from fnmatch.py file distributed with Python 2.6.

Licensed under PSF License (see LICENSE.txt file).

Changes to original fnmatch module:
- translate function supports ``*`` and ``**`` similarly to fnmatch C library
"""

import os
import re

__all__ = ["fnmatch", "fnmatchcase", "translate"]

_cache = {}

def fnmatch(name, pat):
    """Test whether FILENAME matches PATTERN.

    Patterns are Unix shell style:

    - ``*``       matches everything except path separator
    - ``**``      matches everything
    - ``?``       matches any single character
    - ``[seq]``   matches any character in seq
    - ``[!seq]``  matches any char not in seq

    An initial period in FILENAME is not special.
    Both FILENAME and PATTERN are first case-normalized
    if the operating system requires it.
    If you don't want this, use fnmatchcase(FILENAME, PATTERN).
    """

    name = os.path.normcase(name).replace(os.sep, "/")
    pat = pat
    return fnmatchcase(name, pat)

def fnmatchcase(name, pat):
    """Test whether FILENAME matches PATTERN, including case.

    This is a version of fnmatch() which doesn't case-normalize
    its arguments.
    """

    if not pat in _cache:
        res = translate(pat)
        _cache[pat] = re.compile(res)
    return _cache[pat].match(name) is not None

def translate(pat):
    """Translate a shell PATTERN to a regular expression.

    There is no way to quote meta-characters.
    """

    i, n = 0, len(pat)
    res = ''
    while i < n:
        c = pat[i]
        i = i+1
        if c == '*':
            j = i
            if j < n and pat[j] == '*':
                res = res + '.*'
            else:
                res = res + '[^/]*'
        elif c == '?':
            res = res + '.'
        elif c == '[':
            j = i
            if j < n and pat[j] == '!':
                j = j+1
            if j < n and pat[j] == ']':
                j = j+1
            while j < n and pat[j] != ']':
                j = j+1
            if j >= n:
                res = res + '\\['
            else:
                stuff = pat[i:j].replace('\\','\\\\')
                i = j+1
                if stuff[0] == '!':
                    stuff = '^' + stuff[1:]
                elif stuff[0] == '^':
                    stuff = '\\' + stuff
                res = '%s[%s]' % (res, stuff)
        else:
            res = res + re.escape(c)
    return res + '\Z(?ms)'
<|endoftext|>


<commit_before>from django.db.models import FileField
from django.forms import forms
from django.template.defaultfilters import filesizeformat
from django.utils.translation import ugettext_lazy as _

class ContentTypeRestrictedFileField(FileField):
    """
    Same as FileField, but you can specify:
        * content_types - list containing allowed content_types. Example: ['application/pdf', 'image/jpeg']
        * max_upload_size - a number indicating the maximum file size allowed for upload.
            2.5MB - 2621440
            5MB - 5242880
            10MB - 10485760
            20MB - 20971520
            50MB - 5242880
            100MB 104857600
            250MB - 214958080
            500MB - 429916160
    """
    def __init__(self, *args, **kwargs):
        self.content_types = kwargs.pop("content_types")
        self.max_upload_size = kwargs.pop("max_upload_size")

        super(ContentTypeRestrictedFileField, self).__init__(*args, **kwargs)

    def clean(self, *args, **kwargs):
        data = super(ContentTypeRestrictedFileField, self).clean(*args, **kwargs)

        file = data.file
        try:
            content_type = file.content_type
            if content_type in self.content_types:
                if file._size > self.max_upload_size:
                    raise forms.ValidationError(_('Please keep filesize under %s. Current filesize %s') % (filesizeformat(self.max_upload_size), filesizeformat(file._size)))
            else:
                raise forms.ValidationError(_('Filetype not supported.'))
        except AttributeError:
            pass

        return data<commit_msg>add ContentTypeRestrictedFileField field<commit_after>from django.db.models import FileField
from django.forms import forms
from django.template.defaultfilters import filesizeformat
from django.utils.translation import ugettext_lazy as _

class ContentTypeRestrictedFileField(FileField):
    """
    Same as FileField, but you can specify:
        * content_types - list containing allowed content_types. Example: ['application/pdf', 'image/jpeg']
        * max_upload_size - a number indicating the maximum file size allowed for upload.
            2.5MB - 2621440
            5MB - 5242880
            10MB - 10485760
            20MB - 20971520
            50MB - 52428800
            100MB 104857600
            250MB - 214958080
            500MB - 429916160
    """
    def __init__(self, *args, **kwargs):
        self.content_types = kwargs.pop("content_types", [])
        self.max_upload_size = kwargs.pop("max_upload_size", 5242880)

        super(ContentTypeRestrictedFileField, self).__init__(*args, **kwargs)

    def clean(self, *args, **kwargs):
        data = super(ContentTypeRestrictedFileField, self).clean(*args, **kwargs)

        file = data.file
        try:
            content_type = file.content_type
            if content_type in self.content_types:
                if file._size > self.max_upload_size:
                    raise forms.ValidationError(_('Please keep filesize under %s. Current filesize %s') % (filesizeformat(self.max_upload_size), filesizeformat(file._size)))
            else:
                raise forms.ValidationError(_('Filetype not supported.'))
        except AttributeError:
            pass

        return data<|endoftext|>


<commit_before># Author: Edison Suen
# Description: infoSFU aggregates useful information from SFU about courses

from __future__ import print_function
from builtins import input
import requests
import time
import os
import platform

headers = {'User-agent': "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"}
delimiter = "=============================================================================="
system = platform.system()
clear = ''

if system == 'Windows':
	clear = 'cls'
elif system == 'Linux':
	clear = 'clear'

def main():
	os.system(clear)
	print("Welcome to infoSFU")
	ans = True
	while ans:
		print("1. Find general course information\n2. Find information about a section\n3. Exit program")
		ans = input("Please select an option: ")
		if ans == "1":
			course_information()
			print("Would you like to continue? (y/n)")
			user_input = input("").lower()
			print("\n")
			if user_input == "y":
				ans = True
			elif user_input == "n":
				ans = False
			else:
				print("Not a valid option")
		elif ans == "2":
			section_information()
			print("Would you like to continue? (y/n)")
			user_input = input("").lower()
			print("\n")
			if user_input == "y":
				ans = True
			elif user_input == "n":
				ans = False
			else:
				print("Not a valid option")
		elif ans == "3":
			ans = False
		else:
			print("\nInvalid choice.")

def course_information():
	print("[Semester Year Department Course-number]",end="\n")
	user_input = input("").split()
	user_input = [i.lower() for i in user_input]
	try:
		if user_input[0].lower() == 'fall':
			final_digit = '7'
		elif user_input[0].lower() == 'spring':
			final_digit = '1'
		else: # summer
			final_digit = '4'

		term_code = int('1'+user_input[1][-2:]+final_digit)

		calender_url = "http://www.sfu.ca/bin/wcm/academic-calendar?{}/{}/courses/{}/{}".format(user_input[1],user_input[0],user_input[2],user_input[3])
		sections_url = "http://api.lib.sfu.ca/courses/sections?term={}&department={}&number={}".format(term_code,user_input[2],user_input[3])
		check_sections_url = "http://www.sfu.ca/bin/wcm/course-outlines?{}/{}/{}/{}/".format(user_input[1],user_input[0],user_input[2],user_input[3])
		cycle = ['title','description','prerequisites']

		response = requests.get(calender_url,headers=headers)
		if response.status_code != 200:
			print("Error: ",response.status_code)
		data = response.json()
		print("\n")
		print(delimiter)
		for i in cycle:
			if data[i].find('Prerequisite:') > -1 and data[i] != data['prerequisites']:
				data[i] = data[i][0:data[i].find('Prerequisite:')]
				print(data[i])
			else:
				print(data[i])
			print("\n")
		response_sections = requests.get(sections_url,headers=headers)
		if response.status_code != 200:
			print("Error: ",response.status_code)
		sections = response_sections.json()

		lectures = []
		tutorials = []
		labs = []

		response_check_sections = requests.get(check_sections_url,headers=headers)
		if response.status_code != 200:
			print("Error: ",response.status_code)
		check_sections = response_check_sections.json()
		for i in sections:
			for j in check_sections:
				if i == j['text']:
					if j['sectionCode'] == 'LEC':
						lectures.append(j['text'])
					elif j['sectionCode'] == 'TUT':
						tutorials.append(j['text'])
					else:
						labs.append(j['text'])

		if len(lectures) > 0:
			print("Lecture section(s):",end=" ")
			print(", ".join([x for x in lectures]))

		if len(tutorials) > 0:
			print("Tutorial section(s):",end=" ")
			print(", ".join([x for x in tutorials]))
		if len(labs) > 0:
			print("Lecture section(s):",end=" ")
			print(", ".join([x for x in labs]))
		print(delimiter)
		print("\n")
	except IndexError:
		print("You have not entered all the required information as specified.")
		print("\n")

def section_information():
	print("[Semester Year Department Course-number Section]")
	user_input = input("").split()
	user_input = [i.lower() for i in user_input]
	try:
		sec_info_url = 'http://www.sfu.ca/bin/wcm/course-outlines?{}/{}/{}/{}/{}'.format(
					user_input[1],user_input[0],user_input[2],user_input[3],user_input[4])

		response = requests.get(sec_info_url,headers=headers)
		if response.status_code != 200:
			print("Error: ",response.status_code)
		sec_info = response.json()
		print("\n")
		print("Instructor: {}".format(sec_info['instructor'][0]['name']))
		print("Start date: {}".format(sec_info['courseSchedule'][0]['startDate'].replace("00:00:00 PDT","").replace("  "," ")))
		print("End date: {}".format(sec_info['courseSchedule'][0]['endDate'].replace("00:00:00 PST","").replace("  "," ")))
		print("Units: {}".format(sec_info['info']['units']))
		print("\n")
		for i in range(len(sec_info['courseSchedule'])):
			print("Day(s): {}".format(sec_info['courseSchedule'][i]['days']))
			print("Location: {} {}, {} campus".format(
										sec_info['courseSchedule'][i]['buildingCode'],
										sec_info['courseSchedule'][i]['roomNumber'],
										sec_info['courseSchedule'][i]['campus'])
										)
			start_time = time.strptime(sec_info['courseSchedule'][i]['startTime'],"%H:%M")
			start_time = time.strftime( "%-I:%M %p", start_time)
			end_time = time.strptime(sec_info['courseSchedule'][i]['endTime'],"%H:%M")
			end_time = time.strftime( "%-I:%M %p", end_time)
			print("Time: {}-{}".format(start_time, end_time))
			print("\n")

		if len(sec_info['examSchedule']) > 0:
			print(delimiter)
			print("Final exam date: {}".format(sec_info['examSchedule'][0]['startDate'].replace("00:00:00 PST","").replace("  "," ")))
			try:
				print("Final exam location: {} {}, {}".format(
											sec_info['examSchedule'][0]['buildingCode'],
											sec_info['examSchedule'][0]['roomNumber'],
											sec_info['examSchedule'][0]['campus'])
											)
			except KeyError:
				print("No location has been currently listed.")
			finalExam_start_time = time.strptime(sec_info['examSchedule'][0]['startTime'],"%H:%M")
			finalExam_start_time = time.strftime( "%-I:%M %p",finalExam_start_time)
			finalExam_end_time = time.strptime(sec_info['examSchedule'][0]['endTime'],"%H:%M")
			finalExam_end_time = time.strftime( "%-I:%M %p",finalExam_end_time)
			print("Final exam time: {}-{}".format(finalExam_start_time, finalExam_end_time))
			print("\n")
		else:
			print("There is no final exam currently listed.")
	except IndexError:
		print("You have not entered all the required information as specified.")
		print("\n")

# ================================================================================

if __name__ == "__main__":
	main()<commit_msg>Now uses regex<commit_after># Author: Edison Suen
# Description: infoSFU aggregates useful information from SFU about courses

from __future__ import print_function
from builtins import input
import requests
import time
import os
import platform
import re

headers = {'User-agent': "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"}
delimiter = "=============================================================================="
system = platform.system()
clear = ''

if system == 'Windows':
	clear = 'cls'
elif system == 'Linux':
	clear = 'clear'

semesters = ['fall','spring','summer']
regex_year = re.compile(r"20[0-9]{2}$")
regex_dept = re.compile(r"[a-z]{2,4}$")
regex_coursenum = re.compile(r"[1-9][0-9]{2}$")
regex_sectionnum = re.compile(r"[decfjuwxyzv][1-9][0-9]{2}$")

def main():
	os.system(clear)
	print("Welcome to infoSFU")
	ans = True
	while ans:
		print("1. Find general course information\n2. Find information about a section\n3. Exit program")
		ans = input("Please select an option: ")
		if ans == "1":
			course_information()
			print("Would you like to continue? (y/n)")
			user_input = input().lower()
			print("\n")
			if user_input == "y":
				ans = True
			elif user_input == "n":
				ans = False
			else:
				print("Not a valid option")
		elif ans == "2":
			section_information()
			print("Would you like to continue? (y/n)")
			user_input = input().lower()
			print("\n")
			if user_input == "y":
				ans = True
			elif user_input == "n":
				ans = False
			else:
				print("Not a valid option")
		elif ans == "3":
			ans = False
		else:
			print("\nInvalid choice.")

def course_information():
	user_input = input().split()
	user_input = [i.lower() for i in user_input]
	semester = ''
	for i in user_input:
		for j in semesters:
			if i == j:
				semester = j

	year = list(filter(regex_year.match,user_input))[0]
	dept = list(filter(regex_dept.match,user_input))
	dept = [i for i in dept if i not in semesters][0]
	coursenum = list(filter(regex_coursenum.match,user_input))[0]

	try:
		if semester.lower() == 'fall':
			final_digit = '7'
		elif semester.lower() == 'spring':
			final_digit = '1'
		else: # summer
			final_digit = '4'

		term_code = int('1'+year[-2:]+final_digit)

		calender_url = "http://www.sfu.ca/bin/wcm/academic-calendar?{}/{}/courses/{}/{}".format(year,semester,dept,coursenum)
		sections_url = "http://api.lib.sfu.ca/courses/sections?term={}&department={}&number={}".format(term_code,dept,coursenum)
		check_sections_url = "http://www.sfu.ca/bin/wcm/course-outlines?{}/{}/{}/{}/".format(year,semester,dept,coursenum)
		cycle = ['title','description','prerequisites']

		response = requests.get(calender_url,headers=headers)
		if response.status_code != 200:
			print("Error: ",response.status_code)
		data = response.json()
		print("\n")
		print(delimiter)
		for i in cycle:
			if data[i].find('Prerequisite:') > -1 and data[i] != data['prerequisites']:
				data[i] = data[i][0:data[i].find('Prerequisite:')]
				print(data[i])
			else:
				print(data[i])
			print("\n")
		response_sections = requests.get(sections_url,headers=headers)
		if response.status_code != 200:
			print("Error: ",response.status_code)
		sections = response_sections.json()

		lectures = []
		tutorials = []
		labs = []

		response_check_sections = requests.get(check_sections_url,headers=headers)
		if response.status_code != 200:
			print("Error: ",response.status_code)
		check_sections = response_check_sections.json()
		for i in sections:
			for j in check_sections:
				if i == j['text']:
					if j['sectionCode'] == 'LEC':
						lectures.append(j['text'])
					elif j['sectionCode'] == 'TUT':
						tutorials.append(j['text'])
					else:
						labs.append(j['text'])

		if len(lectures) > 0:
			print("Lecture section(s):",end=" ")
			print(", ".join([x for x in lectures]))

		if len(tutorials) > 0:
			print("Tutorial section(s):",end=" ")
			print(", ".join([x for x in tutorials]))
		if len(labs) > 0:
			print("Lecture section(s):",end=" ")
			print(", ".join([x for x in labs]))
		print(delimiter)
		print("\n")

	except IndexError:
		print("You have not entered all the required information as specified.")
		print("\n")

def section_information():
	user_input = input().split()
	user_input = [i.lower() for i in user_input]
	semester = ''
	for i in user_input:
		for j in semesters:
			if i == j:
				semester = j

	year = list(filter(regex_year.match,user_input))[0]
	dept = list(filter(regex_dept.match,user_input))
	dept = [i for i in dept if i not in semesters][0]
	coursenum = list(filter(regex_coursenum.match,user_input))[0]
	sectionnum = list(filter(regex_sectionnum.match,user_input))[0]

	try:
		sec_info_url = 'http://www.sfu.ca/bin/wcm/course-outlines?{}/{}/{}/{}/{}'.format(
					year,semester,dept,coursenum,sectionnum)

		response = requests.get(sec_info_url,headers=headers)
		if response.status_code != 200:
			print("Error: ",response.status_code)
		sec_info = response.json()
		print("\n")
		print("Instructor: {}".format(sec_info['instructor'][0]['name']))
		print("Start date: {}".format(sec_info['courseSchedule'][0]['startDate'].replace("00:00:00 PDT","").replace("  "," ")))
		print("End date: {}".format(sec_info['courseSchedule'][0]['endDate'].replace("00:00:00 PST","").replace("  "," ")))
		print("Units: {}".format(sec_info['info']['units']))
		print("\n")
		for i in range(len(sec_info['courseSchedule'])):
			print("Day(s): {}".format(sec_info['courseSchedule'][i]['days']))
			print("Location: {} {}, {} campus".format(
										sec_info['courseSchedule'][i]['buildingCode'],
										sec_info['courseSchedule'][i]['roomNumber'],
										sec_info['courseSchedule'][i]['campus'])
										)
			start_time = time.strptime(sec_info['courseSchedule'][i]['startTime'],"%H:%M")
			start_time = time.strftime( "%-I:%M %p", start_time)
			end_time = time.strptime(sec_info['courseSchedule'][i]['endTime'],"%H:%M")
			end_time = time.strftime( "%-I:%M %p", end_time)
			print("Time: {}-{}".format(start_time, end_time))
			print("\n")

		if len(sec_info['examSchedule']) > 0:
			print(delimiter)
			print("Final exam date: {}".format(sec_info['examSchedule'][0]['startDate'].replace("00:00:00 PST","").replace("  "," ")))
			try:
				print("Final exam location: {} {}, {}".format(
											sec_info['examSchedule'][0]['buildingCode'],
											sec_info['examSchedule'][0]['roomNumber'],
											sec_info['examSchedule'][0]['campus'])
											)
			except KeyError:
				print("No location has been currently listed.")
			finalExam_start_time = time.strptime(sec_info['examSchedule'][0]['startTime'],"%H:%M")
			finalExam_start_time = time.strftime( "%-I:%M %p",finalExam_start_time)
			finalExam_end_time = time.strptime(sec_info['examSchedule'][0]['endTime'],"%H:%M")
			finalExam_end_time = time.strftime( "%-I:%M %p",finalExam_end_time)
			print("Final exam time: {}-{}".format(finalExam_start_time, finalExam_end_time))
			print("\n")
		else:
			print("There is no final exam currently listed.")
	except IndexError:
		print("You have not entered all the required information as specified.")
		print("\n")

# ================================================================================

if __name__ == "__main__":
	main()<|endoftext|>


<commit_before>#!/usr/bin/python
#    Copyright 2013, 2014 Joshua Charles Campbell, Alex Wilson, Eddie Santos
#
#    This file is part of UnnaturalCode.
#
#    UnnaturalCode is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    UnnaturalCode is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with UnnaturalCode.  If not, see <http://www.gnu.org/licenses/>.

from unnaturalcode.ucUtil import *
from unnaturalcode.unnaturalCode import *
from logging import debug, info, warning, error
import urllib
import re

allWhitespace = re.compile('^\s+$')
whitespace = re.compile('\s')

class genericLexeme(ucLexeme):

    @classmethod
    def stringify_build(cls, t, v):
        """
        Stringify a lexeme: produce a string describing it. For MITLM the
        string can't contain any whitespace.
        """
        if t is None:
            t = "UNKNOWN"
        if len(v) > 20:
            v = v[0:20]
        # quote all whitespace, colons, backslashes, control and unicode, and quotes
        v = urllib.quote(v, "!#$%&()*+,-./;<=>?@[]^_`{|}~")
        assert(re.match('\w+$', t) is not none), "Lexeme type should be a word %s %s." % (t, v)
        return t + ":" + v

class genericSource(ucSource):

    lexemeClass = genericLexeme

    def deLex(self):
        line = 1
        col = 0
        src = ""
        for l in self:
            for i in range(line, l.start.line):
                src += os.linesep
                col = 0
                line += 1
            for i in range(col, l.start.col):
                src += " "
                col += 1
            src += l.val
            col += len(l.val)
            nls = l.val.count(os.linesep)
            if (nls > 0):
                line += nls
                col = len(l.val.splitlines().pop())
        return src

    def scrubbed(self):
        """Clean up generic source code removing extra whitespace tokens and comments"""
        ls = copy(self)
        assert len(ls)
        i = 0
        r = []
        for i in range(0, len(ls)):
            #if allWhitespace.match(ls[i].val) and i < len(ls)-1 and allWhitespace.match(ls[i+1]):
                #continue
            if allWhitespace.match(ls[i].val):
                continue
            else:
                val = ls[i].val
                val = whitespace.sub("", val)
                r.append(ls[i].__class__((ls[i][0], val, ls[i][2], ls[i][3], ls[i][4])))
        assert len(r)
        return genericSource(r)

<commit_msg>fix typeo<commit_after>#!/usr/bin/python
#    Copyright 2013, 2014 Joshua Charles Campbell, Alex Wilson, Eddie Santos
#
#    This file is part of UnnaturalCode.
#
#    UnnaturalCode is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    UnnaturalCode is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with UnnaturalCode.  If not, see <http://www.gnu.org/licenses/>.

from unnaturalcode.ucUtil import *
from unnaturalcode.unnaturalCode import *
from logging import debug, info, warning, error
import urllib
import re

allWhitespace = re.compile('^\s+$')
whitespace = re.compile('\s')

class genericLexeme(ucLexeme):

    @classmethod
    def stringify_build(cls, t, v):
        """
        Stringify a lexeme: produce a string describing it. For MITLM the
        string can't contain any whitespace.
        """
        if t is None:
            t = "UNKNOWN"
        if len(v) > 20:
            v = v[0:20]
        # quote all whitespace, colons, backslashes, control and unicode, and quotes
        v = urllib.quote(v, "!#$%&()*+,-./;<=>?@[]^_`{|}~")
        assert(re.match('\w+$', t) is not None), "Lexeme type should be a word %s %s." % (t, v)
        return t + ":" + v

class genericSource(ucSource):

    lexemeClass = genericLexeme

    def deLex(self):
        line = 1
        col = 0
        src = ""
        for l in self:
            for i in range(line, l.start.line):
                src += os.linesep
                col = 0
                line += 1
            for i in range(col, l.start.col):
                src += " "
                col += 1
            src += l.val
            col += len(l.val)
            nls = l.val.count(os.linesep)
            if (nls > 0):
                line += nls
                col = len(l.val.splitlines().pop())
        return src

    def scrubbed(self):
        """Clean up generic source code removing extra whitespace tokens and comments"""
        ls = copy(self)
        assert len(ls)
        i = 0
        r = []
        for i in range(0, len(ls)):
            #if allWhitespace.match(ls[i].val) and i < len(ls)-1 and allWhitespace.match(ls[i+1]):
                #continue
            if allWhitespace.match(ls[i].val):
                continue
            else:
                val = ls[i].val
                val = whitespace.sub("", val)
                r.append(ls[i].__class__((ls[i][0], val, ls[i][2], ls[i][3], ls[i][4])))
        assert len(r)
        return genericSource(r)

<|endoftext|>


<commit_before>from background_task import background

# generic imports
import glob
import json
import math
import os
import polling
import time
import uuid
from zipfile import ZipFile

# Get an instance of a logger
import logging
logger = logging.getLogger(__name__)

from core import models as models

'''
This file provides background tasks that are performed with Django-Background-Tasks
'''

@background(schedule=1)
def delete_model_instance(instance_model, instance_id, verbose_name=uuid.uuid4().urn):

	'''
	Background task to delete generic DB model instance
	'''

	# try:

	# get model
	m = getattr(models, instance_model, None)

	if m:

		# get model instance
		i = m.objects.get(pk=int(instance_id))
		logger.debug('retrieved %s model, instance ID %s, deleting' % (m.__name__, instance_id))

		# delete
		return i.delete()

	else:
		logger.debug('Combine model %s not found, aborting' % (instance_model))

@background(schedule=1)
def download_and_index_bulk_data(dbdd_id, verbose_name=uuid.uuid4().urn):

	'''
	Background task driver to manage downloading and indexing of bulk data

	Args:
		dbdd_id (int): ID of DPLABulkDataDownload (dbdd) instance
	'''

	# init bulk download instance
	dbdd = models.DPLABulkDataDownload.objects.get(pk=dbdd_id)

	# init data client with filepath
	dbdc = models.DPLABulkDataClient()

	# download data
	logger.debug('downloading %s' % dbdd.s3_key)
	dbdd.status = 'downloading'
	dbdd.save()
	download_results = dbdc.download_bulk_data(dbdd.s3_key, dbdd.filepath)

	# index data
	logger.debug('indexing %s' % dbdd.filepath)
	dbdd.status = 'indexing'
	dbdd.save()
	es_index = dbdc.index_to_es(dbdd.s3_key, dbdd.filepath)

	# update and return
	dbdd.es_index = es_index
	dbdd.status = 'finished'
	dbdd.save()

@background(schedule=1)
def create_validation_report(ct_id):

	'''
	Function to generate a Validation Report for a Job as a bg task

	Args:
		request (django.request): request object with parameters needed for report generation

	Returns:
		location on disk
	'''

	# get CombineTask (ct)
	ct = models.CombineBackgroundTask.objects.get(pk=int(ct_id))

	# get CombineJob
	cjob = models.CombineJob.get_combine_job(int(ct.task_params['job_id']))

	# run report generation
	report_output = cjob.generate_validation_report(
		report_format=ct.task_params['report_format'],
		validation_scenarios=ct.task_params['validation_scenarios'],
		mapped_field_include=ct.task_params['mapped_field_include']
	)
	logger.debug('validation report output: %s' % report_output)

	# save validation report output to Combine Task output
	ct.task_output_json = json.dumps({
		'report_format':ct.task_params['report_format'],
		'report_output':report_output
	})
	ct.save()

@background(schedule=1)
def job_export_mapped_fields(ct_id):

	# get CombineTask (ct)
	ct = models.CombineBackgroundTask.objects.get(pk=int(ct_id))

	# get CombineJob
	cjob = models.CombineJob.get_combine_job(int(ct.task_params['job_id']))

	# set output filename
	export_output = '/tmp/job_%s_mapped_fields.csv' % cjob.job.id

	# issue es2csv as os command
	cmd = "es2csv -q '*' -i 'j%(job_id)s' -D 'record' -o '%(export_output)s'" % {
		'job_id':cjob.job.id,
		'export_output':export_output
	}

	# handle kibana style
	if ct.task_params['kibana_style']:
		cmd += ' -k'

	logger.debug(cmd)
	os.system(cmd)

	# save export output to Combine Task output
	ct.task_output_json = json.dumps({
		'export_output':export_output,
		'name':export_output.split('/')[-1]
	})
	ct.save()

@background(schedule=1)
def job_export_documents(ct_id):

	'''
	- submit livy job and poll until complete
		- use livy session from cjob (works, but awkward way to get this)
	- add wrapper element to file parts
	- rename file parts
	- tar/zip together
	'''

	# get CombineTask (ct)
	try:
		ct = models.CombineBackgroundTask.objects.get(pk=int(ct_id))
		logger.debug('using %s' % ct)

		# get CombineJob
		cjob = models.CombineJob.get_combine_job(int(ct.task_params['job_id']))

		# generate spark code
		output_path = '/tmp/%s' % str(uuid.uuid4())

		spark_code = "import math,uuid\nfrom console import *\ndf = get_job_as_df(spark, %(job_id)d)\ndf.select('document').rdd.repartition(math.ceil(df.count()/%(records_per_file)d)).map(lambda row: row.document.replace('<?xml version=\"1.0\" encoding=\"UTF-8\"?>','')).saveAsTextFile('file://%(output_path)s')" % {
			'job_id':cjob.job.id,
			'output_path':output_path,
			'records_per_file':ct.task_params['records_per_file']
		}
		logger.debug(spark_code)

		# submit to livy
		logger.debug('submitting code to Spark')
		submit = models.LivyClient().submit_job(cjob.livy_session.session_id, {'code':spark_code})

		# poll until complete
		def spark_job_done(response):
			return response['state'] == 'available'

		results = polling.poll(lambda: models.LivyClient().job_status(submit.headers['Location']).json(), check_success=spark_job_done, step=5, poll_forever=True)
		logger.debug(results)

		# loop through parts, group XML docs with rool XML element, and save as new XML file
		logger.debug('grouping documents in XML files')

		export_parts = glob.glob('%s/part*' % output_path)
		logger.debug('found %s documents to write as XML' % len(export_parts))
		for part in export_parts:
			with open('%s.xml' % part, 'w') as f:
				f.write('<?xml version="1.0" encoding="UTF-8"?><documents>')
				with open(part) as f_part:
					f.write(f_part.read())
				f.write('</documents>')

		# save file list pre-archive for cleanup
		pre_archive_files = [ '%s/%s' % (output_path, f) for f in os.listdir(output_path) ]

		# create archive file of loose XML files
		archive_filename_root = 'j_%s_documents' % cjob.job.id

		# zip
		if ct.task_params['archive_type'] == 'zip':
			logger.debug('creating zip archive')
			export_output_archive = '%s/%s.zip' % (output_path, archive_filename_root)
			with ZipFile(export_output_archive,'w') as zip:
				for f in glob.glob('%s/*.xml' % output_path):
					zip.write(f, os.path.basename(f))

			# cleanup directory
			logger.debug('cleaning up: %s' % pre_archive_files)
			for f in pre_archive_files:
				os.remove(f)

		# tar
		if ct.task_params['archive_type'] == 'tar':
			logger.debug('creating tar archive')
			export_output_archive = '%s/%s.tar' % (output_path, archive_filename_root)

		# save export output to Combine Task output
		ct.task_output_json = json.dumps({
			'export_output':export_output_archive
		})
		ct.save()
		logger.debug(ct.task_output_json)

	except Exception as e:

		logger.debug(str(e))

		# attempt to capture error and return for task
		ct.task_output_json = json.dumps({
			'error':str(e)
		})
		ct.save()

@background(schedule=1)
def test_bg_task(duration=5):

	logger.debug('preparing to sleep for: %s' % duration)
	time.sleep(duration)
	return "we had a good nap"

<commit_msg>tar files written<commit_after>from background_task import background

# generic imports
import glob
import json
import math
import os
import polling
import tarfile
import time
import uuid
from zipfile import ZipFile

# Get an instance of a logger
import logging
logger = logging.getLogger(__name__)

from core import models as models

'''
This file provides background tasks that are performed with Django-Background-Tasks
'''

@background(schedule=1)
def delete_model_instance(instance_model, instance_id, verbose_name=uuid.uuid4().urn):

	'''
	Background task to delete generic DB model instance
	'''

	# try:

	# get model
	m = getattr(models, instance_model, None)

	if m:

		# get model instance
		i = m.objects.get(pk=int(instance_id))
		logger.debug('retrieved %s model, instance ID %s, deleting' % (m.__name__, instance_id))

		# delete
		return i.delete()

	else:
		logger.debug('Combine model %s not found, aborting' % (instance_model))

@background(schedule=1)
def download_and_index_bulk_data(dbdd_id, verbose_name=uuid.uuid4().urn):

	'''
	Background task driver to manage downloading and indexing of bulk data

	Args:
		dbdd_id (int): ID of DPLABulkDataDownload (dbdd) instance
	'''

	# init bulk download instance
	dbdd = models.DPLABulkDataDownload.objects.get(pk=dbdd_id)

	# init data client with filepath
	dbdc = models.DPLABulkDataClient()

	# download data
	logger.debug('downloading %s' % dbdd.s3_key)
	dbdd.status = 'downloading'
	dbdd.save()
	download_results = dbdc.download_bulk_data(dbdd.s3_key, dbdd.filepath)

	# index data
	logger.debug('indexing %s' % dbdd.filepath)
	dbdd.status = 'indexing'
	dbdd.save()
	es_index = dbdc.index_to_es(dbdd.s3_key, dbdd.filepath)

	# update and return
	dbdd.es_index = es_index
	dbdd.status = 'finished'
	dbdd.save()

@background(schedule=1)
def create_validation_report(ct_id):

	'''
	Function to generate a Validation Report for a Job as a bg task

	Args:
		request (django.request): request object with parameters needed for report generation

	Returns:
		location on disk
	'''

	# get CombineTask (ct)
	ct = models.CombineBackgroundTask.objects.get(pk=int(ct_id))

	# get CombineJob
	cjob = models.CombineJob.get_combine_job(int(ct.task_params['job_id']))

	# run report generation
	report_output = cjob.generate_validation_report(
		report_format=ct.task_params['report_format'],
		validation_scenarios=ct.task_params['validation_scenarios'],
		mapped_field_include=ct.task_params['mapped_field_include']
	)
	logger.debug('validation report output: %s' % report_output)

	# save validation report output to Combine Task output
	ct.task_output_json = json.dumps({
		'report_format':ct.task_params['report_format'],
		'report_output':report_output
	})
	ct.save()

@background(schedule=1)
def job_export_mapped_fields(ct_id):

	# get CombineTask (ct)
	ct = models.CombineBackgroundTask.objects.get(pk=int(ct_id))

	# get CombineJob
	cjob = models.CombineJob.get_combine_job(int(ct.task_params['job_id']))

	# set output filename
	export_output = '/tmp/job_%s_mapped_fields.csv' % cjob.job.id

	# issue es2csv as os command
	cmd = "es2csv -q '*' -i 'j%(job_id)s' -D 'record' -o '%(export_output)s'" % {
		'job_id':cjob.job.id,
		'export_output':export_output
	}

	# handle kibana style
	if ct.task_params['kibana_style']:
		cmd += ' -k'

	logger.debug(cmd)
	os.system(cmd)

	# save export output to Combine Task output
	ct.task_output_json = json.dumps({
		'export_output':export_output,
		'name':export_output.split('/')[-1]
	})
	ct.save()

@background(schedule=1)
def job_export_documents(ct_id):

	'''
	- submit livy job and poll until complete
		- use livy session from cjob (works, but awkward way to get this)
	- add wrapper element to file parts
	- rename file parts
	- tar/zip together
	'''

	# get CombineTask (ct)
	try:
		ct = models.CombineBackgroundTask.objects.get(pk=int(ct_id))
		logger.debug('using %s' % ct)

		# get CombineJob
		cjob = models.CombineJob.get_combine_job(int(ct.task_params['job_id']))

		# generate spark code
		output_path = '/tmp/%s' % str(uuid.uuid4())

		spark_code = "import math,uuid\nfrom console import *\ndf = get_job_as_df(spark, %(job_id)d)\ndf.select('document').rdd.repartition(math.ceil(df.count()/%(records_per_file)d)).map(lambda row: row.document.replace('<?xml version=\"1.0\" encoding=\"UTF-8\"?>','')).saveAsTextFile('file://%(output_path)s')" % {
			'job_id':cjob.job.id,
			'output_path':output_path,
			'records_per_file':ct.task_params['records_per_file']
		}
		logger.debug(spark_code)

		# submit to livy
		logger.debug('submitting code to Spark')
		submit = models.LivyClient().submit_job(cjob.livy_session.session_id, {'code':spark_code})

		# poll until complete
		def spark_job_done(response):
			return response['state'] == 'available'

		results = polling.poll(lambda: models.LivyClient().job_status(submit.headers['Location']).json(), check_success=spark_job_done, step=5, poll_forever=True)
		logger.debug(results)

		# loop through parts, group XML docs with rool XML element, and save as new XML file
		logger.debug('grouping documents in XML files')

		export_parts = glob.glob('%s/part*' % output_path)
		logger.debug('found %s documents to write as XML' % len(export_parts))
		for part in export_parts:
			with open('%s.xml' % part, 'w') as f:
				f.write('<?xml version="1.0" encoding="UTF-8"?><documents>')
				with open(part) as f_part:
					f.write(f_part.read())
				f.write('</documents>')

		# save file list pre-archive for cleanup
		pre_archive_files = [ '%s/%s' % (output_path, f) for f in os.listdir(output_path) ]

		# create archive file of loose XML files
		archive_filename_root = 'j_%s_documents' % cjob.job.id

		# zip
		if ct.task_params['archive_type'] == 'zip':
			logger.debug('creating zip archive')
			export_output_archive = '%s/%s.zip' % (output_path, archive_filename_root)
			with ZipFile(export_output_archive,'w') as zip:
				for f in glob.glob('%s/*.xml' % output_path):
					zip.write(f, os.path.basename(f))

		# tar
		if ct.task_params['archive_type'] == 'tar':
			logger.debug('creating tar archive')
			export_output_archive = '%s/%s.tar' % (output_path, archive_filename_root)

			with tarfile.open(export_output_archive, 'w') as tar:
				for f in glob.glob('%s/*.xml' % output_path):
					tar.add(f, arcname=os.path.basename(f))

		# cleanup directory
		logger.debug('cleaning up: %s' % pre_archive_files)
		for f in pre_archive_files:
			os.remove(f)

		# save export output to Combine Task output
		ct.task_output_json = json.dumps({
			'export_output':export_output_archive
		})
		ct.save()
		logger.debug(ct.task_output_json)

	except Exception as e:

		logger.debug(str(e))

		# attempt to capture error and return for task
		ct.task_output_json = json.dumps({
			'error':str(e)
		})
		ct.save()

@background(schedule=1)
def test_bg_task(duration=5):

	logger.debug('preparing to sleep for: %s' % duration)
	time.sleep(duration)
	return "we had a good nap"

<|endoftext|>


<commit_before># Copyright 2019 ForgeFlow S.L.
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from datetime import datetime

from odoo.exceptions import UserError, ValidationError
from odoo.tests.common import TransactionCase
from odoo.tools.misc import DEFAULT_SERVER_DATE_FORMAT

class TestAccountLockToDateUpdate(TransactionCase):
    def setUp(self):
        super(TestAccountLockToDateUpdate, self).setUp()
        self.company = self.env.ref("base.main_company")
        self.demo_user = self.env.ref("base.user_demo")
        self.adviser_group = self.env.ref("account.group_account_manager")
        self.UpdateLockToDateUpdateObj = self.env[
            "account.update.lock_to_date"
        ].with_user(self.demo_user)
        self.AccountObj = self.env["account.account"]
        self.AccountJournalObj = self.env["account.journal"]
        self.AccountMoveObj = self.env["account.move"]
        self.bank_journal = self.AccountJournalObj.create(
            {
                "name": "Bank Journal - BJ",
                "code": "BJ",
                "type": "bank",
                "company_id": self.company.id,
            }
        )
        self.account_type_recv = self.env.ref("account.data_account_type_receivable")
        self.account_type_rev = self.env.ref("account.data_account_type_revenue")

        self.account_recv = self.AccountObj.create(
            {
                "code": "RECV_DR",
                "name": "Receivable (test)",
                "reconcile": True,
                "user_type_id": self.account_type_recv.id,
            }
        )
        self.account_sale = self.AccountObj.create(
            {
                "code": "SALE_DR",
                "name": "Receivable (sale)",
                "reconcile": True,
                "user_type_id": self.account_type_rev.id,
            }
        )

    def create_account_move(self, date_str):
        return self.AccountMoveObj.create(
            {
                "journal_id": self.bank_journal.id,
                "date": date_str,
                "line_ids": [
                    (
                        0,
                        0,
                        {
                            "name": "Debit",
                            "debit": 1000,
                            "account_id": self.account_recv.id,
                        },
                    ),
                    (
                        0,
                        0,
                        {
                            "name": "Credit",
                            "credit": 1000,
                            "account_id": self.account_sale.id,
                        },
                    ),
                ],
            }
        )

    def create_account_lock_date_update(self):
        return self.UpdateLockToDateUpdateObj.create({"company_id": self.company.id})

    def test_01_update_without_access(self):
        wizard = self.create_account_lock_date_update()
        wizard.write(
            {
                "period_lock_to_date": "2900-01-01",
                "fiscalyear_lock_to_date": "2900-01-01",
            }
        )
        self.demo_user.write({"groups_id": [(3, self.adviser_group.id)]})
        with self.assertRaises(UserError):
            wizard.with_user(self.demo_user.id).execute()

    def test_02_update_with_access(self):
        wizard = self.create_account_lock_date_update()
        wizard.write(
            {
                "period_lock_to_date": "2900-01-01",
                "fiscalyear_lock_to_date": "2900-02-01",
            }
        )
        self.demo_user.write({"groups_id": [(4, self.adviser_group.id)]})
        wizard.with_user(self.demo_user.id).execute()
        self.assertEqual(
            self.company.period_lock_to_date,
            datetime.strptime("2900-01-01", DEFAULT_SERVER_DATE_FORMAT).date(),
        )
        self.assertEqual(
            self.company.fiscalyear_lock_to_date,
            datetime.strptime("2900-02-01", DEFAULT_SERVER_DATE_FORMAT).date(),
        )

    def test_03_create_move_outside_period(self):
        """We test that we cannot create journal entries after the
        locked date"""
        self.company.period_lock_to_date = "2900-01-01"
        self.company.fiscalyear_lock_to_date = "2900-02-01"
        move = self.create_account_move("2900-01-01")
        with self.assertRaises(UserError):
            move.with_user(self.demo_user.id).action_post()

    def test_04_create_move_inside_period(self):
        """We test that we can successfully create a journal entry
        within period that is not locked"""
        self.company.period_lock_to_date = "2900-01-01"
        self.company.fiscalyear_lock_to_date = "2900-02-01"
        move = self.create_account_move("2800-01-01")
        move.with_user(self.demo_user.id).action_post()
        self.assertEqual(move.state, "posted")

    def test_05_lock_period_with_draft_moves(self):
        """We test that we cannot change the fiscal year lock to date
           if there are draft journal entries after that date."""
        self.create_account_move("2900-02-01")
        with self.assertRaises(ValidationError):
            self.company.period_lock_to_date = "2900-01-01"
            self.company.fiscalyear_lock_to_date = "2900-02-01"
<commit_msg>[FIX] account_lock_to_date. Also adapt tests to ValidationError<commit_after># Copyright 2019 ForgeFlow S.L.
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).

from datetime import datetime

from odoo.exceptions import ValidationError
from odoo.tests.common import TransactionCase
from odoo.tools.misc import DEFAULT_SERVER_DATE_FORMAT

class TestAccountLockToDateUpdate(TransactionCase):
    def setUp(self):
        super(TestAccountLockToDateUpdate, self).setUp()
        self.company = self.env.ref("base.main_company")
        self.demo_user = self.env.ref("base.user_demo")
        self.adviser_group = self.env.ref("account.group_account_manager")
        self.UpdateLockToDateUpdateObj = self.env[
            "account.update.lock_to_date"
        ].with_user(self.demo_user)
        self.AccountObj = self.env["account.account"]
        self.AccountJournalObj = self.env["account.journal"]
        self.AccountMoveObj = self.env["account.move"]
        self.bank_journal = self.AccountJournalObj.create(
            {
                "name": "Bank Journal - BJ",
                "code": "BJ",
                "type": "bank",
                "company_id": self.company.id,
            }
        )
        self.account_type_recv = self.env.ref("account.data_account_type_receivable")
        self.account_type_rev = self.env.ref("account.data_account_type_revenue")

        self.account_recv = self.AccountObj.create(
            {
                "code": "RECV_DR",
                "name": "Receivable (test)",
                "reconcile": True,
                "user_type_id": self.account_type_recv.id,
            }
        )
        self.account_sale = self.AccountObj.create(
            {
                "code": "SALE_DR",
                "name": "Receivable (sale)",
                "reconcile": True,
                "user_type_id": self.account_type_rev.id,
            }
        )

    def create_account_move(self, date_str):
        return self.AccountMoveObj.create(
            {
                "journal_id": self.bank_journal.id,
                "date": date_str,
                "line_ids": [
                    (
                        0,
                        0,
                        {
                            "name": "Debit",
                            "debit": 1000,
                            "account_id": self.account_recv.id,
                        },
                    ),
                    (
                        0,
                        0,
                        {
                            "name": "Credit",
                            "credit": 1000,
                            "account_id": self.account_sale.id,
                        },
                    ),
                ],
            }
        )

    def create_account_lock_date_update(self):
        return self.UpdateLockToDateUpdateObj.create({"company_id": self.company.id})

    def test_01_update_without_access(self):
        wizard = self.create_account_lock_date_update()
        wizard.write(
            {
                "period_lock_to_date": "2900-01-01",
                "fiscalyear_lock_to_date": "2900-01-01",
            }
        )
        self.demo_user.write({"groups_id": [(3, self.adviser_group.id)]})
        with self.assertRaises(ValidationError):
            wizard.with_user(self.demo_user.id).execute()

    def test_02_update_with_access(self):
        wizard = self.create_account_lock_date_update()
        wizard.write(
            {
                "period_lock_to_date": "2900-01-01",
                "fiscalyear_lock_to_date": "2900-02-01",
            }
        )
        self.demo_user.write({"groups_id": [(4, self.adviser_group.id)]})
        wizard.with_user(self.demo_user.id).execute()
        self.assertEqual(
            self.company.period_lock_to_date,
            datetime.strptime("2900-01-01", DEFAULT_SERVER_DATE_FORMAT).date(),
        )
        self.assertEqual(
            self.company.fiscalyear_lock_to_date,
            datetime.strptime("2900-02-01", DEFAULT_SERVER_DATE_FORMAT).date(),
        )

    def test_03_create_move_outside_period(self):
        """We test that we cannot create journal entries after the
        locked date"""
        self.company.period_lock_to_date = "2900-01-01"
        self.company.fiscalyear_lock_to_date = "2900-02-01"
        move = self.create_account_move("2900-01-01")
        with self.assertRaises(ValidationError):
            move.with_user(self.demo_user.id).action_post()

    def test_04_create_move_inside_period(self):
        """We test that we can successfully create a journal entry
        within period that is not locked"""
        self.company.period_lock_to_date = "2900-01-01"
        self.company.fiscalyear_lock_to_date = "2900-02-01"
        move = self.create_account_move("2800-01-01")
        move.with_user(self.demo_user.id).action_post()
        self.assertEqual(move.state, "posted")

    def test_05_lock_period_with_draft_moves(self):
        """We test that we cannot change the fiscal year lock to date
           if there are draft journal entries after that date."""
        self.create_account_move("2900-02-01")
        with self.assertRaises(ValidationError):
            self.company.period_lock_to_date = "2900-01-01"
            self.company.fiscalyear_lock_to_date = "2900-02-01"
<|endoftext|>


<commit_before><commit_msg>Fix UTF8 with ldap sync cmd and Python 3<commit_after><|endoftext|>


<commit_before>print "Hej verden"
<commit_msg>updated to danish<commit_after>print "Hej verden (danish)"
<|endoftext|>


<commit_before>import requests
import re,os
from bs4 import BeautifulSoup

COOKIE = ""

cookies = {
    'gsScrollPos': '',
    'MoodleSession': COOKIE,
}

headers = {
    'Accept-Encoding': 'gzip, deflate, sdch',
    'Accept-Language': 'en-GB,en-US;q=0.8,en;q=0.6',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Referer': 'http://eclassesbyravindra.com/course/view.php?id=26^&section=9',
    'Connection': 'keep-alive',
    'Cache-Control': 'max-age=0',
}

params = (
    ('id', '26'),
)
#Enter the url of the subject. Here: Aptitude
r = requests.get('http://eclassesbyravindra.com/course/view.php', headers=headers, params=params, cookies=cookies)
soup =  BeautifulSoup(r.content, "html.parser")

#All the topics in that subject. Here: 34
all_topics = soup.find("div",{"class":"course-content"}).find_all('a')[1:]

for i in xrange(5,7):
	#URL for each topic and some extra information
	#Example: http://eclassesbyravindra.com/course/view.php?id=26&section=10
	#Example: Partnership
	#Example: 10 .. The number after section=
	topic_link = all_topics[i]['href']
	topic_name = all_topics[i].get_text()
	topic_num = re.findall("\d*$",all_topics[i]['href'])[0]
	try:
		os.mkdir("D:\\Videos\\GATE\\Aptitude\\"+topic_num.strip()+" "+topic_name.strip())
	except:
		pass
	k = requests.get(topic_link, headers=headers, cookies=cookies)
	ksoup = BeautifulSoup(k.content, "html.parser")

	#DEBUGGING
	# print len(soup.find_all("ul",{"class":"section img-text"}))

	#All the subtopics in the chosen topic. Each subtopic contains 1 video
	all_subtopics = ksoup.find_all("ul",{"class":"section img-text"})[1].find_all('a')
	for j in xrange(len(all_subtopics)):

		#DEBUGGING
		# print len(ksoup.find_all("ul",{"class":"section img-text"}))

		#Example:  http://eclassesbyravindra.com/mod/page/view.php?id=2499
		subtopic_link = ksoup.find_all("ul",{"class":"section img-text"})[1].find_all('a')[j]['href']

		#Example:  Introduction to Partnership
		subtopic_name = ksoup.find_all("ul",{"class":"section img-text"})[1].find_all('a')[j].get_text()

		v = requests.get(subtopic_link, headers=headers, cookies=cookies)
		vsoup = BeautifulSoup(v.content, "html.parser")

		#Text file in which the links are stored
		textfile = topic_num.strip()+" "+topic_name.strip()+".txt"

		#The youtube link of the video
		cur_link = vsoup.find("div",{"role":"main"}).p.iframe['src']
		#Name of the video/subtopic_name
		cur_name = vsoup.find("div",{"role":"main"}).h2.get_text()+ ":"
		print cur_name
		print cur_link

		with open("D:\\Videos\\GATE\\Aptitude\\"+topic_num.strip()+" "+topic_name.strip()+"\\"+textfile,"a") as f:
			f.write(str(j+1) + " " + cur_name)
			f.write("\n")
			f.write(cur_link)
			f.write("\n\n")

#NB. Original query string below. It seems impossible to parse and
#reproduce query strings 100% accurately so the one below is given
#in case the reproduced version is not "correct".
# requests.get('http://eclassesbyravindra.com/course/view.php?id=26', headers=headers, cookies=cookies)
<commit_msg>updates gate.py<commit_after>import requests
import re,os
from bs4 import BeautifulSoup

COOKIE = ""

cookies = {
    'gsScrollPos': '',
    'MoodleSession': COOKIE,
}

headers = {
    'Accept-Encoding': 'gzip, deflate, sdch',
    'Accept-Language': 'en-GB,en-US;q=0.8,en;q=0.6',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Referer': 'http://eclassesbyravindra.com/course/view.php?id=26^&section=9',
    'Connection': 'keep-alive',
    'Cache-Control': 'max-age=0',
}

#Replace this number with 'id' of the subject you are scraping
#26 is for aptitude
params = (
    ('id', '26'),
)
#Enter the url of the subject. Here: Aptitude
r = requests.get('http://eclassesbyravindra.com/course/view.php', headers=headers, params=params, cookies=cookies)
soup =  BeautifulSoup(r.content, "html.parser")

#All the topics in that subject. Here: 34
all_topics = soup.find("div",{"class":"course-content"}).find_all('a')[1:]

for i in xrange(0, len(all_topics)):
	#URL for each topic and some extra information
	#Example: http://eclassesbyravindra.com/course/view.php?id=26&section=10
	#Example: Partnership
	#Example: 10 .. The number after section=
	topic_link = all_topics[i]['href']
	topic_name = all_topics[i].get_text()
	topic_num = re.findall("\d*$",all_topics[i]['href'])[0]
	try:
		os.mkdir("D:\\Videos\\GATE\\Aptitude\\"+topic_num.strip()+" "+topic_name.strip())
	except:
		pass
	k = requests.get(topic_link, headers=headers, cookies=cookies)
	ksoup = BeautifulSoup(k.content, "html.parser")

	#DEBUGGING
	# print len(soup.find_all("ul",{"class":"section img-text"}))

	#All the subtopics in the chosen topic. Each subtopic contains 1 video
	all_subtopics = ksoup.find_all("ul",{"class":"section img-text"})[1].find_all('a')
	for j in xrange(len(all_subtopics)):

		#DEBUGGING
		# print len(ksoup.find_all("ul",{"class":"section img-text"}))

		#Example:  http://eclassesbyravindra.com/mod/page/view.php?id=2499
		subtopic_link = ksoup.find_all("ul",{"class":"section img-text"})[1].find_all('a')[j]['href']

		#Example:  Introduction to Partnership
		subtopic_name = ksoup.find_all("ul",{"class":"section img-text"})[1].find_all('a')[j].get_text()

		v = requests.get(subtopic_link, headers=headers, cookies=cookies)
		vsoup = BeautifulSoup(v.content, "html.parser")

		#Text file in which the links are stored
		textfile = topic_num.strip()+" "+topic_name.strip()+".txt"

		#The youtube link of the video
		cur_link = vsoup.find("div",{"role":"main"}).p.iframe['src']
		#Name of the video/subtopic_name
		cur_name = vsoup.find("div",{"role":"main"}).h2.get_text()+ ":"
		print cur_name
		print cur_link

		with open("D:\\Videos\\GATE\\Aptitude\\"+topic_num.strip()+" "+topic_name.strip()+"\\"+textfile,"a") as f:
			f.write(str(j+1) + " " + cur_name)
			f.write("\n")
			f.write(cur_link)
			f.write("\n\n")

#NB. Original query string below. It seems impossible to parse and
#reproduce query strings 100% accurately so the one below is given
#in case the reproduced version is not "correct".
# requests.get('http://eclassesbyravindra.com/course/view.php?id=26', headers=headers, cookies=cookies)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Generated by Django 1.11.3 on 2018-06-20 12:37
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('workflow', '0023_auto_20180425_0136'),
    ]

    operations = [
        migrations.AddField(
            model_name='dashboard',
            name='dashboard_uuid',
            field=models.UUIDField(default=None, verbose_name='Dashboard UUID'),
        ),
        migrations.AddField(
            model_name='dashboard',
            name='public_url_token',
            field=models.CharField(blank=True, max_length=255, null=True),
        ),
        migrations.AlterField(
            model_name='dashboard',
            name='public',
            field=models.CharField(blank=True, choices=[('org', 'Organization'), ('url', 'URL'), ('all', 'All')], max_length=5, null=True),
        ),
    ]
<commit_msg>Fix the dashboard migration for UUID<commit_after># -*- coding: utf-8 -*-
# Generated by Django 1.11.3 on 2018-06-20 12:37
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('workflow', '0023_auto_20180425_0136'),
    ]

    operations = [
        migrations.AddField(
            model_name='dashboard',
            name='dashboard_uuid',
            field=models.UUIDField(blank=True, null=True, default=None, verbose_name='Dashboard UUID'),
        ),
        migrations.AddField(
            model_name='dashboard',
            name='public_url_token',
            field=models.CharField(blank=True, max_length=255, null=True),
        ),
        migrations.AlterField(
            model_name='dashboard',
            name='public',
            field=models.CharField(blank=True, choices=[('org', 'Organization'), ('url', 'URL'), ('all', 'All')], max_length=5, null=True),
        ),
    ]
<|endoftext|>