<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2005 Edgewall Software
# Copyright (C) 2005 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.
#
# Author: Christopher Lenz <cmlenz@gmx.de>

from heapq import heappop, heappush
try:
    import threading
except ImportError:
    import dummy_threading as threading
    threading._get_ident = lambda: 0

from trac.config import Option
from trac.core import *
from trac.perm import PermissionError
from trac.web.api import IRequestFilter

class IRepositoryConnector(Interface):
    """Provide support for a specific version control system."""

    def get_supported_types():
        """Return the types of version control systems that are supported.

        Yields `(repotype, priority)` pairs, where `repotype` is used to
        match against the configured `[trac] repository_type` value in TracIni.

        If multiple provider match a given type, the `priority` is used to
        choose between them (highest number is highest priority).
        """

    def get_repository(repos_type, repos_dir, authname):
        """Return a Repository instance for the given repository type and dir.
        """

class RepositoryManager(Component):
    """Component registering the supported version control systems,

    It provides easy access to the configured implementation.
    """

    implements(IRequestFilter)

    connectors = ExtensionPoint(IRepositoryConnector)

    repository_type = Option('trac', 'repository_type', 'svn',
        """Repository connector type. (''since 0.10'')""")
    repository_dir = Option('trac', 'repository_dir', '',
        """Path to local repository""")

    def __init__(self):
        self._cache = {}
        self._lock = threading.Lock()
        self._connector = None

    # IRequestFilter methods

    def pre_process_request(self, req, handler):
        from trac.web.chrome import Chrome
        if handler is not Chrome(self.env):
            self.get_repository(req.authname) # triggers a sync if applicable
        return handler

    def post_process_request(self, req, template, content_type):
        return (template, content_type)

    # Public API methods

    def get_repository(self, authname):
        if not self._connector:
            candidates = []
            for connector in self.connectors:
                for repos_type_, prio in connector.get_supported_types():
                    if self.repository_type != repos_type_:
                        continue
                    heappush(candidates, (-prio, connector))
            if not candidates:
                raise TracError('Unsupported version control system "%s"'
                                % self.repository_type)
            self._connector = heappop(candidates)[1]
        try:
            self._lock.acquire()
            tid = threading._get_ident()
            if tid in self._cache:
                repos = self._cache[tid]
            else:
                rtype, rdir = self.repository_type, self.repository_dir
                repos = self._connector.get_repository(rtype, rdir, authname)
                self._cache[tid] = repos
            return repos
        finally:
            self._lock.release()

    def shutdown(self, tid=None):
        if tid:
            assert tid == threading._get_ident()
            try:
                self._lock.acquire()
                self._cache.pop(tid, None)
            finally:
                self._lock.release()

class NoSuchChangeset(TracError):
    def __init__(self, rev):
        TracError.__init__(self, "No changeset %s in the repository" % rev)

class NoSuchNode(TracError):
    def __init__(self, path, rev, msg=None):
        TracError.__init__(self, "%sNo node %s at revision %s" \
                           % ((msg and '%s: ' % msg) or '', path, rev))

class Repository(object):
    """Base class for a repository provided by a version control system."""

    def __init__(self, name, authz, log):
        self.name = name
        self.authz = authz or Authorizer()
        self.log = log

    def close(self):
        """Close the connection to the repository."""
        raise NotImplementedError

    def clear(self):
        """Clear any data that may have been cached in instance properties."""
        pass

    def get_quickjump_entries(self, rev):
        """Generate a list of interesting places in the repository.

        `rev` might be used to restrict the list of available locations,
        but in general it's best to produce all known locations.

        The generated results must be of the form (category, name, path, rev).
        """
        return []

    def get_changeset(self, rev):
        """Retrieve a Changeset corresponding to the  given revision `rev`."""
        raise NotImplementedError

    def get_changesets(self, start, stop):
        """Generate Changeset belonging to the given time period (start, stop).
        """
        rev = self.youngest_rev
        while rev:
            if self.authz.has_permission_for_changeset(rev):
                chgset = self.get_changeset(rev)
                if chgset.date < start:
                    return
                if chgset.date < stop:
                    yield chgset
            rev = self.previous_rev(rev)

    def has_node(self, path, rev=None):
        """Tell if there's a node at the specified (path,rev) combination.

        When `rev` is `None`, the latest revision is implied.
        """
        try:
            self.get_node(path, rev)
            return True
        except TracError:
            return False

    def get_node(self, path, rev=None):
        """Retrieve a Node from the repository at the given path.

        A Node represents a directory or a file at a given revision in the
        repository.
        If the `rev` parameter is specified, the Node corresponding to that
        revision is returned, otherwise the Node corresponding to the youngest
        revision is returned.
        """
        raise NotImplementedError

    def get_oldest_rev(self):
        """Return the oldest revision stored in the repository."""
        raise NotImplementedError
    oldest_rev = property(lambda x: x.get_oldest_rev())

    def get_youngest_rev(self):
        """Return the youngest revision in the repository."""
        raise NotImplementedError
    youngest_rev = property(lambda x: x.get_youngest_rev())

    def previous_rev(self, rev):
        """Return the revision immediately preceding the specified revision."""
        raise NotImplementedError

    def next_rev(self, rev, path=''):
        """Return the revision immediately following the specified revision."""
        raise NotImplementedError

    def rev_older_than(self, rev1, rev2):
        """Provides a total order over revisions.

        Return `True` if `rev1` is older than `rev2`, i.e. if `rev1`
        comes before `rev2` in the revision sequence.
        """
        raise NotImplementedError

    def get_youngest_rev_in_cache(self, db):
        """Return the youngest revision currently cached.

        The way revisions are sequenced is version control specific.
        By default, one assumes that the revisions are sequenced in time
        (... which is ''not'' correct for most VCS, including Subversion).
        """
        cursor = db.cursor()
        cursor.execute("SELECT rev FROM revision ORDER BY time DESC LIMIT 1")
        row = cursor.fetchone()
        return row and row[0] or None

    def get_path_history(self, path, rev=None, limit=None):
        """Retrieve all the revisions containing this path

        If given, `rev` is used as a starting point (i.e. no revision
        ''newer'' than `rev` should be returned).
        The result format should be the same as the one of Node.get_history()
        """
        raise NotImplementedError

    def normalize_path(self, path):
        """Return a canonical representation of path in the repos."""
        return NotImplementedError

    def normalize_rev(self, rev):
        """Return a canonical representation of a revision.

        It's up to the backend to decide which string values of `rev`
        (usually provided by the user) should be accepted, and how they
        should be normalized. Some backends may for instance want to match
        against known tags or branch names.

        In addition, if `rev` is `None` or '', the youngest revision should
        be returned.
        """
        return NotImplementedError

    def short_rev(self, rev):
        """Return a compact representation of a revision in the repos."""
        return self.normalize_rev(rev)

    def get_changes(self, old_path, old_rev, new_path, new_rev,
                    ignore_ancestry=1):
        """Generates changes corresponding to generalized diffs.

        Generator that yields change tuples (old_node, new_node, kind, change)
        for each node change between the two arbitrary (path,rev) pairs.

        The old_node is assumed to be None when the change is an ADD,
        the new_node is assumed to be None when the change is a DELETE.
        """
        raise NotImplementedError

class Node(object):
    """Represents a directory or file in the repository at a given revision."""

    DIRECTORY = "dir"
    FILE = "file"

    # created_path and created_rev properties refer to the Node "creation"
    # in the Subversion meaning of a Node in a versioned tree (see #3340).
    #
    # Those properties must be set by subclasses.
    #
    created_rev = None
    created_path = None

    def __init__(self, path, rev, kind):
        assert kind in (Node.DIRECTORY, Node.FILE), \
               "Unknown node kind %s" % kind
        self.path = unicode(path)
        self.rev = rev
        self.kind = kind

    def get_content(self):
        """Return a stream for reading the content of the node.

        This method will return `None` for directories.
        The returned object must support a `read([len])` method.
        """
        raise NotImplementedError

    def get_entries(self):
        """Generator that yields the immediate child entries of a directory.

        The entries are returned in no particular order.
        If the node is a file, this method returns `None`.
        """
        raise NotImplementedError

    def get_history(self, limit=None):
        """Provide backward history for this Node.

        Generator that yields `(path, rev, chg)` tuples, one for each revision
        in which the node was changed. This generator will follow copies and
        moves of a node (if the underlying version control system supports
        that), which will be indicated by the first element of the tuple
        (i.e. the path) changing.
        Starts with an entry for the current revision.
        """
        raise NotImplementedError

    def get_previous(self):
        """Return the change event corresponding to the previous revision.

        This returns a `(path, rev, chg)` tuple.
        """
        skip = True
        for p in self.get_history(2):
            if skip:
                skip = False
            else:
                return p

    def get_annotations(self):
        """Provide detailed backward history for the content of this Node.

        Retrieve an array of revisions, one `rev` for each line of content
        for that node.
        Only expected to work on (text) FILE nodes, of course.
        """
        raise NotImplementedError

    def get_properties(self):
        """Returns the properties (meta-data) of the node, as a dictionary.

        The set of properties depends on the version control system.
        """
        raise NotImplementedError

    def get_content_length(self):
        """The length in bytes of the content.

        Will be `None` for a directory.
        """
        raise NotImplementedError
    content_length = property(lambda x: x.get_content_length())

    def get_content_type(self):
        """The MIME type corresponding to the content, if known.

        Will be `None` for a directory.
        """
        raise NotImplementedError
    content_type = property(lambda x: x.get_content_type())

    def get_name(self):
        return self.path.split('/')[-1]
    name = property(lambda x: x.get_name())

    def get_last_modified(self):
        raise NotImplementedError
    last_modified = property(lambda x: x.get_last_modified())

    isdir = property(lambda x: x.kind == Node.DIRECTORY)
    isfile = property(lambda x: x.kind == Node.FILE)

class Changeset(object):
    """Represents a set of changes committed at once in a repository."""

    ADD = 'add'
    COPY = 'copy'
    DELETE = 'delete'
    EDIT = 'edit'
    MOVE = 'move'

    # change types which can have diff associated to them
    DIFF_CHANGES = (EDIT, COPY, MOVE) # MERGE

    def __init__(self, rev, message, author, date):
        self.rev = rev
        self.message = message or ''
        self.author = author or ''
        self.date = date

    def get_properties(self):
        """Returns the properties (meta-data) of the node, as a dictionary.

        The set of properties depends on the version control system.

        Warning: this used to yield 4-elements tuple (besides `name` and
        `text`, there were `wikiflag` and `htmlclass` values).
        This is now replaced by the usage of IPropertyRenderer (see #1601).
        """

    def get_changes(self):
        """Generator that produces a tuple for every change in the changeset

        The tuple will contain `(path, kind, change, base_path, base_rev)`,
        where `change` can be one of Changeset.ADD, Changeset.COPY,
        Changeset.DELETE, Changeset.EDIT or Changeset.MOVE,
        and `kind` is one of Node.FILE or Node.DIRECTORY.
        The `path` is the targeted path for the `change` (which is
        the ''deleted'' path  for a DELETE change).
        The `base_path` and `base_rev` are the source path and rev for the
        action (`None` and `-1` in the case of an ADD change).
        """
        raise NotImplementedError

class PermissionDenied(PermissionError):
    """Exception raised by an authorizer.

    This exception is raise if the user has insufficient permissions
    to view a specific part of the repository.
    """
    def __str__(self):
        return self.action

class Authorizer(object):
    """Controls the view access to parts of the repository.

    Base class for authorizers that are responsible to granting or denying
    access to view certain parts of a repository.
    """

    def assert_permission(self, path):
        if not self.has_permission(path):
            raise PermissionDenied, \
                  'Insufficient permissions to access %s' % path

    def assert_permission_for_changeset(self, rev):
        if not self.has_permission_for_changeset(rev):
            raise PermissionDenied, \
                  'Insufficient permissions to access changeset %s' % rev

    def has_permission(self, path):
        return True

    def has_permission_for_changeset(self, rev):
        return True
<commit_msg>Improve error message for the much too common ''Unsupported version control system'' error. Fixes #4635.<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2005 Edgewall Software
# Copyright (C) 2005 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.
#
# Author: Christopher Lenz <cmlenz@gmx.de>

from heapq import heappop, heappush
try:
    import threading
except ImportError:
    import dummy_threading as threading
    threading._get_ident = lambda: 0

from trac.config import Option
from trac.core import *
from trac.perm import PermissionError
from trac.web.api import IRequestFilter

class IRepositoryConnector(Interface):
    """Provide support for a specific version control system."""

    def get_supported_types():
        """Return the types of version control systems that are supported.

        Yields `(repotype, priority)` pairs, where `repotype` is used to
        match against the configured `[trac] repository_type` value in TracIni.

        If multiple provider match a given type, the `priority` is used to
        choose between them (highest number is highest priority).
        """

    def get_repository(repos_type, repos_dir, authname):
        """Return a Repository instance for the given repository type and dir.
        """

class RepositoryManager(Component):
    """Component registering the supported version control systems,

    It provides easy access to the configured implementation.
    """

    implements(IRequestFilter)

    connectors = ExtensionPoint(IRepositoryConnector)

    repository_type = Option('trac', 'repository_type', 'svn',
        """Repository connector type. (''since 0.10'')""")
    repository_dir = Option('trac', 'repository_dir', '',
        """Path to local repository""")

    def __init__(self):
        self._cache = {}
        self._lock = threading.Lock()
        self._connector = None

    # IRequestFilter methods

    def pre_process_request(self, req, handler):
        from trac.web.chrome import Chrome
        if handler is not Chrome(self.env):
            self.get_repository(req.authname) # triggers a sync if applicable
        return handler

    def post_process_request(self, req, template, content_type):
        return (template, content_type)

    # Public API methods

    def get_repository(self, authname):
        if not self._connector:
            candidates = []
            for connector in self.connectors:
                for repos_type_, prio in connector.get_supported_types():
                    if self.repository_type != repos_type_:
                        continue
                    heappush(candidates, (-prio, connector))
            if not candidates:
                raise TracError('Unsupported version control system "%s". '
                                'Check that the Python bindings for "%s" are '
                                'correctly installed.' %
                                ((self.repository_type,)*2))
            self._connector = heappop(candidates)[1]
        try:
            self._lock.acquire()
            tid = threading._get_ident()
            if tid in self._cache:
                repos = self._cache[tid]
            else:
                rtype, rdir = self.repository_type, self.repository_dir
                repos = self._connector.get_repository(rtype, rdir, authname)
                self._cache[tid] = repos
            return repos
        finally:
            self._lock.release()

    def shutdown(self, tid=None):
        if tid:
            assert tid == threading._get_ident()
            try:
                self._lock.acquire()
                self._cache.pop(tid, None)
            finally:
                self._lock.release()

class NoSuchChangeset(TracError):
    def __init__(self, rev):
        TracError.__init__(self, "No changeset %s in the repository" % rev)

class NoSuchNode(TracError):
    def __init__(self, path, rev, msg=None):
        TracError.__init__(self, "%sNo node %s at revision %s" \
                           % ((msg and '%s: ' % msg) or '', path, rev))

class Repository(object):
    """Base class for a repository provided by a version control system."""

    def __init__(self, name, authz, log):
        self.name = name
        self.authz = authz or Authorizer()
        self.log = log

    def close(self):
        """Close the connection to the repository."""
        raise NotImplementedError

    def clear(self):
        """Clear any data that may have been cached in instance properties."""
        pass

    def get_quickjump_entries(self, rev):
        """Generate a list of interesting places in the repository.

        `rev` might be used to restrict the list of available locations,
        but in general it's best to produce all known locations.

        The generated results must be of the form (category, name, path, rev).
        """
        return []

    def get_changeset(self, rev):
        """Retrieve a Changeset corresponding to the  given revision `rev`."""
        raise NotImplementedError

    def get_changesets(self, start, stop):
        """Generate Changeset belonging to the given time period (start, stop).
        """
        rev = self.youngest_rev
        while rev:
            if self.authz.has_permission_for_changeset(rev):
                chgset = self.get_changeset(rev)
                if chgset.date < start:
                    return
                if chgset.date < stop:
                    yield chgset
            rev = self.previous_rev(rev)

    def has_node(self, path, rev=None):
        """Tell if there's a node at the specified (path,rev) combination.

        When `rev` is `None`, the latest revision is implied.
        """
        try:
            self.get_node(path, rev)
            return True
        except TracError:
            return False

    def get_node(self, path, rev=None):
        """Retrieve a Node from the repository at the given path.

        A Node represents a directory or a file at a given revision in the
        repository.
        If the `rev` parameter is specified, the Node corresponding to that
        revision is returned, otherwise the Node corresponding to the youngest
        revision is returned.
        """
        raise NotImplementedError

    def get_oldest_rev(self):
        """Return the oldest revision stored in the repository."""
        raise NotImplementedError
    oldest_rev = property(lambda x: x.get_oldest_rev())

    def get_youngest_rev(self):
        """Return the youngest revision in the repository."""
        raise NotImplementedError
    youngest_rev = property(lambda x: x.get_youngest_rev())

    def previous_rev(self, rev):
        """Return the revision immediately preceding the specified revision."""
        raise NotImplementedError

    def next_rev(self, rev, path=''):
        """Return the revision immediately following the specified revision."""
        raise NotImplementedError

    def rev_older_than(self, rev1, rev2):
        """Provides a total order over revisions.

        Return `True` if `rev1` is older than `rev2`, i.e. if `rev1`
        comes before `rev2` in the revision sequence.
        """
        raise NotImplementedError

    def get_youngest_rev_in_cache(self, db):
        """Return the youngest revision currently cached.

        The way revisions are sequenced is version control specific.
        By default, one assumes that the revisions are sequenced in time
        (... which is ''not'' correct for most VCS, including Subversion).
        """
        cursor = db.cursor()
        cursor.execute("SELECT rev FROM revision ORDER BY time DESC LIMIT 1")
        row = cursor.fetchone()
        return row and row[0] or None

    def get_path_history(self, path, rev=None, limit=None):
        """Retrieve all the revisions containing this path

        If given, `rev` is used as a starting point (i.e. no revision
        ''newer'' than `rev` should be returned).
        The result format should be the same as the one of Node.get_history()
        """
        raise NotImplementedError

    def normalize_path(self, path):
        """Return a canonical representation of path in the repos."""
        return NotImplementedError

    def normalize_rev(self, rev):
        """Return a canonical representation of a revision.

        It's up to the backend to decide which string values of `rev`
        (usually provided by the user) should be accepted, and how they
        should be normalized. Some backends may for instance want to match
        against known tags or branch names.

        In addition, if `rev` is `None` or '', the youngest revision should
        be returned.
        """
        return NotImplementedError

    def short_rev(self, rev):
        """Return a compact representation of a revision in the repos."""
        return self.normalize_rev(rev)

    def get_changes(self, old_path, old_rev, new_path, new_rev,
                    ignore_ancestry=1):
        """Generates changes corresponding to generalized diffs.

        Generator that yields change tuples (old_node, new_node, kind, change)
        for each node change between the two arbitrary (path,rev) pairs.

        The old_node is assumed to be None when the change is an ADD,
        the new_node is assumed to be None when the change is a DELETE.
        """
        raise NotImplementedError

class Node(object):
    """Represents a directory or file in the repository at a given revision."""

    DIRECTORY = "dir"
    FILE = "file"

    # created_path and created_rev properties refer to the Node "creation"
    # in the Subversion meaning of a Node in a versioned tree (see #3340).
    #
    # Those properties must be set by subclasses.
    #
    created_rev = None
    created_path = None

    def __init__(self, path, rev, kind):
        assert kind in (Node.DIRECTORY, Node.FILE), \
               "Unknown node kind %s" % kind
        self.path = unicode(path)
        self.rev = rev
        self.kind = kind

    def get_content(self):
        """Return a stream for reading the content of the node.

        This method will return `None` for directories.
        The returned object must support a `read([len])` method.
        """
        raise NotImplementedError

    def get_entries(self):
        """Generator that yields the immediate child entries of a directory.

        The entries are returned in no particular order.
        If the node is a file, this method returns `None`.
        """
        raise NotImplementedError

    def get_history(self, limit=None):
        """Provide backward history for this Node.

        Generator that yields `(path, rev, chg)` tuples, one for each revision
        in which the node was changed. This generator will follow copies and
        moves of a node (if the underlying version control system supports
        that), which will be indicated by the first element of the tuple
        (i.e. the path) changing.
        Starts with an entry for the current revision.
        """
        raise NotImplementedError

    def get_previous(self):
        """Return the change event corresponding to the previous revision.

        This returns a `(path, rev, chg)` tuple.
        """
        skip = True
        for p in self.get_history(2):
            if skip:
                skip = False
            else:
                return p

    def get_annotations(self):
        """Provide detailed backward history for the content of this Node.

        Retrieve an array of revisions, one `rev` for each line of content
        for that node.
        Only expected to work on (text) FILE nodes, of course.
        """
        raise NotImplementedError

    def get_properties(self):
        """Returns the properties (meta-data) of the node, as a dictionary.

        The set of properties depends on the version control system.
        """
        raise NotImplementedError

    def get_content_length(self):
        """The length in bytes of the content.

        Will be `None` for a directory.
        """
        raise NotImplementedError
    content_length = property(lambda x: x.get_content_length())

    def get_content_type(self):
        """The MIME type corresponding to the content, if known.

        Will be `None` for a directory.
        """
        raise NotImplementedError
    content_type = property(lambda x: x.get_content_type())

    def get_name(self):
        return self.path.split('/')[-1]
    name = property(lambda x: x.get_name())

    def get_last_modified(self):
        raise NotImplementedError
    last_modified = property(lambda x: x.get_last_modified())

    isdir = property(lambda x: x.kind == Node.DIRECTORY)
    isfile = property(lambda x: x.kind == Node.FILE)

class Changeset(object):
    """Represents a set of changes committed at once in a repository."""

    ADD = 'add'
    COPY = 'copy'
    DELETE = 'delete'
    EDIT = 'edit'
    MOVE = 'move'

    # change types which can have diff associated to them
    DIFF_CHANGES = (EDIT, COPY, MOVE) # MERGE

    def __init__(self, rev, message, author, date):
        self.rev = rev
        self.message = message or ''
        self.author = author or ''
        self.date = date

    def get_properties(self):
        """Returns the properties (meta-data) of the node, as a dictionary.

        The set of properties depends on the version control system.

        Warning: this used to yield 4-elements tuple (besides `name` and
        `text`, there were `wikiflag` and `htmlclass` values).
        This is now replaced by the usage of IPropertyRenderer (see #1601).
        """

    def get_changes(self):
        """Generator that produces a tuple for every change in the changeset

        The tuple will contain `(path, kind, change, base_path, base_rev)`,
        where `change` can be one of Changeset.ADD, Changeset.COPY,
        Changeset.DELETE, Changeset.EDIT or Changeset.MOVE,
        and `kind` is one of Node.FILE or Node.DIRECTORY.
        The `path` is the targeted path for the `change` (which is
        the ''deleted'' path  for a DELETE change).
        The `base_path` and `base_rev` are the source path and rev for the
        action (`None` and `-1` in the case of an ADD change).
        """
        raise NotImplementedError

class PermissionDenied(PermissionError):
    """Exception raised by an authorizer.

    This exception is raise if the user has insufficient permissions
    to view a specific part of the repository.
    """
    def __str__(self):
        return self.action

class Authorizer(object):
    """Controls the view access to parts of the repository.

    Base class for authorizers that are responsible to granting or denying
    access to view certain parts of a repository.
    """

    def assert_permission(self, path):
        if not self.has_permission(path):
            raise PermissionDenied, \
                  'Insufficient permissions to access %s' % path

    def assert_permission_for_changeset(self, rev):
        if not self.has_permission_for_changeset(rev):
            raise PermissionDenied, \
                  'Insufficient permissions to access changeset %s' % rev

    def has_permission(self, path):
        return True

    def has_permission_for_changeset(self, rev):
        return True
<|endoftext|>


<commit_before>from django.conf import settings
from django.core.urlresolvers import resolve
import sys
import urllib2
import traceback
from lxml import etree
from airbrake.utils.decorators import async

class Client(object):
    API_URL = '%s://airbrake.io/notifier_api/v2/notices'
    ERRORS = {
        403: "Cannot use SSL",
        422: "Invalid XML sent to Airbrake",
        500: "Airbrake has braked too hard",
    }

    DEFAULTS = {
        'TIMEOUT': 5,
        'USE_SSL': False,
    }

    @property
    def url(self):
        scheme = 'http'
        if self.settings['USE_SSL']:
            scheme = 'https'

        return Client.API_URL % scheme

    @property
    def settings(self):
        if getattr(self, '_settings', None):
            return self._settings

        self._settings = Client.DEFAULTS
        self._settings.update(getattr(settings, 'AIRBRAKE', {}))
        return self._settings

    @async
    def notify(self, exception=None, request=None):
        headers = {
            'Content-Type': 'text/xml'
        }

        payload = self._generate_xml(exception=exception, request=request)
        print "PAYLOAD"
        print payload
        req = urllib2.Request(self.url, payload, headers)
        print "REQUEST"
        try:
            resp = urllib2.urlopen(req, timeout=self.settings['TIMEOUT'])
        except:
            print sys.exc_info()[0]
            print traceback.print_exc(file=sys.stdout)

        print "RESPONSE"
        status = resp.getcode()
        if status == 200:
            return True
        elif status in Client.ERRORS:
            raise Exception(Client.ERRORS[status])

    def _generate_xml(self, exception=None, request=None):
        _,_,trace = sys.exc_info()

        notice_em = etree.Element('notice', version='2.0')

        tb_dict = {}
        tb = traceback.extract_tb(trace)

        if tb:
            tb = tb[0]
            tb_dict['filename'] = tb[0]
            tb_dict['line_number'] = tb[1]
            tb_dict['function_name'] = tb[2]

        api_key = etree.SubElement(notice_em, 'api-key').text = self.settings['API_KEY']

        notifier_em = etree.SubElement(notice_em, 'notifier')

        etree.SubElement(notifier_em, 'name').text = 'django-airbrake'
        etree.SubElement(notifier_em, 'version').text = '0.0.2'
        etree.SubElement(notifier_em, 'url').text = 'http://example.com'

        if request:
            request_em = etree.SubElement(notice_em, 'request')

            if request.is_secure():
                scheme = 'https'
            else:
                scheme = 'http'
            url = '%s://%s%s' % (scheme, request.get_host(),
                request.get_full_path())
            etree.SubElement(request_em, 'url').text = str(url)

            cb,_,_ = resolve(request.path)
            etree.SubElement(request_em, 'component').text = str(cb.__module__)
            etree.SubElement(request_em, 'action').text = str(cb.__name__)

            if len(request.POST):
                params_em = etree.SubElement(request_em, 'params')

                for key, val in request.POST.items():
                    var = etree.SubElement(params_em, 'var')
                    var.set('key', str(key))
                    var.text(str(val))

            session = request.session.items()
            if len(session):
                session_em = etree.SubElement(request_em, 'session')
                for key, val in session.items():
                    var = etree.SubElement(session_em, 'var')
                    var.set('key', str(key))
                    var.text(str(val))

            if exception:
                error_em = etree.SubElement(notice_em, 'error')

                etree.SubElement(error_em, 'class').text = str(exception.__class__.__name__)
                etree.SubElement(error_em, 'message').text = str(exception)

                backtrace_em = etree.SubElement(error_em, 'backtrace')

                etree.SubElement(backtrace_em, 'line',
                    file=tb_dict.get('filename', 'unknown'),
                    number=tb_dict.get('line_number', 'unknown'),
                    method=tb_dict.get('function_name', 'unknown'))

        env_em = etree.SubElement(notice_em, 'server-environment')

        etree.SubElement(env_em, 'environment-name').text = settings.get('environment', 'development')

        return '<?xml version="1.0" encoding="UTF-8"?>%s' % etree.tostring(notice_em)
<commit_msg>stupid me<commit_after>from django.conf import settings
from django.core.urlresolvers import resolve
import sys
import urllib2
import traceback
from lxml import etree
from airbrake.utils.decorators import async

class Client(object):
    API_URL = '%s://airbrake.io/notifier_api/v2/notices'
    ERRORS = {
        403: "Cannot use SSL",
        422: "Invalid XML sent to Airbrake",
        500: "Airbrake has braked too hard",
    }

    DEFAULTS = {
        'TIMEOUT': 5,
        'USE_SSL': False,
    }

    @property
    def url(self):
        scheme = 'http'
        if self.settings['USE_SSL']:
            scheme = 'https'

        return Client.API_URL % scheme

    @property
    def settings(self):
        if getattr(self, '_settings', None):
            return self._settings

        self._settings = Client.DEFAULTS
        self._settings.update(getattr(settings, 'AIRBRAKE', {}))
        return self._settings

    @async
    def notify(self, exception=None, request=None):
        headers = {
            'Content-Type': 'text/xml'
        }

        payload = self._generate_xml(exception=exception, request=request)
        req = urllib2.Request(self.url, payload, headers)
        resp = urllib2.urlopen(req, timeout=self.settings['TIMEOUT'])
        status = resp.getcode()

        if status == 200:
            return True
        elif status in Client.ERRORS:
            raise Exception(Client.ERRORS[status])

    def _generate_xml(self, exception=None, request=None):
        _,_,trace = sys.exc_info()

        notice_em = etree.Element('notice', version='2.0')

        tb_dict = {}
        tb = traceback.extract_tb(trace)

        if tb:
            tb = tb[0]
            tb_dict['filename'] = tb[0]
            tb_dict['line_number'] = tb[1]
            tb_dict['function_name'] = tb[2]

        api_key = etree.SubElement(notice_em, 'api-key').text = self.settings['API_KEY']

        notifier_em = etree.SubElement(notice_em, 'notifier')

        etree.SubElement(notifier_em, 'name').text = 'django-airbrake'
        etree.SubElement(notifier_em, 'version').text = '0.0.2'
        etree.SubElement(notifier_em, 'url').text = 'http://example.com'

        if request:
            request_em = etree.SubElement(notice_em, 'request')

            if request.is_secure():
                scheme = 'https'
            else:
                scheme = 'http'
            url = '%s://%s%s' % (scheme, request.get_host(),
                request.get_full_path())
            etree.SubElement(request_em, 'url').text = str(url)

            cb,_,_ = resolve(request.path)
            etree.SubElement(request_em, 'component').text = str(cb.__module__)
            etree.SubElement(request_em, 'action').text = str(cb.__name__)

            if len(request.POST):
                params_em = etree.SubElement(request_em, 'params')

                for key, val in request.POST.items():
                    var = etree.SubElement(params_em, 'var')
                    var.set('key', str(key))
                    var.text(str(val))

            session = request.session.items()
            if len(session):
                session_em = etree.SubElement(request_em, 'session')
                for key, val in session.items():
                    var = etree.SubElement(session_em, 'var')
                    var.set('key', str(key))
                    var.text(str(val))

            if exception:
                error_em = etree.SubElement(notice_em, 'error')

                etree.SubElement(error_em, 'class').text = str(exception.__class__.__name__)
                etree.SubElement(error_em, 'message').text = str(exception)

                backtrace_em = etree.SubElement(error_em, 'backtrace')

                etree.SubElement(backtrace_em, 'line',
                    file=tb_dict.get('filename', 'unknown'),
                    number=tb_dict.get('line_number', 'unknown'),
                    method=tb_dict.get('function_name', 'unknown'))

        env_em = etree.SubElement(notice_em, 'server-environment')

        etree.SubElement(env_em, 'environment-name').text = getattr(settings, 'ENVIRONMENT', 'development')

        return '<?xml version="1.0" encoding="UTF-8"?>%s' % etree.tostring(notice_em)
<|endoftext|>


<commit_before># coding=utf-8

import re
import asyncio
import threading

import pytest
from aiohttp import web

from xpaw.config import Config
from xpaw.http import HttpRequest, HttpResponse
from xpaw.downloadermws import *
from xpaw.errors import IgnoreRequest, ResponseNotMatch

from .helpers import wait_server_start

@pytest.fixture(scope="module")
def loop(request):
    def close():
        loop.close()

    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    request.addfinalizer(close)
    return loop

class TestForwardedForMiddleware:
    def test_handle_request(self, loop):
        mw = ForwardedForMiddleware()
        req = HttpRequest("http://www.example.com")
        loop.run_until_complete(mw.handle_request(req))
        assert re.search(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", req.headers["X-Forwarded-For"])

class TestRequestHeadersMiddleware:
    def test_handle_request(self, loop):
        headers = {"Content-Type": "text/html", "User-Agent": "xpaw", "Connection": "keep-alive"}
        mw = RequestHeadersMiddleware.from_config(dict(request_headers=headers))
        req = HttpRequest("http://www.example.com")
        loop.run_until_complete(mw.handle_request(req))
        assert headers == req.headers

@pytest.fixture(scope="class")
def agent(request):
    async def handle_request(request):
        return web.Response(body=b'["127.0.0.1:3128", "127.0.0.1:8080"]')

    def handle_error(loop, context):
        pass

    def start_loop():
        app = web.Application(loop=loop)
        app.router.add_resource("/").add_route("GET", handle_request)
        loop.run_until_complete(loop.create_server(app.make_handler(access_log=None, loop=loop), "0.0.0.0", 7340))
        try:
            loop.run_forever()
        except Exception:
            pass
        finally:
            loop.close()

    def stop_loop():
        loop.call_soon_threadsafe(loop.stop)

    loop = asyncio.new_event_loop()
    loop.set_exception_handler(handle_error)
    t = threading.Thread(target=start_loop)
    t.start()
    wait_server_start("127.0.0.1:7340")
    request.addfinalizer(stop_loop)

class TestProxyAgentMiddleware:
    def test_handle_request(self, loop, monkeypatch):
        async def _pick_proxy():
            return "http://127.0.0.1"

        mw = ProxyAgentMiddleware.from_config(Config({"proxy_agent": {"addr": "http://127.0.0.1:7340"}}))
        monkeypatch.setattr(mw, "_pick_proxy", _pick_proxy)
        req = HttpRequest("http://www.example.com")
        loop.run_until_complete(mw.handle_request(req))
        assert req.proxy == "http://127.0.0.1"

    def test_pick_proxy(self, loop, monkeypatch):
        async def _update_proxy_list():
            while self.index < 2:
                await asyncio.sleep(0.1)
                self.index += 1
                mw._proxy_list = proxy_list[self.index - 1]

        self.index = 0
        proxy_list = [[], ["127.0.0.1", "127.0.0.2"]]
        res = ["http://127.0.0.1", "http://127.0.0.2"]
        mw = ProxyAgentMiddleware.from_config(
            Config({"proxy_agent": {"addr": "127.0.0.1:7340", "update_interval": 0.1}}))
        monkeypatch.setattr(mw, "_update_proxy_list", _update_proxy_list)
        mw.open()
        for i in range(len(res)):
            req = HttpRequest("http://www.example.com")
            loop.run_until_complete(mw.handle_request(req))
            assert req.proxy in res

    def test_update_proxy_list(self, loop, agent):
        async def _func():
            while mw._proxy_list is None:
                await asyncio.sleep(0.1)

        mw = ProxyAgentMiddleware.from_config(
            Config({"proxy_agent": {"addr": "http://127.0.0.1:7340", "update_interval": 0.1}}))
        mw.open()
        loop.run_until_complete(_func())
        assert mw._proxy_list == ["127.0.0.1:3128", "127.0.0.1:8080"]
        mw.close()
        loop.run_until_complete(asyncio.sleep(0.1))

class TestRetryMiddleware:
    def test_handle_reponse(self, loop, monkeypatch):
        class ErrorFlag(Exception):
            pass

        def _retry(request, reason):
            assert isinstance(request, HttpRequest) and isinstance(reason, str)
            raise ErrorFlag

        mw = RetryMiddleware.from_config(Config())
        monkeypatch.setattr(mw, "retry", _retry)
        req = HttpRequest("http://www.example.com")
        resp = HttpResponse("http://www.example.com", 400)
        loop.run_until_complete(mw.handle_response(req, resp))
        with pytest.raises(ErrorFlag):
            resp = HttpResponse("http://www.example.com", 503)
            loop.run_until_complete(mw.handle_response(req, resp))

    def test_handle_error(self, loop, monkeypatch):
        class ErrorFlag(Exception):
            pass

        def _retry(request, reason):
            assert isinstance(request, HttpRequest) and isinstance(reason, str)
            raise ErrorFlag

        mw = RetryMiddleware.from_config(Config())
        monkeypatch.setattr(mw, "retry", _retry)
        req = HttpRequest("http://www.example.com")
        err = ValueError()
        loop.run_until_complete(mw.handle_error(req, err))
        with pytest.raises(ErrorFlag):
            err = ResponseNotMatch()
            loop.run_until_complete(mw.handle_error(req, err))

    def test_retry(self):
        max_retry_times = 2
        mw = RetryMiddleware.from_config(Config({"retry": {"max_retry_times": max_retry_times}}))
        req = HttpRequest("http://www.example.com")
        for i in range(max_retry_times):
            req = mw.retry(req, "")
            assert isinstance(req, HttpRequest)
        with pytest.raises(IgnoreRequest):
            req = mw.retry(req, "")
<commit_msg>add tests<commit_after># coding=utf-8

import re
import asyncio
import threading

import pytest
from aiohttp import web

from xpaw.config import Config
from xpaw.http import HttpRequest, HttpResponse
from xpaw.downloadermws import *
from xpaw.errors import IgnoreRequest, ResponseNotMatch

from .helpers import wait_server_start

@pytest.fixture(scope="module")
def loop(request):
    def close():
        loop.close()

    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    request.addfinalizer(close)
    return loop

class TestForwardedForMiddleware:
    def test_handle_request(self, loop):
        mw = ForwardedForMiddleware()
        req = HttpRequest("http://www.example.com")
        loop.run_until_complete(mw.handle_request(req))
        assert re.search(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", req.headers["X-Forwarded-For"])

class TestRequestHeadersMiddleware:
    def test_handle_request(self, loop):
        headers = {"Content-Type": "text/html", "User-Agent": "xpaw", "Connection": "keep-alive"}
        mw = RequestHeadersMiddleware.from_config(dict(request_headers=headers))
        req = HttpRequest("http://www.example.com")
        loop.run_until_complete(mw.handle_request(req))
        assert headers == req.headers

@pytest.fixture(scope="module")
def agent(request):
    async def handle_request(request):
        return web.Response(body=b'["127.0.0.1:3128", "127.0.0.1:8080"]')

    def handle_error(loop, context):
        pass

    def start_loop():
        app = web.Application(loop=loop)
        app.router.add_resource("/").add_route("GET", handle_request)
        loop.run_until_complete(loop.create_server(app.make_handler(access_log=None, loop=loop), "0.0.0.0", 7340))
        try:
            loop.run_forever()
        except Exception:
            pass
        finally:
            loop.close()

    def stop_loop():
        loop.call_soon_threadsafe(loop.stop)

    loop = asyncio.new_event_loop()
    loop.set_exception_handler(handle_error)
    t = threading.Thread(target=start_loop)
    t.start()
    wait_server_start("127.0.0.1:7340")
    request.addfinalizer(stop_loop)

class TestProxyMiddleware:
    def test_hanle_request(self, loop):
        mw = ProxyMiddleware(["127.0.0.1"])
        req = HttpRequest("http://www.example.com")
        loop.run_until_complete(mw.handle_request(req))
        assert req.proxy == "http://127.0.0.1"

class TestProxyAgentMiddleware:
    def test_handle_request(self, loop, monkeypatch):
        async def _pick_proxy():
            return "http://127.0.0.1"

        mw = ProxyAgentMiddleware.from_config(Config({"proxy_agent": {"addr": "http://127.0.0.1:7340"}}))
        monkeypatch.setattr(mw, "_pick_proxy", _pick_proxy)
        req = HttpRequest("http://www.example.com")
        loop.run_until_complete(mw.handle_request(req))
        assert req.proxy == "http://127.0.0.1"

    def test_pick_proxy(self, loop, monkeypatch):
        async def _update_proxy_list():
            while self.index < 2:
                await asyncio.sleep(0.1)
                self.index += 1
                mw._proxy_list = proxy_list[self.index - 1]

        self.index = 0
        proxy_list = [[], ["127.0.0.1", "127.0.0.2"]]
        res = ["http://127.0.0.1", "http://127.0.0.2"]
        mw = ProxyAgentMiddleware.from_config(
            Config({"proxy_agent": {"addr": "127.0.0.1:7340", "update_interval": 0.1}}))
        monkeypatch.setattr(mw, "_update_proxy_list", _update_proxy_list)
        mw.open()
        for i in range(len(res)):
            req = HttpRequest("http://www.example.com")
            loop.run_until_complete(mw.handle_request(req))
            assert req.proxy in res

    def test_update_proxy_list(self, loop, agent):
        async def _func():
            while mw._proxy_list is None:
                await asyncio.sleep(0.1)

        mw = ProxyAgentMiddleware.from_config(
            Config({"proxy_agent": {"addr": "http://127.0.0.1:7340", "update_interval": 0.1}}))
        mw.open()
        loop.run_until_complete(_func())
        assert mw._proxy_list == ["127.0.0.1:3128", "127.0.0.1:8080"]
        mw.close()
        loop.run_until_complete(asyncio.sleep(0.1))

class TestRetryMiddleware:
    def test_handle_reponse(self, loop, monkeypatch):
        class ErrorFlag(Exception):
            pass

        def _retry(request, reason):
            assert isinstance(request, HttpRequest) and isinstance(reason, str)
            raise ErrorFlag

        mw = RetryMiddleware.from_config(Config())
        monkeypatch.setattr(mw, "retry", _retry)
        req = HttpRequest("http://www.example.com")
        resp = HttpResponse("http://www.example.com", 400)
        loop.run_until_complete(mw.handle_response(req, resp))
        with pytest.raises(ErrorFlag):
            resp = HttpResponse("http://www.example.com", 503)
            loop.run_until_complete(mw.handle_response(req, resp))

    def test_handle_error(self, loop, monkeypatch):
        class ErrorFlag(Exception):
            pass

        def _retry(request, reason):
            assert isinstance(request, HttpRequest) and isinstance(reason, str)
            raise ErrorFlag

        mw = RetryMiddleware.from_config(Config())
        monkeypatch.setattr(mw, "retry", _retry)
        req = HttpRequest("http://www.example.com")
        err = ValueError()
        loop.run_until_complete(mw.handle_error(req, err))
        with pytest.raises(ErrorFlag):
            err = ResponseNotMatch()
            loop.run_until_complete(mw.handle_error(req, err))

    def test_retry(self):
        max_retry_times = 2
        mw = RetryMiddleware.from_config(Config({"retry": {"max_retry_times": max_retry_times}}))
        req = HttpRequest("http://www.example.com")
        for i in range(max_retry_times):
            req = mw.retry(req, "")
            assert isinstance(req, HttpRequest)
        with pytest.raises(IgnoreRequest):
            mw.retry(req, "")

    def test_match_status(self):
        assert RetryMiddleware.match_status("200", 200) is True
        assert RetryMiddleware.match_status(200, 200) is True
        assert RetryMiddleware.match_status("2xX", 201) is True
        assert RetryMiddleware.match_status("40x", 403) is True
        assert RetryMiddleware.match_status("40X", 403) is True
        assert RetryMiddleware.match_status("50x", 403) is False
        assert RetryMiddleware.match_status("~20X", 200) is False
        assert RetryMiddleware.match_status("!20x", 400) is True

class TestResponseMatchMiddleware:
    def test_handle_response(self, loop):
        req = HttpRequest("http://www.baidu.com")
        resp = HttpResponse("http://www.baidu.com", 200, body="<title>百度一下，你就知道</title>".encode("utf-8"))
        wrong_resp = HttpResponse("http://www.qq.com", 200, body="<title>腾讯QQ</title>".encode("utf-8"))
        mw = ResponseMatchMiddleware.from_config(Config({"response_match": {"url_pattern": "www\\.baidu\\.com",
                                                                            "body_pattern": "百度",
                                                                            "encoding": "utf-8"}}))
        loop.run_until_complete(mw.handle_response(req, resp))
        with pytest.raises(ResponseNotMatch):
            loop.run_until_complete(mw.handle_response(req, wrong_resp))
<|endoftext|>


<commit_before># (C) Datadog, Inc. 2018-present
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
import os
from fnmatch import fnmatch

import click
import toml

from ..config import (
    CONFIG_FILE,
    SECRET_KEYS,
    config_file_exists,
    read_config_file,
    read_config_file_scrubbed,
    restore_config,
    save_config,
    scrub_secrets,
    update_config,
)
from ..utils import string_to_toml_type
from .console import CONTEXT_SETTINGS, echo_info, echo_success

@click.group(context_settings=CONTEXT_SETTINGS, short_help='Manage the config file')
def config():
    pass

@config.command(context_settings=CONTEXT_SETTINGS, short_help='Open the config location in your file manager')
def explore():
    """Open the config location in your file manager."""
    click.launch(CONFIG_FILE, locate=True)

@config.command(context_settings=CONTEXT_SETTINGS, short_help='Show the location of the config file')
def find():
    """Show the location of the config file."""
    if ' ' in CONFIG_FILE:
        echo_info(f'"{CONFIG_FILE}"')
    else:
        echo_info(CONFIG_FILE)

@config.command(context_settings=CONTEXT_SETTINGS, short_help='Show the contents of the config file')
@click.option('--all', '-a', 'all_keys', is_flag=True, help='No not scrub secret fields')
def show(all_keys):
    """Show the contents of the config file."""
    if not config_file_exists():
        echo_info('No config file found! Please try `ddev config restore`.')
    else:
        if all_keys:
            echo_info(read_config_file().rstrip())
        else:
            echo_info(read_config_file_scrubbed().rstrip())

@config.command(context_settings=CONTEXT_SETTINGS, short_help='Update the config file with any new fields')
def update():
    """Update the config file with any new fields."""
    update_config()
    echo_success('Settings were successfully updated.')

@config.command(context_settings=CONTEXT_SETTINGS, short_help='Restore the config file to default settings')
def restore():
    """Restore the config file to default settings."""
    restore_config()
    echo_success('Settings were successfully restored.')

@config.command('set', context_settings=CONTEXT_SETTINGS, short_help='Assign values to config file entries')
@click.argument('key')
@click.argument('value', required=False)
@click.pass_context
def set_value(ctx, key, value):
    """Assigns values to config file entries. If the value is omitted,
    you will be prompted, with the input hidden if it is sensitive.

    \b
    `$ ddev config set github.user foo`

        New setting:
        [github]
        user = "foo"

    You can also assign values on a per-org basis.

    \b
    `$ ddev config set orgs.<ORG_NAME>.api_key`

        New setting:
        [orgs.<ORG_NAME>]
        api_key = "***********"
    """
    scrubbing = False
    if value is None:
        scrubbing = any(fnmatch(key, pattern) for pattern in SECRET_KEYS)
        value = click.prompt(f'Value for `{key}`', hide_input=scrubbing)

    if key in ('repos.core', 'repos.extras', 'repos.agent') and not value.startswith('~'):
        value = os.path.abspath(value)

    user_config = new_config = ctx.obj
    user_config.pop('repo_choice', None)

    data = [value]
    data.extend(reversed(key.split('.')))
    key = data.pop()
    value = data.pop()

    # Use a separate mapping to show only what has changed in the end
    branch_config_root = branch_config = {}

    # Consider dots as keys
    while data:
        default_branch = {value: ''}
        branch_config[key] = default_branch
        branch_config = branch_config[key]

        new_value = new_config.get(key)
        if not hasattr(new_value, 'get'):
            new_value = default_branch

        new_config[key] = new_value
        new_config = new_config[key]

        key = value
        value = data.pop()

    value = string_to_toml_type(value)
    branch_config[key] = new_config[key] = value

    save_config(user_config)

    output_config = scrub_secrets(branch_config_root) if scrubbing else branch_config_root
    echo_success('New setting:')
    echo_info(toml.dumps(output_config).rstrip())
<commit_msg>Add "ddev config edit" subcommand (#7217)<commit_after># (C) Datadog, Inc. 2018-present
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
import os
from fnmatch import fnmatch

import click
import toml

from ..config import (
    CONFIG_FILE,
    SECRET_KEYS,
    config_file_exists,
    read_config_file,
    read_config_file_scrubbed,
    restore_config,
    save_config,
    scrub_secrets,
    update_config,
)
from ..utils import string_to_toml_type
from .console import CONTEXT_SETTINGS, echo_info, echo_success

@click.group(context_settings=CONTEXT_SETTINGS, short_help='Manage the config file')
def config():
    pass

@config.command(context_settings=CONTEXT_SETTINGS, short_help='Edit the config file with your default EDITOR')
def edit():
    """Edit the config file with your default EDITOR."""
    click.edit(filename=CONFIG_FILE)

@config.command(context_settings=CONTEXT_SETTINGS, short_help='Open the config location in your file manager')
def explore():
    """Open the config location in your file manager."""
    click.launch(CONFIG_FILE, locate=True)

@config.command(context_settings=CONTEXT_SETTINGS, short_help='Show the location of the config file')
def find():
    """Show the location of the config file."""
    if ' ' in CONFIG_FILE:
        echo_info(f'"{CONFIG_FILE}"')
    else:
        echo_info(CONFIG_FILE)

@config.command(context_settings=CONTEXT_SETTINGS, short_help='Show the contents of the config file')
@click.option('--all', '-a', 'all_keys', is_flag=True, help='No not scrub secret fields')
def show(all_keys):
    """Show the contents of the config file."""
    if not config_file_exists():
        echo_info('No config file found! Please try `ddev config restore`.')
    else:
        if all_keys:
            echo_info(read_config_file().rstrip())
        else:
            echo_info(read_config_file_scrubbed().rstrip())

@config.command(context_settings=CONTEXT_SETTINGS, short_help='Update the config file with any new fields')
def update():
    """Update the config file with any new fields."""
    update_config()
    echo_success('Settings were successfully updated.')

@config.command(context_settings=CONTEXT_SETTINGS, short_help='Restore the config file to default settings')
def restore():
    """Restore the config file to default settings."""
    restore_config()
    echo_success('Settings were successfully restored.')

@config.command('set', context_settings=CONTEXT_SETTINGS, short_help='Assign values to config file entries')
@click.argument('key')
@click.argument('value', required=False)
@click.pass_context
def set_value(ctx, key, value):
    """Assigns values to config file entries. If the value is omitted,
    you will be prompted, with the input hidden if it is sensitive.

    \b
    `$ ddev config set github.user foo`

        New setting:
        [github]
        user = "foo"

    You can also assign values on a per-org basis.

    \b
    `$ ddev config set orgs.<ORG_NAME>.api_key`

        New setting:
        [orgs.<ORG_NAME>]
        api_key = "***********"
    """
    scrubbing = False
    if value is None:
        scrubbing = any(fnmatch(key, pattern) for pattern in SECRET_KEYS)
        value = click.prompt(f'Value for `{key}`', hide_input=scrubbing)

    if key in ('repos.core', 'repos.extras', 'repos.agent') and not value.startswith('~'):
        value = os.path.abspath(value)

    user_config = new_config = ctx.obj
    user_config.pop('repo_choice', None)

    data = [value]
    data.extend(reversed(key.split('.')))
    key = data.pop()
    value = data.pop()

    # Use a separate mapping to show only what has changed in the end
    branch_config_root = branch_config = {}

    # Consider dots as keys
    while data:
        default_branch = {value: ''}
        branch_config[key] = default_branch
        branch_config = branch_config[key]

        new_value = new_config.get(key)
        if not hasattr(new_value, 'get'):
            new_value = default_branch

        new_config[key] = new_value
        new_config = new_config[key]

        key = value
        value = data.pop()

    value = string_to_toml_type(value)
    branch_config[key] = new_config[key] = value

    save_config(user_config)

    output_config = scrub_secrets(branch_config_root) if scrubbing else branch_config_root
    echo_success('New setting:')
    echo_info(toml.dumps(output_config).rstrip())
<|endoftext|>


<commit_before># Import stuff

import numpy as np
#import iso_lib as il
import sklearn as sk

# Class to contain star's data, chain, etc

class star_posterior:

    # init function

    def __init__(self, r_in, i_in, ha_in, dr_in, di_in, dha_in, isochrones):

        self.r=r_in
        self.i=i_in
        self.ha=ha_in

        self.dr=dr_in
        self.di=di_in
        self.dha=dha_in

        self.isochrones=isochrones

    # initialise MCMC_chain

    def init_chain(self, chain_length):
        self.last_Teff=3.8
        self.last_logg=4.5
        self.last_feh=0.
        self.last_dist_mod=10.
        self.last_logA=1.

        self.last_iso_obj=isochrones.query(self.last_feh, self.last_Teff, self.last_logg)

        self.set_lastprob()

        self.Teff_chain=np.zeros(chain_length)
        self.logg_chain=np.zeros(chain_length)
        self.feh_chain=np.zeros(chain_length)
        self.dist_mod_chain=np.zeros(chain_length)
        self.logA_chain=np.zeros(chain_length)

    # find probability of last param set

    def set_lastprob(self):
        self.last_prob=0.
#        self.last_prob=(np.power(self.r- ,2)
#                        +np.power(self.i- ,2)
#                        +np.power(self.ha- ,2) )

    #  find probability of test param set

    def set_testprob(self):
        self.test_prob=0.
#        self.last_prob=(np.power(self.r- ,2)
#                        +np.power(self.i- ,2)
#                        +np.power(self.ha- ,2) )

    # MCMC sampler

    def MCMC_Metropolis(self, iterations=10000, thin=1, Teff_prop, logg_prop, feh_prop, dist_mod_prop, logA_prop):

        self.init_chain(int(iterations/thin))

        for it in range(iterations):
            # Set test params

            test_Teff=3.8
            test_logg=4.5
            test_feh=0.
            test_dist_mod=10.
            test_logA=1.

            test_iso_obj=isochrones.query(self.test_feh, self.test_Teff, self.test_logg)

            # get probs

            set_testprob()

            # accept/reject

            threshold=np.log(np.random.uniform())

            if (self.test_prob-self.last_prob)>=threshold:
                self.last_Teff=test_Teff
                self.last_logg=test_logg
                self.last_feh=test_feh
                self.last_dist_mod=test_dist_mod
                self.last_logA=test_logA

                self.last_iso_obj=test_iso_obj

            # store in chain

            if it%thin==0:
                self.Teff_chain[it/thin]=self.last_Teff
                self.logg_chain[it/thin]=self.last_logg
                self.feh_chain[it/thin]=self.last_feh
                self.dist_mod_chain[it/thin]=self.last_dist_mod
                self.logA_chain[it/thin]=self.last_logA

                print it

    # Fit Gaussians

#    def gauss_fit(self):
<commit_msg>functioning on most basic level<commit_after># Import stuff

import numpy as np
#import iso_lib as il
import sklearn as sk

# Class to contain star's data, chain, etc

class star_posterior:

    # init function

    def __init__(self, r_in, i_in, ha_in, dr_in, di_in, dha_in, isochrones):

        self.r=r_in
        self.i=i_in
        self.ha=ha_in

        self.dr=dr_in
        self.di=di_in
        self.dha=dha_in

        self.isochrones=isochrones

    # initialise MCMC_chain

    def init_chain(self, chain_length):
        self.last_Teff=3.8
        self.last_logg=4.5
        self.last_feh=0.
        self.last_dist_mod=10.
        self.last_logA=1.

        self.last_iso_obj=self.isochrones.query(self.last_feh, self.last_Teff, self.last_logg)

        self.set_lastprob()

        self.Teff_chain=np.zeros(chain_length)
        self.logg_chain=np.zeros(chain_length)
        self.feh_chain=np.zeros(chain_length)
        self.dist_mod_chain=np.zeros(chain_length)
        self.logA_chain=np.zeros(chain_length)

    # find probability of last param set

    def set_lastprob(self):
        self.last_prob=0.
#        self.last_prob=(np.power(self.r- ,2)
#                        +np.power(self.i- ,2)
#                        +np.power(self.ha- ,2) )

    #  find probability of test param set

    def set_testprob(self):
        self.test_prob=0.
#        self.last_prob=(np.power(self.r- ,2)
#                        +np.power(self.i- ,2)
#                        +np.power(self.ha- ,2) )

    # MCMC sampler

    def MCMC_Metropolis(self, Teff_prop, logg_prop, feh_prop, dist_mod_prop, logA_prop, iterations=10000, thin=1):

        self.init_chain(int(iterations/thin))

        for it in range(iterations):
            # Set test params

            test_Teff=3.8
            test_logg=4.5
            test_feh=0.
            test_dist_mod=10.
            test_logA=1.

            test_iso_obj=self.isochrones.query(test_feh, test_Teff, test_logg)

            # get probs

            self.set_testprob()

            # accept/reject

            threshold=np.log(np.random.uniform())

            if (self.test_prob-self.last_prob)>=threshold:
                self.last_Teff=test_Teff
                self.last_logg=test_logg
                self.last_feh=test_feh
                self.last_dist_mod=test_dist_mod
                self.last_logA=test_logA

                self.last_iso_obj=test_iso_obj

            # store in chain

            if it%thin==0:
                self.Teff_chain[it/thin]=self.last_Teff
                self.logg_chain[it/thin]=self.last_logg
                self.feh_chain[it/thin]=self.last_feh
                self.dist_mod_chain[it/thin]=self.last_dist_mod
                self.logA_chain[it/thin]=self.last_logA

    # Fit Gaussians

#    def gauss_fit(self):
<|endoftext|>


<commit_before>import yaml
import tempfile
import os, re
import logging
import numpy as np
import subprocess
import time
import multiprocessing

from .batch import write_slurm_script, get_job_status

class PipelineStage(object):
    name = None
    id_str_fmt = None
    batch_compatible = True
    single_filter = False
    batch_type = 'slurm'

    _default_kwargs = {}

    def __init__(self, pipeline):
        self.pipeline = pipeline

    @property
    def _id_options(self):
        raise NotImplementedError("Need to define what id keys are valid.")

    @property
    def default_kwargs(self):
        return self._default_kwargs

    def id_str(self, filt=None):
        s = ''
        for key in self._id_options:
            try:
                if key=='visit':
                    s += 'visit={0} '.format(self.pipeline['visit'][filt])
                else:
                    fmt = '{0}={{0[{0}]}} '.format(key)
                    s += fmt.format(self.pipeline)
            except KeyError:
                continue

        if filt is not None:
            s += 'filter={0} '.format(filt)
        return s

    def jobname(self, filt=None):
        job = self.pipeline.rerun.replace('/','-')
        s = '{0}-{1}'.format(job, self.name)
        if filt is not None:
            s += '-{}'.format(filt)
        return s

    def cmd_str(self, filt=None, test=False, **kwargs):
        cmd = '{0}.py {1[data_root]} --rerun {1.rerun} '.format(self.name, self.pipeline)

        if not isinstance(self, ManualBatchStage):
            cmd += '--job {0} '.format(self.jobname(filt))

        cmd += '--id {0} '.format(self.id_str(filt))

        if hasattr(self, 'selectId_str'):
            cmd += '--selectId {0} '.format(self.selectId_str(filt))

        kws = dict(self.default_kwargs)
        kws.update(self.pipeline['kwargs']['all'])
        kws.update(self.pipeline['kwargs'][self.name])
        kws.update(kwargs)
        for kw, val in kws.items():
            if kw == 'filters':
                continue

            if kw == 'config':
                # val should be a dict
                for k, v in val.items():
                    cmd += '--config {0}={1} '.format(k,v)
                continue

            if val is False:
                continue

            # skip using "time" keyword if writing batch script manually
            if isinstance(self, ManualBatchStage) and kw == 'time':
                continue

            cmd += '--{0}'.format(kw)
            if val != '' and val is not True:
                # ensure quotation marks where necessary
                if isinstance(val, basestring):
                    val = '"{0}"'.format(val)

                cmd += '={0} '.format(val)
                if kw == 'nodes':
                    cmd += '--procs {0[cores_per_node]} '.format(self.pipeline)
            else:
                cmd += ' '

        if test:
            cmd = cmd.replace('"', "'")
            np.random.seed()
            jobid = np.random.randint(10000)
            sleep = np.random.random()*2 + 2
            cmd = 'echo "{0}\nbatch job {1}; sleep {2}"'.format(cmd, jobid, sleep)
            print(cmd)

        return cmd

    def submit_cmd(self, filt=None, test=False):
        """
        """
        cmd = self.cmd_str(filt, test=test)
        return cmd

    def _get_dependent_jobids(self, filt=None):
        job_ids = self.pipeline.job_ids
        id_depends = []
        if hasattr(self, 'depends'):
            for key,i in job_ids.items():
                # If we know it's done, then skip.
                if i in self.pipeline.complete_job_ids:
                    continue

                for d in self.depends:
                    m = re.match(d, key)
                    if m:
                        if filt is not None:
                            if (key == '{0}-{1}'.format(d, filt) or
                                key == d):
                                id_depends.append(i)
                        else:
                            id_depends.append(i)
        return id_depends

    def _wait_for_dependencies(self, filt=None, test=False):
        # Make sure all dependencies are satisfied
        id_depends = self._get_dependent_jobids(filt=filt)

        if len(id_depends) > 0:
            name = self.name
            if filt is not None:
                name += ' {0}'.format(filt)
            msg = '{0} waiting for jobids {1}...'.format(name, id_depends)
            print(msg)

        while len(id_depends) > 0:
            completed = []
            for jid in id_depends:
                if test:
                    if np.random.random() < 0.3:
                        status = 'COMPLETED'
                    else:
                        status = 'RUNNING'
                else:
                    status = get_job_status(jid)

                if status in ('RUNNING', 'PENDING'):
                    continue
                elif status == 'COMPLETED':
                    print('jobid {0} completed. '.format(jid))
                    completed.append(jid)
                    self.pipeline.complete_job_ids.append(jid)
                elif status in ('FAILED', 'CANCELLED', 'TIMEOUT'):
                    time.sleep(5)
                    status = get_job_status(jid)
                    if status in ('FAILED', 'CANCELLED', 'TIMEOUT'):
                        raise RuntimeError('Unexpected status: Job {0} is {1}.'.format(jid, status))
                    else:
                        continue

                else:
                    logging.warning('Unknown status for {0}: {1}.'.format(jid, status))

            for jid in completed:
                id_depends.remove(jid)
            time.sleep(2)

    def submit_job(self, filt=None, test=False):
        """Submits job; returns jobid
        """
        self._wait_for_dependencies(filt=filt, test=test)

        cmd = self.submit_cmd(filt, test=test)

        output = subprocess.check_output(cmd, shell=True)
        m = re.search('batch job (\d+)', output)
        if not m:
            logging.error('Cannot find job id: {0}'.format(output))
            logging.error('Command submitted: {0}'.format(cmd))
            raise RuntimeError('Cannot find job id.')
        jobid = int(m.group(1))
        return jobid

class ManualBatchStage(PipelineStage):
    _default_time = 1200
    _override_batch_options = {}

    def write_batch_script(self, filt=None, test=False, **kwargs):
        jobname = self.jobname(filt)
        batchdir = self.pipeline.output_dir
        if not os.path.exists(batchdir):
            os.makedirs(batchdir)
        filename = os.path.join(batchdir, '{0}.sh'.format(jobname))

        try:
            time = self.pipeline['kwargs'][self.name]['time'] / 60 # slurm takes minutes
        except KeyError:
            time = self._default_time / 60

        if time < 65:
            time = 65

        batch_options = {
            'job-name':jobname,
            'output':'{0}/{1}.log'.format(batchdir, jobname),
            'time':time,
            'nodes':1,
            'ntasks-per-node':self.pipeline['cores_per_node'],
        }
        batch_options = dict(batch_options, **self._override_batch_options)
        batch_options = dict(batch_options, **kwargs)

        write_slurm_script(filename, self.cmd_str(filt=filt, test=test), **batch_options)

        self.batchfile = filename
        self.logfile = batch_options['output']
        return filename

    def submit_cmd(self, filt=None, test=False):
        batchfile = self.write_batch_script(filt, test=test)
        cmd = 'sbatch {0} '.format(batchfile)
        return cmd

class BatchStage(PipelineStage):

    @property
    def _default_kwargs(self):
        kws = {'mpiexec':'-bind-to socket',
               'batch-type':self.batch_type}

        kws['batch-output'] = self.pipeline.output_dir
        return kws

class SingleFrameDriverStage(BatchStage):
    name = 'singleFrameDriver'
    id_str_fmt = 'ccd={0[ccd]} '
    single_filter = True
    _id_options = ('ccd', 'visit')

class MakeDiscreteSkyMapStage(ManualBatchStage):
    name = 'makeDiscreteSkyMap'
    # depends = ('singleFrameDriver', )
    _override_batch_options = {'ntasks-per-node':1}

    @property
    def skymap(self):
        try:
            return self.pipeline['skymap']
        except KeyError:
            return None

    def _add_dependency(self, cmd, ids):
        if self.skymap is not None:
            return cmd
        else:
            return super(MakeDiscreteSkyMapStage, self)._add_dependency(cmd, ids)

    def cmd_str(self, filt=None, test=False):
        skymap_file = self.skymap
        if skymap_file is None:
            raise ValueError("No precomputed skymap provided!  " +
                             "Need to implement actual skymap creation to continue.")

        target_dir = '{0.pipeline.rerun_dir}/deepCoadd/'.format(self)
        cmd = 'mkdir -p {0}; ln -sf {1.skymap} {0} '.format(target_dir, self)
        return cmd

class MosaicStage(ManualBatchStage):
    name = 'mosaic'
    _id_options = ('tract', 'patch', 'ccd')
    depends = ('singleFrameDriver', 'makeDiscreteSkyMap')
    single_filter = True

    def diagDir(self, filt):
        return os.path.join(os.path.expanduser('~'), 'mosaicDiag',
                            self.pipeline.rerun, filt)

    def cmd_str(self, filt=None, test=False):
        cmd = super(MosaicStage, self).cmd_str(filt=filt, test=test)
        try:
            if self.pipeline['kwargs']['mosaic']['diagnostics']:
                cmd += '--diagDir {0} '.format(self.diagDir(filt))
        except KeyError:
            pass

        return cmd

class CoaddDriverStage(BatchStage):
    name = 'coaddDriver'
    _id_options = ('tract', 'patch')
    depends = ('singleFrameDriver', 'mosaic')
    single_filter = True

    def selectId_str(self, filt=None):
        s = 'ccd={0[ccd]} '.format(self.pipeline)
        s += 'filter={0} '.format(filt)
        s += 'visit={0} '.format(self.pipeline['visit'][filt])
        return s

class MultiBandDriverStage(BatchStage):
    name = 'multiBandDriver'
    depends = ('coaddDriver')
    _id_options = ('tract', 'patch')

    def id_str(self, filt=None):
        all_filters = '^'.join(self.pipeline.filters)
        return super(MultiBandDriverStage, self).id_str(filt=all_filters)
<commit_msg>kwarg parsing updated<commit_after>import yaml
import tempfile
import os, re
import logging
import numpy as np
import subprocess
import time
import multiprocessing

from .batch import write_slurm_script, get_job_status

class KwargDict(dict):

    def cmd_str(self, skip=['filters']):
        if skip is None:
            skip = []
        cmd = ''
        for kw, val in kws.items():
            if kw in skip:
                continue

            if kw == 'config':
                # val should be a dict
                for k, v in val.items():
                    cmd += '--config {0}={1} '.format(k,v)
                continue

            # If false than this is a flag/switch argument; don't turn it on.
            if val is False:
                continue

            cmd += '--{0}'.format(kw)
            if val != '' and val is not True:
                # ensure quotation marks where necessary
                if isinstance(val, basestring):
                    val = '"{0}"'.format(val)

                cmd += '={0} '.format(val)
                if kw == 'nodes':
                    cmd += '--procs {0[cores_per_node]} '.format(self.pipeline)
            else:
                cmd += ' '

        return cmd

class PipelineStage(object):
    name = None
    id_str_fmt = None
    batch_compatible = True
    single_filter = False
    batch_type = 'slurm'

    _default_kwargs = {}
    _kwarg_skip = ()

    def __init__(self, pipeline):
        self.pipeline = pipeline

    @property
    def _id_options(self):
        raise NotImplementedError("Need to define what id keys are valid.")

    @property
    def default_kwargs(self):
        return self._default_kwargs

    def id_str(self, filt=None):
        s = ''
        for key in self._id_options:
            try:
                if key=='visit':
                    s += 'visit={0} '.format(self.pipeline['visit'][filt])
                else:
                    fmt = '{0}={{0[{0}]}} '.format(key)
                    s += fmt.format(self.pipeline)
            except KeyError:
                continue

        if filt is not None:
            s += 'filter={0} '.format(filt)
        return s

    def kwarg_str(self, filt=None, **kwargs):
        """Returns a string of arguments

        example YAML:

            kwargs:
                singleFrameDriver:
                    time: 1200 # sec
                    cores: 240
                makeDiscreteSkyMap:
                    time: 300
                mosaic:
                    time: 4800
                    numCoresForRead: 24
                    diagnostics: True
                    #loglevel: debug
                coaddDriver:
                    time: 900 #more than necessary, but hopefully enough for Y-band
                    cores: 20
                    HSC-Y:
                        config:
                            assembleCoadd.subregionSize: "10000,50"
                multiBandDriver:
                    time: 18000
                    cores: 864
                all:
                    clobber-config: False

        This returns a `CmdLineTask`-compatible keyword string for this task,
        to be added to the id string to make the whole command line call string.
        """

        kws = KwargDict(self.default_kwargs)
        kws.update(self.pipeline['kwargs']['all'])
        kws.update(self.pipeline['kwargs'][self.name])
        if filt in self.pipeline['kwargs'][self.name]:
            kws.update(self.pipeline['kwargs'][self.name][filt])
        kws.update(kwargs)

        skip = self._kwarg_skip + tuple(self.pipeline.filters)
        return kws.cmd_str(skip=skip)

    def jobname(self, filt=None):
        job = self.pipeline.rerun.replace('/','-')
        s = '{0}-{1}'.format(job, self.name)
        if filt is not None:
            s += '-{}'.format(filt)
        return s

    def cmd_str(self, filt=None, test=False, **kwargs):
        cmd = '{0}.py {1[data_root]} --rerun {1.rerun} '.format(self.name, self.pipeline)

        if not isinstance(self, ManualBatchStage):
            cmd += '--job {0} '.format(self.jobname(filt))

        cmd += '--id {0} '.format(self.id_str(filt))

        if hasattr(self, 'selectId_str'):
            cmd += '--selectId {0} '.format(self.selectId_str(filt))

        cmd += self.kwarg_str(**kwargs)

        if test:
            cmd = cmd.replace('"', "'")
            np.random.seed()
            jobid = np.random.randint(10000)
            sleep = np.random.random()*2 + 2
            cmd = 'echo "{0}\nbatch job {1}; sleep {2}"'.format(cmd, jobid, sleep)
            print(cmd)

        return cmd

    def submit_cmd(self, filt=None, test=False):
        """
        """
        cmd = self.cmd_str(filt, test=test)
        return cmd

    def _get_dependent_jobids(self, filt=None):
        job_ids = self.pipeline.job_ids
        id_depends = []
        if hasattr(self, 'depends'):
            for key,i in job_ids.items():
                # If we know it's done, then skip.
                if i in self.pipeline.complete_job_ids:
                    continue

                for d in self.depends:
                    m = re.match(d, key)
                    if m:
                        if filt is not None:
                            if (key == '{0}-{1}'.format(d, filt) or
                                key == d):
                                id_depends.append(i)
                        else:
                            id_depends.append(i)
        return id_depends

    def _wait_for_dependencies(self, filt=None, test=False):
        # Make sure all dependencies are satisfied
        id_depends = self._get_dependent_jobids(filt=filt)

        if len(id_depends) > 0:
            name = self.name
            if filt is not None:
                name += ' {0}'.format(filt)
            msg = '{0} waiting for jobids {1}...'.format(name, id_depends)
            print(msg)

        while len(id_depends) > 0:
            completed = []
            for jid in id_depends:
                if test:
                    if np.random.random() < 0.3:
                        status = 'COMPLETED'
                    else:
                        status = 'RUNNING'
                else:
                    status = get_job_status(jid)

                if status in ('RUNNING', 'PENDING'):
                    continue
                elif status == 'COMPLETED':
                    print('jobid {0} completed. '.format(jid))
                    completed.append(jid)
                    self.pipeline.complete_job_ids.append(jid)
                elif status in ('FAILED', 'CANCELLED', 'TIMEOUT'):
                    time.sleep(5)
                    status = get_job_status(jid)
                    if status in ('FAILED', 'CANCELLED', 'TIMEOUT'):
                        raise RuntimeError('Unexpected status: Job {0} is {1}.'.format(jid, status))
                    else:
                        continue

                else:
                    logging.warning('Unknown status for {0}: {1}.'.format(jid, status))

            for jid in completed:
                id_depends.remove(jid)
            time.sleep(2)

    def submit_job(self, filt=None, test=False):
        """Submits job; returns jobid
        """
        self._wait_for_dependencies(filt=filt, test=test)

        cmd = self.submit_cmd(filt, test=test)

        output = subprocess.check_output(cmd, shell=True)
        m = re.search('batch job (\d+)', output)
        if not m:
            logging.error('Cannot find job id: {0}'.format(output))
            logging.error('Command submitted: {0}'.format(cmd))
            raise RuntimeError('Cannot find job id.')
        jobid = int(m.group(1))
        return jobid

class ManualBatchStage(PipelineStage):
    _default_time = 1200
    _override_batch_options = {}
    _kwarg_skip = ('time,')

    def write_batch_script(self, filt=None, test=False, **kwargs):
        jobname = self.jobname(filt)
        batchdir = self.pipeline.output_dir
        if not os.path.exists(batchdir):
            os.makedirs(batchdir)
        filename = os.path.join(batchdir, '{0}.sh'.format(jobname))

        try:
            time = self.pipeline['kwargs'][self.name]['time'] / 60 # slurm takes minutes
        except KeyError:
            time = self._default_time / 60

        if time < 65:
            time = 65

        batch_options = {
            'job-name':jobname,
            'output':'{0}/{1}.log'.format(batchdir, jobname),
            'time':time,
            'nodes':1,
            'ntasks-per-node':self.pipeline['cores_per_node'],
        }
        batch_options = dict(batch_options, **self._override_batch_options)
        batch_options = dict(batch_options, **kwargs)

        write_slurm_script(filename, self.cmd_str(filt=filt, test=test), **batch_options)

        self.batchfile = filename
        self.logfile = batch_options['output']
        return filename

    def submit_cmd(self, filt=None, test=False):
        batchfile = self.write_batch_script(filt, test=test)
        cmd = 'sbatch {0} '.format(batchfile)
        return cmd

class BatchStage(PipelineStage):

    @property
    def _default_kwargs(self):
        kws = {'mpiexec':'-bind-to socket',
               'batch-type':self.batch_type}

        kws['batch-output'] = self.pipeline.output_dir
        return kws

class SingleFrameDriverStage(BatchStage):
    name = 'singleFrameDriver'
    id_str_fmt = 'ccd={0[ccd]} '
    single_filter = True
    _id_options = ('ccd', 'visit')

class MakeDiscreteSkyMapStage(ManualBatchStage):
    name = 'makeDiscreteSkyMap'
    # depends = ('singleFrameDriver', )
    _override_batch_options = {'ntasks-per-node':1}

    @property
    def skymap(self):
        try:
            return self.pipeline['skymap']
        except KeyError:
            return None

    def _add_dependency(self, cmd, ids):
        if self.skymap is not None:
            return cmd
        else:
            return super(MakeDiscreteSkyMapStage, self)._add_dependency(cmd, ids)

    def cmd_str(self, filt=None, test=False):
        skymap_file = self.skymap
        if skymap_file is None:
            raise ValueError("No precomputed skymap provided!  " +
                             "Need to implement actual skymap creation to continue.")

        target_dir = '{0.pipeline.rerun_dir}/deepCoadd/'.format(self)
        cmd = 'mkdir -p {0}; ln -sf {1.skymap} {0} '.format(target_dir, self)
        return cmd

class MosaicStage(ManualBatchStage):
    name = 'mosaic'
    _id_options = ('tract', 'patch', 'ccd')
    depends = ('singleFrameDriver', 'makeDiscreteSkyMap')
    single_filter = True

    def diagDir(self, filt):
        return os.path.join(os.path.expanduser('~'), 'mosaicDiag',
                            self.pipeline.rerun, filt)

    def cmd_str(self, filt=None, test=False):
        cmd = super(MosaicStage, self).cmd_str(filt=filt, test=test)
        try:
            if self.pipeline['kwargs']['mosaic']['diagnostics']:
                cmd += '--diagDir {0} '.format(self.diagDir(filt))
        except KeyError:
            pass

        return cmd

class CoaddDriverStage(BatchStage):
    name = 'coaddDriver'
    _id_options = ('tract', 'patch')
    depends = ('singleFrameDriver', 'mosaic')
    single_filter = True

    def selectId_str(self, filt=None):
        s = 'ccd={0[ccd]} '.format(self.pipeline)
        s += 'filter={0} '.format(filt)
        s += 'visit={0} '.format(self.pipeline['visit'][filt])
        return s

class MultiBandDriverStage(BatchStage):
    name = 'multiBandDriver'
    depends = ('coaddDriver')
    _id_options = ('tract', 'patch')

    def id_str(self, filt=None):
        all_filters = '^'.join(self.pipeline.filters)
        return super(MultiBandDriverStage, self).id_str(filt=all_filters)
<|endoftext|>


<commit_before># Skeleton Flask Project
from flask import Flask
app = Flask(__name__)  # Pass in default file name as parameter

# Decorators for methods to execute based on route(s)
@app.route('/')
@app.route('/hello')
def HelloWorld():
    return "Hello World."

# __main__ is the default name given to the application run by the Python
# interpreter. The below if statement only runs if this file is being executed
# by it explicitly. If it's imported, the below won't run
if __name__ == '__main__':
    app.debug = True  # Will reload automatically when code changes
    app.run(host='0.0.0.0', port=5000)  # Run on public IP, port 5000
<commit_msg>feat: Add database imports and restaurant routes<commit_after>from flask import Flask
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from database_setup import Base, Restaurant, MenuItem

def createDBSession():
    """Connect to database and return session"""
    engine = create_engine('sqlite:///restaurantmenu.db', echo=True)
    Base.metadata.bind = engine

    DBSession = sessionmaker(bind=engine)
    session = DBSession()
    return session

app = Flask(__name__)  # Pass in default file name as parameter

# Decorators for methods to execute based on route(s)
@app.route('/')
@app.route('/restaurants/<int:restaurant_id>/')
def restaurantMenu(restaurant_id):
    session = createDBSession()
    restaurant = session.query(Restaurant).\
        filter(Restaurant.id == restaurant_id).one()
    items = session.query(MenuItem).\
        filter(MenuItem.restaurant_id == restaurant.id).all()
    output = ""

    for item in items:
        output += ("
%s
%s
%s
") % (
            item.name, item.price, item.description)

    return output

# __main__ is the default name given to the application run by the Python
# interpreter. The below if statement only runs if this file is being executed
# by it explicitly. If it's imported, the below won't run
if __name__ == '__main__':
    app.debug = True  # Will reload automatically when code changes
    app.run(host='0.0.0.0', port=5000)  # Run on public IP, port 5000
<|endoftext|>