<commit_before>#!/usr/bin/env python

# Akvo RSR is covered by the GNU Affero General Public License.
# See more details in the license.txt file located at the root folder of the Akvo RSR module.
# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.

import os, signal, sys

from selenium import selenium

DEFAULT_SELENIUM_RC_HOST = "localhost"
DEFAULT_SELENIUM_RC_PORT = 4444
DEFAULT_BROWSER_ENVIRONMENT = "*firefox"

RC_SERVER_PID_FILE_NAME = "rc_server.pid"

def display_usage_and_exit():
    default_values = "%s %i %s" % (DEFAULT_SELENIUM_RC_HOST, DEFAULT_SELENIUM_RC_PORT, DEFAULT_BROWSER_ENVIRONMENT)
    print 'Usage: stop_selenium_rc_server <rc_server_log_path> [rc_host] [rc_port] [browser_environment]'
    print "If no server parameters are specified, the following default values will be used: %s" % (default_values)
    sys.exit(1)

def verify_script_parameters():
    if len(sys.argv) <= 1:
        print ">> Missing rc_server_log_path parameter"
        display_usage_and_exit()
    elif len(sys.argv) != 2 and len(sys.argv) != 5:
        display_usage_and_exit()
    elif len(sys.argv) == 5 and not sys.argv[3].isdigit():
        print ">> The rc_port parameter [%s] must be an integer" % sys.argv[3]
        display_usage_and_exit()

    rc_server_log_path = os.path.abspath(sys.argv[1])

    if not os.path.exists(rc_server_log_path):
        print ">> Selenium RC server log path does not exist: %s" % (rc_server_log_path)
        display_usage_and_exit()
    elif not os.path.isdir(rc_server_log_path):
        print ">> Selenium RC server log path is not a directory: %s" % (rc_server_log_path)
        display_usage_and_exit()

def stop_rc_server():
    if rc_server_process_is_not_running():
        print "Selenium RC server does not appear to be running -- process ID file not found: %s" % rc_server_log_file_path()
        sys.exit(0)

    try:
        stop_rc_server_with_shutdown_command()
        print "Server stopped successfully\n"
        os.remove(rc_server_process_id_file_path())
    except Exception, exception:
        print "Unable to stop Selenium RC server with shutdown command: %s" % (exception)
        kill_unresponsive_rc_server_process()

def rc_server_process_is_not_running():
    return not os.path.exists(rc_server_process_id_file_path())

def stop_rc_server_with_shutdown_command():
    selenium_rc_host = DEFAULT_SELENIUM_RC_HOST
    selenium_rc_port = DEFAULT_SELENIUM_RC_PORT
    browser_environment = DEFAULT_BROWSER_ENVIRONMENT

    if len(sys.argv) == 5:
        selenium_rc_host = sys.argv[2]
        selenium_rc_port = int(sys.argv[3])
        browser_environment = sys.argv[4]

    try:
        print "\nStopping Selenium RC server [%s, %i, %s]..." % (selenium_rc_host, selenium_rc_port, browser_environment)
        selenium(selenium_rc_host, selenium_rc_port, browser_environment, '').shut_down_selenium_server()
    except Exception, exception:
        if str(exception).endswith('Connection refused'):
            print "Server already appears to have been stopped: %s" % (exception)
            sys.exit(0)
        else:
            raise exception

def kill_unresponsive_rc_server_process():
    process_id = rc_server_process_id()

    if process_id_exists(process_id):
        print process_id
        print "Killing unresponsive Selenium RC server process"
        os.kill(process_id, signal.SIGTERM) # send terminate signal
    else:
        raise Exception("Server process ID %i no longer exists: %s" % (process_id, os_error))

def rc_server_process_id():
    process_id_file_path = rc_server_process_id_file_path()
    process_id = open(process_id_file_path, 'r').readline().strip()

    if not process_id.isdigit():
        raise Exception("Unexpected Selenium RC server process ID [%s] in: %s" % (process_id, process_id_file_path))

    return int(process_id)

def process_id_exists(process_id):
    try:
        os.kill(process_id, 0) # attempt to send a harmless signal (0) to the process
        return True
    except OSError, os_error:
        if os_error.endswith('No such process'):
            return False
        else:
            raise os_error

def rc_server_process_id_file_path():
    return os.path.join(os.path.abspath(sys.argv[1]), RC_SERVER_PID_FILE_NAME)

if __name__ == "__main__":
    verify_script_parameters()
    stop_rc_server()
<commit_msg>[#3345224] Used os.path.realpath() instead of os.path.abspath() for RC server log path.<commit_after>#!/usr/bin/env python

# Akvo RSR is covered by the GNU Affero General Public License.
# See more details in the license.txt file located at the root folder of the Akvo RSR module.
# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.

import os, signal, sys

from selenium import selenium

DEFAULT_SELENIUM_RC_HOST = "localhost"
DEFAULT_SELENIUM_RC_PORT = 4444
DEFAULT_BROWSER_ENVIRONMENT = "*firefox"

RC_SERVER_PID_FILE_NAME = "rc_server.pid"

def display_usage_and_exit():
    default_values = "%s %i %s" % (DEFAULT_SELENIUM_RC_HOST, DEFAULT_SELENIUM_RC_PORT, DEFAULT_BROWSER_ENVIRONMENT)
    print 'Usage: stop_selenium_rc_server <rc_server_log_path> [rc_host] [rc_port] [browser_environment]'
    print "If no server parameters are specified, the following default values will be used: %s" % (default_values)
    sys.exit(1)

def verify_script_parameters():
    if len(sys.argv) <= 1:
        print ">> Missing rc_server_log_path parameter"
        display_usage_and_exit()
    elif len(sys.argv) != 2 and len(sys.argv) != 5:
        display_usage_and_exit()
    elif len(sys.argv) == 5 and not sys.argv[3].isdigit():
        print ">> The rc_port parameter [%s] must be an integer" % sys.argv[3]
        display_usage_and_exit()

    rc_server_log_path = os.path.realpath(sys.argv[1])

    if not os.path.exists(rc_server_log_path):
        print ">> Selenium RC server log path does not exist: %s" % (rc_server_log_path)
        display_usage_and_exit()
    elif not os.path.isdir(rc_server_log_path):
        print ">> Selenium RC server log path is not a directory: %s" % (rc_server_log_path)
        display_usage_and_exit()

def stop_rc_server():
    if rc_server_process_is_not_running():
        print "Selenium RC server does not appear to be running -- process ID file not found: %s" % rc_server_log_file_path()
        sys.exit(0)

    try:
        stop_rc_server_with_shutdown_command()
        print "Server stopped successfully\n"
        os.remove(rc_server_process_id_file_path())
    except Exception, exception:
        print "Unable to stop Selenium RC server with shutdown command: %s" % (exception)
        kill_unresponsive_rc_server_process()

def rc_server_process_is_not_running():
    return not os.path.exists(rc_server_process_id_file_path())

def stop_rc_server_with_shutdown_command():
    selenium_rc_host = DEFAULT_SELENIUM_RC_HOST
    selenium_rc_port = DEFAULT_SELENIUM_RC_PORT
    browser_environment = DEFAULT_BROWSER_ENVIRONMENT

    if len(sys.argv) == 5:
        selenium_rc_host = sys.argv[2]
        selenium_rc_port = int(sys.argv[3])
        browser_environment = sys.argv[4]

    try:
        print "\nStopping Selenium RC server [%s, %i, %s]..." % (selenium_rc_host, selenium_rc_port, browser_environment)
        selenium(selenium_rc_host, selenium_rc_port, browser_environment, '').shut_down_selenium_server()
    except Exception, exception:
        if str(exception).endswith('Connection refused'):
            print "Server already appears to have been stopped: %s" % (exception)
            sys.exit(0)
        else:
            raise exception

def kill_unresponsive_rc_server_process():
    process_id = rc_server_process_id()

    if process_id_exists(process_id):
        print process_id
        print "Killing unresponsive Selenium RC server process"
        os.kill(process_id, signal.SIGTERM) # send terminate signal
    else:
        raise Exception("Server process ID %i no longer exists: %s" % (process_id, os_error))

def rc_server_process_id():
    process_id_file_path = rc_server_process_id_file_path()
    process_id = open(process_id_file_path, 'r').readline().strip()

    if not process_id.isdigit():
        raise Exception("Unexpected Selenium RC server process ID [%s] in: %s" % (process_id, process_id_file_path))

    return int(process_id)

def process_id_exists(process_id):
    try:
        os.kill(process_id, 0) # attempt to send a harmless signal (0) to the process
        return True
    except OSError, os_error:
        if os_error.endswith('No such process'):
            return False
        else:
            raise os_error

def rc_server_process_id_file_path():
    return os.path.join(os.path.realpath(sys.argv[1]), RC_SERVER_PID_FILE_NAME)

if __name__ == "__main__":
    verify_script_parameters()
    stop_rc_server()
<|endoftext|>


<commit_before>import json
from builtins import super

from django import forms
from django.conf import settings

class JSONEditorWidget(forms.Widget):
    class Media:
        css = {'all': (settings.STATIC_URL + 'dist/jsoneditor.min.css',)}
        js = (settings.STATIC_URL + 'dist/jsoneditor.min.js',)

    template_name = 'django_json_widget.html'

    def __init__(self, attrs=None, mode='code', options=None, width=None, height=None):
        default_options = {
            'modes': ['text', 'code', 'tree', 'form', 'view'],
            'mode': mode,
            'search': True,
        }
        if options:
            default_options.update(options)

        self.options = default_options
        self.width = width
        self.height = height

        super().__init__(attrs=attrs)

    def get_context(self, name, value, attrs):
        context = super().get_context(name, value, attrs)
        context['widget']['options'] = json.dumps(self.options)
        context['widget']['width'] = self.width
        context['widget']['height'] = self.height

        return context
<commit_msg>Fix whitespace<commit_after>import json
from builtins import super

from django import forms
from django.conf import settings

class JSONEditorWidget(forms.Widget):
    class Media:
        css = {'all': (settings.STATIC_URL + 'dist/jsoneditor.min.css',)}
        js = (settings.STATIC_URL + 'dist/jsoneditor.min.js',)

    template_name = 'django_json_widget.html'

    def __init__(self, attrs=None, mode='code', options=None, width=None, height=None):
        default_options = {
            'modes': ['text', 'code', 'tree', 'form', 'view'],
            'mode': mode,
            'search': True,
        }
        if options:
            default_options.update(options)

        self.options = default_options
        self.width = width
        self.height = height

        super().__init__(attrs=attrs)

    def get_context(self, name, value, attrs):
        context = super().get_context(name, value, attrs)
        context['widget']['options'] = json.dumps(self.options)
        context['widget']['width'] = self.width
        context['widget']['height'] = self.height

        return context
<|endoftext|>


<commit_before>"""

"""

import os
import urllib
import itertools

import github3
import cachecontrol
from dateutil.parser import parse
from requests.adapters import HTTPAdapter

from website.addons.github import settings as github_settings
from website.addons.github.exceptions import (
    NotFoundError, EmptyRepoError, TooBigError, GitHubError
)

# Initialize caches
https_cache = cachecontrol.CacheControlAdapter()
default_adapter = HTTPAdapter()

class GitHub(object):

    def __init__(self, access_token=None, token_type=None):

        self.access_token = access_token
        if access_token and token_type:
            self.gh3 = github3.login(token=access_token)
            self.gh3.set_client_id(
                github_settings.CLIENT_ID, github_settings.CLIENT_SECRET
            )
        else:
            self.gh3 = github3.GitHub()

        # Caching libary
        if github_settings.CACHE:
            self.gh3._session.mount('https://api.github.com/user', default_adapter)
            self.gh3._session.mount('https://', https_cache)

    @classmethod
    def from_settings(cls, settings):
        if settings:
            return cls(
                access_token=settings.oauth_access_token,
                token_type=settings.oauth_token_type,
            )
        return cls()

    def user(self, user=None):
        """Fetch a user or the authenticated user.

        :param user: Optional GitHub user name; will fetch authenticated
            user if omitted
        :return dict: GitHub API response

        """
        return self.gh3.user(user)

    def repo(self, user, repo):
        """Get a single Github repo's info.

        :param str user: GitHub user name
        :param str repo: GitHub repo name
        :return: Dict of repo information
            See http://developer.github.com/v3/repos/#get

        """
        rv = self.gh3.repository(user, repo)
        if rv:
            return rv
        raise NotFoundError

    def repos(self):
        return self.gh3.iter_repos(type='all', sort='full_name')

    def user_repos(self, user):
        return self.gh3.iter_user_repos(user, type='all', sort='full_name')

    def my_org_repos(self, permissions=None):
        permissions = permissions or ['push']
        return itertools.chain.from_iterable(
            team.iter_repos()
            for team in self.gh3.iter_user_teams()
            if team.permission in permissions
        )

    def create_repo(self, repo, **kwargs):
        return self.gh3.create_repo(repo, **kwargs)

    def branches(self, user, repo, branch=None):
        """List a repo's branches or get a single branch.

        :param str user: GitHub user name
        :param str repo: GitHub repo name
        :param str branch: Branch name if getting a single branch
        :return: List of branch dicts
            http://developer.github.com/v3/repos/#list-branches

        """
        return self.repo(user, repo).iter_branches() or []

    def commits(self, user, repo, path=None, sha=None):
        """Get commits for a repo or file.

        :param str user: GitHub user name
        :param str repo: GitHub repo name
        :param str path: Path to file within repo
        :param str sha: SHA or branch name
        :return list: List of commit dicts from GitHub; see
            http://developer.github.com/v3/repos/commits/

        """
        return self.repo(user, repo).iter_commits(sha=sha, path=path)

    def history(self, user, repo, path, sha=None):
        """Get commit history for a file.

        :param str user: GitHub user name
        :param str repo: GitHub repo name
        :param str path: Path to file within repo
        :param str sha: SHA or branch name
        :return list: List of dicts summarizing commits

        """
        req = self.commits(user, repo, path=path, sha=sha)

        if req:
            return [
                {
                    'sha': commit.commit.sha,
                    'name': commit.commit.author['name'],
                    'email': commit.commit.author['email'],
                    'date': parse(commit.commit.author['date']).ctime(),
                }
                for commit in req
            ]

    def tree(self, user, repo, sha, recursive=True):
        """Get file tree for a repo.

        :param str user: GitHub user name
        :param str repo: GitHub repo name
        :param str sha: Branch name or SHA
        :param bool recursive: Walk repo recursively
        :param dict registration_data: Registered commit data
        :returns: tuple: Tuple of commit ID and tree JSON; see
            http://developer.github.com/v3/git/trees/

        """
        try:
            tree = self.repo(user, repo).tree(sha)
            if recursive:
                return tree.recurse()
            return tree
        except GitHubError as error:
            if error.code == 409:
                raise EmptyRepoError
            raise

    def file(self, user, repo, path, ref=None):
        """Get a file within a repo and its contents.

        :returns: A tuple of the form (<filename>, <file_content>, <file_size):

        """
        try:
            req = self.contents(user=user, repo=repo, path=path, ref=ref)
        except GitHubError as error:
            if 'requested blob is too large' in error.message:
                raise TooBigError()
            raise
        if req:
            return req.name, req.decoded, req.size
        return None, None, None

    def contents(self, user, repo, path='', ref=None):
        """Get the contents of a path within a repo.
        http://developer.github.com/v3/repos/contents/#get-contents

        """
        return self.repo(user, repo).contents(path.encode('utf-8'), ref)

    # TODO: Test
    def starball(self, user, repo, archive='tar', ref='master'):
        """Get link for archive download.

        :param str user: GitHub user name
        :param str repo: GitHub repo name
        :param str archive: Archive format [tar|zip]
        :param str ref: Git reference
        :returns: tuple: Tuple of headers and file location

        """

        # github3 archive method writes file to disk
        repository = self.repo(user, repo)
        url = repository._build_url(archive + 'ball', ref, base_url=repository._api)
        resp = repository._get(url, allow_redirects=True, stream=True)

        return resp.headers, resp.content

    def set_privacy(self, user, repo, private):
        """Set privacy of GitHub repo.

        :param str user: GitHub user name
        :param str repo: GitHub repo name
        :param bool private: Make repo private; see
            http://developer.github.com/v3/repos/#edit

        """
        return self.repo(user, repo).edit(repo, private=private)

    #########
    # Hooks #
    #########

    def hooks(self, user, repo):
        """List webhooks

        :param str user: GitHub user name
        :param str repo: GitHub repo name
        :return list: List of commit dicts from GitHub; see
            http://developer.github.com/v3/repos/hooks/#json-http

        """
        return self.repo(user, repo).iter_hooks()

    def add_hook(self, user, repo, name, config, events=None, active=True):
        """Create a webhook.

        :param str user: GitHub user name
        :param str repo: GitHub repo name
        :return dict: Hook info from GitHub: see see
            http://developer.github.com/v3/repos/hooks/#json-http

        """

        return self.repo(user, repo).create_hook(name, config, events, active)

    def delete_hook(self, user, repo, _id):
        """Delete a webhook.

        :param str user: GitHub user name
        :param str repo: GitHub repo name
        :return bool: True if successful, False otherwise
        :raises: NotFoundError if repo or hook cannot be located

        """
        repo = self.repo(user, repo)
        hook = repo.hook(_id)
        if hook is None:
            raise NotFoundError
        return repo.hook(_id).delete()

    ########
    # CRUD #
    ########

    def create_file(self, user, repo, path, message, content, branch=None, committer=None, author=None):
        return self.repo(
            user, repo
        ).create_file(
            path, message, content, branch, author, committer
        )

    def update_file(self, user, repo, path, message, content, sha=None, branch=None, committer=None, author=None):
        return self.repo(
            user, repo
        ).update_file(
            path, message, content, sha, branch, author, committer
        )

    def delete_file(self, user, repo, path, message, sha, branch=None, committer=None, author=None):
        return self.repo(user, repo).delete_file(path, message, sha, branch, committer, author)

    ########
    # Auth #
    ########

    def revoke_token(self):

        if self.access_token:
            return self.gh3.revoke_authorization(self.access_token)

def ref_to_params(branch=None, sha=None):

    params = urllib.urlencode({
        key: value
        for key, value in {
            'branch': branch,
            'sha': sha,
        }.iteritems()
        if value
    })
    if params:
        return '?' + params
    return ''

# TODO: Use Node#api_url_for and Node#web_url_for
def build_github_urls(item, node_url, node_api_url, branch, sha):

    quote_path = urllib.quote_plus(item.path.encode('utf-8'))
    params = ref_to_params(branch, sha)

    if item.type in ['tree', 'dir']:
        return {
            'upload': os.path.join(node_api_url, 'github', 'file', quote_path) + '/' + params,
            'fetch': os.path.join(node_api_url, 'github', 'hgrid', item.path) + '/',
        }
    elif item.type in ['file', 'blob']:
        return {
            'view': os.path.join(node_url, 'github', 'file', quote_path) + '/' + params,
            'download': os.path.join(node_url, 'github', 'file', quote_path, 'download') + '/' + params,
            'delete': os.path.join(node_api_url, 'github', 'file', quote_path) + '/' + ref_to_params(branch, item.sha),
        }
    raise ValueError
<commit_msg>Catch an attribute error when tree returns None in the case of an empty private repo<commit_after>"""

"""

import os
import urllib
import itertools

import github3
import cachecontrol
from dateutil.parser import parse
from requests.adapters import HTTPAdapter

from website.addons.github import settings as github_settings
from website.addons.github.exceptions import (
    NotFoundError, EmptyRepoError, TooBigError, GitHubError
)

# Initialize caches
https_cache = cachecontrol.CacheControlAdapter()
default_adapter = HTTPAdapter()

class GitHub(object):

    def __init__(self, access_token=None, token_type=None):

        self.access_token = access_token
        if access_token and token_type:
            self.gh3 = github3.login(token=access_token)
            self.gh3.set_client_id(
                github_settings.CLIENT_ID, github_settings.CLIENT_SECRET
            )
        else:
            self.gh3 = github3.GitHub()

        # Caching libary
        if github_settings.CACHE:
            self.gh3._session.mount('https://api.github.com/user', default_adapter)
            self.gh3._session.mount('https://', https_cache)

    @classmethod
    def from_settings(cls, settings):
        if settings:
            return cls(
                access_token=settings.oauth_access_token,
                token_type=settings.oauth_token_type,
            )
        return cls()

    def user(self, user=None):
        """Fetch a user or the authenticated user.

        :param user: Optional GitHub user name; will fetch authenticated
            user if omitted
        :return dict: GitHub API response

        """
        return self.gh3.user(user)

    def repo(self, user, repo):
        """Get a single Github repo's info.

        :param str user: GitHub user name
        :param str repo: GitHub repo name
        :return: Dict of repo information
            See http://developer.github.com/v3/repos/#get

        """
        rv = self.gh3.repository(user, repo)
        if rv:
            return rv
        raise NotFoundError

    def repos(self):
        return self.gh3.iter_repos(type='all', sort='full_name')

    def user_repos(self, user):
        return self.gh3.iter_user_repos(user, type='all', sort='full_name')

    def my_org_repos(self, permissions=None):
        permissions = permissions or ['push']
        return itertools.chain.from_iterable(
            team.iter_repos()
            for team in self.gh3.iter_user_teams()
            if team.permission in permissions
        )

    def create_repo(self, repo, **kwargs):
        return self.gh3.create_repo(repo, **kwargs)

    def branches(self, user, repo, branch=None):
        """List a repo's branches or get a single branch.

        :param str user: GitHub user name
        :param str repo: GitHub repo name
        :param str branch: Branch name if getting a single branch
        :return: List of branch dicts
            http://developer.github.com/v3/repos/#list-branches

        """
        return self.repo(user, repo).iter_branches() or []

    def commits(self, user, repo, path=None, sha=None):
        """Get commits for a repo or file.

        :param str user: GitHub user name
        :param str repo: GitHub repo name
        :param str path: Path to file within repo
        :param str sha: SHA or branch name
        :return list: List of commit dicts from GitHub; see
            http://developer.github.com/v3/repos/commits/

        """
        return self.repo(user, repo).iter_commits(sha=sha, path=path)

    def history(self, user, repo, path, sha=None):
        """Get commit history for a file.

        :param str user: GitHub user name
        :param str repo: GitHub repo name
        :param str path: Path to file within repo
        :param str sha: SHA or branch name
        :return list: List of dicts summarizing commits

        """
        req = self.commits(user, repo, path=path, sha=sha)

        if req:
            return [
                {
                    'sha': commit.commit.sha,
                    'name': commit.commit.author['name'],
                    'email': commit.commit.author['email'],
                    'date': parse(commit.commit.author['date']).ctime(),
                }
                for commit in req
            ]

    def tree(self, user, repo, sha, recursive=True):
        """Get file tree for a repo.

        :param str user: GitHub user name
        :param str repo: GitHub repo name
        :param str sha: Branch name or SHA
        :param bool recursive: Walk repo recursively
        :param dict registration_data: Registered commit data
        :returns: tuple: Tuple of commit ID and tree JSON; see
            http://developer.github.com/v3/git/trees/

        """
        try:
            tree = self.repo(user, repo).tree(sha)
            if recursive:
                return tree.recurse()
            return tree
        except AttributeError:
            if not tree:
                raise EmptyRepoError
            raise
        except GitHubError as error:
            if error.code == 409:
                raise EmptyRepoError
            raise

    def file(self, user, repo, path, ref=None):
        """Get a file within a repo and its contents.

        :returns: A tuple of the form (<filename>, <file_content>, <file_size):

        """
        try:
            req = self.contents(user=user, repo=repo, path=path, ref=ref)
        except GitHubError as error:
            if 'requested blob is too large' in error.message:
                raise TooBigError()
            raise
        if req:
            return req.name, req.decoded, req.size
        return None, None, None

    def contents(self, user, repo, path='', ref=None):
        """Get the contents of a path within a repo.
        http://developer.github.com/v3/repos/contents/#get-contents

        """
        return self.repo(user, repo).contents(path.encode('utf-8'), ref)

    # TODO: Test
    def starball(self, user, repo, archive='tar', ref='master'):
        """Get link for archive download.

        :param str user: GitHub user name
        :param str repo: GitHub repo name
        :param str archive: Archive format [tar|zip]
        :param str ref: Git reference
        :returns: tuple: Tuple of headers and file location

        """

        # github3 archive method writes file to disk
        repository = self.repo(user, repo)
        url = repository._build_url(archive + 'ball', ref, base_url=repository._api)
        resp = repository._get(url, allow_redirects=True, stream=True)

        return resp.headers, resp.content

    def set_privacy(self, user, repo, private):
        """Set privacy of GitHub repo.

        :param str user: GitHub user name
        :param str repo: GitHub repo name
        :param bool private: Make repo private; see
            http://developer.github.com/v3/repos/#edit

        """
        return self.repo(user, repo).edit(repo, private=private)

    #########
    # Hooks #
    #########

    def hooks(self, user, repo):
        """List webhooks

        :param str user: GitHub user name
        :param str repo: GitHub repo name
        :return list: List of commit dicts from GitHub; see
            http://developer.github.com/v3/repos/hooks/#json-http

        """
        return self.repo(user, repo).iter_hooks()

    def add_hook(self, user, repo, name, config, events=None, active=True):
        """Create a webhook.

        :param str user: GitHub user name
        :param str repo: GitHub repo name
        :return dict: Hook info from GitHub: see see
            http://developer.github.com/v3/repos/hooks/#json-http

        """

        return self.repo(user, repo).create_hook(name, config, events, active)

    def delete_hook(self, user, repo, _id):
        """Delete a webhook.

        :param str user: GitHub user name
        :param str repo: GitHub repo name
        :return bool: True if successful, False otherwise
        :raises: NotFoundError if repo or hook cannot be located

        """
        repo = self.repo(user, repo)
        hook = repo.hook(_id)
        if hook is None:
            raise NotFoundError
        return repo.hook(_id).delete()

    ########
    # CRUD #
    ########

    def create_file(self, user, repo, path, message, content, branch=None, committer=None, author=None):
        return self.repo(
            user, repo
        ).create_file(
            path, message, content, branch, author, committer
        )

    def update_file(self, user, repo, path, message, content, sha=None, branch=None, committer=None, author=None):
        return self.repo(
            user, repo
        ).update_file(
            path, message, content, sha, branch, author, committer
        )

    def delete_file(self, user, repo, path, message, sha, branch=None, committer=None, author=None):
        return self.repo(user, repo).delete_file(path, message, sha, branch, committer, author)

    ########
    # Auth #
    ########

    def revoke_token(self):

        if self.access_token:
            return self.gh3.revoke_authorization(self.access_token)

def ref_to_params(branch=None, sha=None):

    params = urllib.urlencode({
        key: value
        for key, value in {
            'branch': branch,
            'sha': sha,
        }.iteritems()
        if value
    })
    if params:
        return '?' + params
    return ''

# TODO: Use Node#api_url_for and Node#web_url_for
def build_github_urls(item, node_url, node_api_url, branch, sha):

    quote_path = urllib.quote_plus(item.path.encode('utf-8'))
    params = ref_to_params(branch, sha)

    if item.type in ['tree', 'dir']:
        return {
            'upload': os.path.join(node_api_url, 'github', 'file', quote_path) + '/' + params,
            'fetch': os.path.join(node_api_url, 'github', 'hgrid', item.path) + '/',
        }
    elif item.type in ['file', 'blob']:
        return {
            'view': os.path.join(node_url, 'github', 'file', quote_path) + '/' + params,
            'download': os.path.join(node_url, 'github', 'file', quote_path, 'download') + '/' + params,
            'delete': os.path.join(node_api_url, 'github', 'file', quote_path) + '/' + ref_to_params(branch, item.sha),
        }
    raise ValueError
<|endoftext|>


<commit_before>import random

from pyparsing import alphas, empty, oneOf, replaceWith
from pyparsing import CaselessLiteral, OneOrMore, Optional, ParseException
from pyparsing import CaselessKeyword, LineEnd, MatchFirst, Word

from myriad.game.shell import command
from myriad.item import Item

class ShellParseException(ParseException):
    pass

class ShellParser(object):
    def __init__(self, session=None):
        self.session = session
        self.bnf = self.makeBNF()

    def makeCommandParseAction(self, cls):
        def cmdParseAction(s, l, tokens):
            return cls(tokens)
        return cmdParseAction

    def makeBNF(self):
        makeCmd = lambda s: MatchFirst(map(CaselessKeyword, s.split()))
        invVerb = makeCmd("INV INVENTORY I")
        mapVerb = makeCmd("MAP M")
        dropVerb = makeCmd("DROP LEAVE")
        takeVerb = makeCmd("TAKE PICKUP") | \
            (CaselessLiteral("PICK") + CaselessLiteral("UP"))
        moveVerb = makeCmd("MOVE GO") | empty
        useVerb = makeCmd("USE U")
        openVerb = makeCmd("OPEN O")
        quitVerb = makeCmd("QUIT Q")
        lookVerb = makeCmd("LOOK L")
        doorsVerb = CaselessKeyword("DOORS")
        helpVerb = makeCmd("H HELP ?")
        readVerb = CaselessKeyword("READ")

        itemRef = OneOrMore(Word(alphas)).setParseAction(self.validateItemName)
        makeDir = lambda s : makeCmd(s).setParseAction(
            replaceWith(s.split()[0]))
        nDir = makeDir("N NORTH")
        sDir = makeDir("S SOUTH")
        eDir = makeDir("E EAST")
        wDir = makeDir("W WEST")
        neDir = makeDir("NE NORTHEAST")
        seDir = makeDir("SE SOUTHEAST")
        swDir = makeDir("SW SOUTHWEST")
        nwDir = makeDir("NW NORTHWEST")
        moveDirection = nDir | sDir | eDir | wDir | neDir | seDir | swDir \
            | nwDir

        invCommand = invVerb
        mapCommand = mapVerb
        dropCommand = dropVerb + itemRef("item")
        takeCommand = takeVerb + itemRef("item")
        useCommand = useVerb + itemRef("usedObj") + \
            Optional(oneOf("IN ON", caseless=True)) + \
            Optional(itemRef, default=None)("targetObj")
        openCommand = openVerb + itemRef("item")
        moveCommand = moveVerb + moveDirection("direction")
        quitCommand = quitVerb
        lookCommand = lookVerb
        doorsCommand = doorsVerb
        helpCommand = helpVerb
        readCommand = readVerb + itemRef("subjectObj")

        invCommand.setParseAction(command.InventoryCommand)
        mapCommand.setParseAction(command.MapCommand)
        dropCommand.setParseAction(command.DropCommand)
        takeCommand.setParseAction(command.TakeCommand)
        useCommand.setParseAction(command.UseCommand)
        openCommand.setParseAction(command.OpenCommand)
        moveCommand.setParseAction(command.MoveCommand)
        quitCommand.setParseAction(command.QuitCommand)
        lookCommand.setParseAction(command.LookCommand)
        doorsCommand.setParseAction(command.DoorsCommand)
        helpCommand.setParseAction(command.HelpCommand)
        readCommand.setParseAction(command.ReadCommand)
        return (invCommand |
                  mapCommand |
                  useCommand |
                  openCommand |
                  dropCommand |
                  takeCommand |
                  moveCommand |
                  lookCommand |
                  doorsCommand |
                  helpCommand |
                  quitCommand |
                  readCommand).setResultsName("command") + LineEnd()

    def validateItemName(self, s, l, t):
        iname = " ".join(t)
        if iname not in Item.items:
            raise ShellParseException(s, l, "No such item '%s'." % iname)
        return iname

    def parseCmd(self, cmdstr):
        try:
            ret = self.bnf.parseString(cmdstr)
            return ret
        except ShellParseException, parseError:
            print parseError.msg
        except ParseException, parseError:
            print random.choice(["Sorry, I don't understand that.",
                                 "Say what?",
                                 "Whatchyoo talkin' 'bout, Willis?",
                                 "Huh?",
                                 "Garbage in, garbage out. Try again.",
                                 "What was the middle part again?",
                                 "Excuse me?",
                                 "Wtf?",
                                 "Uh... what?"])
<commit_msg>Cleaned up some code.<commit_after>import random

from pyparsing import alphas, empty, oneOf, replaceWith
from pyparsing import CaselessLiteral, OneOrMore, Optional, ParseException
from pyparsing import CaselessKeyword, LineEnd, MatchFirst, Word

from myriad.game.shell import command
from myriad.item import Item

class ShellParseException(ParseException):
    pass

class ShellParser(object):
    def __init__(self, session=None):
        self.session = session
        self.bnf = self.makeBNF()

    def makeCommandParseAction(self, cls):
        def cmdParseAction(s, l, tokens):
            return cls(tokens)
        return cmdParseAction

    def makeBNF(self):
        makeCmd = lambda s: MatchFirst(map(CaselessKeyword, s.split()))
        invVerb = makeCmd("INV INVENTORY I")
        mapVerb = makeCmd("MAP M")
        dropVerb = makeCmd("DROP LEAVE")
        takeVerb = makeCmd("TAKE PICKUP") | \
            (CaselessLiteral("PICK") + CaselessLiteral("UP"))
        moveVerb = makeCmd("MOVE GO") | empty
        useVerb = makeCmd("USE U")
        openVerb = makeCmd("OPEN O")
        quitVerb = makeCmd("QUIT Q")
        lookVerb = makeCmd("LOOK L")
        doorsVerb = CaselessKeyword("DOORS")
        helpVerb = makeCmd("H HELP ?")
        readVerb = CaselessKeyword("READ")

        itemRef = OneOrMore(Word(alphas)).setParseAction(self.validateItemName)
        makeDir = lambda s : makeCmd(s).setParseAction(
            replaceWith(s.split()[0]))
        nDir = makeDir("N NORTH")
        sDir = makeDir("S SOUTH")
        eDir = makeDir("E EAST")
        wDir = makeDir("W WEST")
        neDir = makeDir("NE NORTHEAST")
        seDir = makeDir("SE SOUTHEAST")
        swDir = makeDir("SW SOUTHWEST")
        nwDir = makeDir("NW NORTHWEST")
        moveDirection = nDir | sDir | eDir | wDir | neDir | seDir | swDir \
            | nwDir

        invCommand = invVerb
        mapCommand = mapVerb
        dropCommand = dropVerb + itemRef("item")
        takeCommand = takeVerb + itemRef("item")
        useCommand = useVerb + itemRef("usedObj") + \
            Optional(oneOf("IN ON", caseless=True)) + \
            Optional(itemRef, default=None)("targetObj")
        openCommand = openVerb + itemRef("item")
        moveCommand = moveVerb + moveDirection("direction")
        quitCommand = quitVerb
        lookCommand = lookVerb
        doorsCommand = doorsVerb
        helpCommand = helpVerb
        readCommand = readVerb + itemRef("subjectObj")

        invCommand.setParseAction(command.InventoryCommand)
        mapCommand.setParseAction(command.MapCommand)
        dropCommand.setParseAction(command.DropCommand)
        takeCommand.setParseAction(command.TakeCommand)
        useCommand.setParseAction(command.UseCommand)
        openCommand.setParseAction(command.OpenCommand)
        moveCommand.setParseAction(command.MoveCommand)
        quitCommand.setParseAction(command.QuitCommand)
        lookCommand.setParseAction(command.LookCommand)
        doorsCommand.setParseAction(command.DoorsCommand)
        helpCommand.setParseAction(command.HelpCommand)
        readCommand.setParseAction(command.ReadCommand)
        return (invCommand |
                  mapCommand |
                  useCommand |
                  openCommand |
                  dropCommand |
                  takeCommand |
                  moveCommand |
                  lookCommand |
                  doorsCommand |
                  helpCommand |
                  quitCommand |
                  readCommand).setResultsName("command") + LineEnd()

    def validateItemName(self, s, l, t):
        iname = " ".join(t)
        if iname not in Item.items:
            raise ShellParseException(s, l, "No such item '%s'." % iname)
        return iname

    def parseCmd(self, cmdstr):
        try:
            return self.bnf.parseString(cmdstr)
        except ShellParseException, parseError:
            print parseError.msg
        except ParseException, parseError:
            print random.choice(["Sorry, I don't understand that.",
                                 "Say what?",
                                 "Whatchyoo talkin' 'bout, Willis?",
                                 "Huh?",
                                 "Garbage in, garbage out. Try again.",
                                 "What was the middle part again?",
                                 "Excuse me?",
                                 "Wtf?",
                                 "Uh... what?"])
<|endoftext|>


<commit_before>from django.db import models
from django.conf import settings

class UserSetting(models.Model):
    user = models.OneToOneField(
        settings.AUTH_USER_MODEL,
        related_name="settings",
    )

    def __str__(self):
        return self.user.username

class ArbitrarySetting(models.Model):
    setting_id = models.IntegerField()
    key = models.CharField(max_length=32)
    value = models.CharField(max_length=128)

    def __str__(self):
        return "'%s': '%s' for setting ID %s" % (
            self.key,
            self.value,
            self.setting_id,
        )

<commit_msg>get and set arbitrary setting<commit_after>from django.db import models
from django.conf import settings

class UserSetting(models.Model):
    user = models.OneToOneField(
        settings.AUTH_USER_MODEL,
        related_name="settings",
    )

    def __str__(self):
        return self.user.username

    def __getattr__(self, k):
        try:
            return object.__getattribute__(self, k)
        except AttributeError:
            try:
                asObject = ArbitrarySetting.objects.get(
                    setting_id = self.id,
                    key=k,
                )
                return asObject.value
            except ArbitrarySetting.DoesNotExist:
                raise AttributeError(k)

    def __setattr__(self, k, v):
        try:
            object.__getattribute__(self, k)
        except AttributeError:
            if k.startswith("as_"):
                asObject, created = ArbitrarySetting.objects.get_or_create(
                    setting_id = self.id,
                    key = k,
                )
                asObject.value = v
                asObject.save()
            else:
                object.__setattr__(self, k, v)
        else:
            object.__setattr__(self, k, v)

class ArbitrarySetting(models.Model):
    setting_id = models.IntegerField()
    key = models.CharField(max_length=32)
    value = models.CharField(max_length=128)

    def __str__(self):
        return "'%s': '%s' for setting ID %s" % (
            self.key,
            self.value,
            self.setting_id,
        )

<|endoftext|>


<commit_before>import re
from collections import defaultdict
from optparse import make_option

from datetime import datetime
from uuid import UUID

from django.core.management import BaseCommand

from couchforms.models import XFormInstance, XFormDuplicate

class Command(BaseCommand):
    help = 'Replace xforms missing attachments with xfrom duplicates containing attachments.'
    args = '<ids_file_path> <log_path>'

    option_list = BaseCommand.option_list + (
        make_option(
            '--dry-run',
            action='store_true',
            dest='dry_run',
            default=False,
            help="Don't do the actual modifications, but still log what would be affected."
        ),
        make_option(
            '--no-input',
            action='store_true',
            dest='no_input',
            default=False,
            help='Skip important confirmation warnings.'
        ),
    )

    def handle(self, *args, **options):

        self.dups_by_domain = {}
        # e.g.
        #   {
        #       "foo-domain": {
        #           "an-original-id": {"a-duplicate-id", "another-duplicate-id"}
        #       }
        #   }

        dry_run = options.get("dry_run", True)
        if not options.get('no_input', False) and not dry_run:
            confirm = raw_input(
                u"""
                Are you sure you want to swap the given XFormInstances for duplicates?
                This is NOT a dry run. y/N?
                """
            )
            if confirm != "y":
                print "\n\t\tSwap duplicates cancelled."
                return

        ids_file_path = args[0].strip()
        log_path = args[1].strip()

        num_with_1_dup = 0
        num_with_multi_dup = 0
        num_with_no_dup = 0

        with open(log_path, "w") as log_file:
            with open(ids_file_path, "r") as lines:
                for i, (domain, bad_xform_id) in enumerate(l.split() for l in lines):
                    self._print_progress(i, 1555)
                    duplicates = self.get_duplicates(domain, bad_xform_id)
                    if len(duplicates) == 1:
                        num_with_1_dup += 1
                        self.swap_doc_types(log_file, bad_xform_id, list(duplicates)[0], dry_run)
                    elif len(duplicates) > 1:
                        num_with_multi_dup += 1
                        self.log_too_many_dups(log_file, bad_xform_id, duplicates)
                    else:
                        num_with_no_dup += 1

        print "Found {} forms with no duplicates".format(num_with_no_dup)
        print "Found {} forms with one duplicate".format(num_with_1_dup)
        print "Found {} forms with multiple duplicates".format(num_with_multi_dup)

    def get_duplicates(self, domain, xform_id):

        if domain not in self.dups_by_domain:
            self.populate_dup_map(domain)
        return self.dups_by_domain[domain].get(UUID(xform_id), {})

    def populate_dup_map(self, domain):
        self.dups_by_domain[domain] = defaultdict(set)
        dups = XFormInstance.view(
            'couchforms/all_submissions_by_domain',
            startkey=[domain, 'XFormDuplicate', '2016'],
            endkey=[domain, "XFormDuplicate", {}],
            reduce=False,
            include_docs=True
        )
        for dup in dups:
            match = re.match('Form is a duplicate of another! \((.*)\)', dup.problem or "")
            if match:
                orig_id = match.groups()[0]
                try:
                    orig_id = UUID(orig_id)
                except ValueError:
                    continue
                self.dups_by_domain[domain][orig_id].add(dup._id)

    def swap_doc_types(self, log_file, bad_xform_id, duplicate_xform_id, dry_run):
        bad_xform = XFormInstance.get(bad_xform_id)
        duplicate_xform = XFormInstance.get(duplicate_xform_id)

        # Convert the XFormInstance to an XFormDuplicate
        bad_xform.doc_type = XFormDuplicate.__name__
        bad_xform.problem = "Form was missing multimedia attachments. Replaced by {} on {}".format(
            duplicate_xform_id, datetime.now().isoformat()
        )
        bad_xform = XFormDuplicate.wrap(bad_xform.to_json())

        # Convert the XFormDuplicate to an XFormInstance
        del duplicate_xform.problem
        duplicate_xform.doc_type = XFormInstance.__name__
        duplicate_xform = XFormInstance.wrap(duplicate_xform.to_json())

        self.log_swap(log_file, bad_xform_id, duplicate_xform_id, dry_run)

        if not dry_run:
            duplicate_xform.save()
            bad_xform.save()

    @staticmethod
    def log_too_many_dups(log_file, bad_xform_id, duplicates):
        log_file.write(
            "Multiple duplicates for {}. Duplicates: {}\n".format(
                bad_xform_id,
                ", ".join(d for d in duplicates)
            )
        )

    @staticmethod
    def log_swap(log_file, bad_xform_id, duplicate_xform_id, dry_run):
        if dry_run:
            prefix = "Would have swapped"
        else:
            prefix = "Swapped"
        log_file.write(
            "{} bad xform {} for duplicate {}\n".format(prefix, bad_xform_id, duplicate_xform_id)
        )

    @staticmethod
    def _print_progress(i, total_submissions):
        if i % 20 == 0 and i != 0:
            print "Progress: {} of {} ({})  {}".format(
                i, total_submissions, round(i / float(total_submissions), 2), datetime.now()
            )
<commit_msg>Don't hard code progress<commit_after>import re
from collections import defaultdict
from optparse import make_option

from datetime import datetime
from uuid import UUID

from django.core.management import BaseCommand

from couchforms.models import XFormInstance, XFormDuplicate

class Command(BaseCommand):
    help = 'Replace xforms missing attachments with xfrom duplicates containing attachments.'
    args = '<ids_file_path> <log_path>'

    option_list = BaseCommand.option_list + (
        make_option(
            '--dry-run',
            action='store_true',
            dest='dry_run',
            default=False,
            help="Don't do the actual modifications, but still log what would be affected."
        ),
        make_option(
            '--no-input',
            action='store_true',
            dest='no_input',
            default=False,
            help='Skip important confirmation warnings.'
        ),
    )

    def handle(self, *args, **options):

        self.dups_by_domain = {}
        # e.g.
        #   {
        #       "foo-domain": {
        #           "an-original-id": {"a-duplicate-id", "another-duplicate-id"}
        #       }
        #   }

        dry_run = options.get("dry_run", True)
        if not options.get('no_input', False) and not dry_run:
            confirm = raw_input(
                u"""
                Are you sure you want to swap the given XFormInstances for duplicates?
                This is NOT a dry run. y/N?
                """
            )
            if confirm != "y":
                print "\n\t\tSwap duplicates cancelled."
                return

        ids_file_path = args[0].strip()
        log_path = args[1].strip()

        num_with_1_dup = 0
        num_with_multi_dup = 0
        num_with_no_dup = 0

        with open(log_path, "w") as log_file:
            with open(ids_file_path, "r") as lines:

                total_lines = 0
                for _ in lines:
                    total_lines += 1
                lines.seek(0)

                for i, (domain, bad_xform_id) in enumerate(l.split() for l in lines):
                    self._print_progress(i, total_lines)
                    duplicates = self.get_duplicates(domain, bad_xform_id)
                    if len(duplicates) == 1:
                        num_with_1_dup += 1
                        self.swap_doc_types(log_file, bad_xform_id, list(duplicates)[0], dry_run)
                    elif len(duplicates) > 1:
                        num_with_multi_dup += 1
                        self.log_too_many_dups(log_file, bad_xform_id, duplicates)
                    else:
                        num_with_no_dup += 1

        print "Found {} forms with no duplicates".format(num_with_no_dup)
        print "Found {} forms with one duplicate".format(num_with_1_dup)
        print "Found {} forms with multiple duplicates".format(num_with_multi_dup)

    def get_duplicates(self, domain, xform_id):

        if domain not in self.dups_by_domain:
            self.populate_dup_map(domain)
        return self.dups_by_domain[domain].get(UUID(xform_id), {})

    def populate_dup_map(self, domain):
        self.dups_by_domain[domain] = defaultdict(set)
        dups = XFormInstance.view(
            'couchforms/all_submissions_by_domain',
            startkey=[domain, 'XFormDuplicate', '2016'],
            endkey=[domain, "XFormDuplicate", {}],
            reduce=False,
            include_docs=True
        )
        for dup in dups:
            match = re.match('Form is a duplicate of another! \((.*)\)', dup.problem or "")
            if match:
                orig_id = match.groups()[0]
                try:
                    orig_id = UUID(orig_id)
                except ValueError:
                    continue
                self.dups_by_domain[domain][orig_id].add(dup._id)

    def swap_doc_types(self, log_file, bad_xform_id, duplicate_xform_id, dry_run):
        bad_xform = XFormInstance.get(bad_xform_id)
        duplicate_xform = XFormInstance.get(duplicate_xform_id)

        # Convert the XFormInstance to an XFormDuplicate
        bad_xform.doc_type = XFormDuplicate.__name__
        bad_xform.problem = "Form was missing multimedia attachments. Replaced by {} on {}".format(
            duplicate_xform_id, datetime.now().isoformat()
        )
        bad_xform = XFormDuplicate.wrap(bad_xform.to_json())

        # Convert the XFormDuplicate to an XFormInstance
        del duplicate_xform.problem
        duplicate_xform.doc_type = XFormInstance.__name__
        duplicate_xform = XFormInstance.wrap(duplicate_xform.to_json())

        self.log_swap(log_file, bad_xform_id, duplicate_xform_id, dry_run)

        if not dry_run:
            duplicate_xform.save()
            bad_xform.save()

    @staticmethod
    def log_too_many_dups(log_file, bad_xform_id, duplicates):
        log_file.write(
            "Multiple duplicates for {}. Duplicates: {}\n".format(
                bad_xform_id,
                ", ".join(d for d in duplicates)
            )
        )

    @staticmethod
    def log_swap(log_file, bad_xform_id, duplicate_xform_id, dry_run):
        if dry_run:
            prefix = "Would have swapped"
        else:
            prefix = "Swapped"
        log_file.write(
            "{} bad xform {} for duplicate {}\n".format(prefix, bad_xform_id, duplicate_xform_id)
        )

    @staticmethod
    def _print_progress(i, total_submissions):
        if i % 20 == 0 and i != 0:
            print "Progress: {} of {} ({})  {}".format(
                i, total_submissions, round(i / float(total_submissions), 2), datetime.now()
            )
<|endoftext|>


<commit_before>"""
Created on 7 Oct 2017

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)

example document:
{"location-path": "southcoastscience-dev/test/loc/1", "device-path": "southcoastscience-dev/test/device"}
"""

from collections import OrderedDict

from scs_core.data.json import PersistentJSONable

# --------------------------------------------------------------------------------------------------------------------

class Project(PersistentJSONable):
    """
    classdocs
    """

    # ----------------------------------------------------------------------------------------------------------------

    __FILENAME =                "aws_project.json"

    @classmethod
    def persistence_location(cls, host):
        return host.aws_dir(), cls.__FILENAME

    # ----------------------------------------------------------------------------------------------------------------

    CHANNELS =  ('C', 'G', 'P', 'S', 'X')

    @classmethod
    def is_valid_channel(cls, channel):
        return channel in cls.CHANNELS

    # ----------------------------------------------------------------------------------------------------------------

    @classmethod
    def construct(cls, organisation, group, location):
        path_root = organisation + '/'

        location_path = path_root + group + '/loc/' + str(location)
        device_path = path_root + group + '/device'

        return Project(location_path, device_path)

    @classmethod
    def construct_from_jdict(cls, jdict):
        if not jdict:
            return None

        location_path = jdict.get('location-path')
        device_path = jdict.get('device-path')

        return Project(location_path, device_path)

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self, location_path, device_path):
        """
        Constructor
        """
        self.__location_path = location_path          # string
        self.__device_path = device_path              # string

    # ----------------------------------------------------------------------------------------------------------------

    def as_json(self):
        jdict = OrderedDict()

        jdict['location-path'] = self.__location_path
        jdict['device-path'] = self.__device_path

        return jdict

    # ----------------------------------------------------------------------------------------------------------------

    def channel_path(self, channel, system_id):
        if channel == 'C':
            return self.subject_path('climate', system_id)

        if channel == 'G':
            return self.subject_path('gases', system_id)

        if channel == 'P':
            return self.subject_path('particulates', system_id)

        if channel == 'S':
            return self.subject_path('status', system_id)

        if channel == 'X':
            return self.subject_path('control', system_id)

        raise ValueError(channel)

    def subject_path(self, subject, system_id):
        if subject == 'climate' or subject == 'gases' or subject == 'particulates':
            return '/'.join((self.__location_path, subject))

        if subject == 'status' or subject == 'control':
            return '/'.join((self.__device_path, system_id.topic_label(), subject))

        return None

    # ----------------------------------------------------------------------------------------------------------------

    @property
    def location_path(self):
        return self.__location_path

    @property
    def device_path(self):
        return self.__device_path

    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "Project:{location_path:%s, device_path:%s}" % (self.location_path, self.device_path)
<commit_msg>Channel sophistication<commit_after>"""
Created on 7 Oct 2017

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)

example document:
{"location-path": "southcoastscience-dev/test/loc/1", "device-path": "southcoastscience-dev/test/device"}
"""

from collections import OrderedDict

from scs_core.data.json import PersistentJSONable

# --------------------------------------------------------------------------------------------------------------------

class Project(PersistentJSONable):
    """
    classdocs
    """

    # ----------------------------------------------------------------------------------------------------------------

    __FILENAME =                "aws_project.json"

    @classmethod
    def persistence_location(cls, host):
        return host.aws_dir(), cls.__FILENAME

    # ----------------------------------------------------------------------------------------------------------------

    ENVIRONMENT_CHANNELS =  ('C', 'G', 'P')
    SYSTEM_CHANNELS =  ('S', 'X')

    CHANNELS = ENVIRONMENT_CHANNELS + SYSTEM_CHANNELS

    @classmethod
    def is_valid_channel(cls, channel):
        return channel in cls.CHANNELS

    # ----------------------------------------------------------------------------------------------------------------

    @classmethod
    def construct(cls, organisation, group, location):
        path_root = organisation + '/'

        location_path = path_root + group + '/loc/' + str(location)
        device_path = path_root + group + '/device'

        return Project(location_path, device_path)

    @classmethod
    def construct_from_jdict(cls, jdict):
        if not jdict:
            return None

        location_path = jdict.get('location-path')
        device_path = jdict.get('device-path')

        return Project(location_path, device_path)

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self, location_path, device_path):
        """
        Constructor
        """
        self.__location_path = location_path          # string
        self.__device_path = device_path              # string

    # ----------------------------------------------------------------------------------------------------------------

    def as_json(self):
        jdict = OrderedDict()

        jdict['location-path'] = self.__location_path
        jdict['device-path'] = self.__device_path

        return jdict

    # ----------------------------------------------------------------------------------------------------------------

    def environment_paths(self, system_id):
        return [self.channel_path(channel, system_id) for channel in self.ENVIRONMENT_CHANNELS]

    def channel_path(self, channel, system_id):
        if channel == 'C':
            return self.subject_path('climate', system_id)

        if channel == 'G':
            return self.subject_path('gases', system_id)

        if channel == 'P':
            return self.subject_path('particulates', system_id)

        if channel == 'S':
            return self.subject_path('status', system_id)

        if channel == 'X':
            return self.subject_path('control', system_id)

        raise ValueError(channel)

    def subject_path(self, subject, system_id):
        if subject == 'climate' or subject == 'gases' or subject == 'particulates':
            return '/'.join((self.__location_path, subject))

        if subject == 'status' or subject == 'control':
            return '/'.join((self.__device_path, system_id.topic_label(), subject))

        return None

    # ----------------------------------------------------------------------------------------------------------------

    @property
    def location_path(self):
        return self.__location_path

    @property
    def device_path(self):
        return self.__device_path

    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "Project:{location_path:%s, device_path:%s}" % (self.location_path, self.device_path)
<|endoftext|>


<commit_before># ==============================
# Copyright 2011 Whamcloud, Inc.
# ==============================

from hydra_agent.plugins import DevicePlugin

from hydra_agent.log import agent_log
from hydra_agent import shell
from hydra_agent.plugins import ActionPlugin

#from hydra_agent.plugins import DevicePluginManager

import os
import glob
import re

class LinuxDevicePlugin(DevicePlugin):
    def start_session(self):
        return device_scan()

    def update_session(self):
        pass

def _dev_major_minor(path):
    """Return a string if 'path' is a block device or link to one, else return None"""
    from stat import S_ISBLK
    try:
        s = os.stat(path)
    except OSError:
        return None

    if S_ISBLK(s.st_mode):
        return "%d:%d" % (os.major(s.st_rdev), os.minor(s.st_rdev))
    else:
        return None

def _find_block_devs(folder):
    # Map of major_minor to path
    result = {}
    for path in glob.glob(os.path.join(folder, "*")):
        mm = _dev_major_minor(path)
        if mm:
            result[mm] = path

    return result

def _get_vgs():
    out = shell.try_run(["vgs", "--units", "b", "--noheadings", "-o", "vg_name,vg_uuid,vg_size"])

    lines = [l for l in out.split("\n") if len(l) > 0]
    for line in lines:
        name, uuid, size_str = line.split()
        size = int(size_str[0:-1], 10)
        yield (name, uuid, size)

def _get_lvs(vg_name):
    out = shell.try_run(["lvs", "--units", "b", "--noheadings", "-o", "lv_name,lv_uuid,lv_size,lv_path", vg_name])

    lines = [l for l in out.split("\n") if len(l) > 0]
    for line in lines:
        name, uuid, size_str, path = line.split()
        size = int(size_str[0:-1], 10)
        yield (name, uuid, size, path)

old_udev = None

def _device_node(device_name, major_minor, path, size, parent):
    # This is a WTF, when I declare old_udev at module scope this function can't see it?!?!
    global old_udev
    # Old (RHEL5) version of scsi_id has a different command line
    # syntax to new (RHEL6) version.  Tell them apart with --version
    # (old one doesn't have --version, new one does)
    SCSI_ID_PATH = "/sbin/scsi_id"
    if old_udev == None:
        rc, out, err = shell.run([SCSI_ID_PATH, '--version'])
        old_udev = (rc != 0)

    # Note: we use -p 0x80 with scsi_id in order to get
    # a textual id along the lines of:
    # * SQEMU    QEMU HARDDISK  WD-deadbeef0
    # * SOPNFILERVIRTUAL-DISK   9iI2mH-4Ddf-k3Ov
    # * SDDN     S2A 9550       058C4A531100
    # As well as being more readable than the 0x83 ID, this
    # gets us the serial number for QEMU devices if the user
    # has set one, whereas that doesn't show up in 0x83.

    if old_udev:
        # Old scsi_id, operates on a /sys reference
        rc, out, err = shell.run(["scsi_id", "-g", "-p", "0x80", "-s", "/block/%s" % device_name])
        if rc != 0:
            serial = None
        else:
            serial = out.strip()
    else:
        # New scsi_id, always operates directly on a device
        rc, out, err = shell.run(["scsi_id", "-g", "-p", "0x80", path])
        if rc != 0:
            serial = None
        else:
            serial = out.strip()

    # The downside to using -p 0x80 is that if the user hasn't manually
    # set serials for their scsi devices, multiple different devices on
    # the same host return the same string, so we need an explicit
    # exclusion for that
    if serial == "SQEMU    QEMU HARDDISK  0":
        serial = None

    # FIXME: different controllers may want to use different identifiers,
    # should separate these out so that can pick from e.g. 0x80 vs. 0x83

    info = {'major_minor': major_minor,
            'path': path,
            'serial': serial,
            'size': size,
            'parent': parent}

    #info = DevicePluginManager.annotate_block_device(info)

    return info

def _parse_sys_block():
    mapper_devs = _find_block_devs("/dev/mapper/")
    #by_id_devs = _find_block_devs("/dev/disk/by-path/")
    by_id_devs = []

    def get_path(major_minor, device_name):
        # Try to find device nodes for these:
        fallback_dev_path = os.path.join("/dev/", device_name)
        # * First look in /dev/mapper
        if major_minor in mapper_devs:
            return mapper_devs[major_minor]
        # * Then try /dev/disk/by-id
        elif major_minor in by_id_devs:
            return by_id_devs[major_minor]
        # * Then fall back to just /dev
        elif os.path.exists(fallback_dev_path):
            return fallback_dev_path
        else:
            agent_log.warning("Could not find device node for %s (%s)" % (major_minor, fallback_dev_path))
            return None

    block_device_nodes = {}
    node_block_devices = {}

    def parse_block_dir(dev_dir, parent = None):
        """Parse a dir like /sys/block/sda (must contain 'dev' and 'size')"""
        device_name = dev_dir.split(os.sep)[-1]
        major_minor = open(os.path.join(dev_dir, "dev")).read().strip()
        size = int(open(os.path.join(dev_dir, "size")).read().strip()) * 512

        # Exclude zero-sized devices
        if size == 0:
            return

        # Exclude ramdisks, floppy drives, obvious cdroms
        if re.search("^ram\d+$", device_name) or \
           re.search("^fd\d+$", device_name) or \
           re.search("^sr\d+$", device_name):
            return

        # Exclude read-only or removed devices
        try:
            open("/dev/%s" % device_name, 'w')
        except IOError, e:
            import errno
            # Python errno doesn't include this code
            NO_MEDIA_ERRNO = 123
            if e.errno == errno.EROFS or e.errno == NO_MEDIA_ERRNO:
                return

        # Resolve a major:minor to a /dev/foo
        path = get_path(major_minor, device_name)
        if path:
            block_device_nodes[major_minor] = _device_node(device_name, major_minor, path, size, parent)
            node_block_devices[path] = major_minor

        return major_minor

    for dev_dir in glob.glob("/sys/block/*"):
        major_minor = parse_block_dir(dev_dir)

        partitions = glob.glob(os.path.join(dev_dir, "*/dev"))
        for p in partitions:
            parse_block_dir(os.path.split(p)[0], parent = major_minor)

    return block_device_nodes, node_block_devices

def device_scan(args = None):
    # Map of block devices major:minors to /dev/ path.
    block_device_nodes, node_block_devices = _parse_sys_block()

    vgs = {}
    lvs = {}
    for vg_name, vg_uuid, vg_size in _get_vgs():
        vgs[vg_name] = {
                'name': vg_name,
                'uuid': vg_uuid,
                'size': vg_size,
                'pvs_major_minor': []}
        lvs[vg_name] = {}
        for lv_name, lv_uuid, lv_size, lv_path in _get_lvs(vg_name):
            lvs[vg_name][lv_name] = {
                'name': lv_name,
                'uuid': lv_uuid,
                'size': lv_size}

    # Map multipath name to set of devices
    mpaths = {}

    stdout = shell.try_run(['dmsetup', 'table'])
    if stdout.strip() == "No devices found":
        dm_lines = []
    else:
        dm_lines = [i for i in stdout.split("\n") if len(i) > 0]
    for line in dm_lines:
        tokens = line.split()
        name = tokens[0].strip(":")
        dm_type = tokens[3]

        node_path = os.path.join("/dev/mapper", name)
        block_device = node_block_devices[node_path]

        if dm_type in ['linear', 'striped']:
            # This is an LVM LV
            if dm_type == 'striped':
                # List of striped devices
                dev_indices = range(6, len(tokens), 2)
                devices = [tokens[i] for i in dev_indices]
            elif dm_type == 'linear':
                # Single device linear range
                devices = [tokens[4]]

            # When a name has a "-" in it, DM prints a double hyphen in the output
            # So for an LV called "my-lv" you get VolGroup00-my--lv
            vg_name, lv_name = re.search("(.*[^-])-([^-].*)", name).groups()
            vg_name = vg_name.replace("--", "-")
            lv_name = lv_name.replace("--", "-")

            # Try to get information about LVs from this VG which we queried
            # earlier with _get_vg/_get_lv.  This can fail if the system is in an in-between
            # state where it still has device nodes for some LVs which are
            # no longer really there
            try:
                vg_lv_info = lvs[vg_name]
            except KeyError:
                continue

            if not lv_name in vg_lv_info:
                # This isn't something we saw as a named LV, so its
                # a partition.  Assign its parent and don't store it
                # as an LV.
                result = re.search("(.*)p\d+", lv_name)
                if not result:
                    agent_log.error("Cannot parse LVM device name %s" % name)
                    continue
                parent_lv_name = result.groups()[0]
                if not parent_lv_name in lvs[vg_name]:
                    agent_log.error("Cannot parse LVM device name %s" % name)
                else:
                    # FIXME: compose path in a way that copes with hyphens
                    parent_block_device = node_block_devices["/dev/mapper/%s-%s" % (vg_name, parent_lv_name)]
                    block_device_nodes[block_device]['parent'] = parent_block_device

                continue
            lvs[vg_name][lv_name]['block_device'] = block_device

            devices = [block_device_nodes[i]['major_minor'] for i in devices]
            vgs[vg_name]['pvs_major_minor'] = list(set(vgs[vg_name]['pvs_major_minor']) | set(devices))
        elif dm_type == 'multipath':
            # This is a multipath device, there will be a list of devices like this:
            # "round-robin 0 1 1 8:80 1000"
            dev_indices = range(12, len(tokens), 6)
            devices = [tokens[i] for i in dev_indices]
            devices = [block_device_nodes[i] for i in devices]
            if name in mpaths:
                raise RuntimeError("Duplicated mpath device %s" % name)

            mpaths[name] = {
                    "name": name,
                    "block_device": block_device,
                    "nodes": devices
                    }
        else:
            continue

    # Anything in fstab or that is mounted
    from hydra_agent.utils import Fstab, Mounts
    fstab = Fstab()
    mounts = Mounts()
    from itertools import chain
    bdev_to_local_fs = {}
    for dev, mntpnt, fstype in chain(fstab.all(), mounts.all()):
        mm = _dev_major_minor(dev)
        if mm and fstype != 'lustre' and mm in block_device_nodes:
            bdev_to_local_fs[mm] = (mntpnt, fstype)

    return {"vgs": vgs, "lvs": lvs, "mpath": mpaths, "devs": block_device_nodes, "local_fs": bdev_to_local_fs}

class DeviceScanPlugin(ActionPlugin):
    def register_commands(self, parser):
        p = parser.add_parser("device-scan",
                              help="scan for devices, or something")
        p.set_defaults(func=device_scan)
<commit_msg>Detect and report mdraid information<commit_after># ==============================
# Copyright 2011 Whamcloud, Inc.
# ==============================

from hydra_agent.plugins import DevicePlugin

from hydra_agent.log import agent_log
from hydra_agent import shell
from hydra_agent.plugins import ActionPlugin

#from hydra_agent.plugins import DevicePluginManager

import os
import glob
import re

class LinuxDevicePlugin(DevicePlugin):
    def start_session(self):
        return device_scan()

    def update_session(self):
        pass

def _dev_major_minor(path):
    """Return a string if 'path' is a block device or link to one, else return None"""
    from stat import S_ISBLK
    try:
        s = os.stat(path)
    except OSError:
        return None

    if S_ISBLK(s.st_mode):
        return "%d:%d" % (os.major(s.st_rdev), os.minor(s.st_rdev))
    else:
        return None

def _find_block_devs(folder):
    # Map of major_minor to path
    result = {}
    for path in glob.glob(os.path.join(folder, "*")):
        mm = _dev_major_minor(path)
        if mm:
            result[mm] = path

    return result

def _get_vgs():
    out = shell.try_run(["vgs", "--units", "b", "--noheadings", "-o", "vg_name,vg_uuid,vg_size"])

    lines = [l for l in out.split("\n") if len(l) > 0]
    for line in lines:
        name, uuid, size_str = line.split()
        size = int(size_str[0:-1], 10)
        yield (name, uuid, size)

def _get_lvs(vg_name):
    out = shell.try_run(["lvs", "--units", "b", "--noheadings", "-o", "lv_name,lv_uuid,lv_size,lv_path", vg_name])

    lines = [l for l in out.split("\n") if len(l) > 0]
    for line in lines:
        name, uuid, size_str, path = line.split()
        size = int(size_str[0:-1], 10)
        yield (name, uuid, size, path)

old_udev = None

def _device_node(device_name, major_minor, path, size, parent):
    # This is a WTF, when I declare old_udev at module scope this function can't see it?!?!
    global old_udev
    # Old (RHEL5) version of scsi_id has a different command line
    # syntax to new (RHEL6) version.  Tell them apart with --version
    # (old one doesn't have --version, new one does)
    SCSI_ID_PATH = "/sbin/scsi_id"
    if old_udev == None:
        rc, out, err = shell.run([SCSI_ID_PATH, '--version'])
        old_udev = (rc != 0)

    # Note: we use -p 0x80 with scsi_id in order to get
    # a textual id along the lines of:
    # * SQEMU    QEMU HARDDISK  WD-deadbeef0
    # * SOPNFILERVIRTUAL-DISK   9iI2mH-4Ddf-k3Ov
    # * SDDN     S2A 9550       058C4A531100
    # As well as being more readable than the 0x83 ID, this
    # gets us the serial number for QEMU devices if the user
    # has set one, whereas that doesn't show up in 0x83.

    def scsi_id_command(cmd):
        rc, out, err = shell.run(cmd)
        if rc != 0:
            return None
        else:
            return out.strip()

    if old_udev:
        # Old scsi_id, operates on a /sys reference
        serial_80 = scsi_id_command(["scsi_id", "-g", "-p", "0x80", "-s", "/block/%s" % device_name])
        serial_83 = scsi_id_command(["scsi_id", "-g", "-p", "0x83", "-s", "/block/%s" % device_name])
    else:
        # New scsi_id, always operates directly on a device
        serial_80 = scsi_id_command(["scsi_id", "-g", "-p", "0x80", path])
        serial_83 = scsi_id_command(["scsi_id", "-g", "-p", "0x83", path])

    # The downside to using -p 0x80 is that if the user hasn't manually
    # set serials for their scsi devices, multiple different devices on
    # the same host return the same string, so we need an explicit
    # exclusion for that
    if serial_80 == "SQEMU    QEMU HARDDISK  0":
        serial_80 = None

    info = {'major_minor': major_minor,
            'path': path,
            'serial_80': serial_80,
            'serial_83': serial_83,
            'size': size,
            'parent': parent}

    return info

def _parse_sys_block():
    mapper_devs = _find_block_devs("/dev/mapper/")
    #by_id_devs = _find_block_devs("/dev/disk/by-path/")
    by_id_devs = []

    def get_path(major_minor, device_name):
        # Try to find device nodes for these:
        fallback_dev_path = os.path.join("/dev/", device_name)
        # * First look in /dev/mapper
        if major_minor in mapper_devs:
            return mapper_devs[major_minor]
        # * Then try /dev/disk/by-id
        elif major_minor in by_id_devs:
            return by_id_devs[major_minor]
        # * Then fall back to just /dev
        elif os.path.exists(fallback_dev_path):
            return fallback_dev_path
        else:
            agent_log.warning("Could not find device node for %s (%s)" % (major_minor, fallback_dev_path))
            return None

    block_device_nodes = {}
    node_block_devices = {}

    def parse_block_dir(dev_dir, parent = None):
        """Parse a dir like /sys/block/sda (must contain 'dev' and 'size')"""
        device_name = dev_dir.split(os.sep)[-1]
        major_minor = open(os.path.join(dev_dir, "dev")).read().strip()
        size = int(open(os.path.join(dev_dir, "size")).read().strip()) * 512

        # Exclude zero-sized devices
        if size == 0:
            return

        # Exclude ramdisks, floppy drives, obvious cdroms
        if re.search("^ram\d+$", device_name) or \
           re.search("^fd\d+$", device_name) or \
           re.search("^sr\d+$", device_name):
            return

        # Exclude read-only or removed devices
        try:
            open("/dev/%s" % device_name, 'w')
        except IOError, e:
            import errno
            # Python errno doesn't include this code
            NO_MEDIA_ERRNO = 123
            if e.errno == errno.EROFS or e.errno == NO_MEDIA_ERRNO:
                return

        # Resolve a major:minor to a /dev/foo
        path = get_path(major_minor, device_name)
        if path:
            block_device_nodes[major_minor] = _device_node(device_name, major_minor, path, size, parent)
            node_block_devices[path] = major_minor

        return major_minor

    for dev_dir in glob.glob("/sys/block/*"):
        major_minor = parse_block_dir(dev_dir)

        partitions = glob.glob(os.path.join(dev_dir, "*/dev"))
        for p in partitions:
            parse_block_dir(os.path.split(p)[0], parent = major_minor)

    return block_device_nodes, node_block_devices

def _get_md():
    try:
        matches = re.finditer("^(md\d+) :", open('/proc/mdstat', 'r').read().strip(), flags = re.MULTILINE)
        devs = []
        for match in matches:
            # e.g. md0
            device_name = match.group(1)
            device_path = "/dev/%s" % device_name
            detail = shell.try_run(['mdadm', '--brief', '--detail', '--verbose', device_path])
            device_uuid = re.search("UUID=(.*)[ \\n]", detail.strip(), flags = re.MULTILINE).group(1)
            print detail.strip()
            device_list_csv = re.search("^   devices=(.*)$", detail.strip(), flags = re.MULTILINE).group(1)
            device_list = device_list_csv.split(",")

            devs.append({
                "uuid": device_uuid,
                "path": device_path,
                "device_paths": device_list
                })
        return devs
    except IOError:
        return []

def device_scan(args = None):
    # Map of block devices major:minors to /dev/ path.
    block_device_nodes, node_block_devices = _parse_sys_block()

    vgs = {}
    lvs = {}
    for vg_name, vg_uuid, vg_size in _get_vgs():
        vgs[vg_name] = {
                'name': vg_name,
                'uuid': vg_uuid,
                'size': vg_size,
                'pvs_major_minor': []}
        lvs[vg_name] = {}
        for lv_name, lv_uuid, lv_size, lv_path in _get_lvs(vg_name):
            lvs[vg_name][lv_name] = {
                'name': lv_name,
                'uuid': lv_uuid,
                'size': lv_size}

    # Map multipath name to set of devices
    mpaths = {}

    stdout = shell.try_run(['dmsetup', 'table'])
    if stdout.strip() == "No devices found":
        dm_lines = []
    else:
        dm_lines = [i for i in stdout.split("\n") if len(i) > 0]
    for line in dm_lines:
        tokens = line.split()
        name = tokens[0].strip(":")
        dm_type = tokens[3]

        node_path = os.path.join("/dev/mapper", name)
        block_device = node_block_devices[node_path]

        if dm_type in ['linear', 'striped']:
            # This is an LVM LV
            if dm_type == 'striped':
                # List of striped devices
                dev_indices = range(6, len(tokens), 2)
                devices = [tokens[i] for i in dev_indices]
            elif dm_type == 'linear':
                # Single device linear range
                devices = [tokens[4]]

            # When a name has a "-" in it, DM prints a double hyphen in the output
            # So for an LV called "my-lv" you get VolGroup00-my--lv
            vg_name, lv_name = re.search("(.*[^-])-([^-].*)", name).groups()
            vg_name = vg_name.replace("--", "-")
            lv_name = lv_name.replace("--", "-")

            # Try to get information about LVs from this VG which we queried
            # earlier with _get_vg/_get_lv.  This can fail if the system is in an in-between
            # state where it still has device nodes for some LVs which are
            # no longer really there
            try:
                vg_lv_info = lvs[vg_name]
            except KeyError:
                continue

            if not lv_name in vg_lv_info:
                # This isn't something we saw as a named LV, so its
                # a partition.  Assign its parent and don't store it
                # as an LV.
                result = re.search("(.*)p\d+", lv_name)
                if not result:
                    agent_log.error("Cannot parse LVM device name %s" % name)
                    continue
                parent_lv_name = result.groups()[0]
                if not parent_lv_name in lvs[vg_name]:
                    agent_log.error("Cannot parse LVM device name %s" % name)
                else:
                    # HYD-744: FIXME: compose path in a way that copes with hyphens
                    parent_block_device = node_block_devices["/dev/mapper/%s-%s" % (vg_name, parent_lv_name)]
                    block_device_nodes[block_device]['parent'] = parent_block_device

                continue
            lvs[vg_name][lv_name]['block_device'] = block_device

            devices = [block_device_nodes[i]['major_minor'] for i in devices]
            vgs[vg_name]['pvs_major_minor'] = list(set(vgs[vg_name]['pvs_major_minor']) | set(devices))
        elif dm_type == 'multipath':
            # This is a multipath device, there will be a list of devices like this:
            # "round-robin 0 1 1 8:80 1000"
            dev_indices = range(12, len(tokens), 6)
            devices = [tokens[i] for i in dev_indices]
            devices = [block_device_nodes[i] for i in devices]
            if name in mpaths:
                raise RuntimeError("Duplicated mpath device %s" % name)

            mpaths[name] = {
                    "name": name,
                    "block_device": block_device,
                    "nodes": devices
                    }
        else:
            continue

    mds = {}
    for md in _get_md():
        path = md['path']
        uuid = md['uuid']
        device_paths = md['device_paths']
        drives = [_dev_major_minor(dp) for dp in device_paths]
        mds[uuid] = {'path': path, 'drives': drives}

    # Anything in fstab or that is mounted
    from hydra_agent.utils import Fstab, Mounts
    fstab = Fstab()
    mounts = Mounts()
    from itertools import chain
    bdev_to_local_fs = {}
    for dev, mntpnt, fstype in chain(fstab.all(), mounts.all()):
        mm = _dev_major_minor(dev)
        if mm and fstype != 'lustre' and mm in block_device_nodes:
            bdev_to_local_fs[mm] = (mntpnt, fstype)

    return {"vgs": vgs,
            "lvs": lvs,
            "mpath": mpaths,
            "devs": block_device_nodes,
            "local_fs": bdev_to_local_fs,
            'mds': mds}

class DeviceScanPlugin(ActionPlugin):
    def register_commands(self, parser):
        p = parser.add_parser("device-scan",
                              help="scan for devices, or something")
        p.set_defaults(func=device_scan)
<|endoftext|>