<commit_before># (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

#############################################

import fnmatch
import os
import re

import subprocess
import ansible.constants as C
from ansible.inventory.ini import InventoryParser
from ansible.inventory.script import InventoryScript
from ansible.inventory.dir import InventoryDirectory
from ansible.inventory.group import Group
from ansible.inventory.host import Host
from ansible import errors
from ansible import utils

class Inventory(object):
    """
    Host inventory for ansible.
    """

    __slots__ = [ 'host_list', 'groups', '_restriction', '_also_restriction', '_subset',
                  'parser', '_vars_per_host', '_vars_per_group', '_hosts_cache', '_groups_list',
                  '_vars_plugins', '_playbook_basedir']

    def __init__(self, host_list=C.DEFAULT_HOST_LIST):

        # the host file file, or script path, or list of hosts
        # if a list, inventory data will NOT be loaded
        self.host_list = host_list

        # caching to avoid repeated calculations, particularly with
        # external inventory scripts.

        self._vars_per_host  = {}
        self._vars_per_group = {}
        self._hosts_cache    = {}
        self._groups_list    = {}

        # to be set by calling set_playbook_basedir by ansible-playbook
        self._playbook_basedir = None

        # the inventory object holds a list of groups
        self.groups = []

        # a list of host(names) to contain current inquiries to
        self._restriction = None
        self._also_restriction = None
        self._subset = None

        if isinstance(host_list, basestring):
            if "," in host_list:
                host_list = host_list.split(",")
                host_list = [ h for h in host_list if h and h.strip() ]

        if isinstance(host_list, list):
            self.parser = None
            all = Group('all')
            self.groups = [ all ]
            for x in host_list:
                if ":" in x:
                    tokens = x.split(":", 1)
                    all.add_host(Host(tokens[0], tokens[1]))
                else:
                    all.add_host(Host(x))
        elif os.path.exists(host_list):
            if os.path.isdir(host_list):
                # Ensure basedir is inside the directory
                self.host_list = os.path.join(self.host_list, "")
                self.parser = InventoryDirectory(filename=host_list)
                self.groups = self.parser.groups.values()
            elif utils.is_executable(host_list):
                self.parser = InventoryScript(filename=host_list)
                self.groups = self.parser.groups.values()
            else:
                self.parser = InventoryParser(filename=host_list)
                self.groups = self.parser.groups.values()

            utils.plugins.vars_loader.add_directory(self.basedir(), with_subdir=True)
        else:
            raise errors.AnsibleError("Unable to find an inventory file, specify one with -i ?")

        self._vars_plugins = [ x for x in utils.plugins.vars_loader.all(self) ]

    def _match(self, str, pattern_str):
        if pattern_str.startswith('~'):
            return re.search(pattern_str[1:], str)
        else:
            return fnmatch.fnmatch(str, pattern_str)

    def get_hosts(self, pattern="all"):
        """
        find all host names matching a pattern string, taking into account any inventory restrictions or
        applied subsets.
        """

        # process patterns
        if isinstance(pattern, list):
            pattern = ';'.join(pattern)
        patterns = pattern.replace(";",":").split(":")
        hosts = self._get_hosts(patterns)

        # exclude hosts not in a subset, if defined
        if self._subset:
            subset = self._get_hosts(self._subset)
            hosts.intersection_update(subset)

        # exclude hosts mentioned in any restriction (ex: failed hosts)
        if self._restriction is not None:
            hosts = [ h for h in hosts if h.name in self._restriction ]
        if self._also_restriction is not None:
            hosts = [ h for h in hosts if h.name in self._also_restriction ]

        return sorted(hosts, key=lambda x: x.name)

    def _get_hosts(self, patterns):
        """
        finds hosts that match a list of patterns. Handles negative
        matches as well as intersection matches.
        """

        # Host specifiers should be sorted to ensure consistent behavior
        pattern_regular = []
        pattern_intersection = []
        pattern_exclude = []
        for p in patterns:
            if p.startswith("!"):
                pattern_exclude.append(p)
            elif p.startswith("&"):
                pattern_intersection.append(p)
            else:
                pattern_regular.append(p)

        # if no regular pattern was given, hence only exclude and/or intersection
        # make that magically work
        if pattern_regular == []:
            pattern_regular = ['all']

        # when applying the host selectors, run those without the "&" or "!"
        # first, then the &s, then the !s.
        patterns = pattern_regular + pattern_intersection + pattern_exclude

        hosts = set()
        for p in patterns:
            if p.startswith("!"):
                # Discard excluded hosts
                hosts.difference_update(self.__get_hosts(p))
            elif p.startswith("&"):
                # Only leave the intersected hosts
                hosts.intersection_update(self.__get_hosts(p))
            else:
                # Get all hosts from both patterns
                hosts.update(self.__get_hosts(p))
        return hosts

    def __get_hosts(self, pattern):
        """
        finds hosts that postively match a particular pattern.  Does not
        take into account negative matches.
        """

        (name, enumeration_details) = self._enumeration_info(pattern)
        hpat = self._hosts_in_unenumerated_pattern(name)
        hpat = sorted(hpat, key=lambda x: x.name)

        return set(self._apply_ranges(pattern, hpat))

    def _enumeration_info(self, pattern):
        """
        returns (pattern, limits) taking a regular pattern and finding out
        which parts of it correspond to start/stop offsets.  limits is
        a tuple of (start, stop) or None
        """

        if not "[" in pattern or pattern.startswith('~'):
            return (pattern, None)
        (first, rest) = pattern.split("[")
        rest = rest.replace("]","")
        if "-" in rest:
            (left, right) = rest.split("-",1)
            return (first, (left, right))
        else:
            return (first, (rest, rest))

    def _apply_ranges(self, pat, hosts):
        """
        given a pattern like foo, that matches hosts, return all of hosts
        given a pattern like foo[0:5], where foo matches hosts, return the first 6 hosts
        """

        (loose_pattern, limits) = self._enumeration_info(pat)
        if not limits:
            return hosts

        (left, right) = limits
        enumerated = enumerate(hosts)
        if left == '':
            left = 0
        if right == '':
            right = 0
        left=int(left)
        right=int(right)
        enumerated = [ h for (i,h) in enumerated if i>=left and i<=right ]
        return enumerated

    # TODO: cache this logic so if called a second time the result is not recalculated
    def _hosts_in_unenumerated_pattern(self, pattern):
        """ Get all host names matching the pattern """

        hosts = {}
        # ignore any negative checks here, this is handled elsewhere
        pattern = pattern.replace("!","").replace("&", "")

        groups = self.get_groups()
        for group in groups:
            for host in group.get_hosts():
                if pattern == 'all' or self._match(group.name, pattern) or self._match(host.name, pattern):
                    hosts[host.name] = host
        return sorted(hosts.values(), key=lambda x: x.name)

    def groups_for_host(self, host):
        results = []
        groups = self.get_groups()
        for group in groups:
            for hostn in group.get_hosts():
                if host == hostn.name:
                    results.append(group)
                    continue
        return results

    def groups_list(self):
        if not self._groups_list:
            groups = {}
            for g in self.groups:
                groups[g.name] = [h.name for h in g.get_hosts()]
                ancestors = g.get_ancestors()
                for a in ancestors:
                    if a.name not in groups:
                        groups[a.name] = [h.name for h in a.get_hosts()]
            self._groups_list = groups
        return self._groups_list

    def get_groups(self):
        return self.groups

    def get_host(self, hostname):
        if hostname not in self._hosts_cache:
            self._hosts_cache[hostname] = self._get_host(hostname)
        return self._hosts_cache[hostname]

    def _get_host(self, hostname):
        if hostname in ['localhost','127.0.0.1']:
            for host in self.get_group('all').get_hosts():
                if host.name in ['localhost', '127.0.0.1']:
                    return host
        else:
            for group in self.groups:
                for host in group.get_hosts():
                    if hostname == host.name:
                        return host
        return None

    def get_group(self, groupname):
        for group in self.groups:
            if group.name == groupname:
                return group
        return None

    def get_group_variables(self, groupname):
        if groupname not in self._vars_per_group:
            self._vars_per_group[groupname] = self._get_group_variables(groupname)
        return self._vars_per_group[groupname]

    def _get_group_variables(self, groupname):
        group = self.get_group(groupname)
        if group is None:
            raise Exception("group not found: %s" % groupname)
        return group.get_variables()

    def get_variables(self, hostname):
        if hostname not in self._vars_per_host:
            self._vars_per_host[hostname] = self._get_variables(hostname)
        return self._vars_per_host[hostname]

    def _get_variables(self, hostname):

        host = self.get_host(hostname)
        if host is None:
            raise errors.AnsibleError("host not found: %s" % hostname)

        vars = {}
        vars_results = [ plugin.run(host) for plugin in self._vars_plugins ]
        for updated in vars_results:
            if updated is not None:
                vars.update(updated)

        vars.update(host.get_variables())
        if self.parser is not None:
            vars.update(self.parser.get_host_variables(host))
        return vars

    def add_group(self, group):
        self.groups.append(group)
        self._groups_list = None  # invalidate internal cache

    def list_hosts(self, pattern="all"):
        return [ h.name for h in self.get_hosts(pattern) ]

    def list_groups(self):
        return sorted([ g.name for g in self.groups ], key=lambda x: x)

    # TODO: remove this function
    def get_restriction(self):
        return self._restriction

    def restrict_to(self, restriction):
        """
        Restrict list operations to the hosts given in restriction.  This is used
        to exclude failed hosts in main playbook code, don't use this for other
        reasons.
        """
        if not isinstance(restriction, list):
            restriction = [ restriction ]
        self._restriction = restriction

    def also_restrict_to(self, restriction):
        """
        Works like restict_to but offers an additional restriction.  Playbooks use this
        to implement serial behavior.
        """
        if not isinstance(restriction, list):
            restriction = [ restriction ]
        self._also_restriction = restriction

    def subset(self, subset_pattern):
        """
        Limits inventory results to a subset of inventory that matches a given
        pattern, such as to select a given geographic of numeric slice amongst
        a previous 'hosts' selection that only select roles, or vice versa.
        Corresponds to --limit parameter to ansible-playbook
        """
        if subset_pattern is None:
            self._subset = None
        else:
            subset_pattern = subset_pattern.replace(',',':')
            subset_pattern = subset_pattern.replace(";",":").split(":")
            results = []
            # allow Unix style @filename data
            for x in subset_pattern:
                if x.startswith("@"):
                    fd = open(x[1:])
                    results.extend(fd.read().split("\n"))
                    fd.close()
                else:
                    results.append(x)
            self._subset = results

    def lift_restriction(self):
        """ Do not restrict list operations """
        self._restriction = None

    def lift_also_restriction(self):
        """ Clears the also restriction """
        self._also_restriction = None

    def is_file(self):
        """ did inventory come from a file? """
        if not isinstance(self.host_list, basestring):
            return False
        return os.path.exists(self.host_list)

    def basedir(self):
        """ if inventory came from a file, what's the directory? """
        if not self.is_file():
            return None
        dname = os.path.dirname(self.host_list)
        if dname is None or dname == '' or dname == '.':
            cwd = os.getcwd()
            return cwd
        return dname

    def src(self):
        """ if inventory came from a file, what's the directory and file name? """
        if not self.is_file():
            return None
        return self.host_list

    def playbook_basedir(self):
        """ returns the directory of the current playbook """
        return self._playbook_basedir

    def set_playbook_basedir(self, dir):
        """
        sets the base directory of the playbook so inventory plugins can use it to find
        variable files and other things.
        """
        self._playbook_basedir = dir

<commit_msg>add support for using a ipv6 in -i<commit_after># (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

#############################################

import fnmatch
import os
import re

import subprocess
import ansible.constants as C
from ansible.inventory.ini import InventoryParser
from ansible.inventory.script import InventoryScript
from ansible.inventory.dir import InventoryDirectory
from ansible.inventory.group import Group
from ansible.inventory.host import Host
from ansible import errors
from ansible import utils

class Inventory(object):
    """
    Host inventory for ansible.
    """

    __slots__ = [ 'host_list', 'groups', '_restriction', '_also_restriction', '_subset',
                  'parser', '_vars_per_host', '_vars_per_group', '_hosts_cache', '_groups_list',
                  '_vars_plugins', '_playbook_basedir']

    def __init__(self, host_list=C.DEFAULT_HOST_LIST):

        # the host file file, or script path, or list of hosts
        # if a list, inventory data will NOT be loaded
        self.host_list = host_list

        # caching to avoid repeated calculations, particularly with
        # external inventory scripts.

        self._vars_per_host  = {}
        self._vars_per_group = {}
        self._hosts_cache    = {}
        self._groups_list    = {}

        # to be set by calling set_playbook_basedir by ansible-playbook
        self._playbook_basedir = None

        # the inventory object holds a list of groups
        self.groups = []

        # a list of host(names) to contain current inquiries to
        self._restriction = None
        self._also_restriction = None
        self._subset = None

        if isinstance(host_list, basestring):
            if "," in host_list:
                host_list = host_list.split(",")
                host_list = [ h for h in host_list if h and h.strip() ]

        if isinstance(host_list, list):
            self.parser = None
            all = Group('all')
            self.groups = [ all ]
            ipv6_re = re.compile('\[([a-f:A-F0-9]*)\](?::(\d+))?')
            for x in host_list:
                m = ipv6_re.match(x)
                if m:
                    all.add_host(Host(m.groups()[0], m.groups()[1]))
                else:
                    if ":" in x:
                        tokens = x.rsplit(":", 1)
                        # if there is ':' in the address, then this is a ipv6
                        if ':' in tokens[0]:
                            all.add_host(Host(x))
                        else:
                            all.add_host(Host(tokens[0], tokens[1]))
                    else:
                        all.add_host(Host(x))
        elif os.path.exists(host_list):
            if os.path.isdir(host_list):
                # Ensure basedir is inside the directory
                self.host_list = os.path.join(self.host_list, "")
                self.parser = InventoryDirectory(filename=host_list)
                self.groups = self.parser.groups.values()
            elif utils.is_executable(host_list):
                self.parser = InventoryScript(filename=host_list)
                self.groups = self.parser.groups.values()
            else:
                self.parser = InventoryParser(filename=host_list)
                self.groups = self.parser.groups.values()

            utils.plugins.vars_loader.add_directory(self.basedir(), with_subdir=True)
        else:
            raise errors.AnsibleError("Unable to find an inventory file, specify one with -i ?")

        self._vars_plugins = [ x for x in utils.plugins.vars_loader.all(self) ]

    def _match(self, str, pattern_str):
        if pattern_str.startswith('~'):
            return re.search(pattern_str[1:], str)
        else:
            return fnmatch.fnmatch(str, pattern_str)

    def get_hosts(self, pattern="all"):
        """
        find all host names matching a pattern string, taking into account any inventory restrictions or
        applied subsets.
        """

        # process patterns
        if isinstance(pattern, list):
            pattern = ';'.join(pattern)
        patterns = pattern.replace(";",":").split(":")
        hosts = self._get_hosts(patterns)

        # exclude hosts not in a subset, if defined
        if self._subset:
            subset = self._get_hosts(self._subset)
            hosts.intersection_update(subset)

        # exclude hosts mentioned in any restriction (ex: failed hosts)
        if self._restriction is not None:
            hosts = [ h for h in hosts if h.name in self._restriction ]
        if self._also_restriction is not None:
            hosts = [ h for h in hosts if h.name in self._also_restriction ]

        return sorted(hosts, key=lambda x: x.name)

    def _get_hosts(self, patterns):
        """
        finds hosts that match a list of patterns. Handles negative
        matches as well as intersection matches.
        """

        # Host specifiers should be sorted to ensure consistent behavior
        pattern_regular = []
        pattern_intersection = []
        pattern_exclude = []
        for p in patterns:
            if p.startswith("!"):
                pattern_exclude.append(p)
            elif p.startswith("&"):
                pattern_intersection.append(p)
            else:
                pattern_regular.append(p)

        # if no regular pattern was given, hence only exclude and/or intersection
        # make that magically work
        if pattern_regular == []:
            pattern_regular = ['all']

        # when applying the host selectors, run those without the "&" or "!"
        # first, then the &s, then the !s.
        patterns = pattern_regular + pattern_intersection + pattern_exclude

        hosts = set()
        for p in patterns:
            if p.startswith("!"):
                # Discard excluded hosts
                hosts.difference_update(self.__get_hosts(p))
            elif p.startswith("&"):
                # Only leave the intersected hosts
                hosts.intersection_update(self.__get_hosts(p))
            else:
                # Get all hosts from both patterns
                hosts.update(self.__get_hosts(p))
        return hosts

    def __get_hosts(self, pattern):
        """
        finds hosts that postively match a particular pattern.  Does not
        take into account negative matches.
        """

        (name, enumeration_details) = self._enumeration_info(pattern)
        hpat = self._hosts_in_unenumerated_pattern(name)
        hpat = sorted(hpat, key=lambda x: x.name)

        return set(self._apply_ranges(pattern, hpat))

    def _enumeration_info(self, pattern):
        """
        returns (pattern, limits) taking a regular pattern and finding out
        which parts of it correspond to start/stop offsets.  limits is
        a tuple of (start, stop) or None
        """

        if not "[" in pattern or pattern.startswith('~'):
            return (pattern, None)
        (first, rest) = pattern.split("[")
        rest = rest.replace("]","")
        if "-" in rest:
            (left, right) = rest.split("-",1)
            return (first, (left, right))
        else:
            return (first, (rest, rest))

    def _apply_ranges(self, pat, hosts):
        """
        given a pattern like foo, that matches hosts, return all of hosts
        given a pattern like foo[0:5], where foo matches hosts, return the first 6 hosts
        """

        (loose_pattern, limits) = self._enumeration_info(pat)
        if not limits:
            return hosts

        (left, right) = limits
        enumerated = enumerate(hosts)
        if left == '':
            left = 0
        if right == '':
            right = 0
        left=int(left)
        right=int(right)
        enumerated = [ h for (i,h) in enumerated if i>=left and i<=right ]
        return enumerated

    # TODO: cache this logic so if called a second time the result is not recalculated
    def _hosts_in_unenumerated_pattern(self, pattern):
        """ Get all host names matching the pattern """

        hosts = {}
        # ignore any negative checks here, this is handled elsewhere
        pattern = pattern.replace("!","").replace("&", "")

        groups = self.get_groups()
        for group in groups:
            for host in group.get_hosts():
                if pattern == 'all' or self._match(group.name, pattern) or self._match(host.name, pattern):
                    hosts[host.name] = host
        return sorted(hosts.values(), key=lambda x: x.name)

    def groups_for_host(self, host):
        results = []
        groups = self.get_groups()
        for group in groups:
            for hostn in group.get_hosts():
                if host == hostn.name:
                    results.append(group)
                    continue
        return results

    def groups_list(self):
        if not self._groups_list:
            groups = {}
            for g in self.groups:
                groups[g.name] = [h.name for h in g.get_hosts()]
                ancestors = g.get_ancestors()
                for a in ancestors:
                    if a.name not in groups:
                        groups[a.name] = [h.name for h in a.get_hosts()]
            self._groups_list = groups
        return self._groups_list

    def get_groups(self):
        return self.groups

    def get_host(self, hostname):
        if hostname not in self._hosts_cache:
            self._hosts_cache[hostname] = self._get_host(hostname)
        return self._hosts_cache[hostname]

    def _get_host(self, hostname):
        if hostname in ['localhost','127.0.0.1']:
            for host in self.get_group('all').get_hosts():
                if host.name in ['localhost', '127.0.0.1']:
                    return host
        else:
            for group in self.groups:
                for host in group.get_hosts():
                    if hostname == host.name:
                        return host
        return None

    def get_group(self, groupname):
        for group in self.groups:
            if group.name == groupname:
                return group
        return None

    def get_group_variables(self, groupname):
        if groupname not in self._vars_per_group:
            self._vars_per_group[groupname] = self._get_group_variables(groupname)
        return self._vars_per_group[groupname]

    def _get_group_variables(self, groupname):
        group = self.get_group(groupname)
        if group is None:
            raise Exception("group not found: %s" % groupname)
        return group.get_variables()

    def get_variables(self, hostname):
        if hostname not in self._vars_per_host:
            self._vars_per_host[hostname] = self._get_variables(hostname)
        return self._vars_per_host[hostname]

    def _get_variables(self, hostname):

        host = self.get_host(hostname)
        if host is None:
            raise errors.AnsibleError("host not found: %s" % hostname)

        vars = {}
        vars_results = [ plugin.run(host) for plugin in self._vars_plugins ]
        for updated in vars_results:
            if updated is not None:
                vars.update(updated)

        vars.update(host.get_variables())
        if self.parser is not None:
            vars.update(self.parser.get_host_variables(host))
        return vars

    def add_group(self, group):
        self.groups.append(group)
        self._groups_list = None  # invalidate internal cache

    def list_hosts(self, pattern="all"):
        return [ h.name for h in self.get_hosts(pattern) ]

    def list_groups(self):
        return sorted([ g.name for g in self.groups ], key=lambda x: x)

    # TODO: remove this function
    def get_restriction(self):
        return self._restriction

    def restrict_to(self, restriction):
        """
        Restrict list operations to the hosts given in restriction.  This is used
        to exclude failed hosts in main playbook code, don't use this for other
        reasons.
        """
        if not isinstance(restriction, list):
            restriction = [ restriction ]
        self._restriction = restriction

    def also_restrict_to(self, restriction):
        """
        Works like restict_to but offers an additional restriction.  Playbooks use this
        to implement serial behavior.
        """
        if not isinstance(restriction, list):
            restriction = [ restriction ]
        self._also_restriction = restriction

    def subset(self, subset_pattern):
        """
        Limits inventory results to a subset of inventory that matches a given
        pattern, such as to select a given geographic of numeric slice amongst
        a previous 'hosts' selection that only select roles, or vice versa.
        Corresponds to --limit parameter to ansible-playbook
        """
        if subset_pattern is None:
            self._subset = None
        else:
            subset_pattern = subset_pattern.replace(',',':')
            subset_pattern = subset_pattern.replace(";",":").split(":")
            results = []
            # allow Unix style @filename data
            for x in subset_pattern:
                if x.startswith("@"):
                    fd = open(x[1:])
                    results.extend(fd.read().split("\n"))
                    fd.close()
                else:
                    results.append(x)
            self._subset = results

    def lift_restriction(self):
        """ Do not restrict list operations """
        self._restriction = None

    def lift_also_restriction(self):
        """ Clears the also restriction """
        self._also_restriction = None

    def is_file(self):
        """ did inventory come from a file? """
        if not isinstance(self.host_list, basestring):
            return False
        return os.path.exists(self.host_list)

    def basedir(self):
        """ if inventory came from a file, what's the directory? """
        if not self.is_file():
            return None
        dname = os.path.dirname(self.host_list)
        if dname is None or dname == '' or dname == '.':
            cwd = os.getcwd()
            return cwd
        return dname

    def src(self):
        """ if inventory came from a file, what's the directory and file name? """
        if not self.is_file():
            return None
        return self.host_list

    def playbook_basedir(self):
        """ returns the directory of the current playbook """
        return self._playbook_basedir

    def set_playbook_basedir(self, dir):
        """
        sets the base directory of the playbook so inventory plugins can use it to find
        variable files and other things.
        """
        self._playbook_basedir = dir

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Module for managing ext2/3/4 file systems
'''

# Import python libs
import logging

# Import salt libs
import salt.utils

log = logging.getLogger(__name__)

def __virtual__():
    '''
    Only work on POSIX-like systems
    '''
    if salt.utils.is_windows():
        return False
    return True

def mkfs(device, fs_type, **kwargs):
    '''
    Create a file system on the specified device

    CLI Example:

    .. code-block:: bash

        salt '*' extfs.mkfs /dev/sda1 fs_type=ext4 opts='acl,noexec'

    Valid options are:

    * **block_size**: 1024, 2048 or 4096
    * **check**: check for bad blocks
    * **direct**: use direct IO
    * **ext_opts**: extended file system options (comma-separated)
    * **fragment_size**: size of fragments
    * **force**: setting force to True will cause mke2fs to specify the -F
      option twice (it is already set once); this is truly dangerous
    * **blocks_per_group**: number of blocks in a block group
    * **number_of_groups**: ext4 option for a virtual block group
    * **bytes_per_inode**: set the bytes/inode ratio
    * **inode_size**: size of the inode
    * **journal**: set to True to create a journal (default on ext3/4)
    * **journal_opts**: options for the fs journal (comma separated)
    * **blocks_file**: read bad blocks from file
    * **label**: label to apply to the file system
    * **reserved**: percentage of blocks reserved for super-user
    * **last_dir**: last mounted directory
    * **test**: set to True to not actually create the file system (mke2fs -n)
    * **number_of_inodes**: override default number of inodes
    * **creator_os**: override "creator operating system" field
    * **opts**: mount options (comma separated)
    * **revision**: set the filesystem revision (default 1)
    * **super**: write superblock and group descriptors only
    * **fs_type**: set the filesystem type (REQUIRED)
    * **usage_type**: how the filesystem is going to be used
    * **uuid**: set the UUID for the file system

    See the ``mke2fs(8)`` manpage for a more complete description of these
    options.
    '''
    kwarg_map = {'block_size': 'b',
                 'check': 'c',
                 'direct': 'D',
                 'ext_opts': 'E',
                 'fragment_size': 'f',
                 'force': 'F',
                 'blocks_per_group': 'g',
                 'number_of_groups': 'G',
                 'bytes_per_inode': 'i',
                 'inode_size': 'I',
                 'journal': 'j',
                 'journal_opts': 'J',
                 'blocks_file': 'l',
                 'label': 'L',
                 'reserved': 'm',
                 'last_dir': 'M',
                 'test': 'n',
                 'number_of_inodes': 'N',
                 'creator_os': 'o',
                 'opts': 'O',
                 'revision': 'r',
                 'super': 'S',
                 'usage_type': 'T',
                 'uuid': 'U'}

    opts = ''
    for key in kwargs:
        if key in kwarg_map:
            opt = kwarg_map[key]
            if kwargs[key] == 'True':
                opts += '-{0} '.format(opt)
            else:
                opts += '-{0} {1} '.format(opt, kwargs[key])
    cmd = 'mke2fs -F -t {0} {1}{2}'.format(fs_type, opts, device)
    out = __salt__['cmd.run'](cmd).splitlines()
    ret = []
    for line in out:
        if not line:
            continue
        elif line.startswith('mke2fs'):
            continue
        elif line.startswith('Discarding device blocks'):
            continue
        elif line.startswith('Allocating group tables'):
            continue
        elif line.startswith('Writing inode tables'):
            continue
        elif line.startswith('Creating journal'):
            continue
        elif line.startswith('Writing superblocks'):
            continue
        ret.append(line)
    return ret

def tune(device, **kwargs):
    '''
    Set attributes for the specified device (using tune2fs)

    CLI Example:

    .. code-block:: bash

        salt '*' extfs.tune /dev/sda1 force=True label=wildstallyns opts='acl,noexec'

    Valid options are:

    * **max**: max mount count
    * **count**: mount count
    * **error**: error behavior
    * **extended_opts**: extended options (comma separated)
    * **force**: force, even if there are errors (set to True)
    * **group**: group name or gid that can use the reserved blocks
    * **interval**: interval between checks
    * **journal**: set to True to create a journal (default on ext3/4)
    * **journal_opts**: options for the fs journal (comma separated)
    * **label**: label to apply to the file system
    * **reserved**: percentage of blocks reserved for super-user
    * **last_dir**: last mounted directory
    * **opts**: mount options (comma separated)
    * **feature**: set or clear a feature (comma separated)
    * **mmp_check**: mmp check interval
    * **reserved**: reserved blocks count
    * **quota_opts**: quota options (comma separated)
    * **time**: time last checked
    * **user**: user or uid who can use the reserved blocks
    * **uuid**: set the UUID for the file system

    See the ``mke2fs(8)`` manpage for a more complete description of these
    options.
    '''
    kwarg_map = {'max': 'c',
                 'count': 'C',
                 'error': 'e',
                 'extended_opts': 'E',
                 'force': 'f',
                 'group': 'g',
                 'interval': 'i',
                 'journal': 'j',
                 'journal_opts': 'J',
                 'label': 'L',
                 'last_dir': 'M',
                 'opts': 'o',
                 'feature': 'O',
                 'mmp_check': 'p',
                 'reserved': 'r',
                 'quota_opts': 'Q',
                 'time': 'T',
                 'user': 'u',
                 'uuid': 'U'}
    opts = ''
    for key in kwargs:
        if key in kwarg_map:
            opt = kwarg_map[key]
            if kwargs[key] == 'True':
                opts += '-{0} '.format(opt)
            else:
                opts += '-{0} {1} '.format(opt, kwargs[key])
    cmd = 'tune2fs {0}{1}'.format(opts, device)
    out = __salt__['cmd.run'](cmd).splitlines()
    return out

def attributes(device, args=None):
    '''
    Return attributes from dumpe2fs for a specified device

    CLI Example:

    .. code-block:: bash

        salt '*' extfs.attributes /dev/sda1
    '''
    fsdump = dump(device, args)
    return fsdump['attributes']

def blocks(device, args=None):
    '''
    Return block and inode info from dumpe2fs for a specified device

    CLI Example:

    .. code-block:: bash

        salt '*' extfs.blocks /dev/sda1
    '''
    fsdump = dump(device, args)
    return fsdump['blocks']

def dump(device, args=None):
    '''
    Return all contents of dumpe2fs for a specified device

    CLI Example:

    .. code-block:: bash

        salt '*' extfs.dump /dev/sda1
    '''
    cmd = 'dumpe2fs {0}'.format(device)
    if args:
        cmd = cmd + ' -' + args
    ret = {'attributes': {}, 'blocks': {}}
    out = __salt__['cmd.run'](cmd).splitlines()
    mode = 'opts'
    group = None
    for line in out:
        if not line:
            continue
        if line.startswith('dumpe2fs'):
            continue
        if mode == 'opts':
            line = line.replace('\t', ' ')
            comps = line.split(': ')
            if line.startswith('Filesystem features'):
                ret['attributes'][comps[0]] = comps[1].split()
            elif line.startswith('Group'):
                mode = 'blocks'
            else:
                ret['attributes'][comps[0]] = comps[1].strip()

        if mode == 'blocks':
            if line.startswith('Group'):
                line = line.replace(':', '')
                line = line.replace('(', '')
                line = line.replace(')', '')
                line = line.replace('[', '')
                line = line.replace(']', '')
                comps = line.split()
                blkgrp = comps[1]
                group = 'Group {0}'.format(blkgrp)
                ret['blocks'][group] = {}
                ret['blocks'][group]['group'] = blkgrp
                ret['blocks'][group]['range'] = comps[3]
                # TODO: comps[4:], which may look one one of the following:
                #     ITABLE_ZEROED
                #     INODE_UNINIT, ITABLE_ZEROED
                # Does anyone know what to call these?
                ret['blocks'][group]['extra'] = []
            elif 'Free blocks:' in line:
                comps = line.split(': ')
                free_blocks = comps[1].split(', ')
                ret['blocks'][group]['free blocks'] = free_blocks
            elif 'Free inodes:' in line:
                comps = line.split(': ')
                inodes = comps[1].split(', ')
                ret['blocks'][group]['free inodes'] = inodes
            else:
                line = line.strip()
                ret['blocks'][group]['extra'].append(line)
    return ret
<commit_msg>extfs module python shell kwarg additions<commit_after># -*- coding: utf-8 -*-
'''
Module for managing ext2/3/4 file systems
'''

# Import python libs
import logging

# Import salt libs
import salt.utils

log = logging.getLogger(__name__)

def __virtual__():
    '''
    Only work on POSIX-like systems
    '''
    if salt.utils.is_windows():
        return False
    return True

def mkfs(device, fs_type, **kwargs):
    '''
    Create a file system on the specified device

    CLI Example:

    .. code-block:: bash

        salt '*' extfs.mkfs /dev/sda1 fs_type=ext4 opts='acl,noexec'

    Valid options are:

    * **block_size**: 1024, 2048 or 4096
    * **check**: check for bad blocks
    * **direct**: use direct IO
    * **ext_opts**: extended file system options (comma-separated)
    * **fragment_size**: size of fragments
    * **force**: setting force to True will cause mke2fs to specify the -F
      option twice (it is already set once); this is truly dangerous
    * **blocks_per_group**: number of blocks in a block group
    * **number_of_groups**: ext4 option for a virtual block group
    * **bytes_per_inode**: set the bytes/inode ratio
    * **inode_size**: size of the inode
    * **journal**: set to True to create a journal (default on ext3/4)
    * **journal_opts**: options for the fs journal (comma separated)
    * **blocks_file**: read bad blocks from file
    * **label**: label to apply to the file system
    * **reserved**: percentage of blocks reserved for super-user
    * **last_dir**: last mounted directory
    * **test**: set to True to not actually create the file system (mke2fs -n)
    * **number_of_inodes**: override default number of inodes
    * **creator_os**: override "creator operating system" field
    * **opts**: mount options (comma separated)
    * **revision**: set the filesystem revision (default 1)
    * **super**: write superblock and group descriptors only
    * **fs_type**: set the filesystem type (REQUIRED)
    * **usage_type**: how the filesystem is going to be used
    * **uuid**: set the UUID for the file system

    See the ``mke2fs(8)`` manpage for a more complete description of these
    options.
    '''
    kwarg_map = {'block_size': 'b',
                 'check': 'c',
                 'direct': 'D',
                 'ext_opts': 'E',
                 'fragment_size': 'f',
                 'force': 'F',
                 'blocks_per_group': 'g',
                 'number_of_groups': 'G',
                 'bytes_per_inode': 'i',
                 'inode_size': 'I',
                 'journal': 'j',
                 'journal_opts': 'J',
                 'blocks_file': 'l',
                 'label': 'L',
                 'reserved': 'm',
                 'last_dir': 'M',
                 'test': 'n',
                 'number_of_inodes': 'N',
                 'creator_os': 'o',
                 'opts': 'O',
                 'revision': 'r',
                 'super': 'S',
                 'usage_type': 'T',
                 'uuid': 'U'}

    opts = ''
    for key in kwargs:
        if key in kwarg_map:
            opt = kwarg_map[key]
            if kwargs[key] == 'True':
                opts += '-{0} '.format(opt)
            else:
                opts += '-{0} {1} '.format(opt, kwargs[key])
    cmd = 'mke2fs -F -t {0} {1}{2}'.format(fs_type, opts, device)
    out = __salt__['cmd.run'](cmd, python_shell=False).splitlines()
    ret = []
    for line in out:
        if not line:
            continue
        elif line.startswith('mke2fs'):
            continue
        elif line.startswith('Discarding device blocks'):
            continue
        elif line.startswith('Allocating group tables'):
            continue
        elif line.startswith('Writing inode tables'):
            continue
        elif line.startswith('Creating journal'):
            continue
        elif line.startswith('Writing superblocks'):
            continue
        ret.append(line)
    return ret

def tune(device, **kwargs):
    '''
    Set attributes for the specified device (using tune2fs)

    CLI Example:

    .. code-block:: bash

        salt '*' extfs.tune /dev/sda1 force=True label=wildstallyns opts='acl,noexec'

    Valid options are:

    * **max**: max mount count
    * **count**: mount count
    * **error**: error behavior
    * **extended_opts**: extended options (comma separated)
    * **force**: force, even if there are errors (set to True)
    * **group**: group name or gid that can use the reserved blocks
    * **interval**: interval between checks
    * **journal**: set to True to create a journal (default on ext3/4)
    * **journal_opts**: options for the fs journal (comma separated)
    * **label**: label to apply to the file system
    * **reserved**: percentage of blocks reserved for super-user
    * **last_dir**: last mounted directory
    * **opts**: mount options (comma separated)
    * **feature**: set or clear a feature (comma separated)
    * **mmp_check**: mmp check interval
    * **reserved**: reserved blocks count
    * **quota_opts**: quota options (comma separated)
    * **time**: time last checked
    * **user**: user or uid who can use the reserved blocks
    * **uuid**: set the UUID for the file system

    See the ``mke2fs(8)`` manpage for a more complete description of these
    options.
    '''
    kwarg_map = {'max': 'c',
                 'count': 'C',
                 'error': 'e',
                 'extended_opts': 'E',
                 'force': 'f',
                 'group': 'g',
                 'interval': 'i',
                 'journal': 'j',
                 'journal_opts': 'J',
                 'label': 'L',
                 'last_dir': 'M',
                 'opts': 'o',
                 'feature': 'O',
                 'mmp_check': 'p',
                 'reserved': 'r',
                 'quota_opts': 'Q',
                 'time': 'T',
                 'user': 'u',
                 'uuid': 'U'}
    opts = ''
    for key in kwargs:
        if key in kwarg_map:
            opt = kwarg_map[key]
            if kwargs[key] == 'True':
                opts += '-{0} '.format(opt)
            else:
                opts += '-{0} {1} '.format(opt, kwargs[key])
    cmd = 'tune2fs {0}{1}'.format(opts, device)
    out = __salt__['cmd.run'](cmd, python_shell=False).splitlines()
    return out

def attributes(device, args=None):
    '''
    Return attributes from dumpe2fs for a specified device

    CLI Example:

    .. code-block:: bash

        salt '*' extfs.attributes /dev/sda1
    '''
    fsdump = dump(device, args)
    return fsdump['attributes']

def blocks(device, args=None):
    '''
    Return block and inode info from dumpe2fs for a specified device

    CLI Example:

    .. code-block:: bash

        salt '*' extfs.blocks /dev/sda1
    '''
    fsdump = dump(device, args)
    return fsdump['blocks']

def dump(device, args=None):
    '''
    Return all contents of dumpe2fs for a specified device

    CLI Example:

    .. code-block:: bash

        salt '*' extfs.dump /dev/sda1
    '''
    cmd = 'dumpe2fs {0}'.format(device)
    if args:
        cmd = cmd + ' -' + args
    ret = {'attributes': {}, 'blocks': {}}
    out = __salt__['cmd.run'](cmd, python_shell=False).splitlines()
    mode = 'opts'
    group = None
    for line in out:
        if not line:
            continue
        if line.startswith('dumpe2fs'):
            continue
        if mode == 'opts':
            line = line.replace('\t', ' ')
            comps = line.split(': ')
            if line.startswith('Filesystem features'):
                ret['attributes'][comps[0]] = comps[1].split()
            elif line.startswith('Group'):
                mode = 'blocks'
            else:
                ret['attributes'][comps[0]] = comps[1].strip()

        if mode == 'blocks':
            if line.startswith('Group'):
                line = line.replace(':', '')
                line = line.replace('(', '')
                line = line.replace(')', '')
                line = line.replace('[', '')
                line = line.replace(']', '')
                comps = line.split()
                blkgrp = comps[1]
                group = 'Group {0}'.format(blkgrp)
                ret['blocks'][group] = {}
                ret['blocks'][group]['group'] = blkgrp
                ret['blocks'][group]['range'] = comps[3]
                # TODO: comps[4:], which may look one one of the following:
                #     ITABLE_ZEROED
                #     INODE_UNINIT, ITABLE_ZEROED
                # Does anyone know what to call these?
                ret['blocks'][group]['extra'] = []
            elif 'Free blocks:' in line:
                comps = line.split(': ')
                free_blocks = comps[1].split(', ')
                ret['blocks'][group]['free blocks'] = free_blocks
            elif 'Free inodes:' in line:
                comps = line.split(': ')
                inodes = comps[1].split(', ')
                ret['blocks'][group]['free inodes'] = inodes
            else:
                line = line.strip()
                ret['blocks'][group]['extra'].append(line)
    return ret
<|endoftext|>


<commit_before># coding=utf-8
# Licensed Materials - Property of IBM
# Copyright IBM Corp. 2017
"""
Python APIs for use with Streaming Analytics service on
IBM® Bluemix cloud platform and on-premises IBM Streams.

Python Application API for Streams
----------------------------------
Module that allows the definition and execution of streaming
applications implemented in Python.
Applications use Python code to process tuples and tuples are Python objects.

See :py:mod:`streamsx.topology`

Python functions as SPL operators
---------------------------------
A Python function or class can be simply turned into an SPL primitive operator
to allow tuple processing using Python in an SPL application.

SPL (Streams Processing Language) is a domain specific language for streaming
analytics supported by Streams.

See :py:mod:`streamsx.spl`

Streams Python REST API
-----------------------
Module that allows interaction with an running Streams instance or
service through HTTPS REST APIs.

See :py:mod:`streamsx.rest`

"""

from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
<commit_msg>Fix naming<commit_after># coding=utf-8
# Licensed Materials - Property of IBM
# Copyright IBM Corp. 2017
"""
Python APIs for use with IBM Streaming Analytics service on
IBM Bluemix® cloud platform and on-premises IBM Streams.

Python Application API for Streams
----------------------------------
Module that allows the definition and execution of streaming
applications implemented in Python.
Applications use Python code to process tuples and tuples are Python objects.

See :py:mod:`streamsx.topology`

Python functions as SPL operators
---------------------------------
A Python function or class can be simply turned into an SPL primitive operator
to allow tuple processing using Python in an SPL application.

SPL (Streams Processing Language) is a domain specific language for streaming
analytics supported by Streams.

See :py:mod:`streamsx.spl`

Streams Python REST API
-----------------------
Module that allows interaction with an running Streams instance or
service through HTTPS REST APIs.

See :py:mod:`streamsx.rest`

"""

from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
<|endoftext|>


<commit_before># stdlib
from typing import List
from typing import Type
from typing import Union

# third party
from nacl.signing import VerifyKey
from nacl.encoding import HexEncoder

# syft relative
from syft.core.node.abstract.node import AbstractNode
from syft.core.node.common.service.auth import service_auth
from syft.core.node.common.service.node_service import ImmediateNodeServiceWithReply
from syft.core.node.common.service.node_service import ImmediateNodeServiceWithoutReply
from syft.core.common.message import ImmediateSyftMessageWithReply

from syft.grid.messages.request_messages import (
    CreateRequestMessage,
    CreateRequestResponse,
    GetRequestMessage,
    GetRequestResponse,
    GetRequestsMessage,
    GetRequestsResponse,
    UpdateRequestMessage,
    UpdateRequestResponse,
    DeleteRequestMessage,
    DeleteRequestResponse,
)

from ..exceptions import (
    MissingRequestKeyError,
    AuthorizationError,
    InvalidParameterValueError,
)
from ..database.utils import model_to_json

def create_request_msg(
    msg: CreateRequestMessage,
    node: AbstractNode,
    verify_key: VerifyKey,
) -> CreateRequestResponse:
    # Get Payload Content
    current_user_id = msg.content.get("current_user", None)
    object_id = msg.content.get("object_id", None)
    reason = msg.content.get("reason", None)
    request_type = msg.content.get("request_type", None)

    users = node.users

    if not current_user_id:
        current_user = users.first(
            verify_key=verify_key.encode(encoder=HexEncoder).decode("utf-8")
        )
    else:
        current_user = users.first(id=current_user_id)

    # Check if object_id/reason/request_type fields are empty
    missing_paramaters = not object_id or not reason or not request_type
    if missing_paramaters:
        raise MissingRequestKeyError(
            message="Invalid request payload, empty fields (object_id/reason/request_type)!"
        )

    valid_paramaters = request_type == "permissions" or request_type == "budget"

    if not valid_paramaters:
        raise InvalidParameterValueError(
            message='Request type should be either "permissions” or “budget”.'
        )

    requests = node.data_requests
    request_obj = requests.create_request(
        user_id=current_user.id,
        user_name=current_user.email,
        object_id=object_id,
        reason=reason,
        request_type=request_type,
    )
    request_json = model_to_json(request_obj)

    return CreateRequestResponse(
        address=msg.reply_to,
        status_code=200,
        content=request_json,
    )

def get_request_msg(
    msg: GetRequestMessage,
    node: AbstractNode,
    verify_key: VerifyKey,
) -> GetRequestResponse:

    # Get Payload Content
    request_id = msg.content.get("request_id", None)
    current_user_id = msg.content.get("current_user", None)

    users = node.users

    if not current_user_id:
        current_user_id = users.first(
            verify_key=verify_key.encode(encoder=HexEncoder).decode("utf-8")
        ).id

    requests = node.data_requests
    request = requests.first(id=request_id)

    # A user can get a request if he's the owner of that request
    # or has the can_triage_requests permission
    allowed = request.user_id == current_user_id or users.can_triage_requests(
        user_id=current_user_id
    )

    if allowed:
        request_json = model_to_json(request)
    else:
        raise AuthorizationError("You're not allowed to get Request information!")

    return GetRequestResponse(
        address=msg.reply_to,
        status_code=200,
        content=request_json,
    )

def get_all_request_msg(
    msg: GetRequestsMessage,
    node: AbstractNode,
    verify_key: VerifyKey,
) -> GetRequestsResponse:

    # Get Payload Content
    current_user_id = msg.content.get("current_user", None)

    users = node.users

    if not current_user_id:
        current_user_id = users.first(
            verify_key=verify_key.encode(encoder=HexEncoder).decode("utf-8")
        ).id

    allowed = users.can_triage_requests(user_id=current_user_id)

    if allowed:
        requests = node.data_requests
        requests = requests.all()
        requests_json = [model_to_json(requests) for requests in requests]
    else:
        raise AuthorizationError("You're not allowed to get Request information!")

    return GetRequestsResponse(
        address=msg.reply_to,
        status_code=200,
        content=requests_json,
    )

def update_request_msg(
    msg: DeleteRequestMessage,
    node: AbstractNode,
    verify_key: VerifyKey,
) -> DeleteRequestResponse:

    # Get Payload Content
    request_id = msg.content.get("request_id", None)
    status = msg.content.get("status", None)
    current_user_id = msg.content.get("current_user", None)

    users = node.users

    if not current_user_id:
        current_user_id = users.first(
            verify_key=verify_key.encode(encoder=HexEncoder).decode("utf-8")
        ).id

    # Check if status field is empty
    missing_paramaters = not status
    if missing_paramaters:
        raise MissingRequestKeyError(
            message="Invalid request payload, empty fields (status)!"
        )

    allowed = users.can_triage_requests(user_id=current_user_id)

    if allowed:
        requests = node.data_requests

        if status not in ["accepted", "denied"]:
            raise InvalidParameterValueError(
                message='Request status should be either "accepted" or "denied"'
            )

        if status == "accepted":
            request = requests.first(id=request_id)
            object_id = request.object_id

            # Accessing and updating the datase metadata
            storage = node.disk_store
            read_permission = {
                "verify_key": verify_key.encode(encoder=HexEncoder).decode("utf-8"),
                "request_id": request_id,
            }
            storage.update_dataset_metadata(
                key=object_id, read_permissions=read_permission
            )

        # TODO:
        # 1 - The logic to change a user privacy budget needs to be implemented,
        # as soon as this logic is ready this need to be updated.

        requests.set(request_id=request_id, status=status)
    else:
        raise AuthorizationError("You're not allowed to update Request information!")

    return DeleteRequestResponse(
        address=msg.reply_to,
        status_code=200,
        content={"msg": "Request Updated!"},
    )

def del_request_msg(
    msg: DeleteRequestMessage,
    node: AbstractNode,
    verify_key: VerifyKey,
) -> DeleteRequestResponse:

    # Get Payload Content
    request_id = msg.content.get("request_id", None)
    current_user_id = msg.content.get("current_user", None)

    users = node.users

    if not current_user_id:
        current_user_id = users.first(
            verify_key=verify_key.encode(encoder=HexEncoder).decode("utf-8")
        ).id

    requests = node.data_requests
    request = requests.first(id=request_id)

    # Only the creator of a request may delete their request.
    if request.user_id == current_user_id:
        requests.delete(id=request_id)
    else:
        raise AuthorizationError("You're not allowed to delete this Request!")

    return DeleteRequestResponse(
        address=msg.reply_to,
        status_code=200,
        content={"msg": "Request deleted!"},
    )

class RequestService(ImmediateNodeServiceWithReply):

    msg_handler_map = {
        CreateRequestMessage: create_request_msg,
        GetRequestMessage: get_request_msg,
        GetRequestsMessage: get_all_request_msg,
        UpdateRequestMessage: update_request_msg,
        DeleteRequestMessage: del_request_msg,
    }

    @staticmethod
    @service_auth(guests_welcome=True)
    def process(
        node: AbstractNode,
        msg: Union[
            CreateRequestMessage,
            GetRequestMessage,
            GetRequestsMessage,
            UpdateRequestMessage,
            DeleteRequestMessage,
        ],
        verify_key: VerifyKey,
    ) -> Union[
        CreateRequestResponse,
        GetRequestResponse,
        GetRequestsResponse,
        UpdateRequestResponse,
        DeleteRequestResponse,
    ]:

        return RequestService.msg_handler_map[type(msg)](
            msg=msg, node=node, verify_key=verify_key
        )

    @staticmethod
    def message_handler_types() -> List[Type[ImmediateSyftMessageWithReply]]:
        return [
            CreateRequestMessage,
            GetRequestMessage,
            GetRequestsMessage,
            GetRequestsMessage,
            UpdateRequestMessage,
            DeleteRequestMessage,
        ]
<commit_msg>ADD modified Syft Services<commit_after># stdlib
from typing import List
from typing import Type
from typing import Union
from typing import Optional
import time

# third party
from nacl.signing import VerifyKey
from nacl.encoding import HexEncoder

# syft relative
from syft.core.node.abstract.node import AbstractNode
from syft.core.node.common.service.auth import service_auth
from syft.core.node.common.node import DuplicateRequestException
from syft.core.node.common.service.node_service import ImmediateNodeServiceWithReply
from syft.core.node.common.service.node_service import ImmediateNodeServiceWithoutReply
from syft.core.common.message import ImmediateSyftMessageWithReply
from syft.core.common.message import ImmediateSyftMessageWithoutReply
from syft.core.common.uid import UID
from syft.util import validate_type

# Syft Request services with reply
from syft.core.node.domain.service.request_answer_message import (
    RequestAnswerMessage,
    RequestAnswerResponse,
)
from syft.core.node.domain.service.get_all_requests_service import (
    GetAllRequestsMessage,
    GetAllRequestsResponseMessage,
)
from syft.core.node.domain.service.request_handler_service import (
    GetAllRequestHandlersMessage,
    GetAllRequestHandlersResponseMessage,
)

# Syft Request services without reply
from syft.core.node.domain.service.request_message import RequestMessage
from syft.core.node.domain.service.accept_or_deny_request_service import (
    AcceptOrDenyRequestMessage,
)
from syft.core.node.domain.service.request_handler_service import (
    UpdateRequestHandlerMessage,
)

from syft.grid.messages.request_messages import (
    CreateRequestMessage,
    CreateRequestResponse,
    GetRequestMessage,
    GetRequestResponse,
    GetRequestsMessage,
    GetRequestsResponse,
    UpdateRequestMessage,
    UpdateRequestResponse,
    DeleteRequestMessage,
    DeleteRequestResponse,
)

from ..exceptions import (
    MissingRequestKeyError,
    AuthorizationError,
    InvalidParameterValueError,
)
from ..database.utils import model_to_json

def create_request_msg(
    msg: CreateRequestMessage,
    node: AbstractNode,
    verify_key: VerifyKey,
) -> CreateRequestResponse:
    # Get Payload Content
    current_user_id = msg.content.get("current_user", None)
    object_id = msg.content.get("object_id", None)
    reason = msg.content.get("reason", None)
    request_type = msg.content.get("request_type", None)

    users = node.users

    if not current_user_id:
        current_user = users.first(
            verify_key=verify_key.encode(encoder=HexEncoder).decode("utf-8")
        )
    else:
        current_user = users.first(id=current_user_id)

    # Check if object_id/reason/request_type fields are empty
    missing_paramaters = not object_id or not reason or not request_type
    if missing_paramaters:
        raise MissingRequestKeyError(
            message="Invalid request payload, empty fields (object_id/reason/request_type)!"
        )

    valid_paramaters = request_type == "permissions" or request_type == "budget"

    if not valid_paramaters:
        raise InvalidParameterValueError(
            message='Request type should be either "permissions” or “budget”.'
        )

    requests = node.data_requests
    request_obj = requests.create_request(
        user_id=current_user.id,
        user_name=current_user.email,
        object_id=object_id,
        reason=reason,
        request_type=request_type,
    )
    request_json = model_to_json(request_obj)

    return CreateRequestResponse(
        address=msg.reply_to,
        status_code=200,
        content=request_json,
    )

def get_request_msg(
    msg: GetRequestMessage,
    node: AbstractNode,
    verify_key: VerifyKey,
) -> GetRequestResponse:

    # Get Payload Content
    request_id = msg.content.get("request_id", None)
    current_user_id = msg.content.get("current_user", None)

    users = node.users

    if not current_user_id:
        current_user_id = users.first(
            verify_key=verify_key.encode(encoder=HexEncoder).decode("utf-8")
        ).id

    requests = node.data_requests
    request = requests.first(id=request_id)

    # A user can get a request if he's the owner of that request
    # or has the can_triage_requests permission
    allowed = request.user_id == current_user_id or users.can_triage_requests(
        user_id=current_user_id
    )

    if allowed:
        request_json = model_to_json(request)
    else:
        raise AuthorizationError("You're not allowed to get Request information!")

    return GetRequestResponse(
        address=msg.reply_to,
        status_code=200,
        content=request_json,
    )

def get_all_request_msg(
    msg: GetRequestsMessage,
    node: AbstractNode,
    verify_key: VerifyKey,
) -> GetRequestsResponse:

    # Get Payload Content
    current_user_id = msg.content.get("current_user", None)

    users = node.users

    if not current_user_id:
        current_user_id = users.first(
            verify_key=verify_key.encode(encoder=HexEncoder).decode("utf-8")
        ).id

    allowed = users.can_triage_requests(user_id=current_user_id)

    if allowed:
        requests = node.data_requests
        requests = requests.all()
        requests_json = [model_to_json(requests) for requests in requests]
    else:
        raise AuthorizationError("You're not allowed to get Request information!")

    return GetRequestsResponse(
        address=msg.reply_to,
        status_code=200,
        content=requests_json,
    )

def update_request_msg(
    msg: DeleteRequestMessage,
    node: AbstractNode,
    verify_key: VerifyKey,
) -> DeleteRequestResponse:

    # Get Payload Content
    request_id = msg.content.get("request_id", None)
    status = msg.content.get("status", None)
    current_user_id = msg.content.get("current_user", None)

    users = node.users

    if not current_user_id:
        current_user_id = users.first(
            verify_key=verify_key.encode(encoder=HexEncoder).decode("utf-8")
        ).id

    # Check if status field is empty
    missing_paramaters = not status
    if missing_paramaters:
        raise MissingRequestKeyError(
            message="Invalid request payload, empty fields (status)!"
        )

    allowed = users.can_triage_requests(user_id=current_user_id)

    if allowed:
        requests = node.data_requests

        if status not in ["accepted", "denied"]:
            raise InvalidParameterValueError(
                message='Request status should be either "accepted" or "denied"'
            )

        if status == "accepted":
            request = requests.first(id=request_id)
            object_id = request.object_id

            # Accessing and updating the datase metadata
            storage = node.disk_store
            read_permission = {
                "verify_key": verify_key.encode(encoder=HexEncoder).decode("utf-8"),
                "request_id": request_id,
            }
            storage.update_dataset_metadata(
                key=object_id, read_permissions=read_permission
            )

        # TODO:
        # 1 - The logic to change a user privacy budget needs to be implemented,
        # as soon as this logic is ready this need to be updated.

        requests.set(request_id=request_id, status=status)
    else:
        raise AuthorizationError("You're not allowed to update Request information!")

    return DeleteRequestResponse(
        address=msg.reply_to,
        status_code=200,
        content={"msg": "Request Updated!"},
    )

def del_request_msg(
    msg: DeleteRequestMessage,
    node: AbstractNode,
    verify_key: VerifyKey,
) -> DeleteRequestResponse:

    # Get Payload Content
    request_id = msg.content.get("request_id", None)
    current_user_id = msg.content.get("current_user", None)

    users = node.users

    if not current_user_id:
        current_user_id = users.first(
            verify_key=verify_key.encode(encoder=HexEncoder).decode("utf-8")
        ).id

    requests = node.data_requests
    request = requests.first(id=request_id)

    # Only the creator of a request may delete their request.
    if request.user_id == current_user_id:
        requests.delete(id=request_id)
    else:
        raise AuthorizationError("You're not allowed to delete this Request!")

    return DeleteRequestResponse(
        address=msg.reply_to,
        status_code=200,
        content={"msg": "Request deleted!"},
    )

# PySyft Services

def request_answer_msg(
    msg: RequestAnswerMessage, node: AbstractNode, verify_key: VerifyKey
):
    if verify_key is None:
        raise ValueError(
            "Can't process Request service without a given " "verification key"
        )

    status = node.data_requests.status(id=str(msg.request_id.value))  # type: ignore
    address = msg.reply_to
    return RequestAnswerResponse(
        request_id=msg.request_id, address=address, status=status
    )

# TODO: Check if this method/message should really be a service_without_reply message
def get_all_requests(
    msg: GetAllRequestsMessage, node: AbstractNode, verify_key: VerifyKey
):
    if verify_key is None:
        raise ValueError(
            "Can't process Request service without a given " "verification key"
        )

    current_user = node.users.first(
        verify_key=verify_key.encode(encoder=HexEncoder).decode("utf-8")
    )
    _can_triage_request = node.users.can_triage_requests(user_id=current_user.id)

    _requests = node.data_requests.all()

    if _can_triage_request:
        _requests = node.data_requests.all()
    else:
        _requests = node.data_requests.query(
            verify_key=verify_key.encode(encoder=HexEncoder).decode("utf-8")
        )

    data_requests = [
        RequestMessage(
            request_id=UID.from_string(req.id),
            request_description=req.reason,
            address=node.address,
            owner_address=node.address,
            object_id=UID.from_string(req.object_id),
            object_type=req.object_type,
            object_tags=req.tags,
            requester_verify_key=VerifyKey(
                req.verify_key.encode("utf-8"), encoder=HexEncoder
            ),
            timeout_secs=None,
        )
        for req in _requests
    ]

    return GetAllRequestsResponseMessage(requests=data_requests, address=msg.reply_to)

def get_all_request_handlers(
    msg: GetAllRequestHandlersMessage, node: AbstractNode, verify_key: VerifyKey
):

    if verify_key is None:
        raise ValueError(
            "Can't process Request service without a given " "verification key"
        )

    handlers = []
    if verify_key == node.root_verify_key:
        existing_handlers = getattr(node, "request_handlers", None)
        if existing_handlers is not None:
            handlers = existing_handlers

    return GetAllRequestHandlersResponseMessage(handlers=handlers, address=msg.reply_to)

class RequestService(ImmediateNodeServiceWithReply):

    msg_handler_map = {
        CreateRequestMessage: create_request_msg,
        GetRequestMessage: get_request_msg,
        GetRequestsMessage: get_all_request_msg,
        UpdateRequestMessage: update_request_msg,
        DeleteRequestMessage: del_request_msg,
        RequestAnswerMessage: request_answer_msg,
        GetAllRequestsMessage: get_all_requests,
        GetAllRequestHandlersMessage: get_all_request_handlers,
    }

    @staticmethod
    @service_auth(guests_welcome=True)
    def process(
        node: AbstractNode,
        msg: Union[
            CreateRequestMessage,
            GetRequestMessage,
            GetRequestsMessage,
            UpdateRequestMessage,
            DeleteRequestMessage,
            RequestAnswerMessage,
            GetAllRequestsMessage,
            GetAllRequestHandlersMessage,
        ],
        verify_key: VerifyKey,
    ) -> Union[
        CreateRequestResponse,
        GetRequestResponse,
        GetRequestsResponse,
        UpdateRequestResponse,
        DeleteRequestResponse,
        RequestAnswerResponse,
        GetAllRequestsResponseMessage,
        GetAllRequestHandlersResponseMessage,
    ]:
        return RequestService.msg_handler_map[type(msg)](
                msg=msg, node=node, verify_key=verify_key
            )

    @staticmethod
    def message_handler_types() -> List[Type[ImmediateSyftMessageWithReply]]:
        return [
            CreateRequestMessage,
            GetRequestMessage,
            GetRequestsMessage,
            GetRequestsMessage,
            UpdateRequestMessage,
            DeleteRequestMessage,
            RequestAnswerMessage,
            GetAllRequestsMessage,
            GetAllRequestHandlersMessage,
        ]

def build_request_message(
    msg: RequestMessage, node: AbstractNode, verify_key: VerifyKey
):
    if verify_key is None:
        raise ValueError(
            "Can't process Request service without a given " "verification key"
        )

    if msg.requester_verify_key != verify_key:
        raise Exception(
            "You tried to request access for a key that is not yours!"
            "You cannot do this! Whatever key you want to request access"
            "for must be the verify key that also verifies the message"
            "containing the request."
        )

    # since we reject/accept requests based on the ID, we don't want there to be
    # multiple requests with the same ID because this could cause security problems.
    _duplicate_request = node.data_requests.contain(
        object_id=str(msg.object_id.value),
        verify_key=verify_key.encode(encoder=HexEncoder).decode("utf-8"),
    )

    if _duplicate_request:
        raise DuplicateRequestException(
            f"You have already requested {msg.object_id} ",
            node.data_requests.all(),
            "My Requests",
        )

    current_user = node.users.first(
        verify_key=verify_key.encode(encoder=HexEncoder).decode("utf-8")
    )

    node.data_requests.create_request(
        user_id=current_user.id,
        user_name=current_user.email,
        object_id=str(msg.object_id.value),
        reason=msg.request_description,
        request_type="permissions",
        verify_key=verify_key.encode(encoder=HexEncoder).decode("utf-8"),
        object_type=msg.object_type,
        tags=node.store[msg.object_id]._tags,
    )

def accept_or_deny_request(
    msg: AcceptOrDenyRequestMessage, node: AbstractNode, verify_key: VerifyKey
):
    if verify_key is None:
        raise ValueError(
            "Can't process AcceptOrDenyRequestService without a specified verification key"
        )

    _msg: AcceptOrDenyRequestMessage = validate_type(msg, AcceptOrDenyRequestMessage)

    current_user = node.users.first(
        verify_key=verify_key.encode(encoder=HexEncoder).decode("utf-8")
    )

    _req = node.data_requests.first(id=str(_msg.request_id.value))
    _can_triage_request = node.users.can_triage_requests(user_id=current_user.id)
    if _msg.accept:
        if _req and _can_triage_request:
            tmp_obj = node.store[UID.from_string(_req.object_id)]
            tmp_obj.read_permissions[
                VerifyKey(_req.verify_key.encode("utf-8"), encoder=HexEncoder)
            ] = _req.id
            node.store[UID.from_string(_req.object_id)] = tmp_obj
            node.data_requests.set(request_id=_req.id, status="accepted")
    else:
        _req_owner = current_user.verify_key == _req.verify_key
        if _req and (_can_triage_request or _req_owner):
            node.data_requests.set(request_id=_req.id, status="denied")

def update_req_handler(
    node: AbstractNode,
    msg: UpdateRequestHandlerMessage,
    verify_key: Optional[VerifyKey] = None,
) -> None:
    if verify_key is None:
        raise ValueError(
            "Can't process Request service without a given " "verification key"
        )

    if verify_key == node.root_verify_key:
        replacement_handlers = []

        # find if there exists a handler match the handler passed in
        existing_handlers = getattr(node, "request_handlers", None)
        # debug(f"> Updating Request Handlers with existing: {existing_handlers}")
        if existing_handlers is not None:
            matched = None
            for existing_handler in existing_handlers:
                # we match two handlers according to their tags
                if existing_handler["tags"] == msg.handler["tags"]:
                    matched = existing_handler
                    # if an existing_handler match the passed in handler,
                    # we ignore it in for loop
                    continue
                else:
                    # if an existing_handler does not match the passed in
                    # handler, we keep it
                    replacement_handlers.append(existing_handler)

            if msg.keep:
                msg.handler["created_time"] = time.time()
                replacement_handlers.append(msg.handler)
                if matched is not None:
                    print(
                        f"> Replacing a Request Handler {matched} with: {msg.handler}"
                    )
                    # debug(
                    #    f"> Replacing a Request Handler {matched} with: {msg.handler}"
                    # )
                else:
                    print(f"> Adding a Request Handler {msg.handler}")
                    # debug(f"> Adding a Request Handler {msg.handler}")
            else:
                print(f"> Removing a Request Handler with: {msg.handler}")
                # debug(f"> Removing a Request Handler with: {msg.handler}")

            setattr(node, "request_handlers", replacement_handlers)
            # debug(f"> Finished Updating Request Handlers with: {existing_handlers}")
        else:
            # TODO: Replace line below
            print(f"> Node has no Request Handlers attribute: {type(node)}")
            # error(f"> Node has no Request Handlers attribute: {type(node)}")

    return

class RequestServiceWithoutReply(ImmediateNodeServiceWithoutReply):

    msg_handler_map = {
        RequestMessage: build_request_message,
        AcceptOrDenyRequestMessage: accept_or_deny_request,
        UpdateRequestHandlerMessage: update_req_handler,
    }

    @staticmethod
    @service_auth(guests_welcome=True)
    def process(
        node: AbstractNode,
        msg: Union[
            RequestMessage,
            AcceptOrDenyRequestMessage,
            UpdateRequestHandlerMessage,
        ],
        verify_key: VerifyKey,
    ) -> None:
        RequestServiceWithoutReply.msg_handler_map[type(msg)](
            msg=msg, node=node, verify_key=verify_key
        )

    @staticmethod
    def message_handler_types() -> List[Type[ImmediateSyftMessageWithoutReply]]:
        return [RequestMessage, AcceptOrDenyRequestMessage, UpdateRequestHandlerMessage]
<|endoftext|>