<commit_before># Copyright 2016 The Johns Hopkins University Applied Physics Laboratory
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import glob
import hvac
import json
from pprint import pprint
import traceback

VAULT_TOKEN = "vault_token"
VAULT_KEY = "vault_key."

VAULT_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "vault"))
POLICY_DIR = os.path.join(VAULT_DIR, "policies")
PRIVATE_DIR = os.path.join(VAULT_DIR, "private")

class Vault(object):
    def __init__(self, machine, ip = None, proxy = True):
        # If the machine is X.vault.vpc.boss remove the X.
        if machine and machine.count(".") == 3:
            self.machine = machine.split(".", 1)[1]
        else:
            self.machine = machine
        self.ip = ip

        # if in cluster mode, is_authenticated will http redirect to the cluster leader
        #client = hvac.Client(url="http://localhost:8200")#, allow_redirects=False)
        if ip is not None:
            host = ip
        elif machine is not None:
            host = self.machine
        else:
            host = "localhost"

        self.url = "http://{}:8200".format(host)
        if proxy:
            self.proxy = {"http": "http://localhost:3128"}
        else:
            self.proxy = {} # DP XXX: {} or None???

    def path(self, filename):
        """Get the complete file path for given machine's private file.
        Args:
            filename (string) : Name of the machine's private file
        Returns:
            (string) : Complete file path
        """

        machine = self.machine if self.machine else ""

        path = os.path.join(PRIVATE_DIR, machine, filename)
        os.makedirs(os.path.dirname(path), exist_ok = True)

        return path

    def connect(self, read_token = None):
        client = hvac.Client(url=self.url, proxies=self.proxy)

        if read_token is not None:
            token_file = self.path(read_token)
            if not os.path.exists(token_file):
                raise Exception("Token file '{}' doesn't exist".format(token_file))

            with open(token_file, "r") as fh:
                client.token = fh.read()
                try:
                    if not client.is_authenticated():
                        raise Exception("Vault token is not valid, cannot communicate with the Vault")
                except:
                    raise
        return client

    def status_check(self):
        """Check to see that Vault is up and available. Not checking the configuration
        status of Vault, just that it is ready to receive commands.
        """
        try:
            client = self.connect()
            client.is_initialized() # make an actual network connection
            return True
        except:
            return False

    def shell(self):
        """Create a connection to Vault and then drop the user into an interactive
        shell (just like the python interperter) with 'client' holding the Vault
        connection object.
        """
        import code
        client = self.connect(VAULT_TOKEN)
        code.interact(local=locals())

    def initialize(self, secrets = 5, threashold = 3):
        """Initialize a Vault. Connect using get_client() and if the Vault is not
        initialized then initialize it with 5 secrets and a threashold of 3. The
        keys are stored as VAULT_KEY and root token is stored as VAULT_TOKEN.

        After initializing the Vault it is unsealed for use and vault-configure is called.

        Args:
            secrets (int) : Total number of secrets to split the master key into
            threashold (int) : The number of secrets required to reconstruct the master key
        """

        client = self.connect()
        if client.is_initialized():
            print("Vault is already initialized")
            if client.is_sealed():
                print("Unsealing Vault")
                self.unseal()
            else:
                print("Vault already unsealed")
        else:
            print("Initializing with {} secrets and {} needed to unseal".format(secrets, threashold))
            result = client.initialize(secrets, threashold)

            token_file = self.path(VAULT_TOKEN)
            key_file = self.path(VAULT_KEY)
            with open(token_file, "w") as fh:
                fh.write(result["root_token"])
            for i in range(secrets):
                with open(key_file + str(i+1), "w") as fh:
                    fh.write(result["keys"][i])

            print()
            print("======== WARNING WARNING WARNING ========")
            print("= Vault root token and unseal keys were =")
            print("= written to disk. PROTECT these files. =")
            print("======== WARNING WARNING WARNING ========")

            print()
            print("Unsealing Vault")
            client.unseal_multi(result["keys"])

        self.configure()

    def configure(self):
        """A companion function that will configure a newly initialized Vault
        as needed for BOSS. This includes:
            * Configuring the Audit Backend
            * Adding all of the policies from policies/*.hcl
            * Creating a provisioner token with all of the policies added
                - Required so that the provisioner token can issue tokens
                  for any policy
            * Configure the AWS backend (if there are AWS credentials to use)
            * Configure AWS backend roles from policies/*.iam
            * Configure the PKI backend (if there is a certificate to use)
            * Configure PKI backend roles from policies/*.pki

        Args:
            machine (None|string) : hostname of the machine, used for reading/saving unique data
        """
        print("Configuring Vault")
        client = self.connect(VAULT_TOKEN)

        # Audit Backend
        audit_options = {
            'low_raw': 'True',
        }
        try:
            client.enable_audit_backend('syslog', options=audit_options)
        except hvac.exceptions.InvalidRequest as ex:
            print("audit_backend already created.")

        # Policies
        provisioner_policies = []
        path = os.path.join(POLICY_DIR, "*.hcl")
        for policy in glob.glob(path):
            name = os.path.basename(policy).split('.')[0]
            with open(policy, 'r') as fh:
                client.set_policy(name, fh.read())
            # Add every policy to the provisioner, as it has to have the
            # superset of any policies that it will provision
            provisioner_policies.append(name)

        # AWS Authentication Backend
        #Enable AWS auth in Vault
        if 'aws' not in client.list_auth_backends():
            try:
                client.enable_auth_backend('aws')
            except Exception as e:
                print("Error while enabling auth back end: " + e)
                sys.exit(1)
        else:
            print("aws auth backend already created.")

        #Define policies and arn
        policies = [p for p in provisioner_policies if p not in ('provisioner',)]
        arn = 'arn:aws:iam::{}:instance-profile/'.format(os.environ['AWS_ACCOUNT'])
        #TODO: Find a temporary way of storing the aws account number
        #For each policy configure the policies on a role of the same name
        for policy in policies:
            client.write('/auth/aws/role/' + policy, auth_type='ec2', bound_iam_instance_profile_arn= arn + policy, policies=policy)
            print('Successful write to aws/role/' + policy)

        # AWS Secret Backend
        if 'aws' not in client.list_secret_backends():
            try:
                client.enable_secret_backend('aws')
            except Exception as e:
                print('Error while enabling secret back end: ' + e)
                sys.exit(1)
        else:
            print("aws secret backend already created.")

        path = os.path.join(POLICY_DIR, "*.iam")
        for iam in glob.glob(path):
            name = os.path.basename(iam).split('.')[0]
            with open(iam, 'r') as fh:
                # if we json parse the file first we can use the duplicate key trick for comments
                client.write("aws/roles/" + name, policy = fh.read())

        # PKI Backend
        """
        if True: # Disabled until we either have a CA cert or can generate a CA
            print("Vault PKI cert file does not exist, skipping configuration of PKI secret backend")
        else:
            client.enable_secret_backend('pki')
            # Generate a self signed certificate for CA
            print("Generating self signed CA")
            response = client.write("pki/root/generate/internal", common_name=aws_creds["domain"])
            with open(get_path(machine, "ca.pem"), 'w') as fh:
                fh.write(response["data"]["certificate"])

            # Should we configure CRL?

            path = os.path.join(_CURRENT_DIR, "policies", "*.pki")
            for pki in glob.glob(path):
                name = os.path.basename(pki).split('.')[0]
                with open(pki, 'r') as fh:
                    keys = json.load(fh)
                    client.write("aws/roles/" + name, **keys)
        """

    def unseal(self):
        """Unseal a sealed Vault. Connect using get_client() and if the Vault is
        not sealed read all of the keys defined by VAULT_KEY and unseal.

        If there are not enough keys to completely unseal the Vault, print a
        status message about how many more keys are required to finish the
        process.
        """

        client = self.connect()
        if not client.is_sealed():
            print("Vault is already unsealed")
            return 0

        key_file = self.path(VAULT_KEY)
        keys = []
        for f in glob.glob(key_file + "*"):
            with open(f, "r") as fh:
                keys.append(fh.read())

        if len(keys) == 0:
            raise Exception("Could not locate any key files, not unsealing")

        res = client.unseal_multi(keys)
        if res['sealed']:
            p = res['progress']
            t = res['t']
            print("Vault partly unsealed, {} of {} needed keys entered".format(p,t))
            print("Enter {} more keys to finish unsealing the vault". format(t-p))
            return (t-p)
        else:
            print("Vault unsealed")
            return 0

    def seal(self):
        """Seal an unsealed Vault. Connect using get_client(True) and if the Vault
        is unsealed, seal it.

        Used to quickly protect a Vault without having to stop the Vault service
        on a protected VM.
        """

        client = self.connect(VAULT_TOKEN)
        if client.is_sealed():
            print("Vault is already sealed")
            return

        client.seal()
        print("Vault is sealed")

    def status(self):
        """Print the status of a Vault. Connect using get_client(True) and print
        the status of the following items (if available):
         * Initializing status
         * Seal status
         * Key status
         * High Availability status
         * Secret backends
         * Policies
         * Audit backends
         * Auth backends
        """

        client = self.connect()
        if not client.is_initialized():
            print("Vault is not initialized")
            return
        else:
            print("Vault is initialized")

        if client.is_sealed():
            print("Vault is sealed")
            print(client.seal_status)
            return
        else:
            print("Vault is unsealed")

        # read in the Vault access token
        client = self.connect(VAULT_TOKEN)
        print()
        print("Key Status")
        print(json.dumps(client.key_status))

        print()
        print("HA Status")
        print(json.dumps(client.ha_status))

        print()
        print("Secret Backends")
        print(json.dumps(client.list_secret_backends(), indent=True))

        print()
        print("Policies")
        print(json.dumps(client.list_policies()))

        print()
        print("Audit Backends")
        print(json.dumps(client.list_audit_backends(), indent=True))

        print()
        print("Auth Backends")
        print(json.dumps(client.list_auth_backends(), indent=True))

    def provision(self, policy):
        """Create a new Vault access token.

        Args:
            policy (string) : Name of the policy to attach to the new token

        Returns:
            (string) : String containing the new Vault token
        """
        client = self.connect(VAULT_TOKEN)
        token = client.create_token(policies = [policy])
        return token["auth"]["client_token"]

    def revoke(self, token):
        """Revoke a Vault access token.

        Args:
            token (string) : String containing the Vault token to revoke
        """
        client = self.connect(VAULT_TOKEN)
        client.revoke_token(token)

    def revoke_secret(self, lease_id):
        """Revoke a Vault lease

        Args:
            lease_id (string) : String containing the Vault lease id to revoke
        """
        client = self.connect(VAULT_TOKEN)
        client.revoke_secret(lease_id)

    def revoke_secret_prefix(self, prefix):
        """Revoke a Vault secret by prefix

        Args:
            prefix (string) : String containing the Vault secret prefix to revoke
        """
        client = self.connect(VAULT_TOKEN)
        client.revoke_secret_prefix(prefix)

    def write(self, path, **kwargs):
        """A generic method for writing data into Vault.

            Note: vault-write will override any data already existing at path.
                  There is vault-update that will update data at path instead.

        Args:
            path (string) : Vault path to write data to
            kwargs : Key value pairs to store at path
        """
        client = self.connect(VAULT_TOKEN)
        client.write(path, **kwargs)

    def update(self, path, **kwargs):
        """A generic method for adding/updating data to/in Vault.

        Args:
            path (string) : Vault path to write data to
            kwargs : Key value pairs to store at path
        """
        client = self.connect(VAULT_TOKEN)

        existing = client.read(path)
        if existing is None:
            existing = {}
        else:
            existing = existing["data"]

        existing.update(kwargs)

        client.write(path, **existing)

    def read(self, path):
        """A generic method for reading data from Vault.

        Args:
            path (string) : Vault path to read data from
        """
        client = self.connect(VAULT_TOKEN)
        return client.read(path)

    def delete(self, path):
        """A generic method for deleting data from Vault.

        Args:
            path (string) : Vault path to delete all data from
        """
        client = self.connect(VAULT_TOKEN)
        client.delete(path)

    def export(self, path):
        """A generic method for reading all of the paths and keys from Vault.

        Args:
            path (string) : Vault path to dump data from
        """
        if path[-1] != '/':
            path += '/'

        rtn = {}

        # DP NOTE: not using self.read becuase of the different token needed
        client = self.connect(VAULT_TOKEN)
        results =  client.read(path[:-1])
        if results is not None:
            rtn[path[:-1]] = results['data']

        results = client.list(path)
        for key in results['data']['keys']:
            key = path + key
            if key[-1] == '/':
                data = self.export(key)
                rtn.update(data)
            else:
                # DP NOTE: This will currently do a duplicate read if
                #          data is stored at key and in paths under
                #          key
                #          To prevent this, check to see if (key + '/') is in keys
                data = self.read(key)
                rtn[key] = data['data']

        return rtn

    def import_(self, exported, update=False):
        """A generic method for writing / updating data in multiple paths in Vault.

        Args:
            exported (dict): Dict of Vault path and dict of key / values to store at the path
            update (bool): If an Update should be done or if a Write should be done
        """
        for path in exported:
            kv = exported[path]
            fn = self.update if update else self.write
            fn(path, **kv)
<commit_msg>raise exception in library<commit_after># Copyright 2016 The Johns Hopkins University Applied Physics Laboratory
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import glob
import hvac
import json
from pprint import pprint
import traceback

VAULT_TOKEN = "vault_token"
VAULT_KEY = "vault_key."

VAULT_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "vault"))
POLICY_DIR = os.path.join(VAULT_DIR, "policies")
PRIVATE_DIR = os.path.join(VAULT_DIR, "private")

class Vault(object):
    def __init__(self, machine, ip = None, proxy = True):
        # If the machine is X.vault.vpc.boss remove the X.
        if machine and machine.count(".") == 3:
            self.machine = machine.split(".", 1)[1]
        else:
            self.machine = machine
        self.ip = ip

        # if in cluster mode, is_authenticated will http redirect to the cluster leader
        #client = hvac.Client(url="http://localhost:8200")#, allow_redirects=False)
        if ip is not None:
            host = ip
        elif machine is not None:
            host = self.machine
        else:
            host = "localhost"

        self.url = "http://{}:8200".format(host)
        if proxy:
            self.proxy = {"http": "http://localhost:3128"}
        else:
            self.proxy = {} # DP XXX: {} or None???

    def path(self, filename):
        """Get the complete file path for given machine's private file.
        Args:
            filename (string) : Name of the machine's private file
        Returns:
            (string) : Complete file path
        """

        machine = self.machine if self.machine else ""

        path = os.path.join(PRIVATE_DIR, machine, filename)
        os.makedirs(os.path.dirname(path), exist_ok = True)

        return path

    def connect(self, read_token = None):
        client = hvac.Client(url=self.url, proxies=self.proxy)

        if read_token is not None:
            token_file = self.path(read_token)
            if not os.path.exists(token_file):
                raise Exception("Token file '{}' doesn't exist".format(token_file))

            with open(token_file, "r") as fh:
                client.token = fh.read()
                try:
                    if not client.is_authenticated():
                        raise Exception("Vault token is not valid, cannot communicate with the Vault")
                except:
                    raise
        return client

    def status_check(self):
        """Check to see that Vault is up and available. Not checking the configuration
        status of Vault, just that it is ready to receive commands.
        """
        try:
            client = self.connect()
            client.is_initialized() # make an actual network connection
            return True
        except:
            return False

    def shell(self):
        """Create a connection to Vault and then drop the user into an interactive
        shell (just like the python interperter) with 'client' holding the Vault
        connection object.
        """
        import code
        client = self.connect(VAULT_TOKEN)
        code.interact(local=locals())

    def initialize(self, secrets = 5, threashold = 3):
        """Initialize a Vault. Connect using get_client() and if the Vault is not
        initialized then initialize it with 5 secrets and a threashold of 3. The
        keys are stored as VAULT_KEY and root token is stored as VAULT_TOKEN.

        After initializing the Vault it is unsealed for use and vault-configure is called.

        Args:
            secrets (int) : Total number of secrets to split the master key into
            threashold (int) : The number of secrets required to reconstruct the master key
        """

        client = self.connect()
        if client.is_initialized():
            print("Vault is already initialized")
            if client.is_sealed():
                print("Unsealing Vault")
                self.unseal()
            else:
                print("Vault already unsealed")
        else:
            print("Initializing with {} secrets and {} needed to unseal".format(secrets, threashold))
            result = client.initialize(secrets, threashold)

            token_file = self.path(VAULT_TOKEN)
            key_file = self.path(VAULT_KEY)
            with open(token_file, "w") as fh:
                fh.write(result["root_token"])
            for i in range(secrets):
                with open(key_file + str(i+1), "w") as fh:
                    fh.write(result["keys"][i])

            print()
            print("======== WARNING WARNING WARNING ========")
            print("= Vault root token and unseal keys were =")
            print("= written to disk. PROTECT these files. =")
            print("======== WARNING WARNING WARNING ========")

            print()
            print("Unsealing Vault")
            client.unseal_multi(result["keys"])

        self.configure()

    def configure(self):
        """A companion function that will configure a newly initialized Vault
        as needed for BOSS. This includes:
            * Configuring the Audit Backend
            * Adding all of the policies from policies/*.hcl
            * Creating a provisioner token with all of the policies added
                - Required so that the provisioner token can issue tokens
                  for any policy
            * Configure the AWS backend (if there are AWS credentials to use)
            * Configure AWS backend roles from policies/*.iam
            * Configure the PKI backend (if there is a certificate to use)
            * Configure PKI backend roles from policies/*.pki

        Args:
            machine (None|string) : hostname of the machine, used for reading/saving unique data
        """
        print("Configuring Vault")
        client = self.connect(VAULT_TOKEN)

        # Audit Backend
        audit_options = {
            'low_raw': 'True',
        }
        try:
            client.enable_audit_backend('syslog', options=audit_options)
        except hvac.exceptions.InvalidRequest as ex:
            print("audit_backend already created.")

        # Policies
        provisioner_policies = []
        path = os.path.join(POLICY_DIR, "*.hcl")
        for policy in glob.glob(path):
            name = os.path.basename(policy).split('.')[0]
            with open(policy, 'r') as fh:
                client.set_policy(name, fh.read())
            # Add every policy to the provisioner, as it has to have the
            # superset of any policies that it will provision
            provisioner_policies.append(name)

        # AWS Authentication Backend
        #Enable AWS auth in Vault
        if 'aws' not in client.list_auth_backends():
            try:
                client.enable_auth_backend('aws')
            except Exception as e:
                raise Exception("Error while enabling auth back end: " + e)
        else:
            print("aws auth backend already created.")

        #Define policies and arn
        policies = [p for p in provisioner_policies if p not in ('provisioner',)]
        arn = 'arn:aws:iam::{}:instance-profile/'.format(os.environ['AWS_ACCOUNT'])
        #TODO: Find a temporary way of storing the aws account number
        #For each policy configure the policies on a role of the same name
        for policy in policies:
            client.write('/auth/aws/role/' + policy, auth_type='ec2', bound_iam_instance_profile_arn= arn + policy, policies=policy)
            print('Successful write to aws/role/' + policy)

        # AWS Secret Backend
        if 'aws' not in client.list_secret_backends():
            try:
                client.enable_secret_backend('aws')
            except Exception as e:
                raise Exception('Error while enabling secret back end: ' + e)
        else:
            print("aws secret backend already created.")

        path = os.path.join(POLICY_DIR, "*.iam")
        for iam in glob.glob(path):
            name = os.path.basename(iam).split('.')[0]
            with open(iam, 'r') as fh:
                # if we json parse the file first we can use the duplicate key trick for comments
                client.write("aws/roles/" + name, policy = fh.read())

        # PKI Backend
        """
        if True: # Disabled until we either have a CA cert or can generate a CA
            print("Vault PKI cert file does not exist, skipping configuration of PKI secret backend")
        else:
            client.enable_secret_backend('pki')
            # Generate a self signed certificate for CA
            print("Generating self signed CA")
            response = client.write("pki/root/generate/internal", common_name=aws_creds["domain"])
            with open(get_path(machine, "ca.pem"), 'w') as fh:
                fh.write(response["data"]["certificate"])

            # Should we configure CRL?

            path = os.path.join(_CURRENT_DIR, "policies", "*.pki")
            for pki in glob.glob(path):
                name = os.path.basename(pki).split('.')[0]
                with open(pki, 'r') as fh:
                    keys = json.load(fh)
                    client.write("aws/roles/" + name, **keys)
        """

    def unseal(self):
        """Unseal a sealed Vault. Connect using get_client() and if the Vault is
        not sealed read all of the keys defined by VAULT_KEY and unseal.

        If there are not enough keys to completely unseal the Vault, print a
        status message about how many more keys are required to finish the
        process.
        """

        client = self.connect()
        if not client.is_sealed():
            print("Vault is already unsealed")
            return 0

        key_file = self.path(VAULT_KEY)
        keys = []
        for f in glob.glob(key_file + "*"):
            with open(f, "r") as fh:
                keys.append(fh.read())

        if len(keys) == 0:
            raise Exception("Could not locate any key files, not unsealing")

        res = client.unseal_multi(keys)
        if res['sealed']:
            p = res['progress']
            t = res['t']
            print("Vault partly unsealed, {} of {} needed keys entered".format(p,t))
            print("Enter {} more keys to finish unsealing the vault". format(t-p))
            return (t-p)
        else:
            print("Vault unsealed")
            return 0

    def seal(self):
        """Seal an unsealed Vault. Connect using get_client(True) and if the Vault
        is unsealed, seal it.

        Used to quickly protect a Vault without having to stop the Vault service
        on a protected VM.
        """

        client = self.connect(VAULT_TOKEN)
        if client.is_sealed():
            print("Vault is already sealed")
            return

        client.seal()
        print("Vault is sealed")

    def status(self):
        """Print the status of a Vault. Connect using get_client(True) and print
        the status of the following items (if available):
         * Initializing status
         * Seal status
         * Key status
         * High Availability status
         * Secret backends
         * Policies
         * Audit backends
         * Auth backends
        """

        client = self.connect()
        if not client.is_initialized():
            print("Vault is not initialized")
            return
        else:
            print("Vault is initialized")

        if client.is_sealed():
            print("Vault is sealed")
            print(client.seal_status)
            return
        else:
            print("Vault is unsealed")

        # read in the Vault access token
        client = self.connect(VAULT_TOKEN)
        print()
        print("Key Status")
        print(json.dumps(client.key_status))

        print()
        print("HA Status")
        print(json.dumps(client.ha_status))

        print()
        print("Secret Backends")
        print(json.dumps(client.list_secret_backends(), indent=True))

        print()
        print("Policies")
        print(json.dumps(client.list_policies()))

        print()
        print("Audit Backends")
        print(json.dumps(client.list_audit_backends(), indent=True))

        print()
        print("Auth Backends")
        print(json.dumps(client.list_auth_backends(), indent=True))

    def provision(self, policy):
        """Create a new Vault access token.

        Args:
            policy (string) : Name of the policy to attach to the new token

        Returns:
            (string) : String containing the new Vault token
        """
        client = self.connect(VAULT_TOKEN)
        token = client.create_token(policies = [policy])
        return token["auth"]["client_token"]

    def revoke(self, token):
        """Revoke a Vault access token.

        Args:
            token (string) : String containing the Vault token to revoke
        """
        client = self.connect(VAULT_TOKEN)
        client.revoke_token(token)

    def revoke_secret(self, lease_id):
        """Revoke a Vault lease

        Args:
            lease_id (string) : String containing the Vault lease id to revoke
        """
        client = self.connect(VAULT_TOKEN)
        client.revoke_secret(lease_id)

    def revoke_secret_prefix(self, prefix):
        """Revoke a Vault secret by prefix

        Args:
            prefix (string) : String containing the Vault secret prefix to revoke
        """
        client = self.connect(VAULT_TOKEN)
        client.revoke_secret_prefix(prefix)

    def write(self, path, **kwargs):
        """A generic method for writing data into Vault.

            Note: vault-write will override any data already existing at path.
                  There is vault-update that will update data at path instead.

        Args:
            path (string) : Vault path to write data to
            kwargs : Key value pairs to store at path
        """
        client = self.connect(VAULT_TOKEN)
        client.write(path, **kwargs)

    def update(self, path, **kwargs):
        """A generic method for adding/updating data to/in Vault.

        Args:
            path (string) : Vault path to write data to
            kwargs : Key value pairs to store at path
        """
        client = self.connect(VAULT_TOKEN)

        existing = client.read(path)
        if existing is None:
            existing = {}
        else:
            existing = existing["data"]

        existing.update(kwargs)

        client.write(path, **existing)

    def read(self, path):
        """A generic method for reading data from Vault.

        Args:
            path (string) : Vault path to read data from
        """
        client = self.connect(VAULT_TOKEN)
        return client.read(path)

    def delete(self, path):
        """A generic method for deleting data from Vault.

        Args:
            path (string) : Vault path to delete all data from
        """
        client = self.connect(VAULT_TOKEN)
        client.delete(path)

    def export(self, path):
        """A generic method for reading all of the paths and keys from Vault.

        Args:
            path (string) : Vault path to dump data from
        """
        if path[-1] != '/':
            path += '/'

        rtn = {}

        # DP NOTE: not using self.read becuase of the different token needed
        client = self.connect(VAULT_TOKEN)
        results =  client.read(path[:-1])
        if results is not None:
            rtn[path[:-1]] = results['data']

        results = client.list(path)
        for key in results['data']['keys']:
            key = path + key
            if key[-1] == '/':
                data = self.export(key)
                rtn.update(data)
            else:
                # DP NOTE: This will currently do a duplicate read if
                #          data is stored at key and in paths under
                #          key
                #          To prevent this, check to see if (key + '/') is in keys
                data = self.read(key)
                rtn[key] = data['data']

        return rtn

    def import_(self, exported, update=False):
        """A generic method for writing / updating data in multiple paths in Vault.

        Args:
            exported (dict): Dict of Vault path and dict of key / values to store at the path
            update (bool): If an Update should be done or if a Write should be done
        """
        for path in exported:
            kv = exported[path]
            fn = self.update if update else self.write
            fn(path, **kv)
<|endoftext|>


<commit_before>#!/usr/bin/python

<commit_msg>added example service call to libclient<commit_after>#!/usr/bin/python

import dbus

bus = dbus.SessionBus()
proxy_obj = bus.get_object('org.freedesktop.ipcf.connectionmanager.msn', '/org/freedesktop/ipcf/connectionmanager/msn')
dbus_iface = dbus.Interface(proxy_obj, 'org.freedesktop.ipcf.connectionmanager')

print dbus_iface.make_connection('msn','foo', {'foo':'bar'})

<|endoftext|>


<commit_before>import os
import string
import random
import hashlib
import falcon
import redis

ALPHABET = string.ascii_letters + string.digits
SHORT_LINK_FORMAT = "{}/l/".format(os.environ['WEBHOOK_HOSTNAME'])

redis_client = redis.from_url(os.environ.get('REDIS_URL', 'redis://localhost:6379/0'))

class GetLinkResource:
    def on_get(self, req, resp, query):
        base_key = 'core:shrt:link:{}:'.format(query)
        full_link = redis_client.get(base_key + 'url')
        if full_link is None:
            raise falcon.HTTPMovedPermanently('http://sheldon.ai/')
        redis_client.incr(base_key + 'visits')
        user_hash = redis_client.get(base_key + 'user')
        print('Uber user hash:', user_hash)
        if user_hash:
            resp.set_cookie('user', user_hash.decode('utf-8'), max_age=60 * 60 * 24)
        resp.status = falcon.HTTP_301
        resp.location = full_link.decode('utf-8')

def short_user_link(u_id, link, code_size=11):
    # Generate hash for user
    sha_hash = hashlib.sha256()
    sha_hash.update(str(u_id).encode())
    sha_hash.update(os.environ['BOT_SECRET'].encode())
    user_hash = sha_hash.hexdigest()
    user_hash_key = 'core:shrt:hash:{}'.format(user_hash)
    redis_client.set(user_hash_key, u_id)

    # Save link
    unique = False
    while not unique:
        code = ''
        for _ in range(code_size):
            code += random.choice(ALPHABET)
        link_key = 'core:shrt:link:{}:url'.format(code)
        if not redis_client.exists(link):
            unique = True
    redis_client.set(link_key, link)

    link_hash_key = 'core:shrt:link:{}:user'.format(code)
    redis_client.set(link_hash_key, user_hash)

    return SHORT_LINK_FORMAT + code
<commit_msg>Add debug print<commit_after>import os
import string
import random
import hashlib
import falcon
import redis

ALPHABET = string.ascii_letters + string.digits
SHORT_LINK_FORMAT = "{}/l/".format(os.environ['WEBHOOK_HOSTNAME'])

redis_client = redis.from_url(os.environ.get('REDIS_URL', 'redis://localhost:6379/0'))

class GetLinkResource:
    def on_get(self, req, resp, query):
        base_key = 'core:shrt:link:{}:'.format(query)
        full_link = redis_client.get(base_key + 'url')
        if full_link is None:
            raise falcon.HTTPMovedPermanently('http://sheldon.ai/')
        redis_client.incr(base_key + 'visits')
        user_hash = redis_client.get(base_key + 'user')
        print('Uber user hash:', user_hash)
        if user_hash:
            resp.set_cookie('user', user_hash.decode('utf-8'), max_age=60 * 60 * 24)
        resp.status = falcon.HTTP_301
        resp.location = full_link.decode('utf-8')
        print(resp.headers)

def short_user_link(u_id, link, code_size=11):
    # Generate hash for user
    sha_hash = hashlib.sha256()
    sha_hash.update(str(u_id).encode())
    sha_hash.update(os.environ['BOT_SECRET'].encode())
    user_hash = sha_hash.hexdigest()
    user_hash_key = 'core:shrt:hash:{}'.format(user_hash)
    redis_client.set(user_hash_key, u_id)

    # Save link
    unique = False
    while not unique:
        code = ''
        for _ in range(code_size):
            code += random.choice(ALPHABET)
        link_key = 'core:shrt:link:{}:url'.format(code)
        if not redis_client.exists(link):
            unique = True
    redis_client.set(link_key, link)

    link_hash_key = 'core:shrt:link:{}:user'.format(code)
    redis_client.set(link_hash_key, user_hash)

    return SHORT_LINK_FORMAT + code
<|endoftext|>


<commit_before>"""This module provides general utility methods."""
from collections import ChainMap
from contextlib import contextmanager
from functools import lru_cache, wraps
import locale
import logging
from numbers import Number
import os
import re
import sublime
import subprocess
import time
import threading

MYPY = False
if MYPY:
    from typing import List, MutableMapping, Optional

logger = logging.getLogger(__name__)

STREAM_STDOUT = 1
STREAM_STDERR = 2
STREAM_BOTH = STREAM_STDOUT + STREAM_STDERR

ANSI_COLOR_RE = re.compile(r'\033\[[0-9;]*m')

def printf(*args):
    """Print args to the console, prefixed by the plugin name."""
    print('SublimeLinter: ', end='')
    for arg in args:
        print(arg, end=' ')
    print()

@contextmanager
def print_runtime(message):
    start_time = time.perf_counter()
    yield
    end_time = time.perf_counter()
    duration = round((end_time - start_time) * 1000)
    thread_name = threading.current_thread().name[0]
    print('{} took {}ms [{}]'.format(message, duration, thread_name))

def show_message(message, window=None):
    if window is None:
        window = sublime.active_window()
    window.run_command("sublime_linter_display_panel", {"msg": message})

def clear_message():
    window = sublime.active_window()
    window.run_command("sublime_linter_remove_panel")

def distinct_until_buffer_changed(method):
    # Sublime has problems to hold the distinction between buffers and views.
    # It usually emits multiple identical events if you have multiple views
    # into the same buffer.
    last_call = None

    @wraps(method)
    def wrapper(self, view):
        nonlocal last_call

        this_call = (view.buffer_id(), view.change_count())
        if this_call == last_call:
            return

        last_call = this_call
        method(self, view)

    return wrapper

def canonical_filename(view):
    return (
        os.path.basename(view.file_name()) if view.file_name()
        else '<untitled {}>'.format(view.buffer_id())
    )

def get_filename(view):
    # type: (sublime.View) -> str
    return view.file_name() or '<untitled {}>'.format(view.buffer_id())

def get_syntax(view):
    """
    Return the view's syntax.

    or the syntax it is mapped to in the "syntax_map" setting.
    """
    syntax_re = re.compile(r'(?i)/([^/]+)\.(?:tmLanguage|sublime-syntax)$')
    view_syntax = view.settings().get('syntax') or ''
    mapped_syntax = ''

    if view_syntax:
        match = syntax_re.search(view_syntax)

        if match:
            view_syntax = match.group(1).lower()
            from .persist import settings
            mapped_syntax = settings.get(
                'syntax_map', {}).get(view_syntax, '').lower()
        else:
            view_syntax = ''

    return mapped_syntax or view_syntax

def is_lintable(view):
    """
    Return true when a view is not lintable, e.g. scratch, read_only, etc.

    There is a bug (or feature) in the current ST3 where the Find panel
    is not marked scratch but has no window.

    There is also a bug where settings files opened from within .sublime-package
    files are not marked scratch during the initial on_modified event, so we have
    to check that a view with a filename actually exists on disk if the file
    being opened is in the Sublime Text packages directory.

    """
    if (
        not view.window() or
        view.is_scratch() or
        view.is_read_only() or
        view.settings().get("repl") or
        view.settings().get('is_widget')
    ):
        return False

    filename = view.file_name()
    if (
        filename and
        filename.startswith(sublime.packages_path() + os.path.sep) and
        not os.path.exists(filename)
    ):
        return False

    return True

# file/directory/environment utils

@lru_cache(maxsize=1)  # print once every time the path changes
def debug_print_env(path):
    import textwrap
    logger.info('PATH:\n{}'.format(textwrap.indent(path.replace(os.pathsep, '\n'), '    ')))

def create_environment():
    # type: () -> MutableMapping[str, str]
    """Return a dict with os.environ augmented with a better PATH.

    Platforms paths are added to PATH by getting the "paths" user settings
    for the current platform.
    """
    return ChainMap({'PATH': get_augmented_path()}, os.environ)

def get_augmented_path():
    # type: () -> str
    from . import persist

    paths = [
        os.path.expanduser(path)
        for path in persist.settings.get('paths', {}).get(sublime.platform(), [])
    ]  # type: List[str]

    augmented_path = os.pathsep.join(paths + [os.environ['PATH']])
    if logger.isEnabledFor(logging.INFO):
        debug_print_env(augmented_path)
    return augmented_path

def can_exec(path):
    """Return whether the given path is a file and is executable."""
    return os.path.isfile(path) and os.access(path, os.X_OK)

def which(cmd):
    """Return the full path to an executable searching PATH."""
    for path in find_executables(cmd):
        return path

    return None

def find_executables(executable):
    """Yield full paths to given executable."""
    env = create_environment()

    for base in env.get('PATH', '').split(os.pathsep):
        path = os.path.join(os.path.expanduser(base), executable)

        # On Windows, if path does not have an extension, try .exe, .cmd, .bat
        if sublime.platform() == 'windows' and not os.path.splitext(path)[1]:
            for extension in ('.exe', '.cmd', '.bat'):
                path_ext = path + extension

                if can_exec(path_ext):
                    yield path_ext
        elif can_exec(path):
            yield path

    return None

# popen utils

def check_output(cmd, cwd=None):
    """Short wrapper around subprocess.check_output."""
    logger.info('Running `{}`'.format(' '.join(cmd)))
    env = create_environment()

    try:
        output = subprocess.check_output(
            cmd, env=env, cwd=cwd,
            stderr=subprocess.STDOUT,
            startupinfo=create_startupinfo()
        )
    except Exception as err:
        logger.warning(
            "Executing `{}` failed\n  {}".format(' '.join(cmd), str(err))
        )
        raise
    else:
        return process_popen_output(output)

class popen_output(str):
    """Hybrid of a Popen process and its output.

    Small compatibility layer: It is both the decoded output
    as str and partially the Popen object.
    """

    stdout = ''  # type: Optional[str]
    stderr = ''  # type: Optional[str]
    combined_output = ''

    def __new__(cls, proc, stdout, stderr):
        if stdout is not None:
            stdout = process_popen_output(stdout)
        if stderr is not None:
            stderr = process_popen_output(stderr)

        combined_output = ''.join(filter(None, [stdout, stderr]))

        rv = super().__new__(cls, combined_output)  # type: ignore
        rv.combined_output = combined_output
        rv.stdout = stdout
        rv.stderr = stderr
        rv.proc = proc
        rv.pid = proc.pid
        rv.returncode = proc.returncode
        return rv

def process_popen_output(output):
    # bytes -> string   --> universal newlines
    output = decode(output).replace('\r\n', '\n').replace('\r', '\n')
    return ANSI_COLOR_RE.sub('', output)

def decode(bytes):
    """
    Decode and return a byte string using utf8, falling back to system's encoding if that fails.

    So far we only have to do this because javac is so utterly hopeless it uses CP1252
    for its output on Windows instead of UTF8, even if the input encoding is specified as UTF8.
    Brilliant! But then what else would you expect from Oracle?

    """
    if not bytes:
        return ''

    try:
        return bytes.decode('utf8')
    except UnicodeError:
        return bytes.decode(locale.getpreferredencoding(), errors='replace')

def create_startupinfo():
    if os.name == 'nt':
        info = subprocess.STARTUPINFO()
        info.dwFlags |= subprocess.STARTF_USESTDHANDLES | subprocess.STARTF_USESHOWWINDOW
        info.wShowWindow = subprocess.SW_HIDE
        return info

    return None

def get_creationflags():
    if os.name == 'nt':
        return subprocess.CREATE_NEW_PROCESS_GROUP

    return 0

# misc utils

def convert_type(value, type_value, sep=None, default=None):
    """
    Convert value to the type of type_value.

    If the value cannot be converted to the desired type, default is returned.
    If sep is not None, strings are split by sep (plus surrounding whitespace)
    to make lists/tuples, and tuples/lists are joined by sep to make strings.
    """
    if type_value is None or isinstance(value, type(type_value)):
        return value

    if isinstance(value, str):
        if isinstance(type_value, (tuple, list)):
            if sep is None:
                return [value]
            else:
                if value:
                    return re.split(r'\s*{}\s*'.format(sep), value)
                else:
                    return []
        elif isinstance(type_value, Number):
            return float(value)
        else:
            return default

    if isinstance(value, Number):
        if isinstance(type_value, str):
            return str(value)
        elif isinstance(type_value, (tuple, list)):
            return [value]
        else:
            return default

    if isinstance(value, (tuple, list)):
        if isinstance(type_value, str):
            return sep.join(value)
        else:
            return list(value)

    return default

def load_json(*segments, from_sl_dir=False):
    base_path = "Packages/SublimeLinter/" if from_sl_dir else ""
    full_path = base_path + "/".join(segments)
    return sublime.decode_value(sublime.load_resource(full_path))

def get_sl_version():
    try:
        metadata = load_json("package-metadata.json", from_sl_dir=True)
        return metadata.get("version")
    except Exception:
        return "unknown"
<commit_msg>Use `get_augmented_path` in `find_executables`<commit_after>"""This module provides general utility methods."""
from collections import ChainMap
from contextlib import contextmanager
from functools import lru_cache, wraps
import locale
import logging
from numbers import Number
import os
import re
import sublime
import subprocess
import time
import threading

MYPY = False
if MYPY:
    from typing import List, MutableMapping, Optional

logger = logging.getLogger(__name__)

STREAM_STDOUT = 1
STREAM_STDERR = 2
STREAM_BOTH = STREAM_STDOUT + STREAM_STDERR

ANSI_COLOR_RE = re.compile(r'\033\[[0-9;]*m')

def printf(*args):
    """Print args to the console, prefixed by the plugin name."""
    print('SublimeLinter: ', end='')
    for arg in args:
        print(arg, end=' ')
    print()

@contextmanager
def print_runtime(message):
    start_time = time.perf_counter()
    yield
    end_time = time.perf_counter()
    duration = round((end_time - start_time) * 1000)
    thread_name = threading.current_thread().name[0]
    print('{} took {}ms [{}]'.format(message, duration, thread_name))

def show_message(message, window=None):
    if window is None:
        window = sublime.active_window()
    window.run_command("sublime_linter_display_panel", {"msg": message})

def clear_message():
    window = sublime.active_window()
    window.run_command("sublime_linter_remove_panel")

def distinct_until_buffer_changed(method):
    # Sublime has problems to hold the distinction between buffers and views.
    # It usually emits multiple identical events if you have multiple views
    # into the same buffer.
    last_call = None

    @wraps(method)
    def wrapper(self, view):
        nonlocal last_call

        this_call = (view.buffer_id(), view.change_count())
        if this_call == last_call:
            return

        last_call = this_call
        method(self, view)

    return wrapper

def canonical_filename(view):
    return (
        os.path.basename(view.file_name()) if view.file_name()
        else '<untitled {}>'.format(view.buffer_id())
    )

def get_filename(view):
    # type: (sublime.View) -> str
    return view.file_name() or '<untitled {}>'.format(view.buffer_id())

def get_syntax(view):
    """
    Return the view's syntax.

    or the syntax it is mapped to in the "syntax_map" setting.
    """
    syntax_re = re.compile(r'(?i)/([^/]+)\.(?:tmLanguage|sublime-syntax)$')
    view_syntax = view.settings().get('syntax') or ''
    mapped_syntax = ''

    if view_syntax:
        match = syntax_re.search(view_syntax)

        if match:
            view_syntax = match.group(1).lower()
            from .persist import settings
            mapped_syntax = settings.get(
                'syntax_map', {}).get(view_syntax, '').lower()
        else:
            view_syntax = ''

    return mapped_syntax or view_syntax

def is_lintable(view):
    """
    Return true when a view is not lintable, e.g. scratch, read_only, etc.

    There is a bug (or feature) in the current ST3 where the Find panel
    is not marked scratch but has no window.

    There is also a bug where settings files opened from within .sublime-package
    files are not marked scratch during the initial on_modified event, so we have
    to check that a view with a filename actually exists on disk if the file
    being opened is in the Sublime Text packages directory.

    """
    if (
        not view.window() or
        view.is_scratch() or
        view.is_read_only() or
        view.settings().get("repl") or
        view.settings().get('is_widget')
    ):
        return False

    filename = view.file_name()
    if (
        filename and
        filename.startswith(sublime.packages_path() + os.path.sep) and
        not os.path.exists(filename)
    ):
        return False

    return True

# file/directory/environment utils

@lru_cache(maxsize=1)  # print once every time the path changes
def debug_print_env(path):
    import textwrap
    logger.info('PATH:\n{}'.format(textwrap.indent(path.replace(os.pathsep, '\n'), '    ')))

def create_environment():
    # type: () -> MutableMapping[str, str]
    """Return a dict with os.environ augmented with a better PATH.

    Platforms paths are added to PATH by getting the "paths" user settings
    for the current platform.
    """
    return ChainMap({'PATH': get_augmented_path()}, os.environ)

def get_augmented_path():
    # type: () -> str
    from . import persist

    paths = [
        os.path.expanduser(path)
        for path in persist.settings.get('paths', {}).get(sublime.platform(), [])
    ]  # type: List[str]

    augmented_path = os.pathsep.join(paths + [os.environ['PATH']])
    if logger.isEnabledFor(logging.INFO):
        debug_print_env(augmented_path)
    return augmented_path

def can_exec(path):
    """Return whether the given path is a file and is executable."""
    return os.path.isfile(path) and os.access(path, os.X_OK)

def which(cmd):
    """Return the full path to an executable searching PATH."""
    for path in find_executables(cmd):
        return path

    return None

def find_executables(executable):
    """Yield full paths to given executable."""
    for base in get_augmented_path().split(os.pathsep):
        path = os.path.join(os.path.expanduser(base), executable)

        # On Windows, if path does not have an extension, try .exe, .cmd, .bat
        if sublime.platform() == 'windows' and not os.path.splitext(path)[1]:
            for extension in ('.exe', '.cmd', '.bat'):
                path_ext = path + extension

                if can_exec(path_ext):
                    yield path_ext
        elif can_exec(path):
            yield path

    return None

# popen utils

def check_output(cmd, cwd=None):
    """Short wrapper around subprocess.check_output."""
    logger.info('Running `{}`'.format(' '.join(cmd)))
    env = create_environment()

    try:
        output = subprocess.check_output(
            cmd, env=env, cwd=cwd,
            stderr=subprocess.STDOUT,
            startupinfo=create_startupinfo()
        )
    except Exception as err:
        logger.warning(
            "Executing `{}` failed\n  {}".format(' '.join(cmd), str(err))
        )
        raise
    else:
        return process_popen_output(output)

class popen_output(str):
    """Hybrid of a Popen process and its output.

    Small compatibility layer: It is both the decoded output
    as str and partially the Popen object.
    """

    stdout = ''  # type: Optional[str]
    stderr = ''  # type: Optional[str]
    combined_output = ''

    def __new__(cls, proc, stdout, stderr):
        if stdout is not None:
            stdout = process_popen_output(stdout)
        if stderr is not None:
            stderr = process_popen_output(stderr)

        combined_output = ''.join(filter(None, [stdout, stderr]))

        rv = super().__new__(cls, combined_output)  # type: ignore
        rv.combined_output = combined_output
        rv.stdout = stdout
        rv.stderr = stderr
        rv.proc = proc
        rv.pid = proc.pid
        rv.returncode = proc.returncode
        return rv

def process_popen_output(output):
    # bytes -> string   --> universal newlines
    output = decode(output).replace('\r\n', '\n').replace('\r', '\n')
    return ANSI_COLOR_RE.sub('', output)

def decode(bytes):
    """
    Decode and return a byte string using utf8, falling back to system's encoding if that fails.

    So far we only have to do this because javac is so utterly hopeless it uses CP1252
    for its output on Windows instead of UTF8, even if the input encoding is specified as UTF8.
    Brilliant! But then what else would you expect from Oracle?

    """
    if not bytes:
        return ''

    try:
        return bytes.decode('utf8')
    except UnicodeError:
        return bytes.decode(locale.getpreferredencoding(), errors='replace')

def create_startupinfo():
    if os.name == 'nt':
        info = subprocess.STARTUPINFO()
        info.dwFlags |= subprocess.STARTF_USESTDHANDLES | subprocess.STARTF_USESHOWWINDOW
        info.wShowWindow = subprocess.SW_HIDE
        return info

    return None

def get_creationflags():
    if os.name == 'nt':
        return subprocess.CREATE_NEW_PROCESS_GROUP

    return 0

# misc utils

def convert_type(value, type_value, sep=None, default=None):
    """
    Convert value to the type of type_value.

    If the value cannot be converted to the desired type, default is returned.
    If sep is not None, strings are split by sep (plus surrounding whitespace)
    to make lists/tuples, and tuples/lists are joined by sep to make strings.
    """
    if type_value is None or isinstance(value, type(type_value)):
        return value

    if isinstance(value, str):
        if isinstance(type_value, (tuple, list)):
            if sep is None:
                return [value]
            else:
                if value:
                    return re.split(r'\s*{}\s*'.format(sep), value)
                else:
                    return []
        elif isinstance(type_value, Number):
            return float(value)
        else:
            return default

    if isinstance(value, Number):
        if isinstance(type_value, str):
            return str(value)
        elif isinstance(type_value, (tuple, list)):
            return [value]
        else:
            return default

    if isinstance(value, (tuple, list)):
        if isinstance(type_value, str):
            return sep.join(value)
        else:
            return list(value)

    return default

def load_json(*segments, from_sl_dir=False):
    base_path = "Packages/SublimeLinter/" if from_sl_dir else ""
    full_path = base_path + "/".join(segments)
    return sublime.decode_value(sublime.load_resource(full_path))

def get_sl_version():
    try:
        metadata = load_json("package-metadata.json", from_sl_dir=True)
        return metadata.get("version")
    except Exception:
        return "unknown"
<|endoftext|>


<commit_before>#!/usr/bin/python

"""lintblame!!!"""

import os
import sys
import re
import time
import subprocess
import datetime
from collections import defaultdict

COLORS = {
    'header': '\033[95m',
    'blue': '\033[94m',
    'green': '\033[92m',
    'warn': '\033[93m',
    'fail': '\033[91m',
    'bold': '\033[1m',
    'white': '\033[37m',
    'endc': '\033[0m',
}

PYLINT_COLORS = {
    'C': 'bold',
    'F': 'fail',
    'E': 'fail',
    'W': 'white',
}

REXES = {
    'pep8': re.compile(r'\w+:(\d+):(\d+):\s(\w+)\s(.+)$', re.MULTILINE),
    'pylint': re.compile(r'^(\w):\s+(\d+),\s*(\d+):\s(.+)$', re.MULTILINE),
}

class Environment(object):
    """Holds certain environmental values."""
    def __init__(self):
        self._git_name = False

    @property
    def git_name(self):
        """Returns name from .gitconfig or None."""

        if self._git_name is False:
            path = os.path.expanduser('~/.gitconfig')
            if os.path.isfile(path):
                with open(path, 'r') as open_f:
                    contents = open_f.read()
                    match = re.search(r'name = (.+)$', contents, re.MULTILINE)
                    if match:
                        self._git_name = match.group(1).strip()
                    else:
                        self._git_name = None
            else:
                self._git_name = None
        return self._git_name

ENV = Environment()

class Problem(object):
    """Represents an issue identified by a linter."""
    def __init__(self, lint_type, line, column, problem_code, message):
        self.lint_type = lint_type
        self.line = int(line)
        self.column = column
        self.problem_code = problem_code
        self.message = message

    def __str__(self):
        return '{}, {}: [{}] {}'.format(
            self.line,
            self.column,
            self.problem_code,
            self.message
        )

class TargetFile(object):
    """Represents a code file, and holds its linting issues."""
    blame_name_rex = re.compile(r'\(([\w\s]+)\d{4}')

    def __init__(self, path):
        self.path = path
        self._problems = defaultdict(list)
        self.lines = []

        try:
            self.blame_lines = subprocess.check_output(
                ['git', 'blame', self.path]
            ).splitlines()
        except subprocess.CalledProcessError:
            sys.exit("Unable to blame {}".format(self.path))

    def set_contents(self, contents):
        """Sets contents of the file this object references."""
        self.lines = contents.splitlines()

    @property
    def problems(self):
        """Returns generator that iterates over line, problem."""
        return self._problems.iteritems()

    @property
    def has_problems(self):
        return len(self._problems) > 0

    def add_problem(self, problem):
        self._problems[problem.line].append(problem)

    def author(self, lineno):
        match = self.blame_name_rex.search(self.blame_lines[int(lineno) - 1])
        if match:
            return match.group(1).strip()
        return None

def color(key, text):
    """Returns text wrapped with color."""
    if not key:
        return text
    return ''.join([COLORS[key], str(text), COLORS['endc']])

def validate_file_arg():
    """Returns file path if there's a valid file argument. Exits otherwise."""
    if len(sys.argv) < 2:
        sys.exit("Please provide an argument")

    target_file = os.path.join(os.getcwd(), sys.argv[1])
    if not target_file.endswith('.py'):
        sys.exit("{} is not a python file".format(target_file))
    return target_file

def pylint(path):
    """Returns pylint results."""
    proc = subprocess.Popen(
        'pylint --output-format=text {}'.format(path),
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        shell=True
    )
    out, err = proc.communicate()
    if err:
        sys.exit(err)
    return out

def pep8(path):
    """Returns pep8 results."""
    proc = subprocess.Popen(
        ['pep8', path],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE
    )
    out, err = proc.communicate()
    if err:
        sys.exit(err)
    return out

def pylint_problems(path):
    results = pylint(path)
    for code, line, col, msg in REXES['pylint'].findall(results):
        yield Problem('pylint', line, col, code, msg)

def pep8_problems(path):
    results = pep8(path)
    for line, col, code, msg in REXES['pep8'].findall(results):
        yield Problem('pep8', line, col, code, msg)

def print_results(target_file):
    """Prints formatted results."""
    print(color('header', target_file.path))

    if not target_file.has_problems:
        print(color('bold', 'All clean!'))
    else:
        for line, problems in target_file.problems:
            author = target_file.author(line)
            author_color = 'blue'
            if author == ENV.git_name:
                author_color = 'warn'

            print('{}: [{}] {}'.format(
                color('bold', line),
                color(author_color, author),
                target_file.lines[line - 1].strip(),
            ))

            for problem in problems:
                color_key = PYLINT_COLORS.get(problem.problem_code)
                print('{space} [{code}] {msg}'.format(
                    space=' '.join(['' for i in range(len(str(line)) + 2)]),
                    code=problem.problem_code,
                    msg=color('bold', problem.message)
                ))
            print
    print

def clear():
    os.system('cls' if os.name == 'nt' else 'clear')

def get_git_path():
    return subprocess.check_output(
        ['git', 'rev-parse', '--show-toplevel']
    ).strip()

def get_branch_files():
    changed_files = subprocess.check_output(['git', 'diff', '--name-only'])
    branch_files = subprocess.check_output(
        ['git', 'diff', '--name-only', 'master..HEAD']
    )
    combined = set(changed_files.splitlines() + branch_files.splitlines())
    return [f.strip() for f in combined if f.endswith('py')]

def get_target_files():
    if '--branch' in sys.argv or '-b' in sys.argv:
        files = get_branch_files()
    else:
        if os.path.isdir(sys.argv[1]):
            files = [f for f in os.listdir(sys.argv[1]) if f.endswith('.py')]
        else:
            files = [validate_file_arg()]
    top_path = get_git_path()
    return [os.path.join(top_path, f) for f in files]

def get_additional_files(current_files):
    target_files = get_target_files()
    return (
        [f for f in target_files if f not in current_files],
        [f for f in current_files if f not in target_files],
    )

def run(files):
    start = datetime.datetime.now()
    target_files = []
    for path in files:
        target = TargetFile(path)
        with open(path, 'r') as open_f:
            target.set_contents(open_f.read())
        for problem in pylint_problems(path):
            target.add_problem(problem)
        for problem in pep8_problems(path):
            target.add_problem(problem)
        target_files.append(target)

    clear()
    for target in target_files:
        print_results(target)

    duration = datetime.datetime.now() - start
    print('Finished linting in {} seconds'.format(
        float(duration.seconds) + float(duration.microseconds) / 1000000
    ))

def watch(files):
    modified = {f: os.path.getmtime(f) for f in files}
    loop_count = 0

    while True:
        should_run = False
        add_files, sub_files = get_additional_files(modified.keys())
        if add_files or sub_files:
            should_run = True
        for f in add_files:
            modified[f] = os.path.getmtime(f)
        for f in sub_files:
            modified.pop(f)

        if not should_run:
            for target_file in modified:
                mtime = os.path.getmtime(target_file)
                if mtime != modified.get(target_file):
                    modified[target_file] = mtime
                    should_run = True
                    break

        if should_run or loop_count == 0:
            if loop_count > 0:
                print('Refreshing...')
            run(modified.keys())
        loop_count += 1
        time.sleep(1.5)

if __name__ == '__main__':
    if len(sys.argv) == 1:
        sys.argv.append('--branch')
    watch(get_target_files())
<commit_msg>Tweaks<commit_after>#!/usr/bin/python

"""lintblame!!!"""

import os
import sys
import re
import time
import subprocess
import datetime
from collections import defaultdict

COLORS = {
    'header': '\033[95m',
    'blue': '\033[94m',
    'green': '\033[92m',
    'warn': '\033[93m',
    'fail': '\033[91m',
    'bold': '\033[1m',
    'white': '\033[37m',
    'endc': '\033[0m',
}

PYLINT_COLORS = {
    'C': 'bold',
    'F': 'fail',
    'E': 'fail',
    'W': 'white',
}

REXES = {
    'pep8': re.compile(r'\w+:(\d+):(\d+):\s(\w+)\s(.+)$', re.MULTILINE),
    'pylint': re.compile(r'^(\w):\s+(\d+),\s*(\d+):\s(.+)$', re.MULTILINE),
}

class Environment(object):
    """Holds certain environmental values."""
    def __init__(self):
        self._git_name = False

    @property
    def git_name(self):
        """Returns name from .gitconfig or None."""

        if self._git_name is False:
            path = os.path.expanduser('~/.gitconfig')
            if os.path.isfile(path):
                with open(path, 'r') as open_f:
                    contents = open_f.read()
                    match = re.search(r'name = (.+)$', contents, re.MULTILINE)
                    if match:
                        self._git_name = match.group(1).strip()
                    else:
                        self._git_name = None
            else:
                self._git_name = None
        return self._git_name

ENV = Environment()

class Issue(object):
    """Represents an issue identified by a linter."""
    def __init__(self, lint_type, line, column, issue_code, message):
        self.lint_type = lint_type
        self.line = int(line)
        self.column = column
        self.issue_code = issue_code
        self.message = message

    def __str__(self):
        return '{}, {}: [{}] {}'.format(
            self.line,
            self.column,
            self.issue_code,
            self.message
        )

class TargetFile(object):
    """Represents a code file, and holds its linting issues."""
    blame_name_rex = re.compile(r'\(([\w\s]+)\d{4}')

    def __init__(self, path):
        self.path = path
        self._issues = defaultdict(list)
        self.lines = []

        try:
            self.blame_lines = subprocess.check_output(
                ['git', 'blame', self.path]
            ).splitlines()
        except subprocess.CalledProcessError:
            sys.exit("Unable to blame {}".format(self.path))

    def set_contents(self, contents):
        """Sets contents of the file this object references."""
        self.lines = contents.splitlines()

    @property
    def issues(self):
        """Returns generator that iterates over line, issue."""
        return self._issues.iteritems()

    @property
    def has_issues(self):
        return len(self._issues) > 0

    def add_issue(self, issue):
        self._issues[issue.line].append(issue)

    def author(self, lineno):
        match = self.blame_name_rex.search(self.blame_lines[int(lineno) - 1])
        if match:
            return match.group(1).strip()
        return None

def color(key, text):
    """Returns text wrapped with color."""
    if not key:
        return text
    return ''.join([COLORS[key], str(text), COLORS['endc']])

def validate_file_arg():
    """Returns file path if there's a valid file argument. Exits otherwise."""
    if len(sys.argv) < 2:
        sys.exit("Please provide an argument")

    target_file = os.path.join(os.getcwd(), sys.argv[1])
    if not target_file.endswith('.py'):
        sys.exit("{} is not a python file".format(target_file))
    return target_file

def pylint(path):
    """Returns pylint results."""
    proc = subprocess.Popen(
        'pylint --output-format=text {}'.format(path),
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        shell=True
    )
    out, err = proc.communicate()
    if err:
        sys.exit(err)
    return out

def pep8(path):
    """Returns pep8 results."""
    proc = subprocess.Popen(
        ['pep8', path],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE
    )
    out, err = proc.communicate()
    if err:
        sys.exit(err)
    return out

def pylint_issues(path):
    results = pylint(path)
    for code, line, col, msg in REXES['pylint'].findall(results):
        yield Issue('pylint', line, col, code, msg)

def pep8_issues(path):
    results = pep8(path)
    for line, col, code, msg in REXES['pep8'].findall(results):
        yield Issue('pep8', line, col, code, msg)

def print_results(target_file):
    """Prints formatted results."""
    print(color('header', target_file.path))

    if not target_file.has_issues:
        print(color('bold', 'All clean!'))
    else:
        for line, issues in target_file.issues:
            author = target_file.author(line)
            author_color = 'blue'
            if author == ENV.git_name:
                author_color = 'warn'

            print('{}: [{}] {}'.format(
                color('bold', line),
                color(author_color, author),
                target_file.lines[line - 1].strip(),
            ))

            for issue in issues:
                color_key = PYLINT_COLORS.get(issue.issue_code)
                print('{space} [{code}] {msg}'.format(
                    space=' '.join(['' for i in range(len(str(line)) + 2)]),
                    code=issue.issue_code,
                    msg=color('bold', issue.message)
                ))
            print
    print

def clear():
    os.system('cls' if os.name == 'nt' else 'clear')

def get_git_path():
    return subprocess.check_output(
        ['git', 'rev-parse', '--show-toplevel']
    ).strip()

def get_branch_files():
    changed_files = subprocess.check_output(['git', 'diff', '--name-only'])
    branch_files = subprocess.check_output(
        ['git', 'diff', '--name-only', 'master..HEAD']
    )
    combined = set(changed_files.splitlines() + branch_files.splitlines())
    return [f.strip() for f in combined if f.endswith('py')]

def get_target_files():
    if '--branch' in sys.argv or '-b' in sys.argv:
        files = get_branch_files()
    else:
        if os.path.isdir(sys.argv[1]):
            files = [f for f in os.listdir(sys.argv[1]) if f.endswith('.py')]
        else:
            files = [validate_file_arg()]
    top_path = get_git_path()
    return [os.path.join(top_path, f) for f in files]

def get_additional_files(current_files):
    target_files = get_target_files()
    return (
        [f for f in target_files if f not in current_files],
        [f for f in current_files if f not in target_files],
    )

def run(files):
    start = datetime.datetime.now()
    target_files = []
    for path in files:
        target = TargetFile(path)
        with open(path, 'r') as open_f:
            target.set_contents(open_f.read())
        for issue in pylint_issues(path):
            target.add_issue(issue)
        for issue in pep8_issues(path):
            target.add_issue(issue)
        target_files.append(target)

    clear()
    for target in target_files:
        print_results(target)

    duration = datetime.datetime.now() - start
    print('Finished linting in {} seconds'.format(
        float(duration.seconds) + float(duration.microseconds) / 1000000
    ))

def watch(files):
    modified = {f: os.path.getmtime(f) for f in files}
    loop_count = 0

    while True:
        should_run = False
        add_files, sub_files = get_additional_files(modified.keys())
        if add_files or sub_files:
            should_run = True
        for f in add_files:
            modified[f] = os.path.getmtime(f)
        for f in sub_files:
            modified.pop(f)

        if not should_run:
            for target_file in modified:
                mtime = os.path.getmtime(target_file)
                if mtime != modified.get(target_file):
                    modified[target_file] = mtime
                    should_run = True
                    break

        if should_run or loop_count == 0:
            if loop_count > 0:
                print('Refreshing...')
            run(modified.keys())
        loop_count += 1
        time.sleep(1.5)

if __name__ == '__main__':
    if len(sys.argv) == 1:
        sys.argv.append('--branch')
    watch(get_target_files())
<|endoftext|>


<commit_before>from django.core.management.base import BaseCommand, CommandError
from salesforce.salesforce import Salesforce
from django.contrib.auth.models import User
from django.contrib.auth.models import Group
from django.contrib.auth.models import User, Group
from social.apps.django_app.default.models import DjangoStorage as SocialAuthStorage

class Command(BaseCommand):
    help = "Add user to faculty group if confirmed by salesforce"

    def add_arguments(self, parser):
        parser.add_argument('cms_id', nargs='?', type=int,default=None)
        parser.add_argument('--all', action='store_true', default=False)
    def handle(self, *args, **options):
        if options['all']:
            accounts_id = None
        else:
            social_user = SocialAuthStorage.user.objects.filter(user_id = options['cms_id'])
            accounts_id=social_user[0].uid
        with Salesforce() as sf:
            faculty_list = sf.faculty_status(accounts_id)
            for account_id in faculty_list:
                social_user = SocialAuthStorage.user.objects.filter(uid=account_id)
                if social_user:
                    cms_id = social_user[0].user_id
                    user = User.objects.get(pk=cms_id)
                    faculty_group = Group.objects.get_by_natural_key('Faculty')
                    user.groups.add(faculty_group)
                    user.save()
                    faculty_group.save()
        responce = self.style.SUCCESS("Successfully updated user faculty status")
        self.stdout.write(responce)

<commit_msg>variable update<commit_after>from django.core.management.base import BaseCommand, CommandError
from salesforce.salesforce import Salesforce
from django.contrib.auth.models import User
from django.contrib.auth.models import Group
from django.contrib.auth.models import User, Group
from social.apps.django_app.default.models import DjangoStorage as SocialAuthStorage

class Command(BaseCommand):
    help = "Add user to faculty group if confirmed by salesforce"

    def add_arguments(self, parser):
        parser.add_argument('user_id', nargs='?', type=int,default=None)
        parser.add_argument('--all', action='store_true', default=False)
    def handle(self, *args, **options):
        if options['all']:
            accounts_id = None
        else:
            social_user = SocialAuthStorage.user.objects.filter(user_id = options['user_id'])
            accounts_id=social_user[0].uid
        with Salesforce() as sf:
            faculty_list = sf.faculty_status(accounts_id)
            for account_id in faculty_list:
                social_user = SocialAuthStorage.user.objects.filter(uid=account_id)
                if social_user:
                    cms_id = social_user[0].user_id
                    user = User.objects.get(pk=cms_id)
                    faculty_group = Group.objects.get_by_natural_key('Faculty')
                    user.groups.add(faculty_group)
                    user.save()
                    faculty_group.save()
        responce = self.style.SUCCESS("Successfully updated user faculty status")
        self.stdout.write(responce)

<|endoftext|>


<commit_before>from __future__ import print_function
import sys
import os
import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn
import torchvision.transforms as transforms
from torch.autograd import Variable
import cv2
from sys import platform as sys_pf
from data import VOC_CLASSES as labelmap
import torch.utils.data as data
from data import BaseTransform
from ssd import build_ssd

trained_model = 'weights/ssd_300_VOC0712.pth'
net = build_ssd('test', 300, 21)    # initialize SSD
net.load_state_dict(torch.load(trained_model))
net.eval()
transform = BaseTransform(net.size, (104, 117, 123))
colors = [(255, 0, 0), (0, 255, 0), (0, 0, 255)]
font = cv2.FONT_HERSHEY_SIMPLEX

def predict(frame):
    height, width = frame.shape[:2]
    x = Variable(transform(frame).unsqueeze(0))
    y = net(x)  # forward pass
    detections = y.data
    # scale each detection back up to the image
    scale = torch.Tensor([width, height, width, height])
    for i in range(detections.size(1)):
        j = 0
        while detections[0, i, j, 0] >= 0.6:
            pt = (detections[0, i, j, 1:]*scale).cpu().numpy()
            cv2.rectangle(frame, (int(pt[0]), int(pt[1])), (int(pt[2]),
                          int(pt[3])), colors[i % 3], 2)
            cv2.putText(frame, labelmap[i-1], (int(pt[0]), int(pt[1])), font,
                        2, (255, 255, 255), 2, cv2.LINE_AA)
            j += 1
    return frame

video_capture = cv2.VideoCapture(0)
while True:
    if not video_capture.isOpened():
        print('Unable to load camera.')
        sleep(5)
        pass
    # Capture frame-by-frame
    ret, frame = video_capture.read()
    frame = predict(frame)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break
    # Display the resulting frame
    cv2.imshow('Video', frame)
    cv2.waitKey(1)
# When everything is done, release the capture
video_capture.release()
cv2.destroyAllWindows()
<commit_msg>multithreaded video demo<commit_after>from __future__ import print_function
import sys
import os
import torch
from torch.autograd import Variable
import cv2
import time
from imutils.video import FPS, WebcamVideoStream

from data import BaseTransform
from data import VOC_CLASSES as labelmap
from ssd import build_ssd

trained_model = 'weights/ssd_300_VOC0712.pth'
net = build_ssd('test', 300, 21)    # initialize SSD
net.load_state_dict(torch.load(trained_model))
net.eval()
transform = BaseTransform(net.size, (104, 117, 123))
colors = [(255, 0, 0), (0, 255, 0), (0, 0, 255)]
font = cv2.FONT_HERSHEY_SIMPLEX

def predict(frame):
    height, width = frame.shape[:2]
    x = Variable(transform(frame).unsqueeze(0))
    y = net(x)  # forward pass
    detections = y.data
    # scale each detection back up to the image
    scale = torch.Tensor([width, height, width, height])
    for i in range(detections.size(1)):
        j = 0
        while detections[0, i, j, 0] >= 0.6:
            pt = (detections[0, i, j, 1:] * scale).cpu().numpy()
            cv2.rectangle(frame, (int(pt[0]), int(pt[1])), (int(pt[2]),
                                                            int(pt[3])), colors[i % 3], 2)
            cv2.putText(frame, labelmap[i - 1], (int(pt[0]), int(pt[1])), font,
                        2, (255, 255, 255), 2, cv2.LINE_AA)
            j += 1
    return frame

# start video stream thread, allow buffer to fill
print("[INFO] starting threaded video stream...")
stream = WebcamVideoStream(src=0).start()  # default camera
time.sleep(1.0)

# start fps timer
fps = FPS().start()

# loop over frames from the video file stream
while True:
    # grab next frame
    frame = stream.read()
    key = cv2.waitKey(1) & 0xFF

    # update FPS counter
    fps.update()
    frame = predict(frame)

    # keybindings for display
    if key == ord('p'):  # pause
        while True:
            key2 = cv2.waitKey(1) or 0xff
            cv2.imshow('frame', frame)
            if key2 == ord('p'):  # resume
                break
    cv2.imshow('frame', frame)
    if key == 27:  # exit
        break

# stop the timer and display FPS information
fps.stop()
print("[INFO] elasped time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))

# cleanup
cv2.destroyAllWindows()
stream.stop()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# The MIT License
#
# Copyright 2018 David Pursehouse. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

"""Live server tests."""

import base64
import pytest
import unittest
from pygerrit2 import GerritRestAPI, GerritReview, HTTPBasicAuth, \
    HTTPDigestAuth
from testcontainers.core.container import DockerContainer
from testcontainers.core.waiting_utils import wait_container_is_ready

TEST_TOPIC = "test-topic"

class GerritContainer(DockerContainer):
    """Gerrit container."""

    def __init__(self, version):
        """Construct a GerritContainer with the given version."""
        image = "gerritcodereview/gerrit:" + version
        super(GerritContainer, self).__init__(image)
        self.with_exposed_ports(8080)

@wait_container_is_ready()
def _initialize(api):
    api.get("/changes/")

@pytest.fixture(scope="module",
                params=["2.14.20", "2.15.14", "2.16.9", "3.0.0"])
def gerrit_api(request):
    """Create a Gerrit container for the given version and return an API."""
    with GerritContainer(request.param) as gerrit:
        port = gerrit.get_exposed_port(8080)
        url = "http://localhost:%s" % port
        if request.param == "2.13.11":
            auth = HTTPDigestAuth("admin", "secret")
        else:
            auth = HTTPBasicAuth("admin", "secret")
        api = GerritRestAPI(url=url, auth=auth)
        _initialize(api)
        yield api

class TestGerritAgainstLiveServer(object):
    """Run tests against a live server."""

    def _get_test_change(self, gerrit_api, topic=TEST_TOPIC):
        results = gerrit_api.get("/changes/?q=topic:" + topic)
        assert len(results) == 1
        return results[0]

    def test_put_with_json_dict(self, gerrit_api):
        """Test a PUT request passing data as a dict to `json`.

        Tests that the PUT request works as expected when data is passed
        via the `json` argument as a `dict`.

        Creates the test project which is used by subsequent tests.
        """
        projectinput = {"create_empty_commit": "true"}
        gerrit_api.put("/projects/test-project", json=projectinput)
        gerrit_api.get("/projects/test-project")

    def test_put_with_data_dict(self, gerrit_api):
        """Test a PUT request passing data as a dict to `data`.

        Tests that the PUT request works as expected when data is passed
        via the `data` argument as a `dict`.
        """
        description = {"description": "New Description"}
        gerrit_api.put("/projects/test-project/description", data=description)
        project = gerrit_api.get("/projects/test-project")
        assert project["description"] == "New Description"

    def test_post_with_data_dict_and_no_data(self, gerrit_api):
        """Test a POST request passing data as a dict to `data`.

        Tests that the POST request works as expected when data is passed
        via the `data` argument as a `dict`.
        """
        changeinput = {"project": "test-project",
                       "subject": "subject",
                       "branch": "master",
                       "topic": "post-with-data"}
        result = gerrit_api.post("/changes/", data=changeinput)
        change = self._get_test_change(gerrit_api, "post-with-data")
        assert change["id"] == result["id"]

        # Subsequent post without data or json should not have the Content-Type
        # json header, and should succeed.
        result = gerrit_api.post("/changes/" + change["id"] + "/abandon")
        assert result["status"] == "ABANDONED"

    def test_post_with_json_dict(self, gerrit_api):
        """Test a POST request passing data as a dict to `json`.

        Tests that the POST request works as expected when data is passed
        via the `json` argument as a `dict`.

        Creates the change which is used by subsequent tests.
        """
        changeinput = {"project": "test-project",
                       "subject": "subject",
                       "branch": "master",
                       "topic": TEST_TOPIC}
        result = gerrit_api.post("/changes/", json=changeinput)
        change = self._get_test_change(gerrit_api)
        assert change["id"] == result["id"]

    def test_put_with_data_as_string(self, gerrit_api):
        """Test a PUT request passing data as a string to `data`.

        Tests that the PUT request works as expected when data is passed
        via the `data` parameter as a string.

        Creates a change edit that is checked in the subsequent test.
        """
        change_id = self._get_test_change(gerrit_api)["id"]
        gerrit_api.put("/changes/" + change_id + "/edit/foo",
                       data="Content with non base64 valid chars åäö")

    def test_put_json_content(self, gerrit_api):
        """Test a PUT request with a json file content (issue #54)."""
        change_id = self._get_test_change(gerrit_api)["id"]
        content = """{"foo" : "bar"}"""
        gerrit_api.put("/changes/" + change_id + "/edit/file.json",
                       data=content)

    def test_get_base64_data(self, gerrit_api):
        """Test a GET request on an API that returns base64 encoded response.

        Tests that the headers can be overridden on the GET call, resulting
        in the data being returned as text/plain, and that the content of the
        response can be base64 decoded.
        """
        change_id = self._get_test_change(gerrit_api)["id"]
        response = gerrit_api.get("/changes/" + change_id + "/edit/foo",
                                  headers={'Accept': 'text/plain'})

        # Will raise binascii.Error if content is not properly encoded
        base64.b64decode(response)

    def test_get_patch_zip(self, gerrit_api):
        """Test a GET request to get a patch file (issue #19)."""
        change_id = self._get_test_change(gerrit_api)["id"]
        gerrit_api.get("/changes/" + change_id + "/revisions/current/patch?zip")

    def test_put_with_no_content(self, gerrit_api):
        """Test a PUT request with no content."""
        change_id = self._get_test_change(gerrit_api)["id"]
        gerrit_api.put("/changes/" + change_id + "/edit/foo")

    def test_review(self, gerrit_api):
        """Test that a review can be posted by the review API."""
        change_id = self._get_test_change(gerrit_api)["id"]
        review = GerritReview()
        review.set_message("Review from live test")
        review.add_labels({"Code-Review": 1})
        gerrit_api.review(change_id, "current", review)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Run live tests against 2.16.10 and 3.0.1<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# The MIT License
#
# Copyright 2018 David Pursehouse. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

"""Live server tests."""

import base64
import pytest
import unittest
from pygerrit2 import GerritRestAPI, GerritReview, HTTPBasicAuth, \
    HTTPDigestAuth
from testcontainers.core.container import DockerContainer
from testcontainers.core.waiting_utils import wait_container_is_ready

TEST_TOPIC = "test-topic"

class GerritContainer(DockerContainer):
    """Gerrit container."""

    def __init__(self, version):
        """Construct a GerritContainer with the given version."""
        image = "gerritcodereview/gerrit:" + version
        super(GerritContainer, self).__init__(image)
        self.with_exposed_ports(8080)

@wait_container_is_ready()
def _initialize(api):
    api.get("/changes/")

@pytest.fixture(scope="module",
                params=["2.14.20", "2.15.14", "2.16.10", "3.0.1"])
def gerrit_api(request):
    """Create a Gerrit container for the given version and return an API."""
    with GerritContainer(request.param) as gerrit:
        port = gerrit.get_exposed_port(8080)
        url = "http://localhost:%s" % port
        if request.param == "2.13.11":
            auth = HTTPDigestAuth("admin", "secret")
        else:
            auth = HTTPBasicAuth("admin", "secret")
        api = GerritRestAPI(url=url, auth=auth)
        _initialize(api)
        yield api

class TestGerritAgainstLiveServer(object):
    """Run tests against a live server."""

    def _get_test_change(self, gerrit_api, topic=TEST_TOPIC):
        results = gerrit_api.get("/changes/?q=topic:" + topic)
        assert len(results) == 1
        return results[0]

    def test_put_with_json_dict(self, gerrit_api):
        """Test a PUT request passing data as a dict to `json`.

        Tests that the PUT request works as expected when data is passed
        via the `json` argument as a `dict`.

        Creates the test project which is used by subsequent tests.
        """
        projectinput = {"create_empty_commit": "true"}
        gerrit_api.put("/projects/test-project", json=projectinput)
        gerrit_api.get("/projects/test-project")

    def test_put_with_data_dict(self, gerrit_api):
        """Test a PUT request passing data as a dict to `data`.

        Tests that the PUT request works as expected when data is passed
        via the `data` argument as a `dict`.
        """
        description = {"description": "New Description"}
        gerrit_api.put("/projects/test-project/description", data=description)
        project = gerrit_api.get("/projects/test-project")
        assert project["description"] == "New Description"

    def test_post_with_data_dict_and_no_data(self, gerrit_api):
        """Test a POST request passing data as a dict to `data`.

        Tests that the POST request works as expected when data is passed
        via the `data` argument as a `dict`.
        """
        changeinput = {"project": "test-project",
                       "subject": "subject",
                       "branch": "master",
                       "topic": "post-with-data"}
        result = gerrit_api.post("/changes/", data=changeinput)
        change = self._get_test_change(gerrit_api, "post-with-data")
        assert change["id"] == result["id"]

        # Subsequent post without data or json should not have the Content-Type
        # json header, and should succeed.
        result = gerrit_api.post("/changes/" + change["id"] + "/abandon")
        assert result["status"] == "ABANDONED"

    def test_post_with_json_dict(self, gerrit_api):
        """Test a POST request passing data as a dict to `json`.

        Tests that the POST request works as expected when data is passed
        via the `json` argument as a `dict`.

        Creates the change which is used by subsequent tests.
        """
        changeinput = {"project": "test-project",
                       "subject": "subject",
                       "branch": "master",
                       "topic": TEST_TOPIC}
        result = gerrit_api.post("/changes/", json=changeinput)
        change = self._get_test_change(gerrit_api)
        assert change["id"] == result["id"]

    def test_put_with_data_as_string(self, gerrit_api):
        """Test a PUT request passing data as a string to `data`.

        Tests that the PUT request works as expected when data is passed
        via the `data` parameter as a string.

        Creates a change edit that is checked in the subsequent test.
        """
        change_id = self._get_test_change(gerrit_api)["id"]
        gerrit_api.put("/changes/" + change_id + "/edit/foo",
                       data="Content with non base64 valid chars åäö")

    def test_put_json_content(self, gerrit_api):
        """Test a PUT request with a json file content (issue #54)."""
        change_id = self._get_test_change(gerrit_api)["id"]
        content = """{"foo" : "bar"}"""
        gerrit_api.put("/changes/" + change_id + "/edit/file.json",
                       data=content)

    def test_get_base64_data(self, gerrit_api):
        """Test a GET request on an API that returns base64 encoded response.

        Tests that the headers can be overridden on the GET call, resulting
        in the data being returned as text/plain, and that the content of the
        response can be base64 decoded.
        """
        change_id = self._get_test_change(gerrit_api)["id"]
        response = gerrit_api.get("/changes/" + change_id + "/edit/foo",
                                  headers={'Accept': 'text/plain'})

        # Will raise binascii.Error if content is not properly encoded
        base64.b64decode(response)

    def test_get_patch_zip(self, gerrit_api):
        """Test a GET request to get a patch file (issue #19)."""
        change_id = self._get_test_change(gerrit_api)["id"]
        gerrit_api.get("/changes/" + change_id + "/revisions/current/patch?zip")

    def test_put_with_no_content(self, gerrit_api):
        """Test a PUT request with no content."""
        change_id = self._get_test_change(gerrit_api)["id"]
        gerrit_api.put("/changes/" + change_id + "/edit/foo")

    def test_review(self, gerrit_api):
        """Test that a review can be posted by the review API."""
        change_id = self._get_test_change(gerrit_api)["id"]
        review = GerritReview()
        review.set_message("Review from live test")
        review.add_labels({"Code-Review": 1})
        gerrit_api.review(change_id, "current", review)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>