<commit_before>#!/usr/bin/python

import signal
import subprocess

from termcolor import colored

# Handle keyboard interrupt shortcuts
def signal_handler(signum, frame):
    pass # prevent exiting
#signal.signal(signal.SIGTSTP, signal_handler)  # Disable Ctrl-Z
#signal.signal(signal.SIGINT, signal_handler)   # Disable Ctrl-C

def main():
    print("\n")
    msg("Welcome to Terminal! You must create an account before playing this game.")
    msg("If you have forgotten your password, please email spartandominion@gmail.com.")
    msg("Your username can be between 4 and 16 characters and must be alphanumeric.")

    username = ""
    valid_user = False

    while not valid_user:
        username = raw_input("\nDesired Username: ")
        msg("You entered " + username + ".")

        # name cannot be blank
        if username == "":
            error("You must enter a username.")
        # name must be alphanumeric
        elif not username.isalnum():
            error("Your username can only contain letters and numbers.")
        # name must be between 4 and 16 characters
        elif len(username) < 4 or len(username) > 16:
            error("Your username must be between 4 and 16 characters.")
        # name must not already exist in database

    # Prevent player from exiting game loop
    while True:
        pass

## OUTPUT MESSAGES
# Standard Message
def msg(message):
    print(colored("\t" + message, 'white'))

# Success Message
def success(message):
    print(colored("\t" + message, 'green'))

# Warning Message
def warning(message):
    print(colored("\t" +message, 'yellow'))

# Error Message
def error(message):
    print(colored("\t" + message, 'red'))

if __name__ == '__main__':
    main()

# Prevent user from entering shell
print("Please close this window to disconnect.")
while True:
    pass
<commit_msg>Added database connection; closes #3<commit_after>#!/usr/bin/python

import signal
import subprocess
import MySQLdb

from termcolor import colored

# Handle keyboard interrupt shortcuts
def signal_handler(signum, frame):
    pass # prevent exiting
#signal.signal(signal.SIGTSTP, signal_handler)  # Disable Ctrl-Z
#signal.signal(signal.SIGINT, signal_handler)   # Disable Ctrl-C

def main():
    print("\n")
    msg("Welcome to Terminal! You must create an account before playing this game.")
    msg("If you have forgotten your password, please email spartandominion@gmail.com.")
    msg("Your username can be between 4 and 16 characters and must be alphanumeric.")

    db = connect_database()
    cursor = db.cursor()

    username = ""
    valid_user = False

    while not valid_user:
        username = raw_input("\nDesired Username: ")
        msg("You entered " + username + ".")

        # look up username in database
        sql = "SELECT * FROM users WHERE username = %s"
        cursor.execute(sql, [username])
        response = cursor.fetchall()

        # name cannot be blank
        if username == "":
            error("You must enter a username.")
        # name must be alphanumeric
        elif not username.isalnum():
            error("Your username can only contain letters and numbers.")
        # name must be between 4 and 16 characters
        elif len(username) < 4 or len(username) > 16:
            error("Your username must be between 4 and 16 characters.")
        # name must not already exist in database
        elif len(response) > 0:
            error("Sorry, that name is taken. Please choose another.")
        else:
            msg("Username is available! Please enter your password:")

    # Prevent player from exiting game loop
    while True:
        pass

# Establish a connection to MySQL
def connect_database():
    cred_file = open("/htp/dbcreds.txt", 'r')
    u = cred_file.readline()[:-1]
    p = cred_file.readline()[:-1]
    d = "htp"
    con = MySQLdb.connect(host = "localhost",
                          user = u,
                          passwd = p,
                          db = d)
    return con

## OUTPUT MESSAGES
# Standard Message
def msg(message):
    print(colored("\t" + message, 'white'))

# Success Message
def success(message):
    print(colored("\t" + message, 'green'))

# Warning Message
def warning(message):
    print(colored("\t" +message, 'yellow'))

# Error Message
def error(message):
    print(colored("\t" + message, 'red'))

if __name__ == '__main__':
    main()

# Prevent user from entering shell
print("Please close this window to disconnect.")
while True:
    pass
<|endoftext|>


<commit_before>import evaluate
import env
import misc

class Value():
	def __str__(self):
		return "Generic value (bug ?)"
	def evaluate(self, env):
		raise NotImplementedError

class Nil(Value):
	def __str__(self):
		return '()'
	def evaluate(self, env):
		return self

class Int(Value):
	def __init__(self, value=0):
		self.value = value
	def __str__(self):
		return str(self.value)
	def evaluate(self, env):
		return self

class Bool(Value):
	def __init__(self, value=False):
		self.value = value
	def __str__(self):
		if self.value:
			return "#t"
		else:
			return "#f"
	def evaluate(self, env):
		return self

class String(Value):
	def __init__(self, value=""):
		self.value = value
	def __str__(self):
		return '"' + self.value + '"'
	def evaluate(self, env):
		return self

class Symbol(Value):
	def __init__(self, value):
		self.value = value
	def __str__(self):
		return self.value.upper()
	def evaluate(self, env):
		return env.getValue(self.value)

class Pair(Value):
	def __init__(self, car, cdr):
		self.vcar = car
		self.vcdr = cdr
	def insidePrint(self):
		if type(self.cdr()) is Nil:
			return str(self.car()) + ")"
		elif type(self.cdr()) is Pair:
			return str(self.car()) + " " + self.cdr().insidePrint()
		else:
			return str(self.car()) + " . " + str(self.cdr()) + ")"

	def __str__(self):
		return "(" + self.insidePrint()

	def evaluate(self, env):
		return evaluate.funcall(self.car(), self.cdr(), env)
	def car(self):
		return self.vcar
	def cdr(self):
		return self.vcdr

class Function(Value):
	def __init__(self):
		raise NotImplementedError

class Lambda(Function):
	def __init__(self, env, args, body, varargs = False):
		self.env = env
		self.args = args
		self.body = body
		self.macro = False
	def __str__(self):
		return ("macro " if self.macro else "") + "lambda function : " + str(self.args) + " -> " + str(self.body) + " in " + str(self.env)
	def evaluate(self, env):
		return self
	def apply(self, values):
		e = env.Env(self.env)
		argMappings = misc.filterMap(self.args, values)
		e.addDict(argMappings)
		return self.body.evaluate(e)

class Primitive(Function):
	def __init__(self, env, prim):
		self.env = env
		self.primitive = prim
		self.macro = False
	def __str__(self):
		return ("macro " if self.macro else "") + "primitive function : " + str(self.primitive)
	def evaluate(self, env):
		return self
	def apply(self, values):
		values = misc.pairsToList(values)
		return self.primitive(values)

def tests():
	i = Int(42)
	j = String("hello")
	k = Bool(False)
	l = Nil()
	p1 = Pair(i, k)
	print(p1)
	print(Pair(k, p1))
	print(Pair(p1, p1))
	print(Pair(p1, Pair(p1, Nil)))

if __name__ == '__main__':
	tests()
<commit_msg>removed useless argument<commit_after>import evaluate
import env
import misc

class Value():
	def __str__(self):
		return "Generic value (bug ?)"
	def evaluate(self, env):
		raise NotImplementedError

class Nil(Value):
	def __str__(self):
		return '()'
	def evaluate(self, env):
		return self

class Int(Value):
	def __init__(self, value=0):
		self.value = value
	def __str__(self):
		return str(self.value)
	def evaluate(self, env):
		return self

class Bool(Value):
	def __init__(self, value=False):
		self.value = value
	def __str__(self):
		if self.value:
			return "#t"
		else:
			return "#f"
	def evaluate(self, env):
		return self

class String(Value):
	def __init__(self, value=""):
		self.value = value
	def __str__(self):
		return '"' + self.value + '"'
	def evaluate(self, env):
		return self

class Symbol(Value):
	def __init__(self, value):
		self.value = value
	def __str__(self):
		return self.value.upper()
	def evaluate(self, env):
		return env.getValue(self.value)

class Pair(Value):
	def __init__(self, car, cdr):
		self.vcar = car
		self.vcdr = cdr
	def insidePrint(self):
		if type(self.cdr()) is Nil:
			return str(self.car()) + ")"
		elif type(self.cdr()) is Pair:
			return str(self.car()) + " " + self.cdr().insidePrint()
		else:
			return str(self.car()) + " . " + str(self.cdr()) + ")"

	def __str__(self):
		return "(" + self.insidePrint()

	def evaluate(self, env):
		return evaluate.funcall(self.car(), self.cdr(), env)
	def car(self):
		return self.vcar
	def cdr(self):
		return self.vcdr

class Function(Value):
	def __init__(self):
		raise NotImplementedError

class Lambda(Function):
	def __init__(self, env, args, body):
		self.env = env
		self.args = args
		self.body = body
		self.macro = False
	def __str__(self):
		return ("macro " if self.macro else "") + "lambda function : " + str(self.args) + " -> " + str(self.body) + " in " + str(self.env)
	def evaluate(self, env):
		return self
	def apply(self, values):
		e = env.Env(self.env)
		argMappings = misc.filterMap(self.args, values)
		e.addDict(argMappings)
		return self.body.evaluate(e)

class Primitive(Function):
	def __init__(self, env, prim):
		self.env = env
		self.primitive = prim
		self.macro = False
	def __str__(self):
		return ("macro " if self.macro else "") + "primitive function : " + str(self.primitive)
	def evaluate(self, env):
		return self
	def apply(self, values):
		values = misc.pairsToList(values)
		return self.primitive(values)

def tests():
	i = Int(42)
	j = String("hello")
	k = Bool(False)
	l = Nil()
	p1 = Pair(i, k)
	print(p1)
	print(Pair(k, p1))
	print(Pair(p1, p1))
	print(Pair(p1, Pair(p1, Nil)))

if __name__ == '__main__':
	tests()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import concurrent.futures
import filecmp
import logging
import os
import re
import shutil
import sys
import time

from tqdm import tqdm

from src.date import Date
from src.exif import Exif

UNKNOWN = 'unknown'

logger = logging.getLogger('phockup')

ignored_files = ('.DS_Store', 'Thumbs.db')

class Phockup():
    DEFAULT_DIR_FORMAT = ['%Y', '%m', '%d']
    DEFAULT_NO_DATE_DIRECTORY = "unknown"

    def __init__(self, input_dir, output_dir, **args):
        start_time = time.time()
        self.files_processed = 0
        self.duplicates_found = 0
        self.unknown_found = 0
        self.files_moved = 0
        self.files_copied = 0

        input_dir = os.path.expanduser(input_dir)
        output_dir = os.path.expanduser(output_dir)

        if input_dir.endswith(os.path.sep):
            input_dir = input_dir[:-1]
        if output_dir.endswith(os.path.sep):
            output_dir = output_dir[:-1]

        self.input_dir = input_dir
        self.output_dir = output_dir
        self.no_date_dir = args.get('no_date_dir') or Phockup.DEFAULT_NO_DATE_DIRECTORY
        self.dir_format = args.get('dir_format') or os.path.sep.join(Phockup.DEFAULT_DIR_FORMAT)
        self.move = args.get('move', False)
        self.link = args.get('link', False)
        self.original_filenames = args.get('original_filenames', False)
        self.date_regex = args.get('date_regex', None)
        self.timestamp = args.get('timestamp', False)
        self.date_field = args.get('date_field', False)
        self.skip_unknown = args.get("skip_unknown", False)
        self.dry_run = args.get('dry_run', False)
        self.progress = args.get('progress', False)
        self.max_depth = args.get('max_depth', -1)
        # default to concurrency of one to retain existing behavior
        self.max_concurrency = args.get("max_concurrency", 1)
        if self.max_concurrency > 1:
            logger.info(f"Using {self.max_concurrency} workers to process files.")

        self.stop_depth = self.input_dir.count(os.sep) + self.max_depth \
            if self.max_depth > -1 else sys.maxsize
        self.file_type = args.get('file_type', None)

        if self.dry_run:
            logger.warning("Dry-run phockup (does a trial run with no permanent changes)...")

        self.check_directories()
        # Get the number of files
        if self.progress:
            file_count = self.get_file_count()
            with tqdm(desc=f"Progressing: '{self.input_dir}' ",
                      total=file_count,
                      unit="file",
                      position=0,
                      leave=True,
                      ascii=(sys.platform == 'win32')) as self.pbar:
                self.walk_directory()
        else:
            self.pbar = None
            self.walk_directory()

        run_time = time.time() - start_time
        if self.files_processed and run_time:
            self.print_action_report(run_time)

    def print_action_report(self, run_time):
        logger.info(f"Processed {self.files_processed} files in {run_time:.2f} seconds. Average Throughput: {self.files_processed/run_time:.2f} files/second")
        if self.unknown_found:
            logger.info(f"Found {self.unknown_found} files without EXIF date data.")
        if self.duplicates_found:
            logger.info(f"Found {self.duplicates_found} duplicate files.")
        if self.files_copied:
            if self.dry_run:
                logger.info(f"Would have copied {self.files_copied} files.")
            else:
                logger.info(f"Copied {self.files_copied} files.")
        if self.files_moved:
            if self.dry_run:
                logger.info(f"Would have moved {self.files_moved} files.")
            else:
                logger.info(f"Moved {self.files_moved} files.")

    def check_directories(self):
        """
        Check if input and output directories exist.
        If input does not exists it exits the process.
        If output does not exists it tries to create it or exit with error.
        """

        if not os.path.exists(self.input_dir):
            raise RuntimeError(f"Input directory '{self.input_dir}' does not exist")
        if not os.path.isdir(self.input_dir):
            raise RuntimeError(f"Input directory '{self.input_dir}' is not a directory")
        if not os.path.exists(self.output_dir):
            logger.warning(f"Output directory '{self.output_dir}' does not exist, creating now")
            try:
                if not self.dry_run:
                    os.makedirs(self.output_dir)
            except OSError:
                raise OSError(f"Cannot create output '{self.output_dir}' directory. No write access!")

    def walk_directory(self):
        """
        Walk input directory recursively and call process_file for each file
        except the ignored ones.
        """

        # Walk the directory
        for root, dirnames, files in os.walk(self.input_dir):
            files.sort()
            file_paths_to_process = []
            for filename in files:
                if filename in ignored_files:
                    continue
                file_paths_to_process.append(os.path.join(root, filename))
            if self.max_concurrency > 1:
                if not self.process_files(file_paths_to_process):
                    return
            else:
                try:
                    for file_path in file_paths_to_process:
                        self.process_file(file_path)
                except KeyboardInterrupt:
                    logger.warning("Received interrupt. Shutting down...")
                    return
            if root.count(os.sep) >= self.stop_depth:
                del dirnames[:]

    def get_file_count(self):
        file_count = 0
        for root, dirnames, files in os.walk(self.input_dir):
            file_count += len(files)
            if root.count(os.sep) >= self.stop_depth:
                del dirnames[:]
        return file_count

    def get_file_type(self, mimetype):
        """
        Check if given file_type is image or video
        Return None if other
        Use mimetype to determine if the file is an image or video.
        """
        patternImage = re.compile('^(image/.+|application/vnd.adobe.photoshop)$')
        if patternImage.match(mimetype):
            return 'image'

        patternVideo = re.compile('^(video/.*)$')
        if patternVideo.match(mimetype):
            return 'video'
        return None

    def get_output_dir(self, date):
        """
        Generate output directory path based on the extracted date and
        formatted using dir_format.
        If date is missing from the exifdata the file is going to "unknown"
        directory unless user included a regex from filename or uses timestamp.
        """
        try:
            path = [self.output_dir, date['date'].date().strftime(self.dir_format)]
        except (TypeError, ValueError):
            path = [self.output_dir, self.no_date_dir]

        fullpath = os.path.sep.join(path)

        if not os.path.isdir(fullpath) and not self.dry_run:
            os.makedirs(fullpath)

        return fullpath

    def get_file_name(self, original_filename, date):
        """
        Generate file name based on exif data unless it is missing or
        original filenames are required. Then use original file name
        """
        if self.original_filenames:
            return os.path.basename(original_filename)

        try:
            filename = [
                f'{(date["date"].year):04d}',
                f'{(date["date"].month):02d}',
                f'{(date["date"].day):02d}',
                '-',
                f'{(date["date"].hour):02d}',
                f'{(date["date"].minute):02d}',
                f'{(date["date"].second):02d}',
            ]

            if date['subseconds']:
                filename.append(date['subseconds'])

            return ''.join(filename) + os.path.splitext(original_filename)[1]
        # TODO: Double check if this is correct!
        except TypeError:
            return os.path.basename(original_filename)

    def process_files(self, file_paths_to_process):
        # With all the appropriate files in the directory added to the
        # list, process the directory concurrently using threads
        with concurrent.futures.ThreadPoolExecutor(
                max_workers=self.max_concurrency) as executor:
            try:
                for _ in executor.map(self.process_file,
                                      file_paths_to_process):
                    pass
            except KeyboardInterrupt:
                logger.warning(
                        f"Received interrupt. Shutting down {self.max_concurrency} workers...")
                executor.shutdown(wait=True)
                return False
        return True

    def process_file(self, filename):
        """
        Process the file using the selected strategy
        If file is .xmp skip it so process_xmp method can handle it
        """
        if str.endswith(filename, '.xmp'):
            return None

        progress = f'{filename}'

        output, target_file_name, target_file_path, target_file_type = self.get_file_name_and_path(filename)
        suffix = 1
        target_file = target_file_path

        while True:
            if self.file_type is not None \
                    and self.file_type != target_file_type:
                progress = f"{progress} => skipped, file is '{target_file_type}' \
but looking for '{self.file_type}'"
                logger.info(progress)
                break

            if self.skip_unknown and output.endswith(UNKNOWN):
                # Skip files that didn't generate a path from EXIF data
                progress = f"{progress} => skipped, unknown date EXIF information for '{target_file_name}'"
                self.unknown_found += 1
                if self.progress:
                    self.pbar.write(progress)
                logger.info(progress)
                break

            if os.path.isfile(target_file):
                if filecmp.cmp(filename, target_file, shallow=False):
                    progress = f'{progress} => skipped, duplicated file {target_file}'
                    self.duplicates_found += 1
                    if self.progress:
                        self.pbar.write(progress)
                    logger.info(progress)
                    break
            else:
                if self.move:
                    try:
                        self.files_moved += 1
                        if not self.dry_run:
                            shutil.move(filename, target_file)
                    except FileNotFoundError:
                        progress = f'{progress} => skipped, no such file or directory'
                        if self.progress:
                            self.pbar.write(progress)
                        logger.warning(progress)
                        break
                elif self.link and not self.dry_run:
                    os.link(filename, target_file)
                else:
                    try:
                        self.files_copied += 1
                        if not self.dry_run:
                            shutil.copy2(filename, target_file)
                    except FileNotFoundError:
                        progress = f'{progress} => skipped, no such file or directory'
                        if self.progress:
                            self.pbar.write(progress)
                        logger.warning(progress)
                        break

                progress = f'{progress} => {target_file}'
                if self.progress:
                    self.pbar.write(progress)
                logger.info(progress)

                self.process_xmp(filename, target_file_name, suffix, output)
                break

            suffix += 1
            target_split = os.path.splitext(target_file_path)
            target_file = f'{target_split[0]}-{suffix}{target_split[1]}'

        self.files_processed += 1
        if self.progress:
            self.pbar.update(1)

    def get_file_name_and_path(self, filename):
        """
        Returns target file name and path
        """
        exif_data = Exif(filename).data()
        target_file_type = None

        if exif_data and 'MIMEType' in exif_data:
            target_file_type = self.get_file_type(exif_data['MIMEType'])

        if target_file_type in ['image', 'video']:
            date = Date(filename).from_exif(exif_data, self.timestamp, self.date_regex,
                                            self.date_field)
            output = self.get_output_dir(date)
            target_file_name = self.get_file_name(filename, date)
            if not self.original_filenames:
                target_file_name = target_file_name.lower()
        else:
            output = self.get_output_dir([])
            target_file_name = os.path.basename(filename)

        target_file_path = os.path.sep.join([output, target_file_name])
        return output, target_file_name, target_file_path, target_file_type

    def process_xmp(self, original_filename, file_name, suffix, output):
        """
        Process xmp files. These are meta data for RAW images
        """
        xmp_original_with_ext = original_filename + '.xmp'
        xmp_original_without_ext = os.path.splitext(original_filename)[0] + '.xmp'

        suffix = f'-{suffix}' if suffix > 1 else ''

        xmp_files = {}

        if os.path.isfile(xmp_original_with_ext):
            xmp_target = f'{file_name}{suffix}.xmp'
            xmp_files[xmp_original_with_ext] = xmp_target
        if os.path.isfile(xmp_original_without_ext):
            xmp_target = f'{(os.path.splitext(file_name)[0])}{suffix}.xmp'
            xmp_files[xmp_original_without_ext] = xmp_target

        for original, target in xmp_files.items():
            xmp_path = os.path.sep.join([output, target])
            logger.info(f'{original} => {xmp_path}')

            if not self.dry_run:
                if self.move:
                    shutil.move(original, xmp_path)
                elif self.link:
                    os.link(original, xmp_path)
                else:
                    shutil.copy2(original, xmp_path)
<commit_msg>add check to skip overwriting same file (#151)<commit_after>#!/usr/bin/env python3
import concurrent.futures
import filecmp
import logging
import os
import re
import shutil
import sys
import time

from tqdm import tqdm

from src.date import Date
from src.exif import Exif

UNKNOWN = 'unknown'

logger = logging.getLogger('phockup')

ignored_files = ('.DS_Store', 'Thumbs.db')

class Phockup():
    DEFAULT_DIR_FORMAT = ['%Y', '%m', '%d']
    DEFAULT_NO_DATE_DIRECTORY = "unknown"

    def __init__(self, input_dir, output_dir, **args):
        start_time = time.time()
        self.files_processed = 0
        self.duplicates_found = 0
        self.unknown_found = 0
        self.files_moved = 0
        self.files_copied = 0

        input_dir = os.path.expanduser(input_dir)
        output_dir = os.path.expanduser(output_dir)

        if input_dir.endswith(os.path.sep):
            input_dir = input_dir[:-1]
        if output_dir.endswith(os.path.sep):
            output_dir = output_dir[:-1]

        self.input_dir = input_dir
        self.output_dir = output_dir
        self.no_date_dir = args.get('no_date_dir') or Phockup.DEFAULT_NO_DATE_DIRECTORY
        self.dir_format = args.get('dir_format') or os.path.sep.join(Phockup.DEFAULT_DIR_FORMAT)
        self.move = args.get('move', False)
        self.link = args.get('link', False)
        self.original_filenames = args.get('original_filenames', False)
        self.date_regex = args.get('date_regex', None)
        self.timestamp = args.get('timestamp', False)
        self.date_field = args.get('date_field', False)
        self.skip_unknown = args.get("skip_unknown", False)
        self.dry_run = args.get('dry_run', False)
        self.progress = args.get('progress', False)
        self.max_depth = args.get('max_depth', -1)
        # default to concurrency of one to retain existing behavior
        self.max_concurrency = args.get("max_concurrency", 1)
        if self.max_concurrency > 1:
            logger.info(f"Using {self.max_concurrency} workers to process files.")

        self.stop_depth = self.input_dir.count(os.sep) + self.max_depth \
            if self.max_depth > -1 else sys.maxsize
        self.file_type = args.get('file_type', None)

        if self.dry_run:
            logger.warning("Dry-run phockup (does a trial run with no permanent changes)...")

        self.check_directories()
        # Get the number of files
        if self.progress:
            file_count = self.get_file_count()
            with tqdm(desc=f"Progressing: '{self.input_dir}' ",
                      total=file_count,
                      unit="file",
                      position=0,
                      leave=True,
                      ascii=(sys.platform == 'win32')) as self.pbar:
                self.walk_directory()
        else:
            self.pbar = None
            self.walk_directory()

        run_time = time.time() - start_time
        if self.files_processed and run_time:
            self.print_action_report(run_time)

    def print_action_report(self, run_time):
        logger.info(f"Processed {self.files_processed} files in {run_time:.2f} seconds. Average Throughput: {self.files_processed/run_time:.2f} files/second")
        if self.unknown_found:
            logger.info(f"Found {self.unknown_found} files without EXIF date data.")
        if self.duplicates_found:
            logger.info(f"Found {self.duplicates_found} duplicate files.")
        if self.files_copied:
            if self.dry_run:
                logger.info(f"Would have copied {self.files_copied} files.")
            else:
                logger.info(f"Copied {self.files_copied} files.")
        if self.files_moved:
            if self.dry_run:
                logger.info(f"Would have moved {self.files_moved} files.")
            else:
                logger.info(f"Moved {self.files_moved} files.")

    def check_directories(self):
        """
        Check if input and output directories exist.
        If input does not exists it exits the process.
        If output does not exists it tries to create it or exit with error.
        """

        if not os.path.exists(self.input_dir):
            raise RuntimeError(f"Input directory '{self.input_dir}' does not exist")
        if not os.path.isdir(self.input_dir):
            raise RuntimeError(f"Input directory '{self.input_dir}' is not a directory")
        if not os.path.exists(self.output_dir):
            logger.warning(f"Output directory '{self.output_dir}' does not exist, creating now")
            try:
                if not self.dry_run:
                    os.makedirs(self.output_dir)
            except OSError:
                raise OSError(f"Cannot create output '{self.output_dir}' directory. No write access!")

    def walk_directory(self):
        """
        Walk input directory recursively and call process_file for each file
        except the ignored ones.
        """

        # Walk the directory
        for root, dirnames, files in os.walk(self.input_dir):
            files.sort()
            file_paths_to_process = []
            for filename in files:
                if filename in ignored_files:
                    continue
                file_paths_to_process.append(os.path.join(root, filename))
            if self.max_concurrency > 1:
                if not self.process_files(file_paths_to_process):
                    return
            else:
                try:
                    for file_path in file_paths_to_process:
                        self.process_file(file_path)
                except KeyboardInterrupt:
                    logger.warning("Received interrupt. Shutting down...")
                    return
            if root.count(os.sep) >= self.stop_depth:
                del dirnames[:]

    def get_file_count(self):
        file_count = 0
        for root, dirnames, files in os.walk(self.input_dir):
            file_count += len(files)
            if root.count(os.sep) >= self.stop_depth:
                del dirnames[:]
        return file_count

    def get_file_type(self, mimetype):
        """
        Check if given file_type is image or video
        Return None if other
        Use mimetype to determine if the file is an image or video.
        """
        patternImage = re.compile('^(image/.+|application/vnd.adobe.photoshop)$')
        if patternImage.match(mimetype):
            return 'image'

        patternVideo = re.compile('^(video/.*)$')
        if patternVideo.match(mimetype):
            return 'video'
        return None

    def get_output_dir(self, date):
        """
        Generate output directory path based on the extracted date and
        formatted using dir_format.
        If date is missing from the exifdata the file is going to "unknown"
        directory unless user included a regex from filename or uses timestamp.
        """
        try:
            path = [self.output_dir, date['date'].date().strftime(self.dir_format)]
        except (TypeError, ValueError):
            path = [self.output_dir, self.no_date_dir]

        fullpath = os.path.sep.join(path)

        if not os.path.isdir(fullpath) and not self.dry_run:
            os.makedirs(fullpath)

        return fullpath

    def get_file_name(self, original_filename, date):
        """
        Generate file name based on exif data unless it is missing or
        original filenames are required. Then use original file name
        """
        if self.original_filenames:
            return os.path.basename(original_filename)

        try:
            filename = [
                f'{(date["date"].year):04d}',
                f'{(date["date"].month):02d}',
                f'{(date["date"].day):02d}',
                '-',
                f'{(date["date"].hour):02d}',
                f'{(date["date"].minute):02d}',
                f'{(date["date"].second):02d}',
            ]

            if date['subseconds']:
                filename.append(date['subseconds'])

            return ''.join(filename) + os.path.splitext(original_filename)[1]
        # TODO: Double check if this is correct!
        except TypeError:
            return os.path.basename(original_filename)

    def process_files(self, file_paths_to_process):
        # With all the appropriate files in the directory added to the
        # list, process the directory concurrently using threads
        with concurrent.futures.ThreadPoolExecutor(
                max_workers=self.max_concurrency) as executor:
            try:
                for _ in executor.map(self.process_file,
                                      file_paths_to_process):
                    pass
            except KeyboardInterrupt:
                logger.warning(
                        f"Received interrupt. Shutting down {self.max_concurrency} workers...")
                executor.shutdown(wait=True)
                return False
        return True

    def process_file(self, filename):
        """
        Process the file using the selected strategy
        If file is .xmp skip it so process_xmp method can handle it
        """
        if str.endswith(filename, '.xmp'):
            return None

        progress = f'{filename}'

        output, target_file_name, target_file_path, target_file_type = self.get_file_name_and_path(filename)
        suffix = 1
        target_file = target_file_path

        while True:
            if self.file_type is not None \
                    and self.file_type != target_file_type:
                progress = f"{progress} => skipped, file is '{target_file_type}' \
but looking for '{self.file_type}'"
                logger.info(progress)
                break

            if self.skip_unknown and output.endswith(UNKNOWN):
                # Skip files that didn't generate a path from EXIF data
                progress = f"{progress} => skipped, unknown date EXIF information for '{target_file_name}'"
                self.unknown_found += 1
                if self.progress:
                    self.pbar.write(progress)
                logger.info(progress)
                break

            if os.path.isfile(target_file):
                if filename != target_file and filecmp.cmp(filename, target_file, shallow=False):
                    progress = f'{progress} => skipped, duplicated file {target_file}'
                    self.duplicates_found += 1
                    if self.progress:
                        self.pbar.write(progress)
                    logger.info(progress)
                    break
            else:
                if self.move:
                    try:
                        self.files_moved += 1
                        if not self.dry_run:
                            shutil.move(filename, target_file)
                    except FileNotFoundError:
                        progress = f'{progress} => skipped, no such file or directory'
                        if self.progress:
                            self.pbar.write(progress)
                        logger.warning(progress)
                        break
                elif self.link and not self.dry_run:
                    os.link(filename, target_file)
                else:
                    try:
                        self.files_copied += 1
                        if not self.dry_run:
                            shutil.copy2(filename, target_file)
                    except FileNotFoundError:
                        progress = f'{progress} => skipped, no such file or directory'
                        if self.progress:
                            self.pbar.write(progress)
                        logger.warning(progress)
                        break

                progress = f'{progress} => {target_file}'
                if self.progress:
                    self.pbar.write(progress)
                logger.info(progress)

                self.process_xmp(filename, target_file_name, suffix, output)
                break

            suffix += 1
            target_split = os.path.splitext(target_file_path)
            target_file = f'{target_split[0]}-{suffix}{target_split[1]}'

        self.files_processed += 1
        if self.progress:
            self.pbar.update(1)

    def get_file_name_and_path(self, filename):
        """
        Returns target file name and path
        """
        exif_data = Exif(filename).data()
        target_file_type = None

        if exif_data and 'MIMEType' in exif_data:
            target_file_type = self.get_file_type(exif_data['MIMEType'])

        if target_file_type in ['image', 'video']:
            date = Date(filename).from_exif(exif_data, self.timestamp, self.date_regex,
                                            self.date_field)
            output = self.get_output_dir(date)
            target_file_name = self.get_file_name(filename, date)
            if not self.original_filenames:
                target_file_name = target_file_name.lower()
        else:
            output = self.get_output_dir([])
            target_file_name = os.path.basename(filename)

        target_file_path = os.path.sep.join([output, target_file_name])
        return output, target_file_name, target_file_path, target_file_type

    def process_xmp(self, original_filename, file_name, suffix, output):
        """
        Process xmp files. These are meta data for RAW images
        """
        xmp_original_with_ext = original_filename + '.xmp'
        xmp_original_without_ext = os.path.splitext(original_filename)[0] + '.xmp'

        suffix = f'-{suffix}' if suffix > 1 else ''

        xmp_files = {}

        if os.path.isfile(xmp_original_with_ext):
            xmp_target = f'{file_name}{suffix}.xmp'
            xmp_files[xmp_original_with_ext] = xmp_target
        if os.path.isfile(xmp_original_without_ext):
            xmp_target = f'{(os.path.splitext(file_name)[0])}{suffix}.xmp'
            xmp_files[xmp_original_without_ext] = xmp_target

        for original, target in xmp_files.items():
            xmp_path = os.path.sep.join([output, target])
            logger.info(f'{original} => {xmp_path}')

            if not self.dry_run:
                if self.move:
                    shutil.move(original, xmp_path)
                elif self.link:
                    os.link(original, xmp_path)
                else:
                    shutil.copy2(original, xmp_path)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

#   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

from collections import defaultdict
import matplotlib.pyplot as plt
import re
import os
import pandas as pd
import sys
import traceback

import bayeslite
import bayeslite.core
from bayeslite.loggers import BqlLogger, logged_query
from bayeslite.exception import BayesLiteException as BLE
import bdbcontrib
import bdbcontrib.plot_utils # To helpsub from it.
from bdbcontrib.population import Population
from bdbcontrib.population_method import population_method
from py_utils import helpsub

BqlRecipes = Population

@population_method(population=0)
def quick_explore_vars(self, varnames, plotfile=None, nsimilar=20):
  """Show dependence probabilities and neighborhoods based on those.

  varnames: list of strings
      At least two column names to look at dependence probabilities of,
      and to explore neighborhoods of.
  nsimilar: positive integer
      The size of the neighborhood to explore.
  """
  if len(varnames) < 2:
    raise BLE(ValueError('Need to explore at least two variables.'))
  self.pairplot_vars(varnames)
  query_columns = '''"%s"''' % '''", "'''.join(varnames)
  deps = self.query('''ESTIMATE DEPENDENCE PROBABILITY
                       FROM PAIRWISE COLUMNS OF %s
                       FOR %s;''' % (self.generator_name, query_columns))
  deps.columns = ['genid', 'name0', 'name1', 'value']
  self.logger.plot(plotfile, self.heatmap(deps))
  deps.columns = ['genid', 'name0', 'name1', 'value']
  triangle = deps[deps['name0'] < deps['name1']]
  triangle = triangle.sort_values(ascending=False, by=['value'])
  self.logger.result("Pairwise dependence probability for: %s\n%s\n\n",
                     query_columns, triangle)

  for col in varnames:
    neighborhood = self.query(
    '''ESTIMATE *, DEPENDENCE PROBABILITY WITH "%s"
       AS "Probability of Dependence with %s"
       FROM COLUMNS OF %s
       ORDER BY "Probability of Dependence with %s"
       DESC LIMIT %d;'''
       % (col, col, self.generator_name, col, nsimilar))
    neighbor_columns = ('''"%s"''' %
                        '''", "'''.join(neighborhood["name"].tolist()))
    deps = self.query('''ESTIMATE DEPENDENCE PROBABILITY
        FROM PAIRWISE COLUMNS OF %s
        FOR %s;''' % (self.generator_name, neighbor_columns))
    deps.columns = ['genid', 'name0', 'name1', 'value']
    self.logger.plot(plotfile, self.heatmap(deps))
    self.logger.result("Pairwise dependence probability of %s with its " +
                       "strongest dependents:\n%s\n\n", col, neighborhood)

@population_method(population=0)
def quick_similar_rows(self, identify_row_by, nsimilar=10):
  """Explore rows similar to the identified one.

  identify_row_by : dict
      Dictionary of column names to their values. These will be turned into
      a WHERE clause in BQL, and must identify one unique row.
  nsimilar : positive integer
      The number of similar rows to retrieve.
  """
  import hashlib
  table_name = 'tmptbl_' + hashlib.md5('\x00'.join(
      [repr(identify_row_by), str(self.status)])).hexdigest()
  column_name = 'similarity_to_' + "__".join(
      re.sub(r'\W', '_', str(val)) for val in identify_row_by.values())
  query_params = []
  query_columns = []
  for k, v in identify_row_by.iteritems():
    query_columns.append('''%s = ? ''' % bayeslite.bql_quote_name(k))
    query_params.append(v)
  query_attrs = ' and '.join(query_columns)

  with self.bdb.savepoint():
    row_exists = self.query('SELECT COUNT(*) FROM %s WHERE %s;' %
                            (self.name, query_attrs))
    if row_exists.ix[0][0] != 1:
      raise BLE(NotImplementedError(
          'identify_row_by found %d rows instead of exactly 1 in %s.' %
          (row_exists.ix[0][0], self.csv_path)))
    creation_query = ('''CREATE TEMP TABLE IF NOT EXISTS %s AS ESTIMATE *,
                         SIMILARITY TO (%s) AS %s FROM %%g LIMIT %d;''' %
                      (table_name, query_attrs, column_name, nsimilar))
    self.query(creation_query, query_params)
    result = self.query('''SELECT * FROM %s ORDER BY %s DESC;''' %
                        (table_name, column_name))
  return result

quickstart = bdbcontrib.quickstart
<commit_msg>Cleaning up old imports and aliases.<commit_after># -*- coding: utf-8 -*-

#   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

# pylint: disable=no-member

# These methods are for analyses that involve some amount of
# data-analytical judgment encoded in the procedure and defaults, so
# they are not intended as primitives, but as suggested
# starting-points for a user's more meaningful analysis.

import re

import bayeslite
from bayeslite.exception import BayesLiteException as BLE
import bdbcontrib
from bdbcontrib.population import Population
from bdbcontrib.population_method import population_method
from py_utils import helpsub

@population_method(population=0)
def quick_explore_vars(self, varnames, plotfile=None, nsimilar=20):
  """Show dependence probabilities and neighborhoods based on those.

  varnames: list of strings
      At least two column names to look at dependence probabilities of,
      and to explore neighborhoods of.
  nsimilar: positive integer
      The size of the neighborhood to explore.
  """
  if len(varnames) < 2:
    raise BLE(ValueError('Need to explore at least two variables.'))
  self.pairplot_vars(varnames)
  query_columns = '''"%s"''' % '''", "'''.join(varnames)
  deps = self.query('''ESTIMATE DEPENDENCE PROBABILITY
                       FROM PAIRWISE COLUMNS OF %s
                       FOR %s;''' % (self.generator_name, query_columns))
  deps.columns = ['genid', 'name0', 'name1', 'value']
  self.logger.plot(plotfile, self.heatmap(deps))
  deps.columns = ['genid', 'name0', 'name1', 'value']
  triangle = deps[deps['name0'] < deps['name1']]
  triangle = triangle.sort_values(ascending=False, by=['value'])
  self.logger.result("Pairwise dependence probability for: %s\n%s\n\n",
                     query_columns, triangle)

  for col in varnames:
    neighborhood = self.query(
    '''ESTIMATE *, DEPENDENCE PROBABILITY WITH "%s"
       AS "Probability of Dependence with %s"
       FROM COLUMNS OF %s
       ORDER BY "Probability of Dependence with %s"
       DESC LIMIT %d;'''
       % (col, col, self.generator_name, col, nsimilar))
    neighbor_columns = ('''"%s"''' %
                        '''", "'''.join(neighborhood["name"].tolist()))
    deps = self.query('''ESTIMATE DEPENDENCE PROBABILITY
        FROM PAIRWISE COLUMNS OF %s
        FOR %s;''' % (self.generator_name, neighbor_columns))
    deps.columns = ['genid', 'name0', 'name1', 'value']
    self.logger.plot(plotfile, self.heatmap(deps))
    self.logger.result("Pairwise dependence probability of %s with its " +
                       "strongest dependents:\n%s\n\n", col, neighborhood)

@population_method(population=0)
def quick_similar_rows(self, identify_row_by, nsimilar=10):
  """Explore rows similar to the identified one.

  identify_row_by : dict
      Dictionary of column names to their values. These will be turned into
      a WHERE clause in BQL, and must identify one unique row.
  nsimilar : positive integer
      The number of similar rows to retrieve.
  """
  import hashlib
  table_name = 'tmptbl_' + hashlib.md5('\x00'.join(
      [repr(identify_row_by), str(self.status)])).hexdigest()
  column_name = 'similarity_to_' + "__".join(
      re.sub(r'\W', '_', str(val)) for val in identify_row_by.values())
  query_params = []
  query_columns = []
  for k, v in identify_row_by.iteritems():
    query_columns.append('''%s = ? ''' % bayeslite.bql_quote_name(k))
    query_params.append(v)
  query_attrs = ' and '.join(query_columns)

  with self.bdb.savepoint():
    row_exists = self.query('SELECT COUNT(*) FROM %s WHERE %s;' %
                            (self.name, query_attrs))
    if row_exists.ix[0][0] != 1:
      raise BLE(NotImplementedError(
          'identify_row_by found %d rows instead of exactly 1 in %s.' %
          (row_exists.ix[0][0], self.csv_path)))
    creation_query = ('''CREATE TEMP TABLE IF NOT EXISTS %s AS ESTIMATE *,
                         SIMILARITY TO (%s) AS %s FROM %%g LIMIT %d;''' %
                      (table_name, query_attrs, column_name, nsimilar))
    self.query(creation_query, query_params)
    result = self.query('''SELECT * FROM %s ORDER BY %s DESC;''' %
                        (table_name, column_name))
  return result
<|endoftext|>


<commit_before>from _spotify.session import Session

class SpotifySessionManager:
    def __init__(self):
        pass

    def logged_in(self):
        pass

    def logged_out(self):
        pass

    def metadata_updated(self):
        pass

    def connection_error(self):
        pass

    def message_to_user(self):
        pass

    def notify_main_thread(self):
        pass

    def search(self, query):
        pass

    def music_delivery(self, format, frames, num_frames):
        pass

    def play_token_lost(self):
        pass

    def log_message(self, message):
        pass

    def end_of_track(self):
        pass
<commit_msg>Removed duplicate source file.<commit_after><|endoftext|>


<commit_before># stateSpace.py - state space class for control systems library
#
# Author: Richard M. Murray
# Date: 24 May 09
#
# This file contains the StateSpace class, which is used to represent
# linear systems in state space.  This is the primary representation
# for the control system library.
#
# Copyright (c) 2009 by California Institute of Technology
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# 3. Neither the name of the California Institute of Technology nor
#    the names of its contributors may be used to endorse or promote
#    products derived from this software without specific prior
#    written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CALTECH
# OR THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
# $Id$

import scipy as sp
import scipy.signal as signal
import xferfcn
from scipy import concatenate, zeros

#
# StateSpace class
#
# The StateSpace class is used throughout the control systems library to
# represent systems in statespace form.  This class is derived from
# the ltisys class defined in the scipy.signal package, allowing many
# of the functions that already existing in that package to be used
# directly.
#
class StateSpace(signal.lti):
    """The StateSpace class is used to represent linear input/output systems.
    """
    # Initialization
    def __init__(self, *args, **keywords):
        # First initialize the parent object
        signal.lti.__init__(self, *args, **keywords)

    # Style to use for printing
    def __str__(self):
        str =  "A = " + self.A.__str__() + "\n\n"
        str += "B = " + self.B.__str__() + "\n\n"
        str += "C = " + self.C.__str__() + "\n\n"
        str += "D = " + self.D.__str__() + "\n"
        return str

    # Method for generating the frequency response of the system
    def freqresp(self, omega=None):
        # Generate and save a transfer function matrix
        #! This is currently limited to SISO systems
        nout, nin = self.D.shape

        # Compute the denominator from the A matrix
        den = sp.poly1d(sp.poly(self.A))

        # Compute the numerator based on zeros
        #! This is currently limited to SISO systems
        num = sp.poly1d(\
            sp.poly(self.A - sp.dot(self.B, self.C)) + (self.D[0] - 1) * den)

        # Generate the frequency response at each frequency
        fresp = map(lambda w: num(w*1j) / den(w*1j), omega)
        mag = sp.sqrt(sp.multiply(fresp, sp.conjugate(fresp)))
        phase = sp.angle(fresp)

        return mag, phase, omega

    # Method for evaluating a system at one frequency
    def evalfr(self, freq):
        #! Not implemented
        return None

    # Negation of a system
    def __neg__(self):
        return StateSpace(self.A, self.B, -self.C, -self.D)

    # Addition of two transfer functions (parallel interconnection)
    def __add__(self, other):
        # Check for a couple of special cases
        if (isinstance(other, (int, long, float, complex))):
            # Just adding a scalar; put it in the D matrix
            A, B, C = self.A, self.B, self.C;
            D = self.D + other;

        else:
            # Check to make sure the dimensions are OK
            if ((self.inputs != other.inputs) or \
                    (self.outputs != other.outputs)):
                raise ValueError, "Systems have different shapes."

            # Concatenate the various arrays
            #! Pretty sure this is not correct
            A = concatenate((
                    concatenate((self.A, zeros((self.A.shape[0],
                                                other.A.shape[-1]))), axis=1),
                    concatenate((zeros((other.A.shape[0], self.A.shape[-1])),
                                other.A), axis=1)), axis=0)
            B = concatenate((self.B, other.B))
            C = concatenate((self.C, other.C), axis=1)
            D = self.D + other.D

        return StateSpace(A, B, C, D)

    # Reverse addition - just switch the arguments
    def __radd__(self, other): return self.__add__(other)

    # Subtraction of two transfer functions (parallel interconnection)
    def __sub__(self, other):
        return __add__(self, other.__neg__())

    # Multiplication of two transfer functions (series interconnection)
    def __mul__(self, other):
        # Check for a couple of special cases
        if (isinstance(other, (int, long, float, complex))):
            # Just multiplying by a scalar; change the output
            A, B = self.A, self.B;
            C = self.C * other;
            D = self.D * other;

        else:
            # Check to make sure the dimensions are OK
            if ((self.inputs != other.inputs) or
                (self.outputs != other.outputs)):
                raise ValueError, "State space systems have different shapes."

            # Concatenate the various arrays
            A = concatenate((
                    concatenate((self.A, zeros((self.A.shape[0],
                                                other.A.shape[-1]))), axis=1),
                    concatenate((other.B * self.C, other.A), axis=1)))
            B = concatenate((self.B, other.B * self.D))
            C = concatenate((other.D * self.C, other.C), axis=1)
            D = other.D * self.D

        return StateSpace(A, B, C, D)

    # Reverse multiplication of two transfer functions (series interconnection)
    # Just need to convert LH argument to a state space object
    def __rmul__(self, other):
        # Check for a couple of special cases
        if (isinstance(other, (int, long, float, complex))):
            # Just multiplying by a scalar; change the input
            A, C = self.A, self.C;
            B = self.B * other;
            D = self.D * other;
            return StateSpace(A, B, C, D)

        else:
            raise TypeError("can't interconnect systems")

    # Feedback around a state space system
    def feedback(self, other, sign=-1):
        # Check for special cases
        if (isinstance(other, (int, long, float, complex))):
            # Scalar feedback - easy to include
            A = self.A + sign*self.B*self.C;
            B = self.B + sign*self.B*self.D;
            C, D = self.C, self.D;

        else:
            # Check to make sure the dimensions are OK
            if ((self.inputs != other.inputs) or
                (self.outputs != other.outputs)):
                raise ValueError, "State space systems have different shapes."

            # Concatenate the various arrays
            sys1, sys2 = self, other;

            # Make sure that we don't have an algebraic loop
            #! Not implemented

            #! Pretty sure this is not correct
            A = concatenate((
                concatenate((sys1.A, sys1.B * sys2.C), axis=1),
                concatenate((sys2.B * sys1.C, sys2.A), axis=1)));
            B = concatenate((sys1.B, sys2.B * sys1.D));
            C = concatenate((sys1.C, sys1.D * sys2.C), axis=1);
            D = sys1.D;

        return StateSpace(A, B, C, D)

#
# convertToStateSpace - create a state space system from another type
#
# To allow scalar constants to be used in a simple way (k*P, 1+L), this
# function allows the dimension of the input/output system to be specified
# in the case of a scalar system
#
def convertToStateSpace(sys, inputs=1, outputs=1):
    if (isinstance(sys, StateSpace) or
        isinstance(sys, xferfcn.TransferFunction)):
        # Already a state space system; just return it
        return sys

    elif (isinstance(sys, (int, long, float, complex))):
        # Generate a simple state space system of the desired dimension
        #! Doesn't work due to inconsistencies in ltisys
        # return StateSpace([[]], [[]], [[]], sp.eye(outputs, inputs))
        return StateSpace(-1, 0, 0, sp.eye(outputs, inputs))

    else:
        raise TypeError("can't convert given type to StateSpace system")
<commit_msg>fixed and tested adding systems in parallel and series<commit_after># stateSpace.py - state space class for control systems library
#
# Author: Richard M. Murray
# Date: 24 May 09
#
# This file contains the StateSpace class, which is used to represent
# linear systems in state space.  This is the primary representation
# for the control system library.
#
# Copyright (c) 2009 by California Institute of Technology
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# 3. Neither the name of the California Institute of Technology nor
#    the names of its contributors may be used to endorse or promote
#    products derived from this software without specific prior
#    written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CALTECH
# OR THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
# $Id$

import scipy as sp
import scipy.signal as signal
import xferfcn
from scipy import concatenate, zeros

#
# StateSpace class
#
# The StateSpace class is used throughout the control systems library to
# represent systems in statespace form.  This class is derived from
# the ltisys class defined in the scipy.signal package, allowing many
# of the functions that already existing in that package to be used
# directly.
#
class StateSpace(signal.lti):
    """The StateSpace class is used to represent linear input/output systems.
    """
    # Initialization
    def __init__(self, *args, **keywords):
        # First initialize the parent object
        signal.lti.__init__(self, *args, **keywords)

    # Style to use for printing
    def __str__(self):
        str =  "A = " + self.A.__str__() + "\n\n"
        str += "B = " + self.B.__str__() + "\n\n"
        str += "C = " + self.C.__str__() + "\n\n"
        str += "D = " + self.D.__str__() + "\n"
        return str

    # Method for generating the frequency response of the system
    def freqresp(self, omega=None):
        # Generate and save a transfer function matrix
        #! This is currently limited to SISO systems
        nout, nin = self.D.shape

        # Compute the denominator from the A matrix
        den = sp.poly1d(sp.poly(self.A))

        # Compute the numerator based on zeros
        #! This is currently limited to SISO systems
        num = sp.poly1d(\
            sp.poly(self.A - sp.dot(self.B, self.C)) + (self.D[0] - 1) * den)

        # Generate the frequency response at each frequency
        fresp = map(lambda w: num(w*1j) / den(w*1j), omega)
        mag = sp.sqrt(sp.multiply(fresp, sp.conjugate(fresp)))
        phase = sp.angle(fresp)

        return mag, phase, omega

    # Method for evaluating a system at one frequency
    def evalfr(self, freq):
        #! Not implemented
        return None

    # Negation of a system
    def __neg__(self):
        return StateSpace(self.A, self.B, -self.C, -self.D)

    # Addition of two transfer functions (parallel interconnection)
    def __add__(self, other):
        # Check for a couple of special cases
        if (isinstance(other, (int, long, float, complex))):
            # Just adding a scalar; put it in the D matrix
            A, B, C = self.A, self.B, self.C;
            D = self.D + other;

        else:
            # Check to make sure the dimensions are OK
            if ((self.inputs != other.inputs) or \
                    (self.outputs != other.outputs)):
                raise ValueError, "Systems have different shapes."

            # Concatenate the various arrays
            A = concatenate((
                concatenate((self.A, zeros((self.A.shape[0],
                                           other.A.shape[-1]))),axis=1),
                concatenate((zeros((other.A.shape[0], self.A.shape[-1])),
                                other.A),axis=1)
                            ),axis=0)
            B = concatenate((self.B, other.B), axis=0)
            C = concatenate((self.C, other.C), axis=1)
            D = self.D + other.D
        return StateSpace(A, B, C, D)

    # Reverse addition - just switch the arguments
    def __radd__(self, other): return self.__add__(other)

    # Subtraction of two transfer functions (parallel interconnection)
    def __sub__(self, other):
        return __add__(self, other.__neg__())

    # Multiplication of two transfer functions (series interconnection)
    def __mul__(self, other):
        # Check for a couple of special cases
        if (isinstance(other, (int, long, float, complex))):
            # Just multiplying by a scalar; change the output
            A, B = self.A, self.B;
            C = self.C * other;
            D = self.D * other;

        else:
           # Check to make sure the dimensions are OK
           if (self.outputs != other.inputs):
               raise ValueError, "Number of first's outputs must match number \
                                    of second's inputs."

           # Concatenate the various arrays
           A = concatenate((
                   concatenate(( self.A, zeros((self.A.shape[0],
                                            other.A.shape[-1]))   ),axis=1),
                   concatenate(( other.B*self.C,  other.A  ),axis=1),
                   ),axis=0)
           B = concatenate( (self.B, other.B*self.D), axis=0 )
           C = concatenate( (other.D*self.C, other.C), axis=1 )
           D = other.D*self.D
        return StateSpace(A, B, C, D)

    # Reverse multiplication of two transfer functions (series interconnection)
    # Just need to convert LH argument to a state space object
    def __rmul__(self, other):
        # Check for a couple of special cases
        if (isinstance(other, (int, long, float, complex))):
            # Just multiplying by a scalar; change the input
            A, C = self.A, self.C;
            B = self.B * other;
            D = self.D * other;
            return StateSpace(A, B, C, D)

        else:
            raise TypeError("can't interconnect systems")

    # Feedback around a state space system
    def feedback(self, other, sign=-1):
        # Check for special cases
        if (isinstance(other, (int, long, float, complex))):
            # Scalar feedback - easy to include
            A = self.A + sign*self.B*self.C;
            B = self.B + sign*self.B*self.D;
            C, D = self.C, self.D;

        else:
            # Check to make sure the dimensions are OK
            if ((self.inputs != other.inputs) or
                (self.outputs != other.outputs)):
                raise ValueError, "State space systems have different shapes."

            # Concatenate the various arrays
            sys1, sys2 = self, other;

            # Make sure that we don't have an algebraic loop
            #! Not implemented

            #! Pretty sure this is not correct
            A = concatenate((
                concatenate((sys1.A, sys1.B * sys2.C), axis=1),
                concatenate((sys2.B * sys1.C, sys2.A), axis=1)));
            B = concatenate((sys1.B, sys2.B * sys1.D));
            C = concatenate((sys1.C, sys1.D * sys2.C), axis=1);
            D = sys1.D;

        return StateSpace(A, B, C, D)

#
# convertToStateSpace - create a state space system from another type
#
# To allow scalar constants to be used in a simple way (k*P, 1+L), this
# function allows the dimension of the input/output system to be specified
# in the case of a scalar system
#
def convertToStateSpace(sys, inputs=1, outputs=1):
    if (isinstance(sys, StateSpace) or
        isinstance(sys, xferfcn.TransferFunction)):
        # Already a state space system; just return it
        return sys

    elif (isinstance(sys, (int, long, float, complex))):
        # Generate a simple state space system of the desired dimension
        #! Doesn't work due to inconsistencies in ltisys
        # return StateSpace([[]], [[]], [[]], sp.eye(outputs, inputs))
        return StateSpace(-1, 0, 0, sp.eye(outputs, inputs))

    else:
        raise TypeError("can't convert given type to StateSpace system")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import socket
import time

import pymongo

from src.tools import decode_to_string, in_range

class Tester(object):
    """
    Tester
    Args:
      cred (dict(str: str)): credentials
      tests (Test[]): tests to run
    """

    def __init__(self, cred, tests):
        self.cred = cred
        self.tests = [Test(t, self) for t in tests]
        self.conn = self.get_connection()
        self.info = self.get_info()
        self.database = None

    def run(self, each, end):
        """
        Args:
          each (): function to call before each test runs (used to update and redraw the screen)
          end (function): function to call when all the tests have finished running
        """
        result = []
        for test in self.tests:
            each(test)
            res = test.run()
            result.append(res)
            time.sleep(0.5)
            if test.breaks == bool(res['result']):
                break

        if len(result) < len(self.tests):
            result += [{
                'name': x.name,
                'severity': x.severity,
                'title': x.title,
                'caption': x.caption,
                'message': 'This test was omitted',
                'extra_data': None,
                'result': 3} for x in self.tests[len(result): len(self.tests)]]

        self.conn.close()
        end(result)

    def get_connection(self):
        """
        Returns:
          pymongo.MongoClient: client that takes as arguments (fqdn, port)

        """
        fqdn, port = self.cred['nodelist'][0]
        return pymongo.MongoClient(fqdn, port, serverSelectionTimeoutMS=1)

    def get_info(self):
        """
        Returns:
          dict() or None: Get information about the MongoDB server we’re connected to.
        Note:
          https://docs.mongodb.com/v3.2/reference/command/buildInfo/#dbcmd.buildInfo
        """
        if hasattr(self, 'info'):
            return self.info
        try:
            info = self.conn.server_info()
            self.info = info
            return self.info
        except pymongo.errors.ConnectionFailure:
            return None

    def get_db(self):
        """
        Authenticates to database
        Returns:
          pymongo.database.Database or False:  database(singleton) or false if authentication fails
        """
        if bool(self.database):
            return self.database
        try:
            database = self.conn[self.cred['database']]
            database.authenticate(self.cred['username'], self.cred['password'])
            self.database = database
            return self.database
        except (pymongo.errors.OperationFailure, ValueError, TypeError):
            return False

    def get_roles(self):
        data_base = self.get_db()
        return data_base.command('usersInfo', self.cred['username'])['users'][0]

class Test(object):
    def __init__(self, test, tester):
        """
        Args:
          test dict():
          tester Tester:

        Notes:
          message should contain an array with the messages for the different results that
          the test can return, if message is a function it should return a str[]
        """
        self.name, self.severity, self.title, self.caption, self.message = \
            test['test_name'], test['severity'], test['title'], test['caption'], test['messages']
        self.breaks = test['breaks'] if 'breaks' in test else None
        self.tester = tester

    def run(self):
        """
        Returns:
          tuple(int, str): test result value and message
        """
        result = TEST_FUNCTIONS[self.name](self)
        message = self.message

        if isinstance(result, list):
            value = result[0]
            extra_data = result[1]
        else:
            value = result
            extra_data = None

        return {'name': self.name, 'severity': self.severity, 'title': self.title,
                'caption': self.caption, 'message': message[value], 'result': value,
                'extra_data': extra_data}

def try_socket(test, forced_port=None):
    fqdn, port = test.tester.cred['nodelist'][0]
    try:
        socket.create_connection((fqdn, forced_port or port), 5)
    except socket.error:
        return True
    else:
        return False

def try_authorization(test):
    try:
        test.tester.conn.database_names()
    except pymongo.errors.OperationFailure:
        return True
    else:
        return False

def try_credentials(test):
    """
    verify if the credentials are valid
    """
    return bool(test.tester.get_db())

def try_javascript(test):
    """
    Check if javascript is enabled in MongoDB
    """
    try:
        database = test.tester.get_db()
        database.test.find_one({"$where": "function() { return true;}"})
    except pymongo.errors.OperationFailure:
        return True
    else:
        return False

def try_roles(test):
    """
    Verify that the user roles are not administrative
    Returns:
      [bool or 2 , str ]: True Valid role, False invalid role, 2 custome role ,
    """

    def valid_role(role):
        """
        Args:
          role (str): name of a role
        Returns:
          Bool: True if the role is not administrative
        """
        return role not in [
            'userAdminAnyDatabase',
            'dbAdminAnyDatabase'
            'dbAdmin',
            'dbOwner',
            'userAdmin',
            'clusterAdmin']

    def result_default_value():
        """

        Returns:
          dict(set()): returns an empty dictionary that contains 3 empty
          sets where valid, invalid and custom roles are stored
        """
        return {'invalid': set([]), 'valid': set([]), 'custom': set([])}

    def combine_result(value_1, value_2):
        """
        Args:
          value_1 (dict(set())): result_default_value
          value_2 (dict(set())): result_default_value
        Returns:
          dict(set()): the union of 2 default values
        """
        return {'invalid': value_1['invalid'].union(value_2['invalid']),
                'valid': value_1['valid'].union(value_2['valid']),
                'custom': value_1['custom'].union(value_2['custom'])}

    def reduce_roles(roles):
        """
        Validate and combine a list of roles
        Args:
         roles (list(dict())): roles
        Returns:
          result_default_value: roles sorted by valid, invalid, custom
        """
        return reduce(lambda x, y: combine_result(x, y), [validate_role(r) for r in roles])

    def basic_validation(roles):
        """
        Basic validation in case validate role fails due to lack of permissions
        Returns:
          result_default_value
        """
        validated = result_default_value()
        for role in roles['roles']:
            if valid_role(role['role']):
                validated['valid'].add(role['role'])
            else:
                validated['invalid'].add(role['role'])
        return validated

    def validate_role(role):
        """
        Recursively process the different roles that a role implements or inherits
        Args:
          role (list or dict): value returned by database.command 'usersInfo' or 'rolesInfo'
        """
        if isinstance(role, list) and bool(role):
            return reduce_roles(role)
        elif isinstance(role, dict):
            if 'role' in role and 'isBuiltin' in role:
                result = result_default_value()
                is_valid_role = valid_role(role['role'])
                if is_valid_role and role['isBuiltin']:
                    result['valid'].add(role['role'])
                elif is_valid_role:
                    result['custom'].add(role['role'])
                else:
                    result['invalid'].add(role['role'])
                inherited = validate_role(
                    role['inheritedRoles']) if 'inheritedRoles' in role else result_default_value()
                other_roles = validate_role(
                    role['roles']) if 'roles' in role else result_default_value()
                return combine_result(
                    result, combine_result(
                        inherited, other_roles))
            if 'role' in role:
                return validate_role(database.command('rolesInfo', role)['roles'])
            if 'roles' in role and bool(role['roles']):
                return validate_role(role['roles'])
            else:
                raise Exception('Non exhaustive type case')
        elif isinstance(role, list):
            # empty list
            return result_default_value()
        else:
            raise Exception('Non exhaustive type case')

    database = test.tester.get_db()
    roles = test.tester.get_roles()
    validated = {}

    def get_message(state, text1, text2):
        return text1 + decode_to_string(validated[state]) + text2

    try:
        validated = validate_role(roles)
    except pymongo.errors.OperationFailure:
        # if the users doesn't have permission to run the command 'rolesInfo'
        validated = basic_validation(roles)
        if bool(validated['valid']):
            message = get_message('valid', 'You user permission ',
                                  ' didn\'t allow us to do an exhaustive check')
            return [2, message]

    # when user has permission to run 'rolesInfo'
    if bool(validated['invalid']):
        # if the profile is invalid
        return [False, decode_to_string(validated['invalid'])]
    elif bool(validated['custom']):
        # if the profile has custom permissions
        message = get_message('valid', 'Your user\'s role set ',
                              ' seem to be ok, but we couldn\'t do an exhaustive check.')
        return [2, message]
    # if everything seems to be ok
    return [True, decode_to_string(validated['valid'])]

def try_dedicated_user(test):
    """
    Verify that the role only applies to one database
    """
    roles = test.tester.get_roles()
    user_role_dbs = set()
    for role in roles['roles']:
        user_role_dbs.add(role['db'])

    return [bool(len(user_role_dbs)), decode_to_string(user_role_dbs)]

def try_scram(test):
    try:
        conn = test.tester.get_connection()
        cred = test.tester.cred
        return conn[cred["database"]]\
            .authenticate(cred["username"], cred["password"], mechanism='SCRAM-SHA-1')
    except (pymongo.errors.OperationFailure, ValueError, TypeError):
        return False

# the following functions are for https://www.mongodb.com/alerts security related

def alerts_dec012015(test):
    if "modules" in test.tester.info:
        enterprise = "enterprise" in test.tester.info["modules"]
        version = test.tester.info["version"]
        return not (enterprise and in_range(version, "3.0.0", "3.0.6"))
    else:
        return True

def alerts_mar272015(test):
    return test.tester.info["version"] != "3.0.0"

def alerts_mar252015(test):
    version = test.tester.info["version"]
    return version > "2.6.8" and version != "3.0.0"

def alerts_feb252015(test):
    version = test.tester.info["version"]
    return version > "2.6.7" or version == "2.4.13"

def alerts_jun172014(test):
    version = test.tester.info["version"]
    return version not in ["2.6.0", "2.6.1"]

def alerts_may052014(test):
    version = test.tester.info["version"]
    return version != "2.6.0"

def alerts_jun202013(test):
    version = test.tester.info["version"]
    return not in_range(version, "2.4.0", "2.4.4") and version != "2.5.1"

def alerts_jun052013(test):
    version = test.tester.info["version"]
    return not in_range(version, "2.4.0", "2.4.4")

def alerts_mar062014(test):
    version = test.tester.info["version"]
    return version > "2.3.1"

def alerts_oct012013(test):
    version = test.tester.info["version"]
    return version > "2.2.3"

def alerts_aug152013(test):
    version = test.tester.info["version"]
    return version > "2.5.1"

TEST_FUNCTIONS = {
    "1": lambda test: not (test.tester.cred['nodelist'][0][1] == 27017 and bool(test.tester.info)),
    "2": try_socket,
    "3": lambda test: try_socket(test, 28017),
    "4": lambda test: "version" not in test.tester.info,
    "5": lambda test: [test.tester.info["version"] > "2.4", str(test.tester.info["version"])],
    "6": lambda test: bool(test.tester.info["OpenSSLVersion"])
         if ("OpenSSLVersion" in test.tester.info)
         else test.tester.info["openssl"]["running"] != "disabled",
    "7": try_authorization,
    "8": lambda test: bool(test.tester.get_db()),
    "9": try_javascript,
    "10": try_roles,
    "11": try_dedicated_user,
    "12": try_scram,
    "13": alerts_dec012015,
    "14": alerts_mar272015,
    "15": alerts_mar252015,
    "16": alerts_feb252015,
    "17": alerts_jun172014,
    "18": alerts_may052014,
    "19": alerts_jun202013,
    "20": alerts_jun052013,
    "21": alerts_mar062014,
    "22": alerts_oct012013,
    "23": alerts_aug152013
}
<commit_msg>Reduce complexity of role validation by separating its logic in to smaller functions<commit_after># -*- coding: utf-8 -*-
import socket
import time

import pymongo

from src.tools import decode_to_string, in_range

class Tester(object):
    """
    Tester
    Args:
      cred (dict(str: str)): credentials
      tests (Test[]): tests to run
    """

    def __init__(self, cred, tests):
        self.cred = cred
        self.tests = [Test(t, self) for t in tests]
        self.conn = self.get_connection()
        self.info = self.get_info()
        self.database = None

    def run(self, each, end):
        """
        Args:
          each (): function to call before each test runs (used to update and redraw the screen)
          end (function): function to call when all the tests have finished running
        """
        result = []
        for test in self.tests:
            each(test)
            res = test.run()
            result.append(res)
            time.sleep(0.5)
            if test.breaks == bool(res['result']):
                break

        if len(result) < len(self.tests):
            result += [{
                'name': x.name,
                'severity': x.severity,
                'title': x.title,
                'caption': x.caption,
                'message': 'This test was omitted',
                'extra_data': None,
                'result': 3} for x in self.tests[len(result): len(self.tests)]]

        self.conn.close()
        end(result)

    def get_connection(self):
        """
        Returns:
          pymongo.MongoClient: client that takes as arguments (fqdn, port)

        """
        fqdn, port = self.cred['nodelist'][0]
        return pymongo.MongoClient(fqdn, port, serverSelectionTimeoutMS=1)

    def get_info(self):
        """
        Returns:
          dict() or None: Get information about the MongoDB server we’re connected to.
        Note:
          https://docs.mongodb.com/v3.2/reference/command/buildInfo/#dbcmd.buildInfo
        """
        if hasattr(self, 'info'):
            return self.info
        try:
            info = self.conn.server_info()
            self.info = info
            return self.info
        except pymongo.errors.ConnectionFailure:
            return None

    def get_db(self):
        """
        Authenticates to database
        Returns:
          pymongo.database.Database or False:  database(singleton) or false if authentication fails
        """
        if bool(self.database):
            return self.database
        try:
            database = self.conn[self.cred['database']]
            database.authenticate(self.cred['username'], self.cred['password'])
            self.database = database
            return self.database
        except (pymongo.errors.OperationFailure, ValueError, TypeError):
            return False

    def get_roles(self):
        data_base = self.get_db()
        return data_base.command('usersInfo', self.cred['username'])['users'][0]

class Test(object):
    def __init__(self, test, tester):
        """
        Args:
          test dict():
          tester Tester:

        Notes:
          message should contain an array with the messages for the different results that
          the test can return, if message is a function it should return a str[]
        """
        self.name, self.severity, self.title, self.caption, self.message = \
            test['test_name'], test['severity'], test['title'], test['caption'], test['messages']
        self.breaks = test['breaks'] if 'breaks' in test else None
        self.tester = tester

    def run(self):
        """
        Returns:
          tuple(int, str): test result value and message
        """
        result = TEST_FUNCTIONS[self.name](self)
        message = self.message

        if isinstance(result, list):
            value = result[0]
            extra_data = result[1]
        else:
            value = result
            extra_data = None

        return {'name': self.name, 'severity': self.severity, 'title': self.title,
                'caption': self.caption, 'message': message[value], 'result': value,
                'extra_data': extra_data}

def try_socket(test, forced_port=None):
    fqdn, port = test.tester.cred['nodelist'][0]
    try:
        socket.create_connection((fqdn, forced_port or port), 5)
    except socket.error:
        return True
    else:
        return False

def try_authorization(test):
    try:
        test.tester.conn.database_names()
    except pymongo.errors.OperationFailure:
        return True
    else:
        return False

def try_credentials(test):
    """
    verify if the credentials are valid
    """
    return bool(test.tester.get_db())

def try_javascript(test):
    """
    Check if javascript is enabled in MongoDB
    """
    try:
        database = test.tester.get_db()
        database.test.find_one({"$where": "function() { return true;}"})
    except pymongo.errors.OperationFailure:
        return True
    else:
        return False

def try_roles(test):
    """
    Verify that the user roles are not administrative
    Returns:
      [bool or 2 , str ]: True Valid role, False invalid role, 2 custome role ,
    """

    def valid_role(role):
        """
        Args:
          role (str): name of a role
        Returns:
          Bool: True if the role is not administrative
        """
        return role not in [
            'userAdminAnyDatabase',
            'dbAdminAnyDatabase'
            'dbAdmin',
            'dbOwner',
            'userAdmin',
            'clusterAdmin']

    def result_default_value():
        """

        Returns:
          dict(set()): returns an empty dictionary that contains 3 empty
          sets where valid, invalid and custom roles are stored
        """
        return {'invalid': set([]), 'valid': set([]), 'custom': set([])}

    def combine_result(value_1, value_2):
        """
        Args:
          value_1 (dict(set())): result_default_value
          value_2 (dict(set())): result_default_value
        Returns:
          dict(set()): the union of 2 default values
        """
        return {'invalid': value_1['invalid'].union(value_2['invalid']),
                'valid': value_1['valid'].union(value_2['valid']),
                'custom': value_1['custom'].union(value_2['custom'])}

    def reduce_roles(roles):
        """
        Validate and combine a list of roles
        Args:
         roles (list(dict())): roles
        Returns:
          result_default_value: roles sorted by valid, invalid, custom
        """
        return reduce(lambda x, y: combine_result(x, y), [validate_role(r) for r in roles])

    def basic_validation(roles):
        """
        Basic validation in case validate role fails due to lack of permissions
        Returns:
          result_default_value
        """
        validated = result_default_value()
        for role in roles['roles']:
            if valid_role(role['role']):
                validated['valid'].add(role['role'])
            else:
                validated['invalid'].add(role['role'])
        return validated

    def validate_role_list(role):
        return reduce_roles(role) if bool(role) else result_default_value()

    def get_inherited_roles(role):
        return validate_role(
            role['inheritedRoles']) if 'inheritedRoles' in role else result_default_value()

    def get_other_roles(role):
        return validate_role(
            role['roles']) if 'roles' in role else result_default_value()

    def get_builtin_role(role):
        result = result_default_value()
        is_valid_role = valid_role(role['role'])
        if is_valid_role and role['isBuiltin']:
            result['valid'].add(role['role'])
        elif is_valid_role:
            result['custom'].add(role['role'])
        else:
            result['invalid'].add(role['role'])
        inherited = get_inherited_roles(role)
        other_roles = get_other_roles(role)
        return combine_result(result, combine_result(inherited, other_roles))

    def validate_role_dict(role):
        if 'role' in role:
            return get_builtin_role(role) if 'isBuiltin' in role else  \
                validate_role(database.command('rolesInfo', role)['roles'])
        if 'roles' in role and bool(role['roles']):
            return validate_role(role['roles'])
        else:
            raise Exception('Non exhaustive type case')

    def validate_role(role):
        """
        Recursively process the different roles that a role implements or inherits
        Args:
          role (list or dict): value returned by database.command 'usersInfo' or 'rolesInfo'
        """
        if isinstance(role, list):
            return validate_role_list(role)
        elif isinstance(role, dict):
            return validate_role_dict(role)
        else:
            raise Exception('Non exhaustive type case')

    database = test.tester.get_db()
    roles = test.tester.get_roles()
    validated = {}

    def get_message(state, text1, text2):
        return text1 + decode_to_string(validated[state]) + text2

    try:
        validated = validate_role(roles)
    except pymongo.errors.OperationFailure:
        # if the users doesn't have permission to run the command 'rolesInfo'
        validated = basic_validation(roles)
        if bool(validated['valid']):
            message = get_message('valid', 'You user permission ',
                                  ' didn\'t allow us to do an exhaustive check')
            return [2, message]

    # when user has permission to run 'rolesInfo'
    if bool(validated['invalid']):
        # if the profile is invalid
        return [False, decode_to_string(validated['invalid'])]
    elif bool(validated['custom']):
        # if the profile has custom permissions
        message = get_message('valid', 'Your user\'s role set ',
                              ' seem to be ok, but we couldn\'t do an exhaustive check.')
        return [2, message]
    # if everything seems to be ok
    return [True, decode_to_string(validated['valid'])]

def try_dedicated_user(test):
    """
    Verify that the role only applies to one database
    """
    roles = test.tester.get_roles()
    user_role_dbs = set()
    for role in roles['roles']:
        user_role_dbs.add(role['db'])

    return [bool(len(user_role_dbs)), decode_to_string(user_role_dbs)]

def try_scram(test):
    try:
        conn = test.tester.get_connection()
        cred = test.tester.cred
        return conn[cred["database"]]\
            .authenticate(cred["username"], cred["password"], mechanism='SCRAM-SHA-1')
    except (pymongo.errors.OperationFailure, ValueError, TypeError):
        return False

# the following functions are for https://www.mongodb.com/alerts security related

def alerts_dec012015(test):
    if "modules" in test.tester.info:
        enterprise = "enterprise" in test.tester.info["modules"]
        version = test.tester.info["version"]
        return not (enterprise and in_range(version, "3.0.0", "3.0.6"))
    else:
        return True

def alerts_mar272015(test):
    return test.tester.info["version"] != "3.0.0"

def alerts_mar252015(test):
    version = test.tester.info["version"]
    return version > "2.6.8" and version != "3.0.0"

def alerts_feb252015(test):
    version = test.tester.info["version"]
    return version > "2.6.7" or version == "2.4.13"

def alerts_jun172014(test):
    version = test.tester.info["version"]
    return version not in ["2.6.0", "2.6.1"]

def alerts_may052014(test):
    version = test.tester.info["version"]
    return version != "2.6.0"

def alerts_jun202013(test):
    version = test.tester.info["version"]
    return not in_range(version, "2.4.0", "2.4.4") and version != "2.5.1"

def alerts_jun052013(test):
    version = test.tester.info["version"]
    return not in_range(version, "2.4.0", "2.4.4")

def alerts_mar062014(test):
    version = test.tester.info["version"]
    return version > "2.3.1"

def alerts_oct012013(test):
    version = test.tester.info["version"]
    return version > "2.2.3"

def alerts_aug152013(test):
    version = test.tester.info["version"]
    return version > "2.5.1"

TEST_FUNCTIONS = {
    "1": lambda test: not (test.tester.cred['nodelist'][0][1] == 27017 and bool(test.tester.info)),
    "2": try_socket,
    "3": lambda test: try_socket(test, 28017),
    "4": lambda test: "version" not in test.tester.info,
    "5": lambda test: [test.tester.info["version"] > "2.4", str(test.tester.info["version"])],
    "6": lambda test: bool(test.tester.info["OpenSSLVersion"])
         if ("OpenSSLVersion" in test.tester.info)
         else test.tester.info["openssl"]["running"] != "disabled",
    "7": try_authorization,
    "8": lambda test: bool(test.tester.get_db()),
    "9": try_javascript,
    "10": try_roles,
    "11": try_dedicated_user,
    "12": try_scram,
    "13": alerts_dec012015,
    "14": alerts_mar272015,
    "15": alerts_mar252015,
    "16": alerts_feb252015,
    "17": alerts_jun172014,
    "18": alerts_may052014,
    "19": alerts_jun202013,
    "20": alerts_jun052013,
    "21": alerts_mar062014,
    "22": alerts_oct012013,
    "23": alerts_aug152013
}
<|endoftext|>