<commit_before>import unittest
import datetime
import json
import sys

sys.path.append('..')
import sabre_dev_studio

'''
requires config.json in the same directory for api authentication

{
	"sabre_client_id": -----,
	"sabre_client_secret": -----
}

'''
class TestBasicInstaflights(unittest.TestCase):
    def prepare_client(self):
        raw_data = open('config.json').read()

        data = json.loads(raw_data)

        client_secret = data['sabre_client_secret']
        client_id = data['sabre_client_id']

        sds = sabre_dev_studio.SabreDevStudio()
        sds.set_credentials(client_id, client_secret)
        sds.authenticate()

        self.sds = sds

    def setUp(self):
        self.prepare_client()

    def test_basic_request(self):
        now = datetime.datetime.now()

        # Set departure date to tomorrow
        # Set arrival date to day after
        tomorrow = now + datetime.timedelta(days=1)
        day_after = now + datetime.timedelta(days=2)

        tomorrow_str = tomorrow.strftime('%Y-%m-%d')
        day_after_str = day_after.strftime('%Y-%m-%d')

        options = {
            'origin': 'JFK',
            'destination': 'LAX',
            'departuredate': tomorrow_str,
            'returndate': day_after_str
        }

        instaf = self.sds.instaflights(options)
        self.assertIsNotNone(instaf)

    def test_no_authorization(self):
        sds = sabre_dev_studio.SabreDevStudio()

        with self.assertRaises(sabre_exceptions.NotAuthorizedError):
            resp = sds.instaflights({})

if __name__ == '__main__':
    unittest.main()
<commit_msg>Update instaflights test<commit_after>import unittest
import datetime
import json
import sys

sys.path.append('..')
import sabre_dev_studio

'''
requires config.json in the same directory for api authentication

{
	"sabre_client_id": -----,
	"sabre_client_secret": -----
}

'''
class TestBasicInstaflights(unittest.TestCase):
    def read_config(self):
        raw_data = open('config.json').read()

        data = json.loads(raw_data)

        client_secret = data['sabre_client_secret']
        client_id = data['sabre_client_id']

        return (client_id, client_secret)

    def setUp(self):
        # Read from config
        self.client_id, self.client_secret = self.read_config()
        self.sds = sabre_dev_studio.SabreDevStudio()
        self.sds.set_credentials(self.client_id, self.client_secret)
        self.sds.authenticate()

    def test_basic_request():
        tomorrow_str = tomorrow.strftime('%Y-%m-%d')
        day_after_str = day_after.strftime('%Y-%m-%d')

        options = {
            'origin': 'JFK',
            'destination': 'LAX',
            'departuredate': tomorrow_str,
            'returndate': day_after_str
        }
        instaf = self.sds.instaflights(options)
        self.assertIsNotNone(instaf)

    def test_no_authorization(self):
        sds = sabre_dev_studio.SabreDevStudio()

        with self.assertRaises(sabre_exceptions.NotAuthorizedError):
            resp = sds.instaflights({})

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># Generated from DiceNotation.g4 by ANTLR 4.7.2
from antlr4 import *

import logging
from dice_notation.dice import Dice
from dice_notation.algebra import BinaryOperation

# This class defines a complete listener for a parse tree produced by DiceNotationParser.
class DiceNotationListener(ParseTreeListener):

    def __init__(self):
        super(DiceNotationListener, self).__init__()
        self._logger = logging.getLogger("DiceNotationListener")
        self._nodes = []

    def expression(self):
        expression = None

        self._logger.debug("Checking stack %s for parsed expression", self._nodes)
        if(self._nodes):
            expression = self._nodes.pop()
            self._logger.debug("Got expression %s from stack", expression)
        else:
            self._logger.debug("No expression found")

        return expression

    def binary_operation(self, operators):
        operands = []

        # There are as many operands as operators plus one
        for i in range(0, len(operators)):
            operands.append(self._nodes.pop())

        for operator in operators:
            left = operands.pop()
            right = operands.pop()

            if("+" == operator):
                func = lambda a, b: a + b
            elif("-" == operator):
                func = lambda a, b: a - b
            elif("*" == operator):
                func = lambda a, b: a * b
            elif("/" == operator):
                func = lambda a, b: a / b
            else:
                func = None
                self._logger.error("The %s operator is invalid", operator)
            operation = BinaryOperation(left, right, operator, func)
            self._logger.debug("Parsed operation %s", operation)
            operands.append(operation)

        return operands.pop()

    # Enter a parse tree produced by DiceNotationParser#notation.
    def enterNotation(self, ctx):
        self._logger.debug("Entering notation %s", ctx.getText())
        pass

    # Exit a parse tree produced by DiceNotationParser#notation.
    def exitNotation(self, ctx):
        self._logger.debug("Exiting notation %s", ctx.getText())
        pass

    # Enter a parse tree produced by DiceNotationParser#addOp.
    def enterAddOp(self, ctx):
        self._logger.debug("Entering add %s", ctx.getText())
        pass

    # Exit a parse tree produced by DiceNotationParser#addOp.
    def exitAddOp(self, ctx):
        self._logger.debug("Exiting add %s", ctx.getText())
        pass

    # Enter a parse tree produced by DiceNotationParser#multOp.
    def enterMultOp(self, ctx):
        self._logger.debug("Entering multiplication %s", ctx.getText())
        pass

    # Exit a parse tree produced by DiceNotationParser#multOp.
    def exitMultOp(self, ctx):
        self._logger.debug("Exiting multiplication %s", ctx.getText())
        pass

    # Enter a parse tree produced by DiceNotationParser#operand.
    def enterOperand(self, ctx):
        self._logger.debug("Entering operand %s", ctx.getText())
        pass

    # Exit a parse tree produced by DiceNotationParser#operand.
    def exitOperand(self, ctx):
        self._logger.debug("Exiting operand %s", ctx.getText())
        pass

    # Enter a parse tree produced by DiceNotationParser#dice.
    def enterDice(self, ctx):
        self._logger.debug("Entering dice %s", ctx.getText())
        pass

    # Exit a parse tree produced by DiceNotationParser#dice.
    def exitDice(self, ctx):
        self._logger.debug("Exiting dice %s", ctx.getText())
        self._logger.debug("Quantity %s, sides %s", ctx.DIGIT()[0], ctx.DIGIT()[1])
        digits = iter(ctx.DIGIT())
        if(len(ctx.DIGIT()) > 1):
            # Contains the quantity of dice
            quantity = int(next(digits).getText())
        else:
            # No quantity of dice defined
            # Defaults to 1
            quantity = 1

        sides = int(next(digits).getText())

        dice = Dice(quantity, sides)
        self._nodes.append(dice)
        self._logger.debug("Added dice %s to stack", dice)
        self._logger.debug("Currently the stack contains %s", self._nodes)

    # Enter a parse tree produced by DiceNotationParser#number.
    def enterNumber(self, ctx):
        self._logger.debug("Entering number %s", ctx.getText())
        pass

    # Exit a parse tree produced by DiceNotationParser#number.
    def exitNumber(self, ctx):
        self._logger.debug("Exiting number %s", ctx.getText())
        pass

<commit_msg>Binary operations are parsed<commit_after># Generated from DiceNotation.g4 by ANTLR 4.7.2
from antlr4 import *

import logging
from dice_notation.dice import Dice
from dice_notation.algebra import BinaryOperation

# This class defines a complete listener for a parse tree produced by DiceNotationParser.
class DiceNotationListener(ParseTreeListener):

    def __init__(self):
        super(DiceNotationListener, self).__init__()
        self._logger = logging.getLogger("DiceNotationListener")
        self._nodes = []

    def expression(self):
        expression = None

        self._logger.debug("Checking stack %s for parsed expression", self._nodes)
        if(self._nodes):
            expression = self._nodes.pop()
            self._logger.debug("Got expression %s from stack", expression)
        else:
            self._logger.debug("No expression found")

        return expression

    def binary_operation(self, operators):
        operands = []

        # There are as many operands as operators plus one
        for i in range(0, len(operators)+1):
            operands.append(self._nodes.pop())

        for operator in operators:
            left = operands.pop()
            right = operands.pop()

            if("+" == operator):
                func = lambda a, b: a + b
            elif("-" == operator):
                func = lambda a, b: a - b
            elif("*" == operator):
                func = lambda a, b: a * b
            elif("/" == operator):
                func = lambda a, b: a / b
            else:
                func = None
                self._logger.error("The %s operator is invalid", operator)
            operation = BinaryOperation(left, right, operator, func)
            self._logger.debug("Parsed operation %s", operation)
            operands.append(operation)

        return operands.pop()

    # Enter a parse tree produced by DiceNotationParser#notation.
    def enterNotation(self, ctx):
        self._logger.debug("Entering notation %s", ctx.getText())
        pass

    # Exit a parse tree produced by DiceNotationParser#notation.
    def exitNotation(self, ctx):
        self._logger.debug("Exiting notation %s", ctx.getText())
        pass

    # Enter a parse tree produced by DiceNotationParser#addOp.
    def enterAddOp(self, ctx):
        self._logger.debug("Entering add %s", ctx.getText())
        pass

    # Exit a parse tree produced by DiceNotationParser#addOp.
    def exitAddOp(self, ctx):
        self._logger.debug("Exiting add %s", ctx.getText())
        operators = []
        for operator in ctx.ADDOPERATOR():
            operators.append(operator.getText())
        expression = self.binary_operation(operators)
        self._nodes.append(expression)

    # Enter a parse tree produced by DiceNotationParser#multOp.
    def enterMultOp(self, ctx):
        self._logger.debug("Entering multiplication %s", ctx.getText())
        pass

    # Exit a parse tree produced by DiceNotationParser#multOp.
    def exitMultOp(self, ctx):
        self._logger.debug("Exiting multiplication %s", ctx.getText())
        operators = []
        for operator in ctx.MULTOPERATOR():
            operators.append(operator.getText())
        expression = self.binary_operation(operators)
        self._nodes.append(expression)

    # Enter a parse tree produced by DiceNotationParser#operand.
    def enterOperand(self, ctx):
        self._logger.debug("Entering operand %s", ctx.getText())
        pass

    # Exit a parse tree produced by DiceNotationParser#operand.
    def exitOperand(self, ctx):
        self._logger.debug("Exiting operand %s", ctx.getText())
        pass

    # Enter a parse tree produced by DiceNotationParser#dice.
    def enterDice(self, ctx):
        self._logger.debug("Entering dice %s", ctx.getText())
        pass

    # Exit a parse tree produced by DiceNotationParser#dice.
    def exitDice(self, ctx):
        self._logger.debug("Exiting dice %s", ctx.getText())
        self._logger.debug("Quantity %s, sides %s", ctx.DIGIT()[0], ctx.DIGIT()[1])
        digits = iter(ctx.DIGIT())
        if(len(ctx.DIGIT()) > 1):
            # Contains the quantity of dice
            quantity = int(next(digits).getText())
        else:
            # No quantity of dice defined
            # Defaults to 1
            quantity = 1

        sides = int(next(digits).getText())

        dice = Dice(quantity, sides)
        self._nodes.append(dice)

    # Enter a parse tree produced by DiceNotationParser#number.
    def enterNumber(self, ctx):
        self._logger.debug("Entering number %s", ctx.getText())
        pass

    # Exit a parse tree produced by DiceNotationParser#number.
    def exitNumber(self, ctx):
        self._logger.debug("Exiting number %s", ctx.getText())
        pass

<|endoftext|>


<commit_before># Copyright 2014-present Ivan Kravets <me@ikravets.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

from os.path import isdir, isfile, join

from SCons.Script import COMMAND_LINE_TARGETS

from platformio import exception, util
from platformio.managers.platform import PlatformFactory

@util.memoized
def initDevPlatform(name):
    return PlatformFactory.newPlatform(name)

def DevPlatform(env):
    variables = {}
    for key in ("board", "framework"):
        if key not in env:
            continue
        variables[key] = env[key.upper()]
    p = initDevPlatform(env['PLATFORM_MANIFEST'])
    p.configure_default_packages(variables, COMMAND_LINE_TARGETS)
    return p

def BoardConfig(env, board=None):
    p = initDevPlatform(env['PLATFORM_MANIFEST'])
    try:
        config = p.board_config(board if board else env['BOARD'])
    except exception.UnknownBoard as e:
        env.Exit("Error: %s" % str(e))
    return config

def GetFrameworkScript(env, framework):
    p = env.DevPlatform()
    assert p.frameworks and framework in p.frameworks
    script_path = env.subst(p.frameworks[framework]['script'])
    if not isfile(script_path):
        script_path = join(p.get_dir(), script_path)
    return script_path

def LoadDevPlatform(env, variables):
    p = env.DevPlatform()
    installed_packages = p.get_installed_packages()

    # Add toolchains and uploaders to $PATH
    for name in installed_packages:
        type_ = p.get_package_type(name)
        if type_ not in ("toolchain", "uploader"):
            continue
        path = p.get_package_dir(name)
        if isdir(join(path, "bin")):
            path = join(path, "bin")
        env.PrependENVPath("PATH", path)

    # Platform specific LD Scripts
    if isdir(join(p.get_dir(), "ldscripts")):
        env.Prepend(LIBPATH=[join(p.get_dir(), "ldscripts")])

    if "BOARD" not in env:
        return

    board_config = env.BoardConfig()
    for k in variables:
        if (k in env or
                not any([k.startswith("BOARD_"), k.startswith("UPLOAD_")])):
            continue
        _opt, _val = k.lower().split("_", 1)
        if _opt == "board":
            _opt = "build"
        if _val in board_config.get(_opt):
            env.Replace(**{k: board_config.get("%s.%s" % (_opt, _val))})

    if "build.ldscript" in board_config:
        env.Replace(
            LDSCRIPT_PATH=board_config.get("build.ldscript")
        )

def exists(_):
    return True

def generate(env):
    env.AddMethod(DevPlatform)
    env.AddMethod(BoardConfig)
    env.AddMethod(GetFrameworkScript)
    env.AddMethod(LoadDevPlatform)
    return env
<commit_msg>Typo fix<commit_after># Copyright 2014-present Ivan Kravets <me@ikravets.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

from os.path import isdir, isfile, join

from SCons.Script import COMMAND_LINE_TARGETS

from platformio import exception, util
from platformio.managers.platform import PlatformFactory

@util.memoized
def initDevPlatform(name):
    return PlatformFactory.newPlatform(name)

def DevPlatform(env):
    variables = {}
    for key in ("board", "framework"):
        if key not in env:
            continue
        variables[key] = env[key.upper()]
    p = initDevPlatform(env['PLATFORM_MANIFEST'])
    p.configure_default_packages(variables, COMMAND_LINE_TARGETS)
    return p

def BoardConfig(env, board=None):
    p = initDevPlatform(env['PLATFORM_MANIFEST'])
    try:
        config = p.board_config(board if board else env['BOARD'])
    except exception.UnknownBoard as e:
        env.Exit("Error: %s" % str(e))
    return config

def GetFrameworkScript(env, framework):
    p = env.DevPlatform()
    assert p.frameworks and framework in p.frameworks
    script_path = env.subst(p.frameworks[framework]['script'])
    if not isfile(script_path):
        script_path = join(p.get_dir(), script_path)
    return script_path

def LoadDevPlatform(env, variables):
    p = env.DevPlatform()
    installed_packages = p.get_installed_packages()

    # Add toolchains and uploaders to $PATH
    for name in installed_packages:
        type_ = p.get_package_type(name)
        if type_ not in ("toolchain", "uploader"):
            continue
        path = p.get_package_dir(name)
        if isdir(join(path, "bin")):
            path = join(path, "bin")
        env.PrependENVPath("PATH", path)

    # Platform specific LD Scripts
    if isdir(join(p.get_dir(), "ldscripts")):
        env.Prepend(LIBPATH=[join(p.get_dir(), "ldscripts")])

    if "BOARD" not in env:
        return

    board_config = env.BoardConfig()
    for k in variables.keys():
        if (k in env or
                not any([k.startswith("BOARD_"), k.startswith("UPLOAD_")])):
            continue
        _opt, _val = k.lower().split("_", 1)
        if _opt == "board":
            _opt = "build"
        if _val in board_config.get(_opt):
            env.Replace(**{k: board_config.get("%s.%s" % (_opt, _val))})

    if "build.ldscript" in board_config:
        env.Replace(
            LDSCRIPT_PATH=board_config.get("build.ldscript")
        )

def exists(_):
    return True

def generate(env):
    env.AddMethod(DevPlatform)
    env.AddMethod(BoardConfig)
    env.AddMethod(GetFrameworkScript)
    env.AddMethod(LoadDevPlatform)
    return env
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import argparse
import base64
from cryptography.fernet import Fernet
import json
import requests
import pexpect
import os, os.path
import sys

class KeyRepository():
    repository = {}

    def __init__(self, repository_path, repository_key, init=False):
        self.repository_key = repository_key
        self.repository_path = repository_path
        if init:
            self.write()
        # try a read so we throw an exception if key is wrong
        self.read()

    def init(repository_path, repository_key=None):
        if not repository_key:
            repository_key = Fernet.generate_key()
        zkv = __class__(repository_path, repository_key, init=True)
        return zkv

    def read(self):
        with open(self.repository_path, 'rb') as key_store:
            f = Fernet(self.repository_key)
            data = f.decrypt(key_store.read())
            self.repository = json.loads(data)
        return self.repository.copy()

    def write(self):
        with open(self.repository_path, 'wb') as key_store:
            f = Fernet(self.repository_key)
            data = f.encrypt(json.dumps(self.repository).encode())
            key_store.write(data)

    def get_key(self):
        return self.repository_key

    def set_fs_key(self, fs_name, fs_key):
        self.repository[fs_name] = fs_key

    def remove_fs_key(self, fs_name):
        self.repository.pop(fs_name)

    def has_fs_key(self, fs_name):
        return fs_name in self.repository

class Program:
    def __init__(self, repository_path):
        self.repository_path = repository_path

        root_parser = argparse.ArgumentParser(description="ZFS integration with Azure Key Vault")

        # Create the parsers for top-level commands
        subparser = root_parser.add_subparsers(help='commands', dest='action')
        fs_action_p = subparser.add_parser('fs', help='Configure a local, encrypted repository of ZFS filesystem encryption keys')
        fetch_action_p = subparser.add_parser('fetch', help="Fetch repository's encryption key and print it")
        mount_action_p = subparser.add_parser('mount', help="Fetch repository's encryption key and mount all configured ZFS filesystems")

        # Creat options for top-level commands
        for cmd in [fetch_action_p, mount_action_p]:
            cmd.add_argument("gateway_uri", help="URI to the ZFS Key Vault Gateway providing the keys")

        # Create the parsers for fs commands
        fs_action_sp = fs_action_p.add_subparsers(help='fs commands', dest='fs_action')
        fs_action_init_p = fs_action_sp.add_parser('init', help='Initialize a local, encrypted repository')
        fs_action_list_p = fs_action_sp.add_parser('list', help='List configured filesystems in the repository')
        fs_action_add_p = fs_action_sp.add_parser('add', help="Add a filesystem encryption key to the repository")
        fs_action_remove_p = fs_action_sp.add_parser('remove', help="Remove a filesystem encryption key from the repository")

        # Create options for fs commands
        fs_action_init_p.add_argument("-m", "--machine-readable", action='store_true', help="Machine-readable output (prints only the encryption key)")
        fs_action_init_p.add_argument("-f", "--force", action='store_true', help="Overwrite existing repository, if one exists")
        fs_action_init_p.add_argument("-k", "--repository-key", help="Supply the base64-encoded Fernet key used to encrypt the repository")

        for cmd in [fs_action_list_p, fs_action_add_p, fs_action_remove_p]:
            cmd.add_argument("repository_key", help="Base64-encoded Fernet encryption key for the repository")

        for cmd in [fs_action_add_p, fs_action_remove_p]:
            cmd.add_argument("filesystem", help="Name of ZFS filesystem including pool (e.g. tank/myfs)")
        fs_action_add_p.add_argument("fs_key", help="Encryption key for the ZFS filesystem")
        fs_action_add_p.add_argument("-f", "--force", action='store_true', help="Overwrite existing encryption key for provided filesystem, if one exists")
        fs_action_remove_p.add_argument("-f", "--force", action='store_true', help="Proceed normally if filesystem does not exist in repository")
        fs_action_list_p.add_argument("-p", "--plaintext", action='store_true', help="Print configured encryption key in plaintext")

        self.options = root_parser.parse_args()

    def __fetch_repo_key(self):
        r = requests.get(self.options.gateway_uri)
        if r.status_code == 200:
            return r.text
        else:
            sys.stderr.write("Error: attempt to obtain encryption keys failed due to invalid HTTP status code %d\n" % r.status_code)
            sys.exit(1)

    def __load_key_pexpect(self):
        child = pexpect.spawn("zfs", ["load-key", filesystem])
        child.expect(":", timeout=10)
        child.sendline(encryption_key)
        child.wait()
        child.close()
        if child.exitstatus != 0:
            sys.stderr.write("Error: loading encryption keys for filesystem %(filesystem)s failed due to zfs exit status %(exit_status)d\n" % {'filesystem': filesystem, 'exit_status': child.exitstatus})

    def __read_repo(self, repository_key):
        try:
            zkv = KeyRepository(self.repository_path, repository_key)
            return zkv
        except:
            raise
            sys.stderr.write("Could not decrypt key repository; please verify repository key\n")
            sys.exit(1)

    def fetch(self):
        repository_key = self.__fetch_repo_key()
        print(repository_key)

    def mount(self):
        repository_key = self.__fetch_repo_key()
        zkv = self.__read_repo(repository_key)
        fs = zkv.read()
        for filesystem, encryption_key in fs.items():
            self.__load_key_pexpect(filesystem, encryption_key)

    def fs_add(self):
        zkv = self.__read_repo(self.options.repository_key)
        fs = zkv.read()
        if not self.options.force and zkv.has_fs_key(self.options.filesystem):
            sys.stderr.write("An existing filesystem key for '%s' was found in the keystore\n" % self.options.filesystem)
            sys.exit(1)
        zkv.set_fs_key(self.options.filesystem, self.options.fs_key)
        zkv.write()

    def fs_init(self):
        if not self.options.force and os.path.exists(repository_path):
            if not self.options.machine_readable:
                sys.stderr.write("An existing keystore was found at '%s', use -f to overwrite\n" % repository_path)
            sys.exit(1)
        repository_key = self.options.repository_key.encode() if self.options.repository_key else None
        zkv = KeyRepository.init(repository_path, repository_key=repository_key)
        repository_key = zkv.get_key().decode('ascii')
        if self.options.machine_readable:
            print(repository_key)
        else:
            print("New repository created at '%s'. Take note of its encryption key, it will not be displayed again:" % repository_path)
            print("%s\n" % repository_key)
            print("[!] Loss of this key requires re-creation of the entire repository.")

    def fs_list(self):
        zkv = self.__read_repo(self.options.repository_key)
        fs = zkv.read()
        for filesystem, fs_key in fs.items():
            if self.options.plaintext:
                print("%s: %s" % (filesystem, fs_key))
            else:
                print("%s" % filesystem)

    def fs_remove(self):
        zkv = self.__read_repo(self.options.repository_key)
        fs = zkv.read()
        if not zkv.has_fs_key(self.options.filesystem):
            if self.options.force:
                sys.exit(0)
            else:
                sys.stderr.write("No key for filesystem '%s' was found in the keystore\n" % self.options.filesystem)
                sys.exit(1)
        zkv.remove_fs_key(self.options.filesystem)
        zkv.write()

    def run(self):
        if self.options.action == "fetch":
            p.fetch()
        elif self.options.action == "mount":
            p.mount()
        elif self.options.action == "fs":
            if self.options.fs_action == "init":
                p.fs_init()
            elif self.options.fs_action == "list":
                p.fs_list()
            elif self.options.fs_action == "add":
                p.fs_add()
            elif self.options.fs_action == "remove":
                p.fs_remove()
            else:
                fs_action_p.print_usage()
                sys.exit(1)
        else:
            root_parser.print_usage()
            sys.exit(1)

if __name__ == "__main__":
    repository_path = os.path.expanduser('~/.zkv-repository')
    p = Program(repository_path)
    p.run()
<commit_msg>Move repository_key positional argument of 'fs' subcommand to a '-k' required argument, makes consistent with 'fs init'<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import argparse
import base64
from cryptography.fernet import Fernet
import json
import requests
import pexpect
import os, os.path
import sys

class KeyRepository():
    repository = {}

    def __init__(self, repository_path, repository_key, init=False):
        self.repository_key = repository_key
        self.repository_path = repository_path
        if init:
            self.write()
        # try a read so we throw an exception if key is wrong
        self.read()

    def init(repository_path, repository_key=None):
        if not repository_key:
            repository_key = Fernet.generate_key()
        zkv = __class__(repository_path, repository_key, init=True)
        return zkv

    def read(self):
        with open(self.repository_path, 'rb') as key_store:
            f = Fernet(self.repository_key)
            data = f.decrypt(key_store.read())
            self.repository = json.loads(data)
        return self.repository.copy()

    def write(self):
        with open(self.repository_path, 'wb') as key_store:
            f = Fernet(self.repository_key)
            data = f.encrypt(json.dumps(self.repository).encode())
            key_store.write(data)

    def get_key(self):
        return self.repository_key

    def set_fs_key(self, fs_name, fs_key):
        self.repository[fs_name] = fs_key

    def remove_fs_key(self, fs_name):
        self.repository.pop(fs_name)

    def has_fs_key(self, fs_name):
        return fs_name in self.repository

class Program:
    def __init__(self, repository_path):
        self.repository_path = repository_path

        root_parser = argparse.ArgumentParser(description="ZFS integration with Azure Key Vault")

        # Create the parsers for top-level commands
        subparser = root_parser.add_subparsers(help='commands', dest='action')
        fs_action_p = subparser.add_parser('fs', help='Configure a local, encrypted repository of ZFS filesystem encryption keys')
        fetch_action_p = subparser.add_parser('fetch', help="Fetch repository's encryption key and print it")
        mount_action_p = subparser.add_parser('mount', help="Fetch repository's encryption key and mount all configured ZFS filesystems")

        # Creat options for top-level commands
        for cmd in [fetch_action_p, mount_action_p]:
            cmd.add_argument("gateway_uri", help="URI to the ZFS Key Vault Gateway providing the keys")

        # Create the parsers for fs commands
        fs_action_sp = fs_action_p.add_subparsers(help='fs commands', dest='fs_action')
        fs_action_init_p = fs_action_sp.add_parser('init', help='Initialize a local, encrypted repository')
        fs_action_list_p = fs_action_sp.add_parser('list', help='List configured filesystems in the repository')
        fs_action_add_p = fs_action_sp.add_parser('add', help="Add a filesystem encryption key to the repository")
        fs_action_remove_p = fs_action_sp.add_parser('remove', help="Remove a filesystem encryption key from the repository")

        # Create options for fs commands
        fs_action_init_p.add_argument("-m", "--machine-readable", action='store_true', help="Machine-readable output (prints only the encryption key)")
        fs_action_init_p.add_argument("-f", "--force", action='store_true', help="Overwrite existing repository, if one exists")
        fs_action_init_p.add_argument("-k", "--repository-key", help="Supply the base64-encoded Fernet key used to encrypt the repository")

        for cmd in [fs_action_list_p, fs_action_add_p, fs_action_remove_p]:
            cmd.add_argument("-k", "--repository_key", help="Base64-encoded Fernet encryption key for the repository", required=True)

        for cmd in [fs_action_add_p, fs_action_remove_p]:
            cmd.add_argument("filesystem", help="Name of ZFS filesystem including pool (e.g. tank/myfs)")
        fs_action_add_p.add_argument("fs_key", help="Encryption key for the ZFS filesystem")
        fs_action_add_p.add_argument("-f", "--force", action='store_true', help="Overwrite existing encryption key for provided filesystem, if one exists")
        fs_action_remove_p.add_argument("-f", "--force", action='store_true', help="Proceed normally if filesystem does not exist in repository")
        fs_action_list_p.add_argument("-p", "--plaintext", action='store_true', help="Print configured encryption key in plaintext")

        self.options = root_parser.parse_args()

    def __fetch_repo_key(self):
        r = requests.get(self.options.gateway_uri)
        if r.status_code == 200:
            return r.text
        else:
            sys.stderr.write("Error: attempt to obtain encryption keys failed due to invalid HTTP status code %d\n" % r.status_code)
            sys.exit(1)

    def __load_key_pexpect(self):
        child = pexpect.spawn("zfs", ["load-key", filesystem])
        child.expect(":", timeout=10)
        child.sendline(encryption_key)
        child.wait()
        child.close()
        if child.exitstatus != 0:
            sys.stderr.write("Error: loading encryption keys for filesystem %(filesystem)s failed due to zfs exit status %(exit_status)d\n" % {'filesystem': filesystem, 'exit_status': child.exitstatus})

    def __read_repo(self, repository_key):
        try:
            zkv = KeyRepository(self.repository_path, repository_key)
            return zkv
        except:
            raise
            sys.stderr.write("Could not decrypt key repository; please verify repository key\n")
            sys.exit(1)

    def fetch(self):
        repository_key = self.__fetch_repo_key()
        print(repository_key)

    def mount(self):
        repository_key = self.__fetch_repo_key()
        zkv = self.__read_repo(repository_key)
        fs = zkv.read()
        for filesystem, encryption_key in fs.items():
            self.__load_key_pexpect(filesystem, encryption_key)

    def fs_add(self):
        zkv = self.__read_repo(self.options.repository_key)
        fs = zkv.read()
        if not self.options.force and zkv.has_fs_key(self.options.filesystem):
            sys.stderr.write("An existing filesystem key for '%s' was found in the keystore\n" % self.options.filesystem)
            sys.exit(1)
        zkv.set_fs_key(self.options.filesystem, self.options.fs_key)
        zkv.write()

    def fs_init(self):
        if not self.options.force and os.path.exists(repository_path):
            if not self.options.machine_readable:
                sys.stderr.write("An existing keystore was found at '%s', use -f to overwrite\n" % repository_path)
            sys.exit(1)
        repository_key = self.options.repository_key.encode() if self.options.repository_key else None
        zkv = KeyRepository.init(repository_path, repository_key=repository_key)
        repository_key = zkv.get_key().decode('ascii')
        if self.options.machine_readable:
            print(repository_key)
        else:
            print("New repository created at '%s'. Take note of its encryption key, it will not be displayed again:" % repository_path)
            print("%s\n" % repository_key)
            print("[!] Loss of this key requires re-creation of the entire repository.")

    def fs_list(self):
        zkv = self.__read_repo(self.options.repository_key)
        fs = zkv.read()
        for filesystem, fs_key in fs.items():
            if self.options.plaintext:
                print("%s: %s" % (filesystem, fs_key))
            else:
                print("%s" % filesystem)

    def fs_remove(self):
        zkv = self.__read_repo(self.options.repository_key)
        fs = zkv.read()
        if not zkv.has_fs_key(self.options.filesystem):
            if self.options.force:
                sys.exit(0)
            else:
                sys.stderr.write("No key for filesystem '%s' was found in the keystore\n" % self.options.filesystem)
                sys.exit(1)
        zkv.remove_fs_key(self.options.filesystem)
        zkv.write()

    def run(self):
        if self.options.action == "fetch":
            p.fetch()
        elif self.options.action == "mount":
            p.mount()
        elif self.options.action == "fs":
            if self.options.fs_action == "init":
                p.fs_init()
            elif self.options.fs_action == "list":
                p.fs_list()
            elif self.options.fs_action == "add":
                p.fs_add()
            elif self.options.fs_action == "remove":
                p.fs_remove()
            else:
                fs_action_p.print_usage()
                sys.exit(1)
        else:
            root_parser.print_usage()
            sys.exit(1)

if __name__ == "__main__":
    repository_path = os.path.expanduser('~/.zkv-repository')
    p = Program(repository_path)
    p.run()
<|endoftext|>


<commit_before>import unicodedata
import string
import re

import torch
from torch.autograd import Variable

# Constants
HIDDEN_DIMS = 256
EPOCHS = 10
PATH = "name_data/eng-fra.txt"
ALLOWED_CHARS = string.ascii_letters + " ,.!?'"
MAX_TRAINING_LENGTH = 10
ALLOWED_PREFIXES = (
        "i am", "i m",
        "you are", "you re",
        "he is", "he s",
        "she is", "she s",
        "they are", "they re",
        "we are", "we re",
        "it is", "it s"
)

# Reseverd Tokens
SOS = 0  # Start of Sentence
EOS = 1  # End of Sentence

# Helper methods
def unicode_to_ascii(s):
    return "".join(
            c for c in unicodedata.normalize('NFD', s)
            if unicodedata.category(c) != "Mn"
            and c in ALLOWED_CHARS
        )

def normalize_string(s):
    s = unicode_to_ascii(s.lower().strip())
    s = re.sub(r"([^ ])([.,!?])", r"\1 \2", s)
    s = re.sub(r"[^a-z.,!? ]", r" ", s)
    s = re.sub(r" {2,}", r" ", s)
    return s

def read_data(reverse=False):
    with open(PATH, encoding='utf-8') as text:
        lines = text.read().strip().split('\n')

        pairs = []
        for line in lines:
            pair = []
            for sentence in line.split('\t'):
                pair.append(normalize_string(sentence))

            if is_simple(pair):
                pairs.append(pair)

        if not reverse:
            input_lang = Lang("English")
            output_lang = Lang("French")
        else:
            pairs = [list(reversed(pair)) for pair in pairs]
            input_lang = Lang("French")
            output_lang = Lang("English")

        return input_lang, output_lang, pairs

def is_simple(pair):
    return len(pair[0].split(" ")) <= MAX_TRAINING_LENGTH and \
            len(pair[1].split(" ")) <= MAX_TRAINING_LENGTH and \
            pair[0].startswith(ALLOWED_PREFIXES)

# Define a class to hold information about a language
class Lang:
    def __init__(self, name):
        self.name = name
        self.word2index = {"SOS": 0, "EOS": 1}
        self.word2count = {}
        self.index2word = {0: "SOS", 1: "EOS"}
        self.n_words = 2

    def add_word(self, word):
        if not word in self.word2index:
            self.word2index[word] = self.n_words
            self.index2word[self.n_words] = word
            self.word2count[word] = 1
        else:
            self.word2count[word] += 1

        self.n_words += 1

    def add_sentence(self, sentence):
        for word in sentence.split(" "):
            self.add_word(word)

# Read data into lang objects
print("Reading Data...")
input_lang, output_lang, pairs = read_data()
print("{} filtered training pairs loaded".format(len(pairs)))
print("Indexing...")
for pair in pairs:
    input_lang.add_sentence(pair[0])
    output_lang.add_sentence(pair[1])

# print(pairs[:20])
# print(input_lang.word2index["not"])
# print(output_lang.word2index["mais"])

# Define Encoder Network
class Encoder(torch.nn.Module):
    def __init__(self, input_size, hidden_dims):
        super(Encoder, self).__init__()

        self.input_size = input_size
        self.hidden_dims = hidden_dims

        # Input is one hot vector and can used as proxy for vocabulary
        # Embedding dim is the same size as hidden dim
        self.embeddings = torch.nn.Embedding(input_size, hidden_dims)

        # GRU Input dim is the same size as the hidden dim
        # since the embedding dim is the same as the hidden dim
        self.gru = torch.nn.GRU(hidden_dims, hidden_dims)

    def forward(self, word_var, prev_hidden):
        embed = self.embeddings(word_var).view(1,1,-1)
        output, hidden = self.gru(embed, prev_hidden)
        return output, hidden

    def init_hidden(self):
        hidden = torch.zeros(1, 1, self.hidden_dims)
        return Variable(hidden)

# Define Decoder Network (with attention)
class Decoder(torch.nn.Module):
    def __init__(self, output_dims, hidden_dims, max_length=20):
        super(Decoder, self).__init__()

        self.output_dims = output_dims
        self.hidden_dims = hidden_dims
        self.max_length  = max_length

        # Output dims is proxy for output vocabulary
        self.embedding = torch.nn.Embedding(output_dims, hidden_dims)
        self.attention = torch.nn.Linear(hidden_dims * 2, max_length)
        self.attention_combine = torch.nn.Linear(hidden_dims * 2, hidden_dims)
        self.gru = torch.nn.GRU(hidden_dims, hidden_dims)
        self.out = torch.nn.Linear(hidden_dims, output_dims)

        self.logsoftmax = torch.nn.LogSoftmax()
        self.softmax = torch.nn.Softmax()
        self.relu = torch.nn.ReLU()

    def forward(self, prev_word, prev_hidden, encoder_outputs):
        embedded = self.embedding(prev_word).view(1,1,-1)
        attention_input = torch.cat((embedded[0], prev_hidden[0]), 1)
        attention_weights = self.softmax(self.attention(attention_input))

        focused_encoder_outputs = torch.bmm(attention_weights.unsqueeze(0),
                                            encoder_outputs.unsqueeze(0))
        gru_input = self.attention_combine(torch.cat((focused_encoder_outputs[0],
                                                      embedded[0]), 1)).unsqueeze(0)
        gru_output, hidden = self.gru(gru_input, prev_hidden)
        probabilities = self.logsoftmax(self.out(gru_output[0]))
        return probabilities, hidden

    def init_hidden(self):
        hidden = Variable(torch.zeros(1,1,self.hidden_dims))
        return hidden

# Data Preperation helper functions
def sentence_to_indicies(sentence, lang):
    return [lang.word2index[word] for word in sentence.split(" ")]

def sentence_to_variable(sentence, lang):
    indicies = sentence_to_indicies(sentence, lang)
    indicies.append(EOS)
    variable = Variable(torch.LongTensor(indicies))
    return variable

def prepare_pair(pair, input_lang, output_lang):
    return (sentence_to_variable(pair[0], input_lang),
            sentence_to_variable(pair[1], output_lang))

# Train Model
def train(encoder, decoder,
           encoder_optimizer, decoder_optimizer,
           criterion, training_pair):

    # Clear gradient buffers
    encoder_optimizer.zero_grad()
    decoder_optimizer.zero_grad()

    # Establish tracking variables
    encoder_outputs = Variable(torch.zeros(decoder.max_length, encoder.hidden_dims))
    loss = 0

    # Run Encoder
    encoder_hidden = encoder.init_hidden()
    for i, word in enumerate(training_pair[0]):
        output, encoder_hidden = encoder(word, encoder_hidden)
        encoder_outputs[i] = output[0][0]

    # Run Decoder
    decoder_hidden = decoder.init_hidden()
    start_token = Variable(torch.LongTensor([SOS]))
    prev_word = start_token

    for i, word in enumerate(training_pair[1]):
        output, decoder_hidden = decoder(prev_word, decoder_hidden, encoder_outputs)
        _, max_index = torch.max(output, 1)
        max_index = max_index.data[0][0]
        prev_word = Variable(torch.LongTensor([max_index]))

        loss += criterion(output, word)

        if max_index is EOS:
            break

    loss.backward()
    encoder_optimizer.step()
    decoder_optimizer.step()

encoder = Encoder(input_lang.n_words, HIDDEN_DIMS)
decoder = Decoder(output_lang.n_words, HIDDEN_DIMS, max_length=15)
criterion = torch.nn.NLLLoss()
encoder_optimizer = torch.optim.Adam(encoder.parameters(), lr=0.0005)
decoder_optimizer = torch.optim.Adam(decoder.parameters(), lr=0.0005)
traing_pair = prepare_pair(pairs[5], input_lang, output_lang)

train(encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, traing_pair)
<commit_msg>added training loop and an inference function<commit_after>import unicodedata
import string
import re

import torch
from torch.autograd import Variable

# Constants
HIDDEN_DIMS = 256
EPOCHS = 1
PRINT_EVERY = 10
PATH = "name_data/eng-fra.txt"
ALLOWED_CHARS = string.ascii_letters + " ,.!?'"
MAX_TRAINING_LENGTH = 10
ALLOWED_PREFIXES = (
        "i am", "i m",
        "you are", "you re",
        "he is", "he s",
        "she is", "she s",
        "they are", "they re",
        "we are", "we re",
        "it is", "it s"
)

# Reseverd Tokens
SOS = 0  # Start of Sentence
EOS = 1  # End of Sentence

# Helper methods
def unicode_to_ascii(s):
    return "".join(
            c for c in unicodedata.normalize('NFD', s)
            if unicodedata.category(c) != "Mn"
            and c in ALLOWED_CHARS
        )

def normalize_string(s):
    s = unicode_to_ascii(s.lower().strip())
    s = re.sub(r"([^ ])([.,!?])", r"\1 \2", s)
    s = re.sub(r"[^a-z.,!? ]", r" ", s)
    s = re.sub(r" {2,}", r" ", s)
    return s

def read_data(reverse=False):
    with open(PATH, encoding='utf-8') as text:
        lines = text.read().strip().split('\n')

        pairs = []
        for line in lines:
            pair = []
            for sentence in line.split('\t'):
                pair.append(normalize_string(sentence))

            if is_simple(pair):
                pairs.append(pair)

        if not reverse:
            input_lang = Lang("English")
            output_lang = Lang("French")
        else:
            pairs = [list(reversed(pair)) for pair in pairs]
            input_lang = Lang("French")
            output_lang = Lang("English")

        return input_lang, output_lang, pairs

def is_simple(pair):
    return len(pair[0].split(" ")) <= MAX_TRAINING_LENGTH and \
            len(pair[1].split(" ")) <= MAX_TRAINING_LENGTH and \
            pair[0].startswith(ALLOWED_PREFIXES)

# Define a class to hold information about a language
class Lang:
    def __init__(self, name):
        self.name = name
        self.word2index = {"SOS": 0, "EOS": 1}
        self.word2count = {}
        self.index2word = {0: "SOS", 1: "EOS"}
        self.n_words = 2

    def add_word(self, word):
        if not word in self.word2index:
            self.word2index[word] = self.n_words
            self.index2word[self.n_words] = word
            self.word2count[word] = 1
        else:
            self.word2count[word] += 1

        self.n_words += 1

    def add_sentence(self, sentence):
        for word in sentence.split(" "):
            self.add_word(word)

# Read data into lang objects
print("Reading Data...")
input_lang, output_lang, pairs = read_data()
print("{} filtered training pairs loaded".format(len(pairs)))
print("Indexing...")
for pair in pairs:
    input_lang.add_sentence(pair[0])
    output_lang.add_sentence(pair[1])

# print(pairs[:20])
# print(input_lang.word2index["not"])
# print(output_lang.word2index["mais"])

# Define Encoder Network
class Encoder(torch.nn.Module):
    def __init__(self, input_size, hidden_dims):
        super(Encoder, self).__init__()

        self.input_size = input_size
        self.hidden_dims = hidden_dims

        # Input size is the size of the input vocabulary
        # Embedding dim is the same size as hidden dim
        self.embeddings = torch.nn.Embedding(input_size, hidden_dims)

        # GRU Input dim is the same size as the hidden dim
        # since the embedding dim is the same as the hidden dim
        self.gru = torch.nn.GRU(hidden_dims, hidden_dims)

    def forward(self, word_var, prev_hidden):
        embed = self.embeddings(word_var).view(1,1,-1)
        output, hidden = self.gru(embed, prev_hidden)
        return output, hidden

    def init_hidden(self):
        hidden = torch.zeros(1, 1, self.hidden_dims)
        return Variable(hidden)

# Define Decoder Network (with attention)
class Decoder(torch.nn.Module):
    def __init__(self, output_dims, hidden_dims, max_length=20):
        super(Decoder, self).__init__()

        self.output_dims = output_dims
        self.hidden_dims = hidden_dims
        self.max_length  = max_length

        # Output dims is proxy for output vocabulary
        self.embedding = torch.nn.Embedding(output_dims, hidden_dims)
        self.attention = torch.nn.Linear(hidden_dims * 2, max_length)
        self.attention_combine = torch.nn.Linear(hidden_dims * 2, hidden_dims)
        self.gru = torch.nn.GRU(hidden_dims, hidden_dims)
        self.out = torch.nn.Linear(hidden_dims, output_dims)

        self.logsoftmax = torch.nn.LogSoftmax()
        self.softmax = torch.nn.Softmax()
        self.relu = torch.nn.ReLU()

    def forward(self, prev_word, prev_hidden, encoder_outputs):
        embedded = self.embedding(prev_word).view(1,1,-1)
        attention_input = torch.cat((embedded[0], prev_hidden[0]), 1)
        attention_weights = self.softmax(self.attention(attention_input))

        focused_encoder_outputs = torch.bmm(attention_weights.unsqueeze(0),
                                            encoder_outputs.unsqueeze(0))
        gru_input = self.attention_combine(torch.cat((focused_encoder_outputs[0],
                                                      embedded[0]), 1)).unsqueeze(0)
        gru_output, hidden = self.gru(gru_input, prev_hidden)
        probabilities = self.logsoftmax(self.out(gru_output[0]))
        return probabilities, hidden

    def init_hidden(self):
        hidden = Variable(torch.zeros(1,1,self.hidden_dims))
        return hidden

# Data Preperation helper functions
def sentence_to_indicies(sentence, lang):
    return [lang.word2index[word] for word in sentence.split(" ")]

def sentence_to_variable(sentence, lang):
    indicies = sentence_to_indicies(sentence, lang)
    indicies.append(EOS)
    variable = Variable(torch.LongTensor(indicies))
    return variable

def prepare_pair(pair, input_lang, output_lang):
    return (sentence_to_variable(pair[0], input_lang),
            sentence_to_variable(pair[1], output_lang))

# Train Model
def train(encoder, decoder,
           encoder_optimizer, decoder_optimizer,
           criterion, training_pair):

    # Clear gradient buffers
    encoder_optimizer.zero_grad()
    decoder_optimizer.zero_grad()

    # Establish tracking variables
    encoder_outputs = Variable(torch.zeros(decoder.max_length, encoder.hidden_dims))
    loss = 0

    # Run Encoder
    encoder_hidden = encoder.init_hidden()
    for i, word in enumerate(training_pair[0]):
        output, encoder_hidden = encoder(word, encoder_hidden)
        encoder_outputs[i] = output[0][0]

    # Run Decoder
    decoder_hidden = encoder_hidden
    start_token = Variable(torch.LongTensor([SOS]))
    prev_word = start_token

    for i, word in enumerate(training_pair[1]):
        output, decoder_hidden = decoder(prev_word, decoder_hidden, encoder_outputs)
        _, max_index = torch.max(output, 1)
        max_index = max_index.data[0][0]
        prev_word = Variable(torch.LongTensor([max_index]))

        loss += criterion(output, word)

        if max_index is EOS:
            break

    loss.backward()
    encoder_optimizer.step()
    decoder_optimizer.step()

    return loss / (i + 1)

# encoder = Encoder(input_lang.n_words, HIDDEN_DIMS)
# decoder = Decoder(output_lang.n_words, HIDDEN_DIMS, max_length=15)
# criterion = torch.nn.NLLLoss()
# encoder_optimizer = torch.optim.Adam(encoder.parameters(), lr=0.0005)
# decoder_optimizer = torch.optim.Adam(decoder.parameters(), lr=0.0005)
# traing_pair = prepare_pair(pairs[5], input_lang, output_lang)
#
# print(train(encoder, decoder,
#             encoder_optimizer, decoder_optimizer,
#             criterion, traing_pair).data[0])
#
# traing_pair = prepare_pair(pairs[9], input_lang, output_lang)
# print(train(encoder, decoder,
#             encoder_optimizer, decoder_optimizer,
#             criterion, traing_pair).data[0])

# Training Loop

encoder = Encoder(input_lang.n_words, HIDDEN_DIMS)
decoder = Decoder(output_lang.n_words, HIDDEN_DIMS, max_length=15)
criterion = torch.nn.NLLLoss()
encoder_optimizer = torch.optim.Adam(encoder.parameters(), lr=0.0005)
decoder_optimizer = torch.optim.Adam(decoder.parameters(), lr=0.0005)

print("Starting training with {} epochs".format(EPOCHS))
for epoch in range(EPOCHS):
    print("Epoch {}".format(epoch + 1))
    epoch_losses = []
    reporting_losses = 0
    for i, pair in enumerate(pairs):
        training_pair = prepare_pair(pair, input_lang, output_lang)
        loss = train(encoder, decoder,
                     encoder_optimizer, decoder_optimizer,
                     criterion, training_pair)

        reporting_losses += loss.data[0]
        epoch_losses.append(loss.data[0])

        if (i + 1) % PRINT_EVERY is 0:
            print("Training loss is {}".format(reporting_losses / PRINT_EVERY))
            reporting_losses = 0

        # TEMPORARY HARD BREAK
        if (i + 1) % 120 is 0:
            print("Training Stopped!")
            break

def translate(sentence, encoder, decoder, input_lang, output_lang):
    tokenized = normalize_string(sentence)
    input_variable = sentence_to_variable(tokenized, input_lang)

    encoder_outputs = Variable(torch.zeros(decoder.max_length, encoder.hidden_dims))
    encoder_hidden = encoder.init_hidden()

    for i, word in enumerate(input_variable):
        encoder_out, encoder_hidden = encoder(word, encoder_hidden)
        encoder_outputs[i] = encoder_out[0][0]

    decoder_hidden = encoder_hidden
    prev_word = Variable(torch.LongTensor([SOS]))

    result = ""

    for i in range(decoder.max_length):
        decoder_out, decoder_hidden = decoder(prev_word, decoder_hidden, encoder_outputs)
        _, max_index = torch.max(decoder_out, 1)
        max_index = max_index.data[0][0]

        if max_index is EOS:
            break

        prev_word = Variable(torch.LongTensor([max_index]))
        result += output_lang.index2word[max_index] + " "

    return result

print("IN: I am very cold")
print("OUT: {}".format(translate("I am very cold", encoder, decoder,
                                 input_lang, output_lang)))
print("IN: She is hungry")
print("OUT: {}".format(translate("She is hungry", encoder, decoder,
                                 input_lang, output_lang)))
<|endoftext|>


<commit_before>"""
tests.unit.returners.postgres_local_cache_test
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Unit tests for the postgres_local_cache.
"""

import json
import logging
import os
import shutil

import salt.returners.postgres_local_cache as postgres_local_cache
from tests.support.mixins import LoaderModuleMockMixin
from tests.support.mock import MagicMock, patch
from tests.support.runtests import RUNTIME_VARS
from tests.support.unit import TestCase

log = logging.getLogger(__name__)

class PostgresLocalCacheTestCase(TestCase, LoaderModuleMockMixin):
    """
    Tests for the postgre_local_cache returner.
    """

    @classmethod
    def setUpClass(cls):
        cls.TMP_CACHE_DIR = os.path.join(RUNTIME_VARS.TMP, "salt_test_job_cache")
        cls.TMP_JID_DIR = os.path.join(cls.TMP_CACHE_DIR, "jobs")

    def setup_loader_modules(self):
        return {
            postgres_local_cache: {
                "__opts__": {"cachedir": self.TMP_CACHE_DIR, "keep_jobs": 1}
            }
        }

    def tearDown(self):
        """
        Clean up after tests.

        Note that a setUp function is not used in this TestCase because the
        _make_tmp_jid_dirs replaces it.
        """
        if os.path.exists(self.TMP_CACHE_DIR):
            shutil.rmtree(self.TMP_CACHE_DIR)

    def test_returner(self):
        """
        Tests that the returner function
        """
        load = {
            "tgt_type": "glob",
            "fun_args": [],
            "jid": "20200108221839189167",
            "return": True,
            "retcode": 0,
            "success": True,
            "tgt": "minion",
            "cmd": "_return",
            "_stamp": "2020-01-08T22:18:39.197774",
            "arg": [],
            "fun": "test.ping",
            "id": "minion",
        }

        expected = {"return": "True", "retcode": 0, "success": True}

        connect_mock = MagicMock()
        with patch.object(postgres_local_cache, "_get_conn", connect_mock):
            postgres_local_cache.returner(load)

            return_val = None
            for call in connect_mock.mock_calls:
                for arg in call.args:
                    if isinstance(arg, tuple):
                        for val in arg:
                            if isinstance(val, str) and "return" in val:
                                return_val = json.loads(val)

            self.assertIsNotNone(return_val, None)
            self.assertDictEqual(return_val, expected)

    def test_returner_unicode_exception(self):
        """
        Tests that the returner function
        """
        load = {
            "tgt_type": "glob",
            "fun_args": [],
            "jid": "20200108221839189167",
            "return": return_val,
            "retcode": 0,
            "success": True,
            "tgt": "minion",
            "cmd": "_return",
            "_stamp": "2020-01-08T22:18:39.197774",
            "arg": [],
            "fun": "test.ping",
            "id": "minion",
        }

        expected = {"return": "Trüe", "retcode": 0, "success": True}

        connect_mock = MagicMock()
        with patch.object(postgres_local_cache, "_get_conn", connect_mock):
            postgres_local_cache.returner(load)

            return_val = None
            search_string = "return"
            for call in connect_mock.mock_calls:
                for arg in call.args:
                    if isinstance(arg, tuple):
                        for val in arg:
                            if isinstance(val, str):
                                if search_string in val:
                                    return_val = json.loads(val)

            self.assertIsNotNone(return_val, None)
            self.assertDictEqual(return_val, expected)
<commit_msg>Update test_postgres_local_cache.py<commit_after>"""
tests.unit.returners.postgres_local_cache_test
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Unit tests for the postgres_local_cache.
"""

import json
import logging
import os
import shutil

import salt.returners.postgres_local_cache as postgres_local_cache
from tests.support.mixins import LoaderModuleMockMixin
from tests.support.mock import MagicMock, patch
from tests.support.runtests import RUNTIME_VARS
from tests.support.unit import TestCase

log = logging.getLogger(__name__)

class PostgresLocalCacheTestCase(TestCase, LoaderModuleMockMixin):
    """
    Tests for the postgre_local_cache returner.
    """

    @classmethod
    def setUpClass(cls):
        cls.TMP_CACHE_DIR = os.path.join(RUNTIME_VARS.TMP, "salt_test_job_cache")
        cls.TMP_JID_DIR = os.path.join(cls.TMP_CACHE_DIR, "jobs")

    def setup_loader_modules(self):
        return {
            postgres_local_cache: {
                "__opts__": {"cachedir": self.TMP_CACHE_DIR, "keep_jobs": 1}
            }
        }

    def tearDown(self):
        """
        Clean up after tests.

        Note that a setUp function is not used in this TestCase because the
        _make_tmp_jid_dirs replaces it.
        """
        if os.path.exists(self.TMP_CACHE_DIR):
            shutil.rmtree(self.TMP_CACHE_DIR)

    def test_returner(self):
        """
        Tests that the returner function
        """
        load = {
            "tgt_type": "glob",
            "fun_args": [],
            "jid": "20200108221839189167",
            "return": True,
            "retcode": 0,
            "success": True,
            "tgt": "minion",
            "cmd": "_return",
            "_stamp": "2020-01-08T22:18:39.197774",
            "arg": [],
            "fun": "test.ping",
            "id": "minion",
        }

        expected = {"return": "True", "retcode": 0, "success": True}

        connect_mock = MagicMock()
        with patch.object(postgres_local_cache, "_get_conn", connect_mock):
            postgres_local_cache.returner(load)

            return_val = None
            for call in connect_mock.mock_calls:
                for arg in call.args:
                    if isinstance(arg, tuple):
                        for val in arg:
                            if isinstance(val, str) and "return" in val:
                                return_val = json.loads(val)

            self.assertIsNotNone(return_val, None)
            self.assertDictEqual(return_val, expected)

    def test_returner_unicode_exception(self):
        """
        Tests that the returner function
        """
        return_val = "Trüe"

        load = {
            "tgt_type": "glob",
            "fun_args": [],
            "jid": "20200108221839189167",
            "return": return_val,
            "retcode": 0,
            "success": True,
            "tgt": "minion",
            "cmd": "_return",
            "_stamp": "2020-01-08T22:18:39.197774",
            "arg": [],
            "fun": "test.ping",
            "id": "minion",
        }

        expected = {"return": "Trüe", "retcode": 0, "success": True}

        connect_mock = MagicMock()
        with patch.object(postgres_local_cache, "_get_conn", connect_mock):
            postgres_local_cache.returner(load)

            return_val = None
            search_string = "return"
            for call in connect_mock.mock_calls:
                for arg in call.args:
                    if isinstance(arg, tuple):
                        for val in arg:
                            if isinstance(val, str):
                                if search_string in val:
                                    return_val = json.loads(val)

            self.assertIsNotNone(return_val, None)
            self.assertDictEqual(return_val, expected)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import subprocess
import re

outfile = open('commands.md','w')

# tests:
test=[]
test.append( u"Usage: juju sync-tools [options]\n\nSummary:\ncopy tools from the official tool store into a local model\n\nOptions:\n--all  (= false)\n    copy all versions, not just the latest\n--destination (= \"\")\n    local destination directory\n--dev  (= false)\n    consider development versions as well as released ones\n    DEPRECATED: use --stream instead\n--dry-run  (= false)\n    don't copy, just print what would be copied\n--local-dir (= \"\")\n    local destination directory\n-m, --model (= \"\")\n    Model to operate in\n--public  (= false)\n    tools are for a public cloud, so generate mirrors information\n--source (= \"\")\n    local source directory\n--stream (= \"\")\n    simplestreams stream for which to sync metadata\n--version (= \"\")\n    copy a specific major[.minor] version\n\nDetails:\nThis copies the Juju tools tarball from the official tools store (located\nat https://streams.canonical.com/juju) into your model.\nThis is generally done when you want Juju to be able to run without having to\naccess the Internet. Alternatively you can specify a local directory as source.\n\nSometimes this is because the model does not have public access,\nand sometimes you just want to avoid having to access data outside of\nthe local cloud.\n")
test.append( u"Usage: juju bootstrap [options] <controller name> <cloud name>[/region]\n\nSummary:\nInitializes a cloud environment.\n\nOptions:\n--agent-version (= \"\")\n    Version of tools to use for Juju agents\n--auto-upgrade  (= false)\n    Upgrade to the latest patch release tools on first bootstrap\n--bootstrap-constraints  (= )\n    Specify bootstrap machine constraints\n--bootstrap-series (= \"\")\n    Specify the series of the bootstrap machine\n--config  (= )\n    Specify a controller configuration file, or one or more configuration\n    options\n    (--config config.yaml [--config key=value ...])\n--constraints  (= )\n    Set model constraints\n--credential (= \"\")\n    Credentials to use when bootstrapping\n-d, --default-model (= \"default\")\n    Name of the default hosted model for the controller\n--keep-broken  (= false)\n    Do not destroy the model if bootstrap fails\n--metadata-source (= \"\")\n    Local path to use as tools and/or metadata source\n--no-gui  (= false)\n    Do not install the Juju GUI in the controller when bootstrapping\n--to (= \"\")\n    Placement directive indicating an instance to bootstrap\n--upload-tools  (= false)\n    Upload local version of tools before bootstrapping\n\nDetails:\nInitialization consists of creating an 'admin' model and provisioning a\nmachine to act as controller.\nCredentials are set beforehand and are distinct from any other\nconfiguration (see `juju add-credential`).\nThe 'admin' model typically does not run workloads. It should remain\npristine to run and manage Juju's own infrastructure for the corresponding\ncloud. Additional (hosted) models should be created with `juju create-\nmodel` for workload purposes.\nNote that a 'default' model is also created and becomes the current model\nof the environment once the command completes. It can be discarded if\nother models are created.\nIf '--bootstrap-constraints' is used, its values will also apply to any\nfuture controllers provisioned for high availability (HA).\nIf '--constraints' is used, its values will be set as the default\nconstraints for all future workload machines in the model, exactly as if\nthe constraints were set with `juju set-model-constraints`.\nIt is possible to override constraints and the automatic machine selection\nalgorithm by assigning a \"placement directive\" via the '--to' option. This\ndictates what machine to use for the controller. This would typically be\nused with the MAAS provider ('--to <host>.maas').\nYou can change the default timeout and retry delays used during the\nbootstrap by changing the following settings in your configuration file\n(all values represent number of seconds):\n    # How long to wait for a connection to the controller\n    bootstrap-timeout: 600 # default: 10 minutes\n    # How long to wait between connection attempts to a controller\naddress.\n    bootstrap-retry-delay: 5 # default: 5 seconds\n    # How often to refresh controller addresses from the API server.\n    bootstrap-addresses-delay: 10 # default: 10 seconds\nPrivate clouds may need to specify their own custom image metadata and\ntools/agent. Use '--metadata-source' whose value is a local directory.\nThe value of '--agent-version' will become the default tools version to\nuse in all models for this controller. The full binary version is accepted\n(e.g.: 2.0.1-xenial-amd64) but only the numeric version (e.g.: 2.0.1) is\nused. Otherwise, by default, the version used is that of the client.\n\nExamples:\n    juju bootstrap mycontroller google\n    juju bootstrap --config=~/config-rs.yaml mycontroller rackspace\n    juju bootstrap --config agent-version=1.25.3 mycontroller aws\n    juju bootstrap --config bootstrap-timeout=1200 mycontroller azure\n\nSee also:\n    add-credentials\n    create-model\n    set-constraints")

pad='   '

# c = subprocess.check_output(['juju', 'help', 'commands']).splitlines()
# for command in c:

for t in test:
    p = re.compile(u'Usage:(.+?)\n\nSummary:\n(.+?)\n\nOptions:\n(.+?)Details:\n(.+)', re.DOTALL)
    #p = re.compile(ur'Usage:(.+?)\n\nSummary:\n(.+?)\n\nOptions:\n(.+?)Details:\n(.+)', re.DOTALL)
    #test_str = u"Usage: juju sync-tools [options]\n\nSummary:\ncopy tools from the official tool store into a local model\n\nOptions:\n--all  (= false)\n    copy all versions, not just the latest\n--destination (= \"\")\n    local destination directory\n--dev  (= false)\n    consider development versions as well as released ones\n    DEPRECATED: use --stream instead\n--dry-run  (= false)\n    don't copy, just print what would be copied\n--local-dir (= \"\")\n    local destination directory\n-m, --model (= \"\")\n    Model to operate in\n--public  (= false)\n    tools are for a public cloud, so generate mirrors information\n--source (= \"\")\n    local source directory\n--stream (= \"\")\n    simplestreams stream for which to sync metadata\n--version (= \"\")\n    copy a specific major[.minor] version\n\nDetails:\nThis copies the Juju tools tarball from the official tools store (located\nat https://streams.canonical.com/juju) into your model.\nThis is generally done when you want Juju to be able to run without having to\naccess the Internet. Alternatively you can specify a local directory as source.\n\nSometimes this is because the model does not have public access,\nand sometimes you just want to avoid having to access data outside of\nthe local cloud.\n"

    help=re.findall(p, t)[0]
    usage=pad+"**Usage:** `"+help[0]+"`\n\n"
    summary=pad+"**Summary:**\n\n"+pad+help[1]+"\n\n"
    options = pad+'**Options:**\n\n'
    option_lines = help[2].split('\n')

    for line in option_lines:
       if (line !=''):
          if (line[0]=='-'):
             options = options+pad+"_"+line.strip()+"_\n\n"
          else:
             options = options+pad+pad+line.strip()+"\n\n"

    details=help[3]
    examples=''
    also=''

    # search for see also section
    # if it exists, truncate details and process
    q= re.compile(u'See also:\n(.+?)$',re.DOTALL)
    x = re.search( q, details)
    if x:
      match=details[x.start()+10:].split('\n')
      details=details[:x.start()] # truncate the bit we matched
      also=pad+'**See also:**\n\n'

      for line in match:
          also = also+pad+pad+'`'+line.strip()+'`\n\n'

    # search for Examples section
    # if it exists, truncate details and process
    q= re.compile(u'Examples:\n(.+?)$',re.DOTALL)
    x = re.search( q, details)
    if x:
      match=details[x.start()+10:].split('\n')
      details=details[:x.start()] # truncate the bit we matched
      examples=pad+'**Examples:**\n\n'

      for line in match:
        if (line !=''):
          if (line[0]==' '):
            examples=examples+pad+pad
          examples = examples+pad+line.strip()+'\n'
      examples += '\n\n'

    #process the rest of details section.
    section=''
    iflag=False
    for line in details.split('\n'):
       if (line !=''):
          if (line[0]==' '):
             line= pad+pad+line.strip()
             iflag=True
          elif iflag:
             line= '\n'+pad+line
             print(line)
             iflag=False
          if ((len(line)<70) & ((line[-1:]=='.') | (line[-1:]==':'))):
             line=line+'\n'
          section = section+'\n'+pad +line
    details= '\n   **Details:**\n\n'+section+'\n\n'
    outfile.write(usage+summary+options+details+examples+also)
<commit_msg>unicode nonsense<commit_after>#!/usr/bin/env python3

import subprocess
import re

outfile = open('commands.md','w')

# tests:
test=[]
test.append( u"Usage: juju sync-tools [options]\n\nSummary:\ncopy tools from the official tool store into a local model\n\nOptions:\n--all  (= false)\n    copy all versions, not just the latest\n--destination (= \"\")\n    local destination directory\n--dev  (= false)\n    consider development versions as well as released ones\n    DEPRECATED: use --stream instead\n--dry-run  (= false)\n    don't copy, just print what would be copied\n--local-dir (= \"\")\n    local destination directory\n-m, --model (= \"\")\n    Model to operate in\n--public  (= false)\n    tools are for a public cloud, so generate mirrors information\n--source (= \"\")\n    local source directory\n--stream (= \"\")\n    simplestreams stream for which to sync metadata\n--version (= \"\")\n    copy a specific major[.minor] version\n\nDetails:\nThis copies the Juju tools tarball from the official tools store (located\nat https://streams.canonical.com/juju) into your model.\nThis is generally done when you want Juju to be able to run without having to\naccess the Internet. Alternatively you can specify a local directory as source.\n\nSometimes this is because the model does not have public access,\nand sometimes you just want to avoid having to access data outside of\nthe local cloud.\n")
test.append( u"Usage: juju bootstrap [options] <controller name> <cloud name>[/region]\n\nSummary:\nInitializes a cloud environment.\n\nOptions:\n--agent-version (= \"\")\n    Version of tools to use for Juju agents\n--auto-upgrade  (= false)\n    Upgrade to the latest patch release tools on first bootstrap\n--bootstrap-constraints  (= )\n    Specify bootstrap machine constraints\n--bootstrap-series (= \"\")\n    Specify the series of the bootstrap machine\n--config  (= )\n    Specify a controller configuration file, or one or more configuration\n    options\n    (--config config.yaml [--config key=value ...])\n--constraints  (= )\n    Set model constraints\n--credential (= \"\")\n    Credentials to use when bootstrapping\n-d, --default-model (= \"default\")\n    Name of the default hosted model for the controller\n--keep-broken  (= false)\n    Do not destroy the model if bootstrap fails\n--metadata-source (= \"\")\n    Local path to use as tools and/or metadata source\n--no-gui  (= false)\n    Do not install the Juju GUI in the controller when bootstrapping\n--to (= \"\")\n    Placement directive indicating an instance to bootstrap\n--upload-tools  (= false)\n    Upload local version of tools before bootstrapping\n\nDetails:\nInitialization consists of creating an 'admin' model and provisioning a\nmachine to act as controller.\nCredentials are set beforehand and are distinct from any other\nconfiguration (see `juju add-credential`).\nThe 'admin' model typically does not run workloads. It should remain\npristine to run and manage Juju's own infrastructure for the corresponding\ncloud. Additional (hosted) models should be created with `juju create-\nmodel` for workload purposes.\nNote that a 'default' model is also created and becomes the current model\nof the environment once the command completes. It can be discarded if\nother models are created.\nIf '--bootstrap-constraints' is used, its values will also apply to any\nfuture controllers provisioned for high availability (HA).\nIf '--constraints' is used, its values will be set as the default\nconstraints for all future workload machines in the model, exactly as if\nthe constraints were set with `juju set-model-constraints`.\nIt is possible to override constraints and the automatic machine selection\nalgorithm by assigning a \"placement directive\" via the '--to' option. This\ndictates what machine to use for the controller. This would typically be\nused with the MAAS provider ('--to <host>.maas').\nYou can change the default timeout and retry delays used during the\nbootstrap by changing the following settings in your configuration file\n(all values represent number of seconds):\n    # How long to wait for a connection to the controller\n    bootstrap-timeout: 600 # default: 10 minutes\n    # How long to wait between connection attempts to a controller\naddress.\n    bootstrap-retry-delay: 5 # default: 5 seconds\n    # How often to refresh controller addresses from the API server.\n    bootstrap-addresses-delay: 10 # default: 10 seconds\nPrivate clouds may need to specify their own custom image metadata and\ntools/agent. Use '--metadata-source' whose value is a local directory.\nThe value of '--agent-version' will become the default tools version to\nuse in all models for this controller. The full binary version is accepted\n(e.g.: 2.0.1-xenial-amd64) but only the numeric version (e.g.: 2.0.1) is\nused. Otherwise, by default, the version used is that of the client.\n\nExamples:\n    juju bootstrap mycontroller google\n    juju bootstrap --config=~/config-rs.yaml mycontroller rackspace\n    juju bootstrap --config agent-version=1.25.3 mycontroller aws\n    juju bootstrap --config bootstrap-timeout=1200 mycontroller azure\n\nSee also:\n    add-credentials\n    create-model\n    set-constraints")

# useful text
pad='   '
pagetext=("Title:Juju commands and usage\n\n"
          "# Juju Command reference\n\n"
          "You can get a list of the currently used commands by entering\n"
          "```juju help commands``` from the commandline. The currently"
          " understood commands\nare listed here, with usage and examples.\n\n"
          "Click on the expander to see details for each command. \n\n")

commands = subprocess.check_output(['juju', 'help', 'commands']).splitlines()
outfile.write(pagetext)
for c in commands:
  print(c)
  header = '^# '+str(c.split()[0])+ '\n'
  htext = subprocess.check_output(['juju', 'help', c.split()[0].decode('unicode_escape')]).splitlines()
  for t in htext:
    p = re.compile(u'Usage:(.+?)\n\nSummary:\n(.+?)\n\nOptions:\n(.+?)Details:\n(.+)', re.DOTALL)
    #p = re.compile(ur'Usage:(.+?)\n\nSummary:\n(.+?)\n\nOptions:\n(.+?)Details:\n(.+)', re.DOTALL)
    #test_str = u"Usage: juju sync-tools [options]\n\nSummary:\ncopy tools from the official tool store into a local model\n\nOptions:\n--all  (= false)\n    copy all versions, not just the latest\n--destination (= \"\")\n    local destination directory\n--dev  (= false)\n    consider development versions as well as released ones\n    DEPRECATED: use --stream instead\n--dry-run  (= false)\n    don't copy, just print what would be copied\n--local-dir (= \"\")\n    local destination directory\n-m, --model (= \"\")\n    Model to operate in\n--public  (= false)\n    tools are for a public cloud, so generate mirrors information\n--source (= \"\")\n    local source directory\n--stream (= \"\")\n    simplestreams stream for which to sync metadata\n--version (= \"\")\n    copy a specific major[.minor] version\n\nDetails:\nThis copies the Juju tools tarball from the official tools store (located\nat https://streams.canonical.com/juju) into your model.\nThis is generally done when you want Juju to be able to run without having to\naccess the Internet. Alternatively you can specify a local directory as source.\n\nSometimes this is because the model does not have public access,\nand sometimes you just want to avoid having to access data outside of\nthe local cloud.\n"

    help=re.findall(p, t.decode('unicode_escape'))[0]
    usage=pad+"**Usage:** `"+help[0]+"`\n\n"
    summary=pad+"**Summary:**\n\n"+pad+help[1]+"\n\n"
    options = pad+'**Options:**\n\n'
    option_lines = help[2].split('\n')

    for line in option_lines:
       if (line !=''):
          if (line[0]=='-'):
             options = options+pad+"_"+line.strip()+"_\n\n"
          else:
             options = options+pad+pad+line.strip()+"\n\n"

    details=help[3]
    examples=''
    also=''

    # search for see also section
    # if it exists, truncate details and process
    q= re.compile(u'See also:\n(.+?)$',re.DOTALL)
    x = re.search( q, details)
    if x:
      match=details[x.start()+10:].split('\n')
      details=details[:x.start()] # truncate the bit we matched
      also=pad+'**See also:**\n\n'

      for line in match:
          also = also+pad+pad+'`'+line.strip()+'`\n\n'

    # search for Examples section
    # if it exists, truncate details and process
    q= re.compile(u'Examples:\n(.+?)$',re.DOTALL)
    x = re.search( q, details)
    if x:
      match=details[x.start()+10:].split('\n')
      details=details[:x.start()] # truncate the bit we matched
      examples=pad+'**Examples:**\n\n'

      for line in match:
        if (line !=''):
          if (line[0]==' '):
            examples=examples+pad+pad
          examples = examples+pad+line.strip()+'\n'
      examples += '\n\n'

    #process the rest of details section.
    section=''
    iflag=False
    for line in details.split('\n'):
       if (line !=''):
          if (line[0]==' '):
             line= pad+pad+line.strip()
             iflag=True
          elif iflag:
             line= '\n'+pad+line
             print(line)
             iflag=False
          if ((len(line)<70) & ((line[-1:]=='.') | (line[-1:]==':'))):
             line=line+'\n'
          section = section+'\n'+pad +line
    details= '\n   **Details:**\n\n'+section+'\n\n'
    outfile.write(header+usage+summary+options+details+examples+also+'\n\n')
<|endoftext|>


<commit_before>from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect, Http404
from django.template import RequestContext
from django.shortcuts import get_object_or_404, render_to_response

from django.contrib.auth.decorators import login_required

from agora.models import ForumCategory, Forum, UserPostCount
from agora.models import ForumThread, ForumReply, ThreadSubscription

def ajax(func):
    def _wrapped_view(request, *args, **kwargs):
        try:
            return func(request, *args, **kwargs)
        except:
            import sys
            print sys.exc_info()
            raise
    return _wrapped_view

def forums(request):

    categories = ForumCategory.objects.filter(parent__isnull=True)
    categories = categories.order_by("title")

    most_active_forums = Forum.objects.order_by("-reply_count")[:5]
    most_viewed_forums = Forum.objects.order_by("-view_count")[:5]
    most_active_members = UserPostCount.objects.order_by("-count")[:5]

    latest_posts = ForumReply.objects.order_by("-created")[:10]
    latest_threads = ForumThread.objects.order_by("-last_modified")[:10]
    most_active_threads = ForumThread.objects.order_by("-reply_count")[:10]
    most_viewed_threads = ForumThread.objects.order_by("-view_count")[:10]

    return render_to_response("agora/forums.html", {
        "categories": categories,
        "most_active_forums": most_active_forums,
        "most_viewed_forums": most_viewed_forums,
        "most_active_members": most_active_members,
        "latest_posts": latest_posts,
        "latest_threads": latest_threads,
        "most_active_threads": most_active_threads,
        "most_viewed_threads": most_viewed_threads,
    }, context_instance=RequestContext(request))

def forum_category(request, category_id):

    category = get_object_or_404(ForumCategory, id=category_id)

    forums = category.forums.order_by("title")

    return render_to_response("agora/category.html", {
        "category": category,
        "forums": forums,
    }, context_instance=RequestContext(request))

def forum(request, forum_id):

    forum = get_object_or_404(Forum, id=forum_id)

    threads = forum.threads.order_by("-last_modified")

    return render_to_response("agora/forum.html", {
        "forum": forum,
        "threads": threads,
    }, context_instance=RequestContext(request))

def forum_thread(request, thread_id):
    thread = get_object_or_404(ForumThread, id=thread_id)

    if request.method == "POST" and request.user.is_authenticated:

        content = request.POST.get("content")
        subscribe = {"true": True, "false": False, False: False}.get(request.POST.get("subscribe", False), True)

        reply = ForumReply(thread=thread, author=request.user, content=content)
        reply.save()

        # subscribe the poster to the thread if requested (default value is True)
        if subscribe:
            thread.subscribe(reply.author, "email")

        # all users are automatically subscribed to onsite
        thread.subscribe(reply.author, "onsite")

        return HttpResponseRedirect(reverse("agora_thread", args=[thread_id]))

    order_type = request.GET.get("order_type", "asc")

    posts = ForumThread.objects.posts(thread, reverse=(order_type == "desc"))

    thread.inc_views()

    return render_to_response("agora/thread.html", {
        "thread": thread,
        "posts": posts,
        "order_type": order_type,
        "subscribed": thread.subscribed(request.user, "email"),
    }, context_instance=RequestContext(request))

@ajax
@login_required
def new_post(request, forum_id):

    member = request.user.get_profile()
    forum = get_object_or_404(Forum, id=forum_id)

    if request.method == "POST":

        title = request.POST.get("title")
        content = request.POST.get("content")
        subscribe = {"true": True, "false": False, False: False}.get(request.POST.get("subscribe", False), True)

        thread = ForumThread(forum=forum, title=title, content=content, author=request.user)
        thread.save()

        # subscribe the poster to the thread if requested (default value is True)
        if subscribe:
            thread.subscribe(thread.author, "email")

        # all users are automatically subscribed to onsite
        thread.subscribe(thread.author, "onsite")

        return HttpResponseRedirect(reverse("agora_thread", args=[thread.id]))

    return render_to_response("agora/new_post.html", {
        "member": member,
        "forum_id": forum_id,
    }, context_instance=RequestContext(request))

@ajax
@login_required
def reply(request, thread_id):

    member = request.user.get_profile()
    thread = get_object_or_404(ForumThread, id=thread_id)

    quote = request.GET.get("quote") # thread id to quote

    if quote:
        quote_reply = ForumReply.objects.get(id=int(quote))
        quote_content = "\"%s\"" % quote_reply.content
    else:
        quote_content = ""

    if request.method == "POST":
        content = request.POST.get("content")
        subscribe = {"true": True, "false": False, False: False}.get(request.POST.get("subscribe", False), True)

        reply = ForumReply(thread=thread, author=request.user, content=content)
        reply.save()

        # subscribe the poster to the thread if requested (default value is True)
        if subscribe:
            thread.subscribe(reply.author, "email")

        # all users are automatically subscribed to onsite
        thread.subscribe(reply.author, "onsite")

        return HttpResponseRedirect(reverse("agora_thread", args=[thread_id]))

    first_reply = not ForumReply.objects.filter(thread=thread, author=request.user).exists()

    return render_to_response("agora/reply.html", {
        "member": member,
        "thread_id": thread_id,
        "quote_content": quote_content,
        "subscribed": thread.subscribed(request.user, "email"),
        "first_reply": first_reply,
    }, context_instance=RequestContext(request))

@ajax
@login_required
def post_edit(request, post_kind, post_id):
    member = request.user.get_profile()

    if post_kind == "thread":
        post = get_object_or_404(ForumThread, id=post_id)
        thread_id = post.id
    elif post_kind == "reply":
        post = get_object_or_404(ForumReply, id=post_id)
        thread_id = post.thread.id
    else:
        raise Http404

    if not post.editable(request.user):
        raise Http404()

    if request.method == "POST":
        content = request.POST.get("content")

        post.content = content
        post.save()

        return HttpResponseRedirect(reverse("agora_thread", args=[thread_id]))

    return render_to_response("agora/post_edit.html", {
        "member": member,
        "post_kind": post_kind,
        "post": post,
    }, context_instance=RequestContext(request))

@ajax
@login_required
def subscribe(request, thread_id):
    user = request.user
    thread = get_object_or_404(ForumThread, pk=thread_id)

    if request.method == "POST":
        thread.subscribe(user, "email")
        return HttpResponseRedirect(reverse("agora_thread", args=[thread_id]))
    else:
        ctx = RequestContext(request, {"thread": thread})
        return render_to_response("agora/subscribe.html", ctx)

@ajax
@login_required
def unsubscribe(request, thread_id):
    user = request.user
    thread = get_object_or_404(ForumThread, pk=thread_id)

    if request.method == "POST":
        thread.unsubscribe(user, "email")
        return HttpResponseRedirect(reverse("agora_thread", args=[thread_id]))
    else:
        ctx = RequestContext(request, {"thread": thread})
        return render_to_response("agora/unsubscribe.html", ctx)

@login_required
def thread_updates(request):
    subscriptions = ThreadSubscription.objects.filter(user=request.user, kind="onsite")
    subscriptions = subscriptions.select_related("thread", "user")
    subscriptions = subscriptions.order_by("-thread__last_modified")

    if request.method == "POST":
        subscriptions.filter(pk=request.POST["thread_id"]).delete()

    ctx = {
        "subscriptions": subscriptions,
    }
    ctx = RequestContext(request, ctx)
    return render_to_response("agora/thread_updates.html", ctx)
<commit_msg>removed limiting allowing template to deal with it<commit_after>from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect, Http404
from django.template import RequestContext
from django.shortcuts import get_object_or_404, render_to_response

from django.contrib.auth.decorators import login_required

from agora.models import ForumCategory, Forum, UserPostCount
from agora.models import ForumThread, ForumReply, ThreadSubscription

def ajax(func):
    def _wrapped_view(request, *args, **kwargs):
        try:
            return func(request, *args, **kwargs)
        except:
            import sys
            print sys.exc_info()
            raise
    return _wrapped_view

def forums(request):

    categories = ForumCategory.objects.filter(parent__isnull=True)
    categories = categories.order_by("title")

    most_active_forums = Forum.objects.order_by("-reply_count")[:5]
    most_viewed_forums = Forum.objects.order_by("-view_count")[:5]
    most_active_members = UserPostCount.objects.order_by("-count")[:5]

    latest_posts = ForumReply.objects.order_by("-created")[:10]
    latest_threads = ForumThread.objects.order_by("-last_modified")
    most_active_threads = ForumThread.objects.order_by("-reply_count")
    most_viewed_threads = ForumThread.objects.order_by("-view_count")

    return render_to_response("agora/forums.html", {
        "categories": categories,
        "most_active_forums": most_active_forums,
        "most_viewed_forums": most_viewed_forums,
        "most_active_members": most_active_members,
        "latest_posts": latest_posts,
        "latest_threads": latest_threads,
        "most_active_threads": most_active_threads,
        "most_viewed_threads": most_viewed_threads,
    }, context_instance=RequestContext(request))

def forum_category(request, category_id):

    category = get_object_or_404(ForumCategory, id=category_id)

    forums = category.forums.order_by("title")

    return render_to_response("agora/category.html", {
        "category": category,
        "forums": forums,
    }, context_instance=RequestContext(request))

def forum(request, forum_id):

    forum = get_object_or_404(Forum, id=forum_id)

    threads = forum.threads.order_by("-last_modified")

    return render_to_response("agora/forum.html", {
        "forum": forum,
        "threads": threads,
    }, context_instance=RequestContext(request))

def forum_thread(request, thread_id):
    thread = get_object_or_404(ForumThread, id=thread_id)

    if request.method == "POST" and request.user.is_authenticated:

        content = request.POST.get("content")
        subscribe = {"true": True, "false": False, False: False}.get(request.POST.get("subscribe", False), True)

        reply = ForumReply(thread=thread, author=request.user, content=content)
        reply.save()

        # subscribe the poster to the thread if requested (default value is True)
        if subscribe:
            thread.subscribe(reply.author, "email")

        # all users are automatically subscribed to onsite
        thread.subscribe(reply.author, "onsite")

        return HttpResponseRedirect(reverse("agora_thread", args=[thread_id]))

    order_type = request.GET.get("order_type", "asc")

    posts = ForumThread.objects.posts(thread, reverse=(order_type == "desc"))

    thread.inc_views()

    return render_to_response("agora/thread.html", {
        "thread": thread,
        "posts": posts,
        "order_type": order_type,
        "subscribed": thread.subscribed(request.user, "email"),
    }, context_instance=RequestContext(request))

@ajax
@login_required
def new_post(request, forum_id):

    member = request.user.get_profile()
    forum = get_object_or_404(Forum, id=forum_id)

    if request.method == "POST":

        title = request.POST.get("title")
        content = request.POST.get("content")
        subscribe = {"true": True, "false": False, False: False}.get(request.POST.get("subscribe", False), True)

        thread = ForumThread(forum=forum, title=title, content=content, author=request.user)
        thread.save()

        # subscribe the poster to the thread if requested (default value is True)
        if subscribe:
            thread.subscribe(thread.author, "email")

        # all users are automatically subscribed to onsite
        thread.subscribe(thread.author, "onsite")

        return HttpResponseRedirect(reverse("agora_thread", args=[thread.id]))

    return render_to_response("agora/new_post.html", {
        "member": member,
        "forum_id": forum_id,
    }, context_instance=RequestContext(request))

@ajax
@login_required
def reply(request, thread_id):

    member = request.user.get_profile()
    thread = get_object_or_404(ForumThread, id=thread_id)

    quote = request.GET.get("quote") # thread id to quote

    if quote:
        quote_reply = ForumReply.objects.get(id=int(quote))
        quote_content = "\"%s\"" % quote_reply.content
    else:
        quote_content = ""

    if request.method == "POST":
        content = request.POST.get("content")
        subscribe = {"true": True, "false": False, False: False}.get(request.POST.get("subscribe", False), True)

        reply = ForumReply(thread=thread, author=request.user, content=content)
        reply.save()

        # subscribe the poster to the thread if requested (default value is True)
        if subscribe:
            thread.subscribe(reply.author, "email")

        # all users are automatically subscribed to onsite
        thread.subscribe(reply.author, "onsite")

        return HttpResponseRedirect(reverse("agora_thread", args=[thread_id]))

    first_reply = not ForumReply.objects.filter(thread=thread, author=request.user).exists()

    return render_to_response("agora/reply.html", {
        "member": member,
        "thread_id": thread_id,
        "quote_content": quote_content,
        "subscribed": thread.subscribed(request.user, "email"),
        "first_reply": first_reply,
    }, context_instance=RequestContext(request))

@ajax
@login_required
def post_edit(request, post_kind, post_id):
    member = request.user.get_profile()

    if post_kind == "thread":
        post = get_object_or_404(ForumThread, id=post_id)
        thread_id = post.id
    elif post_kind == "reply":
        post = get_object_or_404(ForumReply, id=post_id)
        thread_id = post.thread.id
    else:
        raise Http404

    if not post.editable(request.user):
        raise Http404()

    if request.method == "POST":
        content = request.POST.get("content")

        post.content = content
        post.save()

        return HttpResponseRedirect(reverse("agora_thread", args=[thread_id]))

    return render_to_response("agora/post_edit.html", {
        "member": member,
        "post_kind": post_kind,
        "post": post,
    }, context_instance=RequestContext(request))

@ajax
@login_required
def subscribe(request, thread_id):
    user = request.user
    thread = get_object_or_404(ForumThread, pk=thread_id)

    if request.method == "POST":
        thread.subscribe(user, "email")
        return HttpResponseRedirect(reverse("agora_thread", args=[thread_id]))
    else:
        ctx = RequestContext(request, {"thread": thread})
        return render_to_response("agora/subscribe.html", ctx)

@ajax
@login_required
def unsubscribe(request, thread_id):
    user = request.user
    thread = get_object_or_404(ForumThread, pk=thread_id)

    if request.method == "POST":
        thread.unsubscribe(user, "email")
        return HttpResponseRedirect(reverse("agora_thread", args=[thread_id]))
    else:
        ctx = RequestContext(request, {"thread": thread})
        return render_to_response("agora/unsubscribe.html", ctx)

@login_required
def thread_updates(request):
    subscriptions = ThreadSubscription.objects.filter(user=request.user, kind="onsite")
    subscriptions = subscriptions.select_related("thread", "user")
    subscriptions = subscriptions.order_by("-thread__last_modified")

    if request.method == "POST":
        subscriptions.filter(pk=request.POST["thread_id"]).delete()

    ctx = {
        "subscriptions": subscriptions,
    }
    ctx = RequestContext(request, ctx)
    return render_to_response("agora/thread_updates.html", ctx)
<|endoftext|>