<commit_before>import getpass
import os
import requests
from datetime import datetime
from textwrap import dedent
from pathlib import Path

from pythonanywhere.exceptions import SanityException
from pythonanywhere.snakesay import snakesay

PYTHON_VERSIONS = {
    '2.7': 'python27',
    '3.4': 'python34',
    '3.5': 'python35',
    '3.6': 'python36',
    '3.7': 'python37',
}

class AuthenticationError(Exception):
    pass

class NoTokenError(Exception):
    pass

def get_api_endpoint():
    domain = os.environ.get('PYTHONANYWHERE_DOMAIN', 'pythonanywhere.com')
    return 'https://www.{domain}/api/v0/user/{{username}}/{{flavor}}/'.format(domain=domain)

def call_api(url, method, **kwargs):
    token = os.environ.get('API_TOKEN')
    if token is None:
        raise NoTokenError(
            "Oops, you don't seem to have an API token.  "
            "Please go to the 'Account' page on PythonAnywhere, then to the 'API Token' "
            "tab.  Click the 'Create a new API token' button to create the token, then "
            "start a new console and try running this script again."
        )
    insecure = os.environ.get('PYTHONANYWHERE_INSECURE_API') == 'true'
    response = requests.request(
        method=method,
        url=url,
        headers={'Authorization': 'Token {token}'.format(token=token)},
        verify=not insecure,
        **kwargs
    )
    if response.status_code == 401:
        print(response, response.text)
        raise AuthenticationError(
            'Authentication error {status_code} calling API: {response_text}'.format(
                status_code=response.status_code,
                response_text=response.text,
            )
        )
    return response

class Webapp:

    def __init__(self, domain):
        self.domain = domain

    def __eq__(self, other):
        return self.domain == other.domain

    def sanity_checks(self, nuke):
        print(snakesay('Running API sanity checks'))
        token = os.environ.get('API_TOKEN')
        if not token:
            raise SanityException(dedent(
                '''
                Could not find your API token.
                You may need to create it on the Accounts page?
                You will also need to close this console and open a new one once you've done that.
                '''
            ))

        if nuke:
            return

        url = get_api_endpoint().format(username=getpass.getuser(), flavor="webapps") + self.domain + '/'
        response = call_api(url, 'get')
        if response.status_code == 200:
            raise SanityException(
                'You already have a webapp for {domain}.\n\nUse the --nuke option if you want to replace it.'.format(
                    domain=self.domain
                )
            )

    def create(self, python_version, virtualenv_path, project_path, nuke):
        print(snakesay('Creating web app via API'))
        if nuke:
            webapp_url = get_api_endpoint().format(username=getpass.getuser(), flavor="webapps") + self.domain + '/'
            call_api(webapp_url, 'delete')
        post_url = get_api_endpoint().format(username=getpass.getuser(), flavor="webapps")
        patch_url = post_url + self.domain + '/'
        response = call_api(post_url, 'post', data={
            'domain_name': self.domain, 'python_version': PYTHON_VERSIONS[python_version]},
        )
        if not response.ok or response.json().get('status') == 'ERROR':
            raise Exception(
                'POST to create webapp via API failed, got {response}:{response_text}'.format(
                    response=response,
                    response_text=response.text,
                )
            )
        response = call_api(
            patch_url, 'patch',
            data={'virtualenv_path': virtualenv_path, 'source_directory': project_path}
        )
        if not response.ok:
            raise Exception(
                "PATCH to set virtualenv path and source directory via API failed,"
                "got {response}:{response_text}".format(
                    response=response,
                    response_text=response.text,
                )
            )

    def add_default_static_files_mappings(self, project_path):
        print(snakesay('Adding static files mappings for /static/ and /media/'))

        url = get_api_endpoint().format(username=getpass.getuser(), flavor="webapps") + self.domain + '/static_files/'
        call_api(url, 'post', json=dict(
            url='/static/', path=str(Path(project_path) / 'static'),
        ))
        call_api(url, 'post', json=dict(
            url='/media/', path=str(Path(project_path) / 'media'),
        ))

    def reload(self):
        print(snakesay('Reloading {domain} via API'.format(domain=self.domain)))
        url = get_api_endpoint().format(username=getpass.getuser(), flavor="webapps") + self.domain + '/reload/'
        response = call_api(url, 'post')
        if not response.ok:
            raise Exception(
                'POST to reload webapp via API failed, got {response}:{response_text}'.format(
                    response=response,
                    response_text=response.text,
                )
            )

    def set_ssl(self, certificate, private_key):
        print(snakesay('Setting up SSL for {domain} via API'.format(domain=self.domain)))
        url = get_api_endpoint().format(username=getpass.getuser(), flavor="webapps") + self.domain + '/ssl/'
        response = call_api(
            url, 'post',
            json={'cert': certificate, 'private_key': private_key}
        )
        if not response.ok:
            raise Exception(
                'POST to set SSL details via API failed, got {response}:{response_text}'.format(
                    response=response,
                    response_text=response.text,
                )
            )

    def get_ssl_info(self):
        url = get_api_endpoint().format(username=getpass.getuser(), flavor="webapps") + self.domain + '/ssl/'
        response = call_api(url, 'get')
        if not response.ok:
            raise Exception(
                'GET SSL details via API failed, got {response}:{response_text}'.format(
                    response=response,
                    response_text=response.text,
                )
            )

        result = response.json()
        result["not_after"] = datetime.strptime(result["not_after"], "%Y%m%dT%H%M%SZ")
        return result

    def delete_log(self, log_type, index=0):
        if index:
            print(snakesay(
                'Deleting old (archive number {index}) {type} log file for {domain} via API'.format(index=index,
                                                                                type=log_type,
                                                                                domain=self.domain)))
        else:
            print(snakesay(
                'Deleting current {type} log file for {domain} via API'.format(type=log_type, domain=self.domain)))

        if index == 1:
            url = get_api_endpoint().format(username=getpass.getuser(), flavor="files") + "path/var/log/{domain}.{type}.log.1/".format(
                domain=self.domain, type=log_type)
        elif index > 1:
            url = get_api_endpoint().format(
                username=getpass.getuser(), flavor="files") + "path/var/log/{domain}.{type}.log.{index}.gz/".format(
                domain=self.domain, type=log_type, index=index)
        else:
            url = get_api_endpoint().format(username=getpass.getuser(), flavor="files") + "path/var/log/{domain}.{type}.log/".format(
                domain=self.domain, type=log_type)
        response = call_api(url, "delete")
        if not response.ok:
            raise Exception(
                "DELETE log file via API failed, got {response}:{response_text}".format(
                    response=response,
                    response_text=response.text,
                )
            )

    def get_log_info(self):
        url = get_api_endpoint().format(username=getpass.getuser(),
                                        flavor="files") + "tree/?path=/var/log/"
        response = call_api(url, "get")
        if not response.ok:
            raise Exception(
                "GET log files info via API failed, got {response}:{response_text}".format(
                    response=response,
                    response_text=response.text,
                )
            )
        if response.text[0] == "[" and response.text[-1] == "]":
            file_list = eval(response.text)
        else:
            raise Exception(
                "GET log files info via API failed, got {text} instead of list of files".format(text=response.text))
        log_types = ["access", "error", "server"]
        logs = {"access": [], "error": [], "server": []}
        log_prefix = "/var/log/{domain}.".format(domain=self.domain)
        for file in file_list:
            if type(file) == str and file.startswith(log_prefix):
                log = file[len(log_prefix):].split(".")
                if log[0] in log_types:
                    log_type = log[0]
                    if log[-1] == "log":
                        log_index = 0
                    elif log[-1] == "1":
                        log_index = 1
                    elif log[-1] == "gz":
                        log_index = int(log[-2])
                    else:
                        continue
                    logs[log_type].append(log_index)
        return logs
<commit_msg>readability improvement: regex instead of slicing<commit_after>import getpass
import os
import re
import requests
from datetime import datetime
from textwrap import dedent
from pathlib import Path

from pythonanywhere.exceptions import SanityException
from pythonanywhere.snakesay import snakesay

PYTHON_VERSIONS = {
    '2.7': 'python27',
    '3.4': 'python34',
    '3.5': 'python35',
    '3.6': 'python36',
    '3.7': 'python37',
}

class AuthenticationError(Exception):
    pass

class NoTokenError(Exception):
    pass

def get_api_endpoint():
    domain = os.environ.get('PYTHONANYWHERE_DOMAIN', 'pythonanywhere.com')
    return 'https://www.{domain}/api/v0/user/{{username}}/{{flavor}}/'.format(domain=domain)

def call_api(url, method, **kwargs):
    token = os.environ.get('API_TOKEN')
    if token is None:
        raise NoTokenError(
            "Oops, you don't seem to have an API token.  "
            "Please go to the 'Account' page on PythonAnywhere, then to the 'API Token' "
            "tab.  Click the 'Create a new API token' button to create the token, then "
            "start a new console and try running this script again."
        )
    insecure = os.environ.get('PYTHONANYWHERE_INSECURE_API') == 'true'
    response = requests.request(
        method=method,
        url=url,
        headers={'Authorization': 'Token {token}'.format(token=token)},
        verify=not insecure,
        **kwargs
    )
    if response.status_code == 401:
        print(response, response.text)
        raise AuthenticationError(
            'Authentication error {status_code} calling API: {response_text}'.format(
                status_code=response.status_code,
                response_text=response.text,
            )
        )
    return response

class Webapp:

    def __init__(self, domain):
        self.domain = domain

    def __eq__(self, other):
        return self.domain == other.domain

    def sanity_checks(self, nuke):
        print(snakesay('Running API sanity checks'))
        token = os.environ.get('API_TOKEN')
        if not token:
            raise SanityException(dedent(
                '''
                Could not find your API token.
                You may need to create it on the Accounts page?
                You will also need to close this console and open a new one once you've done that.
                '''
            ))

        if nuke:
            return

        url = get_api_endpoint().format(username=getpass.getuser(), flavor="webapps") + self.domain + '/'
        response = call_api(url, 'get')
        if response.status_code == 200:
            raise SanityException(
                'You already have a webapp for {domain}.\n\nUse the --nuke option if you want to replace it.'.format(
                    domain=self.domain
                )
            )

    def create(self, python_version, virtualenv_path, project_path, nuke):
        print(snakesay('Creating web app via API'))
        if nuke:
            webapp_url = get_api_endpoint().format(username=getpass.getuser(), flavor="webapps") + self.domain + '/'
            call_api(webapp_url, 'delete')
        post_url = get_api_endpoint().format(username=getpass.getuser(), flavor="webapps")
        patch_url = post_url + self.domain + '/'
        response = call_api(post_url, 'post', data={
            'domain_name': self.domain, 'python_version': PYTHON_VERSIONS[python_version]},
        )
        if not response.ok or response.json().get('status') == 'ERROR':
            raise Exception(
                'POST to create webapp via API failed, got {response}:{response_text}'.format(
                    response=response,
                    response_text=response.text,
                )
            )
        response = call_api(
            patch_url, 'patch',
            data={'virtualenv_path': virtualenv_path, 'source_directory': project_path}
        )
        if not response.ok:
            raise Exception(
                "PATCH to set virtualenv path and source directory via API failed,"
                "got {response}:{response_text}".format(
                    response=response,
                    response_text=response.text,
                )
            )

    def add_default_static_files_mappings(self, project_path):
        print(snakesay('Adding static files mappings for /static/ and /media/'))

        url = get_api_endpoint().format(username=getpass.getuser(), flavor="webapps") + self.domain + '/static_files/'
        call_api(url, 'post', json=dict(
            url='/static/', path=str(Path(project_path) / 'static'),
        ))
        call_api(url, 'post', json=dict(
            url='/media/', path=str(Path(project_path) / 'media'),
        ))

    def reload(self):
        print(snakesay('Reloading {domain} via API'.format(domain=self.domain)))
        url = get_api_endpoint().format(username=getpass.getuser(), flavor="webapps") + self.domain + '/reload/'
        response = call_api(url, 'post')
        if not response.ok:
            raise Exception(
                'POST to reload webapp via API failed, got {response}:{response_text}'.format(
                    response=response,
                    response_text=response.text,
                )
            )

    def set_ssl(self, certificate, private_key):
        print(snakesay('Setting up SSL for {domain} via API'.format(domain=self.domain)))
        url = get_api_endpoint().format(username=getpass.getuser(), flavor="webapps") + self.domain + '/ssl/'
        response = call_api(
            url, 'post',
            json={'cert': certificate, 'private_key': private_key}
        )
        if not response.ok:
            raise Exception(
                'POST to set SSL details via API failed, got {response}:{response_text}'.format(
                    response=response,
                    response_text=response.text,
                )
            )

    def get_ssl_info(self):
        url = get_api_endpoint().format(username=getpass.getuser(), flavor="webapps") + self.domain + '/ssl/'
        response = call_api(url, 'get')
        if not response.ok:
            raise Exception(
                'GET SSL details via API failed, got {response}:{response_text}'.format(
                    response=response,
                    response_text=response.text,
                )
            )

        result = response.json()
        result["not_after"] = datetime.strptime(result["not_after"], "%Y%m%dT%H%M%SZ")
        return result

    def delete_log(self, log_type, index=0):
        if index:
            print(snakesay(
                'Deleting old (archive number {index}) {type} log file for {domain} via API'.format(index=index,
                                                                                type=log_type,
                                                                                domain=self.domain)))
        else:
            print(snakesay(
                'Deleting current {type} log file for {domain} via API'.format(type=log_type, domain=self.domain)))

        if index == 1:
            url = get_api_endpoint().format(username=getpass.getuser(), flavor="files") + "path/var/log/{domain}.{type}.log.1/".format(
                domain=self.domain, type=log_type)
        elif index > 1:
            url = get_api_endpoint().format(
                username=getpass.getuser(), flavor="files") + "path/var/log/{domain}.{type}.log.{index}.gz/".format(
                domain=self.domain, type=log_type, index=index)
        else:
            url = get_api_endpoint().format(username=getpass.getuser(), flavor="files") + "path/var/log/{domain}.{type}.log/".format(
                domain=self.domain, type=log_type)
        response = call_api(url, "delete")
        if not response.ok:
            raise Exception(
                "DELETE log file via API failed, got {response}:{response_text}".format(
                    response=response,
                    response_text=response.text,
                )
            )

    def get_log_info(self):
        url = get_api_endpoint().format(username=getpass.getuser(),
                                        flavor="files") + "tree/?path=/var/log/"
        response = call_api(url, "get")
        if not response.ok:
            raise Exception(
                "GET log files info via API failed, got {response}:{response_text}".format(
                    response=response,
                    response_text=response.text,
                )
            )
        if re.match(r"\[.*\]", response.text):
            file_list = eval(response.text)
        else:
            raise Exception(
                "GET log files info via API failed, got {text} instead of list of files".format(text=response.text))
        log_types = ["access", "error", "server"]
        logs = {"access": [], "error": [], "server": []}
        log_prefix = "/var/log/{domain}.".format(domain=self.domain)
        for file in file_list:
            if type(file) == str and file.startswith(log_prefix):
                log = file[len(log_prefix):].split(".")
                if log[0] in log_types:
                    log_type = log[0]
                    if log[-1] == "log":
                        log_index = 0
                    elif log[-1] == "1":
                        log_index = 1
                    elif log[-1] == "gz":
                        log_index = int(log[-2])
                    else:
                        continue
                    logs[log_type].append(log_index)
        return logs
<|endoftext|>


<commit_before>import glob
import os

from pythonwarrior.config import Config
from pythonwarrior.player_generator import PlayerGenerator
from pythonwarrior.profile import Profile
from pythonwarrior.tower import Tower
from pythonwarrior.ui import UI

class Game(object):
    def __init__(self):
        self._profile = None
        self._current_level = None
        self._next_level = None

    def start(self):
        UI.puts('Welcome to Python Warrior')
        if os.path.exists(Config.path_prefix + '/.profile'):
            self._profile = Profile.load(Config.path_prefix + '/.profile')
        else:
            if not os.path.exists(Config.path_prefix + '/pythonwarrior'):
                self.make_game_directory()
        self.play_normal_mode()

    def make_game_directory(self):
        if UI.ask("No pythonwarrior directory found, \
                  would you like to create one?"):
            os.mkdir(Config.path_prefix + '/pythonwarrior')
        else:
            UI.puts('Unable to continue without directory.')
            raise Exception("Unable to continue without directory.")

    def play_normal_mode(self):
        if Config.practice_level:
            UI.puts("Unable to practice level while not in epic mode,"
                    "remove -l options")
        else:
            if self.current_level().number == 0:
                self.prepare_next_level()
                UI.puts("First level has been generated."
                        "See the pythonwarrior/%s/README for instructions."
                        % self.profile().directory_name())
            else:
                self.play_current_level()

    def play_current_level(self):
        continue_play = True
        UI.puts("Starting level %d" % self.current_level().number)
        self.current_level().play()
        if self.current_level().is_passed():
            if self.next_level().exists():
                UI.puts("Success! You have found the stairs.")
            else:
                UI.puts("CONGRATULATIONS! You have climbed to the top "
                        "of the tower and rescued the fair maiden Python")
                continue_play = False
            self.current_level().tally_points()
            self.request_next_level()
        else:
            continue_play = False
            UI.puts("Sorry, you failed level %d. "
                    "Change your script and try again." %
                    self.current_level().number)
            if (not Config.skip_input) and self.current_level().clue \
                    and UI.ask("Would you like to read the "
                               "additional clues for this level?"):
                        UI.puts(self.current_level().clue)
        return continue_play

    def request_next_level(self):
        if not Config.skip_input:
            if self.next_level().exists():
                if UI.ask('Would you like to continue on to the next level?'):
                    self.prepare_next_level()
                    UI.puts('See the updated README in the pythonwarrior/' +
                            self.profile().directory_name() +
                            ' directory')
                else:
                    UI.puts('Staying on current level.'
                            'Try to earn more points next time.')
            else:
                UI.ask('Would you like to continue on to epic mode?')
                UI.puts('Sorry, epic mode is not yet enabled!')

    def prepare_next_level(self):
        self.next_level().load_level()
        PlayerGenerator(self.next_level(), self.current_level()).generate()
        self.profile().level_number += 1
        self.profile().save()

    def profiles(self):
        return map(lambda profile: Profile.load(profile), self.profile_paths())

    def profile_paths(self):
        return glob.glob(Config.path_prefix + '/pythonwarrior/**/.profile')

    def profile(self):
        if self._profile is None:
            self._profile = self.choose_profile()
        return self._profile

    def new_profile(self):
        profile = Profile()
        profile.tower_path = UI.choose('tower', self.towers()).path
        profile.warrior_name = UI.request('Enter a name for your warrior: ')
        return profile

    def towers(self):
        return map(lambda path: Tower(path), self.tower_paths())

    def tower_paths(self):
        tower_paths = glob.glob(os.path.normpath(os.path.abspath(__file__) +
                                                 '/../towers/*'))
        tower_paths = filter(lambda path: os.path.isdir(path), tower_paths)
        return tower_paths

    def current_level(self):
        if not self._current_level:
            self._current_level = self.profile().current_level()
        return self._current_level

    def next_level(self):
        if not self._next_level:
            self._next_level = self.profile().next_level()
        return self._next_level

    def choose_profile(self):
        profile = UI.choose('profile',
                            self.profiles() + [['new', 'New Profile']])
        if profile == 'new':
            profile = self.new_profile()
            if filter(lambda prof: prof.player_path == profile.player_path,
                      self.profiles()):
                if UI.ask('Are you sure you want to replace your existing'
                          'profile for this tower?'):
                    UI.puts('Replacing existing profile.')
                    return profile
                else:
                    UI.puts('Not replacing profile.')
                    raise Exception('Not replacing profile')
            else:
                return profile
        else:
            return profile
<commit_msg>Add missing message when staying on the current level<commit_after>import glob
import os

from pythonwarrior.config import Config
from pythonwarrior.player_generator import PlayerGenerator
from pythonwarrior.profile import Profile
from pythonwarrior.tower import Tower
from pythonwarrior.ui import UI

class Game(object):
    def __init__(self):
        self._profile = None
        self._current_level = None
        self._next_level = None

    def start(self):
        UI.puts('Welcome to Python Warrior')
        if os.path.exists(Config.path_prefix + '/.profile'):
            self._profile = Profile.load(Config.path_prefix + '/.profile')
        else:
            if not os.path.exists(Config.path_prefix + '/pythonwarrior'):
                self.make_game_directory()
        self.play_normal_mode()

    def make_game_directory(self):
        if UI.ask("No pythonwarrior directory found, \
                  would you like to create one?"):
            os.mkdir(Config.path_prefix + '/pythonwarrior')
        else:
            UI.puts('Unable to continue without directory.')
            raise Exception("Unable to continue without directory.")

    def play_normal_mode(self):
        if Config.practice_level:
            UI.puts("Unable to practice level while not in epic mode,"
                    "remove -l options")
        else:
            if self.current_level().number == 0:
                self.prepare_next_level()
                UI.puts("First level has been generated."
                        "See the pythonwarrior/%s/README for instructions."
                        % self.profile().directory_name())
            else:
                self.play_current_level()

    def play_current_level(self):
        continue_play = True
        UI.puts("Starting level %d" % self.current_level().number)
        self.current_level().play()
        if self.current_level().is_passed():
            if self.next_level().exists():
                UI.puts("Success! You have found the stairs.")
            else:
                UI.puts("CONGRATULATIONS! You have climbed to the top "
                        "of the tower and rescued the fair maiden Python")
                continue_play = False
            self.current_level().tally_points()
            self.request_next_level()
        else:
            continue_play = False
            UI.puts("Sorry, you failed level %d. "
                    "Change your script and try again." %
                    self.current_level().number)
            if (not Config.skip_input) and self.current_level().clue \
                    and UI.ask("Would you like to read the "
                               "additional clues for this level?"):
                        UI.puts(self.current_level().clue)
        return continue_play

    def request_next_level(self):
        if not Config.skip_input:
            if self.next_level().exists():
                if UI.ask('Would you like to continue on to the next level?'):
                    self.prepare_next_level()
                    UI.puts('See the updated README in the pythonwarrior/' +
                            self.profile().directory_name() +
                            ' directory')
                else:
                    UI.puts('Staying on current level.'
                            'Try to earn more points next time.')

            else:
                if UI.ask('Would you like to continue on to epic mode?'):
                    UI.puts('Sorry, epic mode is not yet enabled!')
                else:
                    UI.puts("Staying on current level. "
                            "Try to earn more points next time.")

    def prepare_next_level(self):
        self.next_level().load_level()
        PlayerGenerator(self.next_level(), self.current_level()).generate()
        self.profile().level_number += 1
        self.profile().save()

    def profiles(self):
        return map(lambda profile: Profile.load(profile), self.profile_paths())

    def profile_paths(self):
        return glob.glob(Config.path_prefix + '/pythonwarrior/**/.profile')

    def profile(self):
        if self._profile is None:
            self._profile = self.choose_profile()
        return self._profile

    def new_profile(self):
        profile = Profile()
        profile.tower_path = UI.choose('tower', self.towers()).path
        profile.warrior_name = UI.request('Enter a name for your warrior: ')
        return profile

    def towers(self):
        return map(lambda path: Tower(path), self.tower_paths())

    def tower_paths(self):
        tower_paths = glob.glob(os.path.normpath(os.path.abspath(__file__) +
                                                 '/../towers/*'))
        tower_paths = filter(lambda path: os.path.isdir(path), tower_paths)
        return tower_paths

    def current_level(self):
        if not self._current_level:
            self._current_level = self.profile().current_level()
        return self._current_level

    def next_level(self):
        if not self._next_level:
            self._next_level = self.profile().next_level()
        return self._next_level

    def choose_profile(self):
        profile = UI.choose('profile',
                            self.profiles() + [['new', 'New Profile']])
        if profile == 'new':
            profile = self.new_profile()
            if filter(lambda prof: prof.player_path == profile.player_path,
                      self.profiles()):
                if UI.ask('Are you sure you want to replace your existing'
                          'profile for this tower?'):
                    UI.puts('Replacing existing profile.')
                    return profile
                else:
                    UI.puts('Not replacing profile.')
                    raise Exception('Not replacing profile')
            else:
                return profile
        else:
            return profile
<|endoftext|>


<commit_before>import unittest
from content import *

class Element (object):
    __name = None

    def __init__ (self, name):
        self.__name = name

    def name (self): return self.__name

    def __str__ (self): return self.name()

# Represent state transitions as a map from states to maps from
# symbols to sets of states.  States are integers.

class NFA (dict):
    __stateID = -1
    __start = None
    __end = None

    def __init__ (self):
        self.__end = self.newState()
        self.__start = self.newState()

    def newState (self):
        self.__stateID += 1
        self.setdefault(self.__stateID, {})
        return self.__stateID

    def start (self): return self.__start

    def end (self): return self.__end

    def addTransition (self, key, start, end):
        assert end is not None
        self.setdefault(start, {}).setdefault(key, set()).add(end)
        return self

    def ok (self, key, start, end):
        return end in self[start].get(key, set())

    def oneStep (self, node, key):
        return self.setdefault(node, {}).get(key, set())

    def removeState (self, state):
        del self[state]
        for transitions in self.values():
            for target in transitions.values():
                target.discard(state)

    def addSubNFA (self, nfa):
        nfa_base_id = self.__stateID+1
        #print "Adding subnfa offset by %d:\n%s\n" % (nfa_base_id, nfa)
        self.__stateID += len(nfa)
        for sub_state in nfa.keys():
            ssid = sub_state + nfa_base_id
            ss_map = self.setdefault(ssid, {})
            for key in nfa[sub_state]:
                ss_map.setdefault(key, set())
                ss_map[key] = ss_map[key].union(set([ (nfa_base_id+_i) for _i in nfa[sub_state][key]]))
        return (nfa_base_id+nfa.start(), nfa_base_id+nfa.end())

    def canReach (self, key, start=None, with_epsilon=False):
        if start is None:
            start = [ self.start() ]
        if not isinstance(start, (list, set, frozenset)):
            start = [ start ]
        eps_moves = set(start)
        key_moves = set()

        last_eps_moves = None
        last_key_moves = None

        while ((last_eps_moves != eps_moves) or (last_key_moves != key_moves)):
            last_eps_moves = eps_moves.copy()
            last_key_moves = key_moves.copy()
            for start in eps_moves:
                eps_moves = eps_moves.union(self.oneStep(start, None))
                key_moves = key_moves.union(self.oneStep(start, key))
            for start in last_key_moves:
                key_moves = key_moves.union(self.oneStep(start, None))
        if with_epsilon:
            return eps_moves.union(key_moves)
        return key_moves

    def isFullPath (self, steps):
        reaches = set( [self.start()] )
        #print 'Starting full path from %s\n%s\n' % (reaches, self)
        for s in steps:
            reaches = self.canReach(s, reaches)
            #print 'Add %s reaches %s' % (s, reaches)
        return self.end() in reaches

    def __str__ (self):
        states = set(self.keys())
        elements = set()
        for k in self.keys():
            transitions = self[k]
            elements = elements.union(transitions.keys())
            for step in transitions.keys():
                states = states.union(transitions[step])
        states = list(states)
        states.sort()
        strings = []
        for start in states:
            if start == self.end():
                strings.append('%s terminates' % (start,))
                continue
            transitions = self[start]
            if 0 == len(transitions):
                strings.append('%s dead-ends' % (start,))
                continue
            for step in transitions.keys():
                strings.append('%s via %s to %s' % (start, step, ' '.join([ str(_s) for _s in transitions[step]])))
        return "\n".join(strings)

class Thompson:
    """Create a NFA from a content model."""

    __nfa = None

    def nfa (self): return self.__nfa

    def __init__ (self, term):
        self.__nfa = NFA()
        self.addTransition(term, self.__nfa.start(), self.__nfa.end())

    def addTransition (self, element, start, end):
        if isinstance(element, Particle):
            return self.fromParticle(element, start, end)
        if isinstance(element, ModelGroup):
            return self.fromModelGroup(element, start, end)
        self.__nfa.addTransition(element, start, end)

    def fromParticle (self, particle, start, end):
        #print '# %d to %s of %s' % (particle.minOccurs(), particle.maxOccurs(), particle.term())

        if 0 == particle.minOccurs():
            self.addTransition(None, start, end)

        cur_start = next_end = start
        for step in range(0, particle.minOccurs()):
            cur_start = next_end
            next_end = self.__nfa.newState()
            self.addTransition(particle.term(), cur_start, next_end)

        if None is particle.maxOccurs():
            if next_end == start:
                self.addTransition(particle.term(), start, end)
                next_end = end
            self.addTransition(None, next_end, cur_start)
        else:
            for step in range(particle.minOccurs(), particle.maxOccurs()):
                cur_start = next_end
                next_end = self.__nfa.newState()
                self.addTransition(None, cur_start, end)
                self.addTransition(particle.term(), cur_start, next_end)
        self.addTransition(None, next_end, end)

    def __fromMGSequence (self, particles, start, end):
        for p in particles:
            next_state = self.__nfa.newState()
            self.addTransition(p, start, next_state)
            start = next_state
        self.addTransition(None, start, end)

    def __fromMGChoice (self, particles, start, end):
        for p in particles:
            self.addTransition(p, start, end)

    def __fromMGAll (self, particles, start, end):
        nfa = NFA()
        nfa.addTransition(None, nfa.start(), nfa.end())
        for p in particles:
            next_nfa = NFA()
            (sub_start, sub_end) = next_nfa.addSubNFA(nfa)
            next_nfa.addTransition(p, next_nfa.start(), sub_start)
            next_nfa.addTransition(None, sub_end, next_nfa.end())
            (sub_start, sub_end) = next_nfa.addSubNFA(nfa)
            next_nfa.addTransition(None, next_nfa.start(), sub_start)
            next_nfa.addTransition(p, sub_end, next_nfa.end())
            nfa = next_nfa
        (sub_start, sub_end) = self.__nfa.addSubNFA(nfa)
        self.addTransition(None, start, sub_start)
        self.addTransition(None, sub_end, end)

    def fromModelGroup (self, group, start, end):
        if ModelGroup.C_ALL == group.compositor():
            return self.__fromMGAll(group.particles(), start, end)
        if ModelGroup.C_CHOICE == group.compositor():
            return self.__fromMGChoice(group.particles(), start, end)
        if ModelGroup.C_SEQUENCE == group.compositor():
            return self.__fromMGSequence(group.particles(), start, end)
        assert False

class TestThompson (unittest.TestCase):

    def testParticleOne (self):
        t = Thompson(Particle(1,1,'a'))
        nfa = t.nfa()
        self.assertFalse(nfa.end() in nfa.canReach(None, nfa.start()))
        self.assertTrue(nfa.end() in nfa.canReach('a', nfa.start()))
        self.assertFalse(nfa.end() in nfa.canReach('a', nfa.canReach('a')))

    def testParticleOptional (self):
        t = Thompson(Particle(0,1,'a'))
        nfa = t.nfa()
        self.assertTrue(nfa.end() in nfa.canReach(None, nfa.start()))
        self.assertTrue(nfa.end() in nfa.canReach('a', nfa.start()))
        self.assertFalse(nfa.end() in nfa.canReach('a', nfa.canReach('a')))

    def testParticleAny (self):
        t = Thompson(Particle(0,None,'a'))
        nfa = t.nfa()
        self.assertTrue(nfa.end() in nfa.canReach(None, nfa.start()))
        self.assertTrue(nfa.end() in nfa.canReach('a', nfa.start()))
        reaches = [ nfa.start() ]
        for rep in range(0, 10):
            reaches = nfa.canReach('a', reaches)
            self.assertTrue(nfa.end() in reaches)

    def testParticle2Plus (self):
        particle = Particle(2, None, 'a')
        t = Thompson(particle)
        nfa = t.nfa()
        reaches = [ nfa.start() ]
        for rep in range(1, 10):
            reaches = nfa.canReach('a', reaches)
            if particle.minOccurs() <= rep:
                self.assertTrue(nfa.end() in reaches)
            else:
                self.assertFalse(nfa.end() in reaches)

    def testParticleSome (self):
        particle = Particle(3, 5, 'a')
        t = Thompson(particle)
        nfa = t.nfa()
        reaches = [ nfa.start() ]
        for rep in range(1, 10):
            reaches = nfa.canReach('a', reaches)
            if (particle.minOccurs() <= rep) and (rep <= particle.maxOccurs()):
                self.assertTrue(nfa.end() in reaches)
            else:
                self.assertFalse(nfa.end() in reaches)

    def testSequence1 (self):
        seq = ModelGroup(ModelGroup.C_SEQUENCE, [ 'a' ])
        t = Thompson(seq)
        nfa = t.nfa()
        self.assertFalse(nfa.isFullPath([ ]))
        self.assertTrue(nfa.isFullPath([ 'a' ]))
        self.assertFalse(nfa.isFullPath([ 'a', 'b' ]))

    def testSequence3 (self):
        seq = ModelGroup(ModelGroup.C_SEQUENCE, [ 'a', 'b', 'c' ])
        t = Thompson(seq)
        nfa = t.nfa()
        self.assertFalse(nfa.isFullPath([ ]))
        self.assertFalse(nfa.isFullPath([ 'a' ]))
        self.assertFalse(nfa.isFullPath([ 'a', 'b' ]))
        self.assertTrue(nfa.isFullPath([ 'a', 'b', 'c' ]))
        self.assertFalse(nfa.isFullPath([ 'a', 'b', 'c', 'd' ]))

    def testChoice1 (self):
        seq = ModelGroup(ModelGroup.C_CHOICE, [ 'a' ])
        t = Thompson(seq)
        nfa = t.nfa()
        self.assertFalse(nfa.isFullPath([ ]))
        self.assertTrue(nfa.isFullPath([ 'a' ]))
        self.assertFalse(nfa.isFullPath([ 'a', 'b' ]))

    def testChoice3 (self):
        seq = ModelGroup(ModelGroup.C_CHOICE, [ 'a', 'b', 'c' ])
        t = Thompson(seq)
        nfa = t.nfa()
        self.assertFalse(nfa.isFullPath([ ]))
        self.assertTrue(nfa.isFullPath([ 'a' ]))
        self.assertTrue(nfa.isFullPath([ 'b' ]))
        self.assertTrue(nfa.isFullPath([ 'c' ]))
        self.assertFalse(nfa.isFullPath([ 'a', 'b' ]))

    def testAll1 (self):
        seq = ModelGroup(ModelGroup.C_ALL, [ 'a' ])
        t = Thompson(seq)
        nfa = t.nfa()
        self.assertFalse(nfa.isFullPath([ ]))
        self.assertTrue(nfa.isFullPath([ 'a' ]))
        self.assertFalse(nfa.isFullPath([ 'a', 'a' ]))

    def testAll2 (self):
        seq = ModelGroup(ModelGroup.C_ALL, [ 'a', 'b' ])
        t = Thompson(seq)
        nfa = t.nfa()
        self.assertFalse(nfa.isFullPath([ ]))
        self.assertFalse(nfa.isFullPath([ 'a' ]))
        self.assertFalse(nfa.isFullPath([ 'a', 'a' ]))
        self.assertTrue(nfa.isFullPath([ 'a', 'b' ]))
        self.assertTrue(nfa.isFullPath([ 'b', 'a' ]))
        self.assertFalse(nfa.isFullPath([ 'a', 'b', 'a' ]))
        self.assertFalse(nfa.isFullPath([ 'b', 'a', 'b' ]))

class TestNFA (unittest.TestCase):
    def testSubNFA (self):
        subnfa = NFA()
        subnfa.addTransition('a', subnfa.start(), subnfa.end())
        nfa = NFA()
        ( start, end ) = nfa.addSubNFA(subnfa)
        nfa.addTransition('b', nfa.start(), start)
        nfa.addTransition('c', end, nfa.end())
        self.assertFalse(nfa.isFullPath([ ]))
        self.assertTrue(nfa.isFullPath(['b', 'a', 'c']))

    def testSubNFA (self):
        subnfa = NFA()
        subnfa.addTransition('a', subnfa.start(), subnfa.end())
        nfa = NFA()
        ( start, end ) = nfa.addSubNFA(subnfa)
        nfa.addTransition('b', nfa.start(), start)
        nfa.addTransition(None, end, nfa.end())
        ( start, end ) = nfa.addSubNFA(subnfa)
        nfa.addTransition(None, nfa.start(), start)
        nfa.addTransition('b', end, nfa.end())

        self.assertFalse(nfa.isFullPath([ ]))
        self.assertTrue(nfa.isFullPath(['b', 'a']))
        self.assertTrue(nfa.isFullPath(['a', 'b']))
        self.assertFalse(nfa.isFullPath(['a', 'a']))
        self.assertFalse(nfa.isFullPath(['b', 'a', 'b']))
        self.assertFalse(nfa.isFullPath(['a', 'b', 'a']))

'''
a = Element('a')

t = Thompson(a)
ex = Particle(1, 1, a)
Thompson(Particle(1, 1, a))
Thompson(Particle(0, 1, a))
Thompson(Particle(1, 2, a))
Thompson(Particle(3, 5, a))
Thompson(Particle(2, None, a))
ex = Particle(0, 1, a)
ex = Particle(2, 5, a)
ex = Particle(2, None, a)

ex = Particle(1, 1, ModelGroup(ModelGroup.C_SEQUENCE, [ Particle(1,1,'a'), Particle(1,1,'b')]))
'''
if __name__ == '__main__':
    #Thompson(Particle(0,None,'a'))
    unittest.main()

<commit_msg>Rename classes and methods<commit_after>import unittest
from content import *

class Element (object):
    __name = None

    def __init__ (self, name):
        self.__name = name

    def name (self): return self.__name

    def __str__ (self): return self.name()

# Represent state transitions as a map from states to maps from
# symbols to sets of states.  States are integers.

class FiniteAutomaton (dict):
    __stateID = -1
    __start = None
    __end = None

    def __init__ (self):
        self.__end = self.newState()
        self.__start = self.newState()

    def newState (self):
        self.__stateID += 1
        self.setdefault(self.__stateID, {})
        return self.__stateID

    def start (self): return self.__start

    def end (self): return self.__end

    def addTransition (self, key, start, end):
        assert end is not None
        self.setdefault(start, {}).setdefault(key, set()).add(end)
        return self

    def ok (self, key, start, end):
        return end in self[start].get(key, set())

    def oneStep (self, node, key):
        return self.setdefault(node, {}).get(key, set())

    def removeState (self, state):
        del self[state]
        for transitions in self.values():
            for target in transitions.values():
                target.discard(state)

    def addSubAutomaton (self, nfa):
        nfa_base_id = self.__stateID+1
        #print "Adding subnfa offset by %d:\n%s\n" % (nfa_base_id, nfa)
        self.__stateID += len(nfa)
        for sub_state in nfa.keys():
            ssid = sub_state + nfa_base_id
            ss_map = self.setdefault(ssid, {})
            for key in nfa[sub_state]:
                ss_map.setdefault(key, set())
                ss_map[key] = ss_map[key].union(set([ (nfa_base_id+_i) for _i in nfa[sub_state][key]]))
        return (nfa_base_id+nfa.start(), nfa_base_id+nfa.end())

    def canReach (self, key, start=None, with_epsilon=False):
        if start is None:
            start = [ self.start() ]
        if not isinstance(start, (list, set, frozenset)):
            start = [ start ]
        eps_moves = set(start)
        key_moves = set()

        last_eps_moves = None
        last_key_moves = None

        while ((last_eps_moves != eps_moves) or (last_key_moves != key_moves)):
            last_eps_moves = eps_moves.copy()
            last_key_moves = key_moves.copy()
            for start in eps_moves:
                eps_moves = eps_moves.union(self.oneStep(start, None))
                key_moves = key_moves.union(self.oneStep(start, key))
            for start in last_key_moves:
                key_moves = key_moves.union(self.oneStep(start, None))
        if with_epsilon:
            return eps_moves.union(key_moves)
        return key_moves

    def isFullPath (self, steps):
        reaches = set( [self.start()] )
        #print 'Starting full path from %s\n%s\n' % (reaches, self)
        for s in steps:
            reaches = self.canReach(s, reaches)
            #print 'Add %s reaches %s' % (s, reaches)
        return self.end() in reaches

    def __str__ (self):
        states = set(self.keys())
        elements = set()
        for k in self.keys():
            transitions = self[k]
            elements = elements.union(transitions.keys())
            for step in transitions.keys():
                states = states.union(transitions[step])
        states = list(states)
        states.sort()
        strings = []
        for start in states:
            if start == self.end():
                strings.append('%s terminates' % (start,))
                continue
            transitions = self[start]
            if 0 == len(transitions):
                strings.append('%s dead-ends' % (start,))
                continue
            for step in transitions.keys():
                strings.append('%s via %s to %s' % (start, step, ' '.join([ str(_s) for _s in transitions[step]])))
        return "\n".join(strings)

class Thompson:
    """Create a NFA from a content model."""

    __nfa = None

    def nfa (self): return self.__nfa

    def __init__ (self, term):
        self.__nfa = FiniteAutomaton()
        self.addTransition(term, self.__nfa.start(), self.__nfa.end())

    def addTransition (self, element, start, end):
        if isinstance(element, Particle):
            return self.fromParticle(element, start, end)
        if isinstance(element, ModelGroup):
            return self.fromModelGroup(element, start, end)
        self.__nfa.addTransition(element, start, end)

    def fromParticle (self, particle, start, end):
        #print '# %d to %s of %s' % (particle.minOccurs(), particle.maxOccurs(), particle.term())

        if 0 == particle.minOccurs():
            self.addTransition(None, start, end)

        cur_start = next_end = start
        for step in range(0, particle.minOccurs()):
            cur_start = next_end
            next_end = self.__nfa.newState()
            self.addTransition(particle.term(), cur_start, next_end)

        if None is particle.maxOccurs():
            if next_end == start:
                self.addTransition(particle.term(), start, end)
                next_end = end
            self.addTransition(None, next_end, cur_start)
        else:
            for step in range(particle.minOccurs(), particle.maxOccurs()):
                cur_start = next_end
                next_end = self.__nfa.newState()
                self.addTransition(None, cur_start, end)
                self.addTransition(particle.term(), cur_start, next_end)
        self.addTransition(None, next_end, end)

    def __fromMGSequence (self, particles, start, end):
        for p in particles:
            next_state = self.__nfa.newState()
            self.addTransition(p, start, next_state)
            start = next_state
        self.addTransition(None, start, end)

    def __fromMGChoice (self, particles, start, end):
        for p in particles:
            self.addTransition(p, start, end)

    def __fromMGAll (self, particles, start, end):
        nfa = FiniteAutomaton()
        nfa.addTransition(None, nfa.start(), nfa.end())
        for p in particles:
            next_nfa = FiniteAutomaton()
            (sub_start, sub_end) = next_nfa.addSubAutomaton(nfa)
            next_nfa.addTransition(p, next_nfa.start(), sub_start)
            next_nfa.addTransition(None, sub_end, next_nfa.end())
            (sub_start, sub_end) = next_nfa.addSubAutomaton(nfa)
            next_nfa.addTransition(None, next_nfa.start(), sub_start)
            next_nfa.addTransition(p, sub_end, next_nfa.end())
            nfa = next_nfa
        (sub_start, sub_end) = self.__nfa.addSubAutomaton(nfa)
        self.addTransition(None, start, sub_start)
        self.addTransition(None, sub_end, end)

    def fromModelGroup (self, group, start, end):
        if ModelGroup.C_ALL == group.compositor():
            return self.__fromMGAll(group.particles(), start, end)
        if ModelGroup.C_CHOICE == group.compositor():
            return self.__fromMGChoice(group.particles(), start, end)
        if ModelGroup.C_SEQUENCE == group.compositor():
            return self.__fromMGSequence(group.particles(), start, end)
        assert False

class TestThompson (unittest.TestCase):

    def testParticleOne (self):
        t = Thompson(Particle(1,1,'a'))
        nfa = t.nfa()
        self.assertFalse(nfa.end() in nfa.canReach(None, nfa.start()))
        self.assertTrue(nfa.end() in nfa.canReach('a', nfa.start()))
        self.assertFalse(nfa.end() in nfa.canReach('a', nfa.canReach('a')))

    def testParticleOptional (self):
        t = Thompson(Particle(0,1,'a'))
        nfa = t.nfa()
        self.assertTrue(nfa.end() in nfa.canReach(None, nfa.start()))
        self.assertTrue(nfa.end() in nfa.canReach('a', nfa.start()))
        self.assertFalse(nfa.end() in nfa.canReach('a', nfa.canReach('a')))

    def testParticleAny (self):
        t = Thompson(Particle(0,None,'a'))
        nfa = t.nfa()
        self.assertTrue(nfa.end() in nfa.canReach(None, nfa.start()))
        self.assertTrue(nfa.end() in nfa.canReach('a', nfa.start()))
        reaches = [ nfa.start() ]
        for rep in range(0, 10):
            reaches = nfa.canReach('a', reaches)
            self.assertTrue(nfa.end() in reaches)

    def testParticle2Plus (self):
        particle = Particle(2, None, 'a')
        t = Thompson(particle)
        nfa = t.nfa()
        reaches = [ nfa.start() ]
        for rep in range(1, 10):
            reaches = nfa.canReach('a', reaches)
            if particle.minOccurs() <= rep:
                self.assertTrue(nfa.end() in reaches)
            else:
                self.assertFalse(nfa.end() in reaches)

    def testParticleSome (self):
        particle = Particle(3, 5, 'a')
        t = Thompson(particle)
        nfa = t.nfa()
        reaches = [ nfa.start() ]
        for rep in range(1, 10):
            reaches = nfa.canReach('a', reaches)
            if (particle.minOccurs() <= rep) and (rep <= particle.maxOccurs()):
                self.assertTrue(nfa.end() in reaches)
            else:
                self.assertFalse(nfa.end() in reaches)

    def testSequence1 (self):
        seq = ModelGroup(ModelGroup.C_SEQUENCE, [ 'a' ])
        t = Thompson(seq)
        nfa = t.nfa()
        self.assertFalse(nfa.isFullPath([ ]))
        self.assertTrue(nfa.isFullPath([ 'a' ]))
        self.assertFalse(nfa.isFullPath([ 'a', 'b' ]))

    def testSequence3 (self):
        seq = ModelGroup(ModelGroup.C_SEQUENCE, [ 'a', 'b', 'c' ])
        t = Thompson(seq)
        nfa = t.nfa()
        self.assertFalse(nfa.isFullPath([ ]))
        self.assertFalse(nfa.isFullPath([ 'a' ]))
        self.assertFalse(nfa.isFullPath([ 'a', 'b' ]))
        self.assertTrue(nfa.isFullPath([ 'a', 'b', 'c' ]))
        self.assertFalse(nfa.isFullPath([ 'a', 'b', 'c', 'd' ]))

    def testChoice1 (self):
        seq = ModelGroup(ModelGroup.C_CHOICE, [ 'a' ])
        t = Thompson(seq)
        nfa = t.nfa()
        self.assertFalse(nfa.isFullPath([ ]))
        self.assertTrue(nfa.isFullPath([ 'a' ]))
        self.assertFalse(nfa.isFullPath([ 'a', 'b' ]))

    def testChoice3 (self):
        seq = ModelGroup(ModelGroup.C_CHOICE, [ 'a', 'b', 'c' ])
        t = Thompson(seq)
        nfa = t.nfa()
        self.assertFalse(nfa.isFullPath([ ]))
        self.assertTrue(nfa.isFullPath([ 'a' ]))
        self.assertTrue(nfa.isFullPath([ 'b' ]))
        self.assertTrue(nfa.isFullPath([ 'c' ]))
        self.assertFalse(nfa.isFullPath([ 'a', 'b' ]))

    def testAll1 (self):
        seq = ModelGroup(ModelGroup.C_ALL, [ 'a' ])
        t = Thompson(seq)
        nfa = t.nfa()
        self.assertFalse(nfa.isFullPath([ ]))
        self.assertTrue(nfa.isFullPath([ 'a' ]))
        self.assertFalse(nfa.isFullPath([ 'a', 'a' ]))

    def testAll2 (self):
        seq = ModelGroup(ModelGroup.C_ALL, [ 'a', 'b' ])
        t = Thompson(seq)
        nfa = t.nfa()
        self.assertFalse(nfa.isFullPath([ ]))
        self.assertFalse(nfa.isFullPath([ 'a' ]))
        self.assertFalse(nfa.isFullPath([ 'a', 'a' ]))
        self.assertTrue(nfa.isFullPath([ 'a', 'b' ]))
        self.assertTrue(nfa.isFullPath([ 'b', 'a' ]))
        self.assertFalse(nfa.isFullPath([ 'a', 'b', 'a' ]))
        self.assertFalse(nfa.isFullPath([ 'b', 'a', 'b' ]))

class TestFiniteAutomaton (unittest.TestCase):
    def testSubAutomaton (self):
        subnfa = FiniteAutomaton()
        subnfa.addTransition('a', subnfa.start(), subnfa.end())
        nfa = FiniteAutomaton()
        ( start, end ) = nfa.addSubAutomaton(subnfa)
        nfa.addTransition('b', nfa.start(), start)
        nfa.addTransition('c', end, nfa.end())
        self.assertFalse(nfa.isFullPath([ ]))
        self.assertTrue(nfa.isFullPath(['b', 'a', 'c']))

    def testSubAutomaton (self):
        subnfa = FiniteAutomaton()
        subnfa.addTransition('a', subnfa.start(), subnfa.end())
        nfa = FiniteAutomaton()
        ( start, end ) = nfa.addSubAutomaton(subnfa)
        nfa.addTransition('b', nfa.start(), start)
        nfa.addTransition(None, end, nfa.end())
        ( start, end ) = nfa.addSubAutomaton(subnfa)
        nfa.addTransition(None, nfa.start(), start)
        nfa.addTransition('b', end, nfa.end())

        self.assertFalse(nfa.isFullPath([ ]))
        self.assertTrue(nfa.isFullPath(['b', 'a']))
        self.assertTrue(nfa.isFullPath(['a', 'b']))
        self.assertFalse(nfa.isFullPath(['a', 'a']))
        self.assertFalse(nfa.isFullPath(['b', 'a', 'b']))
        self.assertFalse(nfa.isFullPath(['a', 'b', 'a']))

'''
a = Element('a')

t = Thompson(a)
ex = Particle(1, 1, a)
Thompson(Particle(1, 1, a))
Thompson(Particle(0, 1, a))
Thompson(Particle(1, 2, a))
Thompson(Particle(3, 5, a))
Thompson(Particle(2, None, a))
ex = Particle(0, 1, a)
ex = Particle(2, 5, a)
ex = Particle(2, None, a)

ex = Particle(1, 1, ModelGroup(ModelGroup.C_SEQUENCE, [ Particle(1,1,'a'), Particle(1,1,'b')]))
'''
if __name__ == '__main__':
    #Thompson(Particle(0,None,'a'))
    unittest.main()

<|endoftext|>


<commit_before>r"""
Authors: Chase Coleman, Spencer Lyon, Daisuke Oyama, Tom Sargent,
         John Stachurski

Filename: mc_tools.py

This file contains some useful objects for handling a finite-state
discrete-time Markov chain.

Definitions and Some Basic Facts about Markov Chains
----------------------------------------------------

Let :math:`\{X_t\}` be a Markov chain represented by an :math:`n \times
n` stochastic matrix :math:`P`. State :math:`i` *has access* to state
:math:`j`, denoted :math:`i \to j`, if :math:`i = j` or :math:`P^k[i, j]
> 0` for some :math:`k = 1, 2, \ldots`; :math:`i` and `j` *communicate*,
denoted :math:`i \leftrightarrow j`, if :math:`i \to j` and :math:`j \to
i`. The binary relation :math:`\leftrightarrow` is an equivalent
relation. A *communication class* of the Markov chain :math:`\{X_t\}`,
or of the stochastic matrix :math:`P`, is an equivalent class of
:math:`\leftrightarrow`. Equivalently, a communication class is a
*strongly connected component* (SCC) in the associated *directed graph*
:math:`\Gamma(P)`, a directed graph with :math:`n` nodes where there is
an edge from :math:`i` to :math:`j` if and only if :math:`P[i, j] > 0`.
The Markov chain, or the stochastic matrix, is *irreducible* if it
admits only one communication class, or equivalently, if
:math:`\Gamma(P)` is *strongly connected*.

A state :math:`i` is *recurrent* if :math:`i \to j` implies :math:`j \to
i`; it is *transient* if it is not recurrent. For any :math:`i, j`
contained in a communication class, :math:`i` is recurrent if and only
if :math:`j` is recurrent. Therefore, recurrence is a property of a
communication class. Thus, a communication class is a *recurrent class*
if it contains a recurrent state. Equivalently, a recurrent class is a
SCC that corresponds to a sink node in the *condensation* of the
directed graph :math:`\Gamma(P)`, where the condensation of
:math:`\Gamma(P)` is a directed graph in which each SCC is replaced with
a single node and there is an edge from one SCC :math:`C` to another SCC
:math:`C'` if :math:`C \neq C'` and some node in :math:`C` has access to
some node in :math:`C'`. A recurrent class is also called a *closed
communication class*. The condensation is acyclic, so that there exists
at least one recurrent class.

For example, if the entries of :math:`P` are all strictly positive, then
the whole state space is a communication class as well as a recurrent
class. (More generally, if there is only one communication class, then
it is a recurrent class.) As another example, consider the stochastic
matrix :math:`P = [[1, 0], [0,5, 0.5]]`. This has two communication
classes, :math:`\{0\}` and :math:`\{1\}`, and :math:`\{0\}` is the only
recurrent class.

A *stationary distribution* of the Markov chain :math:`\{X_t\}`, or of
the stochastic matrix :math:`P`, is a nonnegative vector :math:`x` such
that :math:`x' P = x'` and :math:`x' \mathbf{1} = 1`, where
:math:`\mathbf{1}` is the vector of ones. The Markov chain has a unique
stationary distribution if and only if it has a unique recurrent class.
More generally, each recurrent class has a unique stationary
distribution whose support equals that recurrent class. The set of all
stationary distributions is given by the convex hull of these unique
stationary distributions for the recurrent classes.

A natural number :math:`d` is the *period* of state :math:`i` if it is
the greatest common divisor of all :math:`k`'s such that :math:`P^k[i,
i] > 0`; equivalently, it is the GCD of the lengths of the cycles in
:math:`\Gamma(P)` passing through :math:`i`. For any :math:`i, j`
contained in a communication class, :math:`i` has period :math:`d` if
and only if :math:`j` has period :math:`d`. The *period* of an
irreducible Markov chain (or of an irreducible stochastic matrix) is the
period of any state. We define the period of a general (not necessarily
irreducible) Markov chain to be the least common multiple of the periods
of its recurrent classes, where the period of a recurrent class is the
period of any state in that class. A Markov chain is *aperiodic* if its
period is one. A Markov chain is irreducible and aperiodic if and only
if it is *uniformly ergodic*, i.e., there exists some :math:`m` such
that :math:`P^m[i, j] > 0` for all :math:`i, j` (in this case, :math:`P`
is also called *primitive*).

Suppose that an irreducible Markov chain has period :math:`d`. Fix any
state, say state :math:`0`. For each :math:`m = 0, \ldots, d-1`, let
:math:`S_m` be the set of states :math:`i` such that :math:`P^{kd+m}[0,
i] > 0` for some :math:`k`. These sets :math:`S_0, \ldots, S_{d-1}`
constitute a partition of the state space and are called the *cyclic
classes*. For each :math:`S_m` and each :math:`i \in S_m`, we have
:math:`\sum_{j \in S_{m+1}} P[i, j] = 1`, where :math:`S_d = S_0`.

"""
from __future__ import division
import numpy as np
from fractions import gcd
import sys
from .graph_tools import DiGraph
from .gth_solve import gth_solve
from warnings import warn

#-Check if Numba is Available-#
from .external import numba_installed, jit
from .utilities import searchsorted

class MarkovChain(object):
    """
    Class for a finite-state discrete-time Markov chain. It stores
    useful information such as the stationary distributions, and
    communication, recurrent, and cyclic classes, and allows simulation
    of state transitions.

    Parameters
    ----------
    P : array_like(float, ndim=2)
        The transition matrix.  Must be of shape n x n.

    Attributes
    ----------
    P : see Parameters

    stationary_distributions : array_like(float, ndim=2)
        Array containing stationary distributions, one for each
        recurrent class, as rows.

    is_irreducible : bool
        Indicate whether the Markov chain is irreducible.

    num_communication_classes : int
        The number of the communication classes.

    communication_classes : list(ndarray(int))
        List of numpy arrays containing the communication classes.

    num_recurrent_classes : int
        The number of the recurrent classes.

    recurrent_classes : list(ndarray(int))
        List of numpy arrays containing the recurrent classes.

    is_aperiodic : bool
        Indicate whether the Markov chain is aperiodic.

    period : int
        The period of the Markov chain.

    cyclic_classes : list(ndarray(int))
        List of numpy arrays containing the cyclic classes. Defined only
        when the Markov chain is irreducible.

    Methods
    -------
    simulate : Simulates the markov chain for a given initial state or
        distribution.

    """

    def __init__(self, P):
        self.P = np.asarray(P)

        # Check Properties
        # Double check that P is a square matrix
        if len(self.P.shape) != 2 or self.P.shape[0] != self.P.shape[1]:
            raise ValueError('P must be a square matrix')

        # Double check that P is a nonnegative matrix
        if not np.all(self.P >= 0):
            raise ValueError('P must be nonnegative')

        # Double check that the rows of P sum to one
        if not np.allclose(np.sum(self.P, axis=1), np.ones(self.P.shape[0])):
            raise ValueError('The rows of P must sum to 1')

        # The number of states
        self.n = self.P.shape[0]

        # To analyze the structure of P as a directed graph
        self.digraph = DiGraph(self.P)

        self._stationary_dists = None

    def __repr__(self):
        msg = "Markov chain with transition matrix \nP = \n{0}"

        if self._stationary_dists is None:
            return msg.format(self.P)
        else:
            msg = msg + "\nand stationary distributions \n{1}"
            return msg.format(self.P, self._stationary_dists)

    def __str__(self):
        return str(self.__repr__)

    @property
    def is_irreducible(self):
        return self.digraph.is_strongly_connected

    @property
    def num_communication_classes(self):
        return self.digraph.num_strongly_connected_components

    @property
    def communication_classes(self):
        return self.digraph.strongly_connected_components

    @property
    def num_recurrent_classes(self):
        return self.digraph.num_sink_strongly_connected_components

    @property
    def recurrent_classes(self):
        return self.digraph.sink_strongly_connected_components

    @property
    def is_aperiodic(self):
        if self.is_irreducible:
            return self.digraph.is_aperiodic
        else:
            return self.period == 1

    @property
    def period(self):
        if self.is_irreducible:
            return self.digraph.period
        else:
            rec_classes = self.recurrent_classes

            # Determine the period, the LCM of the periods of rec_classes
            d = 1
            for rec_class in rec_classes:
                period = DiGraph(self.P[rec_class, :][:, rec_class]).period
                d = (d * period) // gcd(d, period)

            return d

    @property
    def cyclic_classes(self):
        if not self.is_irreducible:
            raise NotImplementedError(
                'Not defined for a reducible Markov chain'
            )
        else:
            return self.digraph.cyclic_components

    def _compute_stationary(self):
        """
        Store the stationary distributions in self._stationary_distributions.

        """
        if self.is_irreducible:
            stationary_dists = gth_solve(self.P).reshape(1, self.n)
        else:
            rec_classes = self.recurrent_classes
            stationary_dists = np.zeros((len(rec_classes), self.n))
            for i, rec_class in enumerate(rec_classes):
                stationary_dists[i, rec_class] = \
                    gth_solve(self.P[rec_class, :][:, rec_class])

        self._stationary_dists = stationary_dists

    @property
    def stationary_distributions(self):
        if self._stationary_dists is None:
            self._compute_stationary()
        return self._stationary_dists

    def simulate(self, init=0, sample_size=1000):
        X = mc_sample_path(self.P, init, sample_size)
        return X

def mc_compute_stationary(P):
    """
    Computes stationary distributions of P, one for each recurrent
    class. Any stationary distribution is written as a convex
    combination of these distributions.

    Returns
    -------
    stationary_dists : array_like(float, ndim=2)
        Array containing the stationary distributions as its rows.

    """
    return MarkovChain(P).stationary_distributions

def mc_sample_path(P, init=0, sample_size=1000):
    """
    See Section: DocStrings below
    """
    n = len(P)

    # CDFs, one for each row of P
    cdfs = np.empty((n, n), order='C')  # see issue #137#issuecomment-96128186
    np.cumsum(P, axis=-1, out=cdfs)

    # Random values, uniformly sampled from [0, 1)
    u = np.random.random(size=sample_size)

    # === set up array to store output === #
    X = np.empty(sample_size, dtype=int)
    if isinstance(init, int):
        X[0] = init
    else:
        cdf0 = np.cumsum(init)
        X[0] = searchsorted(cdf0, u[0])

    # === generate the sample path === #
    for t in range(sample_size-1):
        X[t+1] = searchsorted(cdfs[X[t]], u[t+1])

    return X

if numba_installed:
    mc_sample_path = jit(mc_sample_path)

#------------#
#-DocStrings-#
#------------#

#-mc_sample_path() function and MarkovChain.simulate() method-#
_sample_path_docstr = \
"""
Generates one sample path from the Markov chain represented by (n x n)
transition matrix P on state space S = {{0,...,n-1}}.

Parameters
----------
{p_arg}
init : array_like(float ndim=1) or scalar(int), optional(default=0)
    If init is an array_like, then it is treated as the initial
    distribution across states.  If init is a scalar, then it treated as
    the deterministic initial state.

sample_size : scalar(int), optional(default=1000)
    The length of the sample path.

Returns
-------
X : array_like(int, ndim=1)
    The simulation of states.

"""

#-Functions-#

#-mc_sample_path-#
mc_sample_path.__doc__ = _sample_path_docstr.format(p_arg="""
P : array_like(float, ndim=2)
    A Markov transition matrix.
""")
mc_sample_path_numpy.__doc__ = _sample_path_docstr.format(p_arg="""
P : array_like(float, ndim=2)
    A Markov transition matrix.
""")

#-Methods-#

#-Markovchain.simulate()-#
if sys.version_info[0] == 3:
    MarkovChain.simulate.__doc__ = _sample_path_docstr.format(p_arg="")
elif sys.version_info[0] == 2:
    MarkovChain.simulate.__func__.__doc__ = _sample_path_docstr.format(p_arg="")
<commit_msg>Minor edits<commit_after>r"""
Authors: Chase Coleman, Spencer Lyon, Daisuke Oyama, Tom Sargent,
         John Stachurski

Filename: mc_tools.py

This file contains some useful objects for handling a finite-state
discrete-time Markov chain.

Definitions and Some Basic Facts about Markov Chains
----------------------------------------------------

Let :math:`\{X_t\}` be a Markov chain represented by an :math:`n \times
n` stochastic matrix :math:`P`. State :math:`i` *has access* to state
:math:`j`, denoted :math:`i \to j`, if :math:`i = j` or :math:`P^k[i, j]
> 0` for some :math:`k = 1, 2, \ldots`; :math:`i` and `j` *communicate*,
denoted :math:`i \leftrightarrow j`, if :math:`i \to j` and :math:`j \to
i`. The binary relation :math:`\leftrightarrow` is an equivalent
relation. A *communication class* of the Markov chain :math:`\{X_t\}`,
or of the stochastic matrix :math:`P`, is an equivalent class of
:math:`\leftrightarrow`. Equivalently, a communication class is a
*strongly connected component* (SCC) in the associated *directed graph*
:math:`\Gamma(P)`, a directed graph with :math:`n` nodes where there is
an edge from :math:`i` to :math:`j` if and only if :math:`P[i, j] > 0`.
The Markov chain, or the stochastic matrix, is *irreducible* if it
admits only one communication class, or equivalently, if
:math:`\Gamma(P)` is *strongly connected*.

A state :math:`i` is *recurrent* if :math:`i \to j` implies :math:`j \to
i`; it is *transient* if it is not recurrent. For any :math:`i, j`
contained in a communication class, :math:`i` is recurrent if and only
if :math:`j` is recurrent. Therefore, recurrence is a property of a
communication class. Thus, a communication class is a *recurrent class*
if it contains a recurrent state. Equivalently, a recurrent class is a
SCC that corresponds to a sink node in the *condensation* of the
directed graph :math:`\Gamma(P)`, where the condensation of
:math:`\Gamma(P)` is a directed graph in which each SCC is replaced with
a single node and there is an edge from one SCC :math:`C` to another SCC
:math:`C'` if :math:`C \neq C'` and some node in :math:`C` has access to
some node in :math:`C'`. A recurrent class is also called a *closed
communication class*. The condensation is acyclic, so that there exists
at least one recurrent class.

For example, if the entries of :math:`P` are all strictly positive, then
the whole state space is a communication class as well as a recurrent
class. (More generally, if there is only one communication class, then
it is a recurrent class.) As another example, consider the stochastic
matrix :math:`P = [[1, 0], [0,5, 0.5]]`. This has two communication
classes, :math:`\{0\}` and :math:`\{1\}`, and :math:`\{0\}` is the only
recurrent class.

A *stationary distribution* of the Markov chain :math:`\{X_t\}`, or of
the stochastic matrix :math:`P`, is a nonnegative vector :math:`x` such
that :math:`x' P = x'` and :math:`x' \mathbf{1} = 1`, where
:math:`\mathbf{1}` is the vector of ones. The Markov chain has a unique
stationary distribution if and only if it has a unique recurrent class.
More generally, each recurrent class has a unique stationary
distribution whose support equals that recurrent class. The set of all
stationary distributions is given by the convex hull of these unique
stationary distributions for the recurrent classes.

A natural number :math:`d` is the *period* of state :math:`i` if it is
the greatest common divisor of all :math:`k`'s such that :math:`P^k[i,
i] > 0`; equivalently, it is the GCD of the lengths of the cycles in
:math:`\Gamma(P)` passing through :math:`i`. For any :math:`i, j`
contained in a communication class, :math:`i` has period :math:`d` if
and only if :math:`j` has period :math:`d`. The *period* of an
irreducible Markov chain (or of an irreducible stochastic matrix) is the
period of any state. We define the period of a general (not necessarily
irreducible) Markov chain to be the least common multiple of the periods
of its recurrent classes, where the period of a recurrent class is the
period of any state in that class. A Markov chain is *aperiodic* if its
period is one. A Markov chain is irreducible and aperiodic if and only
if it is *uniformly ergodic*, i.e., there exists some :math:`m` such
that :math:`P^m[i, j] > 0` for all :math:`i, j` (in this case, :math:`P`
is also called *primitive*).

Suppose that an irreducible Markov chain has period :math:`d`. Fix any
state, say state :math:`0`. For each :math:`m = 0, \ldots, d-1`, let
:math:`S_m` be the set of states :math:`i` such that :math:`P^{kd+m}[0,
i] > 0` for some :math:`k`. These sets :math:`S_0, \ldots, S_{d-1}`
constitute a partition of the state space and are called the *cyclic
classes*. For each :math:`S_m` and each :math:`i \in S_m`, we have
:math:`\sum_{j \in S_{m+1}} P[i, j] = 1`, where :math:`S_d = S_0`.

"""
from __future__ import division
import numpy as np
from fractions import gcd
import sys
from .graph_tools import DiGraph
from .gth_solve import gth_solve

# -Check if Numba is Available- #
from .external import numba_installed, jit
from .utilities import searchsorted

class MarkovChain(object):
    """
    Class for a finite-state discrete-time Markov chain. It stores
    useful information such as the stationary distributions, and
    communication, recurrent, and cyclic classes, and allows simulation
    of state transitions.

    Parameters
    ----------
    P : array_like(float, ndim=2)
        The transition matrix.  Must be of shape n x n.

    Attributes
    ----------
    P : see Parameters

    stationary_distributions : array_like(float, ndim=2)
        Array containing stationary distributions, one for each
        recurrent class, as rows.

    is_irreducible : bool
        Indicate whether the Markov chain is irreducible.

    num_communication_classes : int
        The number of the communication classes.

    communication_classes : list(ndarray(int))
        List of numpy arrays containing the communication classes.

    num_recurrent_classes : int
        The number of the recurrent classes.

    recurrent_classes : list(ndarray(int))
        List of numpy arrays containing the recurrent classes.

    is_aperiodic : bool
        Indicate whether the Markov chain is aperiodic.

    period : int
        The period of the Markov chain.

    cyclic_classes : list(ndarray(int))
        List of numpy arrays containing the cyclic classes. Defined only
        when the Markov chain is irreducible.

    Methods
    -------
    simulate : Simulates the markov chain for a given initial state or
        distribution.

    """

    def __init__(self, P):
        self.P = np.asarray(P)

        # Check Properties
        # Double check that P is a square matrix
        if len(self.P.shape) != 2 or self.P.shape[0] != self.P.shape[1]:
            raise ValueError('P must be a square matrix')

        # Double check that P is a nonnegative matrix
        if not np.all(self.P >= 0):
            raise ValueError('P must be nonnegative')

        # Double check that the rows of P sum to one
        if not np.allclose(np.sum(self.P, axis=1), np.ones(self.P.shape[0])):
            raise ValueError('The rows of P must sum to 1')

        # The number of states
        self.n = self.P.shape[0]

        # To analyze the structure of P as a directed graph
        self.digraph = DiGraph(self.P)

        self._stationary_dists = None

    def __repr__(self):
        msg = "Markov chain with transition matrix \nP = \n{0}"

        if self._stationary_dists is None:
            return msg.format(self.P)
        else:
            msg = msg + "\nand stationary distributions \n{1}"
            return msg.format(self.P, self._stationary_dists)

    def __str__(self):
        return str(self.__repr__)

    @property
    def is_irreducible(self):
        return self.digraph.is_strongly_connected

    @property
    def num_communication_classes(self):
        return self.digraph.num_strongly_connected_components

    @property
    def communication_classes(self):
        return self.digraph.strongly_connected_components

    @property
    def num_recurrent_classes(self):
        return self.digraph.num_sink_strongly_connected_components

    @property
    def recurrent_classes(self):
        return self.digraph.sink_strongly_connected_components

    @property
    def is_aperiodic(self):
        if self.is_irreducible:
            return self.digraph.is_aperiodic
        else:
            return self.period == 1

    @property
    def period(self):
        if self.is_irreducible:
            return self.digraph.period
        else:
            rec_classes = self.recurrent_classes

            # Determine the period, the LCM of the periods of rec_classes
            d = 1
            for rec_class in rec_classes:
                period = DiGraph(self.P[rec_class, :][:, rec_class]).period
                d = (d * period) // gcd(d, period)

            return d

    @property
    def cyclic_classes(self):
        if not self.is_irreducible:
            raise NotImplementedError(
                'Not defined for a reducible Markov chain'
            )
        else:
            return self.digraph.cyclic_components

    def _compute_stationary(self):
        """
        Store the stationary distributions in self._stationary_distributions.

        """
        if self.is_irreducible:
            stationary_dists = gth_solve(self.P).reshape(1, self.n)
        else:
            rec_classes = self.recurrent_classes
            stationary_dists = np.zeros((len(rec_classes), self.n))
            for i, rec_class in enumerate(rec_classes):
                stationary_dists[i, rec_class] = \
                    gth_solve(self.P[rec_class, :][:, rec_class])

        self._stationary_dists = stationary_dists

    @property
    def stationary_distributions(self):
        if self._stationary_dists is None:
            self._compute_stationary()
        return self._stationary_dists

    def simulate(self, init=0, sample_size=1000):
        X = mc_sample_path(self.P, init, sample_size)
        return X

def mc_compute_stationary(P):
    """
    Computes stationary distributions of P, one for each recurrent
    class. Any stationary distribution is written as a convex
    combination of these distributions.

    Returns
    -------
    stationary_dists : array_like(float, ndim=2)
        Array containing the stationary distributions as its rows.

    """
    return MarkovChain(P).stationary_distributions

def mc_sample_path(P, init=0, sample_size=1000):
    """
    See Section: DocStrings below
    """
    n = len(P)

    # CDFs, one for each row of P
    cdfs = np.empty((n, n), order='C')  # see issue #137#issuecomment-96128186
    np.cumsum(P, axis=-1, out=cdfs)

    # Random values, uniformly sampled from [0, 1)
    u = np.random.random(size=sample_size)

    # === set up array to store output === #
    X = np.empty(sample_size, dtype=int)
    if isinstance(init, int):
        X[0] = init
    else:
        cdf0 = np.cumsum(init)
        X[0] = searchsorted(cdf0, u[0])

    # === generate the sample path === #
    for t in range(sample_size-1):
        X[t+1] = searchsorted(cdfs[X[t]], u[t+1])

    return X

if numba_installed:
    mc_sample_path = jit(mc_sample_path)

# ------------ #
# -DocStrings- #
# ------------ #

# -mc_sample_path() function and MarkovChain.simulate() method- #
_sample_path_docstr = \
"""
Generates one sample path from the Markov chain represented by (n x n)
transition matrix P on state space S = {{0,...,n-1}}.

Parameters
----------
{p_arg}
init : array_like(float ndim=1) or scalar(int), optional(default=0)
    If init is an array_like, then it is treated as the initial
    distribution across states.  If init is a scalar, then it treated as
    the deterministic initial state.

sample_size : scalar(int), optional(default=1000)
    The length of the sample path.

Returns
-------
X : array_like(int, ndim=1)
    The simulation of states.

"""

# -Functions- #

# -mc_sample_path- #
mc_sample_path.__doc__ = _sample_path_docstr.format(p_arg="""
P : array_like(float, ndim=2)
    A Markov transition matrix.
""")
mc_sample_path_numpy.__doc__ = _sample_path_docstr.format(p_arg="""
P : array_like(float, ndim=2)
    A Markov transition matrix.
""")

# -Methods- #

# -Markovchain.simulate()- #
if sys.version_info[0] == 3:
    MarkovChain.simulate.__doc__ = _sample_path_docstr.format(p_arg="")
elif sys.version_info[0] == 2:
    MarkovChain.simulate.__func__.__doc__ = \
        _sample_path_docstr.format(p_arg="")
<|endoftext|>