<commit_before>#!/usr/bin/env python
"""Cryptogram.

Description:
  This script statistically analyses a line of text to help solve a cryptogram

Author:
  Andrew Mattheisen

Usage:
  cryptogram.py analyze <cyphertext>...
  cryptogram.py analyze -i <cyphertext>...
  cryptogram.py sub -f <file> <cyphertext>...
  cryptogram.py sub -i -f <file> <cyphertext>...
  cryptogram.py <cyphertext>...
  cryptogram.py -i <cyphertext>...
  cryptogram.py (-h | --help)
  cryptogram.py --version

Options:
  -h --help     Show this screen.
  --version     Show version.
  -i            Ignore spaces. Use this when spaces are not provided between words.

"""
from __future__ import print_function
"""
Additional features to implement:
    IMPLEMENT WORD BOUNDARY ANALYSIS
        Look for words / Autosolve
          Create a map of word character patterns: e.g. workout = ABCDAFG
          Use words file. Algorithm: apply letter maps, count #words in solution.
          Display the solutions containing the most words
        Look for prefixes
          {ex-, over-, un-, or up-}
        Look for suffexes
          {-ed, -er, -man or -men, or -ful}
        Look for 1 letter words (If spaces are provided)
          These are most likely {I, a}
    Many sentences with a ? begin with a 'w' {who, what, when, where, why}

cyphertext of interest from https://twitter.com/NSACareers/status/463321993878994945
  tpfccdlfdtte pcaccplircdt dklpcfrp?qeiq lhpqlipqeodf gpwafopwprti izxndkiqpkii krirrifcapnc dxkdciqcafmd vkfpcadf.

source of strategies:
  http://www.bigfishgames.com/blog/cryptogram-tips-tricks-and-strategies/
  http://www.gtoal.com/wordgames/cryptograms.html
  http://www.muth.org/Robert/Cipher/substitution/
  http://www.quipqiup.com/howwork.php
"""
from docopt import docopt
import string
from termcolor import colored
import re

VERSION='1.1'
DATE='2014-05-09'
NAME='Cryptogram'

def substitute(args):
    """This Function prints the result of a requested substitution

    In the substitution,
        Uppercase blue is CLEARTEXT,
        lowercase green is CYPHERTEXT """
    # BUILD & PRINT CYPHERTEXT
    cyphertext = build_cyphertext(args)
    prettyprint_text('INPUT TEXT', cyphertext)

    filename = args['<file>']
    decodes = read_guess_file(filename)

    # PERFORM SUBSTITUTION
    print("")
    for c in cyphertext:
        if c in decodes.keys():
            print(colored(decodes[c], 'blue', attrs=['bold']), end="")
        else:
            if c in string.uppercase:
                c = c.lower()
            print(colored(c, 'green', attrs=['underline']), end="")
    print("\n")

    return

def read_guess_file(filename):
    """ This Function parses the guess file """
    is_decode = re.compile("^\s*(?P<cypher>[a-zA-Z])\s*=\s*(?P<clear>[a-zA-Z])[\s$#]")
    decodes = {}
    fd = open(filename, 'r')
    for line in fd:
        m = is_decode.search(line)
        if not m:
            continue
        #print(line, m.group('cypher'), m.group('clear'))
        if m.group('cypher') in decodes.keys():
            print("WARNING: Cypherchar \'%s\'"%m.group('cypher'), " specified multiple times in guess file - using last occurance.")
        decodes[m.group('cypher')] = m.group('clear')
    return decodes

def analyze(args):
    # BUILD & PRINT CYPHERTEXT
    cyphertext = build_cyphertext(args)
    prettyprint_text('INPUT TEXT', cyphertext)

    # CHARACTER COUNTS
    cypher_char_counts = count_chars(cyphertext)
    prettyprint_counts('CHARACTER', cypher_char_counts)

    # DOUBLES
    highlight_doubles(cyphertext)

    # CHARACTER SEQUENCES
    sequence_counts = count_sequences(cyphertext)
    prettyprint_counts('SEQUENCE', sequence_counts)

    # IDENTIFY POTENTIAL VOWELLS
    sukhotin(cyphertext)

    return

def sukhotin(text):
    freq = [ [ 0 for ii in range(26) ] for jj in range(26) ]
    for letter in string.uppercase:
        c = "@"
        last_c = "@"
        for next_c in text+"@":
            if c == letter:
                #print("DEBUG: ", "last_c=%s, "%last_c, "c=%s, "%c, "next_c=%s."%next_c)
                if last_c in string.uppercase:
                    freq[(ord(c)-65)][(ord(last_c)-65)] += 1
                    freq[(ord(last_c)-65)][(ord(c)-65)] += 1
                if next_c in string.uppercase:
                    freq[(ord(c)-65)][(ord(next_c)-65)] += 1
                    freq[(ord(next_c)-65)][(ord(c)-65)] += 1
            last_c = c
            c = next_c
    # zero diagonal
    for ii in range(26):
        freq[ii][ii] = 0

    # Determine SUMS:
    sums = [0 for ii in range(26)]
    letter_types = ["C" for ii in range(26)]
    for ii in range(26):
        sums[ii] = sum(freq[ii])

    # Identify Vowells
    vowels = []
    for round_num in range(1,6):
        if round_num > 1:
            # reduce sums for letters connected to last round's winner
            for ii in range(26):
                sums[ii] -= freq[ii][ord(max_letter)-65]*2
        # get max sum
        max_sum = 0
        for ii in range(26):
            if letter_types[ii] == "C" and sums[ii] > max_sum:
                max_sum = sums[ii]
        # get letter
        max_letters = [chr(ii+65) for ii,value in enumerate(sums) if (
          value == max_sum and letter_types[ii] == "C")]
        max_letter = max_letters[0] # pick lowest letter if there is a tie
        letter_types[ord(max_letter)-65] = "V"
        # debugging
        #print_sukhotin_state_debug(freq, sums, letter_types)
        vowels.append(max_letter)
    print ("\n== SUKHOTIN ==")
    print("Vowell(s) in order of confidence are ", vowels)
    return

def print_sukhotin_state_debug(freq, sums, letter_types):
    print ("== SUKHOTIN ==")
    print ("    ", end="")
    for letter in string.uppercase:
        print("%3s"%letter, end = "")
    print("  SUM  C/V")
    for ii,letter in enumerate(string.uppercase):
        print("%3s  "%letter, end = "")
        for value in freq[ii]:
            print("%2d "%value, end="")
        print(" %3d   "%sums[ii], "%s"%letter_types[ii])
    return

def count_sequences(text):
    ''' Count recurring sequences of characters '''
    counts = {}
    last_c = ""
    second_last_c = ""
    for c in text:
        sequence2chars = last_c + c
        sequence3chars = second_last_c + last_c + c
        # check if sequence2chars contains punctuation or space
        sequence2chars_valid = True
        for char in sequence2chars:
            if char not in string.letters and char not in string.digits:
                sequence2chars_valid = False
                break
        # check if sequence3chars contains punctuation or space
        sequence3chars_valid = True
        for char in sequence3chars:
            if char not in string.letters and char not in string.digits:
                sequence3chars_valid = False
                break
        if sequence2chars_valid:
            if sequence2chars not in counts.keys():
                counts[sequence2chars] = 1
            else:
                counts[sequence2chars] += 1
        if sequence3chars_valid:
            if sequence3chars not in counts.keys():
                counts[sequence3chars] = 1
            else:
                counts[sequence3chars] += 1
        second_last_c = last_c
        last_c = c
    # remove all counts that only occured once
    for key, value in counts.items():
        if value == 1:
            del counts[key]
    return counts

def highlight_doubles(text):
    ''' Show where doubles occur in text '''
    print("== DOUBLES ==")
    last_c = ""
    doubles = ""
    for ii,c in enumerate(text):
        if ii !=0 and ii%80 == 0:
            print('\n', doubles)
            doubles = ""
        if c == last_c:
            doubles = doubles[:-2] + '^^ '
        else:
            doubles += ' '
        print(c, sep='', end='')
        last_c = c
    if len(doubles)>0:
        print('\n', doubles)
    print("The most common English doubles are {ll, tt, ss, ee, pp, oo, rr, ff, cc, dd, nn}", '\n')
    return

def prettyprint_counts(title, counts):
    ''' print counts with formatting '''
    print("== %s COUNTS =="%title)
    sorted_keys = sorted(counts.iteritems(), key=lambda (k,v): (v,k), reverse=True)
    for ii,(key,value) in enumerate(sorted_keys):
        if ii !=0 and ii%10 == 0:
            print("")
        print("%4s:%3d"%(key, value), sep='', end=''),
    if title == 'CHARACTER':
        print("\nThe most common English letters are {e, t, a, o, i, n, s, h} in that order",'\n')
    if title == 'SEQUENCE':
        print("") # TODO: Are there a list of most common sequences?
    return

def prettyprint_text(title, text):
    print("\n== %s =="%title)
    for ii,c in enumerate(text):
        if ii !=0 and ii%80 == 0:
            print("")
        print(c, sep='', end='')
    print("\n")
    return

def count_chars(cyphertext):
    counts = {}
    for c in cyphertext:
        if not c in string.ascii_letters:
            # excludes numbers, symbols, spaces, and punctuation
            continue
        if c not in counts.keys():
            counts[c] = 1
        else:
            counts[c] += 1
    return counts

def build_cyphertext(args):
    input_text = ""
    cyphertext = ""
    for ii, cypher in enumerate(args['<cyphertext>']):
        cypher = cypher.upper()
        if ii == 0 or args['-i']: # remove spaces
            input_text += cypher
        else: # preserve spaces
            input_text += ' %s'%cypher
    for c in input_text:
        if c in string.uppercase:
            cyphertext += c
            args['has_letters'] = True
        elif c in string.whitespace and not args['-i']:
            cyphertext += c
            args['has_whitespace'] = True
        elif c in string.punctuation:
            cyphertext += c
            args['has_punctuation'] = True
        elif c in string.digits:
            cyphertext += c
            args['has_digits'] = True
        else:
            # ignore non printable characters
            continue
    return cyphertext

if __name__ == '__main__':
    args = docopt(__doc__, version='%s %s:%s'%(NAME, VERSION, DATE))
    if args['sub']:
        substitute(args)
    else:
        analyze(args)
<commit_msg>add background color to unsolved chars<commit_after>#!/usr/bin/env python
"""Cryptogram.

Description:
  This script statistically analyses a line of text to help solve a cryptogram

Author:
  Andrew Mattheisen

Usage:
  cryptogram.py analyze <cyphertext>...
  cryptogram.py analyze -i <cyphertext>...
  cryptogram.py sub -f <file> <cyphertext>...
  cryptogram.py sub -i -f <file> <cyphertext>...
  cryptogram.py <cyphertext>...
  cryptogram.py -i <cyphertext>...
  cryptogram.py (-h | --help)
  cryptogram.py --version

Options:
  -h --help     Show this screen.
  --version     Show version.
  -i            Ignore spaces. Use this when spaces are not provided between words.
  -f <file>     File containing guess mapping of cypherchar to plainchar.

"""
from __future__ import print_function
"""
Additional features to implement:
    IMPLEMENT WORD BOUNDARY ANALYSIS
        Look for words / Autosolve
          Create a map of word character patterns: e.g. workout = ABCDAFG
          Use words file. Algorithm: apply letter maps, count #words in solution.
          Display the solutions containing the most words
        Look for prefixes
          {ex-, over-, un-, or up-}
        Look for suffexes
          {-ed, -er, -man or -men, or -ful}
        Look for 1 letter words (If spaces are provided)
          These are most likely {I, a}
    Many sentences with a ? begin with a 'w' {who, what, when, where, why}

cyphertext of interest from https://twitter.com/NSACareers/status/463321993878994945
  tpfccdlfdtte pcaccplircdt dklpcfrp?qeiq lhpqlipqeodf gpwafopwprti izxndkiqpkii krirrifcapnc dxkdciqcafmd vkfpcadf.

source of strategies:
  http://www.bigfishgames.com/blog/cryptogram-tips-tricks-and-strategies/
  http://www.gtoal.com/wordgames/cryptograms.html
  http://www.muth.org/Robert/Cipher/substitution/
  http://www.quipqiup.com/howwork.php
"""
from docopt import docopt
import string
from termcolor import colored, cprint
import re

VERSION='1.1'
DATE='2014-05-09'
NAME='Cryptogram'

def substitute(args):
    """This Function prints the result of a requested substitution

    In the substitution,
        Uppercase blue is CLEARTEXT,
        lowercase green is CYPHERTEXT """
    # BUILD & PRINT CYPHERTEXT
    cyphertext = build_cyphertext(args)
    prettyprint_text('INPUT TEXT', cyphertext)

    filename = args['-f']
    decodes = read_guess_file(filename)

    # PERFORM SUBSTITUTION
    print("")
    for c in cyphertext:
        if c in decodes.keys():
            cprint(decodes[c], 'blue', attrs=['bold'], end="")
        else:
            if c in string.uppercase:
                c = c.lower()
            cprint(c, 'green', 'on_grey', attrs=['underline'], end="")
    print("\n")

    return

def read_guess_file(filename):
    """ This Function parses the guess file """
    is_decode = re.compile("^\s*(?P<cypher>[a-zA-Z])\s*=\s*(?P<clear>[a-zA-Z])[\s$#]")
    decodes = {}
    fd = open(filename, 'r')
    for line in fd:
        m = is_decode.search(line)
        if not m:
            continue
        #print(line, m.group('cypher'), m.group('clear'))
        if m.group('cypher') in decodes.keys():
            print("WARNING: Cypherchar \'%s\'"%m.group('cypher'), " specified multiple times in guess file - using last occurance.")
        decodes[m.group('cypher')] = m.group('clear')
    return decodes

def analyze(args):
    # BUILD & PRINT CYPHERTEXT
    cyphertext = build_cyphertext(args)
    prettyprint_text('INPUT TEXT', cyphertext)

    # CHARACTER COUNTS
    cypher_char_counts = count_chars(cyphertext)
    prettyprint_counts('CHARACTER', cypher_char_counts)

    # DOUBLES
    highlight_doubles(cyphertext)

    # CHARACTER SEQUENCES
    sequence_counts = count_sequences(cyphertext)
    prettyprint_counts('SEQUENCE', sequence_counts)

    # IDENTIFY POTENTIAL VOWELLS
    sukhotin(cyphertext)

    return

def sukhotin(text):
    freq = [ [ 0 for ii in range(26) ] for jj in range(26) ]
    for letter in string.uppercase:
        c = "@"
        last_c = "@"
        for next_c in text+"@":
            if c == letter:
                #print("DEBUG: ", "last_c=%s, "%last_c, "c=%s, "%c, "next_c=%s."%next_c)
                if last_c in string.uppercase:
                    freq[(ord(c)-65)][(ord(last_c)-65)] += 1
                    freq[(ord(last_c)-65)][(ord(c)-65)] += 1
                if next_c in string.uppercase:
                    freq[(ord(c)-65)][(ord(next_c)-65)] += 1
                    freq[(ord(next_c)-65)][(ord(c)-65)] += 1
            last_c = c
            c = next_c
    # zero diagonal
    for ii in range(26):
        freq[ii][ii] = 0

    # Determine SUMS:
    sums = [0 for ii in range(26)]
    letter_types = ["C" for ii in range(26)]
    for ii in range(26):
        sums[ii] = sum(freq[ii])

    # Identify Vowells
    vowels = []
    for round_num in range(1,6):
        if round_num > 1:
            # reduce sums for letters connected to last round's winner
            for ii in range(26):
                sums[ii] -= freq[ii][ord(max_letter)-65]*2
        # get max sum
        max_sum = 0
        for ii in range(26):
            if letter_types[ii] == "C" and sums[ii] > max_sum:
                max_sum = sums[ii]
        # get letter
        max_letters = [chr(ii+65) for ii,value in enumerate(sums) if (
          value == max_sum and letter_types[ii] == "C")]
        max_letter = max_letters[0] # pick lowest letter if there is a tie
        letter_types[ord(max_letter)-65] = "V"
        # debugging
        #print_sukhotin_state_debug(freq, sums, letter_types)
        vowels.append(max_letter)
    print ("\n== SUKHOTIN ==")
    print("Vowell(s) in order of confidence are ", vowels)
    return

def print_sukhotin_state_debug(freq, sums, letter_types):
    print ("== SUKHOTIN ==")
    print ("    ", end="")
    for letter in string.uppercase:
        print("%3s"%letter, end = "")
    print("  SUM  C/V")
    for ii,letter in enumerate(string.uppercase):
        print("%3s  "%letter, end = "")
        for value in freq[ii]:
            print("%2d "%value, end="")
        print(" %3d   "%sums[ii], "%s"%letter_types[ii])
    return

def count_sequences(text):
    ''' Count recurring sequences of characters '''
    counts = {}
    last_c = ""
    second_last_c = ""
    for c in text:
        sequence2chars = last_c + c
        sequence3chars = second_last_c + last_c + c
        # check if sequence2chars contains punctuation or space
        sequence2chars_valid = True
        for char in sequence2chars:
            if char not in string.letters and char not in string.digits:
                sequence2chars_valid = False
                break
        # check if sequence3chars contains punctuation or space
        sequence3chars_valid = True
        for char in sequence3chars:
            if char not in string.letters and char not in string.digits:
                sequence3chars_valid = False
                break
        if sequence2chars_valid:
            if sequence2chars not in counts.keys():
                counts[sequence2chars] = 1
            else:
                counts[sequence2chars] += 1
        if sequence3chars_valid:
            if sequence3chars not in counts.keys():
                counts[sequence3chars] = 1
            else:
                counts[sequence3chars] += 1
        second_last_c = last_c
        last_c = c
    # remove all counts that only occured once
    for key, value in counts.items():
        if value == 1:
            del counts[key]
    return counts

def highlight_doubles(text):
    ''' Show where doubles occur in text '''
    print("== DOUBLES ==")
    last_c = ""
    doubles = ""
    for ii,c in enumerate(text):
        if ii !=0 and ii%80 == 0:
            print('\n', doubles)
            doubles = ""
        if c == last_c:
            doubles = doubles[:-2] + '^^ '
        else:
            doubles += ' '
        print(c, sep='', end='')
        last_c = c
    if len(doubles)>0:
        print('\n', doubles)
    print("The most common English doubles are {ll, tt, ss, ee, pp, oo, rr, ff, cc, dd, nn}", '\n')
    return

def prettyprint_counts(title, counts):
    ''' print counts with formatting '''
    print("== %s COUNTS =="%title)
    sorted_keys = sorted(counts.iteritems(), key=lambda (k,v): (v,k), reverse=True)
    for ii,(key,value) in enumerate(sorted_keys):
        if ii !=0 and ii%10 == 0:
            print("")
        print("%4s:%3d"%(key, value), sep='', end=''),
    if title == 'CHARACTER':
        print("\nThe most common English letters are {e, t, a, o, i, n, s, h} in that order",'\n')
    if title == 'SEQUENCE':
        print("") # TODO: Are there a list of most common sequences?
    return

def prettyprint_text(title, text):
    print("\n== %s =="%title)
    for ii,c in enumerate(text):
        if ii !=0 and ii%80 == 0:
            print("")
        print(c, sep='', end='')
    print("\n")
    return

def count_chars(cyphertext):
    counts = {}
    for c in cyphertext:
        if not c in string.ascii_letters:
            # excludes numbers, symbols, spaces, and punctuation
            continue
        if c not in counts.keys():
            counts[c] = 1
        else:
            counts[c] += 1
    return counts

def build_cyphertext(args):
    input_text = ""
    cyphertext = ""
    for ii, cypher in enumerate(args['<cyphertext>']):
        cypher = cypher.upper()
        if ii == 0 or args['-i']: # remove spaces
            input_text += cypher
        else: # preserve spaces
            input_text += ' %s'%cypher
    for c in input_text:
        if c in string.uppercase:
            cyphertext += c
            args['has_letters'] = True
        elif c in string.whitespace and not args['-i']:
            cyphertext += c
            args['has_whitespace'] = True
        elif c in string.punctuation:
            cyphertext += c
            args['has_punctuation'] = True
        elif c in string.digits:
            cyphertext += c
            args['has_digits'] = True
        else:
            # ignore non printable characters
            continue
    return cyphertext

if __name__ == '__main__':
    args = docopt(__doc__, version='%s %s:%s'%(NAME, VERSION, DATE))
    if args['sub']:
        substitute(args)
    else:
        analyze(args)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2013 Hannes Juutilainen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import re

from autopkglib import Processor, ProcessorError

try:
    from urllib.request import urlopen  # For Python 3
except ImportError:
    from urllib2 import urlopen  # For Python 2

__all__ = ["VagrantURLProvider"]

MAIN_DOWNLOAD_URL = "http://www.vagrantup.com/downloads.html"

re_dmg_url = re.compile(r'<a href=[\'\"](?P<dmg_url>https://dl.bintray.com/mitchellh/vagrant/Vagrant[-_][0-9\.]+\.dmg)[\'\"]>', re.IGNORECASE)

class VagrantURLProvider(Processor):
    """Provides a download URL for the latest Vagrant"""
    input_variables = {
        "base_url": {
            "required": False,
            "description": "The Vagrant download site",
        },
    }
    output_variables = {
        "url": {
            "description": "URL to the latest Vagrant release.",
        },
    }
    description = __doc__

    def parse_download_url(self, base_url):
        """Returns the URL"""
        try:
            f = urlopen(base_url)
            html = f.read()
            f.close()
        except BaseException as e:
            raise ProcessorError("Can't download %s: %s" % (base_url, e))

        m = re_dmg_url.search(html)

        if not m:
            raise ProcessorError(
                "Couldn't find dmg link in %s" % base_url)

        dmg_url = m.group("dmg_url")
        self.output("Found dmg link: %s" % dmg_url)
        return dmg_url

    def get_vagrant_dmg_url(self, base_url):
        """Find and return a download URL"""

        # Parse the download page to get a dmg link
        dmg_url = self.parse_download_url(base_url)

        return dmg_url

    def main(self):
        base_url = self.env.get("base_url", MAIN_DOWNLOAD_URL)
        self.env["url"] = self.get_vagrant_dmg_url(base_url)
        self.output("Found URL %s" % self.env["url"])

if __name__ == "__main__":
    processor = VagrantURLProvider()
    processor.execute_shell()
<commit_msg>Use Exception instead of more general BaseException<commit_after>#!/usr/bin/env python
#
# Copyright 2013 Hannes Juutilainen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import re

from autopkglib import Processor, ProcessorError

try:
    from urllib.request import urlopen  # For Python 3
except ImportError:
    from urllib2 import urlopen  # For Python 2

__all__ = ["VagrantURLProvider"]

MAIN_DOWNLOAD_URL = "http://www.vagrantup.com/downloads.html"

re_dmg_url = re.compile(r'<a href=[\'\"](?P<dmg_url>https://dl.bintray.com/mitchellh/vagrant/Vagrant[-_][0-9\.]+\.dmg)[\'\"]>', re.IGNORECASE)

class VagrantURLProvider(Processor):
    """Provides a download URL for the latest Vagrant"""
    input_variables = {
        "base_url": {
            "required": False,
            "description": "The Vagrant download site",
        },
    }
    output_variables = {
        "url": {
            "description": "URL to the latest Vagrant release.",
        },
    }
    description = __doc__

    def parse_download_url(self, base_url):
        """Returns the URL"""
        try:
            f = urlopen(base_url)
            html = f.read()
            f.close()
        except Exception as e:
            raise ProcessorError("Can't download %s: %s" % (base_url, e))

        m = re_dmg_url.search(html)

        if not m:
            raise ProcessorError(
                "Couldn't find dmg link in %s" % base_url)

        dmg_url = m.group("dmg_url")
        self.output("Found dmg link: %s" % dmg_url)
        return dmg_url

    def get_vagrant_dmg_url(self, base_url):
        """Find and return a download URL"""

        # Parse the download page to get a dmg link
        dmg_url = self.parse_download_url(base_url)

        return dmg_url

    def main(self):
        base_url = self.env.get("base_url", MAIN_DOWNLOAD_URL)
        self.env["url"] = self.get_vagrant_dmg_url(base_url)
        self.output("Found URL %s" % self.env["url"])

if __name__ == "__main__":
    processor = VagrantURLProvider()
    processor.execute_shell()
<|endoftext|>


<commit_before>import tokenizer

class Tree:
  def __init__(self, elements):
    self._elements = elements

  def count(self):
    return len(self.elements())

  def __len__(self):
    return len(self.elements())

  def elements(self):
    result = []
    for element in self._elements:
      if element.__class__ == Tree:
        result.append(element.elements())
      elif element.isspace() or element.startswith("#"):
        continue
      else:
        result.append(element)
    return result

  def elements_children_as_trees(self):
    result = []
    for element in self._elements:
      if element.__class__ == Tree:
        result.append(element)
      elif element.isspace() or element.startswith("#"):
        continue
      else:
        result.append(element)
    return result

  def __getitem__(self, index):
    return self.elements_children_as_trees()[index]

  def __setitem__(self, index, value):
    non_whitespace = -1
    j = 0
    while j < len(self._elements):
      element = self._elements[j]
      if element.__class__ == Tree:
        non_whitespace += 1
      elif element.isspace() or element.startswith("#"):
        j += 1
        continue
      else:
        non_whitespace += 1
      if non_whitespace == index:
        self._elements[j] = value
        return
      j += 1

  def to_string(self):
    result = ''
    for element in self._elements:
      if element.__class__ == Tree:
        result += '('
        result += element.to_string()
        result += ')'
      else:
        result += element
    return result

def read_expression(tokenizer1):
    while True:
        token = tokenizer1.next_token()
        if token == None:
            return None
        if token == '(':
            result = []
            while True:
                subsexp = read_expression(tokenizer1)
                if subsexp == ')':
                    break
                elif subsexp == None:
                    raise SyntaxError('eof inside sexp')
                result.append(subsexp)
            return Tree(result)
        else:
            return token

def parse(stream):
  tokenizer1 = tokenizer.Tokenizer(stream)
  expressions = []
  while True:
    expression = read_expression(tokenizer1)
    if expression == None:
      break
    expressions.append(expression)
  return Tree(expressions)

<commit_msg>Implement elements_children_as_trees in terms of list comprehension.<commit_after>import tokenizer

class Tree:
  def __init__(self, elements):
    self._elements = elements

  def count(self):
    return len(self.elements())

  def __len__(self):
    return len(self.elements())

  def elements(self):
    result = []
    for element in self._elements:
      if element.__class__ == Tree:
        result.append(element.elements())
      elif element.isspace() or element.startswith("#"):
        continue
      else:
        result.append(element)
    return result

  def elements_children_as_trees(self):
    return [element for element in self._elements
      if element.__class__ == Tree or
        (not element.isspace() and not element.startswith("#"))
      ]

  def __getitem__(self, index):
    return self.elements_children_as_trees()[index]

  def __setitem__(self, index, value):
    non_whitespace = -1
    j = 0
    while j < len(self._elements):
      element = self._elements[j]
      if element.__class__ == Tree:
        non_whitespace += 1
      elif element.isspace() or element.startswith("#"):
        j += 1
        continue
      else:
        non_whitespace += 1
      if non_whitespace == index:
        self._elements[j] = value
        return
      j += 1

  def to_string(self):
    result = ''
    for element in self._elements:
      if element.__class__ == Tree:
        result += '('
        result += element.to_string()
        result += ')'
      else:
        result += element
    return result

def read_expression(tokenizer1):
    while True:
        token = tokenizer1.next_token()
        if token == None:
            return None
        if token == '(':
            result = []
            while True:
                subsexp = read_expression(tokenizer1)
                if subsexp == ')':
                    break
                elif subsexp == None:
                    raise SyntaxError('eof inside sexp')
                result.append(subsexp)
            return Tree(result)
        else:
            return token

def parse(stream):
  tokenizer1 = tokenizer.Tokenizer(stream)
  expressions = []
  while True:
    expression = read_expression(tokenizer1)
    if expression == None:
      break
    expressions.append(expression)
  return Tree(expressions)

<|endoftext|>


<commit_before>"""
This module contains a set of methods that can be used for page loads and
for waiting for elements to appear on a page.

These methods improve on and expand existing WebDriver commands.
Improvements include making WebDriver commands more robust and more reliable
by giving page elements enough time to load before taking action on them.

The default option for searching for elements is by CSS Selector.
This can be changed by overriding the "By" parameter.
Options are:
By.CSS_SELECTOR
By.CLASS_NAME
By.ID
By.NAME
By.LINK_TEXT
By.XPATH
By.TAG_NAME
By.PARTIAL_LINK_TEXT
"""

import codecs
import os
import sys
import time
import traceback
from seleniumbase.config import settings
from selenium.webdriver.common.by import By
from selenium.webdriver.remote.errorhandler import ElementNotVisibleException
from selenium.webdriver.remote.errorhandler import NoSuchElementException
from selenium.webdriver.remote.errorhandler import NoAlertPresentException

def is_element_present(driver, selector, by=By.CSS_SELECTOR):
    """
    Searches for the specified element by the given selector.  Returns whether
    the element object if the element is present on the page.
    @Params
    driver - the webdriver object (required)
    selector - the locator that is used (required)
    by - the method to search for the locator (Default: By.CSS_SELECTOR)
    @Returns
    Boolean (is element present)
    """
    try:
        driver.find_element(by=by, value=selector)
        return True
    except Exception:
        return False

def is_element_visible(driver, selector, by=By.CSS_SELECTOR):
    """
    Searches for the specified element by the given selector.  Returns whether
    the element object if the element is present and visible on the page.
    @Params
    driver - the webdriver object (required)
    selector - the locator that is used (required)
    by - the method to search for the locator (Default: By.CSS_SELECTOR)
    @Returns
    Boolean (is element visible)
    """
    try:
        element = driver.find_element(by=by, value=selector)
        return element.is_displayed()
    except Exception:
        return False

def is_text_visible(driver, text, selector, by=By.CSS_SELECTOR):
    """
    Searches for the specified element by the given selector. Returns whether
    the element object if the element is present and visible on the page and
    contains the given text.
    @Params
    driver - the webdriver object (required)
    text - the text string to search for
    selector - the locator that is used (required)
    by - the method to search for the locator (Default: By.CSS_SELECTOR)
    @Returns
    Boolean (is text visible)
    """
    try:
        element = driver.find_element(by=by, value=selector)
        return element.is_displayed() and text in element.text
    except Exception:
        return False

def hover_on_element(driver, selector):
    """
    Fires the hover event for the specified element by the given selector.
    @Params
    driver - the webdriver object (required)
    selector - the locator (css selector) that is used (required)
    """
    driver.execute_script("jQuery('%s').mouseover()" % selector)

def hover_and_click(driver, hover_selector, click_selector,
                    click_by=By.CSS_SELECTOR, timeout=settings.SMALL_TIMEOUT):
    """
    Fires the hover event for a specified element by a given selector, then
    clicks on another element specified. Useful for dropdown hover based menus.
    @Params
    driver - the webdriver object (required)
    hover_selector - the css selector to hover over (required)
    click_selector - the css selector to click on (required)
    click_by - the method to search by (Default: By.CSS_SELECTOR)
    timeout - number of seconds to wait for click element to appear after hover
    """
    driver.execute_script("jQuery('%s').mouseover()" % (hover_selector))
    for x in range(int(timeout * 10)):
        try:
            element = driver.find_element(by=click_by,
                                          value="%s" % click_selector).click()
            return element
        except Exception:
            time.sleep(0.1)
    raise NoSuchElementException(
        "Element [%s] was not present after %s seconds!" %
        (click_selector, timeout))

def wait_for_element_present(driver, selector, by=By.CSS_SELECTOR,
                             timeout=settings.LARGE_TIMEOUT):
    """
    Searches for the specified element by the given selector. Returns the
    element object if the element is present on the page. The element can be
    invisible. Raises an exception if the element does not appear in the
    specified timeout.
    @Params
    driver - the webdriver object
    selector - the locator that is used (required)
    by - the method to search for the locator (Default: By.CSS_SELECTOR)
    timeout - the time to wait for elements in seconds
    @Returns
    A web element object
    """

    element = None
    for x in range(int(timeout * 10)):
        try:
            element = driver.find_element(by=by, value=selector)
            return element
        except Exception:
            time.sleep(0.1)
    if not element:
        raise NoSuchElementException(
            "Element [%s] was not present after %s seconds!" % (
                selector, timeout))

def wait_for_element_visible(driver, selector, by=By.CSS_SELECTOR,
                             timeout=settings.LARGE_TIMEOUT):
    """
    Searches for the specified element by the given selector. Returns the
    element object if the element is present and visible on the page.
    Raises an exception if the element does not appear in the
    specified timeout.
    @Params
    driver - the webdriver object (required)
    selector - the locator that is used (required)
    by - the method to search for the locator (Default: By.CSS_SELECTOR)
    timeout - the time to wait for elements in seconds

    @Returns
    A web element object
    """

    element = None
    for x in range(int(timeout * 10)):
        try:
            element = driver.find_element(by=by, value=selector)
            if element.is_displayed():
                return element
            else:
                element = None
                raise Exception()
        except Exception:
            time.sleep(0.1)
    if not element and by != By.LINK_TEXT:
        raise ElementNotVisibleException(
            "Element [%s] was not visible after %s seconds!" % (
                selector, timeout))
    if not element and by == By.LINK_TEXT:
        raise ElementNotVisibleException(
            "Link text [%s] was not visible after %s seconds!" % (
                selector, timeout))

def wait_for_text_visible(driver, text, selector, by=By.CSS_SELECTOR,
                          timeout=settings.LARGE_TIMEOUT):
    """
    Searches for the specified element by the given selector. Returns the
    element object if the text is present in the element and visible
    on the page. Raises an exception if the text or element do not appear
    in the specified timeout.
    @Params
    driver - the webdriver object (required)
    text - the text that is being searched for in the element (required)
    selector - the locator that is used (required)
    by - the method to search for the locator (Default: By.CSS_SELECTOR)
    timeout - the time to wait for elements in seconds
    @Returns
    A web element object that contains the text searched for
    """

    element = None
    for x in range(int(timeout * 10)):
        try:
            element = driver.find_element(by=by, value=selector)
            if element.is_displayed():
                if text in element.text:
                    return element
                else:
                    element = None
                    raise Exception()
        except Exception:
            time.sleep(0.1)
    if not element:
        raise ElementNotVisibleException(
            "Expected text [%s] for [%s] was not visible after %s seconds!" %
            (text, selector, timeout))

def wait_for_element_absent(driver, selector, by=By.CSS_SELECTOR,
                            timeout=settings.LARGE_TIMEOUT):
    """
    Searches for the specified element by the given selector.
    Raises an exception if the element is still present after the
    specified timeout.
    @Params
    driver - the webdriver object
    selector - the locator that is used (required)
    by - the method to search for the locator (Default: By.CSS_SELECTOR)
    timeout - the time to wait for elements in seconds
    """

    for x in range(int(timeout * 10)):
        try:
            driver.find_element(by=by, value=selector)
            time.sleep(0.1)
        except Exception:
            return
    raise Exception("Element [%s] was still present after %s seconds!" %
                    (selector, timeout))

def wait_for_element_not_visible(driver, selector, by=By.CSS_SELECTOR,
                                 timeout=settings.LARGE_TIMEOUT):
    """
    Searches for the specified element by the given selector.
    Raises an exception if the element is still visible after the
    specified timeout.
    @Params
    driver - the webdriver object (required)
    selector - the locator that is used (required)
    by - the method to search for the locator (Default: By.CSS_SELECTOR)
    timeout - the time to wait for the element in seconds
    """

    for x in range(int(timeout * 10)):
        try:
            element = driver.find_element(by=by, value=selector)
            if element.is_displayed():
                time.sleep(0.1)
            else:
                return
        except Exception:
            return
    raise Exception(
        "Element [%s] was still visible after %s seconds!" % (
            selector, timeout))

def find_visible_elements(driver, selector, by=By.CSS_SELECTOR):
    """
    Finds all WebElements that match a selector and are visible.
    Similar to webdriver.find_elements.
    @Params
    driver - the webdriver object (required)
    selector - the locator that is used to search the DOM (required)
    by - the method to search for the locator (Default: By.CSS_SELECTOR)
    """
    elements = driver.find_elements(by=by, value=selector)
    return [element for element in elements if element.is_displayed()]

def save_screenshot(driver, name, folder=None):
    """
    Saves a screenshot to the current directory (or to a subfolder if provided)
    If the folder provided doesn't exist, it will get created.
    """
    if folder:
        abs_path = os.path.abspath('.')
        file_path = abs_path + "/%s" % folder
        if not os.path.exists(file_path):
            os.makedirs(file_path)
        screenshot_file = "%s/%s" % (file_path, name)
    else:
        screenshot_file = name
    driver.get_screenshot_as_file(screenshot_file)

def _get_last_page(driver):
    try:
        last_page = driver.current_url
    except Exception:
        last_page = '[WARNING! Browser Not Open!]'
    if len(last_page) < 5:
        last_page = '[WARNING! Browser Not Open!]'
    return last_page

def save_test_failure_data(driver, name, browser_type, folder=None):
    """
    Saves failure data to the current directory (or to a subfolder if provided)
    If the folder provided doesn't exist, it will get created.
    """
    if folder:
        abs_path = os.path.abspath('.')
        file_path = abs_path + "/%s" % folder
        if not os.path.exists(file_path):
            os.makedirs(file_path)
        failure_data_file_path = "%s/%s" % (file_path, name)
    else:
        failure_data_file_path = name
    failure_data_file = codecs.open(failure_data_file_path, "w+", "utf-8")
    last_page = _get_last_page(driver)
    data_to_save = []
    data_to_save.append("Last_Page: %s" % last_page)
    data_to_save.append("Browser: %s " % browser_type)
    data_to_save.append("Traceback: " + ''.join(
        traceback.format_exception(sys.exc_info()[0],
                                   sys.exc_info()[1],
                                   sys.exc_info()[2])))
    failure_data_file.writelines("\r\n".join(data_to_save))
    failure_data_file.close()

def wait_for_ready_state_complete(driver, timeout=settings.EXTREME_TIMEOUT):
    """
    The DOM (Document Object Model) has a property called "readyState".
    When the value of this becomes "complete", page resources are considered
    fully loaded (although AJAX and other loads might still be happening).
    This method will wait until document.readyState == "complete".
    """

    for x in range(int(timeout * 10)):
        ready_state = driver.execute_script("return document.readyState")
        if ready_state == u'complete':
            return True
        else:
            time.sleep(0.1)
    raise Exception(
        "Page elements never fully loaded after %s seconds!" % timeout)

def wait_for_and_accept_alert(driver, timeout=settings.LARGE_TIMEOUT):
    """
    Wait for and accept an alert. Returns the text from the alert.
    @Params
    driver - the webdriver object (required)
    timeout - the time to wait for the alert in seconds
    """
    alert = wait_for_and_switch_to_alert(driver, timeout)
    alert_text = alert.text
    alert.accept()
    return alert_text

def wait_for_and_dismiss_alert(driver, timeout=settings.LARGE_TIMEOUT):
    """
    Wait for and dismiss an alert. Returns the text from the alert.
    @Params
    driver - the webdriver object (required)
    timeout - the time to wait for the alert in seconds
    """
    alert = wait_for_and_switch_to_alert(driver, timeout)
    alert_text = alert.text
    alert.dismiss()
    return alert_text

def wait_for_and_switch_to_alert(driver, timeout=settings.LARGE_TIMEOUT):
    """
    Wait for a browser alert to appear, and switch to it. This should be usable
    as a drop-in replacement for driver.switch_to.alert when the alert box
    may not exist yet.
    @Params
    driver - the webdriver object (required)
    timeout - the time to wait for the alert in seconds
    """

    for x in range(int(timeout * 10)):
        try:
            alert = driver.switch_to.alert
            # Raises exception if no alert present
            dummy_variable = alert.text  # noqa
            return alert
        except NoAlertPresentException:
            time.sleep(0.1)
    raise Exception("Alert was not present after %s seconds!" % timeout)
<commit_msg>Make sure timeouts keep their limit if webdriver calls are slow<commit_after>"""
This module contains a set of methods that can be used for page loads and
for waiting for elements to appear on a page.

These methods improve on and expand existing WebDriver commands.
Improvements include making WebDriver commands more robust and more reliable
by giving page elements enough time to load before taking action on them.

The default option for searching for elements is by CSS Selector.
This can be changed by overriding the "By" parameter.
Options are:
By.CSS_SELECTOR
By.CLASS_NAME
By.ID
By.NAME
By.LINK_TEXT
By.XPATH
By.TAG_NAME
By.PARTIAL_LINK_TEXT
"""

import codecs
import os
import sys
import time
import traceback
from seleniumbase.config import settings
from selenium.webdriver.common.by import By
from selenium.webdriver.remote.errorhandler import ElementNotVisibleException
from selenium.webdriver.remote.errorhandler import NoSuchElementException
from selenium.webdriver.remote.errorhandler import NoAlertPresentException

def is_element_present(driver, selector, by=By.CSS_SELECTOR):
    """
    Searches for the specified element by the given selector.  Returns whether
    the element object if the element is present on the page.
    @Params
    driver - the webdriver object (required)
    selector - the locator that is used (required)
    by - the method to search for the locator (Default: By.CSS_SELECTOR)
    @Returns
    Boolean (is element present)
    """
    try:
        driver.find_element(by=by, value=selector)
        return True
    except Exception:
        return False

def is_element_visible(driver, selector, by=By.CSS_SELECTOR):
    """
    Searches for the specified element by the given selector.  Returns whether
    the element object if the element is present and visible on the page.
    @Params
    driver - the webdriver object (required)
    selector - the locator that is used (required)
    by - the method to search for the locator (Default: By.CSS_SELECTOR)
    @Returns
    Boolean (is element visible)
    """
    try:
        element = driver.find_element(by=by, value=selector)
        return element.is_displayed()
    except Exception:
        return False

def is_text_visible(driver, text, selector, by=By.CSS_SELECTOR):
    """
    Searches for the specified element by the given selector. Returns whether
    the element object if the element is present and visible on the page and
    contains the given text.
    @Params
    driver - the webdriver object (required)
    text - the text string to search for
    selector - the locator that is used (required)
    by - the method to search for the locator (Default: By.CSS_SELECTOR)
    @Returns
    Boolean (is text visible)
    """
    try:
        element = driver.find_element(by=by, value=selector)
        return element.is_displayed() and text in element.text
    except Exception:
        return False

def hover_on_element(driver, selector):
    """
    Fires the hover event for the specified element by the given selector.
    @Params
    driver - the webdriver object (required)
    selector - the locator (css selector) that is used (required)
    """
    driver.execute_script("jQuery('%s').mouseover()" % selector)

def hover_and_click(driver, hover_selector, click_selector,
                    click_by=By.CSS_SELECTOR, timeout=settings.SMALL_TIMEOUT):
    """
    Fires the hover event for a specified element by a given selector, then
    clicks on another element specified. Useful for dropdown hover based menus.
    @Params
    driver - the webdriver object (required)
    hover_selector - the css selector to hover over (required)
    click_selector - the css selector to click on (required)
    click_by - the method to search by (Default: By.CSS_SELECTOR)
    timeout - number of seconds to wait for click element to appear after hover
    """
    start_ms = time.time() * 1000.0
    stop_ms = start_ms + (timeout * 1000.0)
    driver.execute_script("jQuery('%s').mouseover()" % (hover_selector))
    for x in range(int(timeout * 10)):
        try:
            element = driver.find_element(by=click_by,
                                          value="%s" % click_selector).click()
            return element
        except Exception:
            now_ms = time.time() * 1000.0
            if now_ms >= stop_ms:
                break
            time.sleep(0.1)
    raise NoSuchElementException(
        "Element [%s] was not present after %s seconds!" %
        (click_selector, timeout))

def wait_for_element_present(driver, selector, by=By.CSS_SELECTOR,
                             timeout=settings.LARGE_TIMEOUT):
    """
    Searches for the specified element by the given selector. Returns the
    element object if the element is present on the page. The element can be
    invisible. Raises an exception if the element does not appear in the
    specified timeout.
    @Params
    driver - the webdriver object
    selector - the locator that is used (required)
    by - the method to search for the locator (Default: By.CSS_SELECTOR)
    timeout - the time to wait for elements in seconds
    @Returns
    A web element object
    """

    element = None
    start_ms = time.time() * 1000.0
    stop_ms = start_ms + (timeout * 1000.0)
    for x in range(int(timeout * 10)):
        try:
            element = driver.find_element(by=by, value=selector)
            return element
        except Exception:
            now_ms = time.time() * 1000.0
            if now_ms >= stop_ms:
                break
            time.sleep(0.1)
    if not element:
        raise NoSuchElementException(
            "Element [%s] was not present after %s seconds!" % (
                selector, timeout))

def wait_for_element_visible(driver, selector, by=By.CSS_SELECTOR,
                             timeout=settings.LARGE_TIMEOUT):
    """
    Searches for the specified element by the given selector. Returns the
    element object if the element is present and visible on the page.
    Raises an exception if the element does not appear in the
    specified timeout.
    @Params
    driver - the webdriver object (required)
    selector - the locator that is used (required)
    by - the method to search for the locator (Default: By.CSS_SELECTOR)
    timeout - the time to wait for elements in seconds

    @Returns
    A web element object
    """

    element = None
    start_ms = time.time() * 1000.0
    stop_ms = start_ms + (timeout * 1000.0)
    for x in range(int(timeout * 10)):
        try:
            element = driver.find_element(by=by, value=selector)
            if element.is_displayed():
                return element
            else:
                element = None
                raise Exception()
        except Exception:
            now_ms = time.time() * 1000.0
            if now_ms >= stop_ms:
                break
            time.sleep(0.1)
    if not element and by != By.LINK_TEXT:
        raise ElementNotVisibleException(
            "Element [%s] was not visible after %s seconds!" % (
                selector, timeout))
    if not element and by == By.LINK_TEXT:
        raise ElementNotVisibleException(
            "Link text [%s] was not visible after %s seconds!" % (
                selector, timeout))

def wait_for_text_visible(driver, text, selector, by=By.CSS_SELECTOR,
                          timeout=settings.LARGE_TIMEOUT):
    """
    Searches for the specified element by the given selector. Returns the
    element object if the text is present in the element and visible
    on the page. Raises an exception if the text or element do not appear
    in the specified timeout.
    @Params
    driver - the webdriver object (required)
    text - the text that is being searched for in the element (required)
    selector - the locator that is used (required)
    by - the method to search for the locator (Default: By.CSS_SELECTOR)
    timeout - the time to wait for elements in seconds
    @Returns
    A web element object that contains the text searched for
    """

    element = None
    start_ms = time.time() * 1000.0
    stop_ms = start_ms + (timeout * 1000.0)
    for x in range(int(timeout * 10)):
        try:
            element = driver.find_element(by=by, value=selector)
            if element.is_displayed():
                if text in element.text:
                    return element
                else:
                    element = None
                    raise Exception()
        except Exception:
            now_ms = time.time() * 1000.0
            if now_ms >= stop_ms:
                break
            time.sleep(0.1)
    if not element:
        raise ElementNotVisibleException(
            "Expected text [%s] for [%s] was not visible after %s seconds!" %
            (text, selector, timeout))

def wait_for_element_absent(driver, selector, by=By.CSS_SELECTOR,
                            timeout=settings.LARGE_TIMEOUT):
    """
    Searches for the specified element by the given selector.
    Raises an exception if the element is still present after the
    specified timeout.
    @Params
    driver - the webdriver object
    selector - the locator that is used (required)
    by - the method to search for the locator (Default: By.CSS_SELECTOR)
    timeout - the time to wait for elements in seconds
    """

    start_ms = time.time() * 1000.0
    stop_ms = start_ms + (timeout * 1000.0)
    for x in range(int(timeout * 10)):
        try:
            driver.find_element(by=by, value=selector)
            now_ms = time.time() * 1000.0
            if now_ms >= stop_ms:
                break
            time.sleep(0.1)
        except Exception:
            return
    raise Exception("Element [%s] was still present after %s seconds!" %
                    (selector, timeout))

def wait_for_element_not_visible(driver, selector, by=By.CSS_SELECTOR,
                                 timeout=settings.LARGE_TIMEOUT):
    """
    Searches for the specified element by the given selector.
    Raises an exception if the element is still visible after the
    specified timeout.
    @Params
    driver - the webdriver object (required)
    selector - the locator that is used (required)
    by - the method to search for the locator (Default: By.CSS_SELECTOR)
    timeout - the time to wait for the element in seconds
    """

    start_ms = time.time() * 1000.0
    stop_ms = start_ms + (timeout * 1000.0)
    for x in range(int(timeout * 10)):
        try:
            element = driver.find_element(by=by, value=selector)
            if element.is_displayed():
                now_ms = time.time() * 1000.0
                if now_ms >= stop_ms:
                    break
                time.sleep(0.1)
            else:
                return
        except Exception:
            return
    raise Exception(
        "Element [%s] was still visible after %s seconds!" % (
            selector, timeout))

def find_visible_elements(driver, selector, by=By.CSS_SELECTOR):
    """
    Finds all WebElements that match a selector and are visible.
    Similar to webdriver.find_elements.
    @Params
    driver - the webdriver object (required)
    selector - the locator that is used to search the DOM (required)
    by - the method to search for the locator (Default: By.CSS_SELECTOR)
    """
    elements = driver.find_elements(by=by, value=selector)
    return [element for element in elements if element.is_displayed()]

def save_screenshot(driver, name, folder=None):
    """
    Saves a screenshot to the current directory (or to a subfolder if provided)
    If the folder provided doesn't exist, it will get created.
    """
    if folder:
        abs_path = os.path.abspath('.')
        file_path = abs_path + "/%s" % folder
        if not os.path.exists(file_path):
            os.makedirs(file_path)
        screenshot_file = "%s/%s" % (file_path, name)
    else:
        screenshot_file = name
    driver.get_screenshot_as_file(screenshot_file)

def _get_last_page(driver):
    try:
        last_page = driver.current_url
    except Exception:
        last_page = '[WARNING! Browser Not Open!]'
    if len(last_page) < 5:
        last_page = '[WARNING! Browser Not Open!]'
    return last_page

def save_test_failure_data(driver, name, browser_type, folder=None):
    """
    Saves failure data to the current directory (or to a subfolder if provided)
    If the folder provided doesn't exist, it will get created.
    """
    if folder:
        abs_path = os.path.abspath('.')
        file_path = abs_path + "/%s" % folder
        if not os.path.exists(file_path):
            os.makedirs(file_path)
        failure_data_file_path = "%s/%s" % (file_path, name)
    else:
        failure_data_file_path = name
    failure_data_file = codecs.open(failure_data_file_path, "w+", "utf-8")
    last_page = _get_last_page(driver)
    data_to_save = []
    data_to_save.append("Last_Page: %s" % last_page)
    data_to_save.append("Browser: %s " % browser_type)
    data_to_save.append("Traceback: " + ''.join(
        traceback.format_exception(sys.exc_info()[0],
                                   sys.exc_info()[1],
                                   sys.exc_info()[2])))
    failure_data_file.writelines("\r\n".join(data_to_save))
    failure_data_file.close()

def wait_for_ready_state_complete(driver, timeout=settings.EXTREME_TIMEOUT):
    """
    The DOM (Document Object Model) has a property called "readyState".
    When the value of this becomes "complete", page resources are considered
    fully loaded (although AJAX and other loads might still be happening).
    This method will wait until document.readyState == "complete".
    """

    start_ms = time.time() * 1000.0
    stop_ms = start_ms + (timeout * 1000.0)
    for x in range(int(timeout * 10)):
        ready_state = driver.execute_script("return document.readyState")
        if ready_state == u'complete':
            return True
        else:
            now_ms = time.time() * 1000.0
            if now_ms >= stop_ms:
                break
            time.sleep(0.1)
    raise Exception(
        "Page elements never fully loaded after %s seconds!" % timeout)

def wait_for_and_accept_alert(driver, timeout=settings.LARGE_TIMEOUT):
    """
    Wait for and accept an alert. Returns the text from the alert.
    @Params
    driver - the webdriver object (required)
    timeout - the time to wait for the alert in seconds
    """
    alert = wait_for_and_switch_to_alert(driver, timeout)
    alert_text = alert.text
    alert.accept()
    return alert_text

def wait_for_and_dismiss_alert(driver, timeout=settings.LARGE_TIMEOUT):
    """
    Wait for and dismiss an alert. Returns the text from the alert.
    @Params
    driver - the webdriver object (required)
    timeout - the time to wait for the alert in seconds
    """
    alert = wait_for_and_switch_to_alert(driver, timeout)
    alert_text = alert.text
    alert.dismiss()
    return alert_text

def wait_for_and_switch_to_alert(driver, timeout=settings.LARGE_TIMEOUT):
    """
    Wait for a browser alert to appear, and switch to it. This should be usable
    as a drop-in replacement for driver.switch_to.alert when the alert box
    may not exist yet.
    @Params
    driver - the webdriver object (required)
    timeout - the time to wait for the alert in seconds
    """

    start_ms = time.time() * 1000.0
    stop_ms = start_ms + (timeout * 1000.0)
    for x in range(int(timeout * 10)):
        try:
            alert = driver.switch_to.alert
            # Raises exception if no alert present
            dummy_variable = alert.text  # noqa
            return alert
        except NoAlertPresentException:
            now_ms = time.time() * 1000.0
            if now_ms >= stop_ms:
                break
            time.sleep(0.1)
    raise Exception("Alert was not present after %s seconds!" % timeout)
<|endoftext|>


<commit_before>import random
import sys
random.seed(None)

class Node:

    species = None
    x = None
    y = None
    health = None

    def __init__(self, spec=None):
        if spec is not None:
            self.species = spec
        else:
            self.species = 0
        self.x = None
        self.y = None
        self.health = None

    def set_location(self, pos_x, pos_y):
        self.x = pos_x
        self.y = pos_y

    def set_species(self, num):
        if num == 0 or num == 1 or num == 2:
            self.species = num
            if num is 0:  # Blank
                self.health = None
            if num is 1:  # Predator
                self.health = 4
            if num is 2:  # Prey
                self.health = 5

        else:
            sys.stderr.write('Species not correct, do not do math with them.')
            self.species = 0

    def prey_eaten(self):
        self.species = 2
        self.health = 4

    def prey_reproduce(self):
        self.species = 1
        self.health = 5

class Map:
    height = 100
    width = 100
    play_board = []

    def __init__(self):
        prey = 0
        predator = 0
        empty = 0
        for x in range(self.width):
            row = []
            for y in range(self.height):
                i = random.randint(0, 10)
                if i <= 5:
                    row.append(Node(2))
                    prey += 1
                elif i <= 7:
                    row.append(Node(1))
                    predator += 1
                else:
                    row.append(Node)
                    empty += 1

            self.play_board.append(row)
        print('Map created')
        print('Prey: ', prey)
        print('Predator: ', predator)
        print('Empty: ', empty)

    def get_board(self):
        return self.play_board

    # This is the main logic for the Predator-Prey simulation
    def turn(self):
        pb = self.play_board
        for x in range(0, self.width, 1):
            for y in range(0, self.height, 1):
                self.check_neighbors(x, y, pb[x][y])

    # TODO Check to see if node is on the edge of the board and wrap
    def get_neighbors(self, x, y):
        top = self.play_board[x][y - 1]
        tr = self.play_board[x + 1][y - 1]
        right = self.play_board[x + 1][y]
        br = self.play_board[x + 1][y + 1]
        bottom = self.play_board[x][y + 1]
        bl = self.play_board[x - 1][y - 1]
        left = self.play_board[x - 1][y]
        tl = self.play_board[x - 1][y - 1]
        neighbors = [top, tr, right, br, bottom, bl, left, tl]
        return neighbors

    def check_neighbors(self, x, y, node):
        if node.species is 0:
            # Check for prey for reproduction
            prey = 0
            neighbors = self.get_neighbors(x, y)
            for neigh in neighbors:
                if neigh.species == 2:
                    prey += 1
            if prey >= 4:
                node.set_species(2)

        if node.species is 1:
            # Check for prey if no prey reduce health/die
            neighbors = self.get_neighbors(x, y)
            prey = []
            for neigh in neighbors:
                if neigh.species == 2:
                    prey.append(neigh)
            if len(prey) == 0:
                node.health -= 1
            else:
                prey[random.randint(0, len(prey))].prey_eaten()
            if node.health == 0:
                node.species = 0

        if node.species is 2:
            # Check for overpopulation/health
            neighbors = self.get_neighbors(x, y)
            prey = []
            open_spots = []
            for neigh in neighbors:
                if neigh.species == 2:
                    prey.append(neigh)
                if neigh.species == 0:
                    open_spots.append(neigh)
            count = len(prey)
            if count == 0 or count >= 6:
                node.health -= 1
                return
            if node.health is None:
                node.species = 0
                return
            if node.health > 2 and len(open_spots) >= 1:
                open_spots[random.randint(0, len(open_spots))].prey_reproduce()
                return
<commit_msg>Made neighbor checking wrap when on edge<commit_after>import random
import sys
random.seed(None)

class Node:

    species = None
    x = None
    y = None
    health = None

    def __init__(self, spec=None):
        if spec is not None:
            self.species = spec
        else:
            self.species = 0
        self.x = None
        self.y = None
        self.health = None

    def set_location(self, pos_x, pos_y):
        self.x = pos_x
        self.y = pos_y

    def set_species(self, num):
        if num == 0 or num == 1 or num == 2:
            self.species = num
            if num is 0:  # Blank
                self.health = None
            if num is 1:  # Predator
                self.health = 4
            if num is 2:  # Prey
                self.health = 5

        else:
            sys.stderr.write('Species not correct, do not do math with them.')
            self.species = 0

    def prey_eaten(self):
        self.species = 2
        self.health = 4

    def prey_reproduce(self):
        self.species = 1
        self.health = 5

class Map:
    height = 100
    width = 100
    play_board = []

    def __init__(self):
        prey = 0
        predator = 0
        empty = 0
        for x in range(self.width):
            row = []
            for y in range(self.height):
                i = random.randint(0, 10)
                if i <= 5:
                    row.append(Node(2))
                    prey += 1
                elif i <= 7:
                    row.append(Node(1))
                    predator += 1
                else:
                    row.append(Node)
                    empty += 1

            self.play_board.append(row)
        print('Map created')
        print('Prey: ', prey)
        print('Predator: ', predator)
        print('Empty: ', empty)

    def get_board(self):
        return self.play_board

    # This is the main logic for the Predator-Prey simulation
    def turn(self):
        pb = self.play_board
        for x in range(0, self.width, 1):
            for y in range(0, self.height, 1):
                self.check_neighbors(x, y, pb[x][y])

    # TODO Check to see if node is on any edge of the board and wrap
    def get_neighbors(self, x, y):
        top = self.play_board[x][(y - 1) % self.height]
        tr = self.play_board[(x + 1) % self.width][(y - 1) % self.height]
        right = self.play_board[(x + 1) % self.width][y]
        br = self.play_board[(x + 1) % self.width][(y + 1) % self.height]
        bottom = self.play_board[x][(y + 1) % self.height]
        bl = self.play_board[(x - 1) % self.width][(y + 1) % self.height]
        left = self.play_board[(x - 1) % self.width][y]
        tl = self.play_board[(x - 1) % self.width][(y - 1) % self.height]

        neighbors = [top, tr, right, br, bottom, bl, left, tl]
        return neighbors

    # TODO get accurate Rules and implement
    def check_neighbors(self, x, y, node):
        if node.species is 0:
            # Check for prey for reproduction
            prey = 0
            neighbors = self.get_neighbors(x, y)
            for neigh in neighbors:
                if neigh.species == 2:
                    prey += 1
            if prey >= 4:
                node.set_species(2)

        if node.species is 1:
            # Check for prey if no prey reduce health/die
            neighbors = self.get_neighbors(x, y)
            prey = []
            for neigh in neighbors:
                if neigh.species == 2:
                    prey.append(neigh)
            if len(prey) == 0:
                node.health -= 1
            else:
                prey[random.randint(0, len(prey))].prey_eaten()
            if node.health == 0:
                node.species = 0

        if node.species is 2:
            # Check for overpopulation/health
            neighbors = self.get_neighbors(x, y)
            prey = []
            open_spots = []
            for neigh in neighbors:
                if neigh.species == 2:
                    prey.append(neigh)
                if neigh.species == 0:
                    open_spots.append(neigh)
            count = len(prey)
            if count == 0 or count >= 6:
                node.health -= 1
                return
            if node.health is None:
                node.species = 0
                return
            if node.health > 2 and len(open_spots) >= 1:
                open_spots[random.randint(0, len(open_spots))].prey_reproduce()
                return
<|endoftext|>


<commit_before>"""
Convert XPath selectors into CSS selectors
"""
import re

_sub_regexes = {
    "tag": r"([a-zA-Z][a-zA-Z0-9]{0,10}|\*)",
    "attribute": r"[.a-zA-Z_:][-\w:.]*(\(\))?)",
    "value": r"\s*[\w/:][-/\w\s,:;.\S]*",
}

_validation_re = (
    r"(?P<node>"
    r"("
    r"^id\([\"\']?(?P<idvalue>%(value)s)[\"\']?\)"
    r"|"
    r"(?P<nav>//?)(?P<tag>%(tag)s)"
    r"(\[("
    r"(?P<matched>(?P<mattr>@?%(attribute)s=[\"\']"
    r"(?P<mvalue>%(value)s))[\"\']"
    r"|"
    r"(?P<contained>contains\((?P<cattr>@?%(attribute)s,\s*[\"\']"
    r"(?P<cvalue>%(value)s)[\"\']\))"
    r")\])?"
    r"(\[(?P<nth>\d+)\])?"
    r")"
    r")" % _sub_regexes
)

prog = re.compile(_validation_re)

class XpathException(Exception):
    pass

def _handle_brackets_in_strings(xpath):
    # Edge Case: Brackets in strings.
    # Example from GitHub.com -
    # '<input type="text" id="user[login]">' => '//*[@id="user[login]"]'
    # Need to tell apart string-brackets from regular brackets
    new_xpath = ""
    chunks = xpath.split('"')
    len_chunks = len(chunks)
    for chunk_num in range(len_chunks):
        if chunk_num % 2 != 0:
            chunks[chunk_num] = chunks[chunk_num].replace(
                "[", "_STR_L_bracket_"
            )
            chunks[chunk_num] = chunks[chunk_num].replace(
                "]", "_STR_R_bracket_"
            )
        new_xpath += chunks[chunk_num]
        if chunk_num != len_chunks - 1:
            new_xpath += '"'
    return new_xpath

def _filter_xpath_grouping(xpath):
    """
    This method removes the outer parentheses for xpath grouping.
    The xpath converter will break otherwise.
    Example:
    "(//button[@type='submit'])[1]" becomes "//button[@type='submit'][1]"
    """

    # First remove the first open parentheses
    xpath = xpath[1:]

    # Next remove the last closed parentheses
    index = xpath.rfind(")")
    index_p1 = index + 1  # Make "flake8" and "black" agree
    if index == -1:
        raise XpathException("Invalid or unsupported Xpath: %s" % xpath)
    xpath = xpath[:index] + xpath[index_p1:]
    return xpath

def _get_raw_css_from_xpath(xpath):
    css = ""
    attr = ""
    position = 0

    while position < len(xpath):
        node = prog.match(xpath[position:])
        if node is None:
            raise XpathException("Invalid or unsupported Xpath: %s" % xpath)
        match = node.groupdict()

        if position != 0:
            nav = " " if match["nav"] == "//" else " > "
        else:
            nav = ""

        tag = "" if match["tag"] == "*" else match["tag"] or ""

        if match["idvalue"]:
            attr = "#%s" % match["idvalue"].replace(" ", "#")
        elif match["matched"]:
            if match["mattr"] == "@id":
                attr = "#%s" % match["mvalue"].replace(" ", "#")
            elif match["mattr"] == "@class":
                attr = ".%s" % match["mvalue"].replace(" ", ".")
            elif match["mattr"] in ["text()", "."]:
                attr = ":contains('%s')" % match["mvalue"]
            elif match["mattr"]:
                attr = '[%s="%s"]' % (
                    match["mattr"].replace("@", ""),
                    match["mvalue"],
                )
        elif match["contained"]:
            if match["cattr"].startswith("@"):
                attr = '[%s*="%s"]' % (
                    match["cattr"].replace("@", ""),
                    match["cvalue"],
                )
            elif match["cattr"] == "text()":
                attr = ':contains("%s")' % match["cvalue"]
            elif match["cattr"] == ".":
                attr = ':contains("%s")' % match["cvalue"]
        else:
            attr = ""

        if match["nth"]:
            nth = ":nth-of-type(%s)" % match["nth"]
        else:
            nth = ""

        node_css = nav + tag + attr + nth
        css += node_css
        position += node.end()
    else:
        css = css.strip()
        return css

def convert_xpath_to_css(xpath):
    xpath = xpath.replace(" = '", "='")

    # **** Start of handling special xpath edge cases instantly ****

    # Handle a special edge case that converts to: 'tag.class:contains("TEXT")'
    c3 = "@class and contains(concat(' ', normalize-space(@class), ' '), ' "
    if c3 in xpath and xpath.count(c3) == 1 and xpath.count("[@") == 1:
        p2 = " ') and (contains(., '"
        if (
            xpath.count(p2) == 1
            and xpath.endswith("'))]")
            and xpath.count("//") == 1
            and xpath.count(" ') and (") == 1
        ):
            s_contains = xpath.split(p2)[1].split("'))]")[0]
            s_tag = xpath.split("//")[1].split("[@class")[0]
            s_class = xpath.split(c3)[1].split(" ') and (")[0]
            return '%s.%s:contains("%s")' % (s_tag, s_class, s_contains)

    # Find instance of: //tag[@attribute='value' and (contains(., 'TEXT'))]
    data = re.match(
        r"""^\s*//(\S+)\[@(\S+)='(\S+)'\s+and\s+"""
        r"""\(contains\(\.,\s'(\S+)'\)\)\]""",
        xpath,
    )
    if data:
        s_tag = data.group(1)
        s_atr = data.group(2)
        s_val = data.group(3)
        s_contains = data.group(4)
        return '%s[%s="%s"]:contains("%s")' % (s_tag, s_atr, s_val, s_contains)

    # Find instance of: //tag[@attribute1='value1' and (@attribute2='value2')]
    data = re.match(
        r"""^\s*//(\S+)\[@(\S+)='(\S+)'\s+and\s+"""
        r"""\(@(\S+)='(\S+)'\)\]""",
        xpath,
    )
    if data:
        s_tag = data.group(1)
        s_atr1 = data.group(2)
        s_val1 = data.group(3)
        s_atr2 = data.group(4)
        s_val2 = data.group(5)
        return '%s[%s="%s"][%s="%s"]' % (s_tag, s_atr1, s_val1, s_atr2, s_val2)

    # **** End of handling special xpath edge cases instantly ****

    if xpath[0] != '"' and xpath[-1] != '"' and xpath.count('"') % 2 == 0:
        xpath = _handle_brackets_in_strings(xpath)
    xpath = xpath.replace("descendant-or-self::*/", "descORself/")
    if len(xpath) > 3:
        xpath = xpath[0:3] + xpath[3:].replace("//", "/descORself/")

    if " and contains(@" in xpath and xpath.count(" and contains(@") == 1:
        spot1 = xpath.find(" and contains(@")
        spot1 = spot1 + len(" and contains(@")
        spot2 = xpath.find(",", spot1)
        attr = xpath[spot1:spot2]
        swap = " and contains(@%s, " % attr
        if swap in xpath:
            swap_spot = xpath.find(swap)
            close_paren = xpath.find("]", swap_spot) - 1
            close_paren_p1 = close_paren + 1  # Make "flake8" and "black" agree
            if close_paren > 1:
                xpath = xpath[:close_paren] + xpath[close_paren_p1:]
                xpath = xpath.replace(swap, "_STAR_=")

    if xpath.startswith("("):
        xpath = _filter_xpath_grouping(xpath)

    css = ""
    if "/descORself/" in xpath and ("@id" in xpath or "@class" in xpath):
        css_sections = []
        xpath_sections = xpath.split("/descORself/")
        for xpath_section in xpath_sections:
            if not xpath_section.startswith("//"):
                xpath_section = "//" + xpath_section
            css_sections.append(_get_raw_css_from_xpath(xpath_section))
        css = "/descORself/".join(css_sections)
    else:
        css = _get_raw_css_from_xpath(xpath)

    attribute_defs = re.findall(r"(\[\w+\=\S+\])", css)
    for attr_def in attribute_defs:
        if (
            attr_def.count("[") == 1
            and attr_def.count("]") == 1
            and attr_def.count("=") == 1
            and attr_def.count('"') == 0
            and attr_def.count("'") == 0
            and attr_def.count(" ") == 0
        ):
            # Now safe to manipulate
            q1 = attr_def.find("=") + 1
            q2 = attr_def.find("]")
            new_attr_def = attr_def[:q1] + "'" + attr_def[q1:q2] + "']"
            css = css.replace(attr_def, new_attr_def)

    # Replace the string-brackets with escaped ones
    css = css.replace("_STR_L_bracket_", "\\[")
    css = css.replace("_STR_R_bracket_", "\\]")

    # Handle a lot of edge cases with conversion
    css = css.replace(" > descORself > ", " ")
    css = css.replace(" descORself > ", " ")
    css = css.replace("/descORself/*", " ")
    css = css.replace("/descORself/", " ")
    css = css.replace("descORself > ", "")
    css = css.replace("descORself/", " ")
    css = css.replace("descORself", " ")
    css = css.replace("_STAR_=", "*=")
    css = css.replace("]/", "] ")
    css = css.replace("] *[", "] > [")
    css = css.replace("'", '"')
    css = css.replace("[@", "[")

    return css
<commit_msg>Better error messages when converting from XPath to CSS<commit_after>"""
Convert XPath selectors into CSS selectors
"""
import re

_sub_regexes = {
    "tag": r"([a-zA-Z][a-zA-Z0-9]{0,10}|\*)",
    "attribute": r"[.a-zA-Z_:][-\w:.]*(\(\))?)",
    "value": r"\s*[\w/:][-/\w\s,:;.\S]*",
}

_validation_re = (
    r"(?P<node>"
    r"("
    r"^id\([\"\']?(?P<idvalue>%(value)s)[\"\']?\)"
    r"|"
    r"(?P<nav>//?)(?P<tag>%(tag)s)"
    r"(\[("
    r"(?P<matched>(?P<mattr>@?%(attribute)s=[\"\']"
    r"(?P<mvalue>%(value)s))[\"\']"
    r"|"
    r"(?P<contained>contains\((?P<cattr>@?%(attribute)s,\s*[\"\']"
    r"(?P<cvalue>%(value)s)[\"\']\))"
    r")\])?"
    r"(\[(?P<nth>\d+)\])?"
    r")"
    r")" % _sub_regexes
)

prog = re.compile(_validation_re)

class XpathException(Exception):
    pass

def _handle_brackets_in_strings(xpath):
    # Edge Case: Brackets in strings.
    # Example from GitHub.com -
    # '<input type="text" id="user[login]">' => '//*[@id="user[login]"]'
    # Need to tell apart string-brackets from regular brackets
    new_xpath = ""
    chunks = xpath.split('"')
    len_chunks = len(chunks)
    for chunk_num in range(len_chunks):
        if chunk_num % 2 != 0:
            chunks[chunk_num] = chunks[chunk_num].replace(
                "[", "_STR_L_bracket_"
            )
            chunks[chunk_num] = chunks[chunk_num].replace(
                "]", "_STR_R_bracket_"
            )
        new_xpath += chunks[chunk_num]
        if chunk_num != len_chunks - 1:
            new_xpath += '"'
    return new_xpath

def _filter_xpath_grouping(xpath, original):
    """
    This method removes the outer parentheses for xpath grouping.
    The xpath converter will break otherwise.
    Example:
    "(//button[@type='submit'])[1]" becomes "//button[@type='submit'][1]"
    """

    # First remove the first open parentheses
    xpath = xpath[1:]

    # Next remove the last closed parentheses
    index = xpath.rfind(")")
    index_p1 = index + 1  # Make "flake8" and "black" agree
    if index == -1:
        raise XpathException(
            "\nInvalid or unsupported XPath:\n%s\n"
            "(Unable to convert XPath Selector to CSS Selector)"
            "" % original
        )
    xpath = xpath[:index] + xpath[index_p1:]
    return xpath

def _get_raw_css_from_xpath(xpath, original):
    css = ""
    attr = ""
    position = 0

    while position < len(xpath):
        node = prog.match(xpath[position:])
        if node is None:
            raise XpathException(
                "\nInvalid or unsupported XPath:\n%s\n"
                "(Unable to convert XPath Selector to CSS Selector)"
                "" % original
            )
        match = node.groupdict()

        if position != 0:
            nav = " " if match["nav"] == "//" else " > "
        else:
            nav = ""

        tag = "" if match["tag"] == "*" else match["tag"] or ""

        if match["idvalue"]:
            attr = "#%s" % match["idvalue"].replace(" ", "#")
        elif match["matched"]:
            if match["mattr"] == "@id":
                attr = "#%s" % match["mvalue"].replace(" ", "#")
            elif match["mattr"] == "@class":
                attr = ".%s" % match["mvalue"].replace(" ", ".")
            elif match["mattr"] in ["text()", "."]:
                attr = ":contains('%s')" % match["mvalue"]
            elif match["mattr"]:
                attr = '[%s="%s"]' % (
                    match["mattr"].replace("@", ""),
                    match["mvalue"],
                )
        elif match["contained"]:
            if match["cattr"].startswith("@"):
                attr = '[%s*="%s"]' % (
                    match["cattr"].replace("@", ""),
                    match["cvalue"],
                )
            elif match["cattr"] == "text()":
                attr = ':contains("%s")' % match["cvalue"]
            elif match["cattr"] == ".":
                attr = ':contains("%s")' % match["cvalue"]
        else:
            attr = ""

        if match["nth"]:
            nth = ":nth-of-type(%s)" % match["nth"]
        else:
            nth = ""

        node_css = nav + tag + attr + nth
        css += node_css
        position += node.end()
    else:
        css = css.strip()
        return css

def convert_xpath_to_css(xpath):
    original = xpath
    xpath = xpath.replace(" = '", "='")

    # **** Start of handling special xpath edge cases instantly ****

    # Handle a special edge case that converts to: 'tag.class:contains("TEXT")'
    c3 = "@class and contains(concat(' ', normalize-space(@class), ' '), ' "
    if c3 in xpath and xpath.count(c3) == 1 and xpath.count("[@") == 1:
        p2 = " ') and (contains(., '"
        if (
            xpath.count(p2) == 1
            and xpath.endswith("'))]")
            and xpath.count("//") == 1
            and xpath.count(" ') and (") == 1
        ):
            s_contains = xpath.split(p2)[1].split("'))]")[0]
            s_tag = xpath.split("//")[1].split("[@class")[0]
            s_class = xpath.split(c3)[1].split(" ') and (")[0]
            return '%s.%s:contains("%s")' % (s_tag, s_class, s_contains)

    # Find instance of: //tag[@attribute='value' and (contains(., 'TEXT'))]
    data = re.match(
        r"""^\s*//(\S+)\[@(\S+)='(\S+)'\s+and\s+"""
        r"""\(contains\(\.,\s'(\S+)'\)\)\]""",
        xpath,
    )
    if data:
        s_tag = data.group(1)
        s_atr = data.group(2)
        s_val = data.group(3)
        s_contains = data.group(4)
        return '%s[%s="%s"]:contains("%s")' % (s_tag, s_atr, s_val, s_contains)

    # Find instance of: //tag[@attribute1='value1' and (@attribute2='value2')]
    data = re.match(
        r"""^\s*//(\S+)\[@(\S+)='(\S+)'\s+and\s+"""
        r"""\(@(\S+)='(\S+)'\)\]""",
        xpath,
    )
    if data:
        s_tag = data.group(1)
        s_atr1 = data.group(2)
        s_val1 = data.group(3)
        s_atr2 = data.group(4)
        s_val2 = data.group(5)
        return '%s[%s="%s"][%s="%s"]' % (s_tag, s_atr1, s_val1, s_atr2, s_val2)

    # **** End of handling special xpath edge cases instantly ****

    if xpath[0] != '"' and xpath[-1] != '"' and xpath.count('"') % 2 == 0:
        xpath = _handle_brackets_in_strings(xpath)
    xpath = xpath.replace("descendant-or-self::*/", "descORself/")
    if len(xpath) > 3:
        xpath = xpath[0:3] + xpath[3:].replace("//", "/descORself/")

    if " and contains(@" in xpath and xpath.count(" and contains(@") == 1:
        spot1 = xpath.find(" and contains(@")
        spot1 = spot1 + len(" and contains(@")
        spot2 = xpath.find(",", spot1)
        attr = xpath[spot1:spot2]
        swap = " and contains(@%s, " % attr
        if swap in xpath:
            swap_spot = xpath.find(swap)
            close_paren = xpath.find("]", swap_spot) - 1
            close_paren_p1 = close_paren + 1  # Make "flake8" and "black" agree
            if close_paren > 1:
                xpath = xpath[:close_paren] + xpath[close_paren_p1:]
                xpath = xpath.replace(swap, "_STAR_=")

    if xpath.startswith("("):
        xpath = _filter_xpath_grouping(xpath, original)

    css = ""
    if "/descORself/" in xpath and ("@id" in xpath or "@class" in xpath):
        css_sections = []
        xpath_sections = xpath.split("/descORself/")
        for xpath_section in xpath_sections:
            if not xpath_section.startswith("//"):
                xpath_section = "//" + xpath_section
            css_sections.append(_get_raw_css_from_xpath(
                xpath_section, original)
            )
        css = "/descORself/".join(css_sections)
    else:
        css = _get_raw_css_from_xpath(xpath, original)

    attribute_defs = re.findall(r"(\[\w+\=\S+\])", css)
    for attr_def in attribute_defs:
        if (
            attr_def.count("[") == 1
            and attr_def.count("]") == 1
            and attr_def.count("=") == 1
            and attr_def.count('"') == 0
            and attr_def.count("'") == 0
            and attr_def.count(" ") == 0
        ):
            # Now safe to manipulate
            q1 = attr_def.find("=") + 1
            q2 = attr_def.find("]")
            new_attr_def = attr_def[:q1] + "'" + attr_def[q1:q2] + "']"
            css = css.replace(attr_def, new_attr_def)

    # Replace the string-brackets with escaped ones
    css = css.replace("_STR_L_bracket_", "\\[")
    css = css.replace("_STR_R_bracket_", "\\]")

    # Handle a lot of edge cases with conversion
    css = css.replace(" > descORself > ", " ")
    css = css.replace(" descORself > ", " ")
    css = css.replace("/descORself/*", " ")
    css = css.replace("/descORself/", " ")
    css = css.replace("descORself > ", "")
    css = css.replace("descORself/", " ")
    css = css.replace("descORself", " ")
    css = css.replace("_STAR_=", "*=")
    css = css.replace("]/", "] ")
    css = css.replace("] *[", "] > [")
    css = css.replace("'", '"')
    css = css.replace("[@", "[")

    return css
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# PyCi
#
# Copyright (c) 2009, The PyCi Project
# Authors: Wu Ke <ngu.kho@gmail.com>
#          Chen Xing <cxcxcxcx@gmail.com>
# URL: <http://code.google.com/p/pyci>
# For license information, see COPYING

"""Trie data structure for fast dictionary look-up
"""

class Trie(object):
    """Trie is a data structure ideal for fast dictionary look-up.
    For details, see http://en.wikipedia.org/wiki/Trie
    """

    def __init__(self, keys, value=lambda x:0):
        """Construct a trie from keys.

        @type keys: random accessible object with hashable elements
        @param keys: keys from which the trie is constructed
        @type value: callable
        @param value: method to get values from keys, default sets
        everything to 0
        """
        self._trie_root = TrieNode(None)
        for i in keys:
            self._trie_root.insert(i, value(i))

    def __getitem__(self, key):
        return self._trie_root.lookup(key)

    def __setitem__(self, key, value):
        return self._trie_root.insert(key, value)

    def longest_prefix(self, seq):
        """Find the longest prefix of seq that is in the trie.

        @type seq: random accessible object with hashable elements
        @param seq: from where the longest prefix will be extracted
        """
        return self._trie_root.longest_prefix(seq)

class TrieNode(object):
    """A node for a trie -- you should use class Trie to access data
    stored here.
    """

    # FIXME: first, rest in insert, longest_prefix, lookup is SLOW

    def __init__(self, label):
        """Construct a trie node with label.

        @type label: hashable
        @param label: the label for this trie node
        """
        self._label = label
        self._child = {}
        self._value = None

    def insert(self, key, value):
        """Insert a node with key, if the node already exists, update
        its value.

        @type key: random accessible object of hashable elements
        @param key: the key for updating
        @type value: anything you like
        @param value: the value for the key
        """
        if not key:
            self._value = value
        else:
            first = key[0]
            rest = key[1:]
            if first not in self._child:
                self._child[first] = TrieNode(first)
            self._child[first].insert(rest, value)

    def lookup(self, key):
        """Lookup a node with key.

        @type key: random accessible object of hashable elements
        @param key: the key for updating
        @return: the value
        """
        if not key:
            return self._value
        else:
            first = key[0]
            rest = key[1:]
            if first not in self._child:
                raise KeyError
            return self._child[first].lookup(rest)

    def longest_prefix(self, seq):
        """Find the longest prefix of seq starting at this node.

        @seq: random accessible object with hashable elements
        @param seq: from where the longest prefix will be extracted
        """
        if not seq:
            return self._label
        else:
            first = seq[0]
            rest = seq[1:]
            if first not in self._child:
                return self._label
            elif self._label:
                return self._label + self._child[first].longest_prefix(rest)
            else:
                return self._child[first].longest_prefix(rest)

if __name__ == "__main__":
    # demo
    from corpus import rmrb
    words = [i for i in rmrb.words()]
    trie = Trie(words)
    sent = raw_input("Give me a sentence: ").decode("utf8")
    pref = trie.longest_prefix(sent)
    print pref
    print trie[pref]
    trie[pref] = 1234
    print trie[pref]
<commit_msg>trie.py: faster<commit_after># -*- coding: utf-8 -*-

# PyCi
#
# Copyright (c) 2009, The PyCi Project
# Authors: Wu Ke <ngu.kho@gmail.com>
#          Chen Xing <cxcxcxcx@gmail.com>
# URL: <http://code.google.com/p/pyci>
# For license information, see COPYING

"""Trie data structure for fast dictionary look-up
"""

class Trie(object):
    """Trie is a data structure ideal for fast dictionary look-up.
    For details, see http://en.wikipedia.org/wiki/Trie
    """

    def __init__(self, keys, value=lambda x:0):
        """Construct a trie from keys.

        @type keys: random accessible object with hashable elements
        @param keys: keys from which the trie is constructed
        @type value: callable
        @param value: method to get values from keys, default sets
        everything to 0
        """
        self._trie_root = TrieNode(None)
        for i in keys:
            self[i] = value(i)

    def __getitem__(self, key):
        return self._trie_root.lookup(key, 0)

    def __setitem__(self, key, value):
        return self._trie_root.insert(key, 0, value)

    def longest_prefix(self, seq):
        """Find the longest prefix of seq that is in the trie.

        @type seq: random accessible object with hashable elements
        @param seq: from where the longest prefix will be extracted
        """
        return self._trie_root.longest_prefix(seq, 0)

class TrieNode(object):
    """A node for a trie -- you should use class Trie to access data
    stored here.
    """

    def __init__(self, label):
        """Construct a trie node with label.

        @type label: hashable
        @param label: the label for this trie node
        """
        self._label = label
        self._child = {}
        self._value = None

    def insert(self, key, offset, value):
        """Insert a node with key, if the node already exists, update
        its value.

        @type key: random accessible object of hashable elements
        @param key: the key for updating
        @type offset: non-negative interge
        @param offset: starting part of actual key
        @type value: anything you like
        @param value: the value for the key
        """
        if offset == len(key):
            self._value = value
        else:
            first = key[offset]
            if first not in self._child:
                self._child[first] = TrieNode(first)
            self._child[first].insert(key, offset + 1, value)

    def lookup(self, key, offset):
        """Lookup a node with key.

        @type key: random accessible object of hashable elements
        @param key: the key for updating
        @type offset: non-negative interge
        @param offset: starting part of actual key
        @return: the value
        """
        if offset == len(key):
            return self._value
        else:
            first = key[offset]
            if first not in self._child:
                raise KeyError
            return self._child[first].lookup(key, offset + 1)

    def longest_prefix(self, seq, offset):
        """Find the longest prefix of seq starting at this node.

        @seq: random accessible object with hashable elements
        @param seq: from where the longest prefix will be extracted
        @type offset: non-negative interge
        @param offset: starting part of actual key
        """
        if offset == len(seq):
            return self._label
        else:
            first = seq[offset]
            if first not in self._child:
                return self._label
            elif self._label:
                return self._label + self._child[first].longest_prefix(seq, offset + 1)
            else:
                return self._child[first].longest_prefix(seq, offset + 1)

if __name__ == "__main__":
    # demo
    from corpus import rmrb
    words = [i for i in rmrb.words()]
    trie = Trie(words)
    sent = raw_input("Give me a sentence: ").decode("utf8")
    pref = trie.longest_prefix(sent)
    print pref
    print trie[pref]
    trie[pref] = 1234
    print trie[pref]
<|endoftext|>