<commit_before>#
#  Tachyon
#  Parser.py
#
#  Created on 03/06/17
#  Ryan Maugin <ryan.maugin@adacollege.org.uk>
#

import constants # for constants like tachyon keywords and datatypes

class Parser(object):

    def __init__(self, token_stream):
        # Complete Abstract Syntax tree
        self.source_ast = { 'main_scope': [] }
        # Symbol table fo variable semantical analysis
        self.symbol_tree = [ ['last_name', 'Maugin'], ['test', 'test101']]
        # This will hold all error messages
        self.error_messages = []
        # This will hold all the tokens
        self.token_stream = token_stream
        # This will hold the token index we are parsing at
        self.token_index = 0

    def parse(self, token_stream):
        """ Parsing

        This will parse the tokens given as argument and turn the sequence of tokens into
        abstract syntax trees

        Args:
         token_stream (list) : The tokens produced by lexer
        """

        # Loop through each token
        while self.token_index < len(token_stream):

            # Set the token values in variables for clearer and easier debugging and readability
            token_type = token_stream[self.token_index][0]
            token_value = token_stream[self.token_index][1]

            # This will find the token pattern for a variable decleration
            if token_type == "DATATYPE":
                self.variable_decleration_parsing(token_stream[self.token_index:len(token_stream)])

            # This will find the token pattern for an if statement
            elif token_type == "IDENTIFIER" and token_value == "if":
                self.conditional_statement_parser(token_stream[self.token_index:len(token_stream)])

            self.token_index += 1

        # Check if there were any errors and if so display them all
        if self.error_messages != []: self.send_error_message(self.error_messages)

    def variable_decleration_parsing(self, token_stream):
        """ Variable Decleration Parsing

        This method will parse variable declerations and add themto the source AST

        Args:
            token_stream (list) : The token stream starting from where var decleration was found
        """

        ast = { 'VariableDecleration': [] }  # The abstract syntax tree for var decl
        tokens_checked = 0                   # Number of token checked that made up the var decl

        for x in range(0, len(token_stream)):

            # Create variables for identifying token type and value more easily
            token_type = token_stream[x][0]
            token_value = token_stream[x][1]

            # Skip the '=' operator in var decl
            if x == 2 and token_type == "OPERATOR" and token_value == "=":
                pass
            # This will handle error detection for making sure the '=' is found
            if x == 2 and token_type != "OPERATOR" and token_value != "=":
                self.error_messages.append(["Variable Decleration Missing '='.", self.token_stream[self.token_index:self.token_index + tokens_checked + 1] ])

            # If a statement end is found then break out parsing
            if token_stream[x][0] == "STATEMENT_END": break

            # This will parse the first token which will be the var type
            if x == 0: ast['VariableDecleration'].append({ "type": token_value })

            # This will parse the second token which will be the name of the var
            if x == 1 and token_type == "IDENTIFIER":

                # Check if a variable has already been named the same and is so send an error
                if self.get_variable_value(token_value) != False:
                    self.error_messages.append(["Variable '%s' already exists and cannot be defined again!" % token_value, self.token_stream[self.token_index:self.token_index + tokens_checked + 1] ])
                else:
                    ast['VariableDecleration'].append({ "name": token_value })

            # Error handling for variable name to make sure the naming convention is acceptable
            if x == 1 and token_type != "IDENTIFIER":
                self.error_messages.append(["Invalid Variable Name '%s'" % token_value, self.token_stream[self.token_index:self.token_index + tokens_checked + 1] ])

            # This will parse the 3rd token which is the value of the variable
            if x == 3 and token_stream[x + 1][0] == "STATEMENT_END":

                # Check if the value matches the variable defined type
                if type(eval(token_value)) == eval(token_stream[0][1]):
                    # Add value as a number not a string if it is an int or else add it as a string
                    try: ast['VariableDecleration'].append({ "value": int(token_value) })
                    except ValueError: ast['VariableDecleration'].append({ "value": token_value })
                else:
                    self.error_messages.append(["Variable value does not match defined type!",
                                                self.token_stream[self.token_index:self.token_index + tokens_checked + 1] ])

            # This will parse any variable declerations which have concatenation or arithmetics
            elif x >= 3:

                # Holds the list of ints and perands that will be passed to equation parser
                value_list = []

                for equation_item in range(x, len(token_stream)):
                    # If there is an end statement then break because the var decl is done
                    if token_stream[equation_item][0] == "STATEMENT_END": break

                    # Try to append item as int not string if you can
                    try:               value_list.append(int(token_stream[equation_item][1]))
                    except ValueError: value_list.append(token_stream[equation_item][1])

                    tokens_checked += 1 # Indent the tokens checked within this for loop

                # Call the equation parser and append value returned or try concat parser if an error occurs
                try: ast['VariableDecleration'].append({ "value": self.equation_parser(value_list)})
                except:
                    try:    ast['VariableDecleration'].append({ "value": self.concatenation_parser(value_list) })
                    except: self.error_messages.append(["Invalid variable decleration!", self.token_stream[self.token_index:self.token_index + tokens_checked] ])
                break                   # Break out of the current var parsing loop since we just parsed everything

            tokens_checked += 1         # Indent within overall for loop

        # Last case error validation checking if all needed var decl elements are in ast such as:
        # var type, name and value
        try: ast['VariableDecleration'][0]
        except: self.error_messages.append(["Invalid variable decleration coud not set variable type!", self.token_stream[self.token_index:self.token_index + tokens_checked] ])
        try: ast['VariableDecleration'][1]
        except: self.error_messages.append(["Invalid variable decleration coud not set variable name!", self.token_stream[self.token_index:self.token_index + tokens_checked] ])
        try: ast['VariableDecleration'][2]
        except: self.error_messages.append(["Invalid variable decleration coud not set variable value!", self.token_stream[self.token_index:self.token_index + tokens_checked] ])

        print(ast)
        self.source_ast['main_scope'].append(ast)
        self.token_index += tokens_checked

    def conditional_statement_parser(self, token_stream):
        """ Conditional Statement Parser

        This will parse conditional statements like 'if else' and create an
        abstract sytax tree for it.

        args:
            token_stream: tokens which make up the conditional statement
        """

        tokens_checked = 0
        ast = {'ConditionalStatement': []}

        # This loop will parse the condition e.g. if 12 < 11
        for x in range(0, len(token_stream)):
            tokens_checked += 1

            # Simplification variables that will improve readbility
            token_type  = token_stream[x][0]
            token_value = token_stream[x][1]
            allowed_conditional_token_types = ['INTEGER', 'STRING', 'IDENTIFIER']

            # Break out of loop at the end of the condition
            if token_type == 'SCOPE_DEFINER' and token_value == '{': break

            # Pass if token is the 'if' identifier as it has already been checked
            if token_type == 'IDENTIFIER' and  token_value == 'if':  pass

            # This will check for the first value and add it to the AST
            if x == 1 and token_type in allowed_conditional_token_types:
                # This will check for an identifier (var) and then check if it exists so it can add the value to it
                if self.get_variable_value(token_value) != False:
                    ast['ConditionalStatement'].append( {'value1': self.get_variable_value(token_value)} )
                else:
                    ast['ConditionalStatement'].append( {'value1': token_value} )

            # This will check for the comparison operator and add it to the AST
            if x == 2 and token_type == 'COMPARISON_OPERATOR':
                ast['ConditionalStatement'].append( {'comparison_type': token_value} )

            # This will check for the second value and add it to the AST
            if x == 3 and token_type in allowed_conditional_token_types:
                # This will check for an identifier (var) and then check if it exists so it can add the value to it
                if self.get_variable_value(token_value) != False:
                    ast['ConditionalStatement'].append( {'value2': self.get_variable_value(token_value)} )
                else:
                    ast['ConditionalStatement'].append( {'value2': token_value} )

        # Get condition statament details
        comparison_type = ast['ConditionalStatement'][1]['comparison_type']
        values          = [ ast['ConditionalStatement'][0]['value1'], ast['ConditionalStatement'][2]['value2'] ]

        # Check if condition is true or false and add result to AST
        if self.perform_conditional_checks(comparison_type, values, tokens_checked):
            ast['ConditionalStatement'].append( {'evaluatesTo': True} )
        else:
            ast['ConditionalStatement'].append( {'evaluatesTo': False} )

        print(ast)

    def perform_conditional_checks(self, comparison_type, values, tokens_checked):
        """ Perform Conditional Checks

        This will perform the condtitional checks and see whether the condition evaluates
        to true or false

        args:
            comparison_type (str) : The comparison operator e.g ==, < or >=
            values         (list) : The values that comparison will be applied on
        return:
            boolean               : True or False based on condition evaluation
        """

        if comparison_type == '==':
            if values[0] == values[1]: return True
            else: return False
        elif comparison_type == '!=':
            if values[0] != values[1]: return False
        elif comparison_type == '>':
            try:
                if int(values[0]) > int(values[1]): return True
                else: return False
            except: self.error_messages.append(["ERROR: Cannot perform comparison check '>' on string values",
                                               self.token_stream[self.token_index:self.token_index + tokens_checked] ])
        elif comparison_type == '<':
            try:
                if int(values[0]) < int(values[1]): return True
                else: return False
            except: self.error_messages.append(["ERROR: Cannot perform comparison check '<' on string values",
                                               self.token_stream[self.token_index:self.token_index + tokens_checked] ])
        elif comparison_type == '>=':
            try:
                if int(values[0]) >= int(values[1]): return True
                else: return False
            except: self.error_messages.append(["ERROR: Cannot perform comparison check '>=' on string values",
                                               self.token_stream[self.token_index:self.token_index + tokens_checked] ])
        elif comparison_type == '<=':
            try:
                if int(values[0]) <= int(values[1]): return True
                else: return False
            except: self.error_messages.append(["ERROR: Cannot perform comparison check '<=' on string values",
                                               self.token_stream[self.token_index:self.token_index + tokens_checked] ])

    def equation_parser(self, equation):
        """ Equation parsing

        This will parse equations such as 10 * 10 which comes in as an array with nums
        and operands

        args:
            equation (list) : List of the ints and operands in order
        returns:
            value (int)     : The value of the equation
        """
        total = 0 # Holds equation value

        for item in range(0, len(equation)):

            # Add first value to total as a starting int to perform calculatios on
            if item == 0:
                total += equation[item]
                pass

            # This will check every operator and perform the right calculations based on total
            # and the number that is after the operator
            if item % 2 == 1:
                if equation[item] == "+": total += equation[item + 1]
                elif equation[item] == "-": total += equation[item + 1]
                elif equation[item] == "/": total /= equation[item + 1]
                elif equation[item] == "*": total *= equation[item + 1]
                elif equation[item] == "%": total %= equation[item + 1]
                else: self.error_messages.append(["Error parsing equation, check that you are using correct operand",
                                                 equation])

            # Skip every number since we already check and use them
            elif item % 2 == 0: pass

        return total

    def concatenation_parser(self, concatenation_list):
        """ Concatenaion Parser

        This will parse concatenation of strings and variables with string values or integer
        values to concatenate arithmetics to strings e.g. "Ryan is ", 10 + 6, "!"

        args:
            concatenation_list (list) : Array with all items needed seperated to perform concatenation
        return:
            value (string)            : Full string after concatenation done
            error (list)              : Return False with an error message in a list
        """

        full_string = ""

        for item in range(0, len(concatenation_list)):

            current_value = concatenation_list[item]

            # Add the first item to the string
            if item == 0:

                # This checks if the value being checked is a string or a variable
                # If it is a string then just add it without the surrounding quotes
                if current_value[0] == '"': full_string += current_value[1:len(current_value) - 1]
                # If it isn't a string then get the variable value and append it
                else: full_string += self.get_variable_value(current_value)
                pass

            # This will check for the concatenation operator
            if item % 2 == 1:
                if current_value == "+":
                    # This checks if the value being checked is a string or a variable
                    if concatenation_list[item + 1][0] != '"': full_string += self.get_variable_value(concatenation_list[item + 1])
                    else: full_string += concatenation_list[item + 1][1:len(concatenation_list[item + 1]) - 1]

                elif current_value == ",":
                    full_string += " " + concatenation_list[item + 1]

                else: self.error_messages.append(["Error parsing equation, check that you are using correct operand",
                                                 concatenation_list])

            # This will skip value as it is already being added and dealt with when getting the operand
            if item % 2 == 0: pass

        return full_string

    def get_variable_value(self, name):
        """ Get Variable Value

        This will get the value of a variable from the symbol tree and return the value
        if the variable exists or an error if it doesn't

        args:
            name (string)  : The name which we will search for in symbol tree
        returns:
            value (string) : The value of the variable if it is found
            error (bool)   : Sends back False if it was not found
        """

        for var in self.symbol_tree:
            if var[0] == name: return var[1]
        return False

    def send_error_message(self, error_list):
        """ Send Error Messages

        This will simply send all the found error messages within the source code
        and return a list of error messages and tokens of which part of the source code
        caused that error

        args:
            error_list (list) : List with error message and tokens
        """

        print("------------------------ %s ERROR'S FOUND ------------------------" % len(error_list))
        for error in range(0, len(error_list)):
            print("%s)  %s" % (error, error_list[error][0]))
            print('    ' + '\033[91m' + " ".join(str(x) for x in error_list[error][1]) + '\033[0m')
        print("-----------------------------------------------------------------")
        quit()<commit_msg>Created 'parse_body' method for parsing statements body<commit_after>#
#  Tachyon
#  Parser.py
#
#  Created on 03/06/17
#  Ryan Maugin <ryan.maugin@adacollege.org.uk>
#

import constants # for constants like tachyon keywords and datatypes

class Parser(object):

    def __init__(self, token_stream):
        # Complete Abstract Syntax tree
        self.source_ast = { 'main_scope': [] }
        # Symbol table fo variable semantical analysis
        self.symbol_tree = [ ['last_name', 'Maugin'], ['test', 'test101']]
        # This will hold all error messages
        self.error_messages = []
        # This will hold all the tokens
        self.token_stream = token_stream
        # This will hold the token index we are parsing at
        self.token_index = 0

    def parse(self, token_stream):
        """ Parsing

        This will parse the tokens given as argument and turn the sequence of tokens into
        abstract syntax trees

        Args:
         token_stream (list) : The tokens produced by lexer
        """

        # Loop through each token
        while self.token_index < len(token_stream):

            # Set the token values in variables for clearer and easier debugging and readability
            token_type = token_stream[self.token_index][0]
            token_value = token_stream[self.token_index][1]

            # This will find the token pattern for a variable decleration
            if token_type == "DATATYPE":
                self.variable_decleration_parsing(token_stream[self.token_index:len(token_stream)])

            # This will find the token pattern for an if statement
            elif token_type == "IDENTIFIER" and token_value == "if":
                self.conditional_statement_parser(token_stream[self.token_index:len(token_stream)])

            self.token_index += 1

        # Check if there were any errors and if so display them all
        if self.error_messages != []: self.send_error_message(self.error_messages)

    def variable_decleration_parsing(self, token_stream):
        """ Variable Decleration Parsing

        This method will parse variable declerations and add themto the source AST

        Args:
            token_stream (list) : The token stream starting from where var decleration was found
        """

        ast = { 'VariableDecleration': [] }  # The abstract syntax tree for var decl
        tokens_checked = 0                   # Number of token checked that made up the var decl

        for x in range(0, len(token_stream)):

            # Create variables for identifying token type and value more easily
            token_type = token_stream[x][0]
            token_value = token_stream[x][1]

            # Skip the '=' operator in var decl
            if x == 2 and token_type == "OPERATOR" and token_value == "=":
                pass
            # This will handle error detection for making sure the '=' is found
            if x == 2 and token_type != "OPERATOR" and token_value != "=":
                self.error_messages.append(["Variable Decleration Missing '='.", self.token_stream[self.token_index:self.token_index + tokens_checked + 1] ])

            # If a statement end is found then break out parsing
            if token_stream[x][0] == "STATEMENT_END": break

            # This will parse the first token which will be the var type
            if x == 0: ast['VariableDecleration'].append({ "type": token_value })

            # This will parse the second token which will be the name of the var
            if x == 1 and token_type == "IDENTIFIER":

                # Check if a variable has already been named the same and is so send an error
                if self.get_variable_value(token_value) != False:
                    self.error_messages.append(["Variable '%s' already exists and cannot be defined again!" % token_value, self.token_stream[self.token_index:self.token_index + tokens_checked + 1] ])
                else:
                    ast['VariableDecleration'].append({ "name": token_value })

            # Error handling for variable name to make sure the naming convention is acceptable
            if x == 1 and token_type != "IDENTIFIER":
                self.error_messages.append(["Invalid Variable Name '%s'" % token_value, self.token_stream[self.token_index:self.token_index + tokens_checked + 1] ])

            # This will parse the 3rd token which is the value of the variable
            if x == 3 and token_stream[x + 1][0] == "STATEMENT_END":

                # Check if the value matches the variable defined type
                if type(eval(token_value)) == eval(token_stream[0][1]):
                    # Add value as a number not a string if it is an int or else add it as a string
                    try: ast['VariableDecleration'].append({ "value": int(token_value) })
                    except ValueError: ast['VariableDecleration'].append({ "value": token_value })
                else:
                    self.error_messages.append(["Variable value does not match defined type!",
                                                self.token_stream[self.token_index:self.token_index + tokens_checked + 1] ])

            # This will parse any variable declerations which have concatenation or arithmetics
            elif x >= 3:

                # Holds the list of ints and perands that will be passed to equation parser
                value_list = []

                for equation_item in range(x, len(token_stream)):
                    # If there is an end statement then break because the var decl is done
                    if token_stream[equation_item][0] == "STATEMENT_END": break

                    # Try to append item as int not string if you can
                    try:               value_list.append(int(token_stream[equation_item][1]))
                    except ValueError: value_list.append(token_stream[equation_item][1])

                    tokens_checked += 1 # Indent the tokens checked within this for loop

                # Call the equation parser and append value returned or try concat parser if an error occurs
                try: ast['VariableDecleration'].append({ "value": self.equation_parser(value_list)})
                except:
                    try:    ast['VariableDecleration'].append({ "value": self.concatenation_parser(value_list) })
                    except: self.error_messages.append(["Invalid variable decleration!", self.token_stream[self.token_index:self.token_index + tokens_checked] ])
                break                   # Break out of the current var parsing loop since we just parsed everything

            tokens_checked += 1         # Indent within overall for loop

        # Last case error validation checking if all needed var decl elements are in ast such as:
        # var type, name and value
        try: ast['VariableDecleration'][0]
        except: self.error_messages.append(["Invalid variable decleration coud not set variable type!", self.token_stream[self.token_index:self.token_index + tokens_checked] ])
        try: ast['VariableDecleration'][1]
        except: self.error_messages.append(["Invalid variable decleration coud not set variable name!", self.token_stream[self.token_index:self.token_index + tokens_checked] ])
        try: ast['VariableDecleration'][2]
        except: self.error_messages.append(["Invalid variable decleration coud not set variable value!", self.token_stream[self.token_index:self.token_index + tokens_checked] ])

        print(ast)
        self.source_ast['main_scope'].append(ast)
        self.token_index += tokens_checked

    def conditional_statement_parser(self, token_stream):
        """ Conditional Statement Parser

        This will parse conditional statements like 'if else' and create an
        abstract sytax tree for it.

        args:
            token_stream: tokens which make up the conditional statement
        """

        tokens_checked = 0
        ast = {'ConditionalStatement': []}

        # This loop will parse the condition e.g. if 12 < 11
        for x in range(0, len(token_stream)):
            tokens_checked += 1

            # Simplification variables that will improve readbility
            token_type  = token_stream[x][0]
            token_value = token_stream[x][1]
            allowed_conditional_token_types = ['INTEGER', 'STRING', 'IDENTIFIER']

            # Break out of loop at the end of the condition
            if token_type == 'SCOPE_DEFINER' and token_value == '{': break

            # Pass if token is the 'if' identifier as it has already been checked
            if token_type == 'IDENTIFIER' and  token_value == 'if':  pass

            # This will check for the first value and add it to the AST
            if x == 1 and token_type in allowed_conditional_token_types:
                # This will check for an identifier (var) and then check if it exists so it can add the value to it
                if self.get_variable_value(token_value) != False:
                    ast['ConditionalStatement'].append( {'value1': self.get_variable_value(token_value)} )
                else:
                    ast['ConditionalStatement'].append( {'value1': token_value} )

            # This will check for the comparison operator and add it to the AST
            if x == 2 and token_type == 'COMPARISON_OPERATOR':
                ast['ConditionalStatement'].append( {'comparison_type': token_value} )

            # This will check for the second value and add it to the AST
            if x == 3 and token_type in allowed_conditional_token_types:
                # This will check for an identifier (var) and then check if it exists so it can add the value to it
                if self.get_variable_value(token_value) != False:
                    ast['ConditionalStatement'].append( {'value2': self.get_variable_value(token_value)} )
                else:
                    ast['ConditionalStatement'].append( {'value2': token_value} )

        # Get condition statament details
        comparison_type = ast['ConditionalStatement'][1]['comparison_type']
        values          = [ ast['ConditionalStatement'][0]['value1'], ast['ConditionalStatement'][2]['value2'] ]

        # Check if condition is true or false and add result to AST
        if self.perform_conditional_checks(comparison_type, values, tokens_checked):
            ast['ConditionalStatement'].append( {'evaluatesTo': True} )
        else:
            ast['ConditionalStatement'].append( {'evaluatesTo': False} )

        print(ast)

    def parse_body(self, token_stream):
        """ Parse body

        This will parse the body of conditional, iteration, functions and more in order
        to return a body ast like this --> {'body': []}

        args:
            token_stream (list) : tokens which make up the body
        returns:
             ast       (object) : Abstract Syntax Tree of the body
        """
        print('Parsing body')

    def perform_conditional_checks(self, comparison_type, values, tokens_checked):
        """ Perform Conditional Checks

        This will perform the condtitional checks and see whether the condition evaluates
        to true or false

        args:
            comparison_type (str) : The comparison operator e.g ==, < or >=
            values         (list) : The values that comparison will be applied on
        return:
            boolean               : True or False based on condition evaluation
        """

        if comparison_type == '==':
            if values[0] == values[1]: return True
            else: return False
        elif comparison_type == '!=':
            if values[0] != values[1]: return False
        elif comparison_type == '>':
            try:
                if int(values[0]) > int(values[1]): return True
                else: return False
            except: self.error_messages.append(["ERROR: Cannot perform comparison check '>' on string values",
                                               self.token_stream[self.token_index:self.token_index + tokens_checked] ])
        elif comparison_type == '<':
            try:
                if int(values[0]) < int(values[1]): return True
                else: return False
            except: self.error_messages.append(["ERROR: Cannot perform comparison check '<' on string values",
                                               self.token_stream[self.token_index:self.token_index + tokens_checked] ])
        elif comparison_type == '>=':
            try:
                if int(values[0]) >= int(values[1]): return True
                else: return False
            except: self.error_messages.append(["ERROR: Cannot perform comparison check '>=' on string values",
                                               self.token_stream[self.token_index:self.token_index + tokens_checked] ])
        elif comparison_type == '<=':
            try:
                if int(values[0]) <= int(values[1]): return True
                else: return False
            except: self.error_messages.append(["ERROR: Cannot perform comparison check '<=' on string values",
                                               self.token_stream[self.token_index:self.token_index + tokens_checked] ])

    def equation_parser(self, equation):
        """ Equation parsing

        This will parse equations such as 10 * 10 which comes in as an array with nums
        and operands

        args:
            equation (list) : List of the ints and operands in order
        returns:
            value (int)     : The value of the equation
        """
        total = 0 # Holds equation value

        for item in range(0, len(equation)):

            # Add first value to total as a starting int to perform calculatios on
            if item == 0:
                total += equation[item]
                pass

            # This will check every operator and perform the right calculations based on total
            # and the number that is after the operator
            if item % 2 == 1:
                if equation[item] == "+": total += equation[item + 1]
                elif equation[item] == "-": total += equation[item + 1]
                elif equation[item] == "/": total /= equation[item + 1]
                elif equation[item] == "*": total *= equation[item + 1]
                elif equation[item] == "%": total %= equation[item + 1]
                else: self.error_messages.append(["Error parsing equation, check that you are using correct operand",
                                                 equation])

            # Skip every number since we already check and use them
            elif item % 2 == 0: pass

        return total

    def concatenation_parser(self, concatenation_list):
        """ Concatenaion Parser

        This will parse concatenation of strings and variables with string values or integer
        values to concatenate arithmetics to strings e.g. "Ryan is ", 10 + 6, "!"

        args:
            concatenation_list (list) : Array with all items needed seperated to perform concatenation
        return:
            value (string)            : Full string after concatenation done
            error (list)              : Return False with an error message in a list
        """

        full_string = ""

        for item in range(0, len(concatenation_list)):

            current_value = concatenation_list[item]

            # Add the first item to the string
            if item == 0:

                # This checks if the value being checked is a string or a variable
                # If it is a string then just add it without the surrounding quotes
                if current_value[0] == '"': full_string += current_value[1:len(current_value) - 1]
                # If it isn't a string then get the variable value and append it
                else: full_string += self.get_variable_value(current_value)
                pass

            # This will check for the concatenation operator
            if item % 2 == 1:
                if current_value == "+":
                    # This checks if the value being checked is a string or a variable
                    if concatenation_list[item + 1][0] != '"': full_string += self.get_variable_value(concatenation_list[item + 1])
                    else: full_string += concatenation_list[item + 1][1:len(concatenation_list[item + 1]) - 1]

                elif current_value == ",":
                    full_string += " " + concatenation_list[item + 1]

                else: self.error_messages.append(["Error parsing equation, check that you are using correct operand",
                                                 concatenation_list])

            # This will skip value as it is already being added and dealt with when getting the operand
            if item % 2 == 0: pass

        return full_string

    def get_variable_value(self, name):
        """ Get Variable Value

        This will get the value of a variable from the symbol tree and return the value
        if the variable exists or an error if it doesn't

        args:
            name (string)  : The name which we will search for in symbol tree
        returns:
            value (string) : The value of the variable if it is found
            error (bool)   : Sends back False if it was not found
        """

        for var in self.symbol_tree:
            if var[0] == name: return var[1]
        return False

    def send_error_message(self, error_list):
        """ Send Error Messages

        This will simply send all the found error messages within the source code
        and return a list of error messages and tokens of which part of the source code
        caused that error

        args:
            error_list (list) : List with error message and tokens
        """

        print("------------------------ %s ERROR'S FOUND ------------------------" % len(error_list))
        for error in range(0, len(error_list)):
            print("%s)  %s" % (error, error_list[error][0]))
            print('    ' + '\033[91m' + " ".join(str(x) for x in error_list[error][1]) + '\033[0m')
        print("-----------------------------------------------------------------")
        quit()<|endoftext|>


<commit_before>#
#  Tachyon
#  parser.py
#
#  Created on 03/06/17
#  Ryan Maugin <ryan.maugin@adacollege.org.uk>
#

from ast import literal_eval # To perform ast literal eval to figure out a strings data type
import constants # for constants like tachyon keywords and datatypes

class Parser(object):

    def __init__(self, token_stream):
        # Complete Abstract Syntax tree
        self.source_ast = [{ 'main_scope': [] }]
        # Symbol table fo variable semantical analysis
        self.symbol_tree = []
        # This will hold all the tokens
        self.token_stream = token_stream

    def parse(self, token_stream):
        """ Parsing

        This will parse the tokens given as argument and turn the sequence of tokens into
        abstract syntax trees

        Args:
         token_stream (list) : The tokens produced by lexer
        """
        print('---------------------------------------------')
        print(token_stream)
        print('---------------------------------------------')

        # This will hold the token index we are parsing at
        token_index = 0

        # Loop through each token
        while token_index < len(token_stream):

            # Set the token values in variables for clearer and easier debugging and readability
            token_type = token_stream[token_index][0]
            token_value = token_stream[token_index][1]

            # This will check for an if statement token
            #if token_type == 'IDENTIFIER' and token_value.lower() == 'if':
            #    source_ast['main_scope'][0].append(self.parse_if_statement(token_stream[token_index:len(token_stream)]))

            # This will parse for a vraible decleration token
            if token_type == 'DATATYPE' and token_value.lower() in constants.DATATYPE:
                self.parse_variable_decleration(token_stream[token_index:len(token_stream)], token_index)

            # Increment token index by 1 when a loop finishes
            token_index += 1

        print(self.source_ast)
        print(self.symbol_tree)

    def parse_if_statement(self, token_stream):
        """ Parsing If Statement

        This will parse through an if statement and create it's abstract tree and handle any
        syntax error

        Args:
            token_stream (list) : List of tokens starting from where if statement was found

        Returns:
            AST : The if statement abstract syntax tree
        """
        print("IF STATEMENT")

    def find_variable_scope(self):
        """ Find/set variable scope

        This will set or find the scope of a variable being declared or called
        so that it can create a list of priorotised variables which should be called.
        """
        print('Find it by looping through source_ast and sorting through _scope tagged names')

    def parse_variable_decleration(self, token_stream, found_at_index):
        """ Parsing Variable decleration

        This will parse through a variable decleration and create it's abstract tree and handle any
        syntax errors

        Args:
            token_stream (list) : List of tokens starting from where if statement was found

        Returns:
            AST : The variable decleration abstract syntax tree
        """

        # Will hold the vraible decleration abstract syntax tree being built
        ast = []

        # Keeps track of the index within variable decleration
        index = 0

        for item in token_stream:

            # This will add one every loop to the index of the var decleration
            index += 1

            # This will get the variable type and add it to the AST
            if index == 1: ast.append({ 'VariableDeclerator': [ {'type': item[1]} ]})

            # This will check for the variable name
            if index == 2:
                # This will check to make sure that the name of the doesn't start wih a number
                if not item[1][0].isdigit(): ast[0]['VariableDeclerator'].append({ 'name': item[1] })
                # This will print an error if variable begins with a number
                else: print('Illegal Variable Name "' + item[1] + '" variable name cannot begind with a number')

            # This will check for equal sign
            if index == 3:
                if item[1] == '=': pass
                else: print("SyntaxError: An equal sign '=' was excpexted in variable decleration")

            # This will check the variable value but will skip the equal sign
            if index >= 4 and item[1] != ';':
                # TODO Add the variable to the symbol tree
                # TODO Modify this code to allow for more complex var declerations
                # Check if the value is the same value as the datatype in decleration
                if str(type(literal_eval(item[1]))) == "<class " + "'" + ast[0]['VariableDeclerator'][0]['type'] + "'>":
                    ast[0]['VariableDeclerator'].append({ 'value': item[1] })
                else:
                    print("TypeError: Variable value does not conform to data type of " + str(type(literal_eval(item[1]))))

            # If the for loop reaches the end statement then break because it is the end of the var decleration
            if item[1] == ';': break

        # Append this var declerating ast to the complete source ast and symbol table
        self.source_ast[0]['main_scope'].append(ast[0])
        self.symbol_tree.append( [ ast[0]['VariableDeclerator'][1]['name'], ast[0]['VariableDeclerator'][2]['value'] ] )

    def parse_print(self, token_stream):
        """ Parsing print

        This will parse through a variable decleration and create it's abstract tree and handle any
        syntax errors

        Args:
            token_stream (list) : List of tokens starting from where if statement was found

        Returns:
            AST : The variable decleration abstract syntax tree
        """
        print("PRINT")<commit_msg>Added semantical analysis to check if declared var alraedy exists<commit_after>#
#  Tachyon
#  parser.py
#
#  Created on 03/06/17
#  Ryan Maugin <ryan.maugin@adacollege.org.uk>
#

from ast import literal_eval # To perform ast literal eval to figure out a strings data type
import constants # for constants like tachyon keywords and datatypes

class Parser(object):

    def __init__(self, token_stream):
        # Complete Abstract Syntax tree
        self.source_ast = [{ 'main_scope': [] }]
        # Symbol table fo variable semantical analysis
        self.symbol_tree = []
        # This will hold all the tokens
        self.token_stream = token_stream

    def parse(self, token_stream):
        """ Parsing

        This will parse the tokens given as argument and turn the sequence of tokens into
        abstract syntax trees

        Args:
         token_stream (list) : The tokens produced by lexer
        """
        print('---------------------------------------------')
        print(token_stream)
        print('---------------------------------------------')

        # This will hold the token index we are parsing at
        token_index = 0

        # Loop through each token
        while token_index < len(token_stream):

            # Set the token values in variables for clearer and easier debugging and readability
            token_type = token_stream[token_index][0]
            token_value = token_stream[token_index][1]

            # This will check for an if statement token
            #if token_type == 'IDENTIFIER' and token_value.lower() == 'if':
            #    source_ast['main_scope'][0].append(self.parse_if_statement(token_stream[token_index:len(token_stream)]))

            # This will parse for a vraible decleration token
            if token_type == 'DATATYPE' and token_value.lower() in constants.DATATYPE:
                self.parse_variable_decleration(token_stream[token_index:len(token_stream)], token_index)

            # Increment token index by 1 when a loop finishes
            token_index += 1

        print(self.source_ast)
        print(self.symbol_tree)

    def parse_if_statement(self, token_stream):
        """ Parsing If Statement

        This will parse through an if statement and create it's abstract tree and handle any
        syntax error

        Args:
            token_stream (list) : List of tokens starting from where if statement was found

        Returns:
            AST : The if statement abstract syntax tree
        """
        print("IF STATEMENT")

    def find_variable_scope(self):
        """ Find/set variable scope

        This will set or find the scope of a variable being declared or called
        so that it can create a list of priorotised variables which should be called.
        """
        print('Find it by looping through source_ast and sorting through _scope tagged names')

    def does_var_exist(self, name):
        """ Check to see if a variable exists """
        for x in self.symbol_tree:
            if x[0] == name: return True
        return False

    def parse_variable_decleration(self, token_stream, found_at_index):
        """ Parsing Variable decleration

        This will parse through a variable decleration and create it's abstract tree and handle any
        syntax errors

        Args:
            token_stream (list) : List of tokens starting from where if statement was found

        Returns:
            AST : The variable decleration abstract syntax tree
        """

        # Will hold the vraible decleration abstract syntax tree being built
        ast = []

        # Keeps track of the index within variable decleration
        index = 0

        for item in token_stream:

            # This will add one every loop to the index of the var decleration
            index += 1

            # This will get the variable type and add it to the AST
            if index == 1: ast.append({ 'VariableDeclerator': [ {'type': item[1]} ]})

            # This will check for the variable name
            if index == 2:
                # This wll check if the variable name already exists
                if self.does_var_exist(item[1]) == False:
                    # This will check to make sure that the name of the doesn't start wih a number
                    if not item[1][0].isdigit(): ast[0]['VariableDeclerator'].append({ 'name': item[1] })
                    # This will print an error if variable begins with a number
                    else: print('Illegal Variable Name "' + item[1] + '" variable name cannot begind with a number')
                else:
                    print('Error: Variable name "' + item[1] + '" is already defined!')
                    quit()

            # This will check for equal sign
            if index == 3:
                if item[1] == '=': pass
                else:
                    print("SyntaxError: An equal sign '=' was excpexted in variable decleration")
                    quit()

            # This will check the variable value but will skip the equal sign
            if index >= 4 and item[1] != ';':
                # TODO Add the variable to the symbol tree
                # TODO Modify this code to allow for more complex var declerations
                # Check if the value is the same value as the datatype in decleration
                if str(type(literal_eval(item[1]))) == "<class " + "'" + ast[0]['VariableDeclerator'][0]['type'] + "'>":
                    ast[0]['VariableDeclerator'].append({ 'value': item[1] })
                else:
                    print("TypeError: Variable value does not conform to data type of " + str(type(literal_eval(item[1]))))
                    quit()

            # If the for loop reaches the end statement then break because it is the end of the var decleration
            if item[1] == ';': break

        # Append this var declerating ast to the complete source ast and symbol table
        self.source_ast[0]['main_scope'].append(ast[0])
        self.symbol_tree.append( [ ast[0]['VariableDeclerator'][1]['name'], ast[0]['VariableDeclerator'][2]['value'] ] )

    def parse_print(self, token_stream):
        """ Parsing print

        This will parse through a variable decleration and create it's abstract tree and handle any
        syntax errors

        Args:
            token_stream (list) : List of tokens starting from where if statement was found

        Returns:
            AST : The variable decleration abstract syntax tree
        """
        print("PRINT")<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2005 onwards University of Deusto
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.
#
# This software consists of contributions made by many individuals,
# listed below:
#
# Author: Iban Eguia <iban.eguia@opendeusto.es>
#

import weblab.experiment.experiment as Experiment

from voodoo.override import Override
from voodoo.log import logged

import urllib2
import json
import random
import sqlite3
import time
import copy

# Actually defined through the configuration.
DEBUG = None

class RoMIExperiment(Experiment.Experiment):

    def __init__(self, coord_address, locator, cfg_manager, *args, **kwargs):
        super(RoMIExperiment, self).__init__(*args, **kwargs)
        self._cfg_manager = cfg_manager
        self.read_base_config()

    def read_base_config(self):
        """
        Reads the base config parameters from the config file.
        """

        self.database = self._cfg_manager.get_value('romie_sqlite')

    @Override(Experiment.Experiment)
    @logged("info")
    def do_get_api(self):
        return "2"

    @Override(Experiment.Experiment)
    @logged("info")
    def do_start_experiment(self, client_initial_data, server_initial_data):
        """
        Callback run when the experiment is started.
        """
        if(DEBUG):
            print "[RoMIE] do_start_experiment called"

        data = json.loads(server_initial_data)
        self.username = data['request.username']
        self.server = self._cfg_manager.get_value('romie_server')
        self.questions = copy.deepcopy(self._cfg_manager.get_value('questions'))
        self.question = {}
        self.q_difficulty = 0
        self.points = 0
        self.last_tag = ''
        self.finish_time = 0
        self.last_correct = 0

        return ""

    @Override(Experiment.Experiment)
    @logged("info")
    def do_send_command_to_device(self, command):
        """
        Callback run when the client sends a command to the experiment
        @param command Command sent by the client, as a string.
        """
        if(DEBUG):
            print "[RoMIE] Command received: %s" % command

        if command == 'F':
            tag = urllib2.urlopen("%sf" % self.server).read()
            if tag.startswith('Tag') and tag != self.last_tag:

                self.last_tag = tag

                self.q_difficulty = int(self.points/65000)
                if self.q_difficulty > 10:
                    self.q_difficulty = 10

                index = random.randint(0, len(self.questions[self.q_difficulty])-1)
                self.question = self.questions[self.q_difficulty][index]

                response_question = {
                    'question': self.question['question'],
                    'answers': self.question['answers']
                }

                return json.dumps(response_question)
            else:
                return 'OK'
        elif command == 'L':
            return urllib2.urlopen("%sl" % self.server).read()
        elif command == 'R':
            return urllib2.urlopen("%sr" % self.server).read()
        elif command.startswith("ANSWER"):

            response = int(command.split()[1])
            correct = self.question['correct'] == response

            if correct:
                time_bonus = 30-(time.time()-self.last_correct)
                bonus = (self.q_difficulty/10+1)*(time_bonus/5 if time_bonus > 5 else 1)
                self.last_correct = time.time()
                self.points += self.question['points']*bonus
                self.finish_time += self.question['time']*bonus
                self.update_points()
                self.questions[self.q_difficulty].remove(self.question)
                self.question = {}

            return json.dumps({"correct": correct, "points": self.points, "finish_time": self.finish_time})

        elif command == 'CHECK_REGISTER':
            conn = sqlite3.connect(self.database)
            cur = conn.cursor()

            cur.execute("SELECT COUNT(*) FROM forotech WHERE username = ?", (self.username,));
            count = cur.fetchone()[0]

            result = ''
            if count == 0:
                result = 'REGISTER'
            else:
                self.update_points()
                self.finish_time = round(time.time()+self._cfg_manager.get_value('romie_time'), 3)
                result = self.finish_time

            conn.close()

            return result

        elif command.startswith('REGISTER'):
            data = json.loads(command.split(' ', 1)[1])

            if (self.email_exists(data["email"])):
                return 'ERROR EMAIL'

            conn = sqlite3.connect(self.database)
            conn.execute("INSERT INTO forotech values (?,?,?,?,?,?,?)",
                (self.username, data["name"], data["surname"], data["school"], data["bdate"], data["email"], 0,))
            conn.commit()
            conn.close()

            self.finish_time = round(time.time()+self._cfg_manager.get_value('romie_time'), 3)

            return self.finish_time

        elif command == 'FINISH':

            conn = sqlite3.connect(self.database)
            conn.execute("UPDATE forotech SET points = ? WHERE username = ?", (self.points, self.username,))

            cur = conn.cursor()
            cur.execute("SELECT username, name, surname, school, points FROM forotech ORDER BY points DESC LIMIT 10")
            result = cur.fetchall()
            ranking = list()

            for user in result:
                current = (user[0] == self.username)
                ranking.append({"name":user[1], "surname":user[2], "school":user[3], "points":user[4], "current":current})

            conn.commit()
            conn.close()
            return json.dumps(ranking)

        return "ERROR"

    @Override(Experiment.Experiment)
    @logged("info")
    def do_dispose(self):
        """
        Callback to perform cleaning after the experiment ends.
        """
        if(DEBUG):
            print "[RoMIE] do_dispose called"

        return "OK"

    def update_points(self):
        """
        Update points in the database
        """
        conn = sqlite3.connect(self.database)

        cur = conn.cursor()
        cur.execute('SELECT points FROM forotech WHERE username = ?', (self.username,))
        points = cur.fetchone()[0]

        if (points < self.points):
	        conn.execute("UPDATE forotech SET points = ? WHERE username = ?", (self.points, self.username,))
	        conn.commit()

        conn.close()

    def email_exists(self, email):
        """
        Check if email is already registered
        """
        conn = sqlite3.connect(self.database)
        cur = conn.cursor()
        cur.execute("SELECT COUNT(*) FROM forotech WHERE email = ?", (email,));
        count = cur.fetchone()[0]
        conn.close()

        return count > 0
<commit_msg>Only forotech users in ranking<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2005 onwards University of Deusto
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.
#
# This software consists of contributions made by many individuals,
# listed below:
#
# Author: Iban Eguia <iban.eguia@opendeusto.es>
#

import weblab.experiment.experiment as Experiment

from voodoo.override import Override
from voodoo.log import logged

import urllib2
import json
import random
import sqlite3
import time
import copy

# Actually defined through the configuration.
DEBUG = None

class RoMIExperiment(Experiment.Experiment):

    def __init__(self, coord_address, locator, cfg_manager, *args, **kwargs):
        super(RoMIExperiment, self).__init__(*args, **kwargs)
        self._cfg_manager = cfg_manager
        self.read_base_config()

    def read_base_config(self):
        """
        Reads the base config parameters from the config file.
        """

        self.database = self._cfg_manager.get_value('romie_sqlite')

    @Override(Experiment.Experiment)
    @logged("info")
    def do_get_api(self):
        return "2"

    @Override(Experiment.Experiment)
    @logged("info")
    def do_start_experiment(self, client_initial_data, server_initial_data):
        """
        Callback run when the experiment is started.
        """
        if(DEBUG):
            print "[RoMIE] do_start_experiment called"

        data = json.loads(server_initial_data)
        self.username = data['request.username']
        self.server = self._cfg_manager.get_value('romie_server')
        self.questions = copy.deepcopy(self._cfg_manager.get_value('questions'))
        self.question = {}
        self.q_difficulty = 0
        self.points = 0
        self.last_tag = ''
        self.finish_time = 0
        self.last_correct = 0

        return ""

    @Override(Experiment.Experiment)
    @logged("info")
    def do_send_command_to_device(self, command):
        """
        Callback run when the client sends a command to the experiment
        @param command Command sent by the client, as a string.
        """
        if(DEBUG):
            print "[RoMIE] Command received: %s" % command

        if command == 'F':
            tag = urllib2.urlopen("%sf" % self.server).read()
            if tag.startswith('Tag') and tag != self.last_tag:

                self.last_tag = tag

                self.q_difficulty = int(self.points/65000)
                if self.q_difficulty > 10:
                    self.q_difficulty = 10

                index = random.randint(0, len(self.questions[self.q_difficulty])-1)
                self.question = self.questions[self.q_difficulty][index]

                response_question = {
                    'question': self.question['question'],
                    'answers': self.question['answers']
                }

                return json.dumps(response_question)
            else:
                return 'OK'
        elif command == 'L':
            return urllib2.urlopen("%sl" % self.server).read()
        elif command == 'R':
            return urllib2.urlopen("%sr" % self.server).read()
        elif command.startswith("ANSWER"):

            response = int(command.split()[1])
            correct = self.question['correct'] == response

            if correct:
                time_bonus = 30-(time.time()-self.last_correct)
                bonus = (self.q_difficulty/10+1)*(time_bonus/5 if time_bonus > 5 else 1)
                self.last_correct = time.time()
                self.points += self.question['points']*bonus
                self.finish_time += self.question['time']*bonus
                self.update_points()
                self.questions[self.q_difficulty].remove(self.question)
                self.question = {}

            return json.dumps({"correct": correct, "points": self.points, "finish_time": self.finish_time})

        elif command == 'CHECK_REGISTER':
            conn = sqlite3.connect(self.database)
            cur = conn.cursor()

            cur.execute("SELECT COUNT(*) FROM forotech WHERE username = ?", (self.username,));
            count = cur.fetchone()[0]

            result = ''
            if count == 0:
                result = 'REGISTER'
            else:
                self.update_points()
                self.finish_time = round(time.time()+self._cfg_manager.get_value('romie_time'), 3)
                result = self.finish_time

            conn.close()

            return result

        elif command.startswith('REGISTER'):
            data = json.loads(command.split(' ', 1)[1])

            if (self.email_exists(data["email"])):
                return 'ERROR EMAIL'

            conn = sqlite3.connect(self.database)
            conn.execute("INSERT INTO forotech values (?,?,?,?,?,?,?)",
                (self.username, data["name"], data["surname"], data["school"], data["bdate"], data["email"], 0,))
            conn.commit()
            conn.close()

            self.finish_time = round(time.time()+self._cfg_manager.get_value('romie_time'), 3)

            return self.finish_time

        elif command == 'FINISH':

            conn = sqlite3.connect(self.database)
            conn.execute("UPDATE forotech SET points = ? WHERE username = ?", (self.points, self.username,))

            cur = conn.cursor()
            cur.execute('SELECT username, name, surname, school, points FROM forotech WHERE username LIKE "forotech%" ORDER BY points DESC LIMIT 10')
            result = cur.fetchall()
            ranking = list()

            for user in result:
                current = (user[0] == self.username)
                ranking.append({"name":user[1], "surname":user[2], "school":user[3], "points":user[4], "current":current})

            conn.commit()
            conn.close()
            return json.dumps(ranking)

        return "ERROR"

    @Override(Experiment.Experiment)
    @logged("info")
    def do_dispose(self):
        """
        Callback to perform cleaning after the experiment ends.
        """
        if(DEBUG):
            print "[RoMIE] do_dispose called"

        return "OK"

    def update_points(self):
        """
        Update points in the database
        """
        conn = sqlite3.connect(self.database)

        cur = conn.cursor()
        cur.execute('SELECT points FROM forotech WHERE username = ?', (self.username,))
        points = cur.fetchone()[0]

        if (points < self.points):
	        conn.execute("UPDATE forotech SET points = ? WHERE username = ?", (self.points, self.username,))
	        conn.commit()

        conn.close()

    def email_exists(self, email):
        """
        Check if email is already registered
        """
        conn = sqlite3.connect(self.database)
        cur = conn.cursor()
        cur.execute("SELECT COUNT(*) FROM forotech WHERE email = ?", (email,));
        count = cur.fetchone()[0]
        conn.close()

        return count > 0
<|endoftext|>


<commit_before>from __future__ import unicode_literals

from markdown import markdown

from django import template
from django.utils.safestring import mark_safe

register = template.Library()

#
# Filters
#

@register.filter()
def oneline(value):
    """
    Replace each line break with a single space
    """
    return value.replace('\n', ' ')

@register.filter()
def getlist(value, arg):
    """
    Return all values of a QueryDict key
    """
    return value.getlist(arg)

@register.filter
def getkey(value, key):
    """
    Return a dictionary item specified by key
    """
    return value[key]

@register.filter(is_safe=True)
def gfm(value):
    """
    Render text as GitHub-Flavored Markdown
    """
    html = markdown(value, extensions=['mdx_gfm'])
    return mark_safe(html)

@register.filter()
def contains(value, arg):
    """
    Test whether a value contains any of a given set of strings. `arg` should be a comma-separated list of strings.
    """
    return any(s in value for s in arg.split(','))

@register.filter()
def bettertitle(value):
    """
    Alternative to the builtin title(); uppercases words without replacing letters that are already uppercase.
    """
    return ' '.join([w[0].upper() + w[1:] for w in value.split()])

@register.filter()
def example_choices(field, arg=3):
    """
    Returns a number (default: 3) of example choices for a ChoiceFiled (useful for CSV import forms).
    """
    examples = []
    if hasattr(field, 'queryset'):
        choices = [(obj.pk, getattr(obj, field.to_field_name)) for obj in field.queryset[:arg+1]]
    else:
        choices = field.choices
    for id, label in choices:
        if len(examples) == arg:
            examples.append('etc.')
            break
        if not id:
            continue
        examples.append(label)
    return ', '.join(examples) or 'None'

#
# Tags
#

@register.simple_tag()
def querystring(request, **kwargs):
    """
    Append or update the page number in a querystring.
    """
    querydict = request.GET.copy()
    for k, v in kwargs.items():
        if v is not None:
            querydict[k] = v
        elif k in querydict:
            querydict.pop(k)
    querystring = querydict.urlencode()
    if querystring:
        return '?' + querystring
    else:
        return ''

@register.inclusion_tag('utilities/templatetags/utilization_graph.html')
def utilization_graph(utilization, warning_threshold=75, danger_threshold=90):
    """
    Display a horizontal bar graph indicating a percentage of utilization.
    """
    return {
        'utilization': utilization,
        'warning_threshold': warning_threshold,
        'danger_threshold': danger_threshold,
    }
<commit_msg>PEP8 fix<commit_after>from __future__ import unicode_literals

from markdown import markdown

from django import template
from django.utils.safestring import mark_safe

register = template.Library()

#
# Filters
#

@register.filter()
def oneline(value):
    """
    Replace each line break with a single space
    """
    return value.replace('\n', ' ')

@register.filter()
def getlist(value, arg):
    """
    Return all values of a QueryDict key
    """
    return value.getlist(arg)

@register.filter
def getkey(value, key):
    """
    Return a dictionary item specified by key
    """
    return value[key]

@register.filter(is_safe=True)
def gfm(value):
    """
    Render text as GitHub-Flavored Markdown
    """
    html = markdown(value, extensions=['mdx_gfm'])
    return mark_safe(html)

@register.filter()
def contains(value, arg):
    """
    Test whether a value contains any of a given set of strings. `arg` should be a comma-separated list of strings.
    """
    return any(s in value for s in arg.split(','))

@register.filter()
def bettertitle(value):
    """
    Alternative to the builtin title(); uppercases words without replacing letters that are already uppercase.
    """
    return ' '.join([w[0].upper() + w[1:] for w in value.split()])

@register.filter()
def example_choices(field, arg=3):
    """
    Returns a number (default: 3) of example choices for a ChoiceFiled (useful for CSV import forms).
    """
    examples = []
    if hasattr(field, 'queryset'):
        choices = [(obj.pk, getattr(obj, field.to_field_name)) for obj in field.queryset[:arg + 1]]
    else:
        choices = field.choices
    for id, label in choices:
        if len(examples) == arg:
            examples.append('etc.')
            break
        if not id:
            continue
        examples.append(label)
    return ', '.join(examples) or 'None'

#
# Tags
#

@register.simple_tag()
def querystring(request, **kwargs):
    """
    Append or update the page number in a querystring.
    """
    querydict = request.GET.copy()
    for k, v in kwargs.items():
        if v is not None:
            querydict[k] = v
        elif k in querydict:
            querydict.pop(k)
    querystring = querydict.urlencode()
    if querystring:
        return '?' + querystring
    else:
        return ''

@register.inclusion_tag('utilities/templatetags/utilization_graph.html')
def utilization_graph(utilization, warning_threshold=75, danger_threshold=90):
    """
    Display a horizontal bar graph indicating a percentage of utilization.
    """
    return {
        'utilization': utilization,
        'warning_threshold': warning_threshold,
        'danger_threshold': danger_threshold,
    }
<|endoftext|>


<commit_before>#
# Copyright (c) 2017 ThoughtWorks, Inc.
#
# Pixelated is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Pixelated is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.

from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.logger import Logger
from twisted.mail import smtp

from email.mime.text import MIMEText

log = Logger()

class AccountRecovery(object):
    def __init__(self, session, soledad, smtp_config, backup_email, domain):
        self._bonafide_session = session
        self._soledad = soledad
        self._smtp_config = smtp_config
        self._backup_email = backup_email
        self._domain = domain

    @inlineCallbacks
    def update_recovery_code(self):
        try:
            code = self._soledad.create_recovery_code()
            response = yield self._bonafide_session.update_recovery_code(code)
            yield self._send_mail(code, self._backup_email)

            returnValue(response)

        except Exception as e:
            log.error('Something went wrong when trying to save the recovery code')
            log.error(e)
            raise e

    @inlineCallbacks
    def _send_mail(self, code, backup_email):
        sender = 'team@{}'.format(self._domain)
        msg = MIMEText('Your code %s' % code)
        msg['Subject'] = 'Recovery Code'
        msg['From'] = sender
        msg['To'] = backup_email

        try:
            send_mail_result = yield smtp.sendmail(
                str(self._smtp_config.remote_smtp_host),
                sender,
                [backup_email],
                msg.as_string())
            returnValue(send_mail_result)
        except Exception as e:
            log.error('Failed trying to send the email with the recovery code')
            raise e
<commit_msg>[#927] Log account recovery steps<commit_after>#
# Copyright (c) 2017 ThoughtWorks, Inc.
#
# Pixelated is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Pixelated is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.

from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.logger import Logger
from twisted.mail import smtp

from email.mime.text import MIMEText

log = Logger()

class AccountRecovery(object):
    def __init__(self, session, soledad, smtp_config, backup_email, domain):
        self._bonafide_session = session
        self._soledad = soledad
        self._smtp_config = smtp_config
        self._backup_email = backup_email
        self._domain = domain

    @inlineCallbacks
    def update_recovery_code(self):
        log.info('Updating user\'s recovery code')

        try:
            code = self._soledad.create_recovery_code()
            response = yield self._bonafide_session.update_recovery_code(code)
            yield self._send_mail(code, self._backup_email)

            returnValue(response)

        except Exception as e:
            log.error('Something went wrong when trying to save the recovery code')
            log.error(e)
            raise e

    @inlineCallbacks
    def _send_mail(self, code, backup_email):
        log.info('Sending mail containing the user\'s recovery code')

        sender = 'team@{}'.format(self._domain)
        msg = MIMEText('Your code %s' % code)
        msg['Subject'] = 'Recovery Code'
        msg['From'] = sender
        msg['To'] = backup_email

        try:
            send_mail_result = yield smtp.sendmail(
                str(self._smtp_config.remote_smtp_host),
                sender,
                [backup_email],
                msg.as_string())
            returnValue(send_mail_result)
        except Exception as e:
            log.error('Failed trying to send the email with the recovery code')
            raise e
<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2011 OpenStack Foundation.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Root wrapper for OpenStack services

   Filters which commands a service is allowed to run as another user.

   To use this with neutron, you should set the following in
   neutron.conf:
   rootwrap_config=/etc/neutron/rootwrap.conf

   You also need to let the neutron user run neutron-rootwrap
   as root in sudoers:
   neutron ALL = (root) NOPASSWD: /usr/bin/neutron-rootwrap
                                   /etc/neutron/rootwrap.conf *

   Service packaging should deploy .filters files only on nodes where
   they are needed, to avoid allowing more than is necessary.
"""

from __future__ import print_function

import ConfigParser
import logging
import os
import pwd
import signal
import subprocess
import sys

RC_UNAUTHORIZED = 99
RC_NOCOMMAND = 98
RC_BADCONFIG = 97
RC_NOEXECFOUND = 96

def _subprocess_setup():
    # Python installs a SIGPIPE handler by default. This is usually not what
    # non-Python subprocesses expect.
    signal.signal(signal.SIGPIPE, signal.SIG_DFL)

def _exit_error(execname, message, errorcode, log=True):
    print("%s: %s" % (execname, message))
    if log:
        logging.error(message)
    sys.exit(errorcode)

def main():
    # Split arguments, require at least a command
    execname = sys.argv.pop(0)
    if len(sys.argv) < 2:
        _exit_error(execname, "No command specified", RC_NOCOMMAND, log=False)

    configfile = sys.argv.pop(0)
    userargs = sys.argv[:]

    # Add ../ to sys.path to allow running from branch
    possible_topdir = os.path.normpath(os.path.join(os.path.abspath(execname),
                                                    os.pardir, os.pardir))
    if os.path.exists(os.path.join(possible_topdir, "neutron", "__init__.py")):
        sys.path.insert(0, possible_topdir)

    from neutron.openstack.common.rootwrap import wrapper

    # Load configuration
    try:
        rawconfig = ConfigParser.RawConfigParser()
        rawconfig.read(configfile)
        config = wrapper.RootwrapConfig(rawconfig)
    except ValueError as exc:
        msg = "Incorrect value in %s: %s" % (configfile, exc.message)
        _exit_error(execname, msg, RC_BADCONFIG, log=False)
    except ConfigParser.Error:
        _exit_error(execname, "Incorrect configuration file: %s" % configfile,
                    RC_BADCONFIG, log=False)

    if config.use_syslog:
        wrapper.setup_syslog(execname,
                             config.syslog_log_facility,
                             config.syslog_log_level)

    # Execute command if it matches any of the loaded filters
    filters = wrapper.load_filters(config.filters_path)
    try:
        filtermatch = wrapper.match_filter(filters, userargs,
                                           exec_dirs=config.exec_dirs)
        if filtermatch:
            command = filtermatch.get_command(userargs,
                                              exec_dirs=config.exec_dirs)
            if config.use_syslog:
                logging.info("(%s > %s) Executing %s (filter match = %s)" % (
                    os.getlogin(), pwd.getpwuid(os.getuid())[0],
                    command, filtermatch.name))

            obj = subprocess.Popen(command,
                                   stdin=sys.stdin,
                                   stdout=sys.stdout,
                                   stderr=sys.stderr,
                                   preexec_fn=_subprocess_setup,
                                   env=filtermatch.get_environment(userargs))
            obj.wait()
            sys.exit(obj.returncode)

    except wrapper.FilterMatchNotExecutable as exc:
        msg = ("Executable not found: %s (filter match = %s)"
               % (exc.match.exec_path, exc.match.name))
        _exit_error(execname, msg, RC_NOEXECFOUND, log=config.use_syslog)

    except wrapper.NoFilterMatched:
        msg = ("Unauthorized command: %s (no filter matched)"
               % ' '.join(userargs))
        _exit_error(execname, msg, RC_UNAUTHORIZED, log=config.use_syslog)
<commit_msg>Fixes files with wrong bitmode<commit_after>#!/usr/bin/env python
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2011 OpenStack Foundation.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Root wrapper for OpenStack services

   Filters which commands a service is allowed to run as another user.

   To use this with neutron, you should set the following in
   neutron.conf:
   rootwrap_config=/etc/neutron/rootwrap.conf

   You also need to let the neutron user run neutron-rootwrap
   as root in sudoers:
   neutron ALL = (root) NOPASSWD: /usr/bin/neutron-rootwrap
                                   /etc/neutron/rootwrap.conf *

   Service packaging should deploy .filters files only on nodes where
   they are needed, to avoid allowing more than is necessary.
"""

from __future__ import print_function

import ConfigParser
import logging
import os
import pwd
import signal
import subprocess
import sys

RC_UNAUTHORIZED = 99
RC_NOCOMMAND = 98
RC_BADCONFIG = 97
RC_NOEXECFOUND = 96

def _subprocess_setup():
    # Python installs a SIGPIPE handler by default. This is usually not what
    # non-Python subprocesses expect.
    signal.signal(signal.SIGPIPE, signal.SIG_DFL)

def _exit_error(execname, message, errorcode, log=True):
    print("%s: %s" % (execname, message))
    if log:
        logging.error(message)
    sys.exit(errorcode)

def main():
    # Split arguments, require at least a command
    execname = sys.argv.pop(0)
    if len(sys.argv) < 2:
        _exit_error(execname, "No command specified", RC_NOCOMMAND, log=False)

    configfile = sys.argv.pop(0)
    userargs = sys.argv[:]

    # Add ../ to sys.path to allow running from branch
    possible_topdir = os.path.normpath(os.path.join(os.path.abspath(execname),
                                                    os.pardir, os.pardir))
    if os.path.exists(os.path.join(possible_topdir, "neutron", "__init__.py")):
        sys.path.insert(0, possible_topdir)

    from neutron.openstack.common.rootwrap import wrapper

    # Load configuration
    try:
        rawconfig = ConfigParser.RawConfigParser()
        rawconfig.read(configfile)
        config = wrapper.RootwrapConfig(rawconfig)
    except ValueError as exc:
        msg = "Incorrect value in %s: %s" % (configfile, exc.message)
        _exit_error(execname, msg, RC_BADCONFIG, log=False)
    except ConfigParser.Error:
        _exit_error(execname, "Incorrect configuration file: %s" % configfile,
                    RC_BADCONFIG, log=False)

    if config.use_syslog:
        wrapper.setup_syslog(execname,
                             config.syslog_log_facility,
                             config.syslog_log_level)

    # Execute command if it matches any of the loaded filters
    filters = wrapper.load_filters(config.filters_path)
    try:
        filtermatch = wrapper.match_filter(filters, userargs,
                                           exec_dirs=config.exec_dirs)
        if filtermatch:
            command = filtermatch.get_command(userargs,
                                              exec_dirs=config.exec_dirs)
            if config.use_syslog:
                logging.info("(%s > %s) Executing %s (filter match = %s)" % (
                    os.getlogin(), pwd.getpwuid(os.getuid())[0],
                    command, filtermatch.name))

            obj = subprocess.Popen(command,
                                   stdin=sys.stdin,
                                   stdout=sys.stdout,
                                   stderr=sys.stderr,
                                   preexec_fn=_subprocess_setup,
                                   env=filtermatch.get_environment(userargs))
            obj.wait()
            sys.exit(obj.returncode)

    except wrapper.FilterMatchNotExecutable as exc:
        msg = ("Executable not found: %s (filter match = %s)"
               % (exc.match.exec_path, exc.match.name))
        _exit_error(execname, msg, RC_NOEXECFOUND, log=config.use_syslog)

    except wrapper.NoFilterMatched:
        msg = ("Unauthorized command: %s (no filter matched)"
               % ' '.join(userargs))
        _exit_error(execname, msg, RC_UNAUTHORIZED, log=config.use_syslog)
<|endoftext|>


<commit_before>"""Package Index Tests
"""
import sys
import os
import unittest
import pkg_resources
from setuptools.compat import urllib2, httplib, HTTPError, unicode, pathname2url
import distutils.errors
import setuptools.package_index
from setuptools.tests.server import IndexServer

class TestPackageIndex(unittest.TestCase):

    def test_bad_url_bad_port(self):
        index = setuptools.package_index.PackageIndex()
        url = 'http://127.0.0.1:0/nonesuch/test_package_index'
        try:
            v = index.open_url(url)
        except Exception:
            v = sys.exc_info()[1]
            self.assertTrue(url in str(v))
        else:
            self.assertTrue(isinstance(v, HTTPError))

    def test_bad_url_typo(self):
        # issue 16
        # easy_install inquant.contentmirror.plone breaks because of a typo
        # in its home URL
        index = setuptools.package_index.PackageIndex(
            hosts=('www.example.com',)
        )

        url = 'url:%20https://svn.plone.org/svn/collective/inquant.contentmirror.plone/trunk'
        try:
            v = index.open_url(url)
        except Exception:
            v = sys.exc_info()[1]
            self.assertTrue(url in str(v))
        else:
            self.assertTrue(isinstance(v, HTTPError))

    def test_bad_url_bad_status_line(self):
        index = setuptools.package_index.PackageIndex(
            hosts=('www.example.com',)
        )

        def _urlopen(*args):
            raise httplib.BadStatusLine('line')

        index.opener = _urlopen
        url = 'http://example.com'
        try:
            v = index.open_url(url)
        except Exception:
            v = sys.exc_info()[1]
            self.assertTrue('line' in str(v))
        else:
            raise AssertionError('Should have raise here!')

    def test_bad_url_double_scheme(self):
        """
        A bad URL with a double scheme should raise a DistutilsError.
        """
        index = setuptools.package_index.PackageIndex(
            hosts=('www.example.com',)
        )

        # issue 20
        url = 'http://http://svn.pythonpaste.org/Paste/wphp/trunk'
        try:
            index.open_url(url)
        except distutils.errors.DistutilsError:
            error = sys.exc_info()[1]
            msg = unicode(error)
            assert 'nonnumeric port' in msg or 'getaddrinfo failed' in msg or 'Name or service not known' in msg
            return
        raise RuntimeError("Did not raise")

    def test_bad_url_screwy_href(self):
        index = setuptools.package_index.PackageIndex(
            hosts=('www.example.com',)
        )

        # issue #160
        if sys.version_info[0] == 2 and sys.version_info[1] == 7:
            # this should not fail
            url = 'http://example.com'
            page = ('')
            index.process_index(url, page)

    def test_url_ok(self):
        index = setuptools.package_index.PackageIndex(
            hosts=('www.example.com',)
        )
        url = 'file:///tmp/test_package_index'
        self.assertTrue(index.url_ok(url, True))

    def test_links_priority(self):
        """
        Download links from the pypi simple index should be used before
        external download links.
        https://bitbucket.org/tarek/distribute/issue/163

        Usecase :
        - someone uploads a package on pypi, a md5 is generated
        - someone manually copies this link (with the md5 in the url) onto an
          external page accessible from the package page.
        - someone reuploads the package (with a different md5)
        - while easy_installing, an MD5 error occurs because the external link
          is used
        -> Setuptools should use the link from pypi, not the external one.
        """
        if sys.platform.startswith('java'):
            # Skip this test on jython because binding to :0 fails
            return

        # start an index server
        server = IndexServer()
        server.start()
        index_url = server.base_url() + 'test_links_priority/simple/'

        # scan a test index
        pi = setuptools.package_index.PackageIndex(index_url)
        requirement = pkg_resources.Requirement.parse('foobar')
        pi.find_packages(requirement)
        server.stop()

        # the distribution has been found
        self.assertTrue('foobar' in pi)
        # we have only one link, because links are compared without md5
        self.assertTrue(len(pi['foobar'])==1)
        # the link should be from the index
        self.assertTrue('correct_md5' in pi['foobar'][0].location)

    def test_parse_bdist_wininst(self):
        self.assertEqual(setuptools.package_index.parse_bdist_wininst(
            'reportlab-2.5.win32-py2.4.exe'), ('reportlab-2.5', '2.4', 'win32'))
        self.assertEqual(setuptools.package_index.parse_bdist_wininst(
            'reportlab-2.5.win32.exe'), ('reportlab-2.5', None, 'win32'))
        self.assertEqual(setuptools.package_index.parse_bdist_wininst(
            'reportlab-2.5.win-amd64-py2.7.exe'), ('reportlab-2.5', '2.7', 'win-amd64'))
        self.assertEqual(setuptools.package_index.parse_bdist_wininst(
            'reportlab-2.5.win-amd64.exe'), ('reportlab-2.5', None, 'win-amd64'))

    def test__vcs_split_rev_from_url(self):
        """
        Test the basic usage of _vcs_split_rev_from_url
        """
        vsrfu = setuptools.package_index.PackageIndex._vcs_split_rev_from_url
        url, rev = vsrfu('https://example.com/bar@2995')
        self.assertEqual(url, 'https://example.com/bar')
        self.assertEqual(rev, '2995')

    def test_local_index(self):
        """
        local_open should be able to read an index from the file system.
        """
        f = open('index.html', 'w')
        f.write('
content
')
        f.close()
        try:
            url = 'file:' + pathname2url(os.getcwd()) + '/'
            res = setuptools.package_index.local_open(url)
        finally:
            os.remove('index.html')
        assert 'content' in res.read()

class TestContentCheckers(unittest.TestCase):

    def test_md5(self):
        checker = setuptools.package_index.HashChecker.from_url(
            'http://foo/bar#md5=f12895fdffbd45007040d2e44df98478')
        checker.feed('You should probably not be using MD5'.encode('ascii'))
        self.assertEqual(checker.hash.hexdigest(),
            'f12895fdffbd45007040d2e44df98478')
        self.assertTrue(checker.is_valid())

    def test_other_fragment(self):
        "Content checks should succeed silently if no hash is present"
        checker = setuptools.package_index.HashChecker.from_url(
            'http://foo/bar#something%20completely%20different')
        checker.feed('anything'.encode('ascii'))
        self.assertTrue(checker.is_valid())

    def test_blank_md5(self):
        "Content checks should succeed if a hash is empty"
        checker = setuptools.package_index.HashChecker.from_url(
            'http://foo/bar#md5=')
        checker.feed('anything'.encode('ascii'))
        self.assertTrue(checker.is_valid())

    def test_get_hash_name_md5(self):
        checker = setuptools.package_index.HashChecker.from_url(
            'http://foo/bar#md5=f12895fdffbd45007040d2e44df98478')
        self.assertEqual(checker.hash_name, 'md5')

    def test_report(self):
        checker = setuptools.package_index.HashChecker.from_url(
            'http://foo/bar#md5=f12895fdffbd45007040d2e44df98478')
        rep = checker.report(lambda x: x, 'My message about %s')
        self.assertEqual(rep, 'My message about md5')
<commit_msg>Remove unused imports<commit_after>"""Package Index Tests
"""
import sys
import os
import unittest
import pkg_resources
from setuptools.compat import httplib, HTTPError, unicode, pathname2url
import distutils.errors
import setuptools.package_index
from setuptools.tests.server import IndexServer

class TestPackageIndex(unittest.TestCase):

    def test_bad_url_bad_port(self):
        index = setuptools.package_index.PackageIndex()
        url = 'http://127.0.0.1:0/nonesuch/test_package_index'
        try:
            v = index.open_url(url)
        except Exception:
            v = sys.exc_info()[1]
            self.assertTrue(url in str(v))
        else:
            self.assertTrue(isinstance(v, HTTPError))

    def test_bad_url_typo(self):
        # issue 16
        # easy_install inquant.contentmirror.plone breaks because of a typo
        # in its home URL
        index = setuptools.package_index.PackageIndex(
            hosts=('www.example.com',)
        )

        url = 'url:%20https://svn.plone.org/svn/collective/inquant.contentmirror.plone/trunk'
        try:
            v = index.open_url(url)
        except Exception:
            v = sys.exc_info()[1]
            self.assertTrue(url in str(v))
        else:
            self.assertTrue(isinstance(v, HTTPError))

    def test_bad_url_bad_status_line(self):
        index = setuptools.package_index.PackageIndex(
            hosts=('www.example.com',)
        )

        def _urlopen(*args):
            raise httplib.BadStatusLine('line')

        index.opener = _urlopen
        url = 'http://example.com'
        try:
            v = index.open_url(url)
        except Exception:
            v = sys.exc_info()[1]
            self.assertTrue('line' in str(v))
        else:
            raise AssertionError('Should have raise here!')

    def test_bad_url_double_scheme(self):
        """
        A bad URL with a double scheme should raise a DistutilsError.
        """
        index = setuptools.package_index.PackageIndex(
            hosts=('www.example.com',)
        )

        # issue 20
        url = 'http://http://svn.pythonpaste.org/Paste/wphp/trunk'
        try:
            index.open_url(url)
        except distutils.errors.DistutilsError:
            error = sys.exc_info()[1]
            msg = unicode(error)
            assert 'nonnumeric port' in msg or 'getaddrinfo failed' in msg or 'Name or service not known' in msg
            return
        raise RuntimeError("Did not raise")

    def test_bad_url_screwy_href(self):
        index = setuptools.package_index.PackageIndex(
            hosts=('www.example.com',)
        )

        # issue #160
        if sys.version_info[0] == 2 and sys.version_info[1] == 7:
            # this should not fail
            url = 'http://example.com'
            page = ('')
            index.process_index(url, page)

    def test_url_ok(self):
        index = setuptools.package_index.PackageIndex(
            hosts=('www.example.com',)
        )
        url = 'file:///tmp/test_package_index'
        self.assertTrue(index.url_ok(url, True))

    def test_links_priority(self):
        """
        Download links from the pypi simple index should be used before
        external download links.
        https://bitbucket.org/tarek/distribute/issue/163

        Usecase :
        - someone uploads a package on pypi, a md5 is generated
        - someone manually copies this link (with the md5 in the url) onto an
          external page accessible from the package page.
        - someone reuploads the package (with a different md5)
        - while easy_installing, an MD5 error occurs because the external link
          is used
        -> Setuptools should use the link from pypi, not the external one.
        """
        if sys.platform.startswith('java'):
            # Skip this test on jython because binding to :0 fails
            return

        # start an index server
        server = IndexServer()
        server.start()
        index_url = server.base_url() + 'test_links_priority/simple/'

        # scan a test index
        pi = setuptools.package_index.PackageIndex(index_url)
        requirement = pkg_resources.Requirement.parse('foobar')
        pi.find_packages(requirement)
        server.stop()

        # the distribution has been found
        self.assertTrue('foobar' in pi)
        # we have only one link, because links are compared without md5
        self.assertTrue(len(pi['foobar'])==1)
        # the link should be from the index
        self.assertTrue('correct_md5' in pi['foobar'][0].location)

    def test_parse_bdist_wininst(self):
        self.assertEqual(setuptools.package_index.parse_bdist_wininst(
            'reportlab-2.5.win32-py2.4.exe'), ('reportlab-2.5', '2.4', 'win32'))
        self.assertEqual(setuptools.package_index.parse_bdist_wininst(
            'reportlab-2.5.win32.exe'), ('reportlab-2.5', None, 'win32'))
        self.assertEqual(setuptools.package_index.parse_bdist_wininst(
            'reportlab-2.5.win-amd64-py2.7.exe'), ('reportlab-2.5', '2.7', 'win-amd64'))
        self.assertEqual(setuptools.package_index.parse_bdist_wininst(
            'reportlab-2.5.win-amd64.exe'), ('reportlab-2.5', None, 'win-amd64'))

    def test__vcs_split_rev_from_url(self):
        """
        Test the basic usage of _vcs_split_rev_from_url
        """
        vsrfu = setuptools.package_index.PackageIndex._vcs_split_rev_from_url
        url, rev = vsrfu('https://example.com/bar@2995')
        self.assertEqual(url, 'https://example.com/bar')
        self.assertEqual(rev, '2995')

    def test_local_index(self):
        """
        local_open should be able to read an index from the file system.
        """
        f = open('index.html', 'w')
        f.write('
content
')
        f.close()
        try:
            url = 'file:' + pathname2url(os.getcwd()) + '/'
            res = setuptools.package_index.local_open(url)
        finally:
            os.remove('index.html')
        assert 'content' in res.read()

class TestContentCheckers(unittest.TestCase):

    def test_md5(self):
        checker = setuptools.package_index.HashChecker.from_url(
            'http://foo/bar#md5=f12895fdffbd45007040d2e44df98478')
        checker.feed('You should probably not be using MD5'.encode('ascii'))
        self.assertEqual(checker.hash.hexdigest(),
            'f12895fdffbd45007040d2e44df98478')
        self.assertTrue(checker.is_valid())

    def test_other_fragment(self):
        "Content checks should succeed silently if no hash is present"
        checker = setuptools.package_index.HashChecker.from_url(
            'http://foo/bar#something%20completely%20different')
        checker.feed('anything'.encode('ascii'))
        self.assertTrue(checker.is_valid())

    def test_blank_md5(self):
        "Content checks should succeed if a hash is empty"
        checker = setuptools.package_index.HashChecker.from_url(
            'http://foo/bar#md5=')
        checker.feed('anything'.encode('ascii'))
        self.assertTrue(checker.is_valid())

    def test_get_hash_name_md5(self):
        checker = setuptools.package_index.HashChecker.from_url(
            'http://foo/bar#md5=f12895fdffbd45007040d2e44df98478')
        self.assertEqual(checker.hash_name, 'md5')

    def test_report(self):
        checker = setuptools.package_index.HashChecker.from_url(
            'http://foo/bar#md5=f12895fdffbd45007040d2e44df98478')
        rep = checker.report(lambda x: x, 'My message about %s')
        self.assertEqual(rep, 'My message about md5')
<|endoftext|>