<commit_before># coding= utf-8

# Copyright (c) 2014 Rackspace, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import random
import string

from tests.endtoend import base

class TestWebsiteCDN(base.TestBase):

    """Tests for CDN enabling a website."""

    def _random_string(self, length=12):
        return ''.join([random.choice(string.ascii_letters)
                        for _ in range(length)])

    def setUp(self):
        super(TestWebsiteCDN, self).setUp()

        sub_domain = 'TestCDN-' + self._random_string()
        self.test_domain = sub_domain + '.' + self.dns_config.test_domain

        print('Domain Name', self.test_domain)

        self.origin = self.test_config.wordpress_origin

    def test_enable_cdn(self):

        # Create a Poppy Service for the test website
        domain_list = [{"domain": self.test_domain}]
        origin_list = [{"origin": self.origin,
                        "port": 80,
                        "ssl": False}]
        caching_list = []
        self.service_name = 'testService-' + self._random_string()

        resp = self.poppy_client.create_service(
            service_name=self.service_name,
            domain_list=domain_list,
            origin_list=origin_list,
            caching_list=caching_list,
            flavor_id=self.poppy_config.flavor)

        self.assertEqual(resp.status_code, 202)

        self.service_location = resp.headers['location']
        self.poppy_client.wait_for_service_status(
            location=self.service_location,
            status='DEPLOYED',
            abort_on_status='FAILED',
            retry_interval=self.poppy_config.status_check_retry_interval,
            retry_timeout=self.poppy_config.status_check_retry_timeout)

        resp = self.poppy_client.get_service(location=self.service_location)
        links = resp.json()['links']
        access_url = [link['href'] for link in links if
                      link['rel'] == 'access_url']
        self.cname_rec = self.dns_client.add_cname_rec(
            name=self.test_domain, data=access_url[0])

        origin_url = 'http://' + self.origin
        cdn_enabled_url = 'http://' + self.test_domain

        self.dns_client.wait_cname_propagation(
            target=self.test_domain,
            retry_interval=self.dns_config.retry_interval)
        self.assertSameContent(origin_url=origin_url,
                               cdn_url=cdn_enabled_url)

        # Benchmark page load metrics for the CDN enabled website
        if self.test_config.webpagetest_enabled:
            wpt_test_results = {}
            for location in self.wpt_config.test_locations:
                wpt_test_url = self.wpt_client.start_test(
                    test_url=cdn_enabled_url, test_location=location, runs=2)
                wpt_test_results[location] = wpt_test_url
                self.wpt_client.wait_for_test_status(
                    status='COMPLETE', test_url=wpt_test_url)
                wpt_test_results[location] = self.wpt_client.get_test_details(
                    test_url=wpt_test_url)

            print('Webpage Tests Results', wpt_test_url)

    def tearDown(self):
        self.poppy_client.delete_service(location=self.service_location)
        # self.dns_client.delete_record(self.cname_rec)
        super(TestWebsiteCDN, self).tearDown()
<commit_msg>Multi Domain Test<commit_after># coding= utf-8

# Copyright (c) 2014 Rackspace, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import random
import string

from tests.endtoend import base

class TestWebsiteCDN(base.TestBase):

    """Tests for CDN enabling a website."""

    def _random_string(self, length=12):
        return ''.join([random.choice(string.ascii_letters)
                        for _ in range(length)])

    def setUp(self):
        super(TestWebsiteCDN, self).setUp()

        sub_domain = 'TestCDN-' + self._random_string()
        self.test_domain = sub_domain + '.' + self.dns_config.test_domain

        print('Domain Name', self.test_domain)

        self.origin = self.test_config.wordpress_origin

    def test_enable_cdn(self):

        # Create a Poppy Service for the test website
        domain_list = [{"domain": self.test_domain}]
        origin_list = [{"origin": self.origin,
                        "port": 80,
                        "ssl": False}]
        caching_list = []
        self.service_name = 'testService-' + self._random_string()

        resp = self.poppy_client.create_service(
            service_name=self.service_name,
            domain_list=domain_list,
            origin_list=origin_list,
            caching_list=caching_list,
            flavor_id=self.poppy_config.flavor)

        self.assertEqual(resp.status_code, 202)

        self.service_location = resp.headers['location']
        self.poppy_client.wait_for_service_status(
            location=self.service_location,
            status='DEPLOYED',
            abort_on_status='FAILED',
            retry_interval=self.poppy_config.status_check_retry_interval,
            retry_timeout=self.poppy_config.status_check_retry_timeout)

        resp = self.poppy_client.get_service(location=self.service_location)
        links = resp.json()['links']
        access_url = [link['href'] for link in links if
                      link['rel'] == 'access_url']
        self.cname_rec = self.dns_client.add_cname_rec(
            name=self.test_domain, data=access_url[0])

        origin_url = 'http://' + self.origin
        cdn_enabled_url = 'http://' + self.test_domain

        self.dns_client.wait_cname_propagation(
            target=self.test_domain,
            retry_interval=self.dns_config.retry_interval)
        self.assertSameContent(origin_url=origin_url,
                               cdn_url=cdn_enabled_url)

        self._test_webpage(cdn_url=cdn_enabled_url)

    def test_multiple_domains(self):

        # Create another domain in addition to the one created in setUp
        sub_domain2 = 'TestCDN-' + self._random_string()
        self.test_domain2 = sub_domain2 + '.' + self.dns_config.test_domain

        print('Additional Domain Name', self.test_domain2)

        # Create a Poppy Service for the test website with two domains
        domain_list = [{"domain": self.test_domain},
                       {"domain": self.test_domain2}]
        origin_list = [{"origin": self.origin,
                        "port": 80,
                        "ssl": False}]
        caching_list = []
        self.service_name = 'testService-' + self._random_string()

        resp = self.poppy_client.create_service(
            service_name=self.service_name,
            domain_list=domain_list,
            origin_list=origin_list,
            caching_list=caching_list,
            flavor_id=self.poppy_config.flavor)

        self.assertEqual(resp.status_code, 202)
        self.service_location = resp.headers['location']
        self.poppy_client.wait_for_service_status(
            location=self.service_location,
            status='DEPLOYED',
            abort_on_status='FAILED',
            retry_interval=self.poppy_config.status_check_retry_interval,
            retry_timeout=self.poppy_config.status_check_retry_timeout)

        resp = self.poppy_client.get_service(location=self.service_location)
        links = resp.json()['links']
        access_url = [link['href'] for link in links if
                      link['rel'] == 'access_url']

        # Add a cname to the first domain
        self.cname_rec = self.dns_client.add_cname_rec(
            name=self.test_domain, data=access_url[0])

        # Add a cname to the second domain
        self.cname_rec = self.dns_client.add_cname_rec(
            name=self.test_domain2, data=access_url[0])

        origin_url = 'http://' + self.origin
        cdn_enabled_url1 = 'http://' + self.test_domain
        cdn_enabled_url2 = 'http://' + self.test_domain2

        self.dns_client.wait_cname_propagation(
            target=self.test_domain,
            retry_interval=self.dns_config.retry_interval)

        self.dns_client.wait_cname_propagation(
            target=self.test_domain2,
            retry_interval=self.dns_config.retry_interval)

        self.assertSameContent(origin_url=origin_url,
                               cdn_url=cdn_enabled_url1)

        self.assertSameContent(origin_url=origin_url,
                               cdn_url=cdn_enabled_url2)

        self._test_webpage(cdn_url=cdn_enabled_url1)

        self._test_webpage(cdn_url=cdn_enabled_url2)

    def _test_webpage(self, cdn_url):

        # Benchmark page load metrics for the CDN enabled website
        if self.test_config.webpagetest_enabled:
            wpt_test_results = {}
            for location in self.wpt_config.test_locations:
                wpt_test_url = self.wpt_client.start_test(
                    test_url=cdn_url, test_location=location, runs=2)
                wpt_test_results[location] = wpt_test_url
                self.wpt_client.wait_for_test_status(
                    status='COMPLETE', test_url=wpt_test_url)
                wpt_test_results[location] = self.wpt_client.get_test_details(
                    test_url=wpt_test_url)

            print('Webpage Tests Results', wpt_test_url)

    def tearDown(self):
        self.poppy_client.delete_service(location=self.service_location)
        # self.dns_client.delete_record(self.cname_rec)
        super(TestWebsiteCDN, self).tearDown()
<|endoftext|>


<commit_before>import syntax_tree
import symbol_table
import code_library

INTEGER_SIZE = 8

class Lazy_generator(object):
  def generate(self, tree, table):
    self.tree = tree
    self.table = table
    self.reset_library()
    self.handle = -1
    self.in_procedure = False
    self.procedures = {}
    self.read_only_declarations = []
    self.code = ['\t.globl\tmain\n']
    self.code.append('\t.text')
    self.code.append('main:\n')
    if tree:
      self.generate_instructions(tree.instructions)
    self.code.append('__end_program:')
    self.code.append('\t\tmovq\t$60, %rax')
    self.code.append('\t\tsyscall\n')
    printed = []
    while self.procedures:
      to_print = self.procedures
      self.procedures = {}
      for procedure in to_print.values():
        if not procedure in printed:
          self.generate_procedure(procedure)
          printed.append(procedure)
    self.link_library()
    self.generate_variables()
    if self.read_only_declarations:
      self.code.append('\n\t.section .rodata')
      for declaration in self.read_only_declarations:
        self.code.append(declaration)
    return self.code
  def reset_library(self):
    self.div_by_zero = False
    self.mod_by_zero = False
    self.bad_index = False
    self.write_output = False
    self.read_input = False
  def new_handle(self):
    self.handle += 1
    return self.handle
  def generate_instructions(self, instructions):
    for instruction in instructions.instructions:
      if type(instruction.child) is syntax_tree.Assign:
        self.generate_assign(instruction.child)
      elif type(instruction.child) is syntax_tree.If:
        self.generate_if(instruction.child)
      elif type(instruction.child) is syntax_tree.Repeat:
        self.generate_repeat(instruction.child)
      elif type(instruction.child) is syntax_tree.Read:
        self.generate_read(instruction.child)
      elif type(instruction.child) is syntax_tree.Call:
        self.generate_call(instruction.child)
      else: # Write
        self.generate_write(instruction.child)
  def generate_assign(self, assign):
    self.code.append("__assign_at_{}:".format(assign.line))
    self.generate_location_evaluator(assign.location)
    count = assign.location.type_object.get_size() / INTEGER_SIZE
    if count is 1:
      self.generate_expression_evaluator(assign.expression)
    else:
      self.generate_location_evaluator(assign.expression.child)
    self.code.append('\t\tpopq\t%rcx')
    self.code.append('\t\tpopq\t%rax')
    if count > 1:
      self.code.append("\t\tmovq\t${}, %rdi".format(count))
      handle = self.new_handle()
      self.code.append("_loop_{}_:".format(handle))
      self.code.append('\t\tmovq\t(%rcx), %rsi')
      self.code.append('\t\tmovq\t%rsi, (%rax)')
      self.code.append('\t\taddq\t$8, %rcx')
      self.code.append('\t\taddq\t$8, %rax')
      self.code.append('\t\tdecq\t%rdi')
      self.code.append("\t\tjnz\t\t_loop_{}_".format(handle))
    else:
      self.code.append('\t\tmovq\t%rcx, (%rax)')
  def generate_if(self, if_statement):
    self.code.append("__if_at_{}:".format(if_statement.line))
    self.generate_condition_evaluator(if_statement.condition)
    handle = self.handle
    self.code.append("_true_{}_:".format(handle))
    self.generate_instructions(if_statement.instructions_true)
    if if_statement.instructions_false:
      self.code.append("\t\tjmp\t\t_end_{}_".format(handle))
    self.code.append("_false_{}_:".format(handle))
    if if_statement.instructions_false:
      self.generate_instructions(if_statement.instructions_false)
      self.code.append("_end_{}_:".format(handle))
  def generate_repeat(self, repeat):
    label = "__repeat_at_{}".format(repeat.line)
    self.code.append(label + ':')
    self.generate_condition_evaluator(repeat.condition)
    handle = self.handle
    self.code.append("_false_{}_:".format(handle))
    self.generate_instructions(repeat.instructions)
    self.code.append("\t\tjmp\t\t{}".format(label))
    self.code.append("_true_{}_:".format(handle))
  def generate_read(self, read):
    self.code.append("__read_at_{}:".format(read.line))
    self.code.append('\t\tpushq\t%rbx')
    self.generate_location_evaluator(read.location)
    self.code.append('\t\tpopq\t%rbx')
    self.code.append('\t\tcall\t__read')
    self.code.append('\t\tmovq\t%rax, (%rbx)')
    self.code.append('\t\tpopq\t%rbx')
    self.read_input = True
  def generate_call(self, call):
    if call.definition.instructions or call.definition.return_expression:
      if call.actuals:
        for actual in call.actuals:
          if (type(actual.child) is syntax_tree.Location and
              actual.type_object.get_size() > INTEGER_SIZE):
            self.generate_location_evaluator(actual.child)
          else:
            self.generate_expression_evaluator(actual)
      self.code.append("\t\tcall\t__{}__".format(call.definition.name))
      if not call.definition.name in self.procedures:
        self.procedures[call.definition.name] = call.definition
      if call.definition.formals:
        n = len(call.definition.formals)
        self.code.append("\t\taddq\t${}, %rsp".format(n * INTEGER_SIZE))
  def generate_write(self, write):
    self.code.append("__write_at_{}:".format(write.line))
    self.generate_expression_evaluator(write.expression)
    self.code.append('\t\tpopq\t%rdi')
    self.code.append('\t\tcall\t__write_stdout')
    self.write_output = True
  def generate_procedure(self, procedure):
    self.code.append("__{}__:".format(procedure.name))
    self.code.append('\t\tpushq\t%rbp')
    self.code.append('\t\tmovq\t%rsp, %rbp')
    self.in_procedure = True
    self.formals = procedure.formals
    self.local_variables = []
    offset = 0
    for variable in procedure.scope.symbols:
      if not self.formals or not variable in self.formals:
        self.local_variables.append(variable)
        offset += procedure.scope.symbols[variable].get_size()
    times = offset / 8
    for i in range(times):
      self.code.append('\t\tpushq\t$0')
    self.code.append('\t\tpushq\t%rbx')
    self.code.append('\t\tleaq\t8(%rsp), %rbx')
    if procedure.instructions:
      self.generate_instructions(procedure.instructions)
    if procedure.return_expression:
      self.generate_expresssion_evaluator(procedure.return_expression)
      self.code.append('\t\tpopq\t%rax')
    self.in_procedure = False
    self.code.append('\t\tpopq\t%rbx')
    self.code.append('\t\tmovq\t%rbp, %rsp')
    self.code.append('\t\tpopq\t%rbp')
    self.code.append('\t\tret')
  def generate_variables(self):
    symbols = self.table.scopes[1].symbols
    printed_data = False
    for name, type_object in symbols.iteritems():
      if not type(type_object) in [symbol_table.Variable, symbol_table.Array, symbol_table.Record,
                                   symbol_table.Integer]:
          continue
      if not printed_data:
        self.code.append('\n\t.data')
        printed_data = True
      self.code.append("{}_:\t\t.space {}".format(name, type_object.get_size()))
  def generate_location_evaluator(self, location):
    pass
  def generate_condition_evaluator(self, condition):
    pass
  def generate_expression_evaluator(self, expression):
    if type(expression.child) is syntax_tree.Number:
      self.code.append("\t\tpushq\t${}".format(expression.child.table_entry.value))
    elif type(expression.child) is syntax_tree.Location:
      self.generate_location_evaluator(expression.child.expression_left)
      self.code.append('\t\tpopq\t%rax')
      self.code.append('\t\tmovq\t(%rax), %rcx')
      self.code.append('\t\tpushq\t%rcx')
    elif type(expression.child) is syntax_tree.Binary:
      if expression.child.operator == '+':
        generate_addition_like_evaluator(expression.child, 'addq')
      elif expression.child.operator == '-':
        generate_addition_like_evaluator(expression.child, 'subq')
      elif expression.child.operator == '*':
        generate_addition_like_evaluator(expression.child, 'imulq')
      elif expression.child.operator == 'DIV':
        generate_division_evaluator(expression.child, '__error_div_by_zero', '%rax')
      else: # MOD
        generate_division_evaluator(expression.child, '__error_mod_by_zero', '%rdx')
    elif type(expression.child) is syntax_tree.Call:
      self.generate_call(expression.child)
      self.code.append('\t\tpushq\t%rax')
  def generate_addition_like_evaluator(self, binary, operation):
    self.generate_expression_evaluator(binary.expression_left)
    if not type(binary.expression_right.child) is syntax_tree.Number:
      self.generate_expression_evaluate(binary.expression_right)
      self.code.append('\t\tpopq\t%rcx')
      self.code.append('\t\tpopq\t%rax')
      self.code.append("\t\t{}\t%rcx, %rax".format(operation))
    else:
      self.code.append('\t\tpopq\t%rax')
      value = binary.expression_right.child.table_entry.value
      self.code.append("\t\t{}\t${}, %rax".format(operation, value))
    self.code.append('\t\tpushq\t%rax')
  def generate_division_evaluator(self, binary, error_function, return_register)
    self.generate_expression_evaluator(binary.expression_left)
    self.generate_expression_evaluator(binary.expression_right)
    self.code.append('\t\tpopq\t%rcx')
    self.code.append('\t\tpopq\t%rax')
    if not type(binary.expression_right.child) is syntax_tree.Number:
      self.code.append('\t\tcmpq\t$0, %rcx')
      self.new_handle()
      self.code.append("\t\tjne\t\t_no_error_{}_".format(self.handle))
      self.code.append("\t\tmovq\t${}, %rdi".format(binary.line))
      self.code.append("\t\tjmp\t\t{}".format(error_function))
    self.code.append('\t\tmovq\t%rax, %rdx')
    self.code.append('\t\tsarq\t$63, %rdx')
    self.code.append('\t\tidivq\t%rcx')
    self.code.append("\t\tpushq\t{}".format(return_register))
  def link_library(self):
    evaluated_to_zero = False
    stderr_printing = False
    write_code = False
    if self.div_by_zero:
      self.code.append(code_library.error_div_zero)
      decl = '_div_error:\t\t.ascii "error: The right side of the DIV expression at ("'
      self.read_only_declarations.append(decl)
      evaluated_to_zero = True
      stderr_printing = True
    if self.mod_by_zero:
      self.code.append(code_library.error_mod_zero)
      decl = '_mod_error:\t\t.ascii "error: The right size of the MOD expression at ("'
      self.read_only_declarations.append(decl)
    if evaluated_to_zero:
      self.read_only_declarations.append('_zero_end:\t\t.ascii ") evaluated to zero\\n"')
    if self.bad_index:
      self.code.append(code_library.error_index_range_code)
      decl = '_index_range:\t.ascii "error: Index out of range: the expression at ("'
      self.read_only_declarations.append(decl)
      self.read_only_declarations.append('_evaluated_to:\t.ascii ") evaluated to "')
      stderr_printing = True
    if self.write_output:
      self.code.append(code_library.write_stdout_code)
      write_code = True
    if stderr_printing:
      self.code.append(code_library.write_stderr_code)
      write_code = True
    if write_code:
      self.code.append(code_library.write_code)
    if self.read_input:
      self.code.append(code_library.read_code)
      self.code.append(code_library.error_input)
      decl = '_bad_input:\t\t.ascii "error: The input was not an integer\\n"'
      self.read_only_declarations.append(decl)

<commit_msg>lazy_generator: Add generate_condition_evaluator<commit_after>import syntax_tree
import symbol_table
import code_library

INTEGER_SIZE = 8

class Lazy_generator(object):
  def generate(self, tree, table):
    self.tree = tree
    self.table = table
    self.reset_library()
    self.handle = -1
    self.in_procedure = False
    self.procedures = {}
    self.read_only_declarations = []
    self.code = ['\t.globl\tmain\n']
    self.code.append('\t.text')
    self.code.append('main:\n')
    if tree:
      self.generate_instructions(tree.instructions)
    self.code.append('__end_program:')
    self.code.append('\t\tmovq\t$60, %rax')
    self.code.append('\t\tsyscall\n')
    printed = []
    while self.procedures:
      to_print = self.procedures
      self.procedures = {}
      for procedure in to_print.values():
        if not procedure in printed:
          self.generate_procedure(procedure)
          printed.append(procedure)
    self.link_library()
    self.generate_variables()
    if self.read_only_declarations:
      self.code.append('\n\t.section .rodata')
      for declaration in self.read_only_declarations:
        self.code.append(declaration)
    return self.code
  def reset_library(self):
    self.div_by_zero = False
    self.mod_by_zero = False
    self.bad_index = False
    self.write_output = False
    self.read_input = False
  def new_handle(self):
    self.handle += 1
    return self.handle
  def generate_instructions(self, instructions):
    for instruction in instructions.instructions:
      if type(instruction.child) is syntax_tree.Assign:
        self.generate_assign(instruction.child)
      elif type(instruction.child) is syntax_tree.If:
        self.generate_if(instruction.child)
      elif type(instruction.child) is syntax_tree.Repeat:
        self.generate_repeat(instruction.child)
      elif type(instruction.child) is syntax_tree.Read:
        self.generate_read(instruction.child)
      elif type(instruction.child) is syntax_tree.Call:
        self.generate_call(instruction.child)
      else: # Write
        self.generate_write(instruction.child)
  def generate_assign(self, assign):
    self.code.append("__assign_at_{}:".format(assign.line))
    self.generate_location_evaluator(assign.location)
    count = assign.location.type_object.get_size() / INTEGER_SIZE
    if count is 1:
      self.generate_expression_evaluator(assign.expression)
    else:
      self.generate_location_evaluator(assign.expression.child)
    self.code.append('\t\tpopq\t%rcx')
    self.code.append('\t\tpopq\t%rax')
    if count > 1:
      self.code.append("\t\tmovq\t${}, %rdi".format(count))
      handle = self.new_handle()
      self.code.append("_loop_{}_:".format(handle))
      self.code.append('\t\tmovq\t(%rcx), %rsi')
      self.code.append('\t\tmovq\t%rsi, (%rax)')
      self.code.append('\t\taddq\t$8, %rcx')
      self.code.append('\t\taddq\t$8, %rax')
      self.code.append('\t\tdecq\t%rdi')
      self.code.append("\t\tjnz\t\t_loop_{}_".format(handle))
    else:
      self.code.append('\t\tmovq\t%rcx, (%rax)')
  def generate_if(self, if_statement):
    self.code.append("__if_at_{}:".format(if_statement.line))
    self.generate_condition_evaluator(if_statement.condition)
    handle = self.handle
    self.code.append("_true_{}_:".format(handle))
    self.generate_instructions(if_statement.instructions_true)
    if if_statement.instructions_false:
      self.code.append("\t\tjmp\t\t_end_{}_".format(handle))
    self.code.append("_false_{}_:".format(handle))
    if if_statement.instructions_false:
      self.generate_instructions(if_statement.instructions_false)
      self.code.append("_end_{}_:".format(handle))
  def generate_repeat(self, repeat):
    label = "__repeat_at_{}".format(repeat.line)
    self.code.append(label + ':')
    self.generate_condition_evaluator(repeat.condition)
    handle = self.handle
    self.code.append("_false_{}_:".format(handle))
    self.generate_instructions(repeat.instructions)
    self.code.append("\t\tjmp\t\t{}".format(label))
    self.code.append("_true_{}_:".format(handle))
  def generate_read(self, read):
    self.code.append("__read_at_{}:".format(read.line))
    self.code.append('\t\tpushq\t%rbx')
    self.generate_location_evaluator(read.location)
    self.code.append('\t\tpopq\t%rbx')
    self.code.append('\t\tcall\t__read')
    self.code.append('\t\tmovq\t%rax, (%rbx)')
    self.code.append('\t\tpopq\t%rbx')
    self.read_input = True
  def generate_call(self, call):
    if call.definition.instructions or call.definition.return_expression:
      if call.actuals:
        for actual in call.actuals:
          if (type(actual.child) is syntax_tree.Location and
              actual.type_object.get_size() > INTEGER_SIZE):
            self.generate_location_evaluator(actual.child)
          else:
            self.generate_expression_evaluator(actual)
      self.code.append("\t\tcall\t__{}__".format(call.definition.name))
      if not call.definition.name in self.procedures:
        self.procedures[call.definition.name] = call.definition
      if call.definition.formals:
        n = len(call.definition.formals)
        self.code.append("\t\taddq\t${}, %rsp".format(n * INTEGER_SIZE))
  def generate_write(self, write):
    self.code.append("__write_at_{}:".format(write.line))
    self.generate_expression_evaluator(write.expression)
    self.code.append('\t\tpopq\t%rdi')
    self.code.append('\t\tcall\t__write_stdout')
    self.write_output = True
  def generate_procedure(self, procedure):
    self.code.append("__{}__:".format(procedure.name))
    self.code.append('\t\tpushq\t%rbp')
    self.code.append('\t\tmovq\t%rsp, %rbp')
    self.in_procedure = True
    self.formals = procedure.formals
    self.local_variables = []
    offset = 0
    for variable in procedure.scope.symbols:
      if not self.formals or not variable in self.formals:
        self.local_variables.append(variable)
        offset += procedure.scope.symbols[variable].get_size()
    times = offset / 8
    for i in range(times):
      self.code.append('\t\tpushq\t$0')
    self.code.append('\t\tpushq\t%rbx')
    self.code.append('\t\tleaq\t8(%rsp), %rbx')
    if procedure.instructions:
      self.generate_instructions(procedure.instructions)
    if procedure.return_expression:
      self.generate_expresssion_evaluator(procedure.return_expression)
      self.code.append('\t\tpopq\t%rax')
    self.in_procedure = False
    self.code.append('\t\tpopq\t%rbx')
    self.code.append('\t\tmovq\t%rbp, %rsp')
    self.code.append('\t\tpopq\t%rbp')
    self.code.append('\t\tret')
  def generate_variables(self):
    symbols = self.table.scopes[1].symbols
    printed_data = False
    for name, type_object in symbols.iteritems():
      if not type(type_object) in [symbol_table.Variable, symbol_table.Array, symbol_table.Record,
                                   symbol_table.Integer]:
          continue
      if not printed_data:
        self.code.append('\n\t.data')
        printed_data = True
      self.code.append("{}_:\t\t.space {}".format(name, type_object.get_size()))
  def generate_location_evaluator(self, location):
    pass
  def generate_condition_evaluator(self, condition):
    self.generate_expression_evaluator(condition.expression_left)
    if not type(condition.expression_right.child) is syntax_tree.Number:
      self.generate_expression_evaluator(condition.expression_right)
      self.code.append('\t\tpopq\t%rcx')
      self.code.append('\t\tpopq\t%rax')
      self.code.append('\t\tcmpq\t%rcx, %rax')
    else:
      self.code.append('\t\tpopq\t%rax')
      value = condition.expression.right.child.table_entry.value
      self.code.append("\t\tcmpq\t${}, %rax".format(value))
    self.new_handle()
    if condition.relation == '=':
      self.code.append("\t\tje\t\t_true_{}_".format(self.handle))
      self.code.append("\t\tjmp\t\t_false_{}_".format(self.handle))
    elif condition.relation == '#':
      self.code.append("\t\tjne\t\t_true_{}_".format(self.handle))
      self.code.append("\t\tjmp\t\t_false_{}_".format(self.handle))
    elif condition.relation == '<':
      self.code.append("\t\tjl\t\t_true_{}_".format(self.handle))
      self.code.append("\t\tjmp\t\t_false_{}_".format(self.handle))
    elif condition.relation == '>':
      self.code.append("\t\tjg\t\t_true_{}_".format(self.handle))
      self.code.append("\t\tjmp\t\t_false_{}_".format(self.handle))
    elif condition.relation == '<=':
      self.code.append("\t\tjle\t\t_true_{}_".format(self.handle))
      self.code.append("\t\tjmp\t\t_false_{}_".format(self.handle))
    else: # >=
      self.code.append("\t\tjge\t\t_true_{}_".format(self.handle))
      self.code.append("\t\tjmp\t\t_false_{}_".format(self.handle))
  def generate_expression_evaluator(self, expression):
    if type(expression.child) is syntax_tree.Number:
      self.code.append("\t\tpushq\t${}".format(expression.child.table_entry.value))
    elif type(expression.child) is syntax_tree.Location:
      self.generate_location_evaluator(expression.child.expression_left)
      self.code.append('\t\tpopq\t%rax')
      self.code.append('\t\tmovq\t(%rax), %rcx')
      self.code.append('\t\tpushq\t%rcx')
    elif type(expression.child) is syntax_tree.Binary:
      if expression.child.operator == '+':
        generate_addition_like_evaluator(expression.child, 'addq')
      elif expression.child.operator == '-':
        generate_addition_like_evaluator(expression.child, 'subq')
      elif expression.child.operator == '*':
        generate_addition_like_evaluator(expression.child, 'imulq')
      elif expression.child.operator == 'DIV':
        generate_division_evaluator(expression.child, '__error_div_by_zero', '%rax')
      else: # MOD
        generate_division_evaluator(expression.child, '__error_mod_by_zero', '%rdx')
    elif type(expression.child) is syntax_tree.Call:
      self.generate_call(expression.child)
      self.code.append('\t\tpushq\t%rax')
  def generate_addition_like_evaluator(self, binary, operation):
    self.generate_expression_evaluator(binary.expression_left)
    if not type(binary.expression_right.child) is syntax_tree.Number:
      self.generate_expression_evaluate(binary.expression_right)
      self.code.append('\t\tpopq\t%rcx')
      self.code.append('\t\tpopq\t%rax')
      self.code.append("\t\t{}\t%rcx, %rax".format(operation))
    else:
      self.code.append('\t\tpopq\t%rax')
      value = binary.expression_right.child.table_entry.value
      self.code.append("\t\t{}\t${}, %rax".format(operation, value))
    self.code.append('\t\tpushq\t%rax')
  def generate_division_evaluator(self, binary, error_function, return_register)
    self.generate_expression_evaluator(binary.expression_left)
    self.generate_expression_evaluator(binary.expression_right)
    self.code.append('\t\tpopq\t%rcx')
    self.code.append('\t\tpopq\t%rax')
    if not type(binary.expression_right.child) is syntax_tree.Number:
      self.code.append('\t\tcmpq\t$0, %rcx')
      self.new_handle()
      self.code.append("\t\tjne\t\t_no_error_{}_".format(self.handle))
      self.code.append("\t\tmovq\t${}, %rdi".format(binary.line))
      self.code.append("\t\tjmp\t\t{}".format(error_function))
    self.code.append('\t\tmovq\t%rax, %rdx')
    self.code.append('\t\tsarq\t$63, %rdx')
    self.code.append('\t\tidivq\t%rcx')
    self.code.append("\t\tpushq\t{}".format(return_register))
  def link_library(self):
    evaluated_to_zero = False
    stderr_printing = False
    write_code = False
    if self.div_by_zero:
      self.code.append(code_library.error_div_zero)
      decl = '_div_error:\t\t.ascii "error: The right side of the DIV expression at ("'
      self.read_only_declarations.append(decl)
      evaluated_to_zero = True
      stderr_printing = True
    if self.mod_by_zero:
      self.code.append(code_library.error_mod_zero)
      decl = '_mod_error:\t\t.ascii "error: The right size of the MOD expression at ("'
      self.read_only_declarations.append(decl)
    if evaluated_to_zero:
      self.read_only_declarations.append('_zero_end:\t\t.ascii ") evaluated to zero\\n"')
    if self.bad_index:
      self.code.append(code_library.error_index_range_code)
      decl = '_index_range:\t.ascii "error: Index out of range: the expression at ("'
      self.read_only_declarations.append(decl)
      self.read_only_declarations.append('_evaluated_to:\t.ascii ") evaluated to "')
      stderr_printing = True
    if self.write_output:
      self.code.append(code_library.write_stdout_code)
      write_code = True
    if stderr_printing:
      self.code.append(code_library.write_stderr_code)
      write_code = True
    if write_code:
      self.code.append(code_library.write_code)
    if self.read_input:
      self.code.append(code_library.read_code)
      self.code.append(code_library.error_input)
      decl = '_bad_input:\t\t.ascii "error: The input was not an integer\\n"'
      self.read_only_declarations.append(decl)

<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
   Copyright 2016 The Trustees of Princeton University

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
"""

import os
import sys
import subprocess
import random
import time
import base64
import json

import testlib
import testconf
import shutil

import syndicate

CAT_PATH = os.path.join(testconf.SYNDICATE_UG_ROOT, "syndicate-cat")
AG_PATH = os.path.join(testconf.SYNDICATE_AG_ROOT, "syndicate-ag")
AG_DRIVER = os.path.join(testconf.SYNDICATE_PYTHON_ROOT, "syndicate/ag/drivers/disk" )

if __name__ == "__main__":

    config_dir, output_dir = testlib.test_setup()
    volume_name = testlib.add_test_volume( config_dir )

    # create AG source dataset.  extract config
    ag_driver = syndicate.util.objects.load_driver( AG_DRIVER, None, include_secrets=False )
    assert 'config' in ag_driver
    ag_config_txt = base64.b64decode(ag_driver['config'])
    ag_config = json.loads(ag_config_txt)

    # path to file...
    assert 'DATASET_DIR' in ag_config
    testdir = ag_config['DATASET_DIR']
    if os.path.exists(testdir):
        shutil.rmtree(testdir)

    os.makedirs(testdir)

    # put a file to publish
    local_paths = []
    expected_datas = []
    for i in xrange(0, 3):
        local_path = testlib.make_tmp_file(16384, "abcdef%s\n" % i, dir=testdir)
        local_fd = open(local_path, "r")
        expected_data = local_fd.read()
        local_fd.close()

        local_paths.append(local_path)
        expected_datas.append(expected_data)

    # start AG
    AG_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "AG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
    testlib.update_gateway( config_dir, AG_gateway_name, "port=31112", "driver=%s" % AG_DRIVER )
    ag_proc, ag_out_path = testlib.start_gateway( config_dir, AG_PATH, testconf.SYNDICATE_ADMIN, volume_name, AG_gateway_name, valgrind=True )
    time.sleep(20)

    if ag_proc.poll() is not None:
        ag_exitcode, ag_out = testlib.stop_gateway( ag_proc, ag_out_path )
        testlib.save_output(output_dir, "syndicate-ag", ag_out)
        raise Exception("%s exited %s" % (AG_PATH, ag_proc.poll()))

    # should cause the AG to get updated that there's a new gateway
    cat_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )

    expected_data_error = False
    for i in xrange(0,len(local_paths)):
        local_path = local_paths[i]
        expected_data = expected_datas[i]

        output_path = "/" + os.path.basename(local_path)
        exitcode, out = testlib.run( CAT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', cat_gateway_name, output_path, valgrind=True )

        testlib.save_output( output_dir, 'syndicate-cat-%s' % i, out )

        if exitcode != 0:
            raise Exception("%s exited %s" % (CAT_PATH, exitcode))

        if expected_data not in out:
            expected_data_error = True
            break

    ag_exitcode, ag_out = testlib.stop_gateway( ag_proc, ag_out_path )
    testlib.save_output(output_dir, "syndicate-ag", ag_out)

    if ag_exitcode != 0:
        raise Exception("%s exited %s" % (AG_PATH, ag_exitcode))

    # check for correctnes
    if expected_data_error:
        raise Exception("data not found in output")

    sys.exit(0)
<commit_msg>cat each file twice<commit_after>#!/usr/bin/env python

"""
   Copyright 2016 The Trustees of Princeton University

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
"""

import os
import sys
import subprocess
import random
import time
import base64
import json

import testlib
import testconf
import shutil

import syndicate

CAT_PATH = os.path.join(testconf.SYNDICATE_UG_ROOT, "syndicate-cat")
AG_PATH = os.path.join(testconf.SYNDICATE_AG_ROOT, "syndicate-ag")
AG_DRIVER = os.path.join(testconf.SYNDICATE_PYTHON_ROOT, "syndicate/ag/drivers/disk" )

if __name__ == "__main__":

    config_dir, output_dir = testlib.test_setup()
    volume_name = testlib.add_test_volume( config_dir )

    # create AG source dataset.  extract config
    ag_driver = syndicate.util.objects.load_driver( AG_DRIVER, None, include_secrets=False )
    assert 'config' in ag_driver
    ag_config_txt = base64.b64decode(ag_driver['config'])
    ag_config = json.loads(ag_config_txt)

    # path to file...
    assert 'DATASET_DIR' in ag_config
    testdir = ag_config['DATASET_DIR']
    if os.path.exists(testdir):
        shutil.rmtree(testdir)

    os.makedirs(testdir)

    # put a file to publish
    local_paths = []
    expected_datas = []
    for i in xrange(0, 3):
        local_path = testlib.make_tmp_file(16384, "abcdef%s\n" % i, dir=testdir)
        local_fd = open(local_path, "r")
        expected_data = local_fd.read()
        local_fd.close()

        local_paths.append(local_path)
        expected_datas.append(expected_data)

    # start AG
    AG_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "AG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
    testlib.update_gateway( config_dir, AG_gateway_name, "port=31112", "driver=%s" % AG_DRIVER )
    ag_proc, ag_out_path = testlib.start_gateway( config_dir, AG_PATH, testconf.SYNDICATE_ADMIN, volume_name, AG_gateway_name, valgrind=True )
    time.sleep(20)

    if ag_proc.poll() is not None:
        ag_exitcode, ag_out = testlib.stop_gateway( ag_proc, ag_out_path )
        testlib.save_output(output_dir, "syndicate-ag", ag_out)
        raise Exception("%s exited %s" % (AG_PATH, ag_proc.poll()))

    # should cause the AG to get updated that there's a new gateway
    cat_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )

    expected_data_error = False
    for i in xrange(0,len(local_paths)):
        # do each file twice
        for j in xrange(0, 2):
            local_path = local_paths[i]
            expected_data = expected_datas[i]

            output_path = "/" + os.path.basename(local_path)
            exitcode, out = testlib.run( CAT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', cat_gateway_name, output_path, valgrind=True )

            testlib.save_output( output_dir, 'syndicate-cat-%s-%s' % (i,j), out )

            if exitcode != 0:
                raise Exception("%s exited %s" % (CAT_PATH, exitcode))

            if expected_data not in out:
                expected_data_error = True
                break

    ag_exitcode, ag_out = testlib.stop_gateway( ag_proc, ag_out_path )
    testlib.save_output(output_dir, "syndicate-ag", ag_out)

    if ag_exitcode != 0:
        raise Exception("%s exited %s" % (AG_PATH, ag_exitcode))

    # check for correctnes
    if expected_data_error:
        raise Exception("data not found in output")

    sys.exit(0)
<|endoftext|>


<commit_before>import shutil
from pathlib import Path
import sys
from typing import Text, Set
from unittest.mock import Mock

import pytest
from _pytest.capture import CaptureFixture
from _pytest.monkeypatch import MonkeyPatch

from rasa.constants import CONFIG_AUTOCONFIGURABLE_KEYS
from rasa.importers import autoconfig
from rasa.utils import io as io_utils

CONFIG_FOLDER = Path("data/test_config")

SOME_CONFIG = CONFIG_FOLDER / "stack_config.yml"
DEFAULT_CONFIG_EN = Path("rasa/importers/default_config_en.yml")
DEFAULT_CONFIG_OTHER_LANGUAGE = Path("rasa/importers/default_config_other_language.yml")

@pytest.mark.parametrize(
    "config_path, autoconfig_keys",
    [
        (Path("rasa/cli/initial_project/config.yml"), {"pipeline", "policies"}),
        (CONFIG_FOLDER / "config_policies_empty.yml", {"policies"}),
        (CONFIG_FOLDER / "config_pipeline_empty.yml", {"pipeline"}),
        (CONFIG_FOLDER / "config_policies_missing.yml", {"policies"}),
        (CONFIG_FOLDER / "config_pipeline_missing.yml", {"pipeline"}),
        (SOME_CONFIG, set()),
    ],
)
def test_get_configuration(
    config_path: Path, autoconfig_keys: Set[Text], monkeypatch: MonkeyPatch
):
    def _auto_configure(_, keys_to_configure: Set[Text]) -> Set[Text]:
        return keys_to_configure

    monkeypatch.setattr(autoconfig, "_dump_config", Mock())
    monkeypatch.setattr(autoconfig, "_auto_configure", _auto_configure)

    config = autoconfig.get_configuration(str(config_path))

    if autoconfig_keys:
        expected_config = _auto_configure(config, autoconfig_keys)
    else:
        expected_config = config

    assert sorted(config) == sorted(expected_config)

@pytest.mark.parametrize("config_file", ("non_existent_config.yml", None))
def test_get_configuration_missing_file(tmp_path: Path, config_file: Text):
    if config_file:
        config_file = tmp_path / "non_existent_config.yml"

    config = autoconfig.get_configuration(str(config_file))

    assert config == {}

@pytest.mark.parametrize(
    "language, keys_to_configure",
    [
        ("en", {"policies"}),
        ("en", {"pipeline"}),
        ("fr", {"pipeline"}),
        ("en", {"policies", "pipeline"}),
    ],
)
def test_auto_configure(language: Text, keys_to_configure: Set[Text]):
    if sys.platform == "win32" or not language == "en":
        default_config = io_utils.read_config_file(DEFAULT_CONFIG_OTHER_LANGUAGE)
    else:
        default_config = io_utils.read_config_file(DEFAULT_CONFIG_EN)

    config = autoconfig._auto_configure({"language": language}, keys_to_configure)

    for k in keys_to_configure:
        assert config[k] == default_config[k]  # given keys are configured correctly

    assert config.get("language") == language
    config.pop("language")
    assert len(config) == len(keys_to_configure)  # no other keys are configured

@pytest.mark.parametrize(
    "config_path, missing_keys",
    [
        (CONFIG_FOLDER / "config_language_only.yml", {"pipeline", "policies"}),
        (CONFIG_FOLDER / "config_policies_missing.yml", {"policies"}),
        (CONFIG_FOLDER / "config_pipeline_missing.yml", {"pipeline"}),
        (SOME_CONFIG, []),
    ],
)
def test_add_missing_config_keys_to_file(
    tmp_path: Path, config_path: Path, missing_keys: Set[Text]
):
    config_file = str(tmp_path / "config.yml")
    shutil.copyfile(str(config_path), config_file)

    autoconfig._add_missing_config_keys_to_file(config_file, missing_keys)

    config_after_addition = io_utils.read_config_file(config_file)

    assert all(key in config_after_addition for key in missing_keys)

def test_dump_config_missing_file(tmp_path: Path, capsys: CaptureFixture):

    config_path = tmp_path / "non_existent_config.yml"

    config = io_utils.read_config_file(str(SOME_CONFIG))

    autoconfig._dump_config(config, str(config_path), set(), {"policies"})

    assert not config_path.exists()

    captured = capsys.readouterr()
    assert "has been removed or modified" in captured.out

# Test a few cases that are known to be potentially tricky (have failed in the past)
@pytest.mark.parametrize(
    "input_file, expected_file, expected_file_windows, autoconfig_keys",
    [
        (
            "config_with_comments.yml",
            "config_with_comments_after_dumping.yml",
            "config_with_comments_after_dumping.yml",
            {"policies"},
        ),  # comments in various positions
        (
            "config_empty_en.yml",
            "config_empty_en_after_dumping.yml",
            "config_empty_en_after_dumping_windows.yml",
            {"policies", "pipeline"},
        ),  # no empty lines
        (
            "config_empty_fr.yml",
            "config_empty_fr_after_dumping.yml",
            "config_empty_fr_after_dumping.yml",
            {"policies", "pipeline"},
        ),  # no empty lines, with different language
        (
            "config_with_comments_after_dumping.yml",
            "config_with_comments_after_dumping.yml",
            "config_with_comments_after_dumping.yml",
            {"policies"},
        ),  # with previous auto config that needs to be overwritten
    ],
)
def test_dump_config(
    tmp_path: Path,
    input_file: Text,
    expected_file: Text,
    expected_file_windows: Text,
    capsys: CaptureFixture,
    autoconfig_keys: Set[Text],
):
    config_file = str(tmp_path / "config.yml")
    shutil.copyfile(str(CONFIG_FOLDER / input_file), config_file)

    autoconfig.get_configuration(config_file)

    actual = io_utils.read_file(config_file)

    if sys.platform == "win32":
        expected = io_utils.read_file(str(CONFIG_FOLDER / expected_file_windows))
    else:
        expected = io_utils.read_file(str(CONFIG_FOLDER / expected_file))

    assert actual == expected

    captured = capsys.readouterr()
    assert "does not exist or is empty" not in captured.out

    for k in CONFIG_AUTOCONFIGURABLE_KEYS:
        if k in autoconfig_keys:
            assert k in captured.out
        else:
            assert k not in captured.out

@pytest.mark.parametrize(
    "input_file, expected_file, expected_file_windows, training_type",
    [
        (
                "config_empty_en.yml",
                "config_empty_en_after_dumping.yml",
                "config_empty_en_after_dumping_windows.yml",
                autoconfig.TrainingType.BOTH,
        ),
        (
                "config_empty_en.yml",
                "config_empty_en_after_dumping_core.yml",
                "config_empty_en_after_dumping_windows_core.yml",
                autoconfig.TrainingType.CORE,
        ),
        (
                "config_empty_en.yml",
                "config_empty_en_after_dumping_nlu.yml",
                "config_empty_en_after_dumping_windows_nlu.yml",
                autoconfig.TrainingType.NLU,
        ),
    ],
)
def test_get_configuration_for_different_training_types(
        tmp_path: Path,
        input_file: Text,
        expected_file: Text,
        expected_file_windows: Text,
        training_type: autoconfig.TrainingType,
):
    config_file = str(tmp_path / "config.yml")
    shutil.copyfile(str(CONFIG_FOLDER / input_file), config_file)

    autoconfig.get_configuration(config_file, training_type)

    actual = io_utils.read_file(config_file)

    if sys.platform == "win32":
        expected = io_utils.read_file(str(CONFIG_FOLDER / expected_file_windows))
    else:
        expected = io_utils.read_file(str(CONFIG_FOLDER / expected_file))

    assert actual == expected
<commit_msg>updated test code style<commit_after>import shutil
from pathlib import Path
import sys
from typing import Text, Set
from unittest.mock import Mock

import pytest
from _pytest.capture import CaptureFixture
from _pytest.monkeypatch import MonkeyPatch

from rasa.constants import CONFIG_AUTOCONFIGURABLE_KEYS
from rasa.importers import autoconfig
from rasa.utils import io as io_utils

CONFIG_FOLDER = Path("data/test_config")

SOME_CONFIG = CONFIG_FOLDER / "stack_config.yml"
DEFAULT_CONFIG_EN = Path("rasa/importers/default_config_en.yml")
DEFAULT_CONFIG_OTHER_LANGUAGE = Path("rasa/importers/default_config_other_language.yml")

@pytest.mark.parametrize(
    "config_path, autoconfig_keys",
    [
        (Path("rasa/cli/initial_project/config.yml"), {"pipeline", "policies"}),
        (CONFIG_FOLDER / "config_policies_empty.yml", {"policies"}),
        (CONFIG_FOLDER / "config_pipeline_empty.yml", {"pipeline"}),
        (CONFIG_FOLDER / "config_policies_missing.yml", {"policies"}),
        (CONFIG_FOLDER / "config_pipeline_missing.yml", {"pipeline"}),
        (SOME_CONFIG, set()),
    ],
)
def test_get_configuration(
    config_path: Path, autoconfig_keys: Set[Text], monkeypatch: MonkeyPatch
):
    def _auto_configure(_, keys_to_configure: Set[Text]) -> Set[Text]:
        return keys_to_configure

    monkeypatch.setattr(autoconfig, "_dump_config", Mock())
    monkeypatch.setattr(autoconfig, "_auto_configure", _auto_configure)

    config = autoconfig.get_configuration(str(config_path))

    if autoconfig_keys:
        expected_config = _auto_configure(config, autoconfig_keys)
    else:
        expected_config = config

    assert sorted(config) == sorted(expected_config)

@pytest.mark.parametrize("config_file", ("non_existent_config.yml", None))
def test_get_configuration_missing_file(tmp_path: Path, config_file: Text):
    if config_file:
        config_file = tmp_path / "non_existent_config.yml"

    config = autoconfig.get_configuration(str(config_file))

    assert config == {}

@pytest.mark.parametrize(
    "language, keys_to_configure",
    [
        ("en", {"policies"}),
        ("en", {"pipeline"}),
        ("fr", {"pipeline"}),
        ("en", {"policies", "pipeline"}),
    ],
)
def test_auto_configure(language: Text, keys_to_configure: Set[Text]):
    if sys.platform == "win32" or not language == "en":
        default_config = io_utils.read_config_file(DEFAULT_CONFIG_OTHER_LANGUAGE)
    else:
        default_config = io_utils.read_config_file(DEFAULT_CONFIG_EN)

    config = autoconfig._auto_configure({"language": language}, keys_to_configure)

    for k in keys_to_configure:
        assert config[k] == default_config[k]  # given keys are configured correctly

    assert config.get("language") == language
    config.pop("language")
    assert len(config) == len(keys_to_configure)  # no other keys are configured

@pytest.mark.parametrize(
    "config_path, missing_keys",
    [
        (CONFIG_FOLDER / "config_language_only.yml", {"pipeline", "policies"}),
        (CONFIG_FOLDER / "config_policies_missing.yml", {"policies"}),
        (CONFIG_FOLDER / "config_pipeline_missing.yml", {"pipeline"}),
        (SOME_CONFIG, []),
    ],
)
def test_add_missing_config_keys_to_file(
    tmp_path: Path, config_path: Path, missing_keys: Set[Text]
):
    config_file = str(tmp_path / "config.yml")
    shutil.copyfile(str(config_path), config_file)

    autoconfig._add_missing_config_keys_to_file(config_file, missing_keys)

    config_after_addition = io_utils.read_config_file(config_file)

    assert all(key in config_after_addition for key in missing_keys)

def test_dump_config_missing_file(tmp_path: Path, capsys: CaptureFixture):

    config_path = tmp_path / "non_existent_config.yml"

    config = io_utils.read_config_file(str(SOME_CONFIG))

    autoconfig._dump_config(config, str(config_path), set(), {"policies"})

    assert not config_path.exists()

    captured = capsys.readouterr()
    assert "has been removed or modified" in captured.out

# Test a few cases that are known to be potentially tricky (have failed in the past)
@pytest.mark.parametrize(
    "input_file, expected_file, expected_file_windows, autoconfig_keys",
    [
        (
            "config_with_comments.yml",
            "config_with_comments_after_dumping.yml",
            "config_with_comments_after_dumping.yml",
            {"policies"},
        ),  # comments in various positions
        (
            "config_empty_en.yml",
            "config_empty_en_after_dumping.yml",
            "config_empty_en_after_dumping_windows.yml",
            {"policies", "pipeline"},
        ),  # no empty lines
        (
            "config_empty_fr.yml",
            "config_empty_fr_after_dumping.yml",
            "config_empty_fr_after_dumping.yml",
            {"policies", "pipeline"},
        ),  # no empty lines, with different language
        (
            "config_with_comments_after_dumping.yml",
            "config_with_comments_after_dumping.yml",
            "config_with_comments_after_dumping.yml",
            {"policies"},
        ),  # with previous auto config that needs to be overwritten
    ],
)
def test_dump_config(
    tmp_path: Path,
    input_file: Text,
    expected_file: Text,
    expected_file_windows: Text,
    capsys: CaptureFixture,
    autoconfig_keys: Set[Text],
):
    config_file = str(tmp_path / "config.yml")
    shutil.copyfile(str(CONFIG_FOLDER / input_file), config_file)

    autoconfig.get_configuration(config_file)

    actual = io_utils.read_file(config_file)

    if sys.platform == "win32":
        expected = io_utils.read_file(str(CONFIG_FOLDER / expected_file_windows))
    else:
        expected = io_utils.read_file(str(CONFIG_FOLDER / expected_file))

    assert actual == expected

    captured = capsys.readouterr()
    assert "does not exist or is empty" not in captured.out

    for k in CONFIG_AUTOCONFIGURABLE_KEYS:
        if k in autoconfig_keys:
            assert k in captured.out
        else:
            assert k not in captured.out

@pytest.mark.parametrize(
    "input_file, expected_file, expected_file_windows, training_type",
    [
        (
            "config_empty_en.yml",
            "config_empty_en_after_dumping.yml",
            "config_empty_en_after_dumping_windows.yml",
            autoconfig.TrainingType.BOTH,
        ),
        (
            "config_empty_en.yml",
            "config_empty_en_after_dumping_core.yml",
            "config_empty_en_after_dumping_windows_core.yml",
            autoconfig.TrainingType.CORE,
        ),
        (
            "config_empty_en.yml",
            "config_empty_en_after_dumping_nlu.yml",
            "config_empty_en_after_dumping_windows_nlu.yml",
            autoconfig.TrainingType.NLU,
        ),
    ],
)
def test_get_configuration_for_different_training_types(
    tmp_path: Path,
    input_file: Text,
    expected_file: Text,
    expected_file_windows: Text,
    training_type: autoconfig.TrainingType,
):
    config_file = str(tmp_path / "config.yml")
    shutil.copyfile(str(CONFIG_FOLDER / input_file), config_file)

    autoconfig.get_configuration(config_file, training_type)

    actual = io_utils.read_file(config_file)

    if sys.platform == "win32":
        expected = io_utils.read_file(str(CONFIG_FOLDER / expected_file_windows))
    else:
        expected = io_utils.read_file(str(CONFIG_FOLDER / expected_file))

    assert actual == expected
<|endoftext|>


<commit_before>import aiohttp

async def aiohttp_request(loop, method, url, output='text', **kwargs):  # NOQA: E999
    async with aiohttp.ClientSession(loop=loop) as session:  # NOQA: E999
        response = await session.request(method, url, **kwargs)  # NOQA: E999
        if output == 'text':
            content = await response.text()  # NOQA: E999
        elif output == 'json':
            content = await response.json()  # NOQA: E999
        elif output == 'raw':
            content = await response.read()  # NOQA: E999
        return response, content
<commit_msg>Fix aiohttp_request to properly perform aiohttp requests<commit_after>import aiohttp

async def aiohttp_request(loop, method, url, output='text', **kwargs):  # NOQA: E999
    async with aiohttp.ClientSession(loop=loop) as session:  # NOQA: E999
        async with session.request(method, url, **kwargs) as response:  # NOQA: E999
            if output == 'text':
                content = await response.text()  # NOQA: E999
            elif output == 'json':
                content = await response.json()  # NOQA: E999
            elif output == 'raw':
                content = await response.read()  # NOQA: E999
            return response, content
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Nicole Thomas <nicole@saltstack.com>`
'''

# Import Salt Libs
import traceback
import integration
import os
import copy
from salt.output import display_output

# Import Salt Testing Libs
from salttesting.helpers import ensure_in_syspath

ensure_in_syspath('../../')

class OutputReturnTest(integration.ShellCase):
    '''
    Integration tests to ensure outputters return their expected format.
    Tests against situations where the loader might not be returning the
    right outputter even though it was explicitly requested.
    '''

    def test_output_json(self):
        '''
        Tests the return of json-formatted data
        '''
        expected = ['{', '    "local": true', '}']
        ret = self.run_call('test.ping --out=json')
        self.assertEqual(ret, expected)

    def test_output_nested(self):
        '''
        Tests the return of nested-formatted data
        '''
        expected = ['local:', '    True']
        ret = self.run_call('test.ping --out=nested')
        self.assertEqual(ret, expected)

    def test_output_quiet(self):
        '''
        Tests the return of an out=quiet query
        '''
        expected = []
        ret = self.run_call('test.ping --out=quiet')
        self.assertEqual(ret, expected)

    def test_output_pprint(self):
        '''
        Tests the return of pprint-formatted data
        '''
        expected = ["{'local': True}"]
        ret = self.run_call('test.ping --out=pprint')
        self.assertEqual(ret, expected)

    def test_output_raw(self):
        '''
        Tests the return of raw-formatted data
        '''
        expected = ["{'local': True}"]
        ret = self.run_call('test.ping --out=raw')
        self.assertEqual(ret, expected)

    def test_output_txt(self):
        '''
        Tests the return of txt-formatted data
        '''
        expected = ['local: True']
        ret = self.run_call('test.ping --out=txt')
        self.assertEqual(ret, expected)

    def test_output_yaml(self):
        '''
        Tests the return of yaml-formatted data
        '''
        expected = ['local: true']
        ret = self.run_call('test.ping --out=yaml')
        self.assertEqual(ret, expected)

    def test_output_unicodebad(self):
        '''
        Tests outputter reliability with utf8
        '''
        opts = copy.deepcopy(self.minion_opts)
        opts['output_file'] = os.path.join(
            self.minion_opts['root_dir'], 'outputtest')
        data = {'foo': {'result': False,
                        'aaa': 'azerzaeréééé',
                        'comment': u'ééééàààà'}}
        try:
            # this should not raises UnicodeEncodeError
            display_output(data, opts=self.minion_opts)
            self.assertTrue(True)
        except Exception:
            # display trace in error message for debugging on jenkins
            trace = traceback.format_exc()
            self.assertEqual(trace, '')

if __name__ == '__main__':
    from integration import run_tests
    run_tests(OutputReturnTest)
<commit_msg>Fix imports<commit_after># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Nicole Thomas <nicole@saltstack.com>`
'''

# Import Salt Libs
import traceback
import os
import copy

# Import Salt Testing Libs
from salttesting.helpers import ensure_in_syspath

ensure_in_syspath('../../')

# Import Salt libs
import integration
from salt.output import display_output

class OutputReturnTest(integration.ShellCase):
    '''
    Integration tests to ensure outputters return their expected format.
    Tests against situations where the loader might not be returning the
    right outputter even though it was explicitly requested.
    '''

    def test_output_json(self):
        '''
        Tests the return of json-formatted data
        '''
        expected = ['{', '    "local": true', '}']
        ret = self.run_call('test.ping --out=json')
        self.assertEqual(ret, expected)

    def test_output_nested(self):
        '''
        Tests the return of nested-formatted data
        '''
        expected = ['local:', '    True']
        ret = self.run_call('test.ping --out=nested')
        self.assertEqual(ret, expected)

    def test_output_quiet(self):
        '''
        Tests the return of an out=quiet query
        '''
        expected = []
        ret = self.run_call('test.ping --out=quiet')
        self.assertEqual(ret, expected)

    def test_output_pprint(self):
        '''
        Tests the return of pprint-formatted data
        '''
        expected = ["{'local': True}"]
        ret = self.run_call('test.ping --out=pprint')
        self.assertEqual(ret, expected)

    def test_output_raw(self):
        '''
        Tests the return of raw-formatted data
        '''
        expected = ["{'local': True}"]
        ret = self.run_call('test.ping --out=raw')
        self.assertEqual(ret, expected)

    def test_output_txt(self):
        '''
        Tests the return of txt-formatted data
        '''
        expected = ['local: True']
        ret = self.run_call('test.ping --out=txt')
        self.assertEqual(ret, expected)

    def test_output_yaml(self):
        '''
        Tests the return of yaml-formatted data
        '''
        expected = ['local: true']
        ret = self.run_call('test.ping --out=yaml')
        self.assertEqual(ret, expected)

    def test_output_unicodebad(self):
        '''
        Tests outputter reliability with utf8
        '''
        opts = copy.deepcopy(self.minion_opts)
        opts['output_file'] = os.path.join(
            self.minion_opts['root_dir'], 'outputtest')
        data = {'foo': {'result': False,
                        'aaa': 'azerzaeréééé',
                        'comment': u'ééééàààà'}}
        try:
            # this should not raises UnicodeEncodeError
            display_output(data, opts=self.minion_opts)
            self.assertTrue(True)
        except Exception:
            # display trace in error message for debugging on jenkins
            trace = traceback.format_exc()
            self.assertEqual(trace, '')

if __name__ == '__main__':
    from integration import run_tests
    run_tests(OutputReturnTest)
<|endoftext|>


<commit_before># Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
import collections
from decimal import Decimal

import boto3.session
from boto3.dynamodb.types import Binary
from boto3.dynamodb.conditions import Attr, Key
from tests import unittest, unique_id

class BaseDynamoDBTest(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.session = boto3.session.Session(region_name='us-west-2')
        cls.dynamodb = cls.session.resource('dynamodb')
        cls.table_name = unique_id('boto3db')
        cls.item_data = {
            'MyHashKey': 'mykey',
            'MyNull': None,
            'MyBool': True,
            'MyString': 'mystring',
            'MyNumber': Decimal('1.25'),
            'MyBinary': Binary(b'\x01'),
            'MyStringSet': set(['foo']),
            'MyNumberSet': set([Decimal('1.25')]),
            'MyBinarySet': set([Binary(b'\x01')]),
            'MyList': ['foo'],
            'MyMap': {'foo': 'bar'}
        }
        cls.table = cls.dynamodb.create_table(
            TableName=cls.table_name,
            ProvisionedThroughput={"ReadCapacityUnits": 5,
                                   "WriteCapacityUnits": 5},
            KeySchema=[{"AttributeName": "MyHashKey", "KeyType": "HASH"}],
            AttributeDefinitions=[{"AttributeName": "MyHashKey",
                                   "AttributeType": "S"}])
        waiter = cls.dynamodb.meta.client.get_waiter('table_exists')
        waiter.wait(TableName=cls.table_name)

    @classmethod
    def tearDownClass(cls):
        cls.table.delete()

class TestDynamoDBTypes(BaseDynamoDBTest):
    def test_put_get_item(self):
        self.table.put_item(Item=self.item_data)
        self.addCleanup(self.table.delete_item, Key={'MyHashKey': 'mykey'})
        response = self.table.get_item(Key={'MyHashKey': 'mykey'})
        self.assertEqual(response['Item'], self.item_data)

class TestDynamoDBConditions(BaseDynamoDBTest):
    @classmethod
    def setUpClass(cls):
        super(TestDynamoDBConditions, cls).setUpClass()
        cls.table.put_item(Item=cls.item_data)

    @classmethod
    def tearDownClass(cls):
        cls.table.delete_item(Key={'MyHashKey': 'mykey'})
        super(TestDynamoDBConditions, cls).tearDownClass()

    def test_filter_expression(self):
        r = self.table.scan(
            FilterExpression=Attr('MyHashKey').eq('mykey'))
        self.assertEqual(r['Items'][0]['MyHashKey'], 'mykey')

    def test_key_condition_expression(self):
        r = self.table.query(
            KeyConditionExpression=Key('MyHashKey').eq('mykey'))
        self.assertEqual(r['Items'][0]['MyHashKey'], 'mykey')

    def test_key_condition_with_filter_condition_expression(self):
        r = self.table.query(
            KeyConditionExpression=Key('MyHashKey').eq('mykey'),
            FilterExpression=Attr('MyString').eq('mystring'))
        self.assertEqual(r['Items'][0]['MyString'], 'mystring')

    def test_condition_less_than(self):
        r = self.table.scan(
            FilterExpression=Attr('MyNumber').lt(Decimal('1.26')))
        self.assertTrue(r['Items'][0]['MyNumber'] < Decimal('1.26'))

    def test_condition_less_than_equal(self):
        r = self.table.scan(
            FilterExpression=Attr('MyNumber').lte(Decimal('1.26')))
        self.assertTrue(r['Items'][0]['MyNumber'] <= Decimal('1.26'))

    def test_condition_greater_than(self):
        r = self.table.scan(
            FilterExpression=Attr('MyNumber').gt(Decimal('1.24')))
        self.assertTrue(r['Items'][0]['MyNumber'] > Decimal('1.24'))

    def test_condition_greater_than_equal(self):
        r = self.table.scan(
            FilterExpression=Attr('MyNumber').gte(Decimal('1.24')))
        self.assertTrue(r['Items'][0]['MyNumber'] >= Decimal('1.24'))

    def test_condition_begins_with(self):
        r = self.table.scan(
            FilterExpression=Attr('MyString').begins_with('my'))
        self.assertTrue(r['Items'][0]['MyString'].startswith('my'))

    def test_condition_between(self):
        r = self.table.scan(
            FilterExpression=Attr('MyNumber').between(
                Decimal('1.24'), Decimal('1.26')))
        self.assertTrue(r['Items'][0]['MyNumber'] > Decimal('1.24'))
        self.assertTrue(r['Items'][0]['MyNumber'] < Decimal('1.26'))

    def test_condition_not_equal(self):
        r = self.table.scan(
            FilterExpression=Attr('MyHashKey').ne('notmykey'))
        self.assertNotEqual(r['Items'][0]['MyHashKey'], 'notmykey')

    def test_condition_in(self):
        r = self.table.scan(
            FilterExpression=Attr('MyHashKey').is_in(['notmykey', 'mykey']))
        self.assertIn(r['Items'][0]['MyHashKey'], ['notmykey', 'mykey'])

    def test_condition_exists(self):
        r = self.table.scan(
            FilterExpression=Attr('MyString').exists())
        self.assertIn('MyString', r['Items'][0])

    def test_condition_not_exists(self):
        r = self.table.scan(
            FilterExpression=Attr('MyFakeKey').not_exists())
        self.assertNotIn('MyFakeKey', r['Items'][0])

    def test_condition_contains(self):
        r = self.table.scan(
            FilterExpression=Attr('MyString').contains('my'))
        self.assertIn('my', r['Items'][0]['MyString'])

    def test_condition_size(self):
        r = self.table.scan(
            FilterExpression=Attr('MyString').size().eq(len('mystring')))
        self.assertEqual(len(r['Items'][0]['MyString']), len('mystring'))

    def test_condition_attribute_type(self):
        r = self.table.scan(
            FilterExpression=Attr('MyMap').attribute_type('M'))
        self.assertIsInstance(r['Items'][0]['MyMap'], collections.Mapping)

    def test_condition_and(self):
        r = self.table.scan(
            FilterExpression=(Attr('MyHashKey').eq('mykey') &
                              Attr('MyString').eq('mystring')))
        item = r['Items'][0]
        self.assertTrue(
            item['MyHashKey'] == 'mykey' and item['MyString'] == 'mystring')

    def test_condition_or(self):
        r = self.table.scan(
            FilterExpression=(Attr('MyHashKey').eq('mykey2') |
                              Attr('MyString').eq('mystring')))
        item = r['Items'][0]
        self.assertTrue(
            item['MyHashKey'] == 'mykey2' or item['MyString'] == 'mystring')

    def test_condition_not(self):
        r = self.table.scan(
            FilterExpression=(~Attr('MyHashKey').eq('mykey2')))
        item = r['Items'][0]
        self.assertTrue(item['MyHashKey'] != 'mykey2')

    def test_condition_in_map(self):
        r = self.table.scan(
            FilterExpression=Attr('MyMap.foo').eq('bar'))
        self.assertEqual(r['Items'][0]['MyMap']['foo'], 'bar')

    def test_condition_in_list(self):
        r = self.table.scan(
            FilterExpression=Attr('MyList[0]').eq('foo'))
        self.assertEqual(r['Items'][0]['MyList'][0], 'foo')

class TestDynamodbBatchWrite(BaseDynamoDBTest):
    def test_batch_write_items(self):
        num_elements = 1000
        items = []
        for i in range(num_elements):
            items.append({'MyHashKey': 'foo%s' % i,
                          'OtherKey': 'bar%s' % i})
        with self.table.batch_writer() as batch:
            for item in items:
                batch.put_item(Item=item)

        # Verify all the items were added to dynamodb.
        for obj in self.table.scan()['Items']:
            self.assertIn(obj, items)
<commit_msg>Use consistent reads in integ tests<commit_after># Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
import collections
from decimal import Decimal

import boto3.session
from boto3.dynamodb.types import Binary
from boto3.dynamodb.conditions import Attr, Key
from tests import unittest, unique_id

class BaseDynamoDBTest(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.session = boto3.session.Session(region_name='us-west-2')
        cls.dynamodb = cls.session.resource('dynamodb')
        cls.table_name = unique_id('boto3db')
        cls.item_data = {
            'MyHashKey': 'mykey',
            'MyNull': None,
            'MyBool': True,
            'MyString': 'mystring',
            'MyNumber': Decimal('1.25'),
            'MyBinary': Binary(b'\x01'),
            'MyStringSet': set(['foo']),
            'MyNumberSet': set([Decimal('1.25')]),
            'MyBinarySet': set([Binary(b'\x01')]),
            'MyList': ['foo'],
            'MyMap': {'foo': 'bar'}
        }
        cls.table = cls.dynamodb.create_table(
            TableName=cls.table_name,
            ProvisionedThroughput={"ReadCapacityUnits": 5,
                                   "WriteCapacityUnits": 5},
            KeySchema=[{"AttributeName": "MyHashKey", "KeyType": "HASH"}],
            AttributeDefinitions=[{"AttributeName": "MyHashKey",
                                   "AttributeType": "S"}])
        waiter = cls.dynamodb.meta.client.get_waiter('table_exists')
        waiter.wait(TableName=cls.table_name)

    @classmethod
    def tearDownClass(cls):
        cls.table.delete()

class TestDynamoDBTypes(BaseDynamoDBTest):
    def test_put_get_item(self):
        self.table.put_item(Item=self.item_data)
        self.addCleanup(self.table.delete_item, Key={'MyHashKey': 'mykey'})
        response = self.table.get_item(Key={'MyHashKey': 'mykey'},
                                       ConsistentRead=True)
        self.assertEqual(response['Item'], self.item_data)

class TestDynamoDBConditions(BaseDynamoDBTest):
    @classmethod
    def setUpClass(cls):
        super(TestDynamoDBConditions, cls).setUpClass()
        cls.table.put_item(Item=cls.item_data)

    @classmethod
    def tearDownClass(cls):
        cls.table.delete_item(Key={'MyHashKey': 'mykey'})
        super(TestDynamoDBConditions, cls).tearDownClass()

    def test_filter_expression(self):
        r = self.table.scan(
            FilterExpression=Attr('MyHashKey').eq('mykey'))
        self.assertEqual(r['Items'][0]['MyHashKey'], 'mykey')

    def test_key_condition_expression(self):
        r = self.table.query(
            KeyConditionExpression=Key('MyHashKey').eq('mykey'))
        self.assertEqual(r['Items'][0]['MyHashKey'], 'mykey')

    def test_key_condition_with_filter_condition_expression(self):
        r = self.table.query(
            KeyConditionExpression=Key('MyHashKey').eq('mykey'),
            FilterExpression=Attr('MyString').eq('mystring'))
        self.assertEqual(r['Items'][0]['MyString'], 'mystring')

    def test_condition_less_than(self):
        r = self.table.scan(
            FilterExpression=Attr('MyNumber').lt(Decimal('1.26')))
        self.assertTrue(r['Items'][0]['MyNumber'] < Decimal('1.26'))

    def test_condition_less_than_equal(self):
        r = self.table.scan(
            FilterExpression=Attr('MyNumber').lte(Decimal('1.26')))
        self.assertTrue(r['Items'][0]['MyNumber'] <= Decimal('1.26'))

    def test_condition_greater_than(self):
        r = self.table.scan(
            FilterExpression=Attr('MyNumber').gt(Decimal('1.24')))
        self.assertTrue(r['Items'][0]['MyNumber'] > Decimal('1.24'))

    def test_condition_greater_than_equal(self):
        r = self.table.scan(
            FilterExpression=Attr('MyNumber').gte(Decimal('1.24')))
        self.assertTrue(r['Items'][0]['MyNumber'] >= Decimal('1.24'))

    def test_condition_begins_with(self):
        r = self.table.scan(
            FilterExpression=Attr('MyString').begins_with('my'))
        self.assertTrue(r['Items'][0]['MyString'].startswith('my'))

    def test_condition_between(self):
        r = self.table.scan(
            FilterExpression=Attr('MyNumber').between(
                Decimal('1.24'), Decimal('1.26')))
        self.assertTrue(r['Items'][0]['MyNumber'] > Decimal('1.24'))
        self.assertTrue(r['Items'][0]['MyNumber'] < Decimal('1.26'))

    def test_condition_not_equal(self):
        r = self.table.scan(
            FilterExpression=Attr('MyHashKey').ne('notmykey'))
        self.assertNotEqual(r['Items'][0]['MyHashKey'], 'notmykey')

    def test_condition_in(self):
        r = self.table.scan(
            FilterExpression=Attr('MyHashKey').is_in(['notmykey', 'mykey']))
        self.assertIn(r['Items'][0]['MyHashKey'], ['notmykey', 'mykey'])

    def test_condition_exists(self):
        r = self.table.scan(
            FilterExpression=Attr('MyString').exists())
        self.assertIn('MyString', r['Items'][0])

    def test_condition_not_exists(self):
        r = self.table.scan(
            FilterExpression=Attr('MyFakeKey').not_exists())
        self.assertNotIn('MyFakeKey', r['Items'][0])

    def test_condition_contains(self):
        r = self.table.scan(
            FilterExpression=Attr('MyString').contains('my'))
        self.assertIn('my', r['Items'][0]['MyString'])

    def test_condition_size(self):
        r = self.table.scan(
            FilterExpression=Attr('MyString').size().eq(len('mystring')))
        self.assertEqual(len(r['Items'][0]['MyString']), len('mystring'))

    def test_condition_attribute_type(self):
        r = self.table.scan(
            FilterExpression=Attr('MyMap').attribute_type('M'))
        self.assertIsInstance(r['Items'][0]['MyMap'], collections.Mapping)

    def test_condition_and(self):
        r = self.table.scan(
            FilterExpression=(Attr('MyHashKey').eq('mykey') &
                              Attr('MyString').eq('mystring')))
        item = r['Items'][0]
        self.assertTrue(
            item['MyHashKey'] == 'mykey' and item['MyString'] == 'mystring')

    def test_condition_or(self):
        r = self.table.scan(
            FilterExpression=(Attr('MyHashKey').eq('mykey2') |
                              Attr('MyString').eq('mystring')))
        item = r['Items'][0]
        self.assertTrue(
            item['MyHashKey'] == 'mykey2' or item['MyString'] == 'mystring')

    def test_condition_not(self):
        r = self.table.scan(
            FilterExpression=(~Attr('MyHashKey').eq('mykey2')))
        item = r['Items'][0]
        self.assertTrue(item['MyHashKey'] != 'mykey2')

    def test_condition_in_map(self):
        r = self.table.scan(
            FilterExpression=Attr('MyMap.foo').eq('bar'))
        self.assertEqual(r['Items'][0]['MyMap']['foo'], 'bar')

    def test_condition_in_list(self):
        r = self.table.scan(
            FilterExpression=Attr('MyList[0]').eq('foo'))
        self.assertEqual(r['Items'][0]['MyList'][0], 'foo')

class TestDynamodbBatchWrite(BaseDynamoDBTest):
    def test_batch_write_items(self):
        num_elements = 1000
        items = []
        for i in range(num_elements):
            items.append({'MyHashKey': 'foo%s' % i,
                          'OtherKey': 'bar%s' % i})
        with self.table.batch_writer() as batch:
            for item in items:
                batch.put_item(Item=item)

        # Verify all the items were added to dynamodb.
        for obj in self.table.scan()['Items']:
            self.assertIn(obj, items)
<|endoftext|>


<commit_before># String operations which don't require allocation
import micropython

micropython.heap_lock()

b"" + b""
b"" + b"1"
b"2" + b""

"" + ""
"" + "1"
"2" + ""

micropython.heap_unlock()
<commit_msg>tests/heapalloc_str: Test no-replacement case for str.replace().<commit_after># String operations which don't require allocation
import micropython

micropython.heap_lock()

# Concatenating empty string returns original string
b"" + b""
b"" + b"1"
b"2" + b""

"" + ""
"" + "1"
"2" + ""

# If no replacements done, returns original string
"foo".replace(",", "_")

micropython.heap_unlock()
<|endoftext|>


<commit_before>import datetime
import os
import pickle
import subprocess
import sys
import warnings

from django.core.files.temp import NamedTemporaryFile
from django.db import DJANGO_VERSION_PICKLE_KEY, models
from django.test import TestCase, mock
from django.utils._os import upath
from django.utils.encoding import force_text
from django.utils.version import get_version

from .models import Article

class ModelPickleTestCase(TestCase):
    def test_missing_django_version_unpickling(self):
        """
        #21430 -- Verifies a warning is raised for models that are
        unpickled without a Django version
        """
        class MissingDjangoVersion(models.Model):
            title = models.CharField(max_length=10)

            def __reduce__(self):
                reduce_list = super(MissingDjangoVersion, self).__reduce__()
                data = reduce_list[-1]
                del data[DJANGO_VERSION_PICKLE_KEY]
                return reduce_list

        p = MissingDjangoVersion(title="FooBar")
        with warnings.catch_warnings(record=True) as recorded:
            pickle.loads(pickle.dumps(p))
            msg = force_text(recorded.pop().message)
            self.assertEqual(msg,
                "Pickled model instance's Django version is not specified.")

    def test_unsupported_unpickle(self):
        """
        #21430 -- Verifies a warning is raised for models that are
        unpickled with a different Django version than the current
        """
        class DifferentDjangoVersion(models.Model):
            title = models.CharField(max_length=10)

            def __reduce__(self):
                reduce_list = super(DifferentDjangoVersion, self).__reduce__()
                data = reduce_list[-1]
                data[DJANGO_VERSION_PICKLE_KEY] = '1.0'
                return reduce_list

        p = DifferentDjangoVersion(title="FooBar")
        with warnings.catch_warnings(record=True) as recorded:
            pickle.loads(pickle.dumps(p))
            msg = force_text(recorded.pop().message)
            self.assertEqual(
                msg,
                "Pickled model instance's Django version 1.0 does not "
                "match the current version %s." % get_version()
            )

    def test_unpickling_when_appregistrynotready(self):
        """
        #24007 -- Verifies that a pickled model can be unpickled without having
        to manually setup the apps registry beforehand.
        """
        script_template = """#!/usr/bin/env python
import pickle

from django.conf import settings

data = %r

settings.configure(DEBUG=False, INSTALLED_APPS=['model_regress'], SECRET_KEY = "blah")
article = pickle.loads(data)
print(article.headline)"""
        a = Article.objects.create(
            headline="Some object",
            pub_date=datetime.datetime.now(),
            article_text="This is an article",
        )

        with NamedTemporaryFile(mode='w+', suffix=".py") as script:
            script.write(script_template % pickle.dumps(a))
            script.flush()
            # A path to model_regress must be set in PYTHONPATH
            model_regress_dir = os.path.dirname(upath(__file__))
            model_regress_path = os.path.abspath(model_regress_dir)
            tests_path = os.path.split(model_regress_path)[0]
            pythonpath = os.environ.get('PYTHONPATH', '')
            pythonpath = os.pathsep.join([tests_path, pythonpath])

            with mock.patch.dict('os.environ', {'PYTHONPATH': pythonpath}):
                try:
                    result = subprocess.check_output([sys.executable, script.name])
                except subprocess.CalledProcessError:
                    self.fail("Unable to reload model pickled data")
        self.assertEqual(result.strip().decode(), "Some object")
<commit_msg>Fixed model_regress test failure on non-ASCII path; refs #24717.<commit_after>import datetime
import os
import pickle
import subprocess
import sys
import warnings

from django.core.files.temp import NamedTemporaryFile
from django.db import DJANGO_VERSION_PICKLE_KEY, models
from django.test import TestCase, mock
from django.utils._os import npath, upath
from django.utils.encoding import force_text
from django.utils.version import get_version

from .models import Article

class ModelPickleTestCase(TestCase):
    def test_missing_django_version_unpickling(self):
        """
        #21430 -- Verifies a warning is raised for models that are
        unpickled without a Django version
        """
        class MissingDjangoVersion(models.Model):
            title = models.CharField(max_length=10)

            def __reduce__(self):
                reduce_list = super(MissingDjangoVersion, self).__reduce__()
                data = reduce_list[-1]
                del data[DJANGO_VERSION_PICKLE_KEY]
                return reduce_list

        p = MissingDjangoVersion(title="FooBar")
        with warnings.catch_warnings(record=True) as recorded:
            pickle.loads(pickle.dumps(p))
            msg = force_text(recorded.pop().message)
            self.assertEqual(msg,
                "Pickled model instance's Django version is not specified.")

    def test_unsupported_unpickle(self):
        """
        #21430 -- Verifies a warning is raised for models that are
        unpickled with a different Django version than the current
        """
        class DifferentDjangoVersion(models.Model):
            title = models.CharField(max_length=10)

            def __reduce__(self):
                reduce_list = super(DifferentDjangoVersion, self).__reduce__()
                data = reduce_list[-1]
                data[DJANGO_VERSION_PICKLE_KEY] = '1.0'
                return reduce_list

        p = DifferentDjangoVersion(title="FooBar")
        with warnings.catch_warnings(record=True) as recorded:
            pickle.loads(pickle.dumps(p))
            msg = force_text(recorded.pop().message)
            self.assertEqual(
                msg,
                "Pickled model instance's Django version 1.0 does not "
                "match the current version %s." % get_version()
            )

    def test_unpickling_when_appregistrynotready(self):
        """
        #24007 -- Verifies that a pickled model can be unpickled without having
        to manually setup the apps registry beforehand.
        """
        script_template = """#!/usr/bin/env python
import pickle

from django.conf import settings

data = %r

settings.configure(DEBUG=False, INSTALLED_APPS=['model_regress'], SECRET_KEY = "blah")
article = pickle.loads(data)
print(article.headline)"""
        a = Article.objects.create(
            headline="Some object",
            pub_date=datetime.datetime.now(),
            article_text="This is an article",
        )

        with NamedTemporaryFile(mode='w+', suffix=".py") as script:
            script.write(script_template % pickle.dumps(a))
            script.flush()
            # A path to model_regress must be set in PYTHONPATH
            model_regress_dir = os.path.dirname(upath(__file__))
            model_regress_path = os.path.abspath(model_regress_dir)
            tests_path = os.path.split(model_regress_path)[0]
            pythonpath = os.environ.get('PYTHONPATH', '')
            pythonpath = npath(os.pathsep.join([tests_path, pythonpath]))

            with mock.patch.dict('os.environ', {'PYTHONPATH': pythonpath}):
                try:
                    result = subprocess.check_output([sys.executable, script.name])
                except subprocess.CalledProcessError:
                    self.fail("Unable to reload model pickled data")
        self.assertEqual(result.strip().decode(), "Some object")
<|endoftext|>