<commit_before>#!/usr/bin/env python
#
#  svnfsfs_tests.py:  testing the 'svnfsfs' tool.
#
#  Subversion is a tool for revision control.
#  See http://subversion.apache.org for more information.
#
# ====================================================================
#    Licensed to the Apache Software Foundation (ASF) under one
#    or more contributor license agreements.  See the NOTICE file
#    distributed with this work for additional information
#    regarding copyright ownership.  The ASF licenses this file
#    to you under the Apache License, Version 2.0 (the
#    "License"); you may not use this file except in compliance
#    with the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing,
#    software distributed under the License is distributed on an
#    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#    KIND, either express or implied.  See the License for the
#    specific language governing permissions and limitations
#    under the License.
######################################################################

# General modules
import os
import logging
import re
import shutil
import sys
import threading
import time
import gzip

logger = logging.getLogger()

# Our testing module
import svntest
from svntest.verify import SVNExpectedStdout, SVNExpectedStderr
from svntest.verify import SVNUnexpectedStderr
from svntest.verify import UnorderedOutput
from svntest.main import SVN_PROP_MERGEINFO

# (abbreviation)
Skip = svntest.testcase.Skip_deco
SkipUnless = svntest.testcase.SkipUnless_deco
XFail = svntest.testcase.XFail_deco
Issues = svntest.testcase.Issues_deco
Issue = svntest.testcase.Issue_deco
Wimp = svntest.testcase.Wimp_deco
SkipDumpLoadCrossCheck = svntest.testcase.SkipDumpLoadCrossCheck_deco
Item = svntest.wc.StateItem

#----------------------------------------------------------------------

# How we currently test 'svnfsfs' --
#
#   'svnfsfs stats':      Run this on a greek repo, then verify that the
#                         various sections are present. The section contents
#                         is matched against section-specific patterns.
#
#   'svnfsfs dump-index': Tested implicitly by the load-index test
#
#   'svnfsfs load-index': Create a greek repo but set shard to 2 and pack
#                         it so we can load into a packed shard with more
#                         than one revision to test ordering issues etc.
#                         r1 also contains a non-trival number of items such
#                         that parser issues etc. have a chance to surface.
#
#                         The idea is dump the index of the pack, mess with
#                         it to cover lots of UI guarantees but keep the
#                         semantics of the relevant bits. Then feed it back
#                         to load-index and verify that the result is still
#                         a complete, consistent etc. repo.
#
######################################################################
# Helper routines

def patch_format(repo_dir, shard_size):
  """Rewrite the format of the FSFS repository REPO_DIR so
  that it would use sharding with SHARDS revisions per shard."""

  format_path = os.path.join(repo_dir, "db", "format")
  contents = open(format_path, 'rb').read()
  processed_lines = []

  for line in contents.split("\n"):
    if line.startswith("layout "):
      processed_lines.append("layout sharded %d" % shard_size)
    else:
      processed_lines.append(line)

  new_contents = "\n".join(processed_lines)
  os.chmod(format_path, 0666)
  open(format_path, 'wb').write(new_contents)

######################################################################
# Tests

#----------------------------------------------------------------------

@SkipUnless(svntest.main.is_fs_type_fsfs)
def test_stats(sbox):
  "stats output"

  sbox.build()

  exit_code, output, errput = \
    svntest.actions.run_and_verify_svnfsfs(None, [], 'stats', sbox.repo_dir)

  # split output into sections
  sections = { }

  last_line = ''
  section_name = ''
  section_contents = []
  for line in output:
    line = line.rstrip()
    if line != '':

      # If the first character is not a space, then LINE is a section header
      if line[0] == ' ':
        section_contents.append(line)
      else:

        # Store previous section
        if section_name != '':
          sections[section_name] = section_contents

          # Is the output formatted nicely?
          if last_line != '':
            logger.warn("Error: no empty line before section '" + line + "'")
            raise svntest.Failure

        # start new section
        section_name = line
        section_contents = []

    last_line = line

  sections[section_name] = section_contents

  # verify that these sections exist
  sections_to_find = ['Reading revisions',
                      'Global statistics:',
                      'Noderev statistics:',
                      'Representation statistics:',
                      'Directory representation statistics:',
                      'File representation statistics:',
                      'Directory property representation statistics:',
                      'File property representation statistics:',
                      'Largest representations:',
                      'Extensions by number of representations:',
                      'Extensions by size of changed files:',
                      'Extensions by size of representations:',
                      'Histogram of expanded node sizes:',
                      'Histogram of representation sizes:',
                      'Histogram of file sizes:',
                      'Histogram of file representation sizes:',
                      'Histogram of file property sizes:',
                      'Histogram of file property representation sizes:',
                      'Histogram of directory sizes:',
                      'Histogram of directory representation sizes:',
                      'Histogram of directory property sizes:',
                      'Histogram of directory property representation sizes:']
  patterns_to_find = {
    'Reading revisions' : ['\s+ 0[ 0-9]*'],
    'Global .*'         : ['.*\d+ bytes in .*\d+ revisions',
                           '.*\d+ bytes in .*\d+ changes',
                           '.*\d+ bytes in .*\d+ node revision records',
                           '.*\d+ bytes in .*\d+ representations',
                           '.*\d+ bytes expanded representation size',
                           '.*\d+ bytes with rep-sharing off' ],
    'Noderev .*'        : ['.*\d+ bytes in .*\d+ nodes total',
                           '.*\d+ bytes in .*\d+ directory noderevs',
                           '.*\d+ bytes in .*\d+ file noderevs' ],
    'Representation .*' : ['.*\d+ bytes in .*\d+ representations total',
                           '.*\d+ bytes in .*\d+ directory representations',
                           '.*\d+ bytes in .*\d+ file representations',
                           '.*\d+ bytes in .*\d+ representations of added file nodes',
                           '.*\d+ bytes in .*\d+ directory property representations',
                           '.*\d+ bytes in .*\d+ file property representations',
                           '.*\d+ bytes in header & footer overhead' ],
    '.* representation statistics:' :
                          ['.*\d+ bytes in .*\d+ reps',
                           '.*\d+ bytes in .*\d+ shared reps',
                           '.*\d+ bytes expanded size',
                           '.*\d+ bytes expanded shared size',
                           '.*\d+ bytes with rep-sharing off',
                           '.*\d+ shared references' ],
    'Largest.*:'        : ['.*\d+ r\d+ */\S*'],
    'Extensions by number .*:' :
                          ['.*\d+ \( ?\d+%\) representations'],
    'Extensions by size .*:' :
                          ['.*\d+ \( ?\d+%\) bytes'],
    'Histogram of .*:'  : ['.*\d+ \.\. < \d+.*\d+ \( ?\d+%\) bytes in *\d+ \( ?\d+%\) items']
  }

  # check that the output contains all sections
  for section_name in sections_to_find:
    if not sections.has_key(section_name):
      logger.warn("Error: section '" + section_name + "' not found")
      raise svntest.Failure

  # check section contents
  for section_name in sections.keys():
    patterns = []

    # find the suitable patterns for the current section
    for pattern in patterns_to_find.keys():
      if re.match(pattern, section_name):
        patterns = patterns_to_find[pattern]
        break;

    if len(patterns) == 0:
      logger.warn("Error: unexpected section '" + section_name + "' found'")
      logger.warn(sections[section_name])
      raise svntest.Failure

    # each line in the section must match one of the patterns
    for line in sections[section_name]:
      found = False

      for pattern in patterns:
        if re.match(pattern, line):
          found = True
          break

      if not found:
        logger.warn("Error: unexpected line '" + line + "' in section '"
                    + section_name + "'")
        logger.warn(sections[section_name])
        raise svntest.Failure

#----------------------------------------------------------------------

@SkipUnless(svntest.main.is_fs_type_fsfs)
@SkipUnless(svntest.main.fs_has_pack)
@SkipUnless(svntest.main.is_fs_log_addressing)
def load_index_sharded(sbox):
  "load-index in a packed repo"

  # Configure two files per shard to trigger packing.
  sbox.build()
  patch_format(sbox.repo_dir, shard_size=2)

  # With --fsfs-packing, everything is already packed and we
  # can skip this part.
  if not svntest.main.options.fsfs_packing:
    expected_output = ["Packing revisions in shard 0...done.\n"]
    svntest.actions.run_and_verify_svnadmin(expected_output, [],
                                            "pack", sbox.repo_dir)

  # Read P2L index using svnfsfs.
  exit_code, items, errput = \
    svntest.actions.run_and_verify_svnfsfs(None, [], "dump-index", "-r0",
                                           sbox.repo_dir)

  # load-index promises to deal with input that
  #
  # * uses the same encoding as the dump-index output
  # * is not in ascending item offset order
  # * contains lines with the full table header
  # * invalid or incorrect data in the checksum column and beyond
  # * starts with an item which does not belong to the first revision
  #   in the pack file
  #
  # So, let's mess with the ITEMS list to call in on these promises.

  # not in ascending order
  items.reverse()

  # multiple headers (there is already one now at the bottom)
  items.insert(0, "       Start       Length Type   Revision     Item Checksum\n")

  # make columns have a variable size
  # mess with the checksums
  # add a junk column
  # keep header lines as are
  for i in range(0, len(items)):
    if items[i].find("Start") == -1:
      columns = items[i].split()
      columns[5] = columns[5].replace('f','-').replace('0','9')
      columns.append("junk")
      items[i] = ' '.join(columns) + "\n"

  # first entry is for rev 1, pack starts at rev 0, though
  assert(items[1].split()[3] == "1")

  # Reload the index
  exit_code, output, errput = svntest.main.run_command_stdin(
    svntest.main.svnfsfs_binary, [], 0, False, items,
    "load-index", sbox.repo_dir)

  # Run verify to see whether we broke anything.
  expected_output = ["* Verifying metadata at revision 0 ...\n",
                     "* Verifying repository metadata ...\n",
                     "* Verified revision 0.\n",
                     "* Verified revision 1.\n"]
  svntest.actions.run_and_verify_svnadmin(expected_output, [],
                                          "verify", sbox.repo_dir)

########################################################################
# Run the tests

# list all tests here, starting with None:
test_list = [ None,
              test_stats,
              load_index_sharded,
             ]

if __name__ == '__main__':
  svntest.main.run_tests(test_list)
  # NOTREACHED

### End of file.
<commit_msg>svnfsfs tests don't need a working copy.<commit_after>#!/usr/bin/env python
#
#  svnfsfs_tests.py:  testing the 'svnfsfs' tool.
#
#  Subversion is a tool for revision control.
#  See http://subversion.apache.org for more information.
#
# ====================================================================
#    Licensed to the Apache Software Foundation (ASF) under one
#    or more contributor license agreements.  See the NOTICE file
#    distributed with this work for additional information
#    regarding copyright ownership.  The ASF licenses this file
#    to you under the Apache License, Version 2.0 (the
#    "License"); you may not use this file except in compliance
#    with the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing,
#    software distributed under the License is distributed on an
#    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#    KIND, either express or implied.  See the License for the
#    specific language governing permissions and limitations
#    under the License.
######################################################################

# General modules
import os
import logging
import re
import shutil
import sys
import threading
import time
import gzip

logger = logging.getLogger()

# Our testing module
import svntest
from svntest.verify import SVNExpectedStdout, SVNExpectedStderr
from svntest.verify import SVNUnexpectedStderr
from svntest.verify import UnorderedOutput
from svntest.main import SVN_PROP_MERGEINFO

# (abbreviation)
Skip = svntest.testcase.Skip_deco
SkipUnless = svntest.testcase.SkipUnless_deco
XFail = svntest.testcase.XFail_deco
Issues = svntest.testcase.Issues_deco
Issue = svntest.testcase.Issue_deco
Wimp = svntest.testcase.Wimp_deco
SkipDumpLoadCrossCheck = svntest.testcase.SkipDumpLoadCrossCheck_deco
Item = svntest.wc.StateItem

#----------------------------------------------------------------------

# How we currently test 'svnfsfs' --
#
#   'svnfsfs stats':      Run this on a greek repo, then verify that the
#                         various sections are present. The section contents
#                         is matched against section-specific patterns.
#
#   'svnfsfs dump-index': Tested implicitly by the load-index test
#
#   'svnfsfs load-index': Create a greek repo but set shard to 2 and pack
#                         it so we can load into a packed shard with more
#                         than one revision to test ordering issues etc.
#                         r1 also contains a non-trival number of items such
#                         that parser issues etc. have a chance to surface.
#
#                         The idea is dump the index of the pack, mess with
#                         it to cover lots of UI guarantees but keep the
#                         semantics of the relevant bits. Then feed it back
#                         to load-index and verify that the result is still
#                         a complete, consistent etc. repo.
#
######################################################################
# Helper routines

def patch_format(repo_dir, shard_size):
  """Rewrite the format of the FSFS repository REPO_DIR so
  that it would use sharding with SHARDS revisions per shard."""

  format_path = os.path.join(repo_dir, "db", "format")
  contents = open(format_path, 'rb').read()
  processed_lines = []

  for line in contents.split("\n"):
    if line.startswith("layout "):
      processed_lines.append("layout sharded %d" % shard_size)
    else:
      processed_lines.append(line)

  new_contents = "\n".join(processed_lines)
  os.chmod(format_path, 0666)
  open(format_path, 'wb').write(new_contents)

######################################################################
# Tests

#----------------------------------------------------------------------

@SkipUnless(svntest.main.is_fs_type_fsfs)
def test_stats(sbox):
  "stats output"

  sbox.build(create_wc=False)

  exit_code, output, errput = \
    svntest.actions.run_and_verify_svnfsfs(None, [], 'stats', sbox.repo_dir)

  # split output into sections
  sections = { }

  last_line = ''
  section_name = ''
  section_contents = []
  for line in output:
    line = line.rstrip()
    if line != '':

      # If the first character is not a space, then LINE is a section header
      if line[0] == ' ':
        section_contents.append(line)
      else:

        # Store previous section
        if section_name != '':
          sections[section_name] = section_contents

          # Is the output formatted nicely?
          if last_line != '':
            logger.warn("Error: no empty line before section '" + line + "'")
            raise svntest.Failure

        # start new section
        section_name = line
        section_contents = []

    last_line = line

  sections[section_name] = section_contents

  # verify that these sections exist
  sections_to_find = ['Reading revisions',
                      'Global statistics:',
                      'Noderev statistics:',
                      'Representation statistics:',
                      'Directory representation statistics:',
                      'File representation statistics:',
                      'Directory property representation statistics:',
                      'File property representation statistics:',
                      'Largest representations:',
                      'Extensions by number of representations:',
                      'Extensions by size of changed files:',
                      'Extensions by size of representations:',
                      'Histogram of expanded node sizes:',
                      'Histogram of representation sizes:',
                      'Histogram of file sizes:',
                      'Histogram of file representation sizes:',
                      'Histogram of file property sizes:',
                      'Histogram of file property representation sizes:',
                      'Histogram of directory sizes:',
                      'Histogram of directory representation sizes:',
                      'Histogram of directory property sizes:',
                      'Histogram of directory property representation sizes:']
  patterns_to_find = {
    'Reading revisions' : ['\s+ 0[ 0-9]*'],
    'Global .*'         : ['.*\d+ bytes in .*\d+ revisions',
                           '.*\d+ bytes in .*\d+ changes',
                           '.*\d+ bytes in .*\d+ node revision records',
                           '.*\d+ bytes in .*\d+ representations',
                           '.*\d+ bytes expanded representation size',
                           '.*\d+ bytes with rep-sharing off' ],
    'Noderev .*'        : ['.*\d+ bytes in .*\d+ nodes total',
                           '.*\d+ bytes in .*\d+ directory noderevs',
                           '.*\d+ bytes in .*\d+ file noderevs' ],
    'Representation .*' : ['.*\d+ bytes in .*\d+ representations total',
                           '.*\d+ bytes in .*\d+ directory representations',
                           '.*\d+ bytes in .*\d+ file representations',
                           '.*\d+ bytes in .*\d+ representations of added file nodes',
                           '.*\d+ bytes in .*\d+ directory property representations',
                           '.*\d+ bytes in .*\d+ file property representations',
                           '.*\d+ bytes in header & footer overhead' ],
    '.* representation statistics:' :
                          ['.*\d+ bytes in .*\d+ reps',
                           '.*\d+ bytes in .*\d+ shared reps',
                           '.*\d+ bytes expanded size',
                           '.*\d+ bytes expanded shared size',
                           '.*\d+ bytes with rep-sharing off',
                           '.*\d+ shared references' ],
    'Largest.*:'        : ['.*\d+ r\d+ */\S*'],
    'Extensions by number .*:' :
                          ['.*\d+ \( ?\d+%\) representations'],
    'Extensions by size .*:' :
                          ['.*\d+ \( ?\d+%\) bytes'],
    'Histogram of .*:'  : ['.*\d+ \.\. < \d+.*\d+ \( ?\d+%\) bytes in *\d+ \( ?\d+%\) items']
  }

  # check that the output contains all sections
  for section_name in sections_to_find:
    if not sections.has_key(section_name):
      logger.warn("Error: section '" + section_name + "' not found")
      raise svntest.Failure

  # check section contents
  for section_name in sections.keys():
    patterns = []

    # find the suitable patterns for the current section
    for pattern in patterns_to_find.keys():
      if re.match(pattern, section_name):
        patterns = patterns_to_find[pattern]
        break;

    if len(patterns) == 0:
      logger.warn("Error: unexpected section '" + section_name + "' found'")
      logger.warn(sections[section_name])
      raise svntest.Failure

    # each line in the section must match one of the patterns
    for line in sections[section_name]:
      found = False

      for pattern in patterns:
        if re.match(pattern, line):
          found = True
          break

      if not found:
        logger.warn("Error: unexpected line '" + line + "' in section '"
                    + section_name + "'")
        logger.warn(sections[section_name])
        raise svntest.Failure

#----------------------------------------------------------------------

@SkipUnless(svntest.main.is_fs_type_fsfs)
@SkipUnless(svntest.main.fs_has_pack)
@SkipUnless(svntest.main.is_fs_log_addressing)
def load_index_sharded(sbox):
  "load-index in a packed repo"

  # Configure two files per shard to trigger packing.
  sbox.build(create_wc=False)
  patch_format(sbox.repo_dir, shard_size=2)

  # With --fsfs-packing, everything is already packed and we
  # can skip this part.
  if not svntest.main.options.fsfs_packing:
    expected_output = ["Packing revisions in shard 0...done.\n"]
    svntest.actions.run_and_verify_svnadmin(expected_output, [],
                                            "pack", sbox.repo_dir)

  # Read P2L index using svnfsfs.
  exit_code, items, errput = \
    svntest.actions.run_and_verify_svnfsfs(None, [], "dump-index", "-r0",
                                           sbox.repo_dir)

  # load-index promises to deal with input that
  #
  # * uses the same encoding as the dump-index output
  # * is not in ascending item offset order
  # * contains lines with the full table header
  # * invalid or incorrect data in the checksum column and beyond
  # * starts with an item which does not belong to the first revision
  #   in the pack file
  #
  # So, let's mess with the ITEMS list to call in on these promises.

  # not in ascending order
  items.reverse()

  # multiple headers (there is already one now at the bottom)
  items.insert(0, "       Start       Length Type   Revision     Item Checksum\n")

  # make columns have a variable size
  # mess with the checksums
  # add a junk column
  # keep header lines as are
  for i in range(0, len(items)):
    if items[i].find("Start") == -1:
      columns = items[i].split()
      columns[5] = columns[5].replace('f','-').replace('0','9')
      columns.append("junk")
      items[i] = ' '.join(columns) + "\n"

  # first entry is for rev 1, pack starts at rev 0, though
  assert(items[1].split()[3] == "1")

  # Reload the index
  exit_code, output, errput = svntest.main.run_command_stdin(
    svntest.main.svnfsfs_binary, [], 0, False, items,
    "load-index", sbox.repo_dir)

  # Run verify to see whether we broke anything.
  expected_output = ["* Verifying metadata at revision 0 ...\n",
                     "* Verifying repository metadata ...\n",
                     "* Verified revision 0.\n",
                     "* Verified revision 1.\n"]
  svntest.actions.run_and_verify_svnadmin(expected_output, [],
                                          "verify", sbox.repo_dir)

########################################################################
# Run the tests

# list all tests here, starting with None:
test_list = [ None,
              test_stats,
              load_index_sharded,
             ]

if __name__ == '__main__':
  svntest.main.run_tests(test_list)
  # NOTREACHED

### End of file.
<|endoftext|>


<commit_before>import datetime
import pytest

from sunpy.time import parse_time
from sunpy.time.timerange import TimeRange
from sunpy.net.vso import VSOClient
from sunpy.net.vso.attrs import Time, Instrument, Source, Level
from sunpy.net.dataretriever.client import QueryResponse
import sunpy.net.dataretriever.sources.eve as eve
from sunpy.net.fido_factory import UnifiedResponse
from sunpy.net import Fido
from sunpy.net import attrs as a

from hypothesis import given, settings
from hypothesis.extra.datetime import datetimes
from sunpy.net.tests.strategies import time_attr

LCClient = eve.EVEClient()

@pytest.mark.online
@pytest.mark.parametrize("timerange,url_start,url_end", [
    (TimeRange('2012/4/21', '2012/4/21'),
     'http://lasp.colorado.edu/eve/data_access/evewebdata/quicklook/L0CS/SpWx/2012/20120421_EVE_L0CS_DIODES_1m.txt',
     'http://lasp.colorado.edu/eve/data_access/evewebdata/quicklook/L0CS/SpWx/2012/20120421_EVE_L0CS_DIODES_1m.txt'
     ),
    (TimeRange('2012/5/5', '2012/5/6'),
     'http://lasp.colorado.edu/eve/data_access/evewebdata/quicklook/L0CS/SpWx/2012/20120505_EVE_L0CS_DIODES_1m.txt',
     'http://lasp.colorado.edu/eve/data_access/evewebdata/quicklook/L0CS/SpWx/2012/20120506_EVE_L0CS_DIODES_1m.txt'
     ),
    (TimeRange('2012/7/7', '2012/7/14'),
     'http://lasp.colorado.edu/eve/data_access/evewebdata/quicklook/L0CS/SpWx/2012/20120707_EVE_L0CS_DIODES_1m.txt',
     'http://lasp.colorado.edu/eve/data_access/evewebdata/quicklook/L0CS/SpWx/2012/20120714_EVE_L0CS_DIODES_1m.txt'
     )
])
def test_get_url_for_time_range(timerange, url_start, url_end):
    urls = LCClient._get_url_for_timerange(timerange)
    assert isinstance(urls, list)
    assert urls[0] == url_start
    assert urls[-1] == url_end

def test_can_handle_query():
    ans1 = eve.EVEClient._can_handle_query(
        Time('2012/8/9', '2012/8/10'), Instrument('eve'), Level(0))
    assert ans1 is True
    ans2 = eve.EVEClient._can_handle_query(Time('2012/7/7', '2012/7/7'))
    assert ans2 is False
    ans3 = eve.EVEClient._can_handle_query(
        Time('2012/8/9', '2012/8/10'), Instrument('eve'), Source('sdo'))
    assert ans3 is False

@pytest.mark.online
def test_query():
    qr1 = LCClient.query(Time('2012/8/9', '2012/8/10'), Instrument('eve'))
    assert isinstance(qr1, QueryResponse)
    assert len(qr1) == 2
    assert qr1.time_range().start == parse_time('2012/08/09')
    assert qr1.time_range().end == parse_time('2012/08/10')

@pytest.mark.online
@pytest.mark.parametrize("time,instrument", [
    (Time('2012/11/27', '2012/11/27'), Instrument('eve')),
])
def test_get(time, instrument):
    qr1 = LCClient.query(time, instrument)
    res = LCClient.get(qr1)
    download_list = res.wait(progress=False)
    assert len(download_list) == len(qr1)

@pytest.mark.online
@pytest.mark.parametrize(
    'query',
    [(a.Time('2012/10/4', '2012/10/6') & a.Instrument('eve') & a.Level(0))])
def test_fido(query):
    qr = Fido.search(query)
    client = qr.get_response(0).client
    assert isinstance(qr, UnifiedResponse)
    assert isinstance(client, eve.EVEClient)
    response = Fido.fetch(qr)
    assert len(response) == qr._numfile

@pytest.mark.online
@given(time_attr(time=datetimes(timezones=[], max_year=datetime.datetime.utcnow().year, min_year=2010)))
@settings(max_examples=2, timeout=240)
def test_levels(time):
    """
    Test the correct handling of level 0 / 1.
    The default should be level 1 from VSO, level 0 comes from EVEClient.
    """
    eve_a = a.Instrument('EVE')
    qr = Fido.search(time, eve_a)
    client = qr.get_response(0).client
    assert isinstance(client, VSOClient)

    qr = Fido.search(time, eve_a, a.Level(0))
    client = qr.get_response(0).client
    assert isinstance(client, eve.EVEClient)

    qr = Fido.search(time, eve_a, a.Level(0) | a.Level(1))
    clients = {type(a.client) for a in qr.responses}
    assert clients.symmetric_difference({VSOClient, eve.EVEClient}) == set()

<commit_msg>Fix eve test where the last day was not included<commit_after>import datetime
import pytest

from sunpy.time import parse_time
from sunpy.time.timerange import TimeRange
from sunpy.net.vso import VSOClient
from sunpy.net.vso.attrs import Time, Instrument, Source, Level
from sunpy.net.dataretriever.client import QueryResponse
import sunpy.net.dataretriever.sources.eve as eve
from sunpy.net.fido_factory import UnifiedResponse
from sunpy.net import Fido
from sunpy.net import attrs as a

from hypothesis import given, settings
from hypothesis.extra.datetime import datetimes
from sunpy.net.tests.strategies import time_attr

LCClient = eve.EVEClient()

@pytest.mark.online
@pytest.mark.parametrize("timerange,url_start,url_end", [
    (TimeRange('2012/4/21', '2012/4/21'),
     'http://lasp.colorado.edu/eve/data_access/evewebdata/quicklook/L0CS/SpWx/2012/20120421_EVE_L0CS_DIODES_1m.txt',
     'http://lasp.colorado.edu/eve/data_access/evewebdata/quicklook/L0CS/SpWx/2012/20120421_EVE_L0CS_DIODES_1m.txt'
     ),
    (TimeRange('2012/5/5', '2012/5/6'),
     'http://lasp.colorado.edu/eve/data_access/evewebdata/quicklook/L0CS/SpWx/2012/20120505_EVE_L0CS_DIODES_1m.txt',
     'http://lasp.colorado.edu/eve/data_access/evewebdata/quicklook/L0CS/SpWx/2012/20120506_EVE_L0CS_DIODES_1m.txt'
     ),
    (TimeRange('2012/7/7', '2012/7/14'),
     'http://lasp.colorado.edu/eve/data_access/evewebdata/quicklook/L0CS/SpWx/2012/20120707_EVE_L0CS_DIODES_1m.txt',
     'http://lasp.colorado.edu/eve/data_access/evewebdata/quicklook/L0CS/SpWx/2012/20120714_EVE_L0CS_DIODES_1m.txt'
     )
])
def test_get_url_for_time_range(timerange, url_start, url_end):
    urls = LCClient._get_url_for_timerange(timerange)
    assert isinstance(urls, list)
    assert urls[0] == url_start
    assert urls[-1] == url_end

def test_can_handle_query():
    ans1 = eve.EVEClient._can_handle_query(
        Time('2012/8/9', '2012/8/10'), Instrument('eve'), Level(0))
    assert ans1 is True
    ans2 = eve.EVEClient._can_handle_query(Time('2012/7/7', '2012/7/7'))
    assert ans2 is False
    ans3 = eve.EVEClient._can_handle_query(
        Time('2012/8/9', '2012/8/10'), Instrument('eve'), Source('sdo'))
    assert ans3 is False

@pytest.mark.online
def test_query():
    qr1 = LCClient.query(Time('2012/8/9', '2012/8/10'), Instrument('eve'))
    assert isinstance(qr1, QueryResponse)
    assert len(qr1) == 2
    assert qr1.time_range().start == parse_time('2012/08/09')
    assert qr1.time_range().end == parse_time('2012/08/11') # includes end.

@pytest.mark.online
@pytest.mark.parametrize("time,instrument", [
    (Time('2012/11/27', '2012/11/27'), Instrument('eve')),
])
def test_get(time, instrument):
    qr1 = LCClient.query(time, instrument)
    res = LCClient.get(qr1)
    download_list = res.wait(progress=False)
    assert len(download_list) == len(qr1)

@pytest.mark.online
@pytest.mark.parametrize(
    'query',
    [(a.Time('2012/10/4', '2012/10/6') & a.Instrument('eve') & a.Level(0))])
def test_fido(query):
    qr = Fido.search(query)
    client = qr.get_response(0).client
    assert isinstance(qr, UnifiedResponse)
    assert isinstance(client, eve.EVEClient)
    response = Fido.fetch(qr)
    assert len(response) == qr._numfile

@pytest.mark.online
@given(time_attr(time=datetimes(timezones=[], max_year=datetime.datetime.utcnow().year, min_year=2010)))
@settings(max_examples=2, timeout=240)
def test_levels(time):
    """
    Test the correct handling of level 0 / 1.
    The default should be level 1 from VSO, level 0 comes from EVEClient.
    """
    eve_a = a.Instrument('EVE')
    qr = Fido.search(time, eve_a)
    client = qr.get_response(0).client
    assert isinstance(client, VSOClient)

    qr = Fido.search(time, eve_a, a.Level(0))
    client = qr.get_response(0).client
    assert isinstance(client, eve.EVEClient)

    qr = Fido.search(time, eve_a, a.Level(0) | a.Level(1))
    clients = {type(a.client) for a in qr.responses}
    assert clients.symmetric_difference({VSOClient, eve.EVEClient}) == set()
<|endoftext|>


<commit_before>import numpy

def svd_weighting(data):
    p, _, v = numpy.linalg.svd(data, full_matrices=False)
    channel_weights = p[:, 0].conjugate()

    # try some basic phase correction
    # in our truncation of the SVD to rank 1, we know that v[0] is our FID
    # use the first point of it to phase the signal
    phase_shift = numpy.angle(v[0, 0])

    return channel_weights * numpy.exp(-1j * phase_shift) / numpy.sum(numpy.abs(channel_weights))

def combine_channels(data, weights=None):
    if weights is None:
        weights = svd_weighting(data)
    weighted_data = weights.reshape((len(weights), 1)) * data
    combined_data = weighted_data.sum(axis=0)
    return combined_data
<commit_msg>BUG: channel_combination averages over wrong axis<commit_after>import numpy

def svd_weighting(data):
    p, _, v = numpy.linalg.svd(data, full_matrices=False)
    channel_weights = p[:, 0].conjugate()

    # try some basic phase correction
    # in our truncation of the SVD to rank 1, we know that v[0] is our FID
    # use the first point of it to phase the signal
    phase_shift = numpy.angle(v[0, 0])

    return channel_weights * numpy.exp(-1j * phase_shift) / numpy.sum(numpy.abs(channel_weights))

def combine_channels(data, weights=None):
    if weights is None:
        weights = svd_weighting(data)
    weighted_data = weights.reshape((len(weights), 1)) * data
    combined_data = weighted_data.sum(axis=-2)
    return combined_data
<|endoftext|>


<commit_before>from sympy import I, symbols, sqrt, Add, Mul, Rational, Pow, Symbol, sympify
from sympy import Integer, conjugate, pretty, latex, oo, sin, pi

from sympy.physics.quantum.dagger import Dagger
from sympy.physics.quantum.qexpr import QExpr
from sympy.physics.quantum.state import (
    Ket, Bra, TimeDepKet, TimeDepBra,
    KetBase, BraBase, StateBase, Wavefunction
)
from sympy.physics.quantum.hilbert import HilbertSpace

x,y,t = symbols('x,y,t')

def test_ket():
    k = Ket('0')

    assert isinstance(k, Ket)
    assert isinstance(k, KetBase)
    assert isinstance(k, StateBase)
    assert isinstance(k, QExpr)

    assert k.label == (Symbol('0'),)
    assert k.hilbert_space == HilbertSpace()
    assert k.is_commutative == False

    # Make sure this doesn't get converted to the number pi.
    k = Ket('pi')
    assert k.label == (Symbol('pi'),)

    k = Ket(x,y)
    assert k.label == (x,y)
    assert k.hilbert_space == HilbertSpace()
    assert k.is_commutative == False

    assert k.dual_class == Bra
    assert k.dual == Bra(x,y)
    assert k.subs(x,y) == Ket(y,y)

def test_bra():
    b = Bra('0')

    assert isinstance(b, Bra)
    assert isinstance(b, BraBase)
    assert isinstance(b, StateBase)
    assert isinstance(b, QExpr)

    assert b.label == (Symbol('0'),)
    assert b.hilbert_space == HilbertSpace()
    assert b.is_commutative == False

    # Make sure this doesn't get converted to the number pi.
    b = Bra('pi')
    assert b.label == (Symbol('pi'),)

    b = Bra(x,y)
    assert b.label == (x,y)
    assert b.hilbert_space == HilbertSpace()
    assert b.is_commutative == False

    assert b.dual_class == Ket
    assert b.dual == Ket(x,y)
    assert b.subs(x,y) == Bra(y,y)

def test_ops():
    k0 = Ket(0)
    k1 = Ket(1)
    k = 2*I*k0 - (x/sqrt(2))*k1
    assert k == Add(Mul(2, I, k0),
        Mul(Rational(-1, 2), x, Pow(2, Rational(1, 2)), k1))

def test_time_dep_ket():
    k = TimeDepKet(0,t)

    assert isinstance(k, TimeDepKet)
    assert isinstance(k, KetBase)
    assert isinstance(k, StateBase)
    assert isinstance(k, QExpr)

    assert k.label == (Integer(0),)
    assert k.args == (Integer(0),t)
    assert k.time == t

    assert k.dual_class == TimeDepBra
    assert k.dual == TimeDepBra(0,t)

    assert k.subs(t,2) == TimeDepKet(0,2)

    k = TimeDepKet(x, 0.5)
    assert k.label == (x,)
    assert k.args == (x,sympify(0.5))

def test_time_dep_bra():
    b = TimeDepBra(0,t)

    assert isinstance(b, TimeDepBra)
    assert isinstance(b, BraBase)
    assert isinstance(b, StateBase)
    assert isinstance(b, QExpr)

    assert b.label == (Integer(0),)
    assert b.args == (Integer(0),t)
    assert b.time == t

    assert b.dual_class == TimeDepKet
    assert b.dual == TimeDepKet(0,t)

    k = TimeDepBra(x, 0.5)
    assert k.label == (x,)
    assert k.args == (x,sympify(0.5))

def test_bra_ket_dagger():
    x = symbols('x',complex=True)
    k = Ket('k')
    b = Bra('b')
    assert Dagger(k) == Bra('k')
    assert Dagger(b) == Ket('b')
    assert Dagger(k).is_commutative == False

    k2 = Ket('k2')
    e = 2*I*k + x*k2
    assert Dagger(e) == conjugate(x)*Dagger(k2) - 2*I*Dagger(k)

def test_printing():
    psi = Ket('psi')
    assert pretty(psi, use_unicode=True) == u'\u2758\u03c8\u27e9'
    assert pretty(Dagger(psi), use_unicode=True) == u'\u27e8\u03c8\u2758'
    assert latex(psi) == r"{\left|\psi\right\rangle }"
    assert latex(Dagger(psi)) == r"{\left\langle \psi\right|}"

def test_wavefunction():
    x, L = symbols('x,L', real=True)
    n = symbols('n', integer=True)

    f = Wavefunction(x**2, x)
    p = f.prob()
    lims = f.limits

    assert f.is_normalized == False
    assert f.norm_constant == 0
    assert f(10) == 100
    assert p(10) == 10000
    assert lims[x] == (-oo, oo)

    g = Wavefunction(x**2*y+y**2*x, (x, 0, 3), (y, 0, 5))
    lims_g = g.limits

    assert lims_g[x] == (0, 3)
    assert lims_g[y] == (0, 5)
    assert g.is_normalized == False
    assert g(2,4) == 48

    h = Wavefunction(sqrt(5)*x**2, (x, 0, 1))
    assert h.is_normalized == True

    piab = Wavefunction(sin(n*pi*x/L), (x, 0, L))
    assert piab.norm_constant == sqrt(2/L)
    assert piab(L+1) == 0
    assert piab(0.5) == sin(0.5*n*pi/L)
    assert piab(0.5, n=1, L=1) == sin(0.5*pi)
<commit_msg>A couple more tests for Wavefunction<commit_after>from sympy import I, symbols, sqrt, Add, Mul, Rational, Pow, Symbol, sympify, S
from sympy import Integer, conjugate, pretty, latex, oo, sin, pi

from sympy.physics.quantum.dagger import Dagger
from sympy.physics.quantum.qexpr import QExpr
from sympy.physics.quantum.state import (
    Ket, Bra, TimeDepKet, TimeDepBra,
    KetBase, BraBase, StateBase, Wavefunction
)
from sympy.physics.quantum.hilbert import HilbertSpace

x,y,t = symbols('x,y,t')

def test_ket():
    k = Ket('0')

    assert isinstance(k, Ket)
    assert isinstance(k, KetBase)
    assert isinstance(k, StateBase)
    assert isinstance(k, QExpr)

    assert k.label == (Symbol('0'),)
    assert k.hilbert_space == HilbertSpace()
    assert k.is_commutative == False

    # Make sure this doesn't get converted to the number pi.
    k = Ket('pi')
    assert k.label == (Symbol('pi'),)

    k = Ket(x,y)
    assert k.label == (x,y)
    assert k.hilbert_space == HilbertSpace()
    assert k.is_commutative == False

    assert k.dual_class == Bra
    assert k.dual == Bra(x,y)
    assert k.subs(x,y) == Ket(y,y)

def test_bra():
    b = Bra('0')

    assert isinstance(b, Bra)
    assert isinstance(b, BraBase)
    assert isinstance(b, StateBase)
    assert isinstance(b, QExpr)

    assert b.label == (Symbol('0'),)
    assert b.hilbert_space == HilbertSpace()
    assert b.is_commutative == False

    # Make sure this doesn't get converted to the number pi.
    b = Bra('pi')
    assert b.label == (Symbol('pi'),)

    b = Bra(x,y)
    assert b.label == (x,y)
    assert b.hilbert_space == HilbertSpace()
    assert b.is_commutative == False

    assert b.dual_class == Ket
    assert b.dual == Ket(x,y)
    assert b.subs(x,y) == Bra(y,y)

def test_ops():
    k0 = Ket(0)
    k1 = Ket(1)
    k = 2*I*k0 - (x/sqrt(2))*k1
    assert k == Add(Mul(2, I, k0),
        Mul(Rational(-1, 2), x, Pow(2, Rational(1, 2)), k1))

def test_time_dep_ket():
    k = TimeDepKet(0,t)

    assert isinstance(k, TimeDepKet)
    assert isinstance(k, KetBase)
    assert isinstance(k, StateBase)
    assert isinstance(k, QExpr)

    assert k.label == (Integer(0),)
    assert k.args == (Integer(0),t)
    assert k.time == t

    assert k.dual_class == TimeDepBra
    assert k.dual == TimeDepBra(0,t)

    assert k.subs(t,2) == TimeDepKet(0,2)

    k = TimeDepKet(x, 0.5)
    assert k.label == (x,)
    assert k.args == (x,sympify(0.5))

def test_time_dep_bra():
    b = TimeDepBra(0,t)

    assert isinstance(b, TimeDepBra)
    assert isinstance(b, BraBase)
    assert isinstance(b, StateBase)
    assert isinstance(b, QExpr)

    assert b.label == (Integer(0),)
    assert b.args == (Integer(0),t)
    assert b.time == t

    assert b.dual_class == TimeDepKet
    assert b.dual == TimeDepKet(0,t)

    k = TimeDepBra(x, 0.5)
    assert k.label == (x,)
    assert k.args == (x,sympify(0.5))

def test_bra_ket_dagger():
    x = symbols('x',complex=True)
    k = Ket('k')
    b = Bra('b')
    assert Dagger(k) == Bra('k')
    assert Dagger(b) == Ket('b')
    assert Dagger(k).is_commutative == False

    k2 = Ket('k2')
    e = 2*I*k + x*k2
    assert Dagger(e) == conjugate(x)*Dagger(k2) - 2*I*Dagger(k)

def test_printing():
    psi = Ket('psi')
    assert pretty(psi, use_unicode=True) == u'\u2758\u03c8\u27e9'
    assert pretty(Dagger(psi), use_unicode=True) == u'\u27e8\u03c8\u2758'
    assert latex(psi) == r"{\left|\psi\right\rangle }"
    assert latex(Dagger(psi)) == r"{\left\langle \psi\right|}"

def test_wavefunction():
    x, L = symbols('x,L', real=True)
    n = symbols('n', integer=True)

    f = Wavefunction(x**2, x)
    p = f.prob()
    lims = f.limits

    assert f.is_normalized == False
    assert f.norm_constant == 0
    assert f(10) == 100
    assert p(10) == 10000
    assert lims[x] == (-oo, oo)

    g = Wavefunction(x**2*y+y**2*x, (x, 0, 1), (y, 0, 2))
    lims_g = g.limits

    assert lims_g[x] == (0, 1)
    assert lims_g[y] == (0, 2)
    assert g.is_normalized == False
    assert g.norm_constant == 42**(S(1)/2)/14
    assert g(2,4) == 0
    assert g(1,1) == 2

    h = Wavefunction(sqrt(5)*x**2, (x, 0, 1))
    assert h.is_normalized == True

    piab = Wavefunction(sin(n*pi*x/L), (x, 0, L))
    assert piab.norm_constant == sqrt(2/L)
    assert piab(L+1) == 0
    assert piab(0.5) == sin(0.5*n*pi/L)
    assert piab(0.5, n=1, L=1) == sin(0.5*pi)
<|endoftext|>


<commit_before>import os.path
import xlrd

class XslReader(object):

    def __init__(self):
        pass

    def load_metadata_sheet(self, file_path):
        workbook = xlrd.open_workbook(file_path)
        worksheet = workbook.sheet_by_name("metadata")

        data_dictionary = {}
	data_dictionary["org_acronym"] = worksheet.cell_value(0, 1).decode("UTF-8")
	data_dictionary["dataset_internal_id"] = worksheet.cell_value(1, 1).decode("UTF-8")
	data_dictionary["indicator_internal_id"] = worksheet.cell_value(2, 1).decode("UTF-8")
	data_dictionary["read_as"] = worksheet.cell_value(3, 1).decode("UTF-8")

        return data_dictionary

    def load_xsl_country_year_value(self, file_path):
        workbook = xlrd.open_workbook(file_path)
        worksheet = workbook.sheet_by_name("data")

        data_matrix = [[0 for x in xrange(worksheet.ncols)] for x in xrange(worksheet.nrows)]
        for curr_row in range (0, worksheet.nrows):
            for curr_col in range (0, worksheet.ncols):
                if worksheet.cell_type(curr_row, curr_col) == 1:  # text cell
                    data_matrix[curr_row][curr_col] = worksheet.cell_value(curr_row, curr_col).encode("UTF-8");
                else:
		    data_matrix[curr_row][curr_col] = self._get_value_from_cell(worksheet.cell_value(curr_row, curr_col))

        return data_matrix

    def load_xsl_multiple_columns_by_year(self, file_path):
        workbook = xlrd.open_workbook(file_path)
        worksheet = workbook.sheet_by_name("data")

        data_matrix = [[0 for x in xrange(worksheet.ncols)] for x in xrange(worksheet.nrows)]

        for curr_row in range (0, worksheet.nrows):
            for curr_col in range (0, worksheet.ncols):
                #print "%s,%s   ---- %s" %(curr_row, curr_col, worksheet.cell_value(curr_row, curr_col))
                if worksheet.cell_type(curr_row, curr_col) == 1:  # text cell
                    data_matrix[curr_row][curr_col] = worksheet.cell_value(curr_row, curr_col).encode("UTF-8");
                else:
		    data_matrix[curr_row][curr_col] = self._get_value_from_cell(worksheet.cell_value(curr_row, curr_col))

        return data_matrix

    @staticmethod
    def _get_value_from_cell(cell_value):
      if cell_value == "":
	 return None
      if cell_value.is_integer(): #integer
         return int(cell_value)
      else: #float
         return float(cell_value)
<commit_msg>support only numbers as indicators ID (in case there were saved as numbers)<commit_after>import os.path
import xlrd

class XslReader(object):

    def __init__(self):
        pass

    def load_metadata_sheet(self, file_path):
        workbook = xlrd.open_workbook(file_path)
        worksheet = workbook.sheet_by_name("metadata")

        data_dictionary = {}
	data_dictionary["org_acronym"] = worksheet.cell_value(0, 1).decode("UTF-8")
	data_dictionary["dataset_internal_id"] = worksheet.cell_value(1, 1).decode("UTF-8")
	data_dictionary["indicator_internal_id"] = self._get_string_from_cell_value(worksheet.cell_value(2, 1))
	data_dictionary["read_as"] = worksheet.cell_value(3, 1).decode("UTF-8")

        return data_dictionary

    def load_xsl_country_year_value(self, file_path):
        workbook = xlrd.open_workbook(file_path)
        worksheet = workbook.sheet_by_name("data")

        data_matrix = [[0 for x in xrange(worksheet.ncols)] for x in xrange(worksheet.nrows)]
        for curr_row in range (0, worksheet.nrows):
            for curr_col in range (0, worksheet.ncols):
                if worksheet.cell_type(curr_row, curr_col) == 1:  # text cell
                    data_matrix[curr_row][curr_col] = worksheet.cell_value(curr_row, curr_col).encode("UTF-8");
                else:
		    data_matrix[curr_row][curr_col] = self._get_value_from_cell(worksheet.cell_value(curr_row, curr_col))

        return data_matrix

    def load_xsl_multiple_columns_by_year(self, file_path):
        workbook = xlrd.open_workbook(file_path)
        worksheet = workbook.sheet_by_name("data")

        data_matrix = [[0 for x in xrange(worksheet.ncols)] for x in xrange(worksheet.nrows)]

        for curr_row in range (0, worksheet.nrows):
            for curr_col in range (0, worksheet.ncols):
                #print "%s,%s   ---- %s" %(curr_row, curr_col, worksheet.cell_value(curr_row, curr_col))
                if worksheet.cell_type(curr_row, curr_col) == 1:  # text cell
                    data_matrix[curr_row][curr_col] = worksheet.cell_value(curr_row, curr_col).encode("UTF-8");
                else:
		    data_matrix[curr_row][curr_col] = self._get_value_from_cell(worksheet.cell_value(curr_row, curr_col))

        return data_matrix

    @staticmethod
    def _get_value_from_cell(cell_value):
      if cell_value == "":
	 return None
      if cell_value.is_integer(): #integer
         return int(cell_value)
      else: #float
         return float(cell_value)

    @staticmethod
    def _get_string_from_cell_value(cell_value):
	# in case the cell is read as float, because there is only numbers
        if isinstance(cell_value, float):
	   if cell_value.is_integer():
              return str(int(cell_value))
           else:
              return str(cell_value)
        else:
	   return cell_value.decode("UTF-8")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals, absolute_import

from uuid import UUID

import pytest

from django.contrib import admin
from django.core.exceptions import ImproperlyConfigured
from django.forms import Form, IntegerField
from django.test import override_settings
from django.utils.encoding import force_text
from templateselector.fields import TemplateChoiceField
from templateselector.widgets import TemplateSelector, AdminTemplateSelector

@pytest.yield_fixture
def modelcls():
    from templateselector.tests.models import MyModel
    yield MyModel

def test_field_warns_if_providing_coerce_callable():
    with pytest.raises(ImproperlyConfigured):
        TemplateChoiceField(coerce=int, match="^.*$")

def test_field_warns_if_providing_dumb_display_name():
    with pytest.raises(ImproperlyConfigured):
        TemplateChoiceField(display_name='goose', match="^.*$")

def test_field_warns_if_missing_caret_at_start():
    with pytest.raises(ImproperlyConfigured):
        TemplateChoiceField(match=".*$")

def test_field_warns_if_missing_dollar_at_end():
    with pytest.raises(ImproperlyConfigured):
        TemplateChoiceField(match="^.*")

def test_model_field_yields_correct_formfield(modelcls):
    x = modelcls()
    y = x._meta.get_field('f').formfield()
    assert isinstance(y, TemplateChoiceField)

def test_choices():
    x = TemplateChoiceField(match="^admin/[0-9]+.html$")
    assert set(x.choices) == {('admin/404.html', '404'), ('admin/500.html', '500')}

def test_choices_using_custom_setting_mapping():
    s = {
        'admin/404.html': 'Page Not Found',
        'admin/500.html': 'Server Error',
    }
    with override_settings(TEMPLATESELECTOR_DISPLAY_NAMES=s):
        x = TemplateChoiceField(match="^admin/[0-9]+.html$")
        assert set(x.choices) == set(s.items())

def test_choices_using_custom_namer():
    def namer(data):
        return str(UUID('F'*32))
    x = TemplateChoiceField(match="^admin/[0-9]+.html$", display_name=namer)
    assert set(x.choices) == {('admin/404.html', 'ffffffff-ffff-ffff-ffff-ffffffffffff'),
                              ('admin/500.html', 'ffffffff-ffff-ffff-ffff-ffffffffffff')}

@pytest.yield_fixture
def form_cls():
    class MyForm(Form):
        a = IntegerField()
        b = TemplateChoiceField(match="^admin/[0-9]+.html$")
    return MyForm

def test_form_usage_invalid(form_cls):
    f = form_cls(data={'a': '1', 'b': 'goose'})
    assert f.is_valid() is False
    assert f.errors == {'b': ['Select a valid choice. goose is not one of the available choices.']}

def test_form_usage_ok(form_cls):
    f = form_cls(data={'a': '1', 'b': 'admin/404.html'})
    assert f.is_valid() is True
    assert f.errors == {}

@pytest.mark.xfail
def test_form_usage_render(form_cls):
    f = form_cls(data={'a': '1', 'b': 'admin/404.html'})
    rendered = force_text(f)
    assert rendered == ''

def test_admin_default_formfield(rf, modelcls):
    modeladmin = admin.site._registry[modelcls]
    request = rf.get('/')
    form = modeladmin.get_form(request=request)()
    widget = form.fields['f'].widget
    assert isinstance(widget, TemplateSelector)
    assert isinstance(widget, AdminTemplateSelector)
<commit_msg>Actually test the widget output.<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals, absolute_import

from unittest import skipIf
from uuid import UUID

import django
import pytest

from django.contrib import admin
from django.core.exceptions import ImproperlyConfigured
from django.forms import Form, IntegerField
from django.test import override_settings, SimpleTestCase
from django.utils.encoding import force_text
from templateselector.fields import TemplateChoiceField
from templateselector.widgets import TemplateSelector, AdminTemplateSelector

@pytest.yield_fixture
def modelcls():
    from templateselector.tests.models import MyModel
    yield MyModel

def test_field_warns_if_providing_coerce_callable():
    with pytest.raises(ImproperlyConfigured):
        TemplateChoiceField(coerce=int, match="^.*$")

def test_field_warns_if_providing_dumb_display_name():
    with pytest.raises(ImproperlyConfigured):
        TemplateChoiceField(display_name='goose', match="^.*$")

def test_field_warns_if_missing_caret_at_start():
    with pytest.raises(ImproperlyConfigured):
        TemplateChoiceField(match=".*$")

def test_field_warns_if_missing_dollar_at_end():
    with pytest.raises(ImproperlyConfigured):
        TemplateChoiceField(match="^.*")

def test_model_field_yields_correct_formfield(modelcls):
    x = modelcls()
    y = x._meta.get_field('f').formfield()
    assert isinstance(y, TemplateChoiceField)

def test_choices():
    x = TemplateChoiceField(match="^admin/[0-9]+.html$")
    assert set(x.choices) == {('admin/404.html', '404'), ('admin/500.html', '500')}

def test_choices_using_custom_setting_mapping():
    s = {
        'admin/404.html': 'Page Not Found',
        'admin/500.html': 'Server Error',
    }
    with override_settings(TEMPLATESELECTOR_DISPLAY_NAMES=s):
        x = TemplateChoiceField(match="^admin/[0-9]+.html$")
        assert set(x.choices) == set(s.items())

def test_choices_using_custom_namer():
    def namer(data):
        return str(UUID('F'*32))
    x = TemplateChoiceField(match="^admin/[0-9]+.html$", display_name=namer)
    assert set(x.choices) == {('admin/404.html', 'ffffffff-ffff-ffff-ffff-ffffffffffff'),
                              ('admin/500.html', 'ffffffff-ffff-ffff-ffff-ffffffffffff')}

@pytest.fixture
def form_cls():
    class MyForm(Form):
        a = IntegerField()
        b = TemplateChoiceField(match="^admin/[0-9]+.html$")
    return MyForm

def test_form_usage_invalid(form_cls):
    f = form_cls(data={'a': '1', 'b': 'goose'})
    assert f.is_valid() is False
    assert f.errors == {'b': ['Select a valid choice. goose is not one of the available choices.']}

def test_form_usage_ok(form_cls):
    f = form_cls(data={'a': '1', 'b': 'admin/404.html'})
    assert f.is_valid() is True
    assert f.errors == {}

def test_admin_default_formfield(rf, modelcls):
    modeladmin = admin.site._registry[modelcls]
    request = rf.get('/')
    form = modeladmin.get_form(request=request)()
    widget = form.fields['f'].widget
    assert isinstance(widget, TemplateSelector)
    assert isinstance(widget, AdminTemplateSelector)

class WidgetTestCase(SimpleTestCase):
    @skipIf(django.VERSION[0:2] < (1, 11), "won't render properly in Django pre-template-widgets")
    def test_html_output(self):
        form = form_cls()(data={'a': '1', 'b': 'admin/404.html'})
        STATIC_URL = "/TESTOUTPUT/"
        with override_settings(STATIC_URL=STATIC_URL):
            rendered = force_text(form['b'])

        self.assertHTMLEqual(rendered, """

        <input type="radio" name="b" value="admin/404.html" required id="id_b_0" checked />

        404

        <input type="radio" name="b" value="admin/500.html" required id="id_b_1" />

        500

        """.format(STATIC_URL=STATIC_URL))
<|endoftext|>


<commit_before>import re
import warnings
from django.conf import settings
from django.utils.importlib import import_module
from django.core.exceptions import ImproperlyConfigured
from tenant_schemas.utils import get_public_schema_name

ORIGINAL_BACKEND = getattr(settings, 'ORIGINAL_BACKEND', 'django.db.backends.postgresql_psycopg2')

original_backend = import_module('.base', ORIGINAL_BACKEND)

EXTRA_SEARCH_PATHS = getattr(settings, 'PG_EXTRA_SEARCH_PATHS', [])

# from the postgresql doc
SQL_IDENTIFIER_RE = re.compile('^[_a-zA-Z][_a-zA-Z0-9]{,62}$')

def _check_identifier(identifier):
    if not SQL_IDENTIFIER_RE.match(identifier):
        raise RuntimeError("Invalid string used for the schema name.")

class DatabaseWrapper(original_backend.DatabaseWrapper):
    """
    Adds the capability to manipulate the search_path using set_tenant and set_schema_name
    """
    include_public_schema = True

    def __init__(self, *args, **kwargs):
        super(DatabaseWrapper, self).__init__(*args, **kwargs)
        self.set_schema_to_public()

    def set_tenant(self, tenant, include_public=True):
        """
        Main API method to current database schema,
        but it does not actually modify the db connection.
        """
        self.tenant = tenant
        self.schema_name = tenant.schema_name
        self.include_public_schema = include_public
        self.set_settings_schema(self.schema_name)
        self._set_search_path()

    def set_schema(self, schema_name, include_public=True):
        """
        Main API method to current database schema,
        but it does not actually modify the db connection.
        """
        self.tenant = FakeTenant(schema_name=schema_name)
        self.schema_name = schema_name
        self.include_public_schema = include_public
        self.set_settings_schema(schema_name)
        self._set_search_path()

    def set_schema_to_public(self):
        """
        Instructs to stay in the common 'public' schema.
        """
        self.tenant = FakeTenant(schema_name=get_public_schema_name())
        self.schema_name = get_public_schema_name()
        self.set_settings_schema(self.schema_name)
        self._set_search_path()

    def set_settings_schema(self, schema_name):
        self.settings_dict['SCHEMA'] = schema_name

    def get_schema(self):
        warnings.warn("connection.get_schema() is deprecated, use connection.schema_name instead.",
                      category=DeprecationWarning)
        return self.schema_name

    def get_tenant(self):
        warnings.warn("connection.get_tenant() is deprecated, use connection.tenant instead.",
                      category=DeprecationWarning)
        return self.tenant

    def _set_search_path(self):
        # Actual search_path modification for the cursor. Database will
        # search schemata from left to right when looking for the object
        # (table, index, sequence, etc.).
        if not self.schema_name:
            raise ImproperlyConfigured("Illegal tenant; database schema not set.")
        _check_identifier(self.schema_name)
        public_schema_name = get_public_schema_name()
        search_paths = []

        if self.schema_name == public_schema_name:
            search_paths = [public_schema_name]
        elif self.include_public_schema:
            search_paths = [self.schema_name, public_schema_name]
        else:
            search_paths = [self.schema_name]

        search_paths.extend(EXTRA_SEARCH_PATHS)
        cursor = self._cursor()
        cursor.execute('SET search_path = {0}'.format(','.join(search_paths)))

class FakeTenant:
    """
    We can't import any db model in a backend (apparently?), so this class is used
    for wrapping schema names in a tenant-like structure.
    """
    def __init__(self, schema_name):
        self.schema_name = schema_name

DatabaseError = original_backend.DatabaseError
IntegrityError = original_backend.IntegrityError
<commit_msg>Revert to setting search_path in _cursor()<commit_after>import re
import warnings
from django.conf import settings
from django.utils.importlib import import_module
from django.core.exceptions import ImproperlyConfigured
from tenant_schemas.utils import get_public_schema_name

ORIGINAL_BACKEND = getattr(settings, 'ORIGINAL_BACKEND', 'django.db.backends.postgresql_psycopg2')

original_backend = import_module('.base', ORIGINAL_BACKEND)

EXTRA_SEARCH_PATHS = getattr(settings, 'PG_EXTRA_SEARCH_PATHS', [])

# from the postgresql doc
SQL_IDENTIFIER_RE = re.compile('^[_a-zA-Z][_a-zA-Z0-9]{,62}$')

def _check_identifier(identifier):
    if not SQL_IDENTIFIER_RE.match(identifier):
        raise RuntimeError("Invalid string used for the schema name.")

class DatabaseWrapper(original_backend.DatabaseWrapper):
    """
    Adds the capability to manipulate the search_path using set_tenant and set_schema_name
    """
    include_public_schema = True

    def __init__(self, *args, **kwargs):
        super(DatabaseWrapper, self).__init__(*args, **kwargs)
        self.set_schema_to_public()

    def set_tenant(self, tenant, include_public=True):
        """
        Main API method to current database schema,
        but it does not actually modify the db connection.
        """
        self.tenant = tenant
        self.schema_name = tenant.schema_name
        self.include_public_schema = include_public
        self.set_settings_schema(self.schema_name)
        self.search_path_set = False

    def set_schema(self, schema_name, include_public=True):
        """
        Main API method to current database schema,
        but it does not actually modify the db connection.
        """
        self.tenant = FakeTenant(schema_name=schema_name)
        self.schema_name = schema_name
        self.include_public_schema = include_public
        self.set_settings_schema(schema_name)
        self.search_path_set = False

    def set_schema_to_public(self):
        """
        Instructs to stay in the common 'public' schema.
        """
        self.tenant = FakeTenant(schema_name=get_public_schema_name())
        self.schema_name = get_public_schema_name()
        self.set_settings_schema(self.schema_name)
        self.search_path_set = False

    def set_settings_schema(self, schema_name):
        self.settings_dict['SCHEMA'] = schema_name

    def get_schema(self):
        warnings.warn("connection.get_schema() is deprecated, use connection.schema_name instead.",
                      category=DeprecationWarning)
        return self.schema_name

    def get_tenant(self):
        warnings.warn("connection.get_tenant() is deprecated, use connection.tenant instead.",
                      category=DeprecationWarning)
        return self.tenant

    def _cursor(self):
        """
        Here it happens. We hope every Django db operation using PostgreSQL
        must go through this to get the cursor handle. We change the path.
        """
        cursor = super(DatabaseWrapper, self)._cursor()

        if not self.search_path_set:
            # Actual search_path modification for the cursor. Database will
            # search schemata from left to right when looking for the object
            # (table, index, sequence, etc.).
            if not self.schema_name:
                raise ImproperlyConfigured("Database schema not set. Did you forget "
                                           "to call set_schema() or set_tenant()?")
            _check_identifier(self.schema_name)
            public_schema_name = get_public_schema_name()
            search_paths = []

            if self.schema_name == public_schema_name:
                search_paths = [public_schema_name]
            elif self.include_public_schema:
                search_paths = [self.schema_name, public_schema_name]
            else:
                search_paths = [self.schema_name]

            search_paths.extend(EXTRA_SEARCH_PATHS)
            cursor.execute('SET search_path = {0}'.format(','.join(search_paths)))
            self.search_path_set = True
        return cursor

class FakeTenant:
    """
    We can't import any db model in a backend (apparently?), so this class is used
    for wrapping schema names in a tenant-like structure.
    """
    def __init__(self, schema_name):
        self.schema_name = schema_name

DatabaseError = original_backend.DatabaseError
IntegrityError = original_backend.IntegrityError
<|endoftext|>


<commit_before># encoding: utf-8
import logging
from south.v2 import DataMigration
from django.conf import settings
from dimagi.utils.couch.database import iter_docs
from django.utils.encoding import force_unicode
from django.utils.safestring import mark_safe
from corehq.apps.app_manager.const import APP_V1
from corehq.apps.app_manager.detail_screen import get_column_xpath_generator
from corehq.apps.app_manager.models import Application
from corehq.apps.app_manager.xpath import dot_interpolate

logger = logging.getLogger(__name__)

class Migration(DataMigration):

    def forwards(self, orm):
        if not settings.UNIT_TESTING:
            application_ids = {r['id'] for r in Application.get_db().view(
                'app_manager/applications',
                reduce=False,
            ).all()}

            to_save = []

            for app_doc in iter_docs(Application.get_db(), application_ids):

                if app_doc["doc_type"] in ["Application", "Application-Deleted"]:
                    application = Application.wrap(app_doc)
                    self.migrate_app(application)

                    to_save.append(application)
                    if len(to_save) > 25:
                        self.bulk_save(to_save)
                        to_save = []
            if to_save:
                self.bulk_save(to_save)

    @classmethod
    def bulk_save(cls, apps):
        Application.get_db().bulk_save(apps)
        for app in apps:
            logger.info("Filter migration on app {id} complete.".format(id=app.id))

    @classmethod
    def migrate_app(cls, app):
        filter_combination_func = cls.combine_and_interpolate_V2_filters
        if app.application_version == APP_V1:
            filter_combination_func = \
                cls.combine_and_interpolate_V1_filters

        for module in app.get_modules():
            detail = module.case_details.short
            combined_filter_string = filter_combination_func(
                detail.get_columns(), app, module, detail
            )
            detail.filter = combined_filter_string

    @classmethod
    def combine_and_interpolate_V1_filters(cls, columns, app, module, detail):
        """
        Return a single filter xpath generated by ANDing together the given
        componenets. Also replaces "."s with the corresponding xpath. The
        interpolation here is specific to v1 apps! use
        combine_and_interpolate_V2_filters for V2 apps.
        :param column_filters: A list of columns
        :param app:
        :param module:
        :param detail:
        :return:
        """
        filters = []
        for i, column in enumerate(columns):
            if column.format == 'filter':
                value = dot_interpolate(
                    column.filter_xpath,
                    '%s_%s_%s' % (column.model, column.field, i + 1)
                )
                filters.append("(%s)" % value)
        xpath = ' and '.join(filters)
        return cls.partial_escape(xpath)

    @classmethod
    def partial_escape(cls, xpath):
        """
        Copied from http://stackoverflow.com/questions/275174/how-do-i-perform-html-decoding-encoding-using-python-django
        but without replacing the single quote
        """
        return mark_safe(
            force_unicode(xpath).replace(
                '&', '&'
            ).replace(
                '<', '<'
            ).replace(
                '>', '>'
            ).replace(
                '"', '"'
            )
        )

    @classmethod
    def combine_and_interpolate_V2_filters(cls, columns, app, module, detail):
        """
        Return a single filter xpath generated by ANDing together the given
        componenets. Also replaces "."s with the corresponding xpath.
        The interpolation here is specific to v2 apps! use
        combine_and_interpolate_V1_filters for V1 apps.
        :param column_filters: A list of columns
        :param app:
        :param module:
        :param detail:
        :return:
        """
        interpolated_filters = []
        for column in columns:
            if column.format == "filter":
                # filters might have a "." in them like: . = "VT"
                # We need to replace these dots with the names of the
                # properties that they refer to.
                #
                # So, if we had a case property called "state", the filter
                # xpath would be converted to: state = "VT"

                # The string that will replace "."s
                replacer_xpath = get_column_xpath_generator(
                    app, module, detail, column
                ).xpath

                # The filter with "."s replaced
                interpolated_xpath = dot_interpolate(
                    column.filter_xpath, replacer_xpath
                )

                interpolated_filters.append(interpolated_xpath)

        combined_filter = ' and '.join(
            '(%s)' % f for f in interpolated_filters
        )

        return combined_filter

    def backwards(self, orm):
        pass

    models = {}
    complete_apps = ['app_manager']
<commit_msg>Catch errors<commit_after># encoding: utf-8
import logging
import sys
from south.v2 import DataMigration
from django.conf import settings
from dimagi.utils.couch.database import iter_docs
from django.utils.encoding import force_unicode
from django.utils.safestring import mark_safe
from corehq.apps.app_manager.const import APP_V1
from corehq.apps.app_manager.detail_screen import get_column_xpath_generator
from corehq.apps.app_manager.models import Application
from corehq.apps.app_manager.xpath import dot_interpolate

logger = logging.getLogger(__name__)

class Migration(DataMigration):

    def forwards(self, orm):
        if not settings.UNIT_TESTING:
            application_ids = {r['id'] for r in Application.get_db().view(
                'app_manager/applications',
                reduce=False,
            ).all()}

            to_save = []
            errors = []

            for app_doc in iter_docs(Application.get_db(), application_ids):
                try:
                    if app_doc["doc_type"] in ["Application", "Application-Deleted"]:
                        application = Application.wrap(app_doc)
                        self.migrate_app(application)

                        to_save.append(application)
                        if len(to_save) > 25:
                            self.bulk_save(to_save)
                            to_save = []
                except:
                    errors.append(
                            ("App {id} not properly migrated because {error}".format(id=app_doc['_id'],
                                                                                error=sys.exc_info()[0])))

            if to_save:
                self.bulk_save(to_save)

    @classmethod
    def bulk_save(cls, apps):
        Application.get_db().bulk_save(apps)
        for app in apps:
            logger.info("Filter migration on app {id} complete.".format(id=app.id))

    @classmethod
    def migrate_app(cls, app):
        filter_combination_func = cls.combine_and_interpolate_V2_filters
        if app.application_version == APP_V1:
            filter_combination_func = \
                cls.combine_and_interpolate_V1_filters

        for module in app.get_modules():
            detail = module.case_details.short
            combined_filter_string = filter_combination_func(
                detail.get_columns(), app, module, detail
            )
            detail.filter = combined_filter_string

    @classmethod
    def combine_and_interpolate_V1_filters(cls, columns, app, module, detail):
        """
        Return a single filter xpath generated by ANDing together the given
        componenets. Also replaces "."s with the corresponding xpath. The
        interpolation here is specific to v1 apps! use
        combine_and_interpolate_V2_filters for V2 apps.
        :param column_filters: A list of columns
        :param app:
        :param module:
        :param detail:
        :return:
        """
        filters = []
        for i, column in enumerate(columns):
            if column.format == 'filter':
                value = dot_interpolate(
                    column.filter_xpath,
                    '%s_%s_%s' % (column.model, column.field, i + 1)
                )
                filters.append("(%s)" % value)
        xpath = ' and '.join(filters)
        return cls.partial_escape(xpath)

    @classmethod
    def partial_escape(cls, xpath):
        """
        Copied from http://stackoverflow.com/questions/275174/how-do-i-perform-html-decoding-encoding-using-python-django
        but without replacing the single quote
        """
        return mark_safe(
            force_unicode(xpath).replace(
                '&', '&'
            ).replace(
                '<', '<'
            ).replace(
                '>', '>'
            ).replace(
                '"', '"'
            )
        )

    @classmethod
    def combine_and_interpolate_V2_filters(cls, columns, app, module, detail):
        """
        Return a single filter xpath generated by ANDing together the given
        componenets. Also replaces "."s with the corresponding xpath.
        The interpolation here is specific to v2 apps! use
        combine_and_interpolate_V1_filters for V1 apps.
        :param column_filters: A list of columns
        :param app:
        :param module:
        :param detail:
        :return:
        """
        interpolated_filters = []
        for column in columns:
            if column.format == "filter":
                # filters might have a "." in them like: . = "VT"
                # We need to replace these dots with the names of the
                # properties that they refer to.
                #
                # So, if we had a case property called "state", the filter
                # xpath would be converted to: state = "VT"

                # The string that will replace "."s
                replacer_xpath = get_column_xpath_generator(
                    app, module, detail, column
                ).xpath

                # The filter with "."s replaced
                interpolated_xpath = dot_interpolate(
                    column.filter_xpath, replacer_xpath
                )

                interpolated_filters.append(interpolated_xpath)

        combined_filter = ' and '.join(
            '(%s)' % f for f in interpolated_filters
        )

        return combined_filter

    def backwards(self, orm):
        pass

    models = {}
    complete_apps = ['app_manager']
<|endoftext|>


<commit_before>import os
import zipfile
from collections import Counter

from django.core.management.base import BaseCommand, CommandError

from corehq.apps.dump_reload.couch.load import (
    CouchDataLoader,
    DomainLoader,
    ToggleLoader,
)
from corehq.apps.dump_reload.exceptions import DataExistsException
from corehq.apps.dump_reload.sql import SqlDataLoader

# Domain loader should be first
LOADERS = [DomainLoader, SqlDataLoader, CouchDataLoader, ToggleLoader]

class Command(BaseCommand):
    help = 'Loads data from the give file into the database.'

    def add_arguments(self, parser):
        parser.add_argument('dump_file_path')
        parser.add_argument('--use-extracted', action='store_true', default=False, dest='use_extracted',
                            help="Use already extracted dump if it exists.")
        parser.add_argument('--force', action='store_true', default=False, dest='force',
                            help="Load data for domain that already exists.")
        parser.add_argument('--loader', dest='loaders', action='append', default=[],
                            help='loader slug to run (use multiple --loader to run multiple loaders).'
                                 'domain should always be the first loader to be invoked in case of '
                                 'very first import',
                            choices=[loader.slug for loader in LOADERS])
        parser.add_argument('--object-filter',
                            help="Regular expression to use to selectively load data. Will be matched"
                                 " against a CouchDB 'doc_type' or Django model name: 'app_label.ModelName'."
                                 "Use 'print_domain_stats' command to get a list of available types.")

    def handle(self, dump_file_path, **options):
        self.verbosity = options.get('verbosity')
        self.force = options.get('force')
        self.use_extracted = options.get('use_extracted')

        if not os.path.isfile(dump_file_path):
            raise CommandError("Dump file not found: {}".format(dump_file_path))

        if self.verbosity >= 2:
            self.stdout.write("Loading data from %s." % dump_file_path)

        extracted_dir = self.extract_dump_archive(dump_file_path)

        total_object_count = 0
        model_counts = Counter()
        loaders = options.get('loaders')
        object_filter = options.get('object_filter')
        if loaders:
            loaders = [loader for loader in LOADERS if loader.slug in loaders]
        else:
            loaders = LOADERS

        for loader in loaders:
            loader_total_object_count, loader_model_counts = self._load_data(object_filter, loader, extracted_dir)
            total_object_count += loader_total_object_count
            model_counts.update(loader_model_counts)

        loaded_object_count = sum(model_counts.values())

        if self.verbosity >= 2:
            self.stdout.write('{0} Load Stats {0}'.format('-' * 40))
            for model in sorted(model_counts):
                self.stdout.write("{:<48}: {}".format(model, model_counts[model]))
            self.stdout.write('{0}{0}'.format('-' * 46))
            self.stdout.write('Loaded {}/{} objects'.format(loaded_object_count, total_object_count))
            self.stdout.write('{0}{0}'.format('-' * 46))
        else:
            self.stdout.write("Loaded %d object(s) (of %d)" %
                              (loaded_object_count, total_object_count))

    def extract_dump_archive(self, dump_file_path):
        target_dir = '_tmp_load_{}'.format(dump_file_path)
        if not os.path.exists(target_dir):
            with zipfile.ZipFile(dump_file_path, 'r') as archive:
                archive.extractall(target_dir)
        elif not self.use_extracted:
            raise CommandError(
                "Extracted dump already exists at {}. Delete it or use --use-extracted".format(target_dir))
        return target_dir

    def _load_data(self, loader_class, extracted_dump_path, object_filter):
        try:
            loader = loader_class(object_filter, self.stdout, self.stderr)
            return loader.load_from_file(extracted_dump_path, self.force)
        except DataExistsException as e:
            raise CommandError('Some data already exists. Use --force to load anyway: {}'.format(str(e)))
        except Exception as e:
            if not isinstance(e, CommandError):
                e.args = ("Problem loading data '%s': %s" % (extracted_dump_path, e),)
            raise
<commit_msg>fix arg order<commit_after>import os
import zipfile
from collections import Counter

from django.core.management.base import BaseCommand, CommandError

from corehq.apps.dump_reload.couch.load import (
    CouchDataLoader,
    DomainLoader,
    ToggleLoader,
)
from corehq.apps.dump_reload.exceptions import DataExistsException
from corehq.apps.dump_reload.sql import SqlDataLoader

# Domain loader should be first
LOADERS = [DomainLoader, SqlDataLoader, CouchDataLoader, ToggleLoader]

class Command(BaseCommand):
    help = 'Loads data from the give file into the database.'

    def add_arguments(self, parser):
        parser.add_argument('dump_file_path')
        parser.add_argument('--use-extracted', action='store_true', default=False, dest='use_extracted',
                            help="Use already extracted dump if it exists.")
        parser.add_argument('--force', action='store_true', default=False, dest='force',
                            help="Load data for domain that already exists.")
        parser.add_argument('--loader', dest='loaders', action='append', default=[],
                            help='loader slug to run (use multiple --loader to run multiple loaders).'
                                 'domain should always be the first loader to be invoked in case of '
                                 'very first import',
                            choices=[loader.slug for loader in LOADERS])
        parser.add_argument('--object-filter',
                            help="Regular expression to use to selectively load data. Will be matched"
                                 " against a CouchDB 'doc_type' or Django model name: 'app_label.ModelName'."
                                 "Use 'print_domain_stats' command to get a list of available types.")

    def handle(self, dump_file_path, **options):
        self.verbosity = options.get('verbosity')
        self.force = options.get('force')
        self.use_extracted = options.get('use_extracted')

        if not os.path.isfile(dump_file_path):
            raise CommandError("Dump file not found: {}".format(dump_file_path))

        if self.verbosity >= 2:
            self.stdout.write("Loading data from %s." % dump_file_path)

        extracted_dir = self.extract_dump_archive(dump_file_path)

        total_object_count = 0
        model_counts = Counter()
        loaders = options.get('loaders')
        object_filter = options.get('object_filter')
        if loaders:
            loaders = [loader for loader in LOADERS if loader.slug in loaders]
        else:
            loaders = LOADERS

        for loader in loaders:
            loader_total_object_count, loader_model_counts = self._load_data(loader, extracted_dir, object_filter)
            total_object_count += loader_total_object_count
            model_counts.update(loader_model_counts)

        loaded_object_count = sum(model_counts.values())

        if self.verbosity >= 2:
            self.stdout.write('{0} Load Stats {0}'.format('-' * 40))
            for model in sorted(model_counts):
                self.stdout.write("{:<48}: {}".format(model, model_counts[model]))
            self.stdout.write('{0}{0}'.format('-' * 46))
            self.stdout.write('Loaded {}/{} objects'.format(loaded_object_count, total_object_count))
            self.stdout.write('{0}{0}'.format('-' * 46))
        else:
            self.stdout.write("Loaded %d object(s) (of %d)" %
                              (loaded_object_count, total_object_count))

    def extract_dump_archive(self, dump_file_path):
        target_dir = '_tmp_load_{}'.format(dump_file_path)
        if not os.path.exists(target_dir):
            with zipfile.ZipFile(dump_file_path, 'r') as archive:
                archive.extractall(target_dir)
        elif not self.use_extracted:
            raise CommandError(
                "Extracted dump already exists at {}. Delete it or use --use-extracted".format(target_dir))
        return target_dir

    def _load_data(self, loader_class, extracted_dump_path, object_filter):
        try:
            loader = loader_class(object_filter, self.stdout, self.stderr)
            return loader.load_from_file(extracted_dump_path, self.force)
        except DataExistsException as e:
            raise CommandError('Some data already exists. Use --force to load anyway: {}'.format(str(e)))
        except Exception as e:
            if not isinstance(e, CommandError):
                e.args = ("Problem loading data '%s': %s" % (extracted_dump_path, e),)
            raise
<|endoftext|>


<commit_before><commit_msg>Revert "Adding University rankings route"<commit_after><|endoftext|>


<commit_before>from daybed.tests.support import BaseWebTest

class FunctionalTest(BaseWebTest):
    """These are the functional tests for daybed.

    The goal is to have them reproduce every possible scenario that we want to
    support in the application.

    The test suite is created in a way that each test is a different scenario.
    We reset the database each time we start a new test to avoid sharing
    context between tests.
    """

    def __init__(self, *args, **kwargs):
        super(FunctionalTest, self).__init__(*args, **kwargs)
        self.valid_definition = {
            "title": "todo",
            "description": "A list of my stuff to do",
            "fields": [
                {
                    "name": "item",
                    "type": "string",
                    "description": "The item"
                },
                {
                    "name": "status",
                    "type": "enum",
                    "choices": [
                        "done",
                        "todo"
                    ],
                    "description": "is it done or not"
                }
            ]}

        self.definition_without_title = self.valid_definition.copy()
        self.definition_without_title.pop('title')
        self.valid_data = {'item': 'My task', 'status': 'todo'}
        self.malformed_definition = '{"test":"toto", "titi": "tutu'
        self.invalid_data = {'item': 'Invalid task', 'status': 'yay'}
        self.headers = {'Content-Type': 'application/json'}

    def create_definition(self, data=None):
        if not data:
            data = self.valid_definition

        return self.app.put_json('/definitions/todo', data,
                                 headers=self.headers)

    def create_data(self, data=None):
        if not data:
            data = self.valid_data
        return self.app.post_json('/data/todo', data,
                                  headers=self.headers)

    def create_data_resp(self, data=None):
        if not data:
            data = self.valid_data

        return self.app.post_json('/data/todo',
                                  data,
                                  headers=self.headers)

    def test_normal_definition_creation(self):
        resp = self.create_definition()
        self.assertIn('token', resp.body)

    def test_malformed_definition_creation(self):
        resp = self.app.put_json('/definitions/todo',
                    self.definition_without_title,
                    headers=self.headers,
                    status=400)
        self.assertIn('"name": "title"', resp.body)

    def test_definition_creation_rejects_malformed_data(self):
        resp = self.app.put('/definitions/todo',
                    self.malformed_definition,
                    headers=self.headers,
                    status=400)
        self.assertIn('"status": "error"', resp.body)

    def test_definition_retrieval(self):
        self.create_definition()

        # Verify that the schema is the same
        resp = self.app.get('/definitions/todo', headers=self.headers)
        self.assertEqual(resp.json, self.valid_definition)

    def test_definition_deletion(self):
        resp = self.create_definition()
        token = resp.json['token']
        resp = self.create_data()
        data_item_id = resp.json['id']
        self.app.delete(str('/definitions/todo?token=%s' % token))
        queryset = self.db.get_data_item('todo', data_item_id)
        self.assertIsNone(queryset)
        queryset = self.db.get_definition('todo')
        self.assertIsNone(queryset)

    def test_normal_data_creation(self):
        self.create_definition()

        # Put data against this definition
        resp = self.app.post_json('/data/todo',
                                 self.valid_data,
                                 headers=self.headers)
        self.assertIn('id', resp.body)

    def test_invalid_data_validation(self):
        self.create_definition()

        # Try to put invalid data to this definition
        resp = self.app.post_json('/data/todo',
                                 {'item': 'My task',
                                  'status': 'false'},
                                 headers=self.headers,
                                 status=400)
        self.assertIn('"status": "error"', resp.body)

    def test_data_retrieval(self):
        self.create_definition()
        resp = self.create_data()

        # Put valid data against this definition
        self.assertIn('id', resp.body)

        data_item_id = resp.json['id']
        resp = self.app.get('/data/todo/%s' % data_item_id,
                            headers=self.headers)
        entry = self.valid_data.copy()
        # entry['id'] = str(data_item_id
        self.assertEqual(resp.json, entry)

    def test_data_update(self):
        self.create_definition()
        # Put data against this definition
        entry = self.valid_data.copy()
        resp = self.create_data(entry)
        data_item_id = resp.json['id']

        # Update this data
        entry['status'] = 'done'
        resp = self.app.put_json(str('/data/todo/%s' % data_item_id),
                                 entry,
                                 headers=self.headers)
        self.assertIn('id', resp.body)
        # Todo : Verify DB
        queryset = self.db.get_data('todo')
        self.assertEqual(len(queryset), 1)

    def test_data_deletion(self):
        self.create_definition()
        resp = self.create_data()
        data_item_id = resp.json['id']
        self.app.delete(str('/data/todo/%s' % data_item_id))
        queryset = self.db.get_data_item('todo', data_item_id)
        self.assertIsNone(queryset)

    def test_data_validation(self):
        self.create_definition()
        headers = self.headers.copy()
        headers['X-Daybed-Validate-Only'] = 'true'
        self.app.post_json('/data/todo', self.valid_data,
                           headers=headers, status=200)

        # no data should be added
        self.assertEquals(0, len(self.app.get('/data/todo').json['data']))
        # of course, pushing weird data should tell what's wrong
        self.app.post_json('/data/todo', self.invalid_data,
                           headers=headers, status=400)
<commit_msg>Prepare inheritance of functional tests (ref #31)<commit_after>from daybed.tests.support import BaseWebTest

class FunctionalTest(BaseWebTest):
    """These are the functional tests for daybed.

    The goal is to have them reproduce every possible scenario that we want to
    support in the application.

    The test suite is created in a way that each test is a different scenario.
    We reset the database each time we start a new test to avoid sharing
    context between tests.
    """

    def __init__(self, *args, **kwargs):
        super(FunctionalTest, self).__init__(*args, **kwargs)
        self.model_name = 'todo'
        self.valid_definition = {
            "title": "todo",
            "description": "A list of my stuff to do",
            "fields": [
                {
                    "name": "item",
                    "type": "string",
                    "description": "The item"
                },
                {
                    "name": "status",
                    "type": "enum",
                    "choices": [
                        "done",
                        "todo"
                    ],
                    "description": "is it done or not"
                }
            ]}

        self.definition_without_title = self.valid_definition.copy()
        self.definition_without_title.pop('title')
        self.valid_data = {'item': 'My task', 'status': 'todo'}
        self.malformed_definition = '{"test":"toto", "titi": "tutu'
        self.invalid_data = {'item': 'Invalid task', 'status': 'yay'}
        self.headers = {'Content-Type': 'application/json'}

    def create_definition(self, data=None):
        if not data:
            data = self.valid_definition
        return self.app.put_json('/definitions/%s' % self.model_name,
                                 data,
                                 headers=self.headers)

    def create_data(self, data=None):
        if not data:
            data = self.valid_data
        return self.app.post_json('/data/%s' % self.model_name,
                                  data,
                                  headers=self.headers)

    def create_data_resp(self, data=None):
        if not data:
            data = self.valid_data
        return self.app.post_json('/data/%s' % self.model_name,
                                  data,
                                  headers=self.headers)

    def test_normal_definition_creation(self):
        resp = self.create_definition()
        self.assertIn('token', resp.body)

    def test_malformed_definition_creation(self):
        resp = self.app.put_json('/definitions/%s' % self.model_name,
                    self.definition_without_title,
                    headers=self.headers,
                    status=400)
        self.assertIn('"name": "title"', resp.body)

    def test_definition_creation_rejects_malformed_data(self):
        resp = self.app.put('/definitions/%s' % self.model_name,
                    self.malformed_definition,
                    headers=self.headers,
                    status=400)
        self.assertIn('"status": "error"', resp.body)

    def test_definition_retrieval(self):
        self.create_definition()

        # Verify that the schema is the same
        resp = self.app.get('/definitions/%s' % self.model_name,
                            headers=self.headers)
        self.assertEqual(resp.json, self.valid_definition)

    def test_definition_deletion(self):
        resp = self.create_definition()
        token = resp.json['token']
        resp = self.create_data()
        data_item_id = resp.json['id']
        self.app.delete(str('/definitions/%s?token=%s' % (
                            self.model_name, token)))
        queryset = self.db.get_data_item(self.model_name,
                                         data_item_id)
        self.assertIsNone(queryset)
        queryset = self.db.get_definition(self.model_name)
        self.assertIsNone(queryset)

    def test_normal_data_creation(self):
        self.create_definition()

        # Put data against this definition
        resp = self.app.post_json('/data/%s' % self.model_name,
                                 self.valid_data,
                                 headers=self.headers)
        self.assertIn('id', resp.body)

    def test_invalid_data_validation(self):
        self.create_definition()

        # Try to put invalid data to this definition
        resp = self.app.post_json('/data/%s' % self.model_name,
                                  {'item': 'My task',
                                   'status': 'false'},
                                  headers=self.headers,
                                  status=400)
        self.assertIn('"status": "error"', resp.body)

    def test_data_retrieval(self):
        self.create_definition()
        resp = self.create_data()

        # Put valid data against this definition
        self.assertIn('id', resp.body)

        data_item_id = resp.json['id']
        resp = self.app.get('/data/%s/%s' % (self.model_name,
                                             data_item_id),
                            headers=self.headers)
        entry = self.valid_data.copy()
        # entry['id'] = str(data_item_id
        self.assertEqual(resp.json, entry)

    def test_data_update(self):
        self.create_definition()
        # Put data against this definition
        entry = self.valid_data.copy()
        resp = self.create_data(entry)
        data_item_id = resp.json['id']

        # Update this data
        entry['status'] = 'done'
        resp = self.app.put_json(str('/data/%s/%s' % (
                                     self.model_name,
                                     data_item_id)),
                                 entry,
                                 headers=self.headers)
        self.assertIn('id', resp.body)
        # Todo : Verify DB
        queryset = self.db.get_data(self.model_name)
        self.assertEqual(len(queryset), 1)

    def test_data_deletion(self):
        self.create_definition()
        resp = self.create_data()
        data_item_id = resp.json['id']
        self.app.delete(str('/data/%s/%s' % (self.model_name,
                                             data_item_id)))
        queryset = self.db.get_data_item(self.model_name,
                                         data_item_id)
        self.assertIsNone(queryset)

    def test_data_validation(self):
        self.create_definition()
        headers = self.headers.copy()
        headers['X-Daybed-Validate-Only'] = 'true'
        self.app.post_json('/data/%s' % self.model_name,
                           self.valid_data,
                           headers=headers, status=200)

        # no data should be added
        response = self.app.get('/data/%s' % self.model_name)
        self.assertEquals(0, len(response.json['data']))
        # of course, pushing weird data should tell what's wrong
        self.app.post_json('/data/%s' % self.model_name,
                           self.invalid_data,
                           headers=headers, status=400)
<|endoftext|>