<commit_before># coding: utf-8
# Copyright 2009 Alexandre Fiori
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import, division

import time

from io import BytesIO as StringIO

import os
from bson import objectid, timestamp
from pymongo.write_concern import WriteConcern
import txmongo
from txmongo import database, collection, filter as qf
from txmongo.gridfs import GridFS, GridIn, GridOut, GridOutIterator, errors
from txmongo._gridfs.errors import NoFile
from twisted.trial import unittest
from twisted.internet import defer
from twisted.python.compat import _PY3
from twisted import _version

mongo_host = "127.0.0.1"
mongo_port = 27017

class TestMongoObjects(unittest.TestCase):
    @defer.inlineCallbacks
    def test_MongoObjects(self):
        """ Tests creating mongo objects """
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        mydb = conn.mydb
        self.assertEqual(isinstance(mydb, database.Database), True)
        if _PY3:
            self.assertEqual(repr(mydb), "Database(Connection('127.0.0.1', 27017), 'mydb')")
        else:
            self.assertEqual(repr(mydb), "Database(Connection('127.0.0.1', 27017), u'mydb')")
        self.assertEqual(repr(mydb("mydb2")), repr(mydb.__call__("mydb2")))
        mycol = mydb.mycol
        self.assertEqual(isinstance(mycol, collection.Collection), True)
        mycol2 = yield mydb.create_collection("mycol2")
        self.assertEqual(isinstance(mycol2, collection.Collection), True)
        mycol3 = yield mydb.create_collection("mycol3", {"size": 1000})
        self.assertEqual(isinstance(mycol3, collection.Collection), True)
        yield mydb.drop_collection("mycol3")
        yield mydb.drop_collection(mycol3)
        yield self.assertFailure(mydb.drop_collection(None), TypeError)
        yield conn.disconnect()

    @defer.inlineCallbacks
    def test_Properties(self):
        conn = txmongo.MongoConnection(mongo_host, mongo_port)
        db = conn.mydb
        coll = db.mycol

        try:
            # names
            self.assertEqual(db.name, u"mydb")
            self.assertEqual(coll.name, u"mycol")
            self.assertEqual(coll.full_name, u"mydb.mycol")
            self.assertEqual(coll.subcoll.name, u"mycol.subcoll")
            self.assertEqual(coll.subcoll.full_name, u"mydb.mycol.subcoll")

            # database
            self.assertTrue(coll.database is db)

            # Write concern
            w2 = coll.with_options(write_concern=WriteConcern(w=2))
            dbw2 = database.Database(conn, "mydb", write_concern=WriteConcern(w=2))
            self.assertEqual(w2.write_concern, WriteConcern(w=2))
            self.assertEqual(dbw2.write_concern, WriteConcern(w=2))

            # Connection
            self.assertTrue(db.connection is conn)
        finally:
            yield conn.disconnect()

    @defer.inlineCallbacks
    def test_MongoOperations(self):
        """ Tests mongo operations """
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        test = conn.foo.test

        # insert
        doc = {"foo": "bar", "items": [1, 2, 3]}
        yield test.insert(doc, safe=True)
        result = yield test.find_one(doc)
        self.assertEqual("_id" in result, True)
        self.assertEqual(result["foo"], "bar")
        self.assertEqual(result["items"], [1, 2, 3])

        # insert preserves object id
        doc.update({"_id": objectid.ObjectId()})
        yield test.insert(doc, safe=True)
        result = yield test.find_one(doc)
        self.assertEqual(result.get("_id"), doc.get("_id"))
        self.assertEqual(result["foo"], "bar")
        self.assertEqual(result["items"], [1, 2, 3])

        # update
        yield test.update({"_id": result["_id"]}, {"$set": {"one": "two"}}, safe=True)
        result = yield test.find_one({"_id": result["_id"]})
        self.assertEqual(result["one"], "two")

        # delete
        yield test.remove(result["_id"], safe=True)

        # disconnect
        yield conn.disconnect()

    @defer.inlineCallbacks
    def test_Timestamps(self):
        """Tests mongo operations with Timestamps"""
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        test = conn.foo.test_ts

        test.drop()

        # insert with specific timestamp
        doc1 = {"_id": objectid.ObjectId(),
                "ts": timestamp.Timestamp(1, 2)}
        yield test.insert(doc1, safe=True)

        result = yield test.find_one(doc1)
        self.assertEqual(result.get("ts").time, 1)
        self.assertEqual(result.get("ts").inc, 2)

        # insert with specific timestamp
        doc2 = {"_id": objectid.ObjectId(),
                "ts": timestamp.Timestamp(2, 1)}
        yield test.insert(doc2, safe=True)

        # the objects come back sorted by ts correctly.
        # (test that we stored inc/time in the right fields)
        result = yield test.find(filter=qf.sort(qf.ASCENDING("ts")))
        self.assertEqual(len(result), 2)
        self.assertEqual(result[0]["_id"], doc1["_id"])
        self.assertEqual(result[1]["_id"], doc2["_id"])

        # insert with null timestamp
        doc3 = {"_id": objectid.ObjectId(),
                "ts": timestamp.Timestamp(0, 0)}
        yield test.insert(doc3, safe=True)

        # time field loaded correctly
        result = yield test.find_one(doc3["_id"])
        now = time.time()
        self.assertTrue(now - 2 <= result["ts"].time <= now)

        # delete
        yield test.remove(doc1["_id"], safe=True)
        yield test.remove(doc2["_id"], safe=True)
        yield test.remove(doc3["_id"], safe=True)

        # disconnect
        yield conn.disconnect()

class TestGridFsObjects(unittest.TestCase):
    """ Test the GridFS operations from txmongo._gridfs """
    @defer.inlineCallbacks
    def _disconnect(self, conn):
        """ Disconnect the connection """
        yield conn.disconnect()

    def _drop_gridfs(self, db):
        """
        Drop the default gridfs instance (i.e. ``fs``) associate to this database
        """
        return defer.gatherResults([
            db.drop_collection('fs.files'),
            db.drop_collection('fs.chunks')
        ])

    @defer.inlineCallbacks
    def test_GridFileObjects(self):
        """ Tests gridfs objects """
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        db = conn.test
        yield self._drop_gridfs(db)
        self.assertRaises(TypeError, GridFS, None)
        _ = GridFS(db)  # Default collection
        self.assertRaises(TypeError, GridIn, None)
        with GridIn(db.fs, filename="test_with", contentType="text/plain", chunk_size=1024):
            pass
        grid_in_file = GridIn(db.fs, filename="test_1", contentType="text/plain",
                              content_type="text/plain", chunk_size=65536, length=1048576,
                              upload_date="20150101")
        self.assertFalse(grid_in_file.closed)
        if _version.version.major >= 15:
            with self.assertRaises(TypeError):
                yield grid_in_file.write(1)
            with self.assertRaises(TypeError):
                yield grid_in_file.write(u"0xDEADBEEF")
            with self.assertRaises(AttributeError):
                _ = grid_in_file.test
        grid_in_file.test = 1
        yield grid_in_file.write(b"0xDEADBEEF")
        yield grid_in_file.write(b"0xDEADBEEF"*1048576)
        fake_doc = {"_id": "test_id", "length": 1048576, "filename": "test",
                    "upload_date": "20150101"}
        self.assertRaises(TypeError, GridOut, None, None)
        grid_out_file = GridOut(db.fs, fake_doc)
        if _version.version.major >= 15:
            with self.assertRaises(AttributeError):
                _ = grid_out_file.testing
        self.assertEqual("test", grid_out_file.filename)
        self.assertEqual(0, grid_out_file.tell())
        grid_out_file.seek(1024)
        self.assertEqual(1024, grid_out_file.tell())
        grid_out_file.seek(1024, os.SEEK_CUR)
        self.assertEqual(2048, grid_out_file.tell())
        grid_out_file.seek(0, os.SEEK_END)
        self.assertEqual(1048576, grid_out_file.tell())
        self.assertRaises(IOError, grid_out_file.seek, 0, 4)
        self.assertRaises(IOError, grid_out_file.seek, -1)
        self.assertTrue("'_id': 'test_id'" in repr(grid_out_file))
        self.assertTrue("20150101", grid_in_file.upload_date)
        yield grid_in_file.writelines([b"0xDEADBEEF", b"0xDEADBEAF"])
        yield grid_in_file.close()
        if _version.version.major >= 15:
            with self.assertRaises(AttributeError):
                grid_in_file.length = 1
        self.assertEqual(1, grid_in_file.test)
        if _version.version.major >= 15:
            with self.assertRaises(AttributeError):
                _ = grid_in_file.test_none
        self.assertTrue(grid_in_file.closed)
        if _version.version.major >= 15:
            with self.assertRaises(ValueError):
                yield grid_in_file.write(b"0xDEADBEEF")
        yield conn.disconnect()

    @defer.inlineCallbacks
    def test_GridFsObjects(self):
        """ Tests gridfs objects """
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        db = conn.test
        yield self._drop_gridfs(db)
        gfs = GridFS(db)  # Default collection
        yield gfs.delete(u"test")

        _ = gfs.new_file(filename="test_1", contentType="text/plain", chunk_size=65536)
        yield conn.disconnect()
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        db = conn.test
        yield self._drop_gridfs(db)
        gfs = GridFS(db)  # Default collection
        _ = yield gfs.put(b"0xDEADBEEF", filename="test_2", contentType="text/plain",
                          chunk_size=65536)
        # disconnect
        yield conn.disconnect()

        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        db = conn.test
        gfs = GridFS(db)  # Default collection
        # Missing file raises error
        yield self.assertFailure(gfs.get("test_3"), NoFile)
        # disconnect
        yield conn.disconnect()

    @defer.inlineCallbacks
    def test_GridFsIteration(self):
        """ Tests gridfs iterator """
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        db = conn.test
        yield self._drop_gridfs(db)
        gfs = GridFS(db)  # Default collection
        new_file = gfs.new_file(filename="testName", contentType="text/plain", length=1048576,
                                chunk_size=4096)
        yield new_file.write(b"0xDEADBEEF"*4096*2)
        yield new_file.close()
        fake_doc = {"_id": new_file._id, "name": "testName", "length": 4096*2, "chunkSize": 4096,
                    "contentType": "text/plain"}
        grid_out_file = GridOut(db.fs, fake_doc)
        iterator = GridOutIterator(grid_out_file, db.fs.chunks)
        next_it = yield next(iterator)
        self.assertEqual(len(next_it), 4096)
        _ = yield next(iterator)
        next_it = yield next(iterator)
        self.assertEqual(next_it, None)

        fake_bad_doc = {"_id": "bad_id", "name": "testName", "length": 4096*2,
                        "chunkSize": 4096, "contentType": "text/plain"}
        grid_bad_out_file = GridOut(db.fs, fake_bad_doc)
        bad_iterator = GridOutIterator(grid_bad_out_file, db.fs.chunks)
        if _version.version.major >= 15:
            with self.assertRaises(errors.CorruptGridFile):
                next_it = yield bad_iterator.next()

        # disconnect
        yield conn.disconnect()

    @defer.inlineCallbacks
    def test_GridFsOperations(self):
        """ Tests gridfs operations """
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        db = conn.test
        # Drop files first TODO: iterate through files and delete them
        yield self._drop_gridfs(db)

        # Don't forget to disconnect
        self.addCleanup(self._disconnect, conn)
        try:
            in_file = StringIO(b"Test input string")
            out_file = StringIO()
        except Exception as e:
            self.fail("Failed to create memory files for testing: %s" % e)

        g_out = None

        try:
            # Tests writing to a new gridfs file
            gfs = GridFS(db)  # Default collection
            if _version.version.major >= 15:
                with self.assertRaises(NoFile):
                    yield gfs.get_last_version("optest")

            g_in = gfs.new_file(filename="optest", contentType="text/plain",
                                chunk_size=65536)  # non-default chunk size used
            # yielding to ensure writes complete before we close and close before we try to read
            yield g_in.write(in_file.read())
            yield g_in.close()

            # Tests reading from an existing gridfs file
            g_out = yield gfs.get_last_version("optest")
            data = yield g_out.read()
            out_file.write(data)
            _id = g_out._id
        except Exception as e:
            self.fail("Failed to communicate with the GridFS. " +
                      "Is MongoDB running? %s" % e)
        else:
            self.assertEqual(in_file.getvalue(), out_file.getvalue(),
                             "Could not read the value from writing an input")
        finally:
            in_file.close()
            out_file.close()
            if g_out:
                g_out.close()

        listed_files = yield gfs.list()
        self.assertEqual(["optest"], listed_files,
                         "`optest` is the only expected file and we received %s" % listed_files)

        yield gfs.delete(_id)

    @defer.inlineCallbacks
    def test_GridFsIndexesCreation(self):
        """ Tests gridfs indexes creation"""
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        db = conn.test
        yield self._drop_gridfs(db)

        # Create a new GridFS instance should trigger indexes creation
        gfs = GridFS(db)

        # Multiple calls should return multiple defer not to mix between them
        self.assertNotEqual(gfs.indexes_created(), gfs.indexes_created())

        yield gfs.indexes_created()

        yield conn.disconnect()
<commit_msg>New way of getting Twisted version in trunk version<commit_after># coding: utf-8
# Copyright 2009 Alexandre Fiori
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import, division

import time

from io import BytesIO as StringIO

import os
from bson import objectid, timestamp
from pymongo.write_concern import WriteConcern
import txmongo
from txmongo import database, collection, filter as qf
from txmongo.gridfs import GridFS, GridIn, GridOut, GridOutIterator, errors
from txmongo._gridfs.errors import NoFile
from twisted.trial import unittest
from twisted.internet import defer
from twisted.python.compat import _PY3
from twisted._version import __version__ as twisted_version

mongo_host = "127.0.0.1"
mongo_port = 27017

class TestMongoObjects(unittest.TestCase):
    @defer.inlineCallbacks
    def test_MongoObjects(self):
        """ Tests creating mongo objects """
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        mydb = conn.mydb
        self.assertEqual(isinstance(mydb, database.Database), True)
        if _PY3:
            self.assertEqual(repr(mydb), "Database(Connection('127.0.0.1', 27017), 'mydb')")
        else:
            self.assertEqual(repr(mydb), "Database(Connection('127.0.0.1', 27017), u'mydb')")
        self.assertEqual(repr(mydb("mydb2")), repr(mydb.__call__("mydb2")))
        mycol = mydb.mycol
        self.assertEqual(isinstance(mycol, collection.Collection), True)
        mycol2 = yield mydb.create_collection("mycol2")
        self.assertEqual(isinstance(mycol2, collection.Collection), True)
        mycol3 = yield mydb.create_collection("mycol3", {"size": 1000})
        self.assertEqual(isinstance(mycol3, collection.Collection), True)
        yield mydb.drop_collection("mycol3")
        yield mydb.drop_collection(mycol3)
        yield self.assertFailure(mydb.drop_collection(None), TypeError)
        yield conn.disconnect()

    @defer.inlineCallbacks
    def test_Properties(self):
        conn = txmongo.MongoConnection(mongo_host, mongo_port)
        db = conn.mydb
        coll = db.mycol

        try:
            # names
            self.assertEqual(db.name, u"mydb")
            self.assertEqual(coll.name, u"mycol")
            self.assertEqual(coll.full_name, u"mydb.mycol")
            self.assertEqual(coll.subcoll.name, u"mycol.subcoll")
            self.assertEqual(coll.subcoll.full_name, u"mydb.mycol.subcoll")

            # database
            self.assertTrue(coll.database is db)

            # Write concern
            w2 = coll.with_options(write_concern=WriteConcern(w=2))
            dbw2 = database.Database(conn, "mydb", write_concern=WriteConcern(w=2))
            self.assertEqual(w2.write_concern, WriteConcern(w=2))
            self.assertEqual(dbw2.write_concern, WriteConcern(w=2))

            # Connection
            self.assertTrue(db.connection is conn)
        finally:
            yield conn.disconnect()

    @defer.inlineCallbacks
    def test_MongoOperations(self):
        """ Tests mongo operations """
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        test = conn.foo.test

        # insert
        doc = {"foo": "bar", "items": [1, 2, 3]}
        yield test.insert(doc, safe=True)
        result = yield test.find_one(doc)
        self.assertEqual("_id" in result, True)
        self.assertEqual(result["foo"], "bar")
        self.assertEqual(result["items"], [1, 2, 3])

        # insert preserves object id
        doc.update({"_id": objectid.ObjectId()})
        yield test.insert(doc, safe=True)
        result = yield test.find_one(doc)
        self.assertEqual(result.get("_id"), doc.get("_id"))
        self.assertEqual(result["foo"], "bar")
        self.assertEqual(result["items"], [1, 2, 3])

        # update
        yield test.update({"_id": result["_id"]}, {"$set": {"one": "two"}}, safe=True)
        result = yield test.find_one({"_id": result["_id"]})
        self.assertEqual(result["one"], "two")

        # delete
        yield test.remove(result["_id"], safe=True)

        # disconnect
        yield conn.disconnect()

    @defer.inlineCallbacks
    def test_Timestamps(self):
        """Tests mongo operations with Timestamps"""
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        test = conn.foo.test_ts

        test.drop()

        # insert with specific timestamp
        doc1 = {"_id": objectid.ObjectId(),
                "ts": timestamp.Timestamp(1, 2)}
        yield test.insert(doc1, safe=True)

        result = yield test.find_one(doc1)
        self.assertEqual(result.get("ts").time, 1)
        self.assertEqual(result.get("ts").inc, 2)

        # insert with specific timestamp
        doc2 = {"_id": objectid.ObjectId(),
                "ts": timestamp.Timestamp(2, 1)}
        yield test.insert(doc2, safe=True)

        # the objects come back sorted by ts correctly.
        # (test that we stored inc/time in the right fields)
        result = yield test.find(filter=qf.sort(qf.ASCENDING("ts")))
        self.assertEqual(len(result), 2)
        self.assertEqual(result[0]["_id"], doc1["_id"])
        self.assertEqual(result[1]["_id"], doc2["_id"])

        # insert with null timestamp
        doc3 = {"_id": objectid.ObjectId(),
                "ts": timestamp.Timestamp(0, 0)}
        yield test.insert(doc3, safe=True)

        # time field loaded correctly
        result = yield test.find_one(doc3["_id"])
        now = time.time()
        self.assertTrue(now - 2 <= result["ts"].time <= now)

        # delete
        yield test.remove(doc1["_id"], safe=True)
        yield test.remove(doc2["_id"], safe=True)
        yield test.remove(doc3["_id"], safe=True)

        # disconnect
        yield conn.disconnect()

class TestGridFsObjects(unittest.TestCase):
    """ Test the GridFS operations from txmongo._gridfs """
    @defer.inlineCallbacks
    def _disconnect(self, conn):
        """ Disconnect the connection """
        yield conn.disconnect()

    def _drop_gridfs(self, db):
        """
        Drop the default gridfs instance (i.e. ``fs``) associate to this database
        """
        return defer.gatherResults([
            db.drop_collection('fs.files'),
            db.drop_collection('fs.chunks')
        ])

    @defer.inlineCallbacks
    def test_GridFileObjects(self):
        """ Tests gridfs objects """
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        db = conn.test
        yield self._drop_gridfs(db)
        self.assertRaises(TypeError, GridFS, None)
        _ = GridFS(db)  # Default collection
        self.assertRaises(TypeError, GridIn, None)
        with GridIn(db.fs, filename="test_with", contentType="text/plain", chunk_size=1024):
            pass
        grid_in_file = GridIn(db.fs, filename="test_1", contentType="text/plain",
                              content_type="text/plain", chunk_size=65536, length=1048576,
                              upload_date="20150101")
        self.assertFalse(grid_in_file.closed)
        if twisted_version.major >= 15:
            with self.assertRaises(TypeError):
                yield grid_in_file.write(1)
            with self.assertRaises(TypeError):
                yield grid_in_file.write(u"0xDEADBEEF")
            with self.assertRaises(AttributeError):
                _ = grid_in_file.test
        grid_in_file.test = 1
        yield grid_in_file.write(b"0xDEADBEEF")
        yield grid_in_file.write(b"0xDEADBEEF"*1048576)
        fake_doc = {"_id": "test_id", "length": 1048576, "filename": "test",
                    "upload_date": "20150101"}
        self.assertRaises(TypeError, GridOut, None, None)
        grid_out_file = GridOut(db.fs, fake_doc)
        if twisted_version.major >= 15:
            with self.assertRaises(AttributeError):
                _ = grid_out_file.testing
        self.assertEqual("test", grid_out_file.filename)
        self.assertEqual(0, grid_out_file.tell())
        grid_out_file.seek(1024)
        self.assertEqual(1024, grid_out_file.tell())
        grid_out_file.seek(1024, os.SEEK_CUR)
        self.assertEqual(2048, grid_out_file.tell())
        grid_out_file.seek(0, os.SEEK_END)
        self.assertEqual(1048576, grid_out_file.tell())
        self.assertRaises(IOError, grid_out_file.seek, 0, 4)
        self.assertRaises(IOError, grid_out_file.seek, -1)
        self.assertTrue("'_id': 'test_id'" in repr(grid_out_file))
        self.assertTrue("20150101", grid_in_file.upload_date)
        yield grid_in_file.writelines([b"0xDEADBEEF", b"0xDEADBEAF"])
        yield grid_in_file.close()
        if twisted_version.major >= 15:
            with self.assertRaises(AttributeError):
                grid_in_file.length = 1
        self.assertEqual(1, grid_in_file.test)
        if twisted_version.major >= 15:
            with self.assertRaises(AttributeError):
                _ = grid_in_file.test_none
        self.assertTrue(grid_in_file.closed)
        if twisted_version.major >= 15:
            with self.assertRaises(ValueError):
                yield grid_in_file.write(b"0xDEADBEEF")
        yield conn.disconnect()

    @defer.inlineCallbacks
    def test_GridFsObjects(self):
        """ Tests gridfs objects """
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        db = conn.test
        yield self._drop_gridfs(db)
        gfs = GridFS(db)  # Default collection
        yield gfs.delete(u"test")

        _ = gfs.new_file(filename="test_1", contentType="text/plain", chunk_size=65536)
        yield conn.disconnect()
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        db = conn.test
        yield self._drop_gridfs(db)
        gfs = GridFS(db)  # Default collection
        _ = yield gfs.put(b"0xDEADBEEF", filename="test_2", contentType="text/plain",
                          chunk_size=65536)
        # disconnect
        yield conn.disconnect()

        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        db = conn.test
        gfs = GridFS(db)  # Default collection
        # Missing file raises error
        yield self.assertFailure(gfs.get("test_3"), NoFile)
        # disconnect
        yield conn.disconnect()

    @defer.inlineCallbacks
    def test_GridFsIteration(self):
        """ Tests gridfs iterator """
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        db = conn.test
        yield self._drop_gridfs(db)
        gfs = GridFS(db)  # Default collection
        new_file = gfs.new_file(filename="testName", contentType="text/plain", length=1048576,
                                chunk_size=4096)
        yield new_file.write(b"0xDEADBEEF"*4096*2)
        yield new_file.close()
        fake_doc = {"_id": new_file._id, "name": "testName", "length": 4096*2, "chunkSize": 4096,
                    "contentType": "text/plain"}
        grid_out_file = GridOut(db.fs, fake_doc)
        iterator = GridOutIterator(grid_out_file, db.fs.chunks)
        next_it = yield next(iterator)
        self.assertEqual(len(next_it), 4096)
        _ = yield next(iterator)
        next_it = yield next(iterator)
        self.assertEqual(next_it, None)

        fake_bad_doc = {"_id": "bad_id", "name": "testName", "length": 4096*2,
                        "chunkSize": 4096, "contentType": "text/plain"}
        grid_bad_out_file = GridOut(db.fs, fake_bad_doc)
        bad_iterator = GridOutIterator(grid_bad_out_file, db.fs.chunks)
        if twisted_version.major >= 15:
            with self.assertRaises(errors.CorruptGridFile):
                next_it = yield bad_iterator.next()

        # disconnect
        yield conn.disconnect()

    @defer.inlineCallbacks
    def test_GridFsOperations(self):
        """ Tests gridfs operations """
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        db = conn.test
        # Drop files first TODO: iterate through files and delete them
        yield self._drop_gridfs(db)

        # Don't forget to disconnect
        self.addCleanup(self._disconnect, conn)
        try:
            in_file = StringIO(b"Test input string")
            out_file = StringIO()
        except Exception as e:
            self.fail("Failed to create memory files for testing: %s" % e)

        g_out = None

        try:
            # Tests writing to a new gridfs file
            gfs = GridFS(db)  # Default collection
            if twisted_version.major >= 15:
                with self.assertRaises(NoFile):
                    yield gfs.get_last_version("optest")

            g_in = gfs.new_file(filename="optest", contentType="text/plain",
                                chunk_size=65536)  # non-default chunk size used
            # yielding to ensure writes complete before we close and close before we try to read
            yield g_in.write(in_file.read())
            yield g_in.close()

            # Tests reading from an existing gridfs file
            g_out = yield gfs.get_last_version("optest")
            data = yield g_out.read()
            out_file.write(data)
            _id = g_out._id
        except Exception as e:
            self.fail("Failed to communicate with the GridFS. " +
                      "Is MongoDB running? %s" % e)
        else:
            self.assertEqual(in_file.getvalue(), out_file.getvalue(),
                             "Could not read the value from writing an input")
        finally:
            in_file.close()
            out_file.close()
            if g_out:
                g_out.close()

        listed_files = yield gfs.list()
        self.assertEqual(["optest"], listed_files,
                         "`optest` is the only expected file and we received %s" % listed_files)

        yield gfs.delete(_id)

    @defer.inlineCallbacks
    def test_GridFsIndexesCreation(self):
        """ Tests gridfs indexes creation"""
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        db = conn.test
        yield self._drop_gridfs(db)

        # Create a new GridFS instance should trigger indexes creation
        gfs = GridFS(db)

        # Multiple calls should return multiple defer not to mix between them
        self.assertNotEqual(gfs.indexes_created(), gfs.indexes_created())

        yield gfs.indexes_created()

        yield conn.disconnect()
<|endoftext|>


<commit_before>from django.contrib.auth.signals import user_logged_in, user_logged_out
from django.dispatch import receiver

from hs_core.signals import pre_download_file, post_delete_resource, post_create_resource

from .models import Session
from .models import Variable
from .utils import get_client_ip, get_user_type, get_user_email_domain

@receiver(user_logged_in, dispatch_uid='id_capture_login')
def capture_login(sender, **kwargs):
    session = Session.objects.for_request(kwargs['request'], kwargs['user'])
    session.record('login')

@receiver(user_logged_out, dispatch_uid='id_capture_logout')
def capture_logout(sender, **kwargs):
    session = Session.objects.for_request(kwargs['request'], kwargs['user'])
    session.record('logout')

@receiver(pre_download_file)
def capture_download(**kwargs):

    # get input kwargs
    resource = kwargs['resource']
    filename = kwargs['download_file_name']

    # set session, user, and ip to None if a request object was not passed as an input kwarg.
    if 'request' not in kwargs.keys():
        session = None
        user = None
        ip = None
    else:
        request = kwargs['request']
        user = request.user
        session = Session.objects.for_request(request, user)
        ip = get_client_ip(request)

    # get the user info
    usertype = get_user_type(session)
    emaildomain = get_user_email_domain(session)

    # format the 'create' kwargs
    msg = Variable.format_kwargs(user_ip=ip,
                                 filename=filename,
                                 resource_size_bytes=resource.size,
                                 resource_type=resource.resource_type,
                                 resource_guid=resource.short_id,
                                 user_type=usertype,
                                 user_email_domain=emaildomain
                                 )

    # record the action
    session.record('download', value=msg)

@receiver(post_create_resource)
def capture_resource_create(**kwargs):

    # get input kwargs
    resource = kwargs['resource']
    user = kwargs['user']

    # set session, user, and ip to None if a request object was not passed as an input kwarg.
    if 'request' not in kwargs.keys():
        session = None
        ip = None
    else:
        request = kwargs['request']
        session = Session.objects.for_request(request, user)
        ip = get_client_ip(request)

    # get the user info
    usertype = get_user_type(session)
    emaildomain = get_user_email_domain(session)

    # format the 'download' kwargs
    msg = Variable.format_kwargs(user_ip=ip,
                                 resource_size_bytes=resource.size,
                                 resource_type=resource.resource_type,
                                 resource_guid=resource.short_id,
                                 user_type=usertype,
                                 user_email_domain=emaildomain
                                 )

    # record the action
    session.record('create', value=msg)

@receiver(post_delete_resource)
def capture_resource_delete(**kwargs):

    # get input kwargs
    resource_type = kwargs['resource_type']
    resource_shortid = kwargs['resource_shortkey']
    user = kwargs['user']

    # set session, user, and ip to None if a request object was not passed as an input kwarg.
    if 'request' not in kwargs.keys():
        session = None
        ip = None
    else:
        request = kwargs['request']
        session = Session.objects.for_request(request, user)
        ip = get_client_ip(request)

    # get the user info
    usertype = get_user_type(session)
    emaildomain = get_user_email_domain(session)

    # formate the 'delete' kwargs
    msg = Variable.format_kwargs(user_ip=ip,
                                 resource_type=resource_type,
                                 resource_guid=resource_shortid,
                                 user_type=usertype,
                                 user_email_domain=emaildomain
                                 )

    # record the action
    session.record('delete', value=msg)
<commit_msg>#1531 flake8 fixes<commit_after>from django.contrib.auth.signals import user_logged_in, user_logged_out
from django.dispatch import receiver

from hs_core.signals import pre_download_file, post_delete_resource, post_create_resource

from .models import Session
from .models import Variable
from .utils import get_client_ip, get_user_type, get_user_email_domain

@receiver(user_logged_in, dispatch_uid='id_capture_login')
def capture_login(sender, **kwargs):
    session = Session.objects.for_request(kwargs['request'], kwargs['user'])
    session.record('login')

@receiver(user_logged_out, dispatch_uid='id_capture_logout')
def capture_logout(sender, **kwargs):
    session = Session.objects.for_request(kwargs['request'], kwargs['user'])
    session.record('logout')

@receiver(pre_download_file)
def capture_download(**kwargs):

    # get input kwargs
    resource = kwargs['resource']
    filename = kwargs['download_file_name']

    # set session, user, and ip to None if a request object was not passed as an input kwarg.
    if 'request' not in kwargs.keys():
        session = None
        user = None
        ip = None
    else:
        request = kwargs['request']
        user = request.user
        session = Session.objects.for_request(request, user)
        ip = get_client_ip(request)

    # get the user info
    usertype = get_user_type(session)
    emaildomain = get_user_email_domain(session)

    # format the 'create' kwargs
    msg = Variable.format_kwargs(user_ip=ip,
                                 filename=filename,
                                 resource_size_bytes=resource.size,
                                 resource_type=resource.resource_type,
                                 resource_guid=resource.short_id,
                                 user_type=usertype,
                                 user_email_domain=emaildomain
                                 )

    # record the action
    session.record('download', value=msg)

@receiver(post_create_resource)
def capture_resource_create(**kwargs):

    # get input kwargs
    resource = kwargs['resource']
    user = kwargs['user']

    # set session, user, and ip to None if a request object was not passed as an input kwarg.
    if 'request' not in kwargs.keys():
        session = None
        ip = None
    else:
        request = kwargs['request']
        session = Session.objects.for_request(request, user)
        ip = get_client_ip(request)

    # get the user info
    usertype = get_user_type(session)
    emaildomain = get_user_email_domain(session)

    # format the 'download' kwargs
    msg = Variable.format_kwargs(user_ip=ip,
                                 resource_size_bytes=resource.size,
                                 resource_type=resource.resource_type,
                                 resource_guid=resource.short_id,
                                 user_type=usertype,
                                 user_email_domain=emaildomain
                                 )

    # record the action
    session.record('create', value=msg)

@receiver(post_delete_resource)
def capture_resource_delete(**kwargs):

    # get input kwargs
    resource_type = kwargs['resource_type']
    resource_shortid = kwargs['resource_shortkey']
    user = kwargs['user']

    # set session, user, and ip to None if a request object was not passed as an input kwarg.
    if 'request' not in kwargs.keys():
        session = None
        ip = None
    else:
        request = kwargs['request']
        session = Session.objects.for_request(request, user)
        ip = get_client_ip(request)

    # get the user info
    usertype = get_user_type(session)
    emaildomain = get_user_email_domain(session)

    # formate the 'delete' kwargs
    msg = Variable.format_kwargs(user_ip=ip,
                                 resource_type=resource_type,
                                 resource_guid=resource_shortid,
                                 user_type=usertype,
                                 user_email_domain=emaildomain
                                 )

    # record the action
    session.record('delete', value=msg)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

"""Contains test cases for the parsers module."""

from __future__ import unicode_literals

import sys
import os.path
import unittest

PATH = os.path.realpath(os.path.abspath(__file__))
sys.path.insert(0, os.path.dirname(os.path.dirname(PATH)))

try:
    from youtube_dl_gui.parsers import OptionsParser
except ImportError as error:
    print error
    sys.exit(1)

class TestParse(unittest.TestCase):

    """Test case for OptionsParser parse method."""

    def test_parse_to_audio_requirement_bug(self):
        """Test case for the 'to_audio' requirement."""

        options_dict = {  # Extracted from youtube-dlG settings.json
            'keep_video': False,
            'opts_win_size': (640, 490),
            'open_dl_dir': False,
            'second_video_format': '0',
            'native_hls': False,
            'write_subs': False,
            'workers_number': 3,
            'max_downloads': 0,
            'max_filesize': 0,
            'youtube_dl_debug': False,
            'shutdown': False,
            'selected_format': 'mp3',
            'write_all_subs': False,
            'enable_log': True,
            'embed_thumbnail': True,
            'audio_quality': '9',
            'subs_lang': 'en',
            'audio_format': 'mp3',
            'restrict_filenames': False,
            'min_filesize_unit': '',
            'selected_audio_formats': ['mp3', 'm4a', 'vorbis'],
            'selected_video_formats': ['webm', 'mp4'],
            'save_path': '/home/user/Workplace/test/youtube',
            'output_template': '%(uploader)s/%(title)s.%(ext)s',
            'show_completion_popup': True,
            'locale_name': 'en_US',
            'to_audio': False,
            'confirm_deletion': True,
            'min_filesize': 0,
            'save_path_dirs': ['/home/user/Downloads', '/home/user/Desktop', '/home/user/Videos', '/home/user/Music', '/home/user/Workplace/test/youtube'],
            'sudo_password': '',
            'video_password': '',
            'output_format': 1,
            'embed_subs': False,
            'write_auto_subs': False,
            'video_format': '0',
            'confirm_exit': False,
            'referer': '',
            'proxy': '',
            'add_metadata': False,
            'ignore_errors': False,
            'log_time': True,
            'password': '',
            'playlist_end': 0,
            'write_description': False,
            'retries': 10,
            'cmd_args': '',
            'write_thumbnail': False,
            'playlist_start': 1,
            'nomtime': False,
            'write_info': False,
            'username': '',
            'main_win_size': (930, 560),
            'user_agent': '',
            'max_filesize_unit': '',
            'ignore_config': False,
            'youtubedl_path': '/home/user/.config/youtube-dlg'
        }

        expected_cmd_list = ["--newline",
                             "-x",
                             "--audio-format",
                             "mp3",
                             "--embed-thumbnail",
                             "--audio-quality",
                             "9",
                             "-o",
                             "/home/user/Workplace/test/youtube/%(title)s.%(ext)s"]

        options_parser = OptionsParser()

        self.assertItemsEqual(options_parser.parse(options_dict), expected_cmd_list)

        # Setting 'to_audio' to True should return the same results
        # since the '-x' flag is already set on audio extraction
        options_dict["to_audio"] = True

        self.assertItemsEqual(options_parser.parse(options_dict), expected_cmd_list)

def main():
    unittest.main()

if __name__ == '__main__':
    main()
<commit_msg>Test the 'cmd_args' option parsing when quotes are presented<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

"""Contains test cases for the parsers module."""

from __future__ import unicode_literals

import sys
import os.path
import unittest

PATH = os.path.realpath(os.path.abspath(__file__))
sys.path.insert(0, os.path.dirname(os.path.dirname(PATH)))

try:
    from youtube_dl_gui.parsers import OptionsParser
except ImportError as error:
    print error
    sys.exit(1)

class TestParse(unittest.TestCase):

    """Test case for OptionsParser parse method."""

    def test_parse_to_audio_requirement_bug(self):
        """Test case for the 'to_audio' requirement."""

        options_dict = {  # Extracted from youtube-dlG settings.json
            'keep_video': False,
            'opts_win_size': (640, 490),
            'open_dl_dir': False,
            'second_video_format': '0',
            'native_hls': False,
            'write_subs': False,
            'workers_number': 3,
            'max_downloads': 0,
            'max_filesize': 0,
            'youtube_dl_debug': False,
            'shutdown': False,
            'selected_format': 'mp3',
            'write_all_subs': False,
            'enable_log': True,
            'embed_thumbnail': True,
            'audio_quality': '9',
            'subs_lang': 'en',
            'audio_format': 'mp3',
            'restrict_filenames': False,
            'min_filesize_unit': '',
            'selected_audio_formats': ['mp3', 'm4a', 'vorbis'],
            'selected_video_formats': ['webm', 'mp4'],
            'save_path': '/home/user/Workplace/test/youtube',
            'output_template': '%(uploader)s/%(title)s.%(ext)s',
            'show_completion_popup': True,
            'locale_name': 'en_US',
            'to_audio': False,
            'confirm_deletion': True,
            'min_filesize': 0,
            'save_path_dirs': ['/home/user/Downloads', '/home/user/Desktop', '/home/user/Videos', '/home/user/Music', '/home/user/Workplace/test/youtube'],
            'sudo_password': '',
            'video_password': '',
            'output_format': 1,
            'embed_subs': False,
            'write_auto_subs': False,
            'video_format': '0',
            'confirm_exit': False,
            'referer': '',
            'proxy': '',
            'add_metadata': False,
            'ignore_errors': False,
            'log_time': True,
            'password': '',
            'playlist_end': 0,
            'write_description': False,
            'retries': 10,
            'cmd_args': '',
            'write_thumbnail': False,
            'playlist_start': 1,
            'nomtime': False,
            'write_info': False,
            'username': '',
            'main_win_size': (930, 560),
            'user_agent': '',
            'max_filesize_unit': '',
            'ignore_config': False,
            'youtubedl_path': '/home/user/.config/youtube-dlg'
        }

        expected_cmd_list = ["--newline",
                             "-x",
                             "--audio-format",
                             "mp3",
                             "--embed-thumbnail",
                             "--audio-quality",
                             "9",
                             "-o",
                             "/home/user/Workplace/test/youtube/%(title)s.%(ext)s"]

        options_parser = OptionsParser()

        self.assertItemsEqual(options_parser.parse(options_dict), expected_cmd_list)

        # Setting 'to_audio' to True should return the same results
        # since the '-x' flag is already set on audio extraction
        options_dict["to_audio"] = True

        self.assertItemsEqual(options_parser.parse(options_dict), expected_cmd_list)

    def test_parse_cmd_args_with_quotes(self):
        """Test the youtube-dl cmd line args parsing when quotes are presented.

        See: https://github.com/MrS0m30n3/youtube-dl-gui/issues/54

        """

        options_dict = {
            "keep_video": False,
            "opts_win_size": (640, 490),
            "open_dl_dir": False,
            "second_video_format": "0",
            "native_hls": False,
            "write_subs": False,
            "sudo_password": "",
            "workers_number": 3,
            "max_filesize": 0,
            "youtube_dl_debug": False,
            "shutdown": False,
            "selected_format": "mp4",
            "write_all_subs": False,
            "enable_log": True,
            "embed_thumbnail": False,
            "audio_quality": "5",
            "main_win_size": (930, 560),
            "restrict_filenames": False,
            "min_filesize_unit": "",
            "video_password": "",
            "selected_audio_formats": [
                "mp3",
                "m4a",
                "vorbis"
            ],
            "selected_video_formats": [
                "webm",
                "mp4"
            ],
            "save_path": "/home/user/Workplace/test/youtube",
            "output_template": "%(uploader)s/%(title)s.%(ext)s",
            "show_completion_popup": True,
            "locale_name": "en_US",
            "to_audio": False,
            "confirm_deletion": True,
            "min_filesize": 0,
            "save_path_dirs": [
                "/home/user/Downloads",
                "/home/user/Desktop",
                "/home/user/Videos",
                "/home/user/Music",
                "/home/user/Workplace/test/youtube"
            ],
            "audio_format": "",
            "max_downloads": 0,
            "output_format": 1,
            "embed_subs": False,
            "write_auto_subs": False,
            "video_format": "mp4",
            "confirm_exit": False,
            "referer": "",
            "proxy": "",
            "add_metadata": False,
            "ignore_errors": False,
            "log_time": True,
            "password": "",
            "playlist_end": 0,
            "write_description": False,
            "retries": 10,
            "cmd_args": "--recode-video mkv --postprocessor-args \"-codec copy -report\"",
            "write_thumbnail": False,
            "playlist_start": 1,
            "nomtime": False,
            "write_info": False,
            "username": "",
            "subs_lang": "en",
            "user_agent": "",
            "max_filesize_unit": "",
            "ignore_config": False,
            "youtubedl_path": "/home/user/.config/youtube-dlg"
        }

        expected_cmd_list = ["--newline",
                             "-f",
                             "mp4",
                             "-o",
                             "/home/user/Workplace/test/youtube/%(title)s.%(ext)s",
                             "--recode-video",
                             "mkv",
                             "--postprocessor-args",
                             "-codec copy -report"]

        options_parser = OptionsParser()

        self.assertItemsEqual(options_parser.parse(options_dict), expected_cmd_list)

        # Test the example presented in issue #54
        options_dict["cmd_args"] = "-f \"(mp4)[width<1300]\""
        options_dict["video_format"] = "0"  # Set video format to 'default'

        expected_cmd_list = ["--newline",
                             "-o",
                             "/home/user/Workplace/test/youtube/%(title)s.%(ext)s",
                             "-f",
                             "(mp4)[width<1300]"]

        self.assertItemsEqual(options_parser.parse(options_dict), expected_cmd_list)

        # Test with two quoted 'cmd_args'
        options_dict["cmd_args"] = "--postprocessor-args \"-y -report\""
        options_dict["video_format"] = "mp4"  # Set video format back to 'mp4'

        expected_cmd_list = ["--newline",
                             "-f",
                             "mp4",
                             "-o",
                             "/home/user/Workplace/test/youtube/%(title)s.%(ext)s",
                             "--postprocessor-args",
                             "-y -report"]

        self.assertItemsEqual(options_parser.parse(options_dict), expected_cmd_list)

        # Test with single quoted 'cmd_arg' followed by other cmd line args
        options_dict["cmd_args"] = "--postprocessor-args \"-y\" -v"

        expected_cmd_list = ["--newline",
                             "-f",
                             "mp4",
                             "-o",
                             "/home/user/Workplace/test/youtube/%(title)s.%(ext)s",
                             "--postprocessor-args",
                             "-y",
                             "-v"]

        self.assertItemsEqual(options_parser.parse(options_dict), expected_cmd_list)

def main():
    unittest.main()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import pytest
import os
import numpy as np

import pypiv.piv.process as process

@pytest.fixture(scope='module')

def data():
    script_path = os.path.dirname(os.path.realpath(__file__))
    file_path   = os.path.join(script_path, 'data/correlation.npy')
    window = np.load(file_path)
    return window

def test_1D_elliptical_gauss(data):
    xo, yo = process.find_subpixel_peak(data,subpixel_method='gaussian')
    assert abs(xo-31.5) < 0.01
    assert abs(yo-31.5) < 0.01

def test_2D_elliptical_gauss(data):
    xo, yo = process.find_subpixel_peak_2D(data)
    assert abs(yo-31.5) < 0.01
    assert abs(xo-31.5) < 0.01

<commit_msg>added test for scipy interative 2D gaussian peakfinder<commit_after>import pytest
import os
import numpy as np

import pypiv.piv.process as process

@pytest.fixture(scope='module')
def data():
    script_path = os.path.dirname(os.path.realpath(__file__))
    file_path   = os.path.join(script_path, 'data/correlation.npy')
    window = np.load(file_path)
    return window

def test_1D_elliptical_gauss(data):
    xo, yo = process.find_subpixel_peak(data,subpixel_method='gaussian')
    assert abs(xo-31.5) < 0.01
    assert abs(yo-31.5) < 0.01

def test_2D_elliptical_gauss_9point(data):
    xo, yo = process.find_subpixel_peak_2D(data,subpixel_method='9point')
    assert abs(yo-31.5) < 0.01
    assert abs(xo-31.5) < 0.01

def test_2D_elliptical_gauss_scipy(data):
    xo, yo = process.find_subpixel_peak_2D(data,subpixel_method='scipy')
    assert abs(yo-31.5) < 0.01
    assert abs(xo-31.5) < 0.01

<|endoftext|>


<commit_before># coding: utf-8
# Copyright 2010 Mark L.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from bson import BSON, ObjectId
from twisted.internet import defer
from twisted.trial import unittest
import txmongo
from txmongo.protocol import MongoClientProtocol

mongo_host = "localhost"
mongo_port = 27017

class TestMongoQueries(unittest.TestCase):

    timeout = 5

    @defer.inlineCallbacks
    def setUp(self):
        self.conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        self.db = self.conn.mydb
        self.coll = self.db.mycol

    @defer.inlineCallbacks
    def test_SingleCursorIteration(self):
        yield self.coll.insert([{'v': i} for i in range(10)], safe=True)
        res = yield self.coll.find()
        self.assertEqual(len(res), 10)

    @defer.inlineCallbacks
    def test_MultipleCursorIterations(self):
        yield self.coll.insert([{'v': i} for i in range(450)], safe=True)
        res = yield self.coll.find()
        self.assertEqual(len(res), 450)

    @defer.inlineCallbacks
    def test_FindWithCursor(self):
        yield self.coll.insert([{'v': i} for i in range(750)], safe=True)
        docs, d = yield self.coll.find_with_cursor()
        self.assertEqual(len(docs), 101)
        total = 0
        while docs:
            total += len(docs)
            docs, d = yield d
        self.assertEqual(total, 750)

        # Same thing, but with the "cursor" keyword argument on find()
        docs, d = yield self.coll.find(cursor=True)
        self.assertEqual(len(docs), 101)
        total = 0
        while docs:
            total += len(docs)
            docs, d = yield d
        self.assertEqual(total, 750)

    @defer.inlineCallbacks
    def test_LargeData(self):
        yield self.coll.insert([{'v': ' '*(2**19)} for _ in range(4)], safe=True)
        res = yield self.coll.find()
        self.assertEqual(len(res), 4)

    @defer.inlineCallbacks
    def test_SpecifiedFields(self):
        yield self.coll.insert([{k: v for k in "abcdefg"} for v in range(5)], safe=True)
        res = yield self.coll.find(fields={'a': 1, 'c': 1})
        yield self.coll.count(fields={'a': 1, 'c': 1})
        self.assertEqual(res[0].keys(), ['a', 'c', "_id"])
        res = yield self.coll.find(fields=['a', 'c'])
        yield self.coll.count(fields=['a', 'c'])
        self.assertEqual(res[0].keys(), ['a', 'c', "_id"])
        res = yield self.coll.find(fields=[])
        yield self.coll.count(fields=[])
        self.assertEqual(res[0].keys(), ["_id"])
        self.assertRaises(TypeError, self.coll._fields_list_to_dict, [1])

    @defer.inlineCallbacks
    def test_group(self):
        yield self.coll.insert([{'v': i % 2} for i in range(5)], safe=True)
        reduce_ = """
        function(curr, result) {
            result.total += curr.v;
        }
        """
        keys = {'v': 1}
        initial = {"total": 0}
        cond = {'v': {"$in": [0, 1]}}
        final = """
        function(result) {
            result.five = 5;
        }
        """
        res = yield self.coll.group(keys, initial, reduce_, cond, final)
        self.assertEqual(len(res["retval"]), 2)

        keys = """
        function(doc) {
            return {"value": 5, 'v': 1};
        }
        """

        res = yield self.coll.group(keys, initial, reduce_, cond, final)
        self.assertEqual(len(res["retval"]), 1)

    @defer.inlineCallbacks
    def test_CursorClosing(self):
        def make_object():
            return {"_id": ObjectId(), 'x': 'a' * 1000}

        # Calculate number of objects in 4mb batch
        obj_count_4mb = 4 * 1024**2 / len(BSON.encode(make_object())) + 1

        first_batch = 5
        yield self.coll.insert([make_object() for _ in range(first_batch + obj_count_4mb)])
        _ = yield self.coll.find(limit=first_batch)

        status = yield self.db["$cmd"].find_one({"serverStatus": 1})
        self.assertEqual(status["metrics"]["cursor"]["open"]["total"], 0)

    @defer.inlineCallbacks
    def test_GetMoreCount(self):
        class CallCounter(object):
            def __init__(self, original):
                self.call_count = 0
                self.original = original

            def __call__(self, this, *args, **kwargs):
                self.call_count += 1
                return self.original(this, *args, **kwargs)

        counter = CallCounter(MongoClientProtocol.send_GETMORE)
        MongoClientProtocol.send_GETMORE = counter

        try:
            yield self.coll.insert([{'x': 42} for _ in range(20)])
            yield self.coll.find({}, limit=10)

            self.assertEqual(counter.call_count, 0)
        finally:
            MongoClientProtocol.send_GETMORE = counter.original

    @defer.inlineCallbacks
    def tearDown(self):
        yield self.coll.drop()
        yield self.conn.disconnect()

class TestMongoQueriesEdgeCases(unittest.TestCase):

    timeout = 5

    @defer.inlineCallbacks
    def setUp(self):
        self.conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        self.coll = self.conn.mydb.mycol

    @defer.inlineCallbacks
    def test_BelowBatchThreshold(self):
        yield self.coll.insert([{'v': i} for i in range(100)], safe=True)
        res = yield self.coll.find()
        self.assertEqual(len(res), 100)

    @defer.inlineCallbacks
    def test_EqualToBatchThreshold(self):
        yield self.coll.insert([{'v': i} for i in range(101)], safe=True)
        res = yield self.coll.find()
        self.assertEqual(len(res), 101)

    @defer.inlineCallbacks
    def test_AboveBatchThreshold(self):
        yield self.coll.insert([{'v': i} for i in range(102)], safe=True)
        res = yield self.coll.find()
        self.assertEqual(len(res), 102)

    @defer.inlineCallbacks
    def tearDown(self):
        yield self.coll.drop()
        yield self.conn.disconnect()

class TestLimit(unittest.TestCase):

    timeout = 5

    @defer.inlineCallbacks
    def setUp(self):
        self.conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        self.coll = self.conn.mydb.mycol

    @defer.inlineCallbacks
    def test_LimitBelowBatchThreshold(self):
        yield self.coll.insert([{'v': i} for i in range(50)], safe=True)
        res = yield self.coll.find(limit=20)
        self.assertEqual(len(res), 20)

    @defer.inlineCallbacks
    def test_LimitAboveBatchThreshold(self):
        yield self.coll.insert([{'v': i} for i in range(200)], safe=True)
        res = yield self.coll.find(limit=150)
        self.assertEqual(len(res), 150)

    @defer.inlineCallbacks
    def test_LimitAtBatchThresholdEdge(self):
        yield self.coll.insert([{'v': i} for i in range(200)], safe=True)
        res = yield self.coll.find(limit=100)
        self.assertEqual(len(res), 100)

        yield self.coll.drop(safe=True)

        yield self.coll.insert([{'v': i} for i in range(200)], safe=True)
        res = yield self.coll.find(limit=101)
        self.assertEqual(len(res), 101)

        yield self.coll.drop(safe=True)

        yield self.coll.insert([{'v': i} for i in range(200)], safe=True)
        res = yield self.coll.find(limit=102)
        self.assertEqual(len(res), 102)

    @defer.inlineCallbacks
    def test_LimitAboveMessageSizeThreshold(self):
        yield self.coll.insert([{'v': ' '*(2**20)} for _ in range(8)], safe=True)
        res = yield self.coll.find(limit=5)
        self.assertEqual(len(res), 5)

    @defer.inlineCallbacks
    def test_HardLimit(self):
        yield self.coll.insert([{'v': i} for i in range(200)], safe=True)
        res = yield self.coll.find(limit=-150)
        self.assertEqual(len(res), 150)

    @defer.inlineCallbacks
    def test_HardLimitAboveMessageSizeThreshold(self):
        yield self.coll.insert([{'v': ' '*(2**20)} for _ in range(8)], safe=True)
        res = yield self.coll.find(limit=-6)
        self.assertEqual(len(res), 4)

    @defer.inlineCallbacks
    def tearDown(self):
        yield self.coll.drop(safe=True)
        yield self.conn.disconnect()

class TestSkip(unittest.TestCase):

    timeout = 5

    @defer.inlineCallbacks
    def setUp(self):
        self.conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        self.coll = self.conn.mydb.mycol

    @defer.inlineCallbacks
    def test_Skip(self):
        yield self.coll.insert([{'v': i} for i in range(5)], safe=True)
        res = yield self.coll.find(skip=3)
        self.assertEqual(len(res), 2)

        yield self.coll.drop(safe=True)

        yield self.coll.insert([{'v': i} for i in range(5)], safe=True)
        res = yield self.coll.find(skip=5)
        self.assertEqual(len(res), 0)

        yield self.coll.drop(safe=True)

        yield self.coll.insert([{'v': i} for i in range(5)], safe=True)
        res = yield self.coll.find(skip=6)
        self.assertEqual(len(res), 0)

    @defer.inlineCallbacks
    def test_SkipWithLimit(self):
        yield self.coll.insert([{'v': i} for i in range(5)], safe=True)
        res = yield self.coll.find(skip=3, limit=1)
        self.assertEqual(len(res), 1)

        yield self.coll.drop(safe=True)

        yield self.coll.insert([{'v': i} for i in range(5)], safe=True)
        res = yield self.coll.find(skip=4, limit=2)
        self.assertEqual(len(res), 1)

        yield self.coll.drop(safe=True)

        yield self.coll.insert([{'v': i} for i in range(5)], safe=True)
        res = yield self.coll.find(skip=4, limit=1)
        self.assertEqual(len(res), 1)

        yield self.coll.drop(safe=True)

        yield self.coll.insert([{'v': i} for i in range(5)], safe=True)
        res = yield self.coll.find(skip=5, limit=1)
        self.assertEqual(len(res), 0)

    @defer.inlineCallbacks
    def tearDown(self):
        yield self.coll.drop(safe=True)
        yield self.conn.disconnect()
<commit_msg>one last fix<commit_after># coding: utf-8
# Copyright 2010 Mark L.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from bson import BSON, ObjectId
from twisted.internet import defer
from twisted.trial import unittest
import txmongo
from txmongo.protocol import MongoClientProtocol

mongo_host = "localhost"
mongo_port = 27017

class TestMongoQueries(unittest.TestCase):

    timeout = 5

    @defer.inlineCallbacks
    def setUp(self):
        self.conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        self.db = self.conn.mydb
        self.coll = self.db.mycol

    @defer.inlineCallbacks
    def test_SingleCursorIteration(self):
        yield self.coll.insert([{'v': i} for i in range(10)], safe=True)
        res = yield self.coll.find()
        self.assertEqual(len(res), 10)

    @defer.inlineCallbacks
    def test_MultipleCursorIterations(self):
        yield self.coll.insert([{'v': i} for i in range(450)], safe=True)
        res = yield self.coll.find()
        self.assertEqual(len(res), 450)

    @defer.inlineCallbacks
    def test_FindWithCursor(self):
        yield self.coll.insert([{'v': i} for i in range(750)], safe=True)
        docs, d = yield self.coll.find_with_cursor()
        self.assertEqual(len(docs), 101)
        total = 0
        while docs:
            total += len(docs)
            docs, d = yield d
        self.assertEqual(total, 750)

        # Same thing, but with the "cursor" keyword argument on find()
        docs, d = yield self.coll.find(cursor=True)
        self.assertEqual(len(docs), 101)
        total = 0
        while docs:
            total += len(docs)
            docs, d = yield d
        self.assertEqual(total, 750)

    @defer.inlineCallbacks
    def test_LargeData(self):
        yield self.coll.insert([{'v': ' '*(2**19)} for _ in range(4)], safe=True)
        res = yield self.coll.find()
        self.assertEqual(len(res), 4)

    @defer.inlineCallbacks
    def test_SpecifiedFields(self):
        yield self.coll.insert([{k: v for k in "abcdefg"} for v in range(5)], safe=True)
        res = yield self.coll.find(fields={'a': 1, 'c': 1})
        yield self.coll.count(fields={'a': 1, 'c': 1})
        self.assertEqual(res[0].keys(), ['a', 'c', "_id"])
        res = yield self.coll.find(fields=['a', 'c'])
        yield self.coll.count(fields=['a', 'c'])
        self.assertEqual(res[0].keys(), ['a', 'c', "_id"])
        res = yield self.coll.find(fields=[])
        yield self.coll.count(fields=[])
        self.assertEqual(res[0].keys(), ["_id"])
        self.assertRaises(TypeError, self.coll._fields_list_to_dict, [1])

    @defer.inlineCallbacks
    def test_group(self):
        yield self.coll.insert([{'v': i % 2} for i in range(5)], safe=True)
        reduce_ = """
        function(curr, result) {
            result.total += curr.v;
        }
        """
        keys = {'v': 1}
        initial = {"total": 0}
        cond = {'v': {"$in": [0, 1]}}
        final = """
        function(result) {
            result.five = 5;
        }
        """
        res = yield self.coll.group(keys, initial, reduce_, cond, final)
        self.assertEqual(len(res["retval"]), 2)

        keys = """
        function(doc) {
            return {"value": 5, 'v': 1};
        }
        """

        res = yield self.coll.group(keys, initial, reduce_, cond, final)
        self.assertEqual(len(res["retval"]), 1)

    @defer.inlineCallbacks
    def test_CursorClosing(self):
        def make_object():
            return {"_id": ObjectId(), 'x': 'a' * 1000}

        # Calculate number of objects in 4mb batch
        obj_count_4mb = 4 * 1024**2 / len(BSON.encode(make_object())) + 1

        first_batch = 5
        yield self.coll.insert([make_object() for _ in range(first_batch + obj_count_4mb)])
        yield self.coll.find(limit=first_batch)

        status = yield self.db["$cmd"].find_one({"serverStatus": 1})
        self.assertEqual(status["metrics"]["cursor"]["open"]["total"], 0)

    @defer.inlineCallbacks
    def test_GetMoreCount(self):
        class CallCounter(object):
            def __init__(self, original):
                self.call_count = 0
                self.original = original

            def __call__(self, this, *args, **kwargs):
                self.call_count += 1
                return self.original(this, *args, **kwargs)

        counter = CallCounter(MongoClientProtocol.send_GETMORE)
        MongoClientProtocol.send_GETMORE = counter

        try:
            yield self.coll.insert([{'x': 42} for _ in range(20)])
            yield self.coll.find({}, limit=10)

            self.assertEqual(counter.call_count, 0)
        finally:
            MongoClientProtocol.send_GETMORE = counter.original

    @defer.inlineCallbacks
    def tearDown(self):
        yield self.coll.drop()
        yield self.conn.disconnect()

class TestMongoQueriesEdgeCases(unittest.TestCase):

    timeout = 5

    @defer.inlineCallbacks
    def setUp(self):
        self.conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        self.coll = self.conn.mydb.mycol

    @defer.inlineCallbacks
    def test_BelowBatchThreshold(self):
        yield self.coll.insert([{'v': i} for i in range(100)], safe=True)
        res = yield self.coll.find()
        self.assertEqual(len(res), 100)

    @defer.inlineCallbacks
    def test_EqualToBatchThreshold(self):
        yield self.coll.insert([{'v': i} for i in range(101)], safe=True)
        res = yield self.coll.find()
        self.assertEqual(len(res), 101)

    @defer.inlineCallbacks
    def test_AboveBatchThreshold(self):
        yield self.coll.insert([{'v': i} for i in range(102)], safe=True)
        res = yield self.coll.find()
        self.assertEqual(len(res), 102)

    @defer.inlineCallbacks
    def tearDown(self):
        yield self.coll.drop()
        yield self.conn.disconnect()

class TestLimit(unittest.TestCase):

    timeout = 5

    @defer.inlineCallbacks
    def setUp(self):
        self.conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        self.coll = self.conn.mydb.mycol

    @defer.inlineCallbacks
    def test_LimitBelowBatchThreshold(self):
        yield self.coll.insert([{'v': i} for i in range(50)], safe=True)
        res = yield self.coll.find(limit=20)
        self.assertEqual(len(res), 20)

    @defer.inlineCallbacks
    def test_LimitAboveBatchThreshold(self):
        yield self.coll.insert([{'v': i} for i in range(200)], safe=True)
        res = yield self.coll.find(limit=150)
        self.assertEqual(len(res), 150)

    @defer.inlineCallbacks
    def test_LimitAtBatchThresholdEdge(self):
        yield self.coll.insert([{'v': i} for i in range(200)], safe=True)
        res = yield self.coll.find(limit=100)
        self.assertEqual(len(res), 100)

        yield self.coll.drop(safe=True)

        yield self.coll.insert([{'v': i} for i in range(200)], safe=True)
        res = yield self.coll.find(limit=101)
        self.assertEqual(len(res), 101)

        yield self.coll.drop(safe=True)

        yield self.coll.insert([{'v': i} for i in range(200)], safe=True)
        res = yield self.coll.find(limit=102)
        self.assertEqual(len(res), 102)

    @defer.inlineCallbacks
    def test_LimitAboveMessageSizeThreshold(self):
        yield self.coll.insert([{'v': ' '*(2**20)} for _ in range(8)], safe=True)
        res = yield self.coll.find(limit=5)
        self.assertEqual(len(res), 5)

    @defer.inlineCallbacks
    def test_HardLimit(self):
        yield self.coll.insert([{'v': i} for i in range(200)], safe=True)
        res = yield self.coll.find(limit=-150)
        self.assertEqual(len(res), 150)

    @defer.inlineCallbacks
    def test_HardLimitAboveMessageSizeThreshold(self):
        yield self.coll.insert([{'v': ' '*(2**20)} for _ in range(8)], safe=True)
        res = yield self.coll.find(limit=-6)
        self.assertEqual(len(res), 4)

    @defer.inlineCallbacks
    def tearDown(self):
        yield self.coll.drop(safe=True)
        yield self.conn.disconnect()

class TestSkip(unittest.TestCase):

    timeout = 5

    @defer.inlineCallbacks
    def setUp(self):
        self.conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        self.coll = self.conn.mydb.mycol

    @defer.inlineCallbacks
    def test_Skip(self):
        yield self.coll.insert([{'v': i} for i in range(5)], safe=True)
        res = yield self.coll.find(skip=3)
        self.assertEqual(len(res), 2)

        yield self.coll.drop(safe=True)

        yield self.coll.insert([{'v': i} for i in range(5)], safe=True)
        res = yield self.coll.find(skip=5)
        self.assertEqual(len(res), 0)

        yield self.coll.drop(safe=True)

        yield self.coll.insert([{'v': i} for i in range(5)], safe=True)
        res = yield self.coll.find(skip=6)
        self.assertEqual(len(res), 0)

    @defer.inlineCallbacks
    def test_SkipWithLimit(self):
        yield self.coll.insert([{'v': i} for i in range(5)], safe=True)
        res = yield self.coll.find(skip=3, limit=1)
        self.assertEqual(len(res), 1)

        yield self.coll.drop(safe=True)

        yield self.coll.insert([{'v': i} for i in range(5)], safe=True)
        res = yield self.coll.find(skip=4, limit=2)
        self.assertEqual(len(res), 1)

        yield self.coll.drop(safe=True)

        yield self.coll.insert([{'v': i} for i in range(5)], safe=True)
        res = yield self.coll.find(skip=4, limit=1)
        self.assertEqual(len(res), 1)

        yield self.coll.drop(safe=True)

        yield self.coll.insert([{'v': i} for i in range(5)], safe=True)
        res = yield self.coll.find(skip=5, limit=1)
        self.assertEqual(len(res), 0)

    @defer.inlineCallbacks
    def tearDown(self):
        yield self.coll.drop(safe=True)
        yield self.conn.disconnect()
<|endoftext|>


<commit_before>import pytest
from tests import TestCase, uses_native_versioning

@pytest.mark.skipif('not uses_native_versioning()')
class TestRawSQL(TestCase):
    def test_single_statement(self):
        self.session.execute(
            "INSERT INTO article (name) VALUES ('some article')"
        )
        assert self.session.execute(
            "SELECT COUNT(1) FROM transaction"
        ).scalar() == 1

    def test_multiple_statements(self):
        self.session.execute(
            "INSERT INTO article (name) VALUES ('some article')"
        )
        self.session.execute(
            "INSERT INTO article (name) VALUES ('some article')"
        )
        assert self.session.execute(
            "SELECT COUNT(1) FROM transaction"
        ).scalar() == 1

    def test_raw_statement_with_objects(self):
        self.session.execute(
            "INSERT INTO article (name) VALUES ('some article')"
        )
        self.session.add(self.Article(name=u'some other article'))
        self.session.commit()
<commit_msg>Add more tests<commit_after>import pytest
from tests import TestCase, uses_native_versioning

@pytest.mark.skipif('not uses_native_versioning()')
class TestRawSQL(TestCase):
    def test_single_statement(self):
        self.session.execute(
            "INSERT INTO article (name) VALUES ('some article')"
        )
        assert self.session.execute(
            "SELECT COUNT(1) FROM transaction"
        ).scalar() == 1

    def test_multiple_statements(self):
        self.session.execute(
            "INSERT INTO article (name) VALUES ('some article')"
        )
        self.session.execute(
            "INSERT INTO article (name) VALUES ('some article')"
        )
        assert self.session.execute(
            "SELECT COUNT(1) FROM transaction"
        ).scalar() == 1

    def test_flush_after_raw_insert(self):
        self.session.execute(
            "INSERT INTO article (name) VALUES ('some article')"
        )
        self.session.add(self.Article(name=u'some other article'))
        self.session.commit()
        assert self.session.execute(
            "SELECT COUNT(1) FROM transaction"
        ).scalar() == 1

    def test_raw_insert_after_flush(self):
        self.session.add(self.Article(name=u'some other article'))
        self.session.flush()
        self.session.execute(
            "INSERT INTO article (name) VALUES ('some article')"
        )
        self.session.commit()
        assert self.session.execute(
            "SELECT COUNT(1) FROM transaction"
        ).scalar() == 1
<|endoftext|>


<commit_before>from swimlane.core.auth import Client
from swimlane.records import add_references

def test_add_references():
    Client.set_default("https://swimlane.com", "usr", "pwd", False)

    print add_references(
        record_id="5670dcec0e23ab0e4c363e12",
        keywords="chisrv7008",
        app_acronym="SOC",
        remote_app_acronym="CMDB",
        field_name="CMDB"
    )
<commit_msg>Update test_records.py<commit_after>from swimlane.core.auth import Client
from swimlane.records import add_references

def test_add_references():
    Client.set_default(server", "usr", "pwd", False)

    print add_references(
        record_id="5670dcec0e23ab0e4c363e12",
        keywords="chisrv7008",
        app_acronym="SOC",
        remote_app_acronym="CMDB",
        field_name="CMDB"
    )
<|endoftext|>


<commit_before>import json
import os

from nose.tools import assert_raises, assert_equal, assert_true
from craft_ai import reduce_decision_rules, errors

HERE = os.path.abspath(os.path.dirname(__file__))

# Assuming we are the test folder and the folder hierarchy is correctly
# constructed
EXPECTATIONS_DIR = os.path.join(HERE, "data", "interpreter", "reduce_decision_rules")

def reduce_decision_rules_tests_generator():
    expectations_files = os.listdir(EXPECTATIONS_DIR)
    for expectations_file in expectations_files:
        if os.path.splitext(expectations_file)[1] == ".json":
            with open(os.path.join(EXPECTATIONS_DIR, expectations_file)) as f:
                expectations = json.load(f)

            for expectation in expectations:
                assert_true(
                    "title" in expectation,
                    "Invalid expectation from '{}': missing \"title\".".format(
                        expectations_file
                    ),
                )
                assert_true(
                    "rules" in expectation and "expectation" in expectation,
                    'Invalid expectation from \'{}\': missing "rules" or "expectation".'.format(
                        expectations_file
                    ),
                )

                def test_fn(r, e):
                    return check_expectation(r, e)

                test_fn.description = expectation["title"]
                reduce_decision_rules_tests_generator.compat_func_name = (
                    test_fn.description
                )

                yield test_fn, expectation["rules"], expectation["expectation"]

def check_expectation(rules, expectation):

    if "error" in expectation:
        assert_raises(errors.CraftAiError, reduce_decision_rules, rules)
    else:
        assert_equal(reduce_decision_rules(rules), expectation["rules"])
<commit_msg>reducer moved to unittest and subtest<commit_after>import unittest

import json
import os

from craft_ai import reduce_decision_rules, errors

HERE = os.path.abspath(os.path.dirname(__file__))

# Assuming we are the test folder and the folder hierarchy is correctly
# constructed
EXPECTATIONS_DIR = os.path.join(HERE, "data", "interpreter", "reduce_decision_rules")

class TestReducer(unittest.TestCase):

    def test_reduce_decision_rules_tests_generator(self):
        expectations_files = os.listdir(EXPECTATIONS_DIR)
        for expectations_file in expectations_files:
            if os.path.splitext(expectations_file)[1] == ".json":
                with open(os.path.join(EXPECTATIONS_DIR, expectations_file)) as f:
                    expectations = json.load(f)

                for expectation in expectations:
                    self.assertTrue(
                        "title" in expectation,
                        "Invalid expectation from '{}': missing \"title\".".format(
                            expectations_file
                        ),
                    )
                    self.assertTrue(
                        "rules" in expectation and "expectation" in expectation,
                        'Invalid expectation from \'{}\': missing "rules" or "expectation".'.format(
                            expectations_file
                        ),
                    )

                    for expectation in expectations:
                        with self.subTest():
                            self.check_expectation(expectation["rules"], expectation["expectation"])

    def check_expectation(self, rules, expectation):

        if "error" in expectation:
            self.assertRaises(errors.CraftAiError, reduce_decision_rules, rules)
        else:
            self.assertEqual(reduce_decision_rules(rules), expectation["rules"])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals, print_function
from datetime import datetime
import os
from os.path import dirname, join
import sys
import time
import unittest
import uuid

import logging
logging.basicConfig(level=logging.WARNING)

import six
import django
import pytest
from requests.exceptions import ConnectionError

from qiniu import set_default, BucketManager
from qiniu.config import Zone

zone_overseas = Zone('up.qiniug.com', 'upload.qiniug.com')

from .utils import retry

# Add repo/demo_site to sys.path
DEMO_SITE_DIR = join(dirname(dirname(__file__)), 'demo_site')
sys.path.append(DEMO_SITE_DIR)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "demo_site.settings")

try:
    django.setup()
except AttributeError:
   # Setup isn't necessary in Django < 1.7
   pass

from django.conf import settings
from qiniustorage.backends import QiniuStorage, QiniuFile
from qiniustorage.utils import QiniuError

USING_TRAVIS = os.environ.get('USING_TRAVIS', None) is None
if USING_TRAVIS:
    set_default(default_zone=zone_overseas, connection_retries=20, connection_timeout=100)

UNIQUE_PATH = str(uuid.uuid4())

class QiniuStorageTest(unittest.TestCase):
    def setUp(self):
        self.storage = QiniuStorage()

    def test_file_init(self):
        fil = QiniuFile('foo', self.storage, mode='rb')
        assert fil._mode == 'rb'
        assert fil._name == "foo"

    def test_write_to_read_only_file(self):
        with pytest.raises(AttributeError):
            fil = QiniuFile('foo', self.storage, mode='rb')
            fil.write("goto fail")

    def test_write_and_delete_file(self):
        ASSET_FILE_NAMES = [u'jquery-2.1.1.min.js', u'jquery-2点壹点一.min.js']
        for assert_file_name in ASSET_FILE_NAMES:
            REMOTE_PATH = join(UNIQUE_PATH, assert_file_name)
            assert self.storage.exists(REMOTE_PATH) == False
            fil = QiniuFile(REMOTE_PATH, self.storage, mode='wb')

            with open(join(dirname(__file__),'assets', assert_file_name), 'rb') as assset_file:
                content = assset_file.read()

                assset_file.seek(0, os.SEEK_END)
                assset_file_size = assset_file.tell()

                fil.write(content)
                self.storage._save(REMOTE_PATH, fil)

                assert self.storage.exists(REMOTE_PATH)

            assert self.storage.size(REMOTE_PATH) == assset_file_size

            now = datetime.utcnow()
            modified_time = self.storage.modified_time(REMOTE_PATH)
            # Datetime on Qiniu servers may be faster or slower than the local
            # machine. Thus the absolute delta within 60s should be considered
            # acceptable.
            time_delta = max(now, modified_time) - min(now, modified_time)
            assert time_delta.seconds < 180

            self.storage.delete(REMOTE_PATH)
            assert self.storage.exists(REMOTE_PATH) == False

    def test_read_file(self):
        ASSET_FILE_NAMES =  [u'jquery-1.11.1.min.js', u'jquery-一点十一点壹.js']
        for assert_file_name in ASSET_FILE_NAMES:
            REMOTE_PATH = join(UNIQUE_PATH, assert_file_name)

            with open(join(dirname(__file__),'assets', assert_file_name), 'rb') as assset_file:
                self.storage.save(REMOTE_PATH, assset_file)

            fil = self.storage.open(REMOTE_PATH, 'r')

            assert fil._is_read == False

            content = fil.read()
            assert content.startswith(u"/*!")

            assert fil._is_read == True

            # Test open mode
            fil = self.storage.open(REMOTE_PATH, 'rb')
            bin_content = fil.read()
            assert bin_content.startswith(b"/*!")

    def test_dirty_file(self):
        ASSET_FILE_NAME = 'bootstrap.min.css'
        REMOTE_PATH = join(UNIQUE_PATH, ASSET_FILE_NAME)

        fil = self.storage.open(REMOTE_PATH, 'rw')

        assert fil._is_read == False
        assert fil._is_dirty == False
        assert self.storage.exists(REMOTE_PATH) == False

        with open(join(dirname(__file__),'assets', ASSET_FILE_NAME), 'r') as assset_file:
            content = assset_file.read()
            fil.write(content)

        assert fil._is_read == True
        assert fil._is_dirty == True

        fil.close()
        assert self.storage.exists(REMOTE_PATH) == True

    @retry(QiniuError, tries=10, backoff=1)
    def test_listdir(self):
        raise QiniuError('retry test')
        dirnames = ['', 'foo', 'bar']
        filenames = ['file1', 'file2', 'file3', u'file四']
        for dirname in dirnames:
            for filename in filenames:
                fil = self.storage.open(join(UNIQUE_PATH, dirname, filename), 'w')
                fil.write('test text')
                fil.close()
        time.sleep(3)

        dirs, files = self.storage.listdir(UNIQUE_PATH)
        assert sorted(dirs) == sorted(['foo', 'bar'])

        dirs, files = self.storage.listdir(join(UNIQUE_PATH, 'foo'))
        assert dirs == []
        assert sorted(files) == sorted(filenames)

    @classmethod
    def teardown_class(cls):
        """Delete all files in the test bucket.
        """
        storage = QiniuStorage()
        auth = storage.auth
        bucket = BucketManager(auth)

        while True:
            ret, eof, info = bucket.list(storage.bucket_name, limit=100)

            if ret is None:
                print(info)
                break

            for item in ret['items']:
                name = item['key']
                if six.PY2:
                    name = name.encode('utf-8')
                ret, info = bucket.delete(storage.bucket_name, name)
                if ret is None:
                    print(info)
            if eof:
                break
<commit_msg>Remove debug code<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals, print_function
from datetime import datetime
import os
from os.path import dirname, join
import sys
import time
import unittest
import uuid

import logging
logging.basicConfig(level=logging.WARNING)

import six
import django
import pytest
from requests.exceptions import ConnectionError

from qiniu import set_default, BucketManager
from qiniu.config import Zone

zone_overseas = Zone('up.qiniug.com', 'upload.qiniug.com')

from .utils import retry

# Add repo/demo_site to sys.path
DEMO_SITE_DIR = join(dirname(dirname(__file__)), 'demo_site')
sys.path.append(DEMO_SITE_DIR)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "demo_site.settings")

try:
    django.setup()
except AttributeError:
   # Setup isn't necessary in Django < 1.7
   pass

from django.conf import settings
from qiniustorage.backends import QiniuStorage, QiniuFile
from qiniustorage.utils import QiniuError

USING_TRAVIS = os.environ.get('USING_TRAVIS', None) is None
if USING_TRAVIS:
    set_default(default_zone=zone_overseas, connection_retries=20, connection_timeout=100)

UNIQUE_PATH = str(uuid.uuid4())

class QiniuStorageTest(unittest.TestCase):
    def setUp(self):
        self.storage = QiniuStorage()

    def test_file_init(self):
        fil = QiniuFile('foo', self.storage, mode='rb')
        assert fil._mode == 'rb'
        assert fil._name == "foo"

    def test_write_to_read_only_file(self):
        with pytest.raises(AttributeError):
            fil = QiniuFile('foo', self.storage, mode='rb')
            fil.write("goto fail")

    def test_write_and_delete_file(self):
        ASSET_FILE_NAMES = [u'jquery-2.1.1.min.js', u'jquery-2点壹点一.min.js']
        for assert_file_name in ASSET_FILE_NAMES:
            REMOTE_PATH = join(UNIQUE_PATH, assert_file_name)
            assert self.storage.exists(REMOTE_PATH) == False
            fil = QiniuFile(REMOTE_PATH, self.storage, mode='wb')

            with open(join(dirname(__file__),'assets', assert_file_name), 'rb') as assset_file:
                content = assset_file.read()

                assset_file.seek(0, os.SEEK_END)
                assset_file_size = assset_file.tell()

                fil.write(content)
                self.storage._save(REMOTE_PATH, fil)

                assert self.storage.exists(REMOTE_PATH)

            assert self.storage.size(REMOTE_PATH) == assset_file_size

            now = datetime.utcnow()
            modified_time = self.storage.modified_time(REMOTE_PATH)
            # Datetime on Qiniu servers may be faster or slower than the local
            # machine. Thus the absolute delta within 60s should be considered
            # acceptable.
            time_delta = max(now, modified_time) - min(now, modified_time)
            assert time_delta.seconds < 180

            self.storage.delete(REMOTE_PATH)
            assert self.storage.exists(REMOTE_PATH) == False

    def test_read_file(self):
        ASSET_FILE_NAMES =  [u'jquery-1.11.1.min.js', u'jquery-一点十一点壹.js']
        for assert_file_name in ASSET_FILE_NAMES:
            REMOTE_PATH = join(UNIQUE_PATH, assert_file_name)

            with open(join(dirname(__file__),'assets', assert_file_name), 'rb') as assset_file:
                self.storage.save(REMOTE_PATH, assset_file)

            fil = self.storage.open(REMOTE_PATH, 'r')

            assert fil._is_read == False

            content = fil.read()
            assert content.startswith(u"/*!")

            assert fil._is_read == True

            # Test open mode
            fil = self.storage.open(REMOTE_PATH, 'rb')
            bin_content = fil.read()
            assert bin_content.startswith(b"/*!")

    def test_dirty_file(self):
        ASSET_FILE_NAME = 'bootstrap.min.css'
        REMOTE_PATH = join(UNIQUE_PATH, ASSET_FILE_NAME)

        fil = self.storage.open(REMOTE_PATH, 'rw')

        assert fil._is_read == False
        assert fil._is_dirty == False
        assert self.storage.exists(REMOTE_PATH) == False

        with open(join(dirname(__file__),'assets', ASSET_FILE_NAME), 'r') as assset_file:
            content = assset_file.read()
            fil.write(content)

        assert fil._is_read == True
        assert fil._is_dirty == True

        fil.close()
        assert self.storage.exists(REMOTE_PATH) == True

    @retry(QiniuError, tries=10, backoff=1)
    def test_listdir(self):
        dirnames = ['', 'foo', 'bar']
        filenames = ['file1', 'file2', 'file3', u'file四']
        for dirname in dirnames:
            for filename in filenames:
                fil = self.storage.open(join(UNIQUE_PATH, dirname, filename), 'w')
                fil.write('test text')
                fil.close()
        time.sleep(3)

        dirs, files = self.storage.listdir(UNIQUE_PATH)
        assert sorted(dirs) == sorted(['foo', 'bar'])

        dirs, files = self.storage.listdir(join(UNIQUE_PATH, 'foo'))
        assert dirs == []
        assert sorted(files) == sorted(filenames)

    @classmethod
    def teardown_class(cls):
        """Delete all files in the test bucket.
        """
        storage = QiniuStorage()
        auth = storage.auth
        bucket = BucketManager(auth)

        while True:
            ret, eof, info = bucket.list(storage.bucket_name, limit=100)

            if ret is None:
                print(info)
                break

            for item in ret['items']:
                name = item['key']
                if six.PY2:
                    name = name.encode('utf-8')
                ret, info = bucket.delete(storage.bucket_name, name)
                if ret is None:
                    print(info)
            if eof:
                break
<|endoftext|>