<commit_before>'''
This module holds the constants used for specifying the states of the debugger.
'''

STATE_RUN = 1
STATE_SUSPEND = 2

PYTHON_SUSPEND = 1
DJANGO_SUSPEND = 2

try:
    __setFalse = False
except:
    import __builtin__

    setattr(__builtin__, 'True', 1)
    setattr(__builtin__, 'False', 0)

USE_LIB_COPY = 1

class DebugInfoHolder:
    #we have to put it here because it can be set through the command line (so, the
    #already imported references would not have it).
    DEBUG_RECORD_SOCKET_READS = False
    DEBUG_TRACE_LEVEL = -1
    DEBUG_TRACE_BREAKPOINTS = -1

#Optimize with psyco? This gave a 50% speedup in the debugger in tests
USE_PSYCO_OPTIMIZATION = True

#Hold a reference to the original _getframe (because psyco will change that as soon as it's imported)
import sys #Note: the sys import must be here anyways (others depend on it)
try:
    GetFrame = sys._getframe
except AttributeError:
    def GetFrame():
        raise AssertionError('sys._getframe not available (possible causes: enable -X:Frames on IronPython?)')

#Used to determine the maximum size of each variable passed to eclipse -- having a big value here may make
#the communication slower -- as the variables are being gathered lazily in the latest version of eclipse,
#this value was raised from 200 to 1000.
MAXIMUM_VARIABLE_REPRESENTATION_SIZE = 1000

import pydev_threading as threading
import os

_nextThreadIdLock = threading.Lock()

#=======================================================================================================================
# Python 3?
#=======================================================================================================================
IS_PY3K = False
IS_PY27 = False
IS_PY24 = False
try:
    if sys.version_info[0] >= 3:
        IS_PY3K = True
    elif sys.version_info[0] == 2 and sys.version_info[1] == 7:
        IS_PY27 = True
    elif sys.version_info[0] == 2 and sys.version_info[1] == 4:
        IS_PY24 = True
except AttributeError:
    pass #Not all versions have sys.version_info

try:
    IS_64_BITS = sys.maxsize > 2 ** 32
except AttributeError:
    try:
        import struct
        IS_64_BITS = struct.calcsize("P") * 8 > 32
    except:
        IS_64_BITS = False

#=======================================================================================================================
# Jython?
#=======================================================================================================================
try:
    import org.python.core.PyDictionary #@UnresolvedImport @UnusedImport -- just to check if it could be valid

    def DictContains(d, key):
        return d.has_key(key)
except:
    try:
        #Py3k does not have has_key anymore, and older versions don't have __contains__
        DictContains = dict.__contains__
    except:
        try:
            DictContains = dict.has_key
        except NameError:
            def DictContains(d, key):
                return d.has_key(key)

try:
    xrange
except:
    #Python 3k does not have it
    xrange = range

try:
    object
except NameError:
    class object:
        pass

try:
    enumerate
except:
    def enumerate(lst):
        ret = []
        i=0
        for element in lst:
            ret.append((i, element))
            i+=1
        return ret

#=======================================================================================================================
# StringIO
#=======================================================================================================================
try:
    from StringIO import StringIO
except:
    from io import StringIO

#=======================================================================================================================
# NextId
#=======================================================================================================================
class NextId:

    def __init__(self):
        self._id = 0

    def __call__(self):
        #No need to synchronize here
        self._id += 1
        return self._id

_nextThreadId = NextId()

#=======================================================================================================================
# GetThreadId
#=======================================================================================================================
def GetThreadId(thread):
    try:
        return thread.__pydevd_id__
    except AttributeError:
        _nextThreadIdLock.acquire()
        try:
            #We do a new check with the lock in place just to be sure that nothing changed
            if not hasattr(thread, '__pydevd_id__'):
                try:
                    pid = os.getpid()
                except AttributeError:
                    try:
                        #Jython does not have it!
                        import java.lang.management.ManagementFactory #@UnresolvedImport -- just for jython

                        pid = java.lang.management.ManagementFactory.getRuntimeMXBean().getName()
                        pid = pid.replace('@', '_')
                    except:
                        #ok, no pid available (will be unable to debug multiple processes)
                        pid = '000001'

                thread.__pydevd_id__ = 'pid%s_seq%s' % (pid, _nextThreadId())
        finally:
            _nextThreadIdLock.release()

    return thread.__pydevd_id__

#===============================================================================
# Null
#===============================================================================
class Null:
    """
    Gotten from: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/68205
    """

    def __init__(self, *args, **kwargs):
        return None

    def __call__(self, *args, **kwargs):
        return self

    def __getattr__(self, mname):
        return self

    def __setattr__(self, name, value):
        return self

    def __delattr__(self, name):
        return self

    def __repr__(self):
        return "<Null>"

    def __str__(self):
        return "Null"

    def __len__(self):
        return 0

    def __getitem__(self):
        return self

    def __setitem__(self, *args, **kwargs):
        pass

    def write(self, *args, **kwargs):
        pass

    def __nonzero__(self):
        return 0

if __name__ == '__main__':
    if Null():
        sys.stdout.write('here\n')

<commit_msg>Fixed wrong import.<commit_after>'''
This module holds the constants used for specifying the states of the debugger.
'''

STATE_RUN = 1
STATE_SUSPEND = 2

PYTHON_SUSPEND = 1
DJANGO_SUSPEND = 2

try:
    __setFalse = False
except:
    import __builtin__

    setattr(__builtin__, 'True', 1)
    setattr(__builtin__, 'False', 0)

USE_LIB_COPY = 1

class DebugInfoHolder:
    #we have to put it here because it can be set through the command line (so, the
    #already imported references would not have it).
    DEBUG_RECORD_SOCKET_READS = False
    DEBUG_TRACE_LEVEL = -1
    DEBUG_TRACE_BREAKPOINTS = -1

#Optimize with psyco? This gave a 50% speedup in the debugger in tests
USE_PSYCO_OPTIMIZATION = True

#Hold a reference to the original _getframe (because psyco will change that as soon as it's imported)
import sys #Note: the sys import must be here anyways (others depend on it)
try:
    GetFrame = sys._getframe
except AttributeError:
    def GetFrame():
        raise AssertionError('sys._getframe not available (possible causes: enable -X:Frames on IronPython?)')

#Used to determine the maximum size of each variable passed to eclipse -- having a big value here may make
#the communication slower -- as the variables are being gathered lazily in the latest version of eclipse,
#this value was raised from 200 to 1000.
MAXIMUM_VARIABLE_REPRESENTATION_SIZE = 1000

if USE_LIB_COPY:
    import _pydev_threading as threading
else:
    import threading

import os

_nextThreadIdLock = threading.Lock()

#=======================================================================================================================
# Python 3?
#=======================================================================================================================
IS_PY3K = False
IS_PY27 = False
IS_PY24 = False
try:
    if sys.version_info[0] >= 3:
        IS_PY3K = True
    elif sys.version_info[0] == 2 and sys.version_info[1] == 7:
        IS_PY27 = True
    elif sys.version_info[0] == 2 and sys.version_info[1] == 4:
        IS_PY24 = True
except AttributeError:
    pass #Not all versions have sys.version_info

try:
    IS_64_BITS = sys.maxsize > 2 ** 32
except AttributeError:
    try:
        import struct
        IS_64_BITS = struct.calcsize("P") * 8 > 32
    except:
        IS_64_BITS = False

#=======================================================================================================================
# Jython?
#=======================================================================================================================
try:
    import org.python.core.PyDictionary #@UnresolvedImport @UnusedImport -- just to check if it could be valid

    def DictContains(d, key):
        return d.has_key(key)
except:
    try:
        #Py3k does not have has_key anymore, and older versions don't have __contains__
        DictContains = dict.__contains__
    except:
        try:
            DictContains = dict.has_key
        except NameError:
            def DictContains(d, key):
                return d.has_key(key)

try:
    xrange
except:
    #Python 3k does not have it
    xrange = range

try:
    object
except NameError:
    class object:
        pass

try:
    enumerate
except:
    def enumerate(lst):
        ret = []
        i=0
        for element in lst:
            ret.append((i, element))
            i+=1
        return ret

#=======================================================================================================================
# StringIO
#=======================================================================================================================
try:
    from StringIO import StringIO
except:
    from io import StringIO

#=======================================================================================================================
# NextId
#=======================================================================================================================
class NextId:

    def __init__(self):
        self._id = 0

    def __call__(self):
        #No need to synchronize here
        self._id += 1
        return self._id

_nextThreadId = NextId()

#=======================================================================================================================
# GetThreadId
#=======================================================================================================================
def GetThreadId(thread):
    try:
        return thread.__pydevd_id__
    except AttributeError:
        _nextThreadIdLock.acquire()
        try:
            #We do a new check with the lock in place just to be sure that nothing changed
            if not hasattr(thread, '__pydevd_id__'):
                try:
                    pid = os.getpid()
                except AttributeError:
                    try:
                        #Jython does not have it!
                        import java.lang.management.ManagementFactory #@UnresolvedImport -- just for jython

                        pid = java.lang.management.ManagementFactory.getRuntimeMXBean().getName()
                        pid = pid.replace('@', '_')
                    except:
                        #ok, no pid available (will be unable to debug multiple processes)
                        pid = '000001'

                thread.__pydevd_id__ = 'pid%s_seq%s' % (pid, _nextThreadId())
        finally:
            _nextThreadIdLock.release()

    return thread.__pydevd_id__

#===============================================================================
# Null
#===============================================================================
class Null:
    """
    Gotten from: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/68205
    """

    def __init__(self, *args, **kwargs):
        return None

    def __call__(self, *args, **kwargs):
        return self

    def __getattr__(self, mname):
        return self

    def __setattr__(self, name, value):
        return self

    def __delattr__(self, name):
        return self

    def __repr__(self):
        return "<Null>"

    def __str__(self):
        return "Null"

    def __len__(self):
        return 0

    def __getitem__(self):
        return self

    def __setitem__(self, *args, **kwargs):
        pass

    def write(self, *args, **kwargs):
        pass

    def __nonzero__(self):
        return 0

if __name__ == '__main__':
    if Null():
        sys.stdout.write('here\n')

<|endoftext|>


<commit_before>from AlphaTwirl import WriteListToFile
import pandas
import unittest
import cStringIO

##____________________________________________________________________________||
class MockOpen(object):
    def __init__(self, out): self._out = out
    def __call__(self, path): return self._out

##____________________________________________________________________________||
def mockClose(file): pass

##____________________________________________________________________________||
class TestWriteListToFile(unittest.TestCase):

    def test_deliver(self):
        delivery = WriteListToFile("tbl.txt")

        out = cStringIO.StringIO()
        delivery._open = MockOpen(out)
        delivery._close = mockClose

        results = [
            ('component', 'v1', 'nvar', 'n'),
            ('data1',  100, 6.0,   40),
            ('data1',    2, 9.0, 3.3),
            ('data1', 3124, 3.0, 0.0000001),
            ('data2',  333, 6.0, 300909234),
            ('data2',   11, 2.0, 323432.2234),
        ]

        delivery.deliver(results)

        expected = """ component   v1 nvar           n
     data1  100    6          40
     data1    2    9         3.3
     data1 3124    3       1e-07
     data2  333    6   300909234
     data2   11    2 323432.2234
"""

        self.assertEqual(expected, out.getvalue())

    def test_deliver_empty_dataframe(self):

        delivery = WriteListToFile("tbl.txt")

        out = cStringIO.StringIO()
        delivery._open = MockOpen(out)
        delivery._close = mockClose

        results = [
            ('component', 'v1', 'nvar', 'n'),
        ]

        delivery.deliver(results)

        expected = " component v1 nvar n\n"
        self.assertEqual(expected, out.getvalue())

    def test_deliver_None_results(self):

        delivery = WriteListToFile("tbl.txt")

        out = cStringIO.StringIO()
        delivery._open = MockOpen(out)
        delivery._close = mockClose

        delivery.deliver(None)

##____________________________________________________________________________||
<commit_msg>remove an unnecessary import statement in test_WriteListToFile.py<commit_after>from AlphaTwirl import WriteListToFile
import unittest
import cStringIO

##____________________________________________________________________________||
class MockOpen(object):
    def __init__(self, out): self._out = out
    def __call__(self, path): return self._out

##____________________________________________________________________________||
def mockClose(file): pass

##____________________________________________________________________________||
class TestWriteListToFile(unittest.TestCase):

    def test_deliver(self):
        delivery = WriteListToFile("tbl.txt")

        out = cStringIO.StringIO()
        delivery._open = MockOpen(out)
        delivery._close = mockClose

        results = [
            ('component', 'v1', 'nvar', 'n'),
            ('data1',  100, 6.0,   40),
            ('data1',    2, 9.0, 3.3),
            ('data1', 3124, 3.0, 0.0000001),
            ('data2',  333, 6.0, 300909234),
            ('data2',   11, 2.0, 323432.2234),
        ]

        delivery.deliver(results)

        expected = """ component   v1 nvar           n
     data1  100    6          40
     data1    2    9         3.3
     data1 3124    3       1e-07
     data2  333    6   300909234
     data2   11    2 323432.2234
"""

        self.assertEqual(expected, out.getvalue())

    def test_deliver_empty_dataframe(self):

        delivery = WriteListToFile("tbl.txt")

        out = cStringIO.StringIO()
        delivery._open = MockOpen(out)
        delivery._close = mockClose

        results = [
            ('component', 'v1', 'nvar', 'n'),
        ]

        delivery.deliver(results)

        expected = " component v1 nvar n\n"
        self.assertEqual(expected, out.getvalue())

    def test_deliver_None_results(self):

        delivery = WriteListToFile("tbl.txt")

        out = cStringIO.StringIO()
        delivery._open = MockOpen(out)
        delivery._close = mockClose

        delivery.deliver(None)

##____________________________________________________________________________||
<|endoftext|>


<commit_before>from tapiriik.settings import WEB_ROOT, AGGRESSIVE_CACHE, STRAVA_CLIENT_SECRET, STRAVA_CLIENT_ID
from tapiriik.services.service_base import ServiceAuthenticationType, ServiceBase
from tapiriik.services.service_record import ServiceRecord
from tapiriik.database import cachedb
from tapiriik.services.interchange import UploadedActivity, ActivityType, Waypoint, WaypointType, Location
from tapiriik.services.api import APIException, UserException, UserExceptionType, APIExcludeActivity
from tapiriik.services.tcx import TCXIO

from django.core.urlresolvers import reverse
from datetime import datetime, timedelta
import calendar
import requests
import json
import os
import logging
import pytz
import re
import time

logger = logging.getLogger(__name__)

class StravaService(ServiceBase):
    ID = "strava"
    DisplayName = "Strava"
    AuthenticationType = ServiceAuthenticationType.OAuth
    UserProfileURL = "http://www.strava.com/athletes/{0}"
    UserActivityURL = "http://app.strava.com/activities/{1}"
    AuthenticationNoFrame = True  # They don't prevent the iframe, it just looks really ugly.

    SupportsHR = SupportsCadence = SupportsTemp = SupportsPower = True

    # For mapping Strava->common; no ambiguity in Strava activity type
    _activityTypeMappings = {
        ActivityType.Cycling: "Ride",
        ActivityType.MountainBiking: "Ride",
        ActivityType.Hiking: "Hike",
        ActivityType.Running: "Run",
        ActivityType.Walking: "Walk",
        ActivityType.Snowboarding: "Snowboard",
        ActivityType.Skating: "IceSkate",
        ActivityType.CrossCountrySkiing: "BackcountrySki",
        ActivityType.DownhillSkiing: "NordicSki",
        ActivityType.DownhillSkiing: "AlpineSki",
        ActivityType.Swimming: "Swim"
    }

    # For mapping common->Strava
    _reverseActivityTypeMappings = {
        ActivityType.Cycling: "Ride",
        ActivityType.MountainBiking: "MountainBiking",
        ActivityType.Running: "Run",
        ActivityType.Hiking: "Hike",
        ActivityType.Walking: "Walk",
        ActivityType.DownhillSkiing: "AlpineSki",
        ActivityType.CrossCountrySkiing: "BackcountrySki",
        ActivityType.Swimming: "Swim",
        ActivityType.Skating: "IceSkate"
    }

    SupportedActivities = list(_reverseActivityTypeMappings.keys())

    def WebInit(self):
        self.UserAuthorizationURL = "https://www.strava.com/oauth/authorize?scope=write%20view_private&client_id=" + STRAVA_CLIENT_ID + "&response_type=code&redirect_uri=http://tapiriik.com"  + reverse("oauth_return", kwargs={"service": "strava"})

    def _apiHeaders(self, serviceRecord):
        return {"Authorization": "access_token " + serviceRecord.Authorization["OAuthToken"]}

    def RetrieveAuthorizationToken(self, req, level):
        code = req.GET.get("code")
        params = {"grant_type": "authorization_code", "code": code, "client_id": STRAVA_CLIENT_ID, "client_secret": STRAVA_CLIENT_SECRET, "redirect_uri": WEB_ROOT + reverse("oauth_return", kwargs={"service": "runkeeper"})}

        response = requests.post("https://www.strava.com/oauth/token", data=params)
        if response.status_code != 200:
            raise APIException("Invalid code")
        data = response.json()

        authorizationData = {"OAuthToken": data["access_token"]}
        # Retrieve the user ID, meh.
        id_resp = requests.get("https://www.strava.com/api/v3/athlete", headers=self._apiHeaders(ServiceRecord({"Authorization": authorizationData})))

        return (id_resp.json()["id"], authorizationData)

    def RevokeAuthorization(self, serviceRecord):
        #  you can't revoke the tokens strava distributes :\
        pass

    def DownloadActivityList(self, svcRecord, exhaustive=False):
        activities = []
        exclusions = []
        before = earliestDate = None

        while True:
            logger.debug("Req with before=" + str(before) + "/" + str(earliestDate))
            resp = requests.get("https://www.strava.com/api/v3/athletes/" + str(svcRecord.ExternalID) + "/activities", headers=self._apiHeaders(svcRecord), params={"before": before})
            if resp.status_code == 401:
                raise APIException("No authorization to retrieve activity list", block=True, user_exception=UserException(UserExceptionType.Authorization, intervention_required=True))

            earliestDate = None

            reqdata = resp.json()

            if not len(reqdata):
                break  # No more activities to see

            for ride in reqdata:
                activity = UploadedActivity()
                activity.TZ = pytz.timezone(re.sub("^\([^\)]+\)\s*", "", ride["timezone"]))  # Comes back as "(GMT -13:37) The Stuff/We Want""
                activity.StartTime = pytz.utc.localize(datetime.strptime(ride["start_date"], "%Y-%m-%dT%H:%M:%SZ"))
                logger.debug("\tActivity s/t " + str(activity.StartTime))
                if not earliestDate or activity.StartTime < earliestDate:
                    earliestDate = activity.StartTime
                    before = calendar.timegm(activity.StartTime.astimezone(pytz.utc).timetuple())

                if ride["start_latlng"] is None or ride["end_latlng"] is None or ride["distance"] is None or ride["distance"] == 0:
                    exclusions.append(APIExcludeActivity("No path", activityId=ride["id"]))
                    logger.debug("\t\tNo pts")
                    continue  # stationary activity - no syncing for now

                activity.EndTime = activity.StartTime + timedelta(0, ride["elapsed_time"])
                activity.UploadedTo = [{"Connection": svcRecord, "ActivityID": ride["id"]}]

                actType = [k for k, v in self._reverseActivityTypeMappings.items() if v == ride["type"]]
                if not len(actType):
                    exclusions.append(APIExcludeActivity("Unsupported activity type %s" % ride["type"], activityId=ride["id"]))
                    logger.debug("\t\tUnknown activity")
                    continue

                activity.Type = actType[0]
                activity.Distance = ride["distance"]
                activity.Name = ride["name"]
                activity.Private = ride["private"]
                activity.AdjustTZ()
                activity.CalculateUID()
                activities.append(activity)

            if not exhaustive or not earliestDate:
                break

        return activities, exclusions

    def DownloadActivity(self, svcRecord, activity):
        # thanks to Cosmo Catalano for the API reference code
        activityID = [x["ActivityID"] for x in activity.UploadedTo if x["Connection"] == svcRecord][0]

        streamdata = requests.get("https://www.strava.com/api/v3/activities/" + str(activityID) + "/streams/time,altitude,heartrate,cadence,watts,watts_calc,temp,resting,latlng", headers=self._apiHeaders(svcRecord))
        if streamdata.status_code == 401:
            raise APIException("No authorization to download activity", block=True, user_exception=UserException(UserExceptionType.Authorization, intervention_required=True))

        streamdata = streamdata.json()

        if "message" in streamdata and streamdata["message"] == "Record Not Found":
            raise APIException("Could not find activity")

        ridedata = {}
        for stream in streamdata:
            ridedata[stream["type"]] = stream["data"]

        activity.Waypoints = []

        hasHR = "heartrate" in ridedata and len(ridedata["heartrate"]) > 0
        hasCadence = "cadence" in ridedata and len(ridedata["cadence"]) > 0
        hasTemp = "temp" in ridedata and len(ridedata["temp"]) > 0
        hasPower = ("watts" in ridedata and len(ridedata["watts"]) > 0) or ("watts_calc" in ridedata and len(ridedata["watts_calc"]) > 0)
        hasAltitude = "altitude" in ridedata and len(ridedata["altitude"]) > 0
        hasRestingData = "resting" in ridedata and len(ridedata["resting"]) > 0
        moving = True

        if "error" in ridedata:
            raise APIException("Strava error " + ridedata["error"])

        hasLocation = False
        waypointCt = len(ridedata["time"])
        for idx in range(0, waypointCt - 1):
            latlng = ridedata["latlng"][idx]

            waypoint = Waypoint(activity.StartTime + timedelta(0, ridedata["time"][idx]))
            latlng = ridedata["latlng"][idx]
            waypoint.Location = Location(latlng[0], latlng[1], None)
            if waypoint.Location.Longitude == 0 and waypoint.Location.Latitude == 0:
                waypoint.Location.Longitude = None
                waypoint.Location.Latitude = None
            else:  # strava only returns 0 as invalid coords, so no need to check for null (update: ??)
                hasLocation = True
            if hasAltitude:
                waypoint.Location.Altitude = float(ridedata["altitude"][idx])

            if idx == 0:
                waypoint.Type = WaypointType.Start
            elif idx == waypointCt - 2:
                waypoint.Type = WaypointType.End
            elif hasRestingData and not moving and ridedata["resting"][idx] is False:
                waypoint.Type = WaypointType.Resume
                moving = True
            elif hasRestingData and ridedata["resting"][idx] is True:
                waypoint.Type = WaypointType.Pause
                moving = False

            if hasHR:
                waypoint.HR = ridedata["heartrate"][idx]
            if hasCadence:
                waypoint.Cadence = ridedata["cadence"][idx]
            if hasTemp:
                waypoint.Temp = ridedata["temp"][idx]
            if hasPower:
                waypoint.Power = ridedata["watts"][idx] if "watts" in ridedata else ridedata["watts_calc"][idx]
            activity.Waypoints.append(waypoint)
        if not hasLocation:
            raise APIExcludeActivity("No waypoints with location", activityId=activityID)
        return activity

    def UploadActivity(self, serviceRecord, activity):
        fields = ["time", "latlng", "elevation", "cmd", "heartrate", "cadence", "watts", "temp"]
        points = []
        logger.info("Activity tz " + str(activity.TZ) + " dt tz " + str(activity.StartTime.tzinfo) + " starttime " + str(activity.StartTime))

        req = { "id": 0,
                "data_type": "tcx",
                "external_id": "tap-sync-" + str(os.getpid()) + "-" + activity.UID + "-" + activity.UploadedTo[0]["Connection"].Service.ID,
                "activity_name": activity.Name,
                "activity_type": self._activityTypeMappings[activity.Type],
                "private": activity.Private}

        if "tcx" in activity.PrerenderedFormats:
            logger.debug("Using prerendered TCX")
            tcxData = activity.PrerenderedFormats["tcx"]
        else:
            activity.EnsureTZ()
            tcxData = TCXIO.Dump(activity)
        # TODO: put the tcx back into PrerenderedFormats once there's more RAM to go around and there's a possibility of it actually being used.
        files = {"file":(req["external_id"] + ".tcx", tcxData)}

        response = requests.post("http://www.strava.com/api/v3/uploads", data=req, files=files, headers=self._apiHeaders(serviceRecord))
        if response.status_code != 201:
            if response.status_code == 401:
                raise APIException("No authorization to upload activity " + activity.UID + " response " + response.text + " status " + str(response.status_code), block=True, user_exception=UserException(UserExceptionType.Authorization, intervention_required=True))
            raise APIException("Unable to upload activity " + activity.UID + " response " + response.text + " status " + str(response.status_code))

        upload_id = response.json()["id"]
        while not response.json()["activity_id"]:
            time.sleep(1)
            response = requests.get("http://www.strava.com/api/v3/uploads/%s" % upload_id, headers=self._apiHeaders(serviceRecord))
            logger.debug("Waiting for upload - status %s id %s" % (response.json()["status"], response.json()["activity_id"]))
            if response.json()["error"]:
                error = response.json()["error"]
                if "duplicate of activity" in error:
                    logger.debug("Duplicate")
                    return # I guess we're done here?
                raise APIException("Strava failed while processing activity - last status %s" % response.text)

    def DeleteCachedData(self, serviceRecord):
        cachedb.strava_cache.remove({"Owner": serviceRecord.ExternalID})
        cachedb.strava_activity_cache.remove({"Owner": serviceRecord.ExternalID})
<commit_msg>Don't pretend Strava's estimated power is real power.<commit_after>from tapiriik.settings import WEB_ROOT, AGGRESSIVE_CACHE, STRAVA_CLIENT_SECRET, STRAVA_CLIENT_ID
from tapiriik.services.service_base import ServiceAuthenticationType, ServiceBase
from tapiriik.services.service_record import ServiceRecord
from tapiriik.database import cachedb
from tapiriik.services.interchange import UploadedActivity, ActivityType, Waypoint, WaypointType, Location
from tapiriik.services.api import APIException, UserException, UserExceptionType, APIExcludeActivity
from tapiriik.services.tcx import TCXIO

from django.core.urlresolvers import reverse
from datetime import datetime, timedelta
import calendar
import requests
import json
import os
import logging
import pytz
import re
import time

logger = logging.getLogger(__name__)

class StravaService(ServiceBase):
    ID = "strava"
    DisplayName = "Strava"
    AuthenticationType = ServiceAuthenticationType.OAuth
    UserProfileURL = "http://www.strava.com/athletes/{0}"
    UserActivityURL = "http://app.strava.com/activities/{1}"
    AuthenticationNoFrame = True  # They don't prevent the iframe, it just looks really ugly.

    SupportsHR = SupportsCadence = SupportsTemp = SupportsPower = True

    # For mapping Strava->common; no ambiguity in Strava activity type
    _activityTypeMappings = {
        ActivityType.Cycling: "Ride",
        ActivityType.MountainBiking: "Ride",
        ActivityType.Hiking: "Hike",
        ActivityType.Running: "Run",
        ActivityType.Walking: "Walk",
        ActivityType.Snowboarding: "Snowboard",
        ActivityType.Skating: "IceSkate",
        ActivityType.CrossCountrySkiing: "BackcountrySki",
        ActivityType.DownhillSkiing: "NordicSki",
        ActivityType.DownhillSkiing: "AlpineSki",
        ActivityType.Swimming: "Swim"
    }

    # For mapping common->Strava
    _reverseActivityTypeMappings = {
        ActivityType.Cycling: "Ride",
        ActivityType.MountainBiking: "MountainBiking",
        ActivityType.Running: "Run",
        ActivityType.Hiking: "Hike",
        ActivityType.Walking: "Walk",
        ActivityType.DownhillSkiing: "AlpineSki",
        ActivityType.CrossCountrySkiing: "BackcountrySki",
        ActivityType.Swimming: "Swim",
        ActivityType.Skating: "IceSkate"
    }

    SupportedActivities = list(_reverseActivityTypeMappings.keys())

    def WebInit(self):
        self.UserAuthorizationURL = "https://www.strava.com/oauth/authorize?scope=write%20view_private&client_id=" + STRAVA_CLIENT_ID + "&response_type=code&redirect_uri=http://tapiriik.com"  + reverse("oauth_return", kwargs={"service": "strava"})

    def _apiHeaders(self, serviceRecord):
        return {"Authorization": "access_token " + serviceRecord.Authorization["OAuthToken"]}

    def RetrieveAuthorizationToken(self, req, level):
        code = req.GET.get("code")
        params = {"grant_type": "authorization_code", "code": code, "client_id": STRAVA_CLIENT_ID, "client_secret": STRAVA_CLIENT_SECRET, "redirect_uri": WEB_ROOT + reverse("oauth_return", kwargs={"service": "runkeeper"})}

        response = requests.post("https://www.strava.com/oauth/token", data=params)
        if response.status_code != 200:
            raise APIException("Invalid code")
        data = response.json()

        authorizationData = {"OAuthToken": data["access_token"]}
        # Retrieve the user ID, meh.
        id_resp = requests.get("https://www.strava.com/api/v3/athlete", headers=self._apiHeaders(ServiceRecord({"Authorization": authorizationData})))

        return (id_resp.json()["id"], authorizationData)

    def RevokeAuthorization(self, serviceRecord):
        #  you can't revoke the tokens strava distributes :\
        pass

    def DownloadActivityList(self, svcRecord, exhaustive=False):
        activities = []
        exclusions = []
        before = earliestDate = None

        while True:
            logger.debug("Req with before=" + str(before) + "/" + str(earliestDate))
            resp = requests.get("https://www.strava.com/api/v3/athletes/" + str(svcRecord.ExternalID) + "/activities", headers=self._apiHeaders(svcRecord), params={"before": before})
            if resp.status_code == 401:
                raise APIException("No authorization to retrieve activity list", block=True, user_exception=UserException(UserExceptionType.Authorization, intervention_required=True))

            earliestDate = None

            reqdata = resp.json()

            if not len(reqdata):
                break  # No more activities to see

            for ride in reqdata:
                activity = UploadedActivity()
                activity.TZ = pytz.timezone(re.sub("^\([^\)]+\)\s*", "", ride["timezone"]))  # Comes back as "(GMT -13:37) The Stuff/We Want""
                activity.StartTime = pytz.utc.localize(datetime.strptime(ride["start_date"], "%Y-%m-%dT%H:%M:%SZ"))
                logger.debug("\tActivity s/t " + str(activity.StartTime))
                if not earliestDate or activity.StartTime < earliestDate:
                    earliestDate = activity.StartTime
                    before = calendar.timegm(activity.StartTime.astimezone(pytz.utc).timetuple())

                if ride["start_latlng"] is None or ride["end_latlng"] is None or ride["distance"] is None or ride["distance"] == 0:
                    exclusions.append(APIExcludeActivity("No path", activityId=ride["id"]))
                    logger.debug("\t\tNo pts")
                    continue  # stationary activity - no syncing for now

                activity.EndTime = activity.StartTime + timedelta(0, ride["elapsed_time"])
                activity.UploadedTo = [{"Connection": svcRecord, "ActivityID": ride["id"]}]

                actType = [k for k, v in self._reverseActivityTypeMappings.items() if v == ride["type"]]
                if not len(actType):
                    exclusions.append(APIExcludeActivity("Unsupported activity type %s" % ride["type"], activityId=ride["id"]))
                    logger.debug("\t\tUnknown activity")
                    continue

                activity.Type = actType[0]
                activity.Distance = ride["distance"]
                activity.Name = ride["name"]
                activity.Private = ride["private"]
                activity.AdjustTZ()
                activity.CalculateUID()
                activities.append(activity)

            if not exhaustive or not earliestDate:
                break

        return activities, exclusions

    def DownloadActivity(self, svcRecord, activity):
        # thanks to Cosmo Catalano for the API reference code
        activityID = [x["ActivityID"] for x in activity.UploadedTo if x["Connection"] == svcRecord][0]

        streamdata = requests.get("https://www.strava.com/api/v3/activities/" + str(activityID) + "/streams/time,altitude,heartrate,cadence,watts,watts_calc,temp,resting,latlng", headers=self._apiHeaders(svcRecord))
        if streamdata.status_code == 401:
            raise APIException("No authorization to download activity", block=True, user_exception=UserException(UserExceptionType.Authorization, intervention_required=True))

        streamdata = streamdata.json()

        if "message" in streamdata and streamdata["message"] == "Record Not Found":
            raise APIException("Could not find activity")

        ridedata = {}
        for stream in streamdata:
            ridedata[stream["type"]] = stream["data"]

        activity.Waypoints = []

        hasHR = "heartrate" in ridedata and len(ridedata["heartrate"]) > 0
        hasCadence = "cadence" in ridedata and len(ridedata["cadence"]) > 0
        hasTemp = "temp" in ridedata and len(ridedata["temp"]) > 0
        hasPower = ("watts" in ridedata and len(ridedata["watts"]) > 0)
        hasAltitude = "altitude" in ridedata and len(ridedata["altitude"]) > 0
        hasRestingData = "resting" in ridedata and len(ridedata["resting"]) > 0
        moving = True

        if "error" in ridedata:
            raise APIException("Strava error " + ridedata["error"])

        hasLocation = False
        waypointCt = len(ridedata["time"])
        for idx in range(0, waypointCt - 1):
            latlng = ridedata["latlng"][idx]

            waypoint = Waypoint(activity.StartTime + timedelta(0, ridedata["time"][idx]))
            latlng = ridedata["latlng"][idx]
            waypoint.Location = Location(latlng[0], latlng[1], None)
            if waypoint.Location.Longitude == 0 and waypoint.Location.Latitude == 0:
                waypoint.Location.Longitude = None
                waypoint.Location.Latitude = None
            else:  # strava only returns 0 as invalid coords, so no need to check for null (update: ??)
                hasLocation = True
            if hasAltitude:
                waypoint.Location.Altitude = float(ridedata["altitude"][idx])

            if idx == 0:
                waypoint.Type = WaypointType.Start
            elif idx == waypointCt - 2:
                waypoint.Type = WaypointType.End
            elif hasRestingData and not moving and ridedata["resting"][idx] is False:
                waypoint.Type = WaypointType.Resume
                moving = True
            elif hasRestingData and ridedata["resting"][idx] is True:
                waypoint.Type = WaypointType.Pause
                moving = False

            if hasHR:
                waypoint.HR = ridedata["heartrate"][idx]
            if hasCadence:
                waypoint.Cadence = ridedata["cadence"][idx]
            if hasTemp:
                waypoint.Temp = ridedata["temp"][idx]
            if hasPower:
                waypoint.Power = ridedata["watts"][idx]
            activity.Waypoints.append(waypoint)
        if not hasLocation:
            raise APIExcludeActivity("No waypoints with location", activityId=activityID)
        return activity

    def UploadActivity(self, serviceRecord, activity):
        fields = ["time", "latlng", "elevation", "cmd", "heartrate", "cadence", "watts", "temp"]
        points = []
        logger.info("Activity tz " + str(activity.TZ) + " dt tz " + str(activity.StartTime.tzinfo) + " starttime " + str(activity.StartTime))

        req = { "id": 0,
                "data_type": "tcx",
                "external_id": "tap-sync-" + str(os.getpid()) + "-" + activity.UID + "-" + activity.UploadedTo[0]["Connection"].Service.ID,
                "activity_name": activity.Name,
                "activity_type": self._activityTypeMappings[activity.Type],
                "private": activity.Private}

        if "tcx" in activity.PrerenderedFormats:
            logger.debug("Using prerendered TCX")
            tcxData = activity.PrerenderedFormats["tcx"]
        else:
            activity.EnsureTZ()
            tcxData = TCXIO.Dump(activity)
        # TODO: put the tcx back into PrerenderedFormats once there's more RAM to go around and there's a possibility of it actually being used.
        files = {"file":(req["external_id"] + ".tcx", tcxData)}

        response = requests.post("http://www.strava.com/api/v3/uploads", data=req, files=files, headers=self._apiHeaders(serviceRecord))
        if response.status_code != 201:
            if response.status_code == 401:
                raise APIException("No authorization to upload activity " + activity.UID + " response " + response.text + " status " + str(response.status_code), block=True, user_exception=UserException(UserExceptionType.Authorization, intervention_required=True))
            raise APIException("Unable to upload activity " + activity.UID + " response " + response.text + " status " + str(response.status_code))

        upload_id = response.json()["id"]
        while not response.json()["activity_id"]:
            time.sleep(1)
            response = requests.get("http://www.strava.com/api/v3/uploads/%s" % upload_id, headers=self._apiHeaders(serviceRecord))
            logger.debug("Waiting for upload - status %s id %s" % (response.json()["status"], response.json()["activity_id"]))
            if response.json()["error"]:
                error = response.json()["error"]
                if "duplicate of activity" in error:
                    logger.debug("Duplicate")
                    return # I guess we're done here?
                raise APIException("Strava failed while processing activity - last status %s" % response.text)

    def DeleteCachedData(self, serviceRecord):
        cachedb.strava_cache.remove({"Owner": serviceRecord.ExternalID})
        cachedb.strava_activity_cache.remove({"Owner": serviceRecord.ExternalID})
<|endoftext|>


<commit_before>import moderngl as mgl

from demosys.effects import effect
from demosys import geometry
from pyrr import Vector3
import math

class SimpleRaymarchEffect(effect.Effect):
    """Generated raymarching effect"""
    def __init__(self):
        self.shader = self.get_shader("raymarching_simple.glsl", local=True)

        # create plane to fit whole screen
        self.plane = geometry.plane.plane_xz(size=(self.window_width, self.window_height), resolution=(10, 10))

    @effect.bind_target
    def draw(self, time, frametime, target):
        self.ctx.enable(mgl.DEPTH_TEST)

        # Rotate plane 90 degrees, and move plane back so it will fit correctly on to screen
        backoff = math.tan(math.radians(self.sys_camera.projection.fov / 2)) * (self.window_width / 2)
        m_mv = self.create_transformation(rotation=(math.radians(90), 0.0, 0.0), translation=(0.0, 0.0, -backoff))

        # Uniforms
        fov = 0.60
        alpha = 1.0
        modifier = -0.6
        cPosition = Vector3([(math.sin(time / 6.0) * 16.0),
                             (math.sin(time / 12.0) * 16.0),
                             (math.cos(time / 3.0) * 16.0)])
        cLookAt = Vector3([0.0, 0.0, 0.0])
        lPosition = Vector3([0.0, 17.0, 0.0])
        lItensity = 2.6
        color = Vector3([0.66, 0.96, 0.26])

        # Draw the plane

        # For vertex shader
        self.shader.uniform("m_proj", self.sys_camera.projection.tobytes())
        self.shader.uniform("m_mv", m_mv.astype('f4').tobytes())

        # For fragment shader
        self.shader.uniform("fov", fov)
        self.shader.uniform("alpha", alpha)
        self.shader.uniform("modifier", modifier)

        self.shader.uniform("cPosition", cPosition.astype('f4').tobytes())
        self.shader.uniform("cLookAt", cLookAt.astype('f4').tobytes())
        self.shader.uniform("lPosition", lPosition.astype('f4').tobytes())
        self.shader.uniform("lIntensity", lItensity)
        self.shader.uniform("color", color.astype('f4').tobytes())

        self.shader.uniform("resolution", (self.window_width, self.window_height))

        self.plane.draw(self.shader)
<commit_msg>fixed raymarching template to be consistent with other templates<commit_after>import moderngl as mgl

from demosys.effects import effect
from demosys import geometry
from pyrr import Vector3
import math

class SimpleRaymarchEffect(effect.Effect):
    """Generated raymarching effect"""
    def __init__(self):
        self.shader = self.get_shader("raymarching_simple.glsl", local=True)

        # create plane to fit whole screen
        self.plane = geometry.plane.plane_xz(size=(self.window_width, self.window_height), resolution=(10, 10))

    @effect.bind_target
    def draw(self, time, frametime, target):
        self.ctx.enable(mgl.DEPTH_TEST)

        # Rotate plane 90 degrees, and move plane back so it will fit correctly on to screen
        backoff = math.tan(math.radians(self.sys_camera.projection.fov / 2)) * (self.window_width / 2)
        m_mv = self.create_transformation(rotation=(math.radians(90), 0.0, 0.0), translation=(0.0, 0.0, -backoff))

        # Uniforms
        fov = 0.60
        alpha = 1.0
        modifier = -0.6
        cPosition = Vector3([(math.sin(time / 6.0) * 16.0),
                             (math.sin(time / 12.0) * 16.0),
                             (math.cos(time / 3.0) * 16.0)])
        cLookAt = Vector3([0.0, 0.0, 0.0])
        lPosition = Vector3([0.0, 17.0, 0.0])
        lItensity = 2.6
        color = Vector3([0.66, 0.96, 0.26])

        # Draw the plane

        # For vertex shader
        self.shader.uniform("m_proj", self.sys_camera.projection.tobytes())
        self.shader.uniform("m_mv", m_mv.astype('f4').tobytes())

        # For fragment shader
        self.shader.uniform("fov", fov)
        self.shader.uniform("alpha", alpha)
        self.shader.uniform("modifier", modifier)

        self.shader.uniform("cPosition", cPosition.astype('f4').tobytes())
        self.shader.uniform("cLookAt", cLookAt.astype('f4').tobytes())
        self.shader.uniform("lPosition", lPosition.astype('f4').tobytes())
        self.shader.uniform("lIntensity", lItensity)
        self.shader.uniform("color", color.astype('f4').tobytes())

        self.shader.uniform("resolution", (self.window_width, self.window_height))

        self.plane.draw(self.shader)<|endoftext|>


<commit_before>
class undirectedGraph(object):
	def __init__(self, degrees):
		self.degrees = degrees
		self.adjacent_matrix = []
		for i in range(degrees):
			self.adjacent_matrix.append([0]*degrees)

	def __str__(self):
		output = ""
		for row in self.adjacent_matrix:
			for item in row:
				output += "|"+str(item)
			output += "|\n"
		return output

	def addEdge(self, vertex1, vertex2):
		self.adjacent_matrix[vertex1][vertex2]+=1
		self.adjacent_matrix[vertex2][vertex1]+=1

        def isTree(self):
	   	return True

	def depthFirstSeach(self, start_vertex, finded_vertexes):
		for node in range(self.degrees):
			if self.adjacent_matrix[start_vertex][node] != 0 and node not in finded_vertexes:
				finded_vertexes.append(node)
				finded_vertexes = self.depthFirstSeach(self, start_node, finded_vertexes)
		return finded_vertexes

	def getAllEdges(self):
		return []

	def breathFirstSeach(self, start_vertex, finded_vertexes):
		for node in range(self.degrees):
			if self.adjacent_matrix[start_vertex][node] != 0 and node not in finded_vertexes:
				finded_vertexes.append(node)
		for node in range(self.degrees):
			finded_vertexes = self.breathFirstSeach(self, start_node, finded_vertexes)

<commit_msg>Add get All edges<commit_after>
class undirectedGraph(object):
	def __init__(self, degrees):
		self.degrees = degrees
		self.adjacent_matrix = []
		for i in range(degrees):
			self.adjacent_matrix.append([0]*degrees)

	def __str__(self):
		output = ""
		for row in self.adjacent_matrix:
			for item in row:
				output += "|"+str(item)
			output += "|\n"
		return output

	def addEdge(self, vertex1, vertex2):
		self.adjacent_matrix[vertex1][vertex2]+=1
		self.adjacent_matrix[vertex2][vertex1]+=1

        def isTree(self):
	   	return True

	def depthFirstSeach(self, start_vertex, finded_vertexes):
		for node in range(self.degrees):
			if self.adjacent_matrix[start_vertex][node] != 0 and node not in finded_vertexes:
				finded_vertexes.append(node)
				finded_vertexes = self.depthFirstSeach(self, start_node, finded_vertexes)
		return finded_vertexes

	def getAllEdges(self):
		result = []
		for node_start in range(self.degrees):
			for node_end in range(self.degrees):
				if self.adjacent_matrix[node_start][node_end] != 0:
					result.append([node_start, node_end])
		return result

	def breathFirstSeach(self, start_vertex, finded_vertexes):
		for node in range(self.degrees):
			if self.adjacent_matrix[start_vertex][node] != 0 and node not in finded_vertexes:
				finded_vertexes.append(node)
		for node in range(self.degrees):
			finded_vertexes = self.breathFirstSeach(self, start_node, finded_vertexes)

<|endoftext|>


<commit_before>import mongomock
from six import text_type

from .utils import TestCase

class CollectionAPITest(TestCase):
    def setUp(self):
        super(CollectionAPITest, self).setUp()
        self.conn = mongomock.Connection()
        self.db = self.conn['somedb']

    def test__get_subcollections(self):
        self.db.a.b
        self.assertEquals(self.db.a.b.full_name, "somedb.a.b")
        self.assertEquals(self.db.a.b.name, "a.b")

        self.assertEquals(
            set(self.db.collection_names()),
            set(["a.b", "system.indexes", "a"]))

    def test__get_collection_full_name(self):
        self.assertEquals(self.db.coll.name, "coll")
        self.assertEquals(self.db.coll.full_name, "somedb.coll")

    def test__get_collection_names(self):
        self.db.a
        self.db.b
        self.assertEquals(set(self.db.collection_names()), set(['a', 'b', 'system.indexes']))
        self.assertEquals(set(self.db.collection_names(True)), set(['a', 'b', 'system.indexes']))
        self.assertEquals(set(self.db.collection_names(False)), set(['a', 'b']))

    def test__cursor_collection(self):
        self.assertIs(self.db.a.find().collection, self.db.a)

    def test__drop_collection(self):
        self.db.a
        self.db.b
        self.db.c
        self.db.drop_collection('b')
        self.db.drop_collection('b')
        self.db.drop_collection(self.db.c)
        self.assertEquals(set(self.db.collection_names()), set(['a', 'system.indexes']))

    def test__distinct_nested_field(self):
        self.db.collection.insert({'f1': {'f2': 'v'}})
        cursor = self.db.collection.find()
        self.assertEquals(cursor.distinct('f1.f2'), ['v'])

    def test__cursor_clone(self):
        self.db.collection.insert([{"a": "b"}, {"b": "c"}, {"c": "d"}])
        cursor1 = self.db.collection.find()
        iterator1 = iter(cursor1)
        first_item = next(iterator1)
        cursor2 = cursor1.clone()
        iterator2 = iter(cursor2)
        self.assertEquals(next(iterator2), first_item)
        for item in iterator1:
            self.assertEquals(item, next(iterator2))

        with self.assertRaises(StopIteration):
            next(iterator2)

    def test__update_retval(self):
        self.db.col.save({"a": 1})
        retval = self.db.col.update({"a": 1}, {"b": 2})
        self.assertIsInstance(retval, dict)
        self.assertIsInstance(retval[text_type("connectionId")], int)
        self.assertIsNone(retval[text_type("err")])
        self.assertEquals(retval[text_type("n")], 1)
        self.assertTrue(retval[text_type("updatedExisting")])
        self.assertEquals(retval["ok"], 1.0)

        self.assertEquals(self.db.col.update({"bla": 1}, {"bla": 2})["n"], 0)

    def test__remove_retval(self):
        self.db.col.save({"a": 1})
        retval = self.db.col.remove({"a": 1})
        self.assertIsInstance(retval, dict)
        self.assertIsInstance(retval[text_type("connectionId")], int)
        self.assertIsNone(retval[text_type("err")])
        self.assertEquals(retval[text_type("n")], 1)
        self.assertEquals(retval[text_type("ok")], 1.0)

        self.assertEquals(self.db.col.remove({"bla": 1})["n"], 0)

    def test__getting_collection_via_getattr(self):
        col1 = self.db.some_collection_here
        col2 = self.db.some_collection_here
        self.assertIs(col1, col2)
        self.assertIs(col1, self.db['some_collection_here'])
        self.assertIsInstance(col1, mongomock.Collection)

    def test__save_class_deriving_from_dict(self):
        # See https://github.com/vmalloc/mongomock/issues/52
        class Document(dict):
            def __init__(self, collection):
                self.collection = collection
                super(Document, self).__init__()
                self.save()

            def save(self):
                self.collection.save(self)

        doc = Document(self.db.collection)
        self.assertIn("_id", doc)
        self.assertNotIn("collection", doc)

    def test__getting_collection_via_getitem(self):
        col1 = self.db['some_collection_here']
        col2 = self.db['some_collection_here']
        self.assertIs(col1, col2)
        self.assertIs(col1, self.db.some_collection_here)
        self.assertIsInstance(col1, mongomock.Collection)

    def test__cannot_save_non_string_keys(self):
        for key in [2, 2.0, True, object()]:
            with self.assertRaises(ValueError):
                self.db.col1.save({key: "value"})

    def test__find_returns_cursors(self):
        collection = self.db.collection
        self.assertEquals(type(collection.find()).__name__, "Cursor")
        self.assertNotIsInstance(collection.find(), list)
        self.assertNotIsInstance(collection.find(), tuple)

    def test__find_and_modify_cannot_remove_and_new(self):
        with self.assertRaises(mongomock.OperationFailure):
            self.db.collection.find_and_modify({}, remove=True, new=True)

    def test__find_and_modify_cannot_remove_and_update(self):
        with self.assertRaises(ValueError): # this is also what pymongo raises
            self.db.collection.find_and_modify({"a": 2}, {"a": 3}, remove=True)

    def test__update_interns_lists_and_dicts(self):
        obj = {}
        obj_id = self.db.collection.save(obj)
        d = {}
        l = []
        self.db.collection.update({"_id": obj_id}, {"d": d, "l": l})
        d["a"] = "b"
        l.append(1)
        self.assertEquals(list(self.db.collection.find()), [{"_id": obj_id, "d": {}, "l": []}])

    def test__string_matching(self):
        """
        Make sure strings are not treated as collections on find
        """
        self.db['abc'].save({'name':'test1'})
        self.db['abc'].save({'name':'test2'})
        #now searching for 'name':'e' returns test1
        self.assertIsNone(self.db['abc'].find_one({'name':'e'}))

    def test__collection_is_indexable(self):
        self.db['def'].save({'name':'test1'})
        self.assertTrue(self.db['def'].find({'name':'test1'}).count() > 0)
        self.assertEquals(self.db['def'].find({'name':'test1'})[0]['name'], 'test1')

    def test__cursor_distinct(self):
        larry_bob = {'name':'larry'}
        larry = {'name':'larry'}
        gary = {'name':'gary'}
        self.db['coll_name'].insert([larry_bob, larry, gary])
        ret_val = self.db['coll_name'].find().distinct('name')
        self.assertTrue(isinstance(ret_val,list))
        self.assertTrue(set(ret_val) == set(['larry','gary']))

    def test__find_with_skip_param(self):
        """
        Make sure that find() will take in account skip parametter
        """

        u1 = {'name': 'first'}
        u2 = {'name': 'second'}
        self.db['users'].insert([u1, u2])
        self.assertEquals(self.db['users'].find(skip=1).count(), 1)
        self.assertEquals(self.db['users'].find(skip=1)[0]['name'], 'second')

<commit_msg>Fix test<commit_after>import mongomock
from six import text_type

from .utils import TestCase

class CollectionAPITest(TestCase):
    def setUp(self):
        super(CollectionAPITest, self).setUp()
        self.conn = mongomock.Connection()
        self.db = self.conn['somedb']

    def test__get_subcollections(self):
        self.db.a.b
        self.assertEquals(self.db.a.b.full_name, "somedb.a.b")
        self.assertEquals(self.db.a.b.name, "a.b")

        self.assertEquals(
            set(self.db.collection_names()),
            set(["a.b", "system.indexes", "a"]))

    def test__get_collection_full_name(self):
        self.assertEquals(self.db.coll.name, "coll")
        self.assertEquals(self.db.coll.full_name, "somedb.coll")

    def test__get_collection_names(self):
        self.db.a
        self.db.b
        self.assertEquals(set(self.db.collection_names()), set(['a', 'b', 'system.indexes']))
        self.assertEquals(set(self.db.collection_names(True)), set(['a', 'b', 'system.indexes']))
        self.assertEquals(set(self.db.collection_names(False)), set(['a', 'b']))

    def test__cursor_collection(self):
        self.assertIs(self.db.a.find().collection, self.db.a)

    def test__drop_collection(self):
        self.db.a
        self.db.b
        self.db.c
        self.db.drop_collection('b')
        self.db.drop_collection('b')
        self.db.drop_collection(self.db.c)
        self.assertEquals(set(self.db.collection_names()), set(['a', 'system.indexes']))

    def test__distinct_nested_field(self):
        self.db.collection.insert({'f1': {'f2': 'v'}})
        cursor = self.db.collection.find()
        self.assertEquals(cursor.distinct('f1.f2'), ['v'])

    def test__cursor_clone(self):
        self.db.collection.insert([{"a": "b"}, {"b": "c"}, {"c": "d"}])
        cursor1 = self.db.collection.find()
        iterator1 = iter(cursor1)
        first_item = next(iterator1)
        cursor2 = cursor1.clone()
        iterator2 = iter(cursor2)
        self.assertEquals(next(iterator2), first_item)
        for item in iterator1:
            self.assertEquals(item, next(iterator2))

        with self.assertRaises(StopIteration):
            next(iterator2)

    def test__update_retval(self):
        self.db.col.save({"a": 1})
        retval = self.db.col.update({"a": 1}, {"b": 2})
        self.assertIsInstance(retval, dict)
        self.assertIsInstance(retval[text_type("connectionId")], int)
        self.assertIsNone(retval[text_type("err")])
        self.assertEquals(retval[text_type("n")], 1)
        self.assertTrue(retval[text_type("updatedExisting")])
        self.assertEquals(retval["ok"], 1.0)

        self.assertEquals(self.db.col.update({"bla": 1}, {"bla": 2})["n"], 0)

    def test__remove_retval(self):
        self.db.col.save({"a": 1})
        retval = self.db.col.remove({"a": 1})
        self.assertIsInstance(retval, dict)
        self.assertIsInstance(retval[text_type("connectionId")], int)
        self.assertIsNone(retval[text_type("err")])
        self.assertEquals(retval[text_type("n")], 1)
        self.assertEquals(retval[text_type("ok")], 1.0)

        self.assertEquals(self.db.col.remove({"bla": 1})["n"], 0)

    def test__getting_collection_via_getattr(self):
        col1 = self.db.some_collection_here
        col2 = self.db.some_collection_here
        self.assertIs(col1, col2)
        self.assertIs(col1, self.db['some_collection_here'])
        self.assertIsInstance(col1, mongomock.Collection)

    def test__save_class_deriving_from_dict(self):
        # See https://github.com/vmalloc/mongomock/issues/52
        class Document(dict):
            def __init__(self, collection):
                self.collection = collection
                super(Document, self).__init__()
                self.save()

            def save(self):
                self.collection.save(self)

        doc = Document(self.db.collection)
        self.assertIn("_id", doc)
        self.assertNotIn("collection", doc)

    def test__getting_collection_via_getitem(self):
        col1 = self.db['some_collection_here']
        col2 = self.db['some_collection_here']
        self.assertIs(col1, col2)
        self.assertIs(col1, self.db.some_collection_here)
        self.assertIsInstance(col1, mongomock.Collection)

    def test__cannot_save_non_string_keys(self):
        for key in [2, 2.0, True, object()]:
            with self.assertRaises(ValueError):
                self.db.col1.save({key: "value"})

    def test__find_returns_cursors(self):
        collection = self.db.collection
        self.assertEquals(type(collection.find()).__name__, "Cursor")
        self.assertNotIsInstance(collection.find(), list)
        self.assertNotIsInstance(collection.find(), tuple)

    def test__find_and_modify_cannot_remove_and_new(self):
        with self.assertRaises(mongomock.OperationFailure):
            self.db.collection.find_and_modify({}, remove=True, new=True)

    def test__find_and_modify_cannot_remove_and_update(self):
        with self.assertRaises(ValueError): # this is also what pymongo raises
            self.db.collection.find_and_modify({"a": 2}, {"a": 3}, remove=True)

    def test__update_interns_lists_and_dicts(self):
        obj = {}
        obj_id = self.db.collection.save(obj)
        d = {}
        l = []
        self.db.collection.update({"_id": obj_id}, {"d": d, "l": l})
        d["a"] = "b"
        l.append(1)
        self.assertEquals(list(self.db.collection.find()), [{"_id": obj_id, "d": {}, "l": []}])

    def test__string_matching(self):
        """
        Make sure strings are not treated as collections on find
        """
        self.db['abc'].save({'name':'test1'})
        self.db['abc'].save({'name':'test2'})
        #now searching for 'name':'e' returns test1
        self.assertIsNone(self.db['abc'].find_one({'name':'e'}))

    def test__collection_is_indexable(self):
        self.db['def'].save({'name':'test1'})
        self.assertTrue(self.db['def'].find({'name':'test1'}).count() > 0)
        self.assertEquals(self.db['def'].find({'name':'test1'})[0]['name'], 'test1')

    def test__cursor_distinct(self):
        larry_bob = {'name':'larry'}
        larry = {'name':'larry'}
        gary = {'name':'gary'}
        self.db['coll_name'].insert([larry_bob, larry, gary])
        ret_val = self.db['coll_name'].find().distinct('name')
        self.assertTrue(isinstance(ret_val,list))
        self.assertTrue(set(ret_val) == set(['larry','gary']))

    def test__find_with_skip_param(self):
        """
        Make sure that find() will take in account skip parametter
        """

        u1 = {'name': 'first'}
        u2 = {'name': 'second'}
        self.db['users'].insert([u1, u2])
        self.assertEquals(self.db['users'].find(sort=[("name", 1)], skip=1).count(), 1)
        self.assertEquals(self.db['users'].find(sort=[("name", 1)], skip=1)[0]['name'], 'second')
<|endoftext|>


<commit_before>import os
import unittest
import unicodecsv

from bson import json_util
from tests.config import config
from smappdragon import BsonCollection
from smappdragon import MongoCollection
from smappdragon import JsonCollection

class TestBaseCollection(unittest.TestCase):

	def test_base_top_entities_returns_dict(self):
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		returndict = collection.top_entities({'hashtags':5})
		self.assertTrue(isinstance(returndict, dict))

	def test_base_top_entities_returns_hashtags(self):
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		returndict = collection.top_entities({'hashtags':5})
		self.assertTrue('hashtags' in returndict)

	def test_base_top_entities_returns_hashtags_and_media(self):
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		returndict = collection.top_entities({'user_mentions':5, 'media':3})
		self.assertTrue('user_mentions' in returndict and 'media' in returndict)

	def test_base_top_entities_returns_counts(self):
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		returndict = collection.top_entities({'urls':5, 'symbols':3})
		if len(returndict['urls']) > 0:
			self.assertTrue(len(returndict['urls']) == 5)
		if len(returndict['symbols']) > 0:
			self.assertTrue(len(returndict['symbols']) == 3)

	def test_limit_is_set(self):
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		collection.set_limit(5)
		self.assertEqual(5, collection.limit)
		collection.set_limit(0)

	def test_limit_actually_limits(self):
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		collection.get_iterator()
		count = len(list(tweet for tweet in collection.set_limit(5).get_iterator()))
		self.assertEqual(5, count)

	def test_filter_is_set(self):
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		collection.set_filter({'a':'b', 'c':'d', 'e':{'f':'g', 'h':'i'}})
		self.assertEqual(collection.filter, {'a':'b', 'c':'d', 'e':{'f':'g', 'h':'i'}})

	def test_dump_to_bson_dumps(self):
		if os.path.exists(os.path.dirname(os.path.abspath(__file__))+'/bson/output.bson'):
			os.remove(os.path.dirname(os.path.abspath(__file__))+'/bson/output.bson')

		output_path = os.path.dirname(os.path.realpath(__file__)) + '/' + 'bson/output.bson'
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		collection.dump_to_bson(output_path)
		self.assertTrue(os.path.getsize(output_path) > 0)

		if os.path.exists(os.path.dirname(os.path.abspath(__file__))+'/bson/output.bson'):
			os.remove(os.path.dirname(os.path.abspath(__file__))+'/bson/output.bson')

	def test_dump_to_json_dumps(self):
		if os.path.exists(os.path.dirname(os.path.abspath(__file__))+'/bson/output.bson.json'):
			os.remove(os.path.dirname(os.path.abspath(__file__))+'/bson/output.bson.json')

		output_path = os.path.dirname(os.path.realpath(__file__)) + '/' + 'bson/output.bson.json'
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		collection.dump_to_json(output_path)
		self.assertTrue(os.path.getsize(output_path) > 0)

		if os.path.exists(os.path.dirname(os.path.abspath(__file__))+'/bson/output.bson.json'):
			os.remove(os.path.dirname(os.path.abspath(__file__))+'/bson/output.bson.json')

	def test_dump_to_csv_orders_and_encodes_properly(self):
		if os.path.exists(os.path.dirname(os.path.abspath(__file__))+'/bson/output.csv'):
			os.remove(os.path.dirname(os.path.abspath(__file__))+'/bson/output.csv')

		output_path = os.path.dirname(os.path.realpath(__file__)) + '/' + 'bson/output.csv'
		collection = JsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['json']['valid-single'])
		collection.dump_to_csv(output_path, ['id_str', 'entities.hashtags.0', 'entities.hashtags.1', 'source', 'user.id', 'timestamp.$date', 'text'])
		with open(os.path.dirname(os.path.abspath(__file__))+'/bson/output.csv', 'rb') as filehandle:
			count = 0
			for line in unicodecsv.reader(filehandle):
				if count != 0:
					val_count = 0
					for csv_row_value in line:
						everything_in_order = True
						if val_count == 0:
							self.assertEqual(csv_row_value, '661275583813431296')
						elif val_count == 1:
							loaded_dict = json_util.loads(csv_row_value)
							if not all(k in loaded_dict for k in ['text', 'indices']) and loaded_dict['text'] == 'jadehelm' and loaded_dict['indices'] == [74, 83]:
								everything_in_order = False
						elif val_count == 2:
							loaded_dict = json_util.loads(csv_row_value)
							if not all(k in loaded_dict for k in ['text', 'indices']) and loaded_dict['text'] == 'newworldorder' and loaded_dict['indices'] == [84, 98]:
								everything_in_order = False
						elif val_count == 3:
							self.assertEqual(csv_row_value, 'Colin\'s Autotweeterpro5.3')
						elif val_count == 4:
							self.assertEqual(csv_row_value, '379851447')
						elif val_count == 5:
							self.assertEqual(csv_row_value, '1446495359000')
						elif val_count == 6:
							self.assertEqual(csv_row_value, 'Susan Lindauer, Rtd US Army LTC Potter: Jade Helm https://t.co/VA4bQRudLt #jadehelm #newworldorder #usa #tyranny #threat')
						if everything_in_order:
							self.assertTrue(True)
						val_count += 1
				else:
					count += 1
		filehandle.close()

		if os.path.exists(os.path.dirname(os.path.abspath(__file__))+'/bson/output.csv'):
			os.remove(os.path.dirname(os.path.abspath(__file__))+'/bson/output.csv')

	def test_dump_to_csv_dumps(self):
		if os.path.exists(os.path.dirname(os.path.abspath(__file__))+'/bson/output.csv'):
			os.remove(os.path.dirname(os.path.abspath(__file__))+'/bson/output.csv')

		output_path = os.path.dirname(os.path.realpath(__file__)) + '/' + 'bson/output.csv'
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		collection.dump_to_csv(output_path, ['id_str', 'entities.hashtags.0', 'entities.hashtags.1'])
		self.assertTrue(os.path.getsize(output_path) > 0)

		if os.path.exists(os.path.dirname(os.path.abspath(__file__))+'/bson/output.csv'):
			os.remove(os.path.dirname(os.path.abspath(__file__))+'/bson/output.csv')

	def test_set_custom_filter_is_set(self):
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		def is_tweet_a_retweet(tweet):
			if 'retweeted' in tweet and tweet['retweeted']:
				return True
			else:
				return False
		collection.set_custom_filter(is_tweet_a_retweet)
		self.assertTrue(len(collection.custom_filters) == 1)

	def test_set_custom_filter_is_not_double_set(self):
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		def is_tweet_a_retweet(tweet):
			if 'retweeted' in tweet and tweet['retweeted']:
				return True
			else:
				return False
		collection.set_custom_filter(is_tweet_a_retweet)
		self.assertFalse(len(collection.custom_filters) > 1)

	def test_collection_filters_custom_filter_filters_something(self):
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		long_len = len(list(collection.get_iterator()))
		def is_tweet_a_retweet(tweet):
			if 'retweeted' in tweet and tweet['retweeted']:
				return True
			else:
				return False
		collection.set_custom_filter(is_tweet_a_retweet)
		shorter_len = len(list(collection.get_iterator()))

		#there should be fewer retweets than all tweets.
		self.assertTrue(long_len > shorter_len)

	def test_collection_filters_custom_filter_properly_applies_filter(self):
		collectionone = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		full_collection_len = len(list(collectionone.get_iterator()))
		def is_tweet_a_retweet(tweet):
			if 'retweeted' in tweet and tweet['retweeted']:
				return True
			else:
				return False
		num_retweets = len(list(collectionone.set_custom_filter(is_tweet_a_retweet).get_iterator()))

		collectiontwo = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		def is_not_a_retweet(tweet):
			if 'retweeted' in tweet and tweet['retweeted']:
				return False
			else:
				return True
		num_non_retweets = len(list(collectiontwo.set_custom_filter(is_not_a_retweet).get_iterator()))

		#the numbes of retweets and non retweets should add up to the whole collection
		self.assertEqual(num_retweets + num_non_retweets, full_collection_len)

	def test_strip_tweets_strips_many_tweets_totally(self):
		collectionone = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		iterator = collectionone.strip_tweets([]).get_iterator()
		first_tweet = next(iterator)
		second_tweet = next(iterator)
		#exhaust the iterator
		len(list(iterator))
		self.assertTrue(first_tweet == {} and second_tweet == {})

if __name__ == '__main__':
	unittest.main()
<commit_msg>add a base collection<commit_after>import os
import unittest
import unicodecsv

from bson import json_util
from tests.config import config
from smappdragon import BsonCollection
from smappdragon import MongoCollection
from smappdragon import JsonCollection

class TestBaseCollection(unittest.TestCase):

	def test_base_top_entities_returns_dict(self):
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		returndict = collection.top_entities({'hashtags':5})
		self.assertTrue(isinstance(returndict, dict))

	def test_base_top_entities_returns_hashtags(self):
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		returndict = collection.top_entities({'hashtags':5})
		self.assertTrue('hashtags' in returndict)

	def test_base_top_entities_returns_hashtags_and_media(self):
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		returndict = collection.top_entities({'user_mentions':5, 'media':3})
		self.assertTrue('user_mentions' in returndict and 'media' in returndict)

	def test_base_top_entities_returns_counts(self):
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		returndict = collection.top_entities({'urls':5, 'symbols':3})
		if len(returndict['urls']) > 0:
			self.assertTrue(len(returndict['urls']) == 5)
		if len(returndict['symbols']) > 0:
			self.assertTrue(len(returndict['symbols']) == 3)

	def test_limit_is_set(self):
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		collection.set_limit(5)
		self.assertEqual(5, collection.limit)
		collection.set_limit(0)

	def test_limit_actually_limits(self):
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		collection.get_iterator()
		count = len(list(tweet for tweet in collection.set_limit(5).get_iterator()))
		self.assertEqual(5, count)

	def test_filter_is_set(self):
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		collection.set_filter({'a':'b', 'c':'d', 'e':{'f':'g', 'h':'i'}})
		self.assertEqual(collection.filter, {'a':'b', 'c':'d', 'e':{'f':'g', 'h':'i'}})

	def test_dump_to_bson_dumps(self):
		if os.path.exists(os.path.dirname(os.path.abspath(__file__))+'/data/output.bson'):
			os.remove(os.path.dirname(os.path.abspath(__file__))+'/data/output.bson')

		output_path = os.path.dirname(os.path.realpath(__file__)) + '/' + 'data/output.bson'
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		collection.dump_to_bson(output_path)
		self.assertTrue(os.path.getsize(output_path) > 0)

		if os.path.exists(os.path.dirname(os.path.abspath(__file__))+'/data/output.bson'):
			os.remove(os.path.dirname(os.path.abspath(__file__))+'/data/output.bson')

	def test_dump_to_json_dumps(self):
		if os.path.exists(os.path.dirname(os.path.abspath(__file__))+'/data/output.bson.json'):
			os.remove(os.path.dirname(os.path.abspath(__file__))+'/data/output.bson.json')

		output_path = os.path.dirname(os.path.realpath(__file__)) + '/' + 'bson/output.bson.json'
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		collection.dump_to_json(output_path)
		self.assertTrue(os.path.getsize(output_path) > 0)

		if os.path.exists(os.path.dirname(os.path.abspath(__file__))+'/data/output.bson.json'):
			os.remove(os.path.dirname(os.path.abspath(__file__))+'/data/output.bson.json')

	def test_dump_to_csv_orders_and_encodes_properly(self):
		if os.path.exists(os.path.dirname(os.path.abspath(__file__))+'/data/output.csv'):
			os.remove(os.path.dirname(os.path.abspath(__file__))+'/data/output.csv')

		output_path = os.path.dirname(os.path.realpath(__file__)) + '/' + 'data/output.csv'
		collection = JsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['json']['valid-single'])
		collection.dump_to_csv(output_path, ['id_str', 'entities.hashtags.0', 'entities.hashtags.1', 'source', 'user.id', 'timestamp.$date', 'text'])
		with open(os.path.dirname(os.path.abspath(__file__))+'/data/output.csv', 'rb') as filehandle:
			count = 0
			for line in unicodecsv.reader(filehandle):
				if count != 0:
					val_count = 0
					for csv_row_value in line:
						everything_in_order = True
						if val_count == 0:
							self.assertEqual(csv_row_value, '661275583813431296')
						elif val_count == 1:
							loaded_dict = json_util.loads(csv_row_value)
							if not all(k in loaded_dict for k in ['text', 'indices']) and loaded_dict['text'] == 'jadehelm' and loaded_dict['indices'] == [74, 83]:
								everything_in_order = False
						elif val_count == 2:
							loaded_dict = json_util.loads(csv_row_value)
							if not all(k in loaded_dict for k in ['text', 'indices']) and loaded_dict['text'] == 'newworldorder' and loaded_dict['indices'] == [84, 98]:
								everything_in_order = False
						elif val_count == 3:
							self.assertEqual(csv_row_value, 'Colin\'s Autotweeterpro5.3')
						elif val_count == 4:
							self.assertEqual(csv_row_value, '379851447')
						elif val_count == 5:
							self.assertEqual(csv_row_value, '1446495359000')
						elif val_count == 6:
							self.assertEqual(csv_row_value, 'Susan Lindauer, Rtd US Army LTC Potter: Jade Helm https://t.co/VA4bQRudLt #jadehelm #newworldorder #usa #tyranny #threat')
						if everything_in_order:
							self.assertTrue(True)
						val_count += 1
				else:
					count += 1
		filehandle.close()

		if os.path.exists(os.path.dirname(os.path.abspath(__file__))+'/data/output.csv'):
			os.remove(os.path.dirname(os.path.abspath(__file__))+'/data/output.csv')

	def test_dump_to_csv_dumps(self):
		if os.path.exists(os.path.dirname(os.path.abspath(__file__))+'/data/output.csv'):
			os.remove(os.path.dirname(os.path.abspath(__file__))+'/data/output.csv')

		output_path = os.path.dirname(os.path.realpath(__file__)) + '/' + 'data/output.csv'
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		collection.dump_to_csv(output_path, ['id_str', 'entities.hashtags.0', 'entities.hashtags.1'])
		self.assertTrue(os.path.getsize(output_path) > 0)

		if os.path.exists(os.path.dirname(os.path.abspath(__file__))+'/data/output.csv'):
			os.remove(os.path.dirname(os.path.abspath(__file__))+'/data/output.csv')

	def test_set_custom_filter_is_set(self):
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		def is_tweet_a_retweet(tweet):
			if 'retweeted' in tweet and tweet['retweeted']:
				return True
			else:
				return False
		collection.set_custom_filter(is_tweet_a_retweet)
		self.assertTrue(len(collection.custom_filters) == 1)

	def test_set_custom_filter_is_not_double_set(self):
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		def is_tweet_a_retweet(tweet):
			if 'retweeted' in tweet and tweet['retweeted']:
				return True
			else:
				return False
		collection.set_custom_filter(is_tweet_a_retweet)
		self.assertFalse(len(collection.custom_filters) > 1)

	def test_collection_filters_custom_filter_filters_something(self):
		collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		long_len = len(list(collection.get_iterator()))
		def is_tweet_a_retweet(tweet):
			if 'retweeted' in tweet and tweet['retweeted']:
				return True
			else:
				return False
		collection.set_custom_filter(is_tweet_a_retweet)
		shorter_len = len(list(collection.get_iterator()))

		#there should be fewer retweets than all tweets.
		self.assertTrue(long_len > shorter_len)

	def test_collection_filters_custom_filter_properly_applies_filter(self):
		collectionone = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		full_collection_len = len(list(collectionone.get_iterator()))
		def is_tweet_a_retweet(tweet):
			if 'retweeted' in tweet and tweet['retweeted']:
				return True
			else:
				return False
		num_retweets = len(list(collectionone.set_custom_filter(is_tweet_a_retweet).get_iterator()))

		collectiontwo = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		def is_not_a_retweet(tweet):
			if 'retweeted' in tweet and tweet['retweeted']:
				return False
			else:
				return True
		num_non_retweets = len(list(collectiontwo.set_custom_filter(is_not_a_retweet).get_iterator()))

		#the numbes of retweets and non retweets should add up to the whole collection
		self.assertEqual(num_retweets + num_non_retweets, full_collection_len)

	def test_strip_tweets_strips_many_tweets_totally(self):
		collectionone = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid'])
		iterator = collectionone.strip_tweets([]).get_iterator()
		first_tweet = next(iterator)
		second_tweet = next(iterator)
		#exhaust the iterator
		len(list(iterator))
		self.assertTrue(first_tweet == {} and second_tweet == {})

if __name__ == '__main__':
	unittest.main()
<|endoftext|>