<commit_before>#!/usr/bin/env python
"""

"""
import os, glob,getopt,sys
import scipy as sp
import matplotlib
matplotlib.use('Agg') # for use where you're running on a command line
import pdb
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
from GeoData.plotting import scatterGD, slice2DGD,insertinfo
from GeoData.GeoData import GeoData
from GeoData.utilityfuncs import readIonofiles, readAllskyFITS,readSRI_h5

def main(allskydir,ionofdir,plotdir,wl = str(558),tint=5,reinterp=False):

    wlstr ='*_0'+wl+'_*.FITS'

    TEClist = []
    TECfiles = glob.glob(os.path.join(ionofdir,'*.iono'))
    TECtime = [sp.Inf,-sp.Inf];
    Geolatlim = [sp.Inf,-sp.Inf];
    Geolonlim = [sp.Inf,-sp.Inf];
    for ifile in TECfiles:
        TECGD = GeoData(readIonofiles,(ifile,))
        TEClist.append(TECGD)
        TECtime[0] = min(min(TECGD.times[:,0]),TECtime[0])
        TECtime[1] = max(max(TECGD.times[:,0]),TECtime[1])
        Geolatlim[0] = min(min(TECGD.dataloc[:,0]),Geolatlim[0])
        Geolatlim[1] = max(max(TECGD.dataloc[:,0]),Geolatlim[1])

        Geolonlim[0] = min(min(TECGD.dataloc[:,1]),Geolonlim[0])
        Geolonlim[1] = max(max(TECGD.dataloc[:,1]),Geolonlim[1])

    latlim2 = [45.,75.]
    lonlim2 = [-185.,-125.]
    interpsavedfile = os.path.join(allskydir,'interp'+wl+'.h5')
    if reinterp or (not os.path.isfile(interpsavedfile)):
        pfalla = sp.array([65.136667,-147.447222,689.])

        flist558 = glob.glob(os.path.join(allskydir,wlstr))
        allsky_data = GeoData(readAllskyFITS,(flist558,'PKR_DASC_20110112_AZ_10deg.FITS','PKR_DASC_20110112_EL_10deg.FITS',150.,pfalla))

            # reduce the size of the allskydata
        allskytime = allsky_data.times[:,0]
        allsky_data=allsky_data.timeslice(sp.where(sp.logical_and(allskytime>=TECtime[0],allskytime<TECtime[1] ))[0])
        allskytime=allsky_data.times[:,0]
        latlim2 = [45.,75.]
        lonlim2 = [-185.,-125.]
        xcoords = allsky_data.__changecoords__('WGS84')
        latlim=[xcoords[:,0].min(),xcoords[:,0].max()]
        lonlim=[xcoords[:,1].min(),xcoords[:,1].max()]
        nlat = 256
        nlon = 256

        latvec = sp.linspace(latlim[0],latlim[1],nlat)
        lonvec = sp.linspace(lonlim[0],lonlim[1],nlon)
        [LATM,LONM] = sp.meshgrid(latvec,lonvec)

        newcoords = sp.column_stack((LATM.flatten(),LONM.flatten(),150.*sp.ones(LONM.size)))
        allsky_data.interpolate(newcoords,'WGS84',method='linear',twodinterp=True)
        allsky_data.write_h5(interpsavedfile)
    else:
        allsky_data = GeoData.read_h5(interpsavedfile)
        allskytime=allsky_data.times[:,0]
    #%% Make map
    fig = plt.figure(figsize=(8,8))
    ax = fig.add_axes([0.1,0.1,0.8,0.8])
    # create polar stereographic Basemap instance.
    m = Basemap(projection='merc',lon_0=sp.mean(lonlim2),lat_0=sp.mean(latlim2),\
        lat_ts=sp.mean(latlim2),llcrnrlat=latlim2[0],urcrnrlat=latlim2[1],\
        llcrnrlon=lonlim2[0],urcrnrlon=lonlim2[1],\
        rsphere=6371200.,resolution='i',ax=ax)
    # draw coastlines, state and country boundaries, edge of map.
    #m.drawcoastlines()
#    m.drawstates()
#    m.drawcountries()
    shp_info = m.readshapefile('st99_d00','states',drawbounds=True)
    meridians=sp.arange(lonlim2[0],lonlim2[1],10.)
    parallels = sp.arange(latlim2[0],latlim2[1],5.)
    parhand=m.drawparallels(parallels,labels=[1,0,0,0],fontsize=10)
    mrdhand = m.drawmeridians(meridians,labels=[0,0,0,1],fontsize=10)
    plt.hold(True)
    #%% make lists for plotting
    tectime = sp.arange(TECtime[0],TECtime[1],60.*tint)
    nptimes= len(tectime)

    allskylist = []
    gpslist = []
    tlistkeep = sp.zeros(nptimes-1,dtype=bool)
    for itasn in range(nptimes-1):
        techlist = []
        tlistemp=True
        itback=tectime[itasn]
        itfor = tectime[itasn+1]
        itas = sp.where(sp.logical_and(allskytime>=itback, allskytime<itfor))[0]
        if len(itas)==0:
            continue

        for k in range(len(TEClist)):
            Geoone=TEClist[k];
            timevec = Geoone.times[:,0];

            itgps = sp.where(sp.logical_and(timevec>=itback, timevec<itfor))[0]
            if len(itgps)>0:
                tlistemp=False
            techlist.append(itgps)
        if tlistemp:
            continue
        allskylist.append(itas)
        gpslist.append(techlist)
        tlistkeep[itasn]=True
    plotgpsnoptics(allsky_data,TEClist,allskylist,gpslist,plotdir,m,ax,fig)
    plt.close(fig)

def plotgpsnoptics(allsky_data,TEClist,allskylist,gpslist,plotdir,m,ax,fig):
    maxplot = len(allsky_data.times)
    strlen = int(sp.ceil(sp.log10(maxplot))+1)
    fmstr = '{0:0>'+str(strlen)+'}_'
    plotnum=0
    for (optic_times,gps_cur)in zip(allskylist,gpslist):
        gpshands = []
        gpsmin = sp.inf
        gpsmax = -sp.inf
        for igpsn, (igps,igpslist) in enumerate(zip(TEClist,gps_cur)):
            print('Plotting GPS data from rec {0} of {1}'.format(igpsn,len(gps_cur)))
            # check if there's anything to plot
            if len(igpslist)==0:
                continue

            (sctter,scatercb) = scatterGD(igps,'alt',3.5e5,vbounds=[0,15],time = igpslist,gkey = 'vTEC',cmap='jet',fig=fig,
                  ax=ax,title='',cbar=True,err=.1,m=m)
            gpsmin = sp.minimum(igps.times[igpslist,0].min(),gpsmin)
            gpsmax = sp.maximum(igps.times[igpslist,1].max(),gpsmax)
            gpshands.append(sctter)
        scatercb.set_label('vTEC in TECu')
        #change he z order
        minz = gpshands[0].get_zorder()
        for i in reversed(gpshands):
            i.set_zorder(i.get_zorder()+1)

        for iop in optic_times:
            (slice3,cbar3) = slice2DGD(allsky_data,'alt',150,[100,800],title='',
                                time = iop,cmap='Greens',gkey = 'image',fig=fig,ax=ax,cbar=False,m=m)
            slice3.set_zorder(minz)
            plt.title(insertinfo('GPS $tmdy $thmsehms',posix=gpsmin,posixend=gpsmax)+'\n'+insertinfo('All Sky $tmdy $thmsehms',posix=allsky_data.times[iop,0],posixend=allsky_data.times[iop,1]))
            print('Ploting {0} of {1} plots'.format(plotnum,maxplot))
            plt.savefig(os.path.join(plotdir,fmstr.format(plotnum)+'ASwGPS.png'))

            plotnum+=1
            slice3.remove()
        for i in reversed(gpshands):
            i.remove()

def getSRIhdf5(filename):
    """ """
    paramstr = ['Ne','Ti','Ti']
    SRIh5 = GeoData(readSRI_h5,(filename,paramstr))
if __name__== '__main__':
    argv = sys.argv[1:]

    outstr = 'plotdata.py -a <all skypath> -w <wavelength>, -i <ionofile dir>, -t <time interval>,  -p <plotdirectory> -r <type y to reinterpolate all sky data>'

    try:
        opts, args = getopt.gnu_getopt(argv,"ha:w:i:t:r:p:")
    except getopt.GetoptError:
        print(outstr)
        sys.exit(2)

    remakealldata = False
    for opt, arg in opts:
        if opt == '-h':
            print(outstr)
            sys.exit()
        elif opt in ("-i", "--ifile"):
            ionofdir = arg

        elif opt in ("-w", "--wlen"):
            wl = arg
        elif opt in ("-a", "--asky"):
            allskydir=arg
        elif opt in ("-t", "--tin"):
            tint=float(arg)
        elif opt in ("-p", "--pdir"):
            plotdir=arg
        elif opt in ('-r', "--re"):
            if arg.lower() == 'y':
                remakealldata = True

#    plotdir = os.path.expanduser('~/Documents/python/mahali/plots10172015')
    if not os.path.isdir(plotdir):
        os.mkdir(plotdir)
#    allskydir = os.path.expanduser('~/DATA/Mahali/allsky')
#    ionofdir = os.path.expanduser('~/DATA/Mahali/GPS_Processed_IonoFiles/site_280_2015')
    cmdrun=True
    if cmdrun:
        matplotlib.use('Agg') # for use where you're running on a command line

    main(allskydir,ionofdir,plotdir,wl = wl,tint=tint,reinterp=remakealldata)
<commit_msg>added stuff for sri data<commit_after>#!/usr/bin/env python
"""

"""
import os, glob,getopt,sys
import scipy as sp
import matplotlib
matplotlib.use('Agg') # for use where you're running on a command line
import pdb
import matplotlib.pyplot as plt
import pytz
from datetime import datetime
from dateutil import parser
from mpl_toolkits.basemap import Basemap
from GeoData.plotting import scatterGD, slice2DGD,insertinfo
from GeoData.GeoData import GeoData
from GeoData.utilityfuncs import readIonofiles, readAllskyFITS,readSRI_h5

def main(allskydir,ionofdir,plotdir,wl = str(558),tint=5,reinterp=False):

    wlstr ='*_0'+wl+'_*.FITS'

    TEClist = []
    TECfiles = glob.glob(os.path.join(ionofdir,'*.iono'))
    TECtime = [sp.Inf,-sp.Inf];
    Geolatlim = [sp.Inf,-sp.Inf];
    Geolonlim = [sp.Inf,-sp.Inf];
    for ifile in TECfiles:
        TECGD = GeoData(readIonofiles,(ifile,))
        TEClist.append(TECGD)
        TECtime[0] = min(min(TECGD.times[:,0]),TECtime[0])
        TECtime[1] = max(max(TECGD.times[:,0]),TECtime[1])
        Geolatlim[0] = min(min(TECGD.dataloc[:,0]),Geolatlim[0])
        Geolatlim[1] = max(max(TECGD.dataloc[:,0]),Geolatlim[1])

        Geolonlim[0] = min(min(TECGD.dataloc[:,1]),Geolonlim[0])
        Geolonlim[1] = max(max(TECGD.dataloc[:,1]),Geolonlim[1])

    latlim2 = [45.,75.]
    lonlim2 = [-185.,-125.]
    interpsavedfile = os.path.join(allskydir,'interp'+wl+'.h5')
    if reinterp or (not os.path.isfile(interpsavedfile)):
        pfalla = sp.array([65.136667,-147.447222,689.])

        flist558 = glob.glob(os.path.join(allskydir,wlstr))
        allsky_data = GeoData(readAllskyFITS,(flist558,'PKR_DASC_20110112_AZ_10deg.FITS','PKR_DASC_20110112_EL_10deg.FITS',150.,pfalla))

            # reduce the size of the allskydata
        allskytime = allsky_data.times[:,0]
        allsky_data=allsky_data.timeslice(sp.where(sp.logical_and(allskytime>=TECtime[0],allskytime<TECtime[1] ))[0])
        allskytime=allsky_data.times[:,0]
        latlim2 = [45.,75.]
        lonlim2 = [-185.,-125.]
        xcoords = allsky_data.__changecoords__('WGS84')
        latlim=[xcoords[:,0].min(),xcoords[:,0].max()]
        lonlim=[xcoords[:,1].min(),xcoords[:,1].max()]
        nlat = 256
        nlon = 256

        latvec = sp.linspace(latlim[0],latlim[1],nlat)
        lonvec = sp.linspace(lonlim[0],lonlim[1],nlon)
        [LATM,LONM] = sp.meshgrid(latvec,lonvec)

        newcoords = sp.column_stack((LATM.flatten(),LONM.flatten(),150.*sp.ones(LONM.size)))
        allsky_data.interpolate(newcoords,'WGS84',method='linear',twodinterp=True)
        allsky_data.write_h5(interpsavedfile)
    else:
        allsky_data = GeoData.read_h5(interpsavedfile)
        allskytime=allsky_data.times[:,0]
    #%% Make map
    fig = plt.figure(figsize=(8,8))
    ax = fig.add_axes([0.1,0.1,0.8,0.8])
    # create polar stereographic Basemap instance.
    m = Basemap(projection='merc',lon_0=sp.mean(lonlim2),lat_0=sp.mean(latlim2),\
        lat_ts=sp.mean(latlim2),llcrnrlat=latlim2[0],urcrnrlat=latlim2[1],\
        llcrnrlon=lonlim2[0],urcrnrlon=lonlim2[1],\
        rsphere=6371200.,resolution='i',ax=ax)
    # draw coastlines, state and country boundaries, edge of map.
    #m.drawcoastlines()
#    m.drawstates()
#    m.drawcountries()
    shp_info = m.readshapefile('st99_d00','states',drawbounds=True)
    meridians=sp.arange(lonlim2[0],lonlim2[1],10.)
    parallels = sp.arange(latlim2[0],latlim2[1],5.)
    parhand=m.drawparallels(parallels,labels=[1,0,0,0],fontsize=10)
    mrdhand = m.drawmeridians(meridians,labels=[0,0,0,1],fontsize=10)
    plt.hold(True)
    #%% make lists for plotting
    tectime = sp.arange(TECtime[0],TECtime[1],60.*tint)
    nptimes= len(tectime)

    allskylist = []
    gpslist = []
    tlistkeep = sp.zeros(nptimes-1,dtype=bool)
    for itasn in range(nptimes-1):
        techlist = []
        tlistemp=True
        itback=tectime[itasn]
        itfor = tectime[itasn+1]
        itas = sp.where(sp.logical_and(allskytime>=itback, allskytime<itfor))[0]
        if len(itas)==0:
            continue

        for k in range(len(TEClist)):
            Geoone=TEClist[k];
            timevec = Geoone.times[:,0];

            itgps = sp.where(sp.logical_and(timevec>=itback, timevec<itfor))[0]
            if len(itgps)>0:
                tlistemp=False
            techlist.append(itgps)
        if tlistemp:
            continue
        allskylist.append(itas)
        gpslist.append(techlist)
        tlistkeep[itasn]=True
    plotgpsnoptics(allsky_data,TEClist,allskylist,gpslist,plotdir,m,ax,fig)
    plt.close(fig)

def plotgpsnoptics(allsky_data,TEClist,allskylist,gpslist,plotdir,m,ax,fig):
    maxplot = len(allsky_data.times)
    strlen = int(sp.ceil(sp.log10(maxplot))+1)
    fmstr = '{0:0>'+str(strlen)+'}_'
    plotnum=0
    for (optic_times,gps_cur)in zip(allskylist,gpslist):
        gpshands = []
        gpsmin = sp.inf
        gpsmax = -sp.inf
        for igpsn, (igps,igpslist) in enumerate(zip(TEClist,gps_cur)):
            print('Plotting GPS data from rec {0} of {1}'.format(igpsn,len(gps_cur)))
            # check if there's anything to plot
            if len(igpslist)==0:
                continue

            (sctter,scatercb) = scatterGD(igps,'alt',3.5e5,vbounds=[0,15],time = igpslist,gkey = 'vTEC',cmap='jet',fig=fig,
                  ax=ax,title='',cbar=True,err=.1,m=m)
            gpsmin = sp.minimum(igps.times[igpslist,0].min(),gpsmin)
            gpsmax = sp.maximum(igps.times[igpslist,1].max(),gpsmax)
            gpshands.append(sctter)
        scatercb.set_label('vTEC in TECu')
        #change he z order
        minz = gpshands[0].get_zorder()
        for i in reversed(gpshands):
            i.set_zorder(i.get_zorder()+1)

        for iop in optic_times:
            (slice3,cbar3) = slice2DGD(allsky_data,'alt',150,[100,800],title='',
                                time = iop,cmap='Greens',gkey = 'image',fig=fig,ax=ax,cbar=False,m=m)
            slice3.set_zorder(minz)
            plt.title(insertinfo('GPS $tmdy $thmsehms',posix=gpsmin,posixend=gpsmax)+'\n'+insertinfo('All Sky $tmdy $thmsehms',posix=allsky_data.times[iop,0],posixend=allsky_data.times[iop,1]))
            print('Ploting {0} of {1} plots'.format(plotnum,maxplot))
            plt.savefig(os.path.join(plotdir,fmstr.format(plotnum)+'ASwGPS.png'))

            plotnum+=1
            slice3.remove()
        for i in reversed(gpshands):
            i.remove()

def getSRIhdf5(filename,times,heights):
    """ """
    paramstr = ['Ne','Ti','Te']
    SRIh5 = GeoData(readSRI_h5,(filename,paramstr))
    (dt1,dt2) = parser.parse(times[0]),parser.parse(times[1])
    dt1 =dt1.replace(tzinfo=pytz.utc)
    dt2 = dt2.replace(tzinfo=pytz.utc)
    dt1ts = (dt1 -datetime(1970,1,1,0,0,0,tzinfo=pytz.utc)).total_seconds()
    dt2ts = (dt2 -datetime(1970,1,1,0,0,0,tzinfo=pytz.utc)).total_seconds()

if __name__== '__main__':
    argv = sys.argv[1:]

    outstr = 'plotdata.py -a <all skypath> -w <wavelength>, -i <ionofile dir>, -t <time interval>,  -p <plotdirectory> -r <type y to reinterpolate all sky data>'

    try:
        opts, args = getopt.gnu_getopt(argv,"ha:w:i:t:r:p:")
    except getopt.GetoptError:
        print(outstr)
        sys.exit(2)

    remakealldata = False
    for opt, arg in opts:
        if opt == '-h':
            print(outstr)
            sys.exit()
        elif opt in ("-i", "--ifile"):
            ionofdir = arg

        elif opt in ("-w", "--wlen"):
            wl = arg
        elif opt in ("-a", "--asky"):
            allskydir=arg
        elif opt in ("-t", "--tin"):
            tint=float(arg)
        elif opt in ("-p", "--pdir"):
            plotdir=arg
        elif opt in ('-r', "--re"):
            if arg.lower() == 'y':
                remakealldata = True

#    plotdir = os.path.expanduser('~/Documents/python/mahali/plots10172015')
    if not os.path.isdir(plotdir):
        os.mkdir(plotdir)
#    allskydir = os.path.expanduser('~/DATA/Mahali/allsky')
#    ionofdir = os.path.expanduser('~/DATA/Mahali/GPS_Processed_IonoFiles/site_280_2015')
    cmdrun=True
    if cmdrun:
        matplotlib.use('Agg') # for use where you're running on a command line

    main(allskydir,ionofdir,plotdir,wl = wl,tint=tint,reinterp=remakealldata)
<|endoftext|>


<commit_before># vim: set fileencoding=utf-8 :
import pycarddav.pycard

vcard1 = """BEGIN:VCARD
VERSION:3.0
N:Gödel;François
FN:François Gödel
TEL;TYPE=WORK,VOICE:+49-123-678901
TEL;TYPE=HOME,VOICE:(101) 1234 4123
ADR;TYPE=WORK:;;Essalág 100;Torshavn;50800;Færøerne
EMAIL;TYPE=PREF,INTERNET:francois@goedel.net
END:VCARD
"""

vcard1_vcf = """BEGIN:VCARD
VERSION:3.0
FN:François Gödel
N:Gödel;François;;;
ADR;TYPE=WORK:;;Essalág 100;Torshavn;50800;Færøerne;
EMAIL;TYPE=PREF,INTERNET:francois@goedel.net
TEL;TYPE=WORK,VOICE:+49-123-678901
TEL;TYPE=HOME,VOICE:(101) 1234 4123
END:VCARD"""

def get_vcard1():
    return pycarddav.pycard.vcard_from_string(vcard1)

def test_serialize_to_vcf():
    assert get_vcard1().vcf.encode('utf-8') == vcard1_vcf
<commit_msg>THE test should pass again<commit_after># vim: set fileencoding=utf-8 :
import pycarddav.model

vcard1 = """BEGIN:VCARD
VERSION:3.0
N:Gödel;François
FN:François Gödel
TEL;TYPE=WORK,VOICE:+49-123-678901
TEL;TYPE=HOME,VOICE:(101) 1234 4123
ADR;TYPE=WORK:;;Essalág 100;Torshavn;50800;Færøerne
EMAIL;TYPE=PREF,INTERNET:francois@goedel.net
END:VCARD
"""

vcard1_vcf = """BEGIN:VCARD
VERSION:3.0
FN:François Gödel
N:Gödel;François;;;
ADR;TYPE=WORK:;;Essalág 100;Torshavn;50800;Færøerne;
EMAIL;TYPE=PREF,INTERNET:francois@goedel.net
TEL;TYPE=WORK,VOICE:+49-123-678901
TEL;TYPE=HOME,VOICE:(101) 1234 4123
END:VCARD"""

def get_vcard1():
    return pycarddav.model.vcard_from_string(vcard1)

def test_serialize_to_vcf():
    assert get_vcard1().vcf.encode('utf-8') == vcard1_vcf
<|endoftext|>


<commit_before>"""Context processor for checking if in deployed environment."""

from django.conf import settings

def deployed(request):
    """Return a dictionary containing boolean if deployed environment.
    Returns:
        Dictionary containing deployed boolean to add to context.
    """
    return {
        "DEPLOYED": settings.DJANGO_PRODUCTION
    }
<commit_msg>PEP8 fix<commit_after>"""Context processor for checking if in deployed environment."""

from django.conf import settings

def deployed(request):
    """Return a dictionary containing boolean if deployed environment.

    Args:
        request (Request): The HTTP request.

    Returns:
        Dictionary containing deployed boolean to add to context.
    """
    return {
        "DEPLOYED": settings.DJANGO_PRODUCTION
    }
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from future import standard_library
standard_library.install_aliases()
from builtins import range
from builtins import *
import enum
import pandas as pd
import numpy as np
import arrow as arrow
import logging

import emission.storage.decorations.analysis_timeseries_queries as esda
import emission.storage.decorations.local_date_queries as esdl

import emission.storage.timeseries.tcquery as esttc
import emission.storage.timeseries.timequery as estt
import emission.storage.timeseries.abstract_timeseries as esta
import emission.storage.timeseries.aggregate_timeseries as estag
import emission.core.wrapper.motionactivity as ecwm
import emission.core.wrapper.modestattimesummary as ecwms
import emission.core.wrapper.modeprediction as ecwmp
import emission.core.wrapper.localdate as ecwl

import emission.analysis.config as eac

def group_by_timestamp(user_id, start_ts, end_ts, freq, summary_fn_list):
    """
    Get grouped dataframes for the specific time range and at the specified frequency
    :param user_id: The user for whom we are computing this information. None for all users.
    :param from_ld: The start timestamp
    :param to_ld: The end timestamp
    :param freq: The frequency as specified in a pandas date_range frequency string.
    We only support frequencies of a day or longer in order to return the data
    in a format that makes sense
    http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
    The canonical list can be found at:
    > pandas.tseries.offsets.prefix_mapping
    :return: a dict containing the last start_ts of the last section processed
        and a result list of ModeStatTimeSummary objects
        If there were no matching sections, the last start_ts is None
        and the list is empty.
    """
    time_query = estt.TimeQuery("data.start_ts", start_ts, end_ts)
    section_df = esda.get_data_df(eac.get_section_key_for_analysis_results(),
                                  user_id=user_id, time_query=time_query,
                                  geo_query=None)
    if len(section_df) == 0:
        logging.info("Found no entries for user %s, time_query %s" % (user_id, time_query))
        return {
            "last_ts_processed": None,
            "result": [[] for i in range(len(summary_fn_list))]
        }
    logging.debug("first row is %s" % section_df.iloc[0])
    secs_to_nanos = lambda x: x * 10 ** 9
    section_df['start_dt'] = pd.to_datetime(secs_to_nanos(section_df.start_ts))
    adjusted_df = adjust_for_user_inputs_if_needed(section_df)
    time_grouped_df = adjusted_df.groupby(pd.Grouper(freq=freq, key='start_dt'))
    return {
        "last_ts_processed": adjusted_df.iloc[-1].start_ts,
        "result": [grouped_to_summary(time_grouped_df, timestamp_fill_times, summary_fn)
                   for summary_fn in summary_fn_list]
    }

def timestamp_fill_times(key, ignored, metric_summary):
    dt = arrow.get(key)
    metric_summary.ts = dt.timestamp
    metric_summary.local_dt = ecwl.LocalDate.get_local_date(dt.timestamp, 'UTC')
    metric_summary.fmt_time = dt.isoformat()

class LocalFreq(enum.Enum):
    DAILY = 0
    MONTHLY = 1
    YEARLY = 2

def group_by_local_date(user_id, from_dt, to_dt, freq, summary_fn_list):
    """
    Get grouped data frames for the specified local date range and frequency
    :param user_id: id for the user. None for aggregate.
    :param from_dt: start local dt object. We assume that only the year, month
    and date entries are filled in and represent a date range.
    :param to_dt: end local dt object. We assume that only the year, month
    and date entries are filled in and represent a date range.
    :param freq: since we only expand certain local_dt fields, we can only
    support frequencies corresponding to them. These are represented in the
    `LocalFreq` enum.
    :return: a dict containing the last start_ts of the last section processed
        and a result list of ModeStatTimeSummary objects
        If there were no matching sections, the last start_ts is None
        and the list is empty.
    """
    time_query = esttc.TimeComponentQuery("data.start_local_dt", from_dt, to_dt)
    section_df = esda.get_data_df(eac.get_section_key_for_analysis_results(),
                                  user_id=user_id, time_query=time_query,
                                  geo_query=None)
    if len(section_df) == 0:
        logging.info("Found no entries for user %s, time_query %s" % (user_id, time_query))
        return {
            "last_ts_processed": None,
            "result": [[] for i in range(len(summary_fn_list))]
        }

    groupby_arr = _get_local_group_by(freq)
    time_grouped_df = section_df.groupby(groupby_arr)
    local_dt_fill_fn = _get_local_key_to_fill_fn(freq)
    return {
        "last_ts_processed": section_df.iloc[-1].start_ts,
        "result": [grouped_to_summary(time_grouped_df, local_dt_fill_fn, summary_fn)
                        for summary_fn in summary_fn_list]
    }

# by default, the incoming values for the keys are `numpy.int64` for reasons
# that I don't understand. This breaks serialization
# (https://github.com/e-mission/e-mission-docs/issues/530)
# converting to regular ints to avoid this issue

def fix_int64_key_if_needed(key):
    if isinstance(key, tuple):
        logging.debug("Converting %d fields from int64 to regular integer" % len(key))
        # print("before conversion, types = %s" % str(tuple([type(k) for k in key])))
        mod_keys = tuple([int(k) for k in key])
        # print("after conversion, types = %s" % str(tuple([type(k) for k in key])))
        return mod_keys
    else:
        return key

def adjust_for_user_inputs_if_needed(section_df):
    result_section_key = eac.get_section_key_for_analysis_results()
    if result_section_key == "analysis/confirmed_trip":
        import emission.storage.decorations.trip_queries as esdt
        adjusted_df = esdt.expand_finallabels(section_df)
        # if none of the trips in the time grouping are labeled,
        # we add a dummy column
        # pandas checks in TestMetricsConfirmedTripsPandas.testPandasConcatModeConfirm
        if "mode_confirm" not in adjusted_df.columns:
            # If we don't reset the index and the index doesn't start from 1,
            # the concat will end up with additional rows
            # see TestMetricsConfirmedTripsPandas.testPandasConcatModeConfirm
            adjusted_df.reset_index(inplace=True)
            dummy_col = pd.Series([np.NaN] * len(adjusted_df), name="mode_confirm")
            adjusted_df = pd.concat([adjusted_df, dummy_col],
                axis = 1, copy=True)
        # pandas ignores NaN entries while grouping
        # (see TestMetricsConfirmedTripsPandas.testPandasNaNHandlingAndWorkaround)
        # so we convert them to "unlabeled" first
        adjusted_df.fillna("unlabeled", inplace=True)
        logging.debug("After replacing unlabeled, we get %s " % list(adjusted_df.mode_confirm))
        return adjusted_df
    else:
        return section_df

def grouped_to_summary(time_grouped_df, key_to_fill_fn, summary_fn):
    ret_list = []
    # When we group by a time range, the key is the end of the range
    for key, section_group_df in time_grouped_df:
        logging.debug("For key %s, found %s sections" % (key, len(section_group_df)))
        curr_msts = ecwms.ModeStatTimeSummary()
        key = fix_int64_key_if_needed(key)
        key_to_fill_fn(key, section_group_df, curr_msts)
        curr_msts.nUsers = len(section_group_df.user_id.unique())
        result_section_key = eac.get_section_key_for_analysis_results()
        if result_section_key == "analysis/confirmed_trip":
            grouping_field = "mode_confirm"
        else:
            grouping_field = "sensed_mode"

        mode_grouped_df = section_group_df.groupby(grouping_field)
        mode_results = summary_fn(mode_grouped_df)
        for mode, result in mode_results.items():
            if eac.get_section_key_for_analysis_results() == "analysis/confirmed_trip":
                curr_msts["label_"+mode] = result
            elif eac.get_section_key_for_analysis_results() == "analysis/inferred_section":
                curr_msts[ecwmp.PredictedModeTypes(mode).name] = result
            else:
                curr_msts[ecwm.MotionTypes(mode).name] = result
        ret_list.append(curr_msts)
#         import bson.json_util as bju
#         logging.debug("After appending %s, ret_list = %s" % (curr_msts, ret_list))
#         for k in curr_msts.keys():
#             print("Serializing key = %s" % k)
#             logging.debug("Serializing key %s = %s" %
#                 (k, bju.dumps(curr_msts[k])))
    return ret_list

def _get_local_group_by(local_freq):
    if (local_freq == LocalFreq.DAILY):
        return ['start_local_dt_year', 'start_local_dt_month', 'start_local_dt_day']
    elif (local_freq == LocalFreq.MONTHLY):
        return ['start_local_dt_year', 'start_local_dt_month']
    assert(local_freq == LocalFreq.YEARLY)
    return ['start_local_dt_year']

def _get_local_key_to_fill_fn(local_freq):
    """
    If we group by the local time, then the key is a tuple, e.g.
    (year, month, day). We return the beginning of the period to be consistent
    with the date_range grouping
    :param local_freq: the frequency enum
    :return: the function used to fill in the times
    """
    if (local_freq == LocalFreq.DAILY):
        # The key should be (year, month, day)
        return local_dt_fill_times_daily
    elif (local_freq == LocalFreq.MONTHLY):
        # The key should be (year, month)
        return local_dt_fill_times_monthly
    assert(local_freq == LocalFreq.YEARLY)
    # The key should be (year)
    return local_dt_fill_times_yearly

# When we are summarizing by local time, it is challenging to figure out what
# the start timestamp should be. If the whole day was spent in the same timezone
# then we can use that timezone. But what if we switched timezones during the
# course of the day?
#
# For now, we use the start of the day in the timezone of the first section of
# the day.
# Since timestamps do increase monotonically, sections are binned by start time,
# and we know that the user was in the first timezone at the beginning of the day,
# this seems to make sense.
#
# Concretely, India is 5 hours ahead of UTC and the US West Coast is 7-8 hours
# behind UTC.
# So it is possible to fly from India to the US and arrive at the same local time
# i.e. leave on 1st March and arrive on 1st March.
#
# This is how that works wrt timezones
#
# >>> print arrow.Arrow(year=2016, month=3, day=1, tzinfo=tz.gettz("Asia/Calcutta"))
# 2016-03-01T00:00:00+05:30
# >>> print arrow.Arrow(year=2016, month=3, day=1, tzinfo=tz.gettz("Asia/Calcutta")).ceil('day').to('UTC')
# 2016-03-01T18:29:59.999999+00:00
# >>> print arrow.Arrow(year=2016, month=3, day=1, tzinfo=tz.gettz("Asia/Calcutta")).ceil('day').timestamp
# 1456856999
#
# >>> print arrow.Arrow(year=2016, month=3, day=1, tzinfo=tz.gettz("America/Los_Angeles"))
# 2016-03-01T00:00:00-08:00
# >>> print arrow.Arrow(year=2016, month=3, day=1, tzinfo=tz.gettz("America/Los_Angeles")).ceil('day').to('UTC')
# 2016-03-02T07:59:59.999999+00:00
# >>> print arrow.Arrow(year=2016, month=3, day=1, tzinfo=tz.gettz("America/Los_Angeles")).ceil('day').timestamp
# 1456905599
#
# So the end timestamp for that very long day would be 1456905599

def local_dt_fill_times_daily(key, section_group_df, metric_summary):
    first_tz = _get_tz(section_group_df)
    ld = ecwl.LocalDate({'year': key[0],
                         'month': key[1],
                         'day': key[2],
                         'timezone': first_tz})
    dt = arrow.Arrow(ld.year, ld.month, ld.day, tzinfo=first_tz
                     ).floor('day')
    metric_summary.ts = dt.timestamp
    metric_summary.local_dt = ld
    metric_summary.fmt_time = dt.format("YYYY-MM-DD")

def local_dt_fill_times_monthly(key, section_group_df, metric_summary):
    first_tz = _get_tz(section_group_df)
    ld = ecwl.LocalDate({'year': key[0],
                         'month': key[1],
                         'timezone': first_tz})
    dt = arrow.Arrow(ld.year, ld.month, 1,
                     tzinfo=first_tz).floor('month')
    metric_summary.ts = dt.timestamp
    metric_summary.local_dt = ld
    metric_summary.fmt_time = dt.format("YYYY-MM")

def local_dt_fill_times_yearly(key, section_group_df, metric_summary):
    first_tz = _get_tz(section_group_df)
    ld = ecwl.LocalDate({'year': key,
                         'timezone': first_tz})
    dt = arrow.Arrow(ld.year, 1, 1, tzinfo=first_tz
                     ).floor('year')
    metric_summary.ts = dt.timestamp
    metric_summary.local_dt = ld
    metric_summary.fmt_time = dt.format("YYYY")

def _get_tz(section_group_df):
    return section_group_df.sort_values(by='start_ts').head(1).start_local_dt_timezone.iloc[0]
<commit_msg>Pre-process for the user input at the beginning even for local date queries<commit_after>from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from future import standard_library
standard_library.install_aliases()
from builtins import range
from builtins import *
import enum
import pandas as pd
import numpy as np
import arrow as arrow
import logging

import emission.storage.decorations.analysis_timeseries_queries as esda
import emission.storage.decorations.local_date_queries as esdl

import emission.storage.timeseries.tcquery as esttc
import emission.storage.timeseries.timequery as estt
import emission.storage.timeseries.abstract_timeseries as esta
import emission.storage.timeseries.aggregate_timeseries as estag
import emission.core.wrapper.motionactivity as ecwm
import emission.core.wrapper.modestattimesummary as ecwms
import emission.core.wrapper.modeprediction as ecwmp
import emission.core.wrapper.localdate as ecwl

import emission.analysis.config as eac

def group_by_timestamp(user_id, start_ts, end_ts, freq, summary_fn_list):
    """
    Get grouped dataframes for the specific time range and at the specified frequency
    :param user_id: The user for whom we are computing this information. None for all users.
    :param from_ld: The start timestamp
    :param to_ld: The end timestamp
    :param freq: The frequency as specified in a pandas date_range frequency string.
    We only support frequencies of a day or longer in order to return the data
    in a format that makes sense
    http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
    The canonical list can be found at:
    > pandas.tseries.offsets.prefix_mapping
    :return: a dict containing the last start_ts of the last section processed
        and a result list of ModeStatTimeSummary objects
        If there were no matching sections, the last start_ts is None
        and the list is empty.
    """
    time_query = estt.TimeQuery("data.start_ts", start_ts, end_ts)
    section_df = esda.get_data_df(eac.get_section_key_for_analysis_results(),
                                  user_id=user_id, time_query=time_query,
                                  geo_query=None)
    if len(section_df) == 0:
        logging.info("Found no entries for user %s, time_query %s" % (user_id, time_query))
        return {
            "last_ts_processed": None,
            "result": [[] for i in range(len(summary_fn_list))]
        }
    logging.debug("first row is %s" % section_df.iloc[0])
    secs_to_nanos = lambda x: x * 10 ** 9
    section_df['start_dt'] = pd.to_datetime(secs_to_nanos(section_df.start_ts))
    adjusted_df = adjust_for_user_inputs_if_needed(section_df)
    time_grouped_df = adjusted_df.groupby(pd.Grouper(freq=freq, key='start_dt'))
    return {
        "last_ts_processed": adjusted_df.iloc[-1].start_ts,
        "result": [grouped_to_summary(time_grouped_df, timestamp_fill_times, summary_fn)
                   for summary_fn in summary_fn_list]
    }

def timestamp_fill_times(key, ignored, metric_summary):
    dt = arrow.get(key)
    metric_summary.ts = dt.timestamp
    metric_summary.local_dt = ecwl.LocalDate.get_local_date(dt.timestamp, 'UTC')
    metric_summary.fmt_time = dt.isoformat()

class LocalFreq(enum.Enum):
    DAILY = 0
    MONTHLY = 1
    YEARLY = 2

def group_by_local_date(user_id, from_dt, to_dt, freq, summary_fn_list):
    """
    Get grouped data frames for the specified local date range and frequency
    :param user_id: id for the user. None for aggregate.
    :param from_dt: start local dt object. We assume that only the year, month
    and date entries are filled in and represent a date range.
    :param to_dt: end local dt object. We assume that only the year, month
    and date entries are filled in and represent a date range.
    :param freq: since we only expand certain local_dt fields, we can only
    support frequencies corresponding to them. These are represented in the
    `LocalFreq` enum.
    :return: a dict containing the last start_ts of the last section processed
        and a result list of ModeStatTimeSummary objects
        If there were no matching sections, the last start_ts is None
        and the list is empty.
    """
    time_query = esttc.TimeComponentQuery("data.start_local_dt", from_dt, to_dt)
    section_df = esda.get_data_df(eac.get_section_key_for_analysis_results(),
                                  user_id=user_id, time_query=time_query,
                                  geo_query=None)
    if len(section_df) == 0:
        logging.info("Found no entries for user %s, time_query %s" % (user_id, time_query))
        return {
            "last_ts_processed": None,
            "result": [[] for i in range(len(summary_fn_list))]
        }

    groupby_arr = _get_local_group_by(freq)
    adjusted_df = adjust_for_user_inputs_if_needed(section_df)
    time_grouped_df = adjusted_df.groupby(groupby_arr)
    local_dt_fill_fn = _get_local_key_to_fill_fn(freq)
    return {
        "last_ts_processed": section_df.iloc[-1].start_ts,
        "result": [grouped_to_summary(time_grouped_df, local_dt_fill_fn, summary_fn)
                        for summary_fn in summary_fn_list]
    }

# by default, the incoming values for the keys are `numpy.int64` for reasons
# that I don't understand. This breaks serialization
# (https://github.com/e-mission/e-mission-docs/issues/530)
# converting to regular ints to avoid this issue

def fix_int64_key_if_needed(key):
    if isinstance(key, tuple):
        logging.debug("Converting %d fields from int64 to regular integer" % len(key))
        # print("before conversion, types = %s" % str(tuple([type(k) for k in key])))
        mod_keys = tuple([int(k) for k in key])
        # print("after conversion, types = %s" % str(tuple([type(k) for k in key])))
        return mod_keys
    else:
        return key

def adjust_for_user_inputs_if_needed(section_df):
    result_section_key = eac.get_section_key_for_analysis_results()
    if result_section_key == "analysis/confirmed_trip":
        import emission.storage.decorations.trip_queries as esdt
        adjusted_df = esdt.expand_finallabels(section_df)
        # if none of the trips in the time grouping are labeled,
        # we add a dummy column
        # pandas checks in TestMetricsConfirmedTripsPandas.testPandasConcatModeConfirm
        if "mode_confirm" not in adjusted_df.columns:
            # If we don't reset the index and the index doesn't start from 1,
            # the concat will end up with additional rows
            # see TestMetricsConfirmedTripsPandas.testPandasConcatModeConfirm
            adjusted_df.reset_index(inplace=True)
            dummy_col = pd.Series([np.NaN] * len(adjusted_df), name="mode_confirm")
            adjusted_df = pd.concat([adjusted_df, dummy_col],
                axis = 1, copy=True)
        # pandas ignores NaN entries while grouping
        # (see TestMetricsConfirmedTripsPandas.testPandasNaNHandlingAndWorkaround)
        # so we convert them to "unlabeled" first
        adjusted_df.fillna("unlabeled", inplace=True)
        logging.debug("After replacing unlabeled, we get %s " % list(adjusted_df.mode_confirm))
        return adjusted_df
    else:
        return section_df

def grouped_to_summary(time_grouped_df, key_to_fill_fn, summary_fn):
    ret_list = []
    # When we group by a time range, the key is the end of the range
    for key, section_group_df in time_grouped_df:
        logging.debug("For key %s, found %s sections" % (key, len(section_group_df)))
        curr_msts = ecwms.ModeStatTimeSummary()
        key = fix_int64_key_if_needed(key)
        key_to_fill_fn(key, section_group_df, curr_msts)
        curr_msts.nUsers = len(section_group_df.user_id.unique())
        result_section_key = eac.get_section_key_for_analysis_results()
        if result_section_key == "analysis/confirmed_trip":
            grouping_field = "mode_confirm"
        else:
            grouping_field = "sensed_mode"

        mode_grouped_df = section_group_df.groupby(grouping_field)
        mode_results = summary_fn(mode_grouped_df)
        for mode, result in mode_results.items():
            if eac.get_section_key_for_analysis_results() == "analysis/confirmed_trip":
                curr_msts["label_"+mode] = result
            elif eac.get_section_key_for_analysis_results() == "analysis/inferred_section":
                curr_msts[ecwmp.PredictedModeTypes(mode).name] = result
            else:
                curr_msts[ecwm.MotionTypes(mode).name] = result
        ret_list.append(curr_msts)
#         import bson.json_util as bju
#         logging.debug("After appending %s, ret_list = %s" % (curr_msts, ret_list))
#         for k in curr_msts.keys():
#             print("Serializing key = %s" % k)
#             logging.debug("Serializing key %s = %s" %
#                 (k, bju.dumps(curr_msts[k])))
    return ret_list

def _get_local_group_by(local_freq):
    if (local_freq == LocalFreq.DAILY):
        return ['start_local_dt_year', 'start_local_dt_month', 'start_local_dt_day']
    elif (local_freq == LocalFreq.MONTHLY):
        return ['start_local_dt_year', 'start_local_dt_month']
    assert(local_freq == LocalFreq.YEARLY)
    return ['start_local_dt_year']

def _get_local_key_to_fill_fn(local_freq):
    """
    If we group by the local time, then the key is a tuple, e.g.
    (year, month, day). We return the beginning of the period to be consistent
    with the date_range grouping
    :param local_freq: the frequency enum
    :return: the function used to fill in the times
    """
    if (local_freq == LocalFreq.DAILY):
        # The key should be (year, month, day)
        return local_dt_fill_times_daily
    elif (local_freq == LocalFreq.MONTHLY):
        # The key should be (year, month)
        return local_dt_fill_times_monthly
    assert(local_freq == LocalFreq.YEARLY)
    # The key should be (year)
    return local_dt_fill_times_yearly

# When we are summarizing by local time, it is challenging to figure out what
# the start timestamp should be. If the whole day was spent in the same timezone
# then we can use that timezone. But what if we switched timezones during the
# course of the day?
#
# For now, we use the start of the day in the timezone of the first section of
# the day.
# Since timestamps do increase monotonically, sections are binned by start time,
# and we know that the user was in the first timezone at the beginning of the day,
# this seems to make sense.
#
# Concretely, India is 5 hours ahead of UTC and the US West Coast is 7-8 hours
# behind UTC.
# So it is possible to fly from India to the US and arrive at the same local time
# i.e. leave on 1st March and arrive on 1st March.
#
# This is how that works wrt timezones
#
# >>> print arrow.Arrow(year=2016, month=3, day=1, tzinfo=tz.gettz("Asia/Calcutta"))
# 2016-03-01T00:00:00+05:30
# >>> print arrow.Arrow(year=2016, month=3, day=1, tzinfo=tz.gettz("Asia/Calcutta")).ceil('day').to('UTC')
# 2016-03-01T18:29:59.999999+00:00
# >>> print arrow.Arrow(year=2016, month=3, day=1, tzinfo=tz.gettz("Asia/Calcutta")).ceil('day').timestamp
# 1456856999
#
# >>> print arrow.Arrow(year=2016, month=3, day=1, tzinfo=tz.gettz("America/Los_Angeles"))
# 2016-03-01T00:00:00-08:00
# >>> print arrow.Arrow(year=2016, month=3, day=1, tzinfo=tz.gettz("America/Los_Angeles")).ceil('day').to('UTC')
# 2016-03-02T07:59:59.999999+00:00
# >>> print arrow.Arrow(year=2016, month=3, day=1, tzinfo=tz.gettz("America/Los_Angeles")).ceil('day').timestamp
# 1456905599
#
# So the end timestamp for that very long day would be 1456905599

def local_dt_fill_times_daily(key, section_group_df, metric_summary):
    first_tz = _get_tz(section_group_df)
    ld = ecwl.LocalDate({'year': key[0],
                         'month': key[1],
                         'day': key[2],
                         'timezone': first_tz})
    dt = arrow.Arrow(ld.year, ld.month, ld.day, tzinfo=first_tz
                     ).floor('day')
    metric_summary.ts = dt.timestamp
    metric_summary.local_dt = ld
    metric_summary.fmt_time = dt.format("YYYY-MM-DD")

def local_dt_fill_times_monthly(key, section_group_df, metric_summary):
    first_tz = _get_tz(section_group_df)
    ld = ecwl.LocalDate({'year': key[0],
                         'month': key[1],
                         'timezone': first_tz})
    dt = arrow.Arrow(ld.year, ld.month, 1,
                     tzinfo=first_tz).floor('month')
    metric_summary.ts = dt.timestamp
    metric_summary.local_dt = ld
    metric_summary.fmt_time = dt.format("YYYY-MM")

def local_dt_fill_times_yearly(key, section_group_df, metric_summary):
    first_tz = _get_tz(section_group_df)
    ld = ecwl.LocalDate({'year': key,
                         'timezone': first_tz})
    dt = arrow.Arrow(ld.year, 1, 1, tzinfo=first_tz
                     ).floor('year')
    metric_summary.ts = dt.timestamp
    metric_summary.local_dt = ld
    metric_summary.fmt_time = dt.format("YYYY")

def _get_tz(section_group_df):
    return section_group_df.sort_values(by='start_ts').head(1).start_local_dt_timezone.iloc[0]
<|endoftext|>


<commit_before>import logging
import pandas as pd
import pymongo

import emission.core.get_database as edb
import emission.storage.timeseries.abstract_timeseries as esta

class BuiltinTimeSeries(esta.TimeSeries):
    def __init__(self, user_id):
        super(BuiltinTimeSeries, self).__init__(user_id)
        self.key_query = lambda(key): {"metadata.key": key}
        self.ts_query = lambda(tq): {"metadata.%s" % tq.timeType: {"$gte": tq.startTs, "$lte": tq.endTs}}
        self.type_query = lambda(entry_type): {"metadata.type": entry_type}

    @staticmethod
    def get_uuid_list():
        return edb.get_timeseries_db().distinct("user_id")

    def _get_query(self, key_list = None, time_query = None):
        ret_query = {'user_id': self.user_id} # UUID is mandatory
        if key_list is not None and len(key_list) > 0:
            key_query_list = []
            for key in key_list:
                key_query_list.append(self.key_query(key))
            ret_query.update({"$or": key_query_list})
        if time_query is not None:
            ret_query.update(self.ts_query(time_query))
        return ret_query

    @staticmethod
    def _get_sort_key(time_query = None):
        if time_query is None:
            return "metadata.write_ts"
        else:
            return "metadata.%s" % time_query.timeType

    @staticmethod
    def _to_df_entry(entry):
        return entry["data"]

    def find_entries(self, key_list = None, time_query = None):
        sort_key = self._get_sort_key(time_query)
        logging.debug("sort_key = %s" % sort_key)
        return edb.get_timeseries_db().find(self._get_query(key_list, time_query)).sort(sort_key, pymongo.ASCENDING)

    def get_entry_at_ts(self, key, ts_key, ts):
        return edb.get_timeseries_db().find_one({"user_id": self.user_id,
                                                 "metadata.key": key,
                                                 ts_key: ts})

    def get_data_df(self, key, time_query = None):
        sort_key = self._get_sort_key(time_query)
        logging.debug("sort_key = %s" % sort_key)
        result_it = edb.get_timeseries_db().find(self._get_query([key], time_query), {"data": True})\
                                            .sort(sort_key, pymongo.ASCENDING)
        # Dataframe doesn't like to work off an iterator - it wants everything in memory
        return pd.DataFrame([BuiltinTimeSeries._to_df_entry(e) for e in list(result_it)])

    def insert(self, entry):
        """
        """
        logging.debug("insert called")
        if "user_id" not in entry:
            entry["user_id"] = self.user_id
        elif entry["user_id"] != self.user_id:
            raise AttributeError("Saving entry for %s in timeseries for %s" % (entry["user_id"], self.user_id))
        else:
            logging.debug("entry was fine, no need to fix it")

        logging.debug("Inserting entry %s into timeseries" % entry)
        edb.get_timeseries_db().insert(entry)

    def insert_error(self, entry):
        """
        """
        logging.debug("insert_error called")
        if "user_id" not in entry:
            entry["user_id"] = self.user_id
        elif entry["user_id"] != self.user_id:
            raise AttributeError("Saving entry for %s in timeseries for %s" % (entry["user_id"], self.user_id))
        else:
            logging.debug("entry was fine, no need to fix it")

        logging.debug("Inserting entry %s into error timeseries" % entry)
        edb.get_timeseries_error_db().insert(entry)
<commit_msg>Include the _id in the data that is converted to a dataframe<commit_after>import logging
import pandas as pd
import pymongo

import emission.core.get_database as edb
import emission.storage.timeseries.abstract_timeseries as esta

class BuiltinTimeSeries(esta.TimeSeries):
    def __init__(self, user_id):
        super(BuiltinTimeSeries, self).__init__(user_id)
        self.key_query = lambda(key): {"metadata.key": key}
        self.ts_query = lambda(tq): {"metadata.%s" % tq.timeType: {"$gte": tq.startTs, "$lte": tq.endTs}}
        self.type_query = lambda(entry_type): {"metadata.type": entry_type}

    @staticmethod
    def get_uuid_list():
        return edb.get_timeseries_db().distinct("user_id")

    def _get_query(self, key_list = None, time_query = None):
        ret_query = {'user_id': self.user_id} # UUID is mandatory
        if key_list is not None and len(key_list) > 0:
            key_query_list = []
            for key in key_list:
                key_query_list.append(self.key_query(key))
            ret_query.update({"$or": key_query_list})
        if time_query is not None:
            ret_query.update(self.ts_query(time_query))
        return ret_query

    @staticmethod
    def _get_sort_key(time_query = None):
        if time_query is None:
            return "metadata.write_ts"
        else:
            return "metadata.%s" % time_query.timeType

    @staticmethod
    def _to_df_entry(entry):
        ret_val = entry["data"]
        ret_val["_id"] = entry["_id"]
        logging.debug("ret_val = %s " % ret_val)
        return ret_val

    def find_entries(self, key_list = None, time_query = None):
        sort_key = self._get_sort_key(time_query)
        logging.debug("sort_key = %s" % sort_key)
        return edb.get_timeseries_db().find(self._get_query(key_list, time_query)).sort(sort_key, pymongo.ASCENDING)

    def get_entry_at_ts(self, key, ts_key, ts):
        return edb.get_timeseries_db().find_one({"user_id": self.user_id,
                                                 "metadata.key": key,
                                                 ts_key: ts})

    def get_data_df(self, key, time_query = None):
        sort_key = self._get_sort_key(time_query)
        logging.debug("sort_key = %s" % sort_key)
        result_it = edb.get_timeseries_db().find(self._get_query([key], time_query), {"data": True})\
                                            .sort(sort_key, pymongo.ASCENDING)
        # Dataframe doesn't like to work off an iterator - it wants everything in memory
        return pd.DataFrame([BuiltinTimeSeries._to_df_entry(e) for e in list(result_it)])

    def insert(self, entry):
        """
        """
        logging.debug("insert called")
        if "user_id" not in entry:
            entry["user_id"] = self.user_id
        elif entry["user_id"] != self.user_id:
            raise AttributeError("Saving entry for %s in timeseries for %s" % (entry["user_id"], self.user_id))
        else:
            logging.debug("entry was fine, no need to fix it")

        logging.debug("Inserting entry %s into timeseries" % entry)
        edb.get_timeseries_db().insert(entry)

    def insert_error(self, entry):
        """
        """
        logging.debug("insert_error called")
        if "user_id" not in entry:
            entry["user_id"] = self.user_id
        elif entry["user_id"] != self.user_id:
            raise AttributeError("Saving entry for %s in timeseries for %s" % (entry["user_id"], self.user_id))
        else:
            logging.debug("entry was fine, no need to fix it")

        logging.debug("Inserting entry %s into error timeseries" % entry)
        edb.get_timeseries_error_db().insert(entry)
<|endoftext|>


<commit_before>"""
    test_base32
    ~~~~~~~~~~~

    Tests for the :mod:`~ulid.base32` module.
"""
import pytest

from ulid import base32

NON_BASE_32_EXP = r'^Non-base32 character found'

@pytest.fixture(scope='session')
def decoding_alphabet():
    """
    Fixture that yields the entire alphabet that is valid for base32 decoding.
    """
    return base32.ENCODING + 'lLiIoO'

def test_encode_handles_ulid_and_returns_26_char_string(valid_bytes_128):
    """
    Assert that :func:`~ulid.base32.encode` encodes a valid 128 bit bytes object into a :class:`~str`
    that is 26 characters long.
    """
    encoded = base32.encode(valid_bytes_128)
    assert isinstance(encoded, str)
    assert len(encoded) == 26

def test_encode_handles_timestamp_and_returns_10_char_string(valid_bytes_48):
    """
    Assert that :func:`~ulid.base32.encode` encodes a valid 48 bit bytes object into a :class:`~str`
    that is 10 characters long.
    """
    encoded = base32.encode(valid_bytes_48)
    assert isinstance(encoded, str)
    assert len(encoded) == 10

def test_encode_handles_randomness_and_returns_16_char_string(valid_bytes_80):
    """
    Assert that :func:`~ulid.base32.encode` encodes a valid 80 bit bytes object into a :class:`~str`
    that is 16 characters long.
    """
    encoded = base32.encode(valid_bytes_80)
    assert isinstance(encoded, str)
    assert len(encoded) == 16

def test_encode_raises_on_bytes_length_mismatch(invalid_bytes_48_80_128):
    """
    Assert that :func:`~ulid.base32.encode` raises a :class:`~ValueError` when given a :class:`~bytes`
    instance that is not exactly 48, 80, or 128 bits in length.
    """
    with pytest.raises(ValueError):
        base32.encode(invalid_bytes_48_80_128)

def test_encode_ulid_returns_26_char_string(valid_bytes_128):
    """
    Assert that :func:`~ulid.base32.encode_ulid` encodes a valid 128 bit bytes object into a :class:`~str`
    that is 26 characters long.
    """
    encoded = base32.encode_ulid(valid_bytes_128)
    assert isinstance(encoded, str)
    assert len(encoded) == 26

def test_encode_ulid_raises_on_bytes_length_mismatch(invalid_bytes_128):
    """
    Assert that :func:`~ulid.base32.encode_ulid` raises a :class:`~ValueError` when given a :class:`~bytes`
    instance that is not exactly 128 bit.
    """
    with pytest.raises(ValueError):
        base32.encode_ulid(invalid_bytes_128)

def test_encode_timestamp_returns_10_char_string(valid_bytes_48):
    """
    Assert that :func:`~ulid.base32.encode_timestamp` encodes a valid 48 bit bytes object into a :class:`~str`
    that is 10 characters long.
    """
    encoded = base32.encode_timestamp(valid_bytes_48)
    assert isinstance(encoded, str)
    assert len(encoded) == 10

def test_encode_timestamp_raises_on_bytes_length_mismatch(invalid_bytes_48):
    """
    Assert that :func:`~ulid.base32.encode_timestamp` raises a :class:`~ValueError` when given a :class:`~bytes`
    instance that is not exactly 48 bit.
    """
    with pytest.raises(ValueError):
        base32.encode_timestamp(invalid_bytes_48)

def test_encode_randomness_returns_16_char_string(valid_bytes_80):
    """
    Assert that :func:`~ulid.base32.encode_randomness` encodes a valid 80 bit bytes object into a :class:`~str`
    that is 16 characters long.
    """
    encoded = base32.encode_randomness(valid_bytes_80)
    assert isinstance(encoded, str)
    assert len(encoded) == 16

def test_encode_randomness_raises_on_bytes_length_mismatch(invalid_bytes_80):
    """
    Assert that :func:`~ulid.base32.encode_randomness` raises a :class:`~ValueError` when given a :class:`~bytes`
    instance that is not exactly 80 bit.
    """
    with pytest.raises(ValueError):
        base32.encode_randomness(invalid_bytes_80)

def test_decode_handles_ulid_and_returns_16_bytes(valid_str_26):
    """
    Assert that :func:`~ulid.base32.decode` decodes a valid 26 character string into a :class:`~bytes`
    instance that is 128 bit.
    """
    decoded = base32.decode(valid_str_26)
    assert isinstance(decoded, bytes)
    assert len(decoded) == 16

def test_decode_handles_timestamp_and_returns_6_bytes(valid_str_10):
    """
    Assert that :func:`~ulid.base32.decode` decodes a valid 10 character string into a :class:`~bytes`
    instance that is 48 bit.
    """
    decoded = base32.decode(valid_str_10)
    assert isinstance(decoded, bytes)
    assert len(decoded) == 6

def test_decode_handles_randomness_and_returns_10_bytes(valid_str_16):
    """
    Assert that :func:`~ulid.base32.decode` decodes a valid 16 character string into a :class:`~bytes`
    instance that is 80 bit.
    """
    decoded = base32.decode(valid_str_16)
    assert isinstance(decoded, bytes)
    assert len(decoded) == 10

def test_decode_raises_on_str_length_mismatch(invalid_str_10_16_26):
    """
    Assert that :func:`~ulid.base32.decode` raises a :class:`~ValueError` when given a :class:`~str`
    instance that is not exactly 10, 16, 26 characters in length.
    """
    with pytest.raises(ValueError):
        base32.decode(invalid_str_10_16_26)

def test_decode_raises_on_extended_ascii_str(extended_ascii_str_valid_length):
    """
    Assert that :func:`~ulid.base32.decode` raises a :class:`~ValueError` when given a :class:`~str`
    instance that contains extended ascii characters.
    """
    with pytest.raises(ValueError):
        base32.decode(extended_ascii_str_valid_length)

def test_decode_raises_on_non_base32_decode_char(ascii_non_base32_str_valid_length):
    """
    Assert that :func:`~ulid.base32.decode_ulid` raises a :class:`~ValueError` when given a :class:`~str`
    instance that includes ASCII characters not part of the Base 32 decoding character set.
    """
    with pytest.raises(ValueError) as ex:
        base32.decode(ascii_non_base32_str_valid_length)
    ex.match(NON_BASE_32_EXP)

def test_decode_ulid_returns_16_bytes(valid_str_26):
    """
    Assert that :func:`~ulid.base32.decode_ulid` decodes a valid 26 character string into a :class:`~bytes`
    instance that is 128 bit.
    """
    decoded = base32.decode_ulid(valid_str_26)
    assert isinstance(decoded, bytes)
    assert len(decoded) == 16

def test_decode_ulid_raises_on_str_length_mismatch(invalid_str_26):
    """
    Assert that :func:`~ulid.base32.decode_ulid` raises a :class:`~ValueError` when given a :class:`~str`
    instance that is not exactly 26 chars.
    """
    with pytest.raises(ValueError):
        base32.decode_ulid(invalid_str_26)

def test_decode_ulid_raises_on_non_ascii_str(extended_ascii_str_26):
    """
    Assert that :func:`~ulid.base32.decode_ulid` raises a :class:`~ValueError` when given a :class:`~str`
    instance that contains extended ascii characters.
    """
    with pytest.raises(ValueError):
        base32.decode_ulid(extended_ascii_str_26)

def test_decode_ulid_raises_on_non_base32_decode_char(ascii_non_base32_str_26):
    """
    Assert that :func:`~ulid.base32.decode_ulid` raises a :class:`~ValueError` when given a :class:`~str`
    instance that includes ASCII characters not part of the Base 32 decoding character set.
    """
    with pytest.raises(ValueError) as ex:
        base32.decode_ulid(ascii_non_base32_str_26)
    ex.match(NON_BASE_32_EXP)

def test_decode_timestamp_returns_6_bytes(valid_str_10):
    """
    Assert that :func:`~ulid.base32.decode_timestamp` decodes a valid 10 character string into a :class:`~bytes`
    instance that is 48 bit.
    """
    decoded = base32.decode_timestamp(valid_str_10)
    assert isinstance(decoded, bytes)
    assert len(decoded) == 6

def test_decode_timestamp_raises_on_str_length_mismatch(invalid_str_10):
    """
    Assert that :func:`~ulid.base32.decode_timestamp` raises a :class:`~ValueError` when given a :class:`~str`
    instance that is not exactly 10 chars.
    """
    with pytest.raises(ValueError):
        base32.decode_timestamp(invalid_str_10)

def test_decode_timestamp_raises_on_non_ascii_str(extended_ascii_str_10):
    """
    Assert that :func:`~ulid.base32.decode_timestamp` raises a :class:`~ValueError` when given a :class:`~str`
    instance that contains extended ascii characters.
    """
    with pytest.raises(ValueError):
        base32.decode_timestamp(extended_ascii_str_10)

def test_decode_timestamp_raises_on_non_base32_decode_char(ascii_non_base32_str_10):
    """
    Assert that :func:`~ulid.base32.decode_timestamp` raises a :class:`~ValueError` when given a :class:`~str`
    instance that includes ASCII characters not part of the Base 32 decoding character set.
    """
    with pytest.raises(ValueError) as ex:
        base32.decode_timestamp(ascii_non_base32_str_10)
    ex.match(NON_BASE_32_EXP)

def test_decode_randomness_returns_10_bytes(valid_str_16):
    """
    Assert that :func:`~ulid.base32.decode_randomness` decodes a valid 16 character string into a :class:`~bytes`
    instance that is 80 bit.
    """
    decoded = base32.decode_randomness(valid_str_16)
    assert isinstance(decoded, bytes)
    assert len(decoded) == 10

def test_decode_randomness_raises_on_str_length_mismatch(invalid_str_16):
    """
    Assert that :func:`~ulid.base32.decode_randomness` raises a :class:`~ValueError` when given a :class:`~str`
    instance that is not exactly 16 chars.
    """
    with pytest.raises(ValueError):
        base32.decode_randomness(invalid_str_16)

def test_decode_randomness_raises_on_non_ascii_str(extended_ascii_str_16):
    """
    Assert that :func:`~ulid.base32.decode_randomness` raises a :class:`~ValueError` when given a :class:`~str`
    instance that contains extended ascii characters.
    """
    with pytest.raises(ValueError):
        base32.decode_randomness(extended_ascii_str_16)

def test_decode_randomness_raises_on_non_base32_decode_char(ascii_non_base32_str_16):
    """
    Assert that :func:`~ulid.base32.decode_randomness` raises a :class:`~ValueError` when given a :class:`~str`
    instance that includes ASCII characters not part of the Base 32 decoding character set.
    """
    with pytest.raises(ValueError) as ex:
        base32.decode_randomness(ascii_non_base32_str_16)
    ex.match(NON_BASE_32_EXP)

def test_decode_table_has_value_for_entire_decoding_alphabet(decoding_alphabet):
    """
    Assert that :attr:`~ulid.base32.DECODING` stores a valid value mapping for all characters that
    can be base32 decoded.
    """
    for char in decoding_alphabet:
        assert base32.DECODING[ord(char)] != 0xFF, 'Character "{}" decoded improperly'.format(char)

def test_str_to_bytes_returns_expected_bytes(valid_str_valid_length):
    """
    Assert that :func:`~ulid.base32.str_to_bytes` decodes a valid string that is 10, 16, or 26 characters long
    into a :class:`~bytes` instance.
    """
    decoded = base32.str_to_bytes(valid_str_valid_length, len(valid_str_valid_length))
    assert isinstance(decoded, bytes)
    assert len(decoded) == len(valid_str_valid_length)

def test_str_to_bytes_raises_on_unexpected_length(invalid_str_26):
    """
    Assert that :func:`~ulid.base32.str_to_bytes` raises a :class:`~ValueError` when given a :class:`~str`
    instance that does not match the expected length.
    """
    with pytest.raises(ValueError):
        base32.str_to_bytes(invalid_str_26, 26)

def test_str_to_bytes_raises_on_extended_ascii_str(extended_ascii_str_valid_length):
    """
    Assert that :func:`~ulid.base32.str_to_bytes` raises a :class:`~ValueError` when given a :class:`~str`
    instance that includes extended ascii characters.
    """
    with pytest.raises(ValueError):
        base32.str_to_bytes(extended_ascii_str_valid_length, len(extended_ascii_str_valid_length))

def test_str_to_bytes_raises_on_non_base32_decode_char(ascii_non_base32_str_valid_length):
    """
    Assert that :func:`~ulid.base32.str_to_bytes` raises a :class:`~ValueError` when given a :class:`~str`
    instance that includes ASCII characters not part of the Base 32 decoding character set.
    """
    with pytest.raises(ValueError) as ex:
        base32.str_to_bytes(ascii_non_base32_str_valid_length, len(ascii_non_base32_str_valid_length))
    ex.match(NON_BASE_32_EXP)
<commit_msg>Refactor NON_BASE_32_EXP to NON_BASE_32_EXC_REGEX.<commit_after>"""
    test_base32
    ~~~~~~~~~~~

    Tests for the :mod:`~ulid.base32` module.
"""
import pytest

from ulid import base32

NON_BASE_32_EXC_REGEX = r'^Non-base32 character found'

@pytest.fixture(scope='session')
def decoding_alphabet():
    """
    Fixture that yields the entire alphabet that is valid for base32 decoding.
    """
    return base32.ENCODING + 'lLiIoO'

def test_encode_handles_ulid_and_returns_26_char_string(valid_bytes_128):
    """
    Assert that :func:`~ulid.base32.encode` encodes a valid 128 bit bytes object into a :class:`~str`
    that is 26 characters long.
    """
    encoded = base32.encode(valid_bytes_128)
    assert isinstance(encoded, str)
    assert len(encoded) == 26

def test_encode_handles_timestamp_and_returns_10_char_string(valid_bytes_48):
    """
    Assert that :func:`~ulid.base32.encode` encodes a valid 48 bit bytes object into a :class:`~str`
    that is 10 characters long.
    """
    encoded = base32.encode(valid_bytes_48)
    assert isinstance(encoded, str)
    assert len(encoded) == 10

def test_encode_handles_randomness_and_returns_16_char_string(valid_bytes_80):
    """
    Assert that :func:`~ulid.base32.encode` encodes a valid 80 bit bytes object into a :class:`~str`
    that is 16 characters long.
    """
    encoded = base32.encode(valid_bytes_80)
    assert isinstance(encoded, str)
    assert len(encoded) == 16

def test_encode_raises_on_bytes_length_mismatch(invalid_bytes_48_80_128):
    """
    Assert that :func:`~ulid.base32.encode` raises a :class:`~ValueError` when given a :class:`~bytes`
    instance that is not exactly 48, 80, or 128 bits in length.
    """
    with pytest.raises(ValueError):
        base32.encode(invalid_bytes_48_80_128)

def test_encode_ulid_returns_26_char_string(valid_bytes_128):
    """
    Assert that :func:`~ulid.base32.encode_ulid` encodes a valid 128 bit bytes object into a :class:`~str`
    that is 26 characters long.
    """
    encoded = base32.encode_ulid(valid_bytes_128)
    assert isinstance(encoded, str)
    assert len(encoded) == 26

def test_encode_ulid_raises_on_bytes_length_mismatch(invalid_bytes_128):
    """
    Assert that :func:`~ulid.base32.encode_ulid` raises a :class:`~ValueError` when given a :class:`~bytes`
    instance that is not exactly 128 bit.
    """
    with pytest.raises(ValueError):
        base32.encode_ulid(invalid_bytes_128)

def test_encode_timestamp_returns_10_char_string(valid_bytes_48):
    """
    Assert that :func:`~ulid.base32.encode_timestamp` encodes a valid 48 bit bytes object into a :class:`~str`
    that is 10 characters long.
    """
    encoded = base32.encode_timestamp(valid_bytes_48)
    assert isinstance(encoded, str)
    assert len(encoded) == 10

def test_encode_timestamp_raises_on_bytes_length_mismatch(invalid_bytes_48):
    """
    Assert that :func:`~ulid.base32.encode_timestamp` raises a :class:`~ValueError` when given a :class:`~bytes`
    instance that is not exactly 48 bit.
    """
    with pytest.raises(ValueError):
        base32.encode_timestamp(invalid_bytes_48)

def test_encode_randomness_returns_16_char_string(valid_bytes_80):
    """
    Assert that :func:`~ulid.base32.encode_randomness` encodes a valid 80 bit bytes object into a :class:`~str`
    that is 16 characters long.
    """
    encoded = base32.encode_randomness(valid_bytes_80)
    assert isinstance(encoded, str)
    assert len(encoded) == 16

def test_encode_randomness_raises_on_bytes_length_mismatch(invalid_bytes_80):
    """
    Assert that :func:`~ulid.base32.encode_randomness` raises a :class:`~ValueError` when given a :class:`~bytes`
    instance that is not exactly 80 bit.
    """
    with pytest.raises(ValueError):
        base32.encode_randomness(invalid_bytes_80)

def test_decode_handles_ulid_and_returns_16_bytes(valid_str_26):
    """
    Assert that :func:`~ulid.base32.decode` decodes a valid 26 character string into a :class:`~bytes`
    instance that is 128 bit.
    """
    decoded = base32.decode(valid_str_26)
    assert isinstance(decoded, bytes)
    assert len(decoded) == 16

def test_decode_handles_timestamp_and_returns_6_bytes(valid_str_10):
    """
    Assert that :func:`~ulid.base32.decode` decodes a valid 10 character string into a :class:`~bytes`
    instance that is 48 bit.
    """
    decoded = base32.decode(valid_str_10)
    assert isinstance(decoded, bytes)
    assert len(decoded) == 6

def test_decode_handles_randomness_and_returns_10_bytes(valid_str_16):
    """
    Assert that :func:`~ulid.base32.decode` decodes a valid 16 character string into a :class:`~bytes`
    instance that is 80 bit.
    """
    decoded = base32.decode(valid_str_16)
    assert isinstance(decoded, bytes)
    assert len(decoded) == 10

def test_decode_raises_on_str_length_mismatch(invalid_str_10_16_26):
    """
    Assert that :func:`~ulid.base32.decode` raises a :class:`~ValueError` when given a :class:`~str`
    instance that is not exactly 10, 16, 26 characters in length.
    """
    with pytest.raises(ValueError):
        base32.decode(invalid_str_10_16_26)

def test_decode_raises_on_extended_ascii_str(extended_ascii_str_valid_length):
    """
    Assert that :func:`~ulid.base32.decode` raises a :class:`~ValueError` when given a :class:`~str`
    instance that contains extended ascii characters.
    """
    with pytest.raises(ValueError):
        base32.decode(extended_ascii_str_valid_length)

def test_decode_raises_on_non_base32_decode_char(ascii_non_base32_str_valid_length):
    """
    Assert that :func:`~ulid.base32.decode_ulid` raises a :class:`~ValueError` when given a :class:`~str`
    instance that includes ASCII characters not part of the Base 32 decoding character set.
    """
    with pytest.raises(ValueError) as ex:
        base32.decode(ascii_non_base32_str_valid_length)
    ex.match(NON_BASE_32_EXC_REGEX)

def test_decode_ulid_returns_16_bytes(valid_str_26):
    """
    Assert that :func:`~ulid.base32.decode_ulid` decodes a valid 26 character string into a :class:`~bytes`
    instance that is 128 bit.
    """
    decoded = base32.decode_ulid(valid_str_26)
    assert isinstance(decoded, bytes)
    assert len(decoded) == 16

def test_decode_ulid_raises_on_str_length_mismatch(invalid_str_26):
    """
    Assert that :func:`~ulid.base32.decode_ulid` raises a :class:`~ValueError` when given a :class:`~str`
    instance that is not exactly 26 chars.
    """
    with pytest.raises(ValueError):
        base32.decode_ulid(invalid_str_26)

def test_decode_ulid_raises_on_non_ascii_str(extended_ascii_str_26):
    """
    Assert that :func:`~ulid.base32.decode_ulid` raises a :class:`~ValueError` when given a :class:`~str`
    instance that contains extended ascii characters.
    """
    with pytest.raises(ValueError):
        base32.decode_ulid(extended_ascii_str_26)

def test_decode_ulid_raises_on_non_base32_decode_char(ascii_non_base32_str_26):
    """
    Assert that :func:`~ulid.base32.decode_ulid` raises a :class:`~ValueError` when given a :class:`~str`
    instance that includes ASCII characters not part of the Base 32 decoding character set.
    """
    with pytest.raises(ValueError) as ex:
        base32.decode_ulid(ascii_non_base32_str_26)
    ex.match(NON_BASE_32_EXC_REGEX)

def test_decode_timestamp_returns_6_bytes(valid_str_10):
    """
    Assert that :func:`~ulid.base32.decode_timestamp` decodes a valid 10 character string into a :class:`~bytes`
    instance that is 48 bit.
    """
    decoded = base32.decode_timestamp(valid_str_10)
    assert isinstance(decoded, bytes)
    assert len(decoded) == 6

def test_decode_timestamp_raises_on_str_length_mismatch(invalid_str_10):
    """
    Assert that :func:`~ulid.base32.decode_timestamp` raises a :class:`~ValueError` when given a :class:`~str`
    instance that is not exactly 10 chars.
    """
    with pytest.raises(ValueError):
        base32.decode_timestamp(invalid_str_10)

def test_decode_timestamp_raises_on_non_ascii_str(extended_ascii_str_10):
    """
    Assert that :func:`~ulid.base32.decode_timestamp` raises a :class:`~ValueError` when given a :class:`~str`
    instance that contains extended ascii characters.
    """
    with pytest.raises(ValueError):
        base32.decode_timestamp(extended_ascii_str_10)

def test_decode_timestamp_raises_on_non_base32_decode_char(ascii_non_base32_str_10):
    """
    Assert that :func:`~ulid.base32.decode_timestamp` raises a :class:`~ValueError` when given a :class:`~str`
    instance that includes ASCII characters not part of the Base 32 decoding character set.
    """
    with pytest.raises(ValueError) as ex:
        base32.decode_timestamp(ascii_non_base32_str_10)
    ex.match(NON_BASE_32_EXC_REGEX)

def test_decode_randomness_returns_10_bytes(valid_str_16):
    """
    Assert that :func:`~ulid.base32.decode_randomness` decodes a valid 16 character string into a :class:`~bytes`
    instance that is 80 bit.
    """
    decoded = base32.decode_randomness(valid_str_16)
    assert isinstance(decoded, bytes)
    assert len(decoded) == 10

def test_decode_randomness_raises_on_str_length_mismatch(invalid_str_16):
    """
    Assert that :func:`~ulid.base32.decode_randomness` raises a :class:`~ValueError` when given a :class:`~str`
    instance that is not exactly 16 chars.
    """
    with pytest.raises(ValueError):
        base32.decode_randomness(invalid_str_16)

def test_decode_randomness_raises_on_non_ascii_str(extended_ascii_str_16):
    """
    Assert that :func:`~ulid.base32.decode_randomness` raises a :class:`~ValueError` when given a :class:`~str`
    instance that contains extended ascii characters.
    """
    with pytest.raises(ValueError):
        base32.decode_randomness(extended_ascii_str_16)

def test_decode_randomness_raises_on_non_base32_decode_char(ascii_non_base32_str_16):
    """
    Assert that :func:`~ulid.base32.decode_randomness` raises a :class:`~ValueError` when given a :class:`~str`
    instance that includes ASCII characters not part of the Base 32 decoding character set.
    """
    with pytest.raises(ValueError) as ex:
        base32.decode_randomness(ascii_non_base32_str_16)
    ex.match(NON_BASE_32_EXC_REGEX)

def test_decode_table_has_value_for_entire_decoding_alphabet(decoding_alphabet):
    """
    Assert that :attr:`~ulid.base32.DECODING` stores a valid value mapping for all characters that
    can be base32 decoded.
    """
    for char in decoding_alphabet:
        assert base32.DECODING[ord(char)] != 0xFF, 'Character "{}" decoded improperly'.format(char)

def test_str_to_bytes_returns_expected_bytes(valid_str_valid_length):
    """
    Assert that :func:`~ulid.base32.str_to_bytes` decodes a valid string that is 10, 16, or 26 characters long
    into a :class:`~bytes` instance.
    """
    decoded = base32.str_to_bytes(valid_str_valid_length, len(valid_str_valid_length))
    assert isinstance(decoded, bytes)
    assert len(decoded) == len(valid_str_valid_length)

def test_str_to_bytes_raises_on_unexpected_length(invalid_str_26):
    """
    Assert that :func:`~ulid.base32.str_to_bytes` raises a :class:`~ValueError` when given a :class:`~str`
    instance that does not match the expected length.
    """
    with pytest.raises(ValueError):
        base32.str_to_bytes(invalid_str_26, 26)

def test_str_to_bytes_raises_on_extended_ascii_str(extended_ascii_str_valid_length):
    """
    Assert that :func:`~ulid.base32.str_to_bytes` raises a :class:`~ValueError` when given a :class:`~str`
    instance that includes extended ascii characters.
    """
    with pytest.raises(ValueError):
        base32.str_to_bytes(extended_ascii_str_valid_length, len(extended_ascii_str_valid_length))

def test_str_to_bytes_raises_on_non_base32_decode_char(ascii_non_base32_str_valid_length):
    """
    Assert that :func:`~ulid.base32.str_to_bytes` raises a :class:`~ValueError` when given a :class:`~str`
    instance that includes ASCII characters not part of the Base 32 decoding character set.
    """
    with pytest.raises(ValueError) as ex:
        base32.str_to_bytes(ascii_non_base32_str_valid_length, len(ascii_non_base32_str_valid_length))
    ex.match(NON_BASE_32_EXC_REGEX)
<|endoftext|>


<commit_before>import json
from dock.constants import CONTAINER_RESULTS_JSON_PATH
from dock.inner import BuildResultsEncoder
from dock.plugin import PostBuildPlugin

__all__ = ('PostBuildRPMqaPlugin', )

class StoreLogsToFilePlugin(PostBuildPlugin):
    key = "store_logs_to_file"

    def __init__(self, tasker, workflow, file_path):
        """
        constructor

        :param tasker: DockerTasker instance
        :param workflow: DockerBuildWorkflow instance
        :param file_path: str, path to file where logs should be stored
        """
        # call parent constructor
        super(StoreLogsToFilePlugin, self).__init__(tasker, workflow)
        self.file_path = file_path

    def run(self):
        file_path = self.file_path or CONTAINER_RESULTS_JSON_PATH
        results = {
            'prebuild_plugins': self.workflow.prebuild_results,
            'postbuild_plugins': self.workflow.postbuild_results,
        }

        with open(file_path, 'w') as results_json_fd:
            json.dump(results, results_json_fd, cls=BuildResultsEncoder)
<commit_msg>store logs plugin: fix __all__<commit_after>import json
from dock.constants import CONTAINER_RESULTS_JSON_PATH
from dock.inner import BuildResultsEncoder
from dock.plugin import PostBuildPlugin

__all__ = ('StoreLogsToFilePlugin', )

class StoreLogsToFilePlugin(PostBuildPlugin):
    key = "store_logs_to_file"

    def __init__(self, tasker, workflow, file_path):
        """
        constructor

        :param tasker: DockerTasker instance
        :param workflow: DockerBuildWorkflow instance
        :param file_path: str, path to file where logs should be stored
        """
        # call parent constructor
        super(StoreLogsToFilePlugin, self).__init__(tasker, workflow)
        self.file_path = file_path

    def run(self):
        file_path = self.file_path or CONTAINER_RESULTS_JSON_PATH
        results = {
            'prebuild_plugins': self.workflow.prebuild_results,
            'postbuild_plugins': self.workflow.postbuild_results,
        }

        with open(file_path, 'w') as results_json_fd:
            json.dump(results, results_json_fd, cls=BuildResultsEncoder)
<|endoftext|>


<commit_before>__version__ = '0.6.0.dev1'
__author__ = 'D-Wave Systems Inc.'
__authoremail__ = 'acondello@dwavesys.com'
__description__ = 'A shared API for binary quadratic model samplers.'
<commit_msg>Update version 0.6.0.dev1 -> 0.6.0.dev2<commit_after>__version__ = '0.6.0.dev2'
__author__ = 'D-Wave Systems Inc.'
__authoremail__ = 'acondello@dwavesys.com'
__description__ = 'A shared API for binary quadratic model samplers.'
<|endoftext|>


<commit_before>"""Deprecated; use Graph."""

from rdflib import Graph

from rdflib.backends.Memory import Memory

class TripleStore(Graph):
    """
    Depcrecated. Use Graph instead.
    """

    def __init__(self, location=None, backend=None):
        if backend==None:
            backend = Memory()
        super(TripleStore, self).__init__(backend=backend)
        if location:
            self.load(location)

    def prefix_mapping(self, prefix, namespace):
        self.bind(prefix, namespace)

<commit_msg>backends -> store<commit_after>"""Deprecated; use Graph."""

from rdflib import Graph

from rdflib.store.Memory import Memory

class TripleStore(Graph):
    """
    Depcrecated. Use Graph instead.
    """

    def __init__(self, location=None, backend=None):
        if backend==None:
            backend = Memory()
        super(TripleStore, self).__init__(backend=backend)
        if location:
            self.load(location)

    def prefix_mapping(self, prefix, namespace):
        self.bind(prefix, namespace)

<|endoftext|>