<commit_before>__doc__ = """ Test basic sync and conflbfnicts: files are modified and deleted by one or both workers (winner and loser); optionally remove local state db on one of the clients (loser).

There are four clients (workers):

 - creator - populates the directory initially and also performs a final check
 - winner  - is syncing its local changes first
 - loser   - is syncing its local changes second (and optionally it looses the local sync database before doing the sync)
 - checker - only performs a final check (without having interacted with the system before)

Note: in 1.6 client conflict files are excluded by default - so they should be never propagated to the server
Note: in 1.5 the exclusion list should be provided separately to the client (FIXME)

Note on effects of removing local state db (1.6):

 - any files modified remotely or locally get a conflict if both remote and local replicas exist (FIXME: this could be possibly more refined in the future based on timestamps or content checksums)
 - any files not present locally but present remotely will be downloaded (so a deletion won't be propagated)

"""

from smashbox.utilities import *

import glob

filesizeKB = int(config.get('basicSync_filesizeKB',10000))

# True => remove local sync db on the loser
# False => keep the loser
rmLocalStateDB = bool(config.get('basicSync_rmLocalStateDB',False))

testsets = [
        { 'basicSync_filesizeKB': 1,
          'basicSync_rmLocalStateDB':False
        },
        { 'basicSync_filesizeKB': 5000,
          'basicSync_rmLocalStateDB':False
        },
        { 'basicSync_filesizeKB': 15000,
          'basicSync_rmLocalStateDB':False
        },
        { 'basicSync_filesizeKB': 50000,
          'basicSync_rmLocalStateDB':False
        },

        { 'basicSync_filesizeKB': 1,
          'basicSync_rmLocalStateDB':True
        },
        { 'basicSync_filesizeKB': 5000,
          'basicSync_rmLocalStateDB':True
        },
        { 'basicSync_filesizeKB': 15000,
          'basicSync_rmLocalStateDB':True
        },
        { 'basicSync_filesizeKB': 50000,
          'basicSync_rmLocalStateDB':True
        }
]

def expect_content(fn,md5):
    actual_md5 = md5sum(fn)
    error_check(actual_md5 == md5, "inconsistent md5 of %s: expected %s, got %s"%(fn,md5,actual_md5))

def expect_no_deleted_files(d):
    expect_deleted_files(d,[])

def expect_deleted_files(d,expected_deleted_files):
    actual_deleted_files = glob.glob(os.path.join(d,'*_DELETED*'))
    logger.debug('deleted files in %s: %s',d,actual_deleted_files)

    error_check(len(expected_deleted_files) == len(actual_deleted_files), "expected %d got %d deleted files"%(len(expected_deleted_files),len(actual_deleted_files)))

    for fn in expected_deleted_files:
        error_check(any([fn in dfn for dfn in actual_deleted_files]), "expected deleted file for %s not found"%fn)

def expect_conflict_files(d,expected_conflict_files):
    actual_conflict_files = glob.glob(os.path.join(d,'*_conflict-*-*'))

    logger.debug('conflict files in %s: %s',d,actual_conflict_files)

    error_check(len(expected_conflict_files) == len(actual_conflict_files), "expected %d got %d conflict files"%(len(expected_conflict_files),len(actual_conflict_files)))

    exp_basefns = [os.path.splitext(fn)[0] for fn in expected_conflict_files]

    logger.debug(exp_basefns)
    logger.debug(actual_conflict_files)

    for bfn in exp_basefns:
        error_check(any([bfn in fn for fn in actual_conflict_files]), "expected conflict file for %s not found"%bfn)

def expect_no_conflict_files(d):
    expect_conflict_files(d,[])

@add_worker
def creator(step):

    reset_owncloud_account()
    reset_rundir()

    step(1,'create initial content and sync')

    d = make_workdir()

    # files *_NONE are not modified by anyone after initial sync
    # files *_LOSER are modified by the loser but not by the winner
    # files *_WINNER are modified by the winner but not by the loser
    # files *_BOTH are modified both by the winner and by the loser (always conflict on the loser)

    createfile(os.path.join(d,'TEST_FILE_MODIFIED_NONE.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(d,'TEST_FILE_MODIFIED_LOSER.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(d,'TEST_FILE_MODIFIED_WINNER.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(d,'TEST_FILE_MODIFIED_BOTH.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(d,'TEST_FILE_DELETED_LOSER.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(d,'TEST_FILE_DELETED_WINNER.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(d,'TEST_FILE_DELETED_BOTH.dat'),'0',count=1000,bs=filesizeKB)

    shared = reflection.getSharedObject()
    shared['md5_creator'] = md5sum(os.path.join(d,'TEST_FILE_MODIFIED_NONE.dat'))
    logger.info('md5_creator: %s',shared['md5_creator'])

    list_files(d)
    run_ocsync(d)
    list_files(d)

    step(7,'download the repository')
    run_ocsync(d,n=3)

    step(8,'final check')

    final_check(d,shared)
    expect_no_conflict_files(d)

@add_worker
def winner(step):
    step(2,'initial sync')

    d = make_workdir()
    run_ocsync(d)

    step(3,'modify locally and sync to server')

    list_files(d)

    remove_file(os.path.join(d,'TEST_FILE_DELETED_WINNER.dat'))
    remove_file(os.path.join(d,'TEST_FILE_DELETED_BOTH.dat'))

    createfile(os.path.join(d,'TEST_FILE_MODIFIED_WINNER.dat'),'1',count=1000,bs=filesizeKB)
    createfile(os.path.join(d,'TEST_FILE_MODIFIED_BOTH.dat'),'1',count=1000,bs=filesizeKB)

    createfile(os.path.join(d,'TEST_FILE_ADDED_WINNER.dat'),'1',count=1000,bs=filesizeKB)
    createfile(os.path.join(d,'TEST_FILE_ADDED_BOTH.dat'),'1',count=1000,bs=filesizeKB)

    shared = reflection.getSharedObject()
    shared['md5_winner'] = md5sum(os.path.join(d,'TEST_FILE_ADDED_WINNER.dat'))
    logger.info('md5_winner: %s',shared['md5_winner'])

    run_ocsync(d)

    step(5,'final sync')

    run_ocsync(d,n=3)

    step(8,'final check')

    final_check(d,shared)
    expect_no_conflict_files(d)

# this is the loser which lost it's local state db after initial sync

@add_worker
def loser(step):

    step(2,'initial sync')

    d = make_workdir()
    run_ocsync(d)

    step(4,'modify locally and sync to the server')

    list_files(d)

    # now do the local changes

    remove_file(os.path.join(d,'TEST_FILE_DELETED_LOSER.dat'))
    remove_file(os.path.join(d,'TEST_FILE_DELETED_BOTH.dat'))

    createfile(os.path.join(d,'TEST_FILE_MODIFIED_LOSER.dat'),'2',count=1000,bs=filesizeKB)
    createfile(os.path.join(d,'TEST_FILE_MODIFIED_BOTH.dat'),'2',count=1000,bs=filesizeKB)

    createfile(os.path.join(d,'TEST_FILE_ADDED_LOSER.dat'),'2',count=1000,bs=filesizeKB)
    createfile(os.path.join(d,'TEST_FILE_ADDED_BOTH.dat'),'2',count=1000,bs=filesizeKB)

    shared = reflection.getSharedObject()
    shared['md5_loser'] = md5sum(os.path.join(d,'TEST_FILE_ADDED_LOSER.dat'))
    logger.info('md5_loser: %s',shared['md5_loser'])

    #os.system('curl -v -s -k -XPROPFIND --data @/b/eos/CURL-TEST/p2.dat %s| xmllint --format -'%oc_webdav_url(remote_folder='TEST_FILE_MODIFIED_BOTH.dat'))
    #os.system('sqlite3 -line /tmp/smashdir/test_basicSync/loser/.csync_journal.db  \'select * from metadata where path like "%TEST_FILE_MODIFIED_BOTH%"\'')

    # remove the sync db
    if rmLocalStateDB:
        remove_file(os.path.join(d,'.csync_journal.db'))

    run_ocsync(d,n=3) # conflict file will be synced to the server but it requires more than one sync run

    step(6,'final sync')
    run_ocsync(d)

    step(8,'final check')

    #os.system('sqlite3 -line /tmp/smashdir/test_basicSync/loser/.csync_journal.db  \'select * from metadata where path like "%TEST_FILE_MODIFIED_BOTH%"\'')

    final_check(d,shared)
    if not rmLocalStateDB:
        expect_conflict_files(d, ['TEST_FILE_ADDED_BOTH.dat', 'TEST_FILE_MODIFIED_BOTH.dat' ])
    else:
        expect_conflict_files(d, ['TEST_FILE_ADDED_BOTH.dat', 'TEST_FILE_MODIFIED_BOTH.dat',
                                  'TEST_FILE_MODIFIED_LOSER.dat', 'TEST_FILE_MODIFIED_WINNER.dat']) # because the local and remote state is different and it is assumed that this is a conflict (FIXME: in the future timestamp-based last-restort check could improve this situation)

@add_worker
def checker(step):
    shared = reflection.getSharedObject()

    step(7,'download the repository for final verification')
    d = make_workdir()
    run_ocsync(d,n=3)

    step(8,'final check')

    final_check(d,shared)
    expect_no_conflict_files(d)

def final_check(d,shared):
    """ This is the final check applicable to all workers - this reflects the status of the remote repository so everyone should be in sync.
    The only potential differences are with locally generated conflict files.
    """

    list_files(d)
    expect_content(os.path.join(d,'TEST_FILE_MODIFIED_NONE.dat'), shared['md5_creator'])

    expect_content(os.path.join(d,'TEST_FILE_ADDED_LOSER.dat'), shared['md5_loser'])

    if not rmLocalStateDB:
        expect_content(os.path.join(d,'TEST_FILE_MODIFIED_LOSER.dat'), shared['md5_loser'])
    else:
        expect_content(os.path.join(d,'TEST_FILE_MODIFIED_LOSER.dat'), shared['md5_creator']) # in this case, a conflict is created on the loser and file on the server stays the same

    expect_content(os.path.join(d,'TEST_FILE_ADDED_WINNER.dat'), shared['md5_winner'])
    expect_content(os.path.join(d,'TEST_FILE_MODIFIED_WINNER.dat'), shared['md5_winner'])
    expect_content(os.path.join(d,'TEST_FILE_ADDED_BOTH.dat'), shared['md5_winner'])     # a conflict on the loser, server not changed
    expect_content(os.path.join(d,'TEST_FILE_MODIFIED_BOTH.dat'), shared['md5_winner'])  # a conflict on the loser, server not changed

    if not rmLocalStateDB:
        expect_no_deleted_files(d) # normally any deleted files should not come back
    else:
        expect_deleted_files(d, ['TEST_FILE_DELETED_LOSER.dat', 'TEST_FILE_DELETED_WINNER.dat']) # but not TEST_FILE_DELETED_BOTH.dat !
        expect_content(os.path.join(d,'TEST_FILE_DELETED_LOSER.dat'), shared['md5_creator']) # this file should be downloaded by the loser because it has no other choice (no previous state to compare with)
        expect_content(os.path.join(d,'TEST_FILE_DELETED_WINNER.dat'), shared['md5_creator']) # this file should be re-uploaded by the loser because it has no other choice (no previous state to compare with)

###############################################################################

def final_check_1_5(d): # this logic applies for 1.5.x client and owncloud server...
    """ Final verification: all local sync folders should look the same. We expect conflicts and handling of deleted files depending on the rmLocalStateDB option. See code for details.
    """
    import glob

    list_files(d)

    conflict_files = glob.glob(os.path.join(d,'*_conflict-*-*'))

    logger.debug('conflict files in %s: %s',d,conflict_files)

    if not rmLocalStateDB:
        # we expect exactly 1 conflict file

        logger.warning("FIXME: currently winner gets a conflict file - exclude list should be updated and this assert modified for the winner")

        error_check(len(conflict_files) == 1, "there should be exactly 1 conflict file (%d)"%len(conflict_files))
    else:
        # we expect exactly 3 conflict files
        error_check(len(conflict_files) == 3, "there should be exactly 3 conflict files (%d)"%len(conflict_files))

    for fn in conflict_files:

        if not rmLocalStateDB:
            error_check('_BOTH' in fn, """only files modified in BOTH workers have a conflict -  all other files should be conflict-free""")

        else:
            error_check('_BOTH' in fn or '_LOSER' in fn or '_WINNER' in fn, """files which are modified by ANY worker have a conflict now;  files which are not modified should not have a conflict""")

    deleted_files = glob.glob(os.path.join(d,'*_DELETED*'))

    logger.debug('deleted files in %s: %s',d,deleted_files)

    if not rmLocalStateDB:
        error_check(len(deleted_files) == 0, 'deleted files should not be there normally')
    else:
        # deleted files "reappear" if local sync db is lost on the loser, the only file that does not reappear is the DELETED_BOTH which was deleted on *all* local clients

        error_check(len(deleted_files) == 2, "we expect exactly 2 deleted files")

        for fn in deleted_files:
            error_check('_LOSER' in fn or '_WINNER' in fn, "deleted files should only reappear if delete on only one client (but not on both at the same time) ")
<commit_msg>added subdirPath option which allows to define a subdirectory in which file operations take place; this allows to detect etag propagation issues; the testsets are defined as cartesian product of possible option values<commit_after>__doc__ = """ Test basic sync and conflbfnicts: files are modified and deleted by one or both workers (winner and loser); optionally remove local state db on one of the clients (loser).

There are four clients (workers):

 - creator - populates the directory initially and also performs a final check
 - winner  - is syncing its local changes first
 - loser   - is syncing its local changes second (and optionally it looses the local sync database before doing the sync)
 - checker - only performs a final check (without having interacted with the system before)

Note: in 1.6 client conflict files are excluded by default - so they should be never propagated to the server
Note: in 1.5 the exclusion list should be provided separately to the client (FIXME)

Note on effects of removing local state db (1.6):

 - any files modified remotely or locally get a conflict if both remote and local replicas exist (FIXME: this could be possibly more refined in the future based on timestamps or content checksums)
 - any files not present locally but present remotely will be downloaded (so a deletion won't be propagated)

"""

from smashbox.utilities import *

import glob

filesizeKB = int(config.get('basicSync_filesizeKB',10000))

# True => remove local sync db on the loser
# False => keep the loser
rmLocalStateDB = bool(config.get('basicSync_rmLocalStateDB',False))

# subdirectory where to put files (if empty then use top level workdir)
subdirPath = config.get('basicSync_subdirPath',"")

#### testsets = [
####         { 'basicSync_filesizeKB': 1,
####           'basicSync_rmLocalStateDB':False
####         },
####         { 'basicSync_filesizeKB': 5000,
####           'basicSync_rmLocalStateDB':False
####         },
####         { 'basicSync_filesizeKB': 15000,
####           'basicSync_rmLocalStateDB':False
####         },
####         { 'basicSync_filesizeKB': 50000,
####           'basicSync_rmLocalStateDB':False
####         },
####
####         { 'basicSync_filesizeKB': 1,
####           'basicSync_rmLocalStateDB':True
####         },
####         { 'basicSync_filesizeKB': 5000,
####           'basicSync_rmLocalStateDB':True
####         },
####         { 'basicSync_filesizeKB': 15000,
####           'basicSync_rmLocalStateDB':True
####         },
####         { 'basicSync_filesizeKB': 50000,
####           'basicSync_rmLocalStateDB':True
####         }
#### ]
####

testsets = []

# create cartesian product of all test configurations
for s in [1, 5000, 15000, 50000]:
  for t in [True, False]:
      for p in [ "", "abc", "abc/abc", "abc/def/ghi" ]:
          testsets.append( { 'basicSync_filesizeKB':s,
                             'basicSync_rmLocalStateDB':t,
                             'basicSync_subdirPath':p } )

def expect_content(fn,md5):
    actual_md5 = md5sum(fn)
    error_check(actual_md5 == md5, "inconsistent md5 of %s: expected %s, got %s"%(fn,md5,actual_md5))

def expect_no_deleted_files(d):
    expect_deleted_files(d,[])

def expect_deleted_files(d,expected_deleted_files):
    actual_deleted_files = glob.glob(os.path.join(d,'*_DELETED*'))
    logger.debug('deleted files in %s: %s',d,actual_deleted_files)

    error_check(len(expected_deleted_files) == len(actual_deleted_files), "expected %d got %d deleted files"%(len(expected_deleted_files),len(actual_deleted_files)))

    for fn in expected_deleted_files:
        error_check(any([fn in dfn for dfn in actual_deleted_files]), "expected deleted file for %s not found"%fn)

def expect_conflict_files(d,expected_conflict_files):
    actual_conflict_files = glob.glob(os.path.join(d,'*_conflict-*-*'))

    logger.debug('conflict files in %s: %s',d,actual_conflict_files)

    error_check(len(expected_conflict_files) == len(actual_conflict_files), "expected %d got %d conflict files"%(len(expected_conflict_files),len(actual_conflict_files)))

    exp_basefns = [os.path.splitext(fn)[0] for fn in expected_conflict_files]

    logger.debug(exp_basefns)
    logger.debug(actual_conflict_files)

    for bfn in exp_basefns:
        error_check(any([bfn in fn for fn in actual_conflict_files]), "expected conflict file for %s not found"%bfn)

def expect_no_conflict_files(d):
    expect_conflict_files(d,[])

@add_worker
def creator(step):

    reset_owncloud_account()
    reset_rundir()

    step(1,'create initial content and sync')

    d = make_workdir()

    # we put the files in the subdir
    subdir = os.path.join(d,subdirPath)

    mkdir(subdir)

    # files *_NONE are not modified by anyone after initial sync
    # files *_LOSER are modified by the loser but not by the winner
    # files *_WINNER are modified by the winner but not by the loser
    # files *_BOTH are modified both by the winner and by the loser (always conflict on the loser)

    createfile(os.path.join(subdir,'TEST_FILE_MODIFIED_NONE.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_MODIFIED_LOSER.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_MODIFIED_WINNER.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_MODIFIED_BOTH.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_DELETED_LOSER.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_DELETED_WINNER.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_DELETED_BOTH.dat'),'0',count=1000,bs=filesizeKB)

    shared = reflection.getSharedObject()
    shared['md5_creator'] = md5sum(os.path.join(subdir,'TEST_FILE_MODIFIED_NONE.dat'))
    logger.info('md5_creator: %s',shared['md5_creator'])

    list_files(subdir)
    run_ocsync(d)
    list_files(subdir)

    step(7,'download the repository')
    run_ocsync(d,n=3)

    step(8,'final check')

    final_check(subdir,shared)
    expect_no_conflict_files(subdir)

@add_worker
def winner(step):
    step(2,'initial sync')

    d = make_workdir()
    subdir = os.path.join(d,subdirPath)

    run_ocsync(d)

    step(3,'modify locally and sync to server')

    list_files(subdir)

    remove_file(os.path.join(subdir,'TEST_FILE_DELETED_WINNER.dat'))
    remove_file(os.path.join(subdir,'TEST_FILE_DELETED_BOTH.dat'))

    createfile(os.path.join(subdir,'TEST_FILE_MODIFIED_WINNER.dat'),'1',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_MODIFIED_BOTH.dat'),'1',count=1000,bs=filesizeKB)

    createfile(os.path.join(subdir,'TEST_FILE_ADDED_WINNER.dat'),'1',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_ADDED_BOTH.dat'),'1',count=1000,bs=filesizeKB)

    shared = reflection.getSharedObject()
    shared['md5_winner'] = md5sum(os.path.join(subdir,'TEST_FILE_ADDED_WINNER.dat'))
    logger.info('md5_winner: %s',shared['md5_winner'])

    run_ocsync(d)

    step(5,'final sync')

    run_ocsync(d,n=3)

    step(8,'final check')

    final_check(subdir,shared)
    expect_no_conflict_files(subdir)

# this is the loser which lost it's local state db after initial sync

@add_worker
def loser(step):

    step(2,'initial sync')

    d = make_workdir()
    subdir = os.path.join(d,subdirPath)

    run_ocsync(d)

    step(4,'modify locally and sync to the server')

    list_files(subdir)

    # now do the local changes

    remove_file(os.path.join(subdir,'TEST_FILE_DELETED_LOSER.dat'))
    remove_file(os.path.join(subdir,'TEST_FILE_DELETED_BOTH.dat'))

    createfile(os.path.join(subdir,'TEST_FILE_MODIFIED_LOSER.dat'),'2',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_MODIFIED_BOTH.dat'),'2',count=1000,bs=filesizeKB)

    createfile(os.path.join(subdir,'TEST_FILE_ADDED_LOSER.dat'),'2',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_ADDED_BOTH.dat'),'2',count=1000,bs=filesizeKB)

    shared = reflection.getSharedObject()
    shared['md5_loser'] = md5sum(os.path.join(subdir,'TEST_FILE_ADDED_LOSER.dat'))
    logger.info('md5_loser: %s',shared['md5_loser'])

    #os.system('curl -v -s -k -XPROPFIND --data @/b/eos/CURL-TEST/p2.dat %s| xmllint --format -'%oc_webdav_url(remote_folder='TEST_FILE_MODIFIED_BOTH.dat'))
    #os.system('sqlite3 -line /tmp/smashdir/test_basicSync/loser/.csync_journal.db  \'select * from metadata where path like "%TEST_FILE_MODIFIED_BOTH%"\'')

    # remove the sync db
    if rmLocalStateDB:
        remove_file(os.path.join(d,'.csync_journal.db'))

    run_ocsync(d,n=3) # conflict file will be synced to the server but it requires more than one sync run

    step(6,'final sync')
    run_ocsync(d)

    step(8,'final check')

    #os.system('sqlite3 -line /tmp/smashdir/test_basicSync/loser/.csync_journal.db  \'select * from metadata where path like "%TEST_FILE_MODIFIED_BOTH%"\'')

    final_check(subdir,shared)
    if not rmLocalStateDB:
        expect_conflict_files(subdir, ['TEST_FILE_ADDED_BOTH.dat', 'TEST_FILE_MODIFIED_BOTH.dat' ])
    else:
        expect_conflict_files(subdir, ['TEST_FILE_ADDED_BOTH.dat', 'TEST_FILE_MODIFIED_BOTH.dat',
                                  'TEST_FILE_MODIFIED_LOSER.dat', 'TEST_FILE_MODIFIED_WINNER.dat']) # because the local and remote state is different and it is assumed that this is a conflict (FIXME: in the future timestamp-based last-restort check could improve this situation)

@add_worker
def checker(step):
    shared = reflection.getSharedObject()

    step(7,'download the repository for final verification')
    d = make_workdir()
    subdir = os.path.join(d,subdirPath)

    run_ocsync(d,n=3)

    step(8,'final check')

    final_check(subdir,shared)
    expect_no_conflict_files(subdir)

def final_check(d,shared):
    """ This is the final check applicable to all workers - this reflects the status of the remote repository so everyone should be in sync.
    The only potential differences are with locally generated conflict files.
    """

    list_files(d)
    expect_content(os.path.join(d,'TEST_FILE_MODIFIED_NONE.dat'), shared['md5_creator'])

    expect_content(os.path.join(d,'TEST_FILE_ADDED_LOSER.dat'), shared['md5_loser'])

    if not rmLocalStateDB:
        expect_content(os.path.join(d,'TEST_FILE_MODIFIED_LOSER.dat'), shared['md5_loser'])
    else:
        expect_content(os.path.join(d,'TEST_FILE_MODIFIED_LOSER.dat'), shared['md5_creator']) # in this case, a conflict is created on the loser and file on the server stays the same

    expect_content(os.path.join(d,'TEST_FILE_ADDED_WINNER.dat'), shared['md5_winner'])
    expect_content(os.path.join(d,'TEST_FILE_MODIFIED_WINNER.dat'), shared['md5_winner'])
    expect_content(os.path.join(d,'TEST_FILE_ADDED_BOTH.dat'), shared['md5_winner'])     # a conflict on the loser, server not changed
    expect_content(os.path.join(d,'TEST_FILE_MODIFIED_BOTH.dat'), shared['md5_winner'])  # a conflict on the loser, server not changed

    if not rmLocalStateDB:
        expect_no_deleted_files(d) # normally any deleted files should not come back
    else:
        expect_deleted_files(d, ['TEST_FILE_DELETED_LOSER.dat', 'TEST_FILE_DELETED_WINNER.dat']) # but not TEST_FILE_DELETED_BOTH.dat !
        expect_content(os.path.join(d,'TEST_FILE_DELETED_LOSER.dat'), shared['md5_creator']) # this file should be downloaded by the loser because it has no other choice (no previous state to compare with)
        expect_content(os.path.join(d,'TEST_FILE_DELETED_WINNER.dat'), shared['md5_creator']) # this file should be re-uploaded by the loser because it has no other choice (no previous state to compare with)

###############################################################################

def final_check_1_5(d): # this logic applies for 1.5.x client and owncloud server...
    """ Final verification: all local sync folders should look the same. We expect conflicts and handling of deleted files depending on the rmLocalStateDB option. See code for details.
    """
    import glob

    list_files(d)

    conflict_files = glob.glob(os.path.join(d,'*_conflict-*-*'))

    logger.debug('conflict files in %s: %s',d,conflict_files)

    if not rmLocalStateDB:
        # we expect exactly 1 conflict file

        logger.warning("FIXME: currently winner gets a conflict file - exclude list should be updated and this assert modified for the winner")

        error_check(len(conflict_files) == 1, "there should be exactly 1 conflict file (%d)"%len(conflict_files))
    else:
        # we expect exactly 3 conflict files
        error_check(len(conflict_files) == 3, "there should be exactly 3 conflict files (%d)"%len(conflict_files))

    for fn in conflict_files:

        if not rmLocalStateDB:
            error_check('_BOTH' in fn, """only files modified in BOTH workers have a conflict -  all other files should be conflict-free""")

        else:
            error_check('_BOTH' in fn or '_LOSER' in fn or '_WINNER' in fn, """files which are modified by ANY worker have a conflict now;  files which are not modified should not have a conflict""")

    deleted_files = glob.glob(os.path.join(d,'*_DELETED*'))

    logger.debug('deleted files in %s: %s',d,deleted_files)

    if not rmLocalStateDB:
        error_check(len(deleted_files) == 0, 'deleted files should not be there normally')
    else:
        # deleted files "reappear" if local sync db is lost on the loser, the only file that does not reappear is the DELETED_BOTH which was deleted on *all* local clients

        error_check(len(deleted_files) == 2, "we expect exactly 2 deleted files")

        for fn in deleted_files:
            error_check('_LOSER' in fn or '_WINNER' in fn, "deleted files should only reappear if delete on only one client (but not on both at the same time) ")
<|endoftext|>


<commit_before>from Cura.Qt.QtApplication import QtApplication
from Cura.Scene.SceneNode import SceneNode
from Cura.Scene.Camera import Camera
from Cura.Math.Vector import Vector

import os.path

class PrinterApplication(QtApplication):
    def __init__(self):
        super(PrinterApplication, self).__init__()

    def run(self):
        self._plugin_registry.loadPlugins({ "type": "Logger"})
        self._plugin_registry.loadPlugins({ "type": "StorageDevice" })
        self._plugin_registry.loadPlugins({ "type": "View" })
        self._plugin_registry.loadPlugins({ "type": "MeshHandler" })
        self._plugin_registry.loadPlugins({ "type": "Tool" })

        self.getController().setActiveView("MeshView")
        self.getController().setActiveTool("TransformTool")

        root = self.getController().getScene().getRoot()
        mesh = SceneNode(root)
        mesh.setMeshData(self.getMeshFileHandler().read("plugins/FileHandlers/STLReader/simpleTestCube.stl",self.getStorageDevice('local')))

        camera = Camera('3d', root)
        camera.translate(Vector(0, 5, 5))
        camera.rotateByAngleAxis(45, Vector(1, 0, 0))

        camera = Camera('left', root)
        camera.translate(Vector(5, 0, 0))
        camera.rotateByAngleAxis(-90, Vector(0, 1, 0))
        camera.setLocked(True)

        self.getController().getScene().setActiveCamera('3d')

        self.log('i',"Application started")

        self.setMainQml(os.path.dirname(__file__) + "/Printer.qml")
        self.initializeEngine()

        if self._engine.rootObjects:
            self.exec_()

    def registerObjects(self, engine):
        pass
<commit_msg>Don't set an active tool on application start<commit_after>from Cura.Qt.QtApplication import QtApplication
from Cura.Scene.SceneNode import SceneNode
from Cura.Scene.Camera import Camera
from Cura.Math.Vector import Vector

import os.path

class PrinterApplication(QtApplication):
    def __init__(self):
        super(PrinterApplication, self).__init__()

    def run(self):
        self._plugin_registry.loadPlugins({ "type": "Logger"})
        self._plugin_registry.loadPlugins({ "type": "StorageDevice" })
        self._plugin_registry.loadPlugins({ "type": "View" })
        self._plugin_registry.loadPlugins({ "type": "MeshHandler" })
        self._plugin_registry.loadPlugins({ "type": "Tool" })

        self.getController().setActiveView("MeshView")

        root = self.getController().getScene().getRoot()
        mesh = SceneNode(root)
        mesh.setMeshData(self.getMeshFileHandler().read("plugins/FileHandlers/STLReader/simpleTestCube.stl",self.getStorageDevice('local')))

        camera = Camera('3d', root)
        camera.translate(Vector(0, 5, 5))
        camera.rotateByAngleAxis(45, Vector(1, 0, 0))

        camera = Camera('left', root)
        camera.translate(Vector(5, 0, 0))
        camera.rotateByAngleAxis(-90, Vector(0, 1, 0))
        camera.setLocked(True)

        self.getController().getScene().setActiveCamera('3d')

        self.log('i',"Application started")

        self.setMainQml(os.path.dirname(__file__) + "/Printer.qml")
        self.initializeEngine()

        if self._engine.rootObjects:
            self.exec_()

    def registerObjects(self, engine):
        pass
<|endoftext|>


<commit_before>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Josh Hagins
# Copyright (c) 2014 Josh Hagins
#
# License: MIT
#

"""This module exports the Scalac plugin class."""

import json
from os import path
from distutils.versionpredicate import VersionPredicate

from SublimeLinter.lint import Linter, util

def memoize(function):
    """Memoization decorator for a function taking arbitrary arguments."""

    memo = {}

    def wrapper(*args):
        if args in memo:
            return memo[args]
        else:
            rv = function(*args)
            memo[args] = rv
            return rv

    return wrapper

class Scalac(Linter):

    """Provides an interface to scalac."""

    syntax = 'scala'
    executable = 'scalac'
    cmd = None
    version_args = '-version'
    version_re = r'(?P<version>\d+\.\d+\.\d+)'
    version_requirement = '>= 2.9.1'
    regex = (
        r'^(?P<file>.+?):(?P<line>\d+): '
        r'(?:(?P<error>error)|(?P<warning>warning)): '
        r'(?:\[.+?\] )?(?P<message>[^\r\n]+)\r?\n'
        r'[^\r\n]+\r?\n'
        r'(?P<col>[^\^]*)\^'
    )
    multiline = True
    tempfile_suffix = '-'
    error_stream = util.STREAM_STDERR
    defaults = {
        'lint': '',
        'classpath_filename': ''
    }
    inline_settings = 'classpath_filename'
    inline_overrides = 'lint'
    comment_re = r'\s*/[/*]'

    # Internal
    __version_satisfies = {}

    @classmethod
    def version_satisfies(cls, req):
        """Return whether executable_version satisfies req."""

        if req not in cls.__version_satisfies:
            predicate = VersionPredicate('SublimeLinter.scalac ({})'.format(req))
            cls.__version_satisfies[req] = predicate.satisfied_by(cls.executable_version)

        return cls.__version_satisfies[req]

    def cmd(self):
        """
        Return the command line to execute.

        We override this because we have to munge the -Xlint argument
        based on the 'lint' setting.

        """

        command = [self.executable_path, '-encoding', 'UTF8']

        settings = self.get_view_settings()
        user_rules = set(settings.get('lint').split(','))

        # Positive overrides negative
        user_rules = {
            name for name in user_rules
            if not (name.startswith('-') and name[1:] in user_rules)
        }

        valid_rules = (rule for rule in self.all_rules if rule.is_valid)

        for name in user_rules:
            neg = name.startswith('-')
            name = name[1:] if neg else name
            rule = valid_rules.get(name)
            if rule is not None:
                rule.disable() if neg else rule.enable()

        rule_flags = ' '.join(rule.flag for rule in valid_rules)

        classpath_filename = settings.get('classpath_filename')

        if classpath_filename:
            classpath = self.get_classpath(classpath_filename)
            if classpath:
                command += ['-classpath', classpath]

        return command + [rule_flags, '*']

    @property
    def all_rules(self):
        """Return dict containing all rules as name: rule."""

        if not hasattr(self, '__rules'):
            with open('rules.json') as json_data:
                rules = json.load(json_data)
                self.__rules = {rule['name']: Rule(rule) for rule in rules}

        return self.__rules

    @memoize
    def get_classpath(self, classpath_filename):
        """Read classpath from file and return as str."""

        classpath_file = util.find_file(
            path.dirname(self.filename),
            classpath_filename
        )

        if classpath_file:
            with open(classpath_file, 'r') as cp:
                return cp.read().strip()

    def split_match(self, match):
        """
        Return the components of the match.

        We override this because scalac lints all referenced files,
        and we only want errors from the linted file.

        """

        if match:
            if match.group('file') != self.filename:
                match = None

        return super().split_match(match)

class Rule:

    """
    A scalac rule.

    Attributes:
        name (str): The name of this rule.
        description (str): A description of this rule.

    """

    def __init__(self, json_data):
        """
        Initialize a new Rule from the given JSON data.

        Args:
            json_data (dict): JSON representation of rule.

        """

        self.name = json_data['name']
        self.description = json_data['description']
        self.__flag = json_data['flag']
        self.__version_req = json_data['version']
        self.__valid = None
        self.__default = json_data['default']
        self.__enabled = self.is_default

    @property
    def flag(self):
        """Get the formatted CLI flag for this rule."""

        if self.__flag == '-Xlint':
            neg = '-' if not self.__enabled else ''
            return '{}:{}{}'.format(self.__flag, neg, self.name)
        else:
            neg = ':false' if not self.__enabled else ''
            return '{}{}'.format(self.__flag, neg)

    @property
    def is_default(self):
        """Return whether this rule is enabled by default for the current Scala version."""

        if not self.is_valid:
            return False
        elif isinstance(self.__default, str):
            self.__default = Scalac.version_satisfies(self.__default)

        return self.__default

    @property
    def is_valid(self):
        """Return whether this rule is valid for the current Scala version."""

        if self.__valid is None:
            self.__valid = Scalac.version_satisfies(self.__version_req)

        return self.__valid

    def enable(self):
        """Enable this rule."""
        self.__enabled = True

    def disable(self):
        """Disable this rule."""
        self.__enabled = False
<commit_msg>Stupid mistake; need to use a dict generator<commit_after>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Josh Hagins
# Copyright (c) 2014 Josh Hagins
#
# License: MIT
#

"""This module exports the Scalac plugin class."""

import json
from os import path
from distutils.versionpredicate import VersionPredicate

from SublimeLinter.lint import Linter, util

def memoize(function):
    """Memoization decorator for a function taking arbitrary arguments."""

    memo = {}

    def wrapper(*args):
        if args in memo:
            return memo[args]
        else:
            rv = function(*args)
            memo[args] = rv
            return rv

    return wrapper

class Scalac(Linter):

    """Provides an interface to scalac."""

    syntax = 'scala'
    executable = 'scalac'
    cmd = None
    version_args = '-version'
    version_re = r'(?P<version>\d+\.\d+\.\d+)'
    version_requirement = '>= 2.9.1'
    regex = (
        r'^(?P<file>.+?):(?P<line>\d+): '
        r'(?:(?P<error>error)|(?P<warning>warning)): '
        r'(?:\[.+?\] )?(?P<message>[^\r\n]+)\r?\n'
        r'[^\r\n]+\r?\n'
        r'(?P<col>[^\^]*)\^'
    )
    multiline = True
    tempfile_suffix = '-'
    error_stream = util.STREAM_STDERR
    defaults = {
        'lint': '',
        'classpath_filename': ''
    }
    inline_settings = 'classpath_filename'
    inline_overrides = 'lint'
    comment_re = r'\s*/[/*]'

    # Internal
    __version_satisfies = {}

    @classmethod
    def version_satisfies(cls, req):
        """Return whether executable_version satisfies req."""

        if req not in cls.__version_satisfies:
            predicate = VersionPredicate('SublimeLinter.scalac ({})'.format(req))
            cls.__version_satisfies[req] = predicate.satisfied_by(cls.executable_version)

        return cls.__version_satisfies[req]

    def cmd(self):
        """
        Return the command line to execute.

        We override this because we have to munge the -Xlint argument
        based on the 'lint' setting.

        """

        command = [self.executable_path, '-encoding', 'UTF8']

        settings = self.get_view_settings()
        user_rules = set(settings.get('lint').split(','))

        # Positive overrides negative
        user_rules = {
            name for name in user_rules
            if not (name.startswith('-') and name[1:] in user_rules)
        }

        valid_rules = {
            name: rule for (name, rule) in self.all_rules.items()
            if rule.is_valid
        }

        for name in user_rules:
            neg = name.startswith('-')
            name = name[1:] if neg else name
            rule = valid_rules.get(name)
            if rule is not None:
                rule.disable() if neg else rule.enable()

        rule_flags = ' '.join(rule.flag for rule in valid_rules)

        classpath_filename = settings.get('classpath_filename')

        if classpath_filename:
            classpath = self.get_classpath(classpath_filename)
            if classpath:
                command += ['-classpath', classpath]

        return command + [rule_flags, '*']

    @property
    def all_rules(self):
        """Return dict containing all rules as name: rule."""

        if not hasattr(self, '__rules'):
            with open('rules.json') as json_data:
                rules = json.load(json_data)
                self.__rules = {rule['name']: Rule(rule) for rule in rules}

        return self.__rules

    @memoize
    def get_classpath(self, classpath_filename):
        """Read classpath from file and return as str."""

        classpath_file = util.find_file(
            path.dirname(self.filename),
            classpath_filename
        )

        if classpath_file:
            with open(classpath_file, 'r') as cp:
                return cp.read().strip()

    def split_match(self, match):
        """
        Return the components of the match.

        We override this because scalac lints all referenced files,
        and we only want errors from the linted file.

        """

        if match:
            if match.group('file') != self.filename:
                match = None

        return super().split_match(match)

class Rule:

    """
    A scalac rule.

    Attributes:
        name (str): The name of this rule.
        description (str): A description of this rule.

    """

    def __init__(self, json_data):
        """
        Initialize a new Rule from the given JSON data.

        Args:
            json_data (dict): JSON representation of rule.

        """

        self.name = json_data['name']
        self.description = json_data['description']
        self.__flag = json_data['flag']
        self.__version_req = json_data['version']
        self.__valid = None
        self.__default = json_data['default']
        self.__enabled = self.is_default

    @property
    def flag(self):
        """Get the formatted CLI flag for this rule."""

        if self.__flag == '-Xlint':
            neg = '-' if not self.__enabled else ''
            return '{}:{}{}'.format(self.__flag, neg, self.name)
        else:
            neg = ':false' if not self.__enabled else ''
            return '{}{}'.format(self.__flag, neg)

    @property
    def is_default(self):
        """Return whether this rule is enabled by default for the current Scala version."""

        if not self.is_valid:
            return False
        elif isinstance(self.__default, str):
            self.__default = Scalac.version_satisfies(self.__default)

        return self.__default

    @property
    def is_valid(self):
        """Return whether this rule is valid for the current Scala version."""

        if self.__valid is None:
            self.__valid = Scalac.version_satisfies(self.__version_req)

        return self.__valid

    def enable(self):
        """Enable this rule."""
        self.__enabled = True

    def disable(self):
        """Disable this rule."""
        self.__enabled = False
<|endoftext|>


<commit_before>from IPython.kernel.zmq.kernelbase import Kernel
from pexpect import replwrap
import pexpect

import signal
from subprocess import check_output
import re

__version__ = '0.2'

version_pat = re.compile(r'version (\d+(\.\d+)+)')

class BashKernel(Kernel):
    implementation = 'bash_kernel'
    implementation_version = __version__
    language = 'bash'
    @property
    def language_version(self):
        m = version_pat.search(self.banner)
        return m.group(1)

    _banner = None
    @property
    def banner(self):
        if self._banner is None:
            self._banner = check_output(['bash', '--version']).decode('utf-8')
        return self._banner

    def __init__(self, **kwargs):
        Kernel.__init__(self, **kwargs)
        # Signal handlers are inherited by forked processes, and we can't easily
        # reset it from the subprocess. Since kernelapp ignores SIGINT except in
        # message handlers, we need to temporarily reset the SIGINT handler here
        # so that bash and its children are interruptible.
        sig = signal.signal(signal.SIGINT, signal.SIG_DFL)
        try:
            self.bashwrapper = replwrap.bash()
        finally:
            signal.signal(signal.SIGINT, sig)

    def do_execute(self, code, silent, store_history=True, user_expressions=None,
                   allow_stdin=False):
        if not code.strip():
            return {'status': 'ok', 'execution_count': self.execution_count,
                    'payload': [], 'user_expressions': {}}

        interrupted = False
        try:
            output = self.bashwrapper.run_command(code.rstrip(), timeout=None)
        except KeyboardInterrupt:
            self.bashwrapper.child.sendintr()
            interrupted = True
            self.bashwrapper._expect_prompt()
            output = self.bashwrapper.child.before
        except pexpect.EOF:
            # TODO: how do we shut down gracefully here?
            output = ''

        if not silent:
            stream_content = {'name': 'stdout', 'data': output}
            self.send_response(self.iopub_socket, 'stream', stream_content)

        if interrupted:
            return {'status': 'abort', 'execution_count': self.execution_count}

        try:
            exitcode = int(self.bashwrapper.run_command('echo $?').rstrip())
        except Exception:
            exitcode = 1

        if exitcode:
            return {'status': 'error', 'execution_count': self.execution_count,
                    'ename': '', 'evalue': str(exitcode), 'traceback': []}
        else:
            return {'status': 'ok', 'execution_count': self.execution_count,
                    'payload': [], 'user_expressions': {}}

    def do_complete(self, code, cursor_pos):
        code = code[:cursor_pos]
        default = {'matches': [], 'cursor_start': 0,
                   'cursor_end': cursor_pos, 'metadata': dict(),
                   'status': 'ok'}
        if code[-1] == ' ':
            return default
        tokens = code.replace(';', ' ').split()
        if not tokens:
            return default
        token = tokens[-1]
        start = cursor_pos - len(token)
        cmd = 'compgen -cdfa %s' % token
        output = self.bashwrapper.run_command(cmd).rstrip()
        matches = output.split()
        if not matches:
            return default
        matches = [m for m in matches if m.startswith(token)]
        return {'matches': matches, 'cursor_start': start,
                'cursor_end': cursor_pos, 'metadata': dict(),
                'status': 'ok'}

if __name__ == '__main__':
    from IPython.kernel.zmq.kernelapp import IPKernelApp
    IPKernelApp.launch_instance(kernel_class=BashKernel)
<commit_msg>Fix possible IndexError and add whitespace<commit_after>from IPython.kernel.zmq.kernelbase import Kernel
from pexpect import replwrap
import pexpect

import signal
from subprocess import check_output
import re

__version__ = '0.2'

version_pat = re.compile(r'version (\d+(\.\d+)+)')

class BashKernel(Kernel):
    implementation = 'bash_kernel'
    implementation_version = __version__
    language = 'bash'
    @property
    def language_version(self):
        m = version_pat.search(self.banner)
        return m.group(1)

    _banner = None
    @property
    def banner(self):
        if self._banner is None:
            self._banner = check_output(['bash', '--version']).decode('utf-8')
        return self._banner

    def __init__(self, **kwargs):
        Kernel.__init__(self, **kwargs)
        # Signal handlers are inherited by forked processes, and we can't easily
        # reset it from the subprocess. Since kernelapp ignores SIGINT except in
        # message handlers, we need to temporarily reset the SIGINT handler here
        # so that bash and its children are interruptible.
        sig = signal.signal(signal.SIGINT, signal.SIG_DFL)
        try:
            self.bashwrapper = replwrap.bash()
        finally:
            signal.signal(signal.SIGINT, sig)

    def do_execute(self, code, silent, store_history=True, user_expressions=None,
                   allow_stdin=False):
        if not code.strip():
            return {'status': 'ok', 'execution_count': self.execution_count,
                    'payload': [], 'user_expressions': {}}

        interrupted = False
        try:
            output = self.bashwrapper.run_command(code.rstrip(), timeout=None)
        except KeyboardInterrupt:
            self.bashwrapper.child.sendintr()
            interrupted = True
            self.bashwrapper._expect_prompt()
            output = self.bashwrapper.child.before
        except pexpect.EOF:
            # TODO: how do we shut down gracefully here?
            output = ''

        if not silent:
            stream_content = {'name': 'stdout', 'data': output}
            self.send_response(self.iopub_socket, 'stream', stream_content)

        if interrupted:
            return {'status': 'abort', 'execution_count': self.execution_count}

        try:
            exitcode = int(self.bashwrapper.run_command('echo $?').rstrip())
        except Exception:
            exitcode = 1

        if exitcode:
            return {'status': 'error', 'execution_count': self.execution_count,
                    'ename': '', 'evalue': str(exitcode), 'traceback': []}
        else:
            return {'status': 'ok', 'execution_count': self.execution_count,
                    'payload': [], 'user_expressions': {}}

    def do_complete(self, code, cursor_pos):
        code = code[:cursor_pos]
        default = {'matches': [], 'cursor_start': 0,
                   'cursor_end': cursor_pos, 'metadata': dict(),
                   'status': 'ok'}

        if not code or code[-1] == ' ':
            return default

        tokens = code.replace(';', ' ').split()
        if not tokens:
            return default

        token = tokens[-1]
        start = cursor_pos - len(token)
        cmd = 'compgen -cdfa %s' % token
        output = self.bashwrapper.run_command(cmd).rstrip()

        matches = output.split()
        if not matches:
            return default
        matches = [m for m in matches if m.startswith(token)]

        return {'matches': matches, 'cursor_start': start,
                'cursor_end': cursor_pos, 'metadata': dict(),
                'status': 'ok'}

if __name__ == '__main__':
    from IPython.kernel.zmq.kernelapp import IPKernelApp
    IPKernelApp.launch_instance(kernel_class=BashKernel)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Parse the NCES Data File Format Doc and create parsing instructions for the main file
"""

import argparse
import re
import os
import csv

# Unit testing
import unittest

# ==============================================================================
# Constants and RegEx
# ==============================================================================
re_idx_header = re.compile(r'Name\s+Order\s+Type\s+Description')

# Name    Type   Position  Size  Description
re_definition = re.compile(r'^(\w+)\s+(\w+)\s+(\d+)[-](\d+)\s+(\d+)[*]?\s+(.*)$')
re_sub_definition = re.compile(r'^\s*[+](\w+)\s+(\w+)\s+(\d+)[-](\d+)\s+(\d+)\s+(.*)$')

# Variable    Start   End     Field   Data
# Name        Pos.    Pos.    Length  Type    Description
re_alt_definition = re.compile(r'^(\w+)\s+(\d+)\s+(\d+)\s+(\d+)*?\s+(\w+)\s+(.*)$')
re_alt_sub_definition = re.compile(r'^\s*[+](\w+)\s+(\d+)\s+(\d+)\s+(\d+)*?\s+(\w+)\s+(.*)$')

# Index Data Format
# Variable             Data
# Name          Order  Type   Description
re_idx_definition = re.compile(r'^(\w+)\s+(\d+)[*]?\s+(\w+)\s+(.*)$')

datafile_name = "nces%02d-%02d.txt"
formatfile_name = "nces%02d-%02d_layout.txt"

# ==============================================================================
# Utility Functions
# ==============================================================================
# --------------------------------------
def strip_comment(line, comment_char='#'):
    """
    Strip inline comments -
    """
    idx = line.find(comment_char)
    if idx != -1:
        line = line[:idx]
    return line.strip()

# --------------------------------------
def longlines(rawdata):
    """
    Generator to merge lines in a text file that end with a "\"
    """
    lines = []
    for i in rawdata.splitlines():
        if i.endswith("\\"):
            lines.append(i.rstrip("\\"))
        else:
            lines.append(i)
            yield "".join(lines)
            lines = []
    if len(lines) > 0: yield "".join(lines)

# ==============================================================================
# Parser Class
# ==============================================================================
class NCESParser(object):
    """
    Parsing Instructions for an NCES data file.

    The instructions consist of list of columns in the database along
    with information needed to pull the column from the file.  Typically
    it will be a string index:

        [('COLUMN_NAME', idx), ('COLUMN_NAME', idx), ...]

    """
    def __init__(self, year=None, formatfile="", debug=False):
        self.debug = debug
        self.parse_instr = []
        self.header_count = 0
        self.headers = []
        self.descriptions = {}
        self.index_mode = 0
        self.save_names = [
            "FIPS",
            "FIPST",
            "BLACK",
            "WHITE",
            "MEMBER",
            "GSHI",
            "LEAID"
        ]

        if year:
            self.year = year
            self.formatfile = self.get_formatfile_name()
        else:
            self.formatfile = formatfile
            self.year = 0
        # Load the network
        self.read_formatfile(self.formatfile)

    def __repr__(self):
        results = ""
        for instr in self.parse_instr:
            if self.index_mode:
                results += "Name:  %s, Index: %d\n" % (instr[0], instr[2])
            else:
                results += "Name:  %s, Size: %d\n" % (instr[0], instr[3] - instr[2] + 1)
        return results

    # --------------------------------------
    def get_formatfile_name(self):
        return self.get_filename(formatfile_name)

    # --------------------------------------
    def get_datafile_name(self):
        return self.get_filename(datafile_name)

    # --------------------------------------
    def get_filename(self, name_str):
        """
        Construct a filename from the base name, a year and
        the local directory structure.
        """
        fname = name_str % (self.year%100, (self.year+1)%100)
        this_dir, this_filename = os.path.split(__file__)
        fname = os.path.join(this_dir, 'data', fname)
        return fname

    # ==============================================================================
    # Read the Format file into a usable data structure
    # ==============================================================================
    # --------------------------------------
    def read_formatfile(self, formatfile):
        if self.debug:
            print "=" * 80
            print "Reading Format File:  %s" % formatfile
            print "=" * 80
        fh = open(formatfile, 'rb')

        for line in fh:
            if re_idx_header.search(line):
                if self.index_mode == 0:
                    print "Switching to Index MODE!!!"
                    self.index_mode = 1

            if re_definition.search(line):
                col_name, type, loidx, hiidx, size, description = re_definition.search(line).groups()
            elif re_sub_definition.search(line):
                col_name, type, loidx, hiidx, size, description = re_sub_definition.search(line).groups()
            elif re_alt_definition.search(line):
                col_name, loidx, hiidx, size, type, description = re_alt_definition.search(line).groups()
            elif re_alt_sub_definition.search(line):
                col_name, loidx, hiidx, size, type, description = re_alt_sub_definition.search(line).groups()
            elif self.index_mode and re_idx_definition.search(line):
                col_name, loidx, type, description = re_idx_definition.search(line).groups()
                hiidx = loidx
            else:
                if self.debug:
                    print line
                continue

            self.add_instr(col_name, type, loidx, hiidx, size, description)

        if self.debug:
            print "=" * 80
            print "Format File Parsing Complete"
            print "=" * 80
            import pprint
            pprint.pprint(self.parse_instr)
            print "=" * 80
            print "\n"

    # --------------------------------------
    def add_instr(self, col_name, type, loidx, hiidx, size, description):
        if self.debug:
            print "Found Column:  %s - %s" % (col_name, size)

        # Strip the year off the column if it is present
        # We store the year in the main data object
        if col_name[-2:].isdigit():
            col_name = col_name[:-2]

        # Is it a number?
        if type == 'N':
            pass

        if col_name in self.save_names:
            self.parse_instr.append((col_name, type, int(loidx)-1, int(hiidx), description.strip()))
            self.add_column(col_name, description)

    # --------------------------------------
    def add_column(self, col_name, description):
        self.headers.append(col_name)
        desc = description.strip().split('\t')[0]   # Filter out any Tab characters
        self.descriptions[col_name] = desc
        self.header_count += 1

    # --------------------------------------
    def get_headers(self):
        return ",".join(self.header.keys())

    # --------------------------------------
    def get_idx(self, col_name):
        for i, name in enumerate(self.headers):
            if name == col_name:
                return i
        raise Exception("Invalid Column Name:  %s" % col_name)

    # --------------------------------------
    def get_descriptions(self):
        return ",".join(self.descriptions)

    # --------------------------------------
    def parse_line(self, line):
        entry = []
        for instr in self.parse_instr:
            field = line[instr[2]:instr[3]]   # Python array slicing rules  low_idx : high_idx + 1
            if self.debug:
                print field
            if instr[1] == 'N':   # Number Type
                try:
                    field = int(field)
                except ValueError:
                    field = 0    # Squash invalid values to 0 for fields of type 'Number'
            else:
                field.strip()    # Otherwise clean up the string
            entry.append(field)

        return entry

    # --------------------------------------
    def parse(self, datafile="", make_dict=False):

        if datafile:
            fname = datafile
        else:
            fname = self.get_datafile_name()
        fh = open(fname, 'rb')
        if self.index_mode:
            fh = csv.reader(fh, dialect='excel-tab')
            line = fh.next() # Pop the header line

        self.schools = []
        if self.index_mode:
            for line in fh:
                if make_dict:
                    self.schools.append(self.make_dict(line))  # CSV already breaks it apart
                else:
                    self.schools.append(line)  # CSV already breaks it apart
        else:
            for line in fh:
                if make_dict:
                    self.schools.append(self.make_dict(self.parse_line(line)))
                else:
                    self.schools.append(self.parse_line(line))

        if self.debug:
            print len(self.schools)
        return self.schools

    # --------------------------------------
    def make_dict(self, school):
        if self.debug:
            print school
        return dict(zip(self.headers, school))

# *****************************************************************************
# Unit Tests
# *****************************************************************************
class TestBasicNetwork(unittest.TestCase):

    def setUp(self):
        self.parse = NCESParser('data/school.test', 'data/school.format')

    def test_old_style(self):
        self.assertEqual(self.parse.something, ["Test"])

# *****************************************************************************
# Program Flow
# *****************************************************************************
# -------------------------------------
# Parse the command line options
# -------------------------------------
def main():
    parser = argparse.ArgumentParser(description='NCES Data File Parser')
    parser.add_argument('--year', action='store', dest='year', required=False, type=int,
            help='NCES Data Year - Standard filenames assumed')
    parser.add_argument('--formatfile', action='store', dest='formatfile', required=False,
            help='NCES Data File Record Layout')
    parser.add_argument('--datafile', action='store', dest='datafile', required=False,
            help='NCES Data File')
    parser.add_argument('-debug', action='store_true',
            help='Print Debug Messages')
    args = parser.parse_args()
    # print args

    # -------------------------------------
    # Actually do the work we intend to do here
    # -------------------------------------
    if args.year:
        parse = NCESParser(year=args.year, debug=args.debug)
    elif args.formatfile:
        parse = NCESParser(formatfile=args.formatfile, debug=args.debug)
    else:
        raise Exception("Please select a year or an NCES Record Layout file name")

    print "=" * 80
    print parse
    print "=" * 80
    print parse.get_idx('GSHI')

    if args.year:
        schools = parse.parse()
    elif args.formatfile:
        schools = parse.parse(args.datafile)
    else:
        raise Exception("Please define a year when constructing or specify an NCES Datefile Layout file name")

    print "=" * 80
    print len(schools)

    print "=" * 80
    print schools[0]
    print schools[22000]
    import pprint
    pprint.pprint(parse.make_dict(schools[22000]))

    print "=" * 80
    print "Headers"
    print "=" * 80
    print parse.headers

    print "=" * 80
    print "Descriptions"
    print "=" * 80
    pprint.pprint(parse.descriptions)

# -------------------------------------
# Drop the script name from the args
# and call our command line parser
# -------------------------------------
if __name__ == "__main__":
    main()

<commit_msg>Not sure that this was what I want exactly, but I modified nces_parse to filter out much of the data.  This can be overriden from an calling object, but it is not so nicely built.<commit_after>#!/usr/bin/env python
"""
Parse the NCES Data File Format Doc and create parsing instructions for the main file
"""

import argparse
import re
import os
import csv

# Unit testing
import unittest

# ==============================================================================
# Constants and RegEx
# ==============================================================================
re_idx_header = re.compile(r'Name\s+Order\s+Type\s+Description')

# Name    Type   Position  Size  Description
re_definition = re.compile(r'^(\w+)\s+(\w+)\s+(\d+)[-](\d+)\s+(\d+)[*]?\s+(.*)$')
re_sub_definition = re.compile(r'^\s*[+](\w+)\s+(\w+)\s+(\d+)[-](\d+)\s+(\d+)\s+(.*)$')

# Variable    Start   End     Field   Data
# Name        Pos.    Pos.    Length  Type    Description
re_alt_definition = re.compile(r'^(\w+)\s+(\d+)\s+(\d+)\s+(\d+)*?\s+(\w+)\s+(.*)$')
re_alt_sub_definition = re.compile(r'^\s*[+](\w+)\s+(\d+)\s+(\d+)\s+(\d+)*?\s+(\w+)\s+(.*)$')

# Index Data Format
# Variable             Data
# Name          Order  Type   Description
re_idx_definition = re.compile(r'^[+]?(\w+)\s+(\d+)[*]?\s+(\w+)\s+(.*)$')

datafile_name = "nces%02d-%02d.txt"
formatfile_name = "nces%02d-%02d_layout.txt"

# ==============================================================================
# Utility Functions
# ==============================================================================
# --------------------------------------
def strip_comment(line, comment_char='#'):
    """
    Strip inline comments -
    """
    idx = line.find(comment_char)
    if idx != -1:
        line = line[:idx]
    return line.strip()

# --------------------------------------
def longlines(rawdata):
    """
    Generator to merge lines in a text file that end with a "\"
    """
    lines = []
    for i in rawdata.splitlines():
        if i.endswith("\\"):
            lines.append(i.rstrip("\\"))
        else:
            lines.append(i)
            yield "".join(lines)
            lines = []
    if len(lines) > 0: yield "".join(lines)

# ==============================================================================
# Parser Class
# ==============================================================================
class NCESParser(object):
    """
    Parsing Instructions for an NCES data file.

    The instructions consist of list of columns in the database along
    with information needed to pull the column from the file.  Typically
    it will be a string index:

        [('COLUMN_NAME', idx), ('COLUMN_NAME', idx), ...]

    """
    def __init__(self, year=None, formatfile="", debug=False):
        self.debug = debug
        self.parse_instr = []
        self.header_count = 0
        self.headers = []
        self.descriptions = {}
        self.index_mode = 0
        self.save_names = [
            "FIPS",
            "BLACK",
            "WHITE",
            "MEMBER",
            "GSHI",
            "LEAID"
        ]

        if year:
            self.year = year
            self.formatfile = self.get_formatfile_name()
        else:
            self.formatfile = formatfile
            self.year = 0
        # Load the network
        self.read_formatfile(self.formatfile)

    def __repr__(self):
        results = ""
        for instr in self.parse_instr:
            if self.index_mode:
                results += "Name:  %s, Index: %d\n" % (instr[0], instr[2])
            else:
                results += "Name:  %s, Size: %d\n" % (instr[0], instr[3] - instr[2] + 1)
        return results

    # --------------------------------------
    def get_formatfile_name(self):
        return self.get_filename(formatfile_name)

    # --------------------------------------
    def get_datafile_name(self):
        return self.get_filename(datafile_name)

    # --------------------------------------
    def get_filename(self, name_str):
        """
        Construct a filename from the base name, a year and
        the local directory structure.
        """
        fname = name_str % (self.year%100, (self.year+1)%100)
        this_dir, this_filename = os.path.split(__file__)
        fname = os.path.join(this_dir, 'data', fname)
        return fname

    # ==============================================================================
    # Read the Format file into a usable data structure
    # ==============================================================================
    # --------------------------------------
    def read_formatfile(self, formatfile):
        if self.debug:
            print "=" * 80
            print "Reading Format File:  %s" % formatfile
            print "=" * 80
        fh = open(formatfile, 'rb')

        for line in fh:
            if re_idx_header.search(line):
                if self.index_mode == 0:
                    print "Switching to Index MODE!!!"
                    self.index_mode = 1

            if re_definition.search(line):
                col_name, type, loidx, hiidx, size, description = re_definition.search(line).groups()
            elif re_sub_definition.search(line):
                col_name, type, loidx, hiidx, size, description = re_sub_definition.search(line).groups()
            elif re_alt_definition.search(line):
                col_name, loidx, hiidx, size, type, description = re_alt_definition.search(line).groups()
            elif re_alt_sub_definition.search(line):
                col_name, loidx, hiidx, size, type, description = re_alt_sub_definition.search(line).groups()
            elif self.index_mode and re_idx_definition.search(line):
                col_name, loidx, type, description = re_idx_definition.search(line).groups()
                hiidx = loidx
                size = 0
            else:
                if self.debug:
                    print line
                continue

            # Filter out a problematic NCES year to year change
            if col_name == "FIPST":
                col_name = "FIPS"
            self.add_instr(col_name, type, loidx, hiidx, size, description)

        if self.debug:
            print "=" * 80
            print "Format File Parsing Complete"
            print "=" * 80
            import pprint
            pprint.pprint(self.parse_instr)
            print "=" * 80
            print "\n"

    # --------------------------------------
    def add_instr(self, col_name, type, loidx, hiidx, size, description):
        if self.debug:
            print "Found Column:  %s - %s" % (col_name, size)

        # Strip the year off the column if it is present
        # We store the year in the main data object
        if col_name[-2:].isdigit():
            col_name = col_name[:-2]

        # Is it a number?
        if type == 'N':
            pass

        if col_name in self.save_names:
            self.parse_instr.append((col_name, type, int(loidx)-1, int(hiidx), description.strip()))
            self.add_column(col_name, description)

    # --------------------------------------
    def add_column(self, col_name, description):
        self.headers.append(col_name)
        desc = description.strip().split('\t')[0]   # Filter out any Tab characters
        self.descriptions[col_name] = desc
        self.header_count += 1

    # --------------------------------------
    def get_headers(self):
        return ",".join(self.header.keys())

    # --------------------------------------
    def get_idx(self, col_name):
        for i, name in enumerate(self.headers):
            if name == col_name:
                return i
        raise Exception("Invalid Column Name:  %s" % col_name)

    # --------------------------------------
    def get_descriptions(self):
        return ",".join(self.descriptions)

    # --------------------------------------
    def parse_line(self, line):
        entry = []
        for instr in self.parse_instr:
            if self.index_mode:
                field = line[instr[2]]
            else:
                field = line[instr[2]:instr[3]]   # Python array slicing rules  low_idx : high_idx + 1
            if self.debug:
                print field
            if instr[1] == 'N':   # Number Type
                try:
                    field = int(field)
                except ValueError:
                    field = 0    # Squash invalid values to 0 for fields of type 'Number'
            else:
                field.strip()    # Otherwise clean up the string
            entry.append(field)

        return entry

    # --------------------------------------
    def parse(self, datafile="", make_dict=False):

        if datafile:
            fname = datafile
        else:
            fname = self.get_datafile_name()
        fh = open(fname, 'rb')
        if self.index_mode:
            fh = csv.reader(fh, dialect='excel-tab')
            line = fh.next() # Pop the header line

        self.schools = []
        for line in fh:
            if make_dict:
                self.schools.append(self.make_dict(self.parse_line(line)))
            else:
                self.schools.append(self.parse_line(line))

        if self.debug:
            print len(self.schools)
        return self.schools

    # --------------------------------------
    def make_dict(self, school):
        if self.debug:
            print school
        return dict(zip(self.headers, school))

# *****************************************************************************
# Unit Tests
# *****************************************************************************
class TestBasicNetwork(unittest.TestCase):

    def setUp(self):
        self.parse = NCESParser('data/school.test', 'data/school.format')

    def test_old_style(self):
        self.assertEqual(self.parse.something, ["Test"])

# *****************************************************************************
# Program Flow
# *****************************************************************************
# -------------------------------------
# Parse the command line options
# -------------------------------------
def main():
    parser = argparse.ArgumentParser(description='NCES Data File Parser')
    parser.add_argument('--year', action='store', dest='year', required=False, type=int,
            help='NCES Data Year - Standard filenames assumed')
    parser.add_argument('--formatfile', action='store', dest='formatfile', required=False,
            help='NCES Data File Record Layout')
    parser.add_argument('--datafile', action='store', dest='datafile', required=False,
            help='NCES Data File')
    parser.add_argument('-debug', action='store_true',
            help='Print Debug Messages')
    args = parser.parse_args()
    # print args

    # -------------------------------------
    # Actually do the work we intend to do here
    # -------------------------------------
    if args.year:
        parse = NCESParser(year=args.year, debug=args.debug)
    elif args.formatfile:
        parse = NCESParser(formatfile=args.formatfile, debug=args.debug)
    else:
        raise Exception("Please select a year or an NCES Record Layout file name")

    print "=" * 80
    print parse
    print "=" * 80
    print parse.get_idx('GSHI')

    if args.year:
        schools = parse.parse()
    elif args.formatfile:
        schools = parse.parse(args.datafile)
    else:
        raise Exception("Please define a year when constructing or specify an NCES Datefile Layout file name")

    print "=" * 80
    print len(schools)

    print "=" * 80
    print schools[0]
    print schools[22000]
    import pprint
    pprint.pprint(parse.make_dict(schools[22000]))

    print "=" * 80
    print "Headers"
    print "=" * 80
    print parse.headers

    print "=" * 80
    print "Descriptions"
    print "=" * 80
    pprint.pprint(parse.descriptions)

# -------------------------------------
# Drop the script name from the args
# and call our command line parser
# -------------------------------------
if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by NotSqrt
# Copyright (c) 2013 NotSqrt
#
# License: MIT
#

"""This module exports the Pyyaml plugin class."""

from SublimeLinter.lint import PythonLinter, persist

class Pyyaml(PythonLinter):

    """Provides an interface to pyyaml."""

    syntax = 'yaml'
    cmd = None
    regex = r'^:(?P<line>\d+):(?P<col>\d+): (?P<message>.+)'
    line_col_base = (0, 0)  # the lines and columns are 0-based
    module = 'yaml'

    def check(self, code, filename):
        """
        Call directly the yaml module, and handles the exception. Return str.

        Very similar to the SublimeLinter-json linter, except yaml is not in the python core library.

        """
        yaml = self.module

        try:
            for x in yaml.safe_load_all(code):
                # exhausting generator so all documents are checked
                pass
        except yaml.error.YAMLError as exc:
            if persist.settings.get('debug'):
                persist.printf('{} - {} : {}'.format(self.name, type(exc), exc))
            message = '{} : {} {}'.format(type(exc).__name__, exc.problem, exc.context)
            return ':{}:{}: {}\n'.format(exc.problem_mark.line, exc.problem_mark.column, message)
        except Exception as exc:
            persist.printf('{} - uncaught exception - {} : {}'.format(self.name, type(exc), exc))
        return ''
<commit_msg>Use yaml.parse to allow special YAML tags<commit_after>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by NotSqrt
# Copyright (c) 2013 NotSqrt
#
# License: MIT
#

"""This module exports the Pyyaml plugin class."""

from SublimeLinter.lint import PythonLinter, persist

class Pyyaml(PythonLinter):

    """Provides an interface to pyyaml."""

    syntax = 'yaml'
    cmd = None
    regex = r'^:(?P<line>\d+):(?P<col>\d+): (?P<message>.+)'
    line_col_base = (0, 0)  # the lines and columns are 0-based
    module = 'yaml'

    def check(self, code, filename):
        """
        Call directly the yaml module, and handles the exception. Return str.

        Very similar to the SublimeLinter-json linter, except yaml is not in the python core library.

        """
        yaml = self.module

        try:
            list(yaml.parse(code, Loader=yaml.BaseLoader))
        except yaml.error.YAMLError as exc:
            if persist.settings.get('debug'):
                persist.printf('{} - {} : {}'.format(self.name, type(exc), exc))
            message = '{} : {} {}'.format(type(exc).__name__, exc.problem, exc.context)
            return ':{}:{}: {}\n'.format(exc.problem_mark.line, exc.problem_mark.column, message)
        except Exception as exc:
            persist.printf('{} - uncaught exception - {} : {}'.format(self.name, type(exc), exc))
        return ''
<|endoftext|>


<commit_before>#!/usr/bin/python
from LaneCorrectorPipeline import Config
import os
import subprocess
import sys
import shlex
import shutil
import os.path
import numpy as np
import json
import glob
from zipfile import ZipFile, ZIP_DEFLATED
from RadarTransforms import loadRDR

def get_transforms(day, run, mark='mark1', absolute=False):
    """ Gets the IMU transforms for a run """
    gps_reader = GPSReader(args['gps_' + mark])
    gps_data = gps_reader.getNumericData()
    imu_transforms = IMUTransforms(gps_data)
    return imu_transforms

def unsafe_mkdir(name):
    try:
        os.mkdir(name)
    except OSError:
        pass

if __name__ == '__main__'
    if len(sys.argv) < 2:
        print """Usage:
        download.py -d -f folder/
        -d = dry-run
        -f = force"""
        sys.exit(-1)

    for day in sys.argv[1:]:
        driverseat_folder = '/deep/group/driving/driverseat_data/' + day + '/'
        remote_folder = '/deep/group/driving_data/jkiske/data/' + day + '/'
        q50_data_folder = '/deep/group/driving_data/q50_data/' + day + '/'

        unsafe_mkdir(driverseat_folder)

        config_file_name = driverseat_folder + '/progress.ini'
        configurator = Config(config_file_name)

        configurator.get('maps_json')
        configurator.get('radar')

        print configurator.config

        if configurator.config['maps_json'] == False:
            for remote_run in sorted(glob.glob(remote_folder + '*/')):
                run = remote_run.split('/')[-2]
                driverseat_run = driverseat_folder + run + '/'
                unsafe_mkdir(driverseat_run)
                print driverseat_run

                npz_map_name = remote_run + run + '_bg.npz'
                json_map_name = driverseat_run + run + '_map.json'
                zip_map_name = json_map_name + '.zip'

                if not os.path.isfile(json_map_name):
                    print '\tExporting map JSON...'
                    data = np.load(npz_map_name)['data']
                    # Only includes x,y,z data for the maps
                    json.dump(data[:, :3].tolist(), open(json_map_name, 'w'))

                if not os.path.isfile(zip_map_name):
                    print '\tZipping...'
                    json_zip = ZipFile(zip_map_name, 'w', ZIP_DEFLATED)
                    json_zip.write(json_map_name)
                    json_zip.close()

            configurator.set('maps_json', True)

        if configurator.config['radar'] == False:
            for radar_folder in sorted(glob.glob(q50_data_folder + '*_radar/')):
                run = radar_folder.split('/')[-2][:-6] # Remove '_radar'
                radar_data = []
                driverseat_run = driverseat_folder + run + '/'
                json_radar_name = driverseat_run + run + '_radar.json'
                zip_radar_name = json_radar_name + '.zip'
                print driverseat_run
                if not os.path.isfile(json_radar_name):
                    print '\tExporting radar JSON...'
                    for radar_file_name in sorted(glob.glob(radar_folder + '*')):
                        rdr = loadRDR(radar_file_name)
                        time = radar_file_name.split('/')[-1][:-4] # Remove '.rdr'
                        data = {time:{'O':rdr[0].tolist(), 'T':rdr[1].tolist()}}
                        radar_data.append(data)
                    json.dump(radar_data, open(json_radar_name, 'w'))

                if not os.path.isfile(zip_radar_name):
                    print '\tZipping...'
                    json_zip = ZipFile(zip_radar_name, 'w', ZIP_DEFLATED)
                    json_zip.write(json_radar_name)
                    json_zip.close()

            configurator.set('radar', True)
<commit_msg>Updated usage instructions<commit_after>#!/usr/bin/python
from LaneCorrectorPipeline import Config
import os
import subprocess
import sys
import shlex
import shutil
import os.path
import numpy as np
import json
import glob
from zipfile import ZipFile, ZIP_DEFLATED
from RadarTransforms import loadRDR

def get_transforms(day, run, mark='mark1', absolute=False):
    """ Gets the IMU transforms for a run """
    gps_reader = GPSReader(args['gps_' + mark])
    gps_data = gps_reader.getNumericData()
    imu_transforms = IMUTransforms(gps_data)
    return imu_transforms

def unsafe_mkdir(name):
    try:
        os.mkdir(name)
    except OSError:
        pass

if __name__ == '__main__'
    if len(sys.argv) < 2:
        print """Usage:
        DriverseatPipeline.py run ...
        """
        sys.exit(-1)

    for day in sys.argv[1:]:
        driverseat_folder = '/deep/group/driving/driverseat_data/' + day + '/'
        remote_folder = '/deep/group/driving_data/jkiske/data/' + day + '/'
        q50_data_folder = '/deep/group/driving_data/q50_data/' + day + '/'

        unsafe_mkdir(driverseat_folder)

        config_file_name = driverseat_folder + '/progress.ini'
        configurator = Config(config_file_name)

        configurator.get('maps_json')
        configurator.get('radar')

        print configurator.config

        if configurator.config['maps_json'] == False:
            for remote_run in sorted(glob.glob(remote_folder + '*/')):
                run = remote_run.split('/')[-2]
                driverseat_run = driverseat_folder + run + '/'
                unsafe_mkdir(driverseat_run)
                print driverseat_run

                npz_map_name = remote_run + run + '_bg.npz'
                json_map_name = driverseat_run + run + '_map.json'
                zip_map_name = json_map_name + '.zip'

                if not os.path.isfile(json_map_name):
                    print '\tExporting map JSON...'
                    data = np.load(npz_map_name)['data']
                    # Only includes x,y,z data for the maps
                    json.dump(data[:, :3].tolist(), open(json_map_name, 'w'))

                if not os.path.isfile(zip_map_name):
                    print '\tZipping...'
                    json_zip = ZipFile(zip_map_name, 'w', ZIP_DEFLATED)
                    json_zip.write(json_map_name)
                    json_zip.close()

            configurator.set('maps_json', True)

        if configurator.config['radar'] == False:
            for radar_folder in sorted(glob.glob(q50_data_folder + '*_radar/')):
                run = radar_folder.split('/')[-2][:-6] # Remove '_radar'
                radar_data = []
                driverseat_run = driverseat_folder + run + '/'
                json_radar_name = driverseat_run + run + '_radar.json'
                zip_radar_name = json_radar_name + '.zip'
                print driverseat_run
                if not os.path.isfile(json_radar_name):
                    print '\tExporting radar JSON...'
                    for radar_file_name in sorted(glob.glob(radar_folder + '*')):
                        rdr = loadRDR(radar_file_name)
                        time = radar_file_name.split('/')[-1][:-4] # Remove '.rdr'
                        data = {time:{'O':rdr[0].tolist(), 'T':rdr[1].tolist()}}
                        radar_data.append(data)
                    json.dump(radar_data, open(json_radar_name, 'w'))

                if not os.path.isfile(zip_radar_name):
                    print '\tZipping...'
                    json_zip = ZipFile(zip_radar_name, 'w', ZIP_DEFLATED)
                    json_zip.write(json_radar_name)
                    json_zip.close()

            configurator.set('radar', True)
<|endoftext|>


<commit_before>#!/usr/bin/python -tt
# Copyright 2010 Google Inc.
# Licensed under the Apache License, Version 2.0
# http://www.apache.org/licenses/LICENSE-2.0

# Google's Python Class
# http://code.google.com/edu/languages/google-python-class/

# Additional basic list exercises

# D. Given a list of numbers, return a list where
# all adjacent == elements have been reduced to a single element,
# so [1, 2, 2, 3] returns [1, 2, 3]. You may create a new list or
# modify the passed in list.
def remove_adjacent(nums):
  sololist = []
  for i in range(len(nums)-1):
    if nums[i] != nums[i+1]:
      sololist.append(nums[i])
  if sololist[-1:] != nums[-1:]:
    sololist.append(nums[-1])
  return sololist

# E. Given two lists sorted in increasing order, create and return a merged
# list of all the elements in sorted order. You may modify the passed in lists.
# Ideally, the solution should work in "linear" time, making a single
# pass of both lists.
#(linear_merge(['aa', 'xx', 'zz'], ['bb', 'cc']),
       #['aa', 'bb', 'cc', 'xx', 'zz'])
def linear_merge(list1, list2):
  merged = []
  # len1 = len(list1)
  # len2 = len(list2)
  while len(list1) > 0 and len(list2) > 0:
    if(list1[0] < list2[0]):
      merged.append(list1.pop(0))
    else:
      merged.append(list2.pop(0))
  if len(list1) > 0:
    merged.extend(list1)
  else:
    merged.extend(list2)
  return merged
  #return sorted(list1 + list2) the easy way, but slower

# Note: the solution above is kind of cute, but unforunately list.pop(0)
# is not constant time with the standard python list implementation, so
# the above is not strictly linear time.
# An alternate approach uses pop(-1) to remove the endmost elements
# from each list, building a solution list which is backwards.
# Then use reversed() to put the result back in the correct order. That
# solution works in linear time, but is more ugly.

# Simple provided test() function used in main() to print
# what each function returns vs. what it's supposed to return.
def test(got, expected):
  if got == expected:
    prefix = ' OK '
  else:
    prefix = '  X '
  print ('%s got: %s expected: %s' % (prefix, repr(got), repr(expected)))

# Calls the above functions with interesting inputs.
def main():
  print ('remove_adjacent')
  test(remove_adjacent([1, 2, 2, 3]), [1, 2, 3])
  test(remove_adjacent([2, 2, 3, 3, 3]), [2, 3])
  test(remove_adjacent([]), [])

  print
  print ('linear_merge')
  test(linear_merge(['aa', 'xx', 'zz'], ['bb', 'cc']),
       ['aa', 'bb', 'cc', 'xx', 'zz'])
  test(linear_merge(['aa', 'xx'], ['bb', 'cc', 'zz']),
       ['aa', 'bb', 'cc', 'xx', 'zz'])
  test(linear_merge(['aa', 'aa'], ['aa', 'bb', 'bb']),
       ['aa', 'aa', 'aa', 'bb', 'bb'])

if __name__ == '__main__':
  main()
<commit_msg>Rewrote remove_adjacent to not be terrible, removed unneeded code from merge<commit_after>#!/usr/bin/python -tt
# Copyright 2010 Google Inc.
# Licensed under the Apache License, Version 2.0
# http://www.apache.org/licenses/LICENSE-2.0

# Google's Python Class
# http://code.google.com/edu/languages/google-python-class/

# Additional basic list exercises

# D. Given a list of numbers, return a list where
# all adjacent == elements have been reduced to a single element,
# so [1, 2, 2, 3] returns [1, 2, 3]. You may create a new list or
# modify the passed in list.
def remove_adjacent(nums):
  sololist = []
  for num in nums:
    if len(sololist) == 0 or num != sololist[-1]:
      sololist.append(num)
  return sololist

# E. Given two lists sorted in increasing order, create and return a merged
# list of all the elements in sorted order. You may modify the passed in lists.
# Ideally, the solution should work in "linear" time, making a single
# pass of both lists.
#(linear_merge(['aa', 'xx', 'zz'], ['bb', 'cc']),
       #['aa', 'bb', 'cc', 'xx', 'zz'])
def linear_merge(list1, list2):
  merged = []
  # len1 = len(list1)
  # len2 = len(list2)
  while len(list1) and len(list2):
    if(list1[0] < list2[0]):
      merged.append(list1.pop(0))
    else:
      merged.append(list2.pop(0))
  merged.extend(list1)
  merged.extend(list2)
  return merged
  #return sorted(list1 + list2) the easy way, but slower

# Note: the solution above is kind of cute, but unforunately list.pop(0)
# is not constant time with the standard python list implementation, so
# the above is not strictly linear time.
# An alternate approach uses pop(-1) to remove the endmost elements
# from each list, building a solution list which is backwards.
# Then use reversed() to put the result back in the correct order. That
# solution works in linear time, but is more ugly.

# Simple provided test() function used in main() to print
# what each function returns vs. what it's supposed to return.
def test(got, expected):
  if got == expected:
    prefix = ' OK '
  else:
    prefix = '  X '
  print ('%s got: %s expected: %s' % (prefix, repr(got), repr(expected)))

# Calls the above functions with interesting inputs.
def main():
  print ('remove_adjacent')
  test(remove_adjacent([1, 2, 2, 3]), [1, 2, 3])
  test(remove_adjacent([2, 2, 3, 3, 3]), [2, 3])
  test(remove_adjacent([]), [])

  print
  print ('linear_merge')
  test(linear_merge(['aa', 'xx', 'zz'], ['bb', 'cc']),
       ['aa', 'bb', 'cc', 'xx', 'zz'])
  test(linear_merge(['aa', 'xx'], ['bb', 'cc', 'zz']),
       ['aa', 'bb', 'cc', 'xx', 'zz'])
  test(linear_merge(['aa', 'aa'], ['aa', 'bb', 'bb']),
       ['aa', 'aa', 'aa', 'bb', 'bb'])

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before><commit_msg>remove requests package install in tests<commit_after><|endoftext|>


<commit_before>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Clifton Kaznocha
# Copyright (c) 2014 Clifton Kaznocha
#
# License: MIT
#

"""This module exports the Flow plugin class."""

import os
from SublimeLinter.lint import Linter

class Flow(Linter):

    """Provides an interface to flow."""

    syntax = ('javascript', 'html', 'javascriptnext', 'javascript (babel)', 'javascript (jsx)', 'jsx')
    executable = 'flow'
    version_args = '--version'
    version_re = r'(?P<version>\d+\.\d+\.\d+)'
    version_requirement = '>= 0.1.0'
    regex = r'''(?xi)
        # Warning location and optional title for the message
        ^.+/(?P<file_name>[^/]+\.(js|html)):(?P<line>\d+):(?P<col>\d+),\d+:\s*(?P<message_title>.+)$\r?\n

        # Main lint message
        ^(?P<message>.+)$

        # Optional message, only extract the text, leave the path
        (\r?\n\s\s/.+:\s(?P<message_footer>.+))?
    '''
    multiline = True
    defaults = {
        # Allows the user to lint *all* files, regardless of whether they have the `/* @flow */` declaration at the top.
        'all': False,

        # Allow to bypass the 50 errors cap
        'show-all-errors': True,

        # Allows flow to start server (makes things faster on larger projects)
        'use-server': True,

        # Options for flow
        '--lib:,': ''
    }
    word_re = r'^((\'|")?[^"\']+(\'|")?)(?=[\s\,\)\]])'
    selectors = {
        'html': 'source.js.embedded.html'
    }

    def cmd(self):
        """Return the command line to execute."""
        command = [self.executable_path]

        if self.get_merged_settings()['use-server']:
            command.append('status')
        else:
            command.append('check')

        if self.get_merged_settings()['show-all-errors']:
            command.append('--show-all-errors')

        if self.get_merged_settings()['all']:
            command.append('--all')

        return command

    def split_match(self, match):
        """
        Return the components of the match.

        We override this to catch linter error messages and return better
        error messages.
        """

        if match:
            open_file_name = os.path.basename(self.view.file_name())
            linted_file_name = match.group('file_name')

            if linted_file_name == open_file_name:
                message_title = match.group('message_title')
                message = match.group('message')
                message_footer = match.group('message_footer') or ""

                if message_title and message_title.strip():
                    message = '"{0}"" {1} {2}'.format(
                        message_title,
                        message,
                        message_footer
                    )

                line = max(int(match.group('line')) - 1, 0)
                col = int(match.group('col')) - 1

                # match, line, col, error, warning, message, near
                return match, line, col, True, False, message, None

        return match, None, None, None, None, '', None
<commit_msg>add jsx file extension<commit_after>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Clifton Kaznocha
# Copyright (c) 2014 Clifton Kaznocha
#
# License: MIT
#

"""This module exports the Flow plugin class."""

import os
from SublimeLinter.lint import Linter

class Flow(Linter):

    """Provides an interface to flow."""

    syntax = ('javascript', 'html', 'javascriptnext', 'javascript (babel)', 'javascript (jsx)', 'jsx')
    executable = 'flow'
    version_args = '--version'
    version_re = r'(?P<version>\d+\.\d+\.\d+)'
    version_requirement = '>= 0.1.0'
    regex = r'''(?xi)
        # Warning location and optional title for the message
        ^.+/(?P<file_name>[^/]+\.(js|html|jsx)):(?P<line>\d+):(?P<col>\d+),\d+:\s*(?P<message_title>.+)$\r?\n

        # Main lint message
        ^(?P<message>.+)$

        # Optional message, only extract the text, leave the path
        (\r?\n\s\s/.+:\s(?P<message_footer>.+))?
    '''
    multiline = True
    defaults = {
        # Allows the user to lint *all* files, regardless of whether they have the `/* @flow */` declaration at the top.
        'all': False,

        # Allow to bypass the 50 errors cap
        'show-all-errors': True,

        # Allows flow to start server (makes things faster on larger projects)
        'use-server': True,

        # Options for flow
        '--lib:,': ''
    }
    word_re = r'^((\'|")?[^"\']+(\'|")?)(?=[\s\,\)\]])'
    selectors = {
        'html': 'source.js.embedded.html'
    }

    def cmd(self):
        """Return the command line to execute."""
        command = [self.executable_path]

        if self.get_merged_settings()['use-server']:
            command.append('status')
        else:
            command.append('check')

        if self.get_merged_settings()['show-all-errors']:
            command.append('--show-all-errors')

        if self.get_merged_settings()['all']:
            command.append('--all')

        return command

    def split_match(self, match):
        """
        Return the components of the match.

        We override this to catch linter error messages and return better
        error messages.
        """

        if match:
            open_file_name = os.path.basename(self.view.file_name())
            linted_file_name = match.group('file_name')

            if linted_file_name == open_file_name:
                message_title = match.group('message_title')
                message = match.group('message')
                message_footer = match.group('message_footer') or ""

                if message_title and message_title.strip():
                    message = '"{0}"" {1} {2}'.format(
                        message_title,
                        message,
                        message_footer
                    )

                line = max(int(match.group('line')) - 1, 0)
                col = int(match.group('col')) - 1

                # match, line, col, error, warning, message, near
                return match, line, col, True, False, message, None

        return match, None, None, None, None, '', None
<|endoftext|>