<commit_before>#!/usr/bin/env python2.7
# TODO: ADD COPYRIGHT TAG
from __future__ import absolute_import, division, print_function
import utool
from vtool import geometry
from itertools import izip
print, print_, printDBG, rrr, profile = utool.inject(__name__, '[TEST_IBS_CONTROL]')

def TEST_IBS_CONTROL(ibs):
    ibs.compute_encounters()

    """ get_image_eids / get_encounter_gids """
    eid_list = ibs.get_valid_eids()
    assert eid_list, 'eid_list is empty'
    gids_list = ibs.get_encounter_gids(eid_list)
    #print('[TEST] gids_list = %r' % gids_list)
    assert gids_list, 'gids_list is empty'
    for gids, eid in izip(gids_list, eid_list):
        eid_list2 = ibs.get_image_eids(gids)
        assert ([[eid]] * len(eid_list2)) == eid_list2
        #print('[TEST] eid_list2 = %r' % eid_list2)
        #print('[TEST] eid = %r' % eid)

    """ set_annotation_notes / get_annotation_notes """
    aid_list = ibs.get_valid_aids()
    annotation_notes_list = len(aid_list) * ["test text"]
    assert aid_list, 'aid_list is empty'
    ibs.set_annotation_notes(aid_list, annotation_notes_list)
    annotation_notes_list2 = ibs.get_annotation_notes(aid_list)
    assert annotation_notes_list2, 'get_annotation_notes returned an empty list'
    #print('[TEST] annotation_notes_list = %r' % annotation_notes_list)
    #print('[TEST] annotation_notes_list2 = %r' % annotation_notes_list2)
    assert annotation_notes_list == annotation_notes_list2, 'annotation notes lists do not match'

    """ set_name_notes / get_name_notes """
    nid_list = ibs.get_valid_nids()
    assert nid_list, 'nid_list is empty'
    nid_notes_list = len(nid_list) * ['nid notes test']
    ibs.set_name_notes(nid_list, nid_notes_list)
    nid_notes_list2 = ibs.get_name_notes(nid_list)
    print('[TEST] nid_notes_list = %r' % nid_notes_list)
    print('[TEST] nid_notes_list2 = %r' % nid_notes_list2)
    assert nid_notes_list == nid_notes_list2, 'nid notes lists do not match'

    """ set_image_notes / get_image_notes """
    gid_list = ibs.get_valid_gids()
    assert gid_list, 'gid_list is empty'
    gid_notes_list = len(gid_list) * ['image note test']
    ibs.set_image_notes(gid_list, gid_notes_list)
    gid_notes_list2 = ibs.get_image_notes(gid_list)
    print('[TEST] gid_notes_list = %r' % gid_notes_list)
    print('[TEST] gid_notes_list2 = %r' % gid_notes_list2)
    assert gid_notes_list == gid_notes_list2, 'images notes lists do not match'

    """ set_annotation_bboxes / get_annotation_bboxes """
    aid_list = ibs.get_valid_aids()
    assert aid_list, 'aid_list is empty'
    bbox_list_orig = ibs.get_annotation_bboxes(aid_list)
    bbox_list = [(1, 2, 3, 4)] * len(aid_list)
    ibs.set_annotation_bboxes(aid_list, bbox_list)
    bbox_list2 = ibs.get_annotation_bboxes(aid_list)
    print('[TEST] aid_list = %r' % (aid_list,))
    print('[TEST] bbox_list = %r' % (bbox_list,))
    print('[TEST] bbox_list2 = %r' % (bbox_list2,))
    assert bbox_list == bbox_list2, 'bbox lists do not match'
    # put bboxes back to original state
    # (otherwise other tests will fail on the second run of run_tests.sh)
    ibs.set_annotation_bboxes(aid_list, bbox_list_orig)

    """ set_annotation_verts / get_annotation_verts """
    aid_list = ibs.get_valid_aids()
    assert aid_list, 'aid_list is empty'
    bbox_list_orig = ibs.get_annotation_bboxes(aid_list)
    vert_list_orig = ibs.get_annotation_verts(aid_list)
    vert_list = [((1, 2), (3, 4), (5, 6), (7, 8))] * len(aid_list)
    print('[TEST] vert_list = %r' % vert_list)
    assert len(aid_list) == len(vert_list), 'lengths do not match, malformed input'
    ibs.set_annotation_verts(aid_list, vert_list)
    vert_list2 = ibs.get_annotation_verts(aid_list)
    assert vert_list == vert_list2, 'vert lists do not match'

    """ set_annotation_verts / get_annotation_bboxes """
    bbox_list = ibs.get_annotation_bboxes(aid_list)
    bbox_list2 = geometry.bboxes_from_vert_list(vert_list2)
    assert bbox_list == bbox_list2, 'bbox lists do not match'
    vert_list = [((10, 10), (120, 10), (120, 120), (10, 120))] * len(aid_list)
    ibs.set_annotation_verts(aid_list, vert_list)
    bbox_list3 = [(10, 10, 110, 110)] * len(aid_list)
    bbox_list4 = ibs.get_annotation_bboxes(aid_list)
    assert bbox_list3 == bbox_list4, 'bbox lists do not match'
    # finish this test here

    """ set_annotation_bboxes / get_annotation_verts  """
    bbox_list = [(10, 10, 110, 110)] * len(aid_list)
    ibs.set_annotation_bboxes(aid_list, bbox_list)
    # test that setting the bounding boxes overrides the vertices
    vert_list = [((10, 10), (120, 10), (120, 120), (10, 120))] * len(aid_list)
    vert_list2 = ibs.get_annotation_verts(aid_list)
    assert vert_list == vert_list2, 'vert lists do not match'

    # put verts back to original state
    # (otherwise other tests will fail on the second run of run_tests.sh)
    ibs.set_annotation_verts(aid_list, vert_list_orig)
    assert vert_list_orig == ibs.get_annotation_verts(aid_list), 'Verts were not reset to original state'
    assert bbox_list_orig == ibs.get_annotation_bboxes(aid_list), 'Bboxes were not reset to original state'

    """ set_image_gps / get_image_gps """
    gid_list = ibs.get_valid_gids()
    assert gids, 'gid_list is empty'
    gps_list_orig = ibs.get_image_gps(gid_list)
    gps_list = [(x, y) for (x, y) in zip(range(len(gid_list)), range(len(gid_list)))]
    ibs.set_image_gps(gid_list, gps_list)
    gps_list2 = ibs.get_image_gps(gid_list)
    assert gps_list == gps_list2, 'gps lists do not match'
    ibs.set_image_gps(gid_list, gps_list_orig)
    assert gps_list_orig == ibs.get_image_gps(gid_list), 'gps was not reset to original state'

    eid_list = ibs.get_valid_eids()
    enc_text_list_orig = ibs.get_encounter_enctext(eid_list)
    enc_text_list = [str(x) for x in range(len(eid_list))]
    assert eid_list, 'eid_list is empty'
    print('len eid_list: %d' % len(eid_list))
    ibs.set_encounter_enctext(eid_list, enc_text_list)
    enc_text_list2 = ibs.get_encounter_enctext(eid_list)
    print('enc_text_list = %r' % enc_text_list)
    print('enc_text_list2 = %r' % enc_text_list2)
    assert enc_text_list == enc_text_list2, 'encounter text lists do not match'
    ibs.set_encounter_enctext(eid_list, enc_text_list_orig)
    assert enc_text_list_orig == ibs.get_encounter_enctext(eid_list), 'enc text was not reset'

    return locals()

if __name__ == '__main__':
    import multiprocessing
    multiprocessing.freeze_support()  # For win32
    import ibeis
    # Initialize database
    main_locals = ibeis.main(defaultdb='testdb1', gui=False)
    ibs = main_locals['ibs']
    test_locals = utool.run_test(TEST_IBS_CONTROL, ibs)
    execstr = utool.execstr_dict(test_locals, 'test_locals')
    exec(execstr)
<commit_msg>Tests for get and set annotation names and species<commit_after>#!/usr/bin/env python2.7
# TODO: ADD COPYRIGHT TAG
from __future__ import absolute_import, division, print_function
import utool
from vtool import geometry
from itertools import izip
print, print_, printDBG, rrr, profile = utool.inject(__name__, '[TEST_IBS_CONTROL]')

def TEST_IBS_CONTROL(ibs):
    ibs.compute_encounters()

    """ get_image_eids / get_encounter_gids """
    eid_list = ibs.get_valid_eids()
    assert eid_list, 'eid_list is empty'
    gids_list = ibs.get_encounter_gids(eid_list)
    #print('[TEST] gids_list = %r' % gids_list)
    assert gids_list, 'gids_list is empty'
    for gids, eid in izip(gids_list, eid_list):
        eid_list2 = ibs.get_image_eids(gids)
        assert ([[eid]] * len(eid_list2)) == eid_list2
        #print('[TEST] eid_list2 = %r' % eid_list2)
        #print('[TEST] eid = %r' % eid)

    """ set_annotation_notes / get_annotation_notes """
    aid_list = ibs.get_valid_aids()
    annotation_notes_list = len(aid_list) * ["test text"]
    assert aid_list, 'aid_list is empty'
    ibs.set_annotation_notes(aid_list, annotation_notes_list)
    annotation_notes_list2 = ibs.get_annotation_notes(aid_list)
    assert annotation_notes_list2, 'get_annotation_notes returned an empty list'
    #print('[TEST] annotation_notes_list = %r' % annotation_notes_list)
    #print('[TEST] annotation_notes_list2 = %r' % annotation_notes_list2)
    assert annotation_notes_list == annotation_notes_list2, 'annotation notes lists do not match'

    """ set_name_notes / get_name_notes """
    nid_list = ibs.get_valid_nids()
    assert nid_list, 'nid_list is empty'
    nid_notes_list = len(nid_list) * ['nid notes test']
    ibs.set_name_notes(nid_list, nid_notes_list)
    nid_notes_list2 = ibs.get_name_notes(nid_list)
    print('[TEST] nid_notes_list = %r' % nid_notes_list)
    print('[TEST] nid_notes_list2 = %r' % nid_notes_list2)
    assert nid_notes_list == nid_notes_list2, 'nid notes lists do not match'

    """ set_image_notes / get_image_notes """
    gid_list = ibs.get_valid_gids()
    assert gid_list, 'gid_list is empty'
    gid_notes_list = len(gid_list) * ['image note test']
    ibs.set_image_notes(gid_list, gid_notes_list)
    gid_notes_list2 = ibs.get_image_notes(gid_list)
    print('[TEST] gid_notes_list = %r' % gid_notes_list)
    print('[TEST] gid_notes_list2 = %r' % gid_notes_list2)
    assert gid_notes_list == gid_notes_list2, 'images notes lists do not match'

    """ set_annotation_bboxes / get_annotation_bboxes """
    aid_list = ibs.get_valid_aids()
    assert aid_list, 'aid_list is empty'
    bbox_list_orig = ibs.get_annotation_bboxes(aid_list)
    bbox_list = [(1, 2, 3, 4)] * len(aid_list)
    ibs.set_annotation_bboxes(aid_list, bbox_list)
    bbox_list2 = ibs.get_annotation_bboxes(aid_list)
    print('[TEST] aid_list = %r' % (aid_list,))
    print('[TEST] bbox_list = %r' % (bbox_list,))
    print('[TEST] bbox_list2 = %r' % (bbox_list2,))
    assert bbox_list == bbox_list2, 'bbox lists do not match'
    # put bboxes back to original state
    # (otherwise other tests will fail on the second run of run_tests.sh)
    ibs.set_annotation_bboxes(aid_list, bbox_list_orig)

    """ set_annotation_verts / get_annotation_verts """
    aid_list = ibs.get_valid_aids()
    assert aid_list, 'aid_list is empty'
    bbox_list_orig = ibs.get_annotation_bboxes(aid_list)
    vert_list_orig = ibs.get_annotation_verts(aid_list)
    vert_list = [((1, 2), (3, 4), (5, 6), (7, 8))] * len(aid_list)
    print('[TEST] vert_list = %r' % vert_list)
    assert len(aid_list) == len(vert_list), 'lengths do not match, malformed input'
    ibs.set_annotation_verts(aid_list, vert_list)
    vert_list2 = ibs.get_annotation_verts(aid_list)
    assert vert_list == vert_list2, 'vert lists do not match'

    """ set_annotation_verts / get_annotation_bboxes """
    bbox_list = ibs.get_annotation_bboxes(aid_list)
    bbox_list2 = geometry.bboxes_from_vert_list(vert_list2)
    assert bbox_list == bbox_list2, 'bbox lists do not match'
    vert_list = [((10, 10), (120, 10), (120, 120), (10, 120))] * len(aid_list)
    ibs.set_annotation_verts(aid_list, vert_list)
    bbox_list3 = [(10, 10, 110, 110)] * len(aid_list)
    bbox_list4 = ibs.get_annotation_bboxes(aid_list)
    assert bbox_list3 == bbox_list4, 'bbox lists do not match'
    # finish this test here

    """ set_annotation_bboxes / get_annotation_verts  """
    bbox_list = [(10, 10, 110, 110)] * len(aid_list)
    ibs.set_annotation_bboxes(aid_list, bbox_list)
    # test that setting the bounding boxes overrides the vertices
    vert_list = [((10, 10), (120, 10), (120, 120), (10, 120))] * len(aid_list)
    vert_list2 = ibs.get_annotation_verts(aid_list)
    assert vert_list == vert_list2, 'vert lists do not match'

    # put verts back to original state
    # (otherwise other tests will fail on the second run of run_tests.sh)
    ibs.set_annotation_verts(aid_list, vert_list_orig)
    assert vert_list_orig == ibs.get_annotation_verts(aid_list), 'Verts were not reset to original state'
    assert bbox_list_orig == ibs.get_annotation_bboxes(aid_list), 'Bboxes were not reset to original state'

    """ set_image_gps / get_image_gps """
    gid_list = ibs.get_valid_gids()
    assert gids, 'gid_list is empty'
    gps_list_orig = ibs.get_image_gps(gid_list)
    gps_list = [(x, y) for (x, y) in zip(range(len(gid_list)), range(len(gid_list)))]
    ibs.set_image_gps(gid_list, gps_list)
    gps_list2 = ibs.get_image_gps(gid_list)
    assert gps_list == gps_list2, 'gps lists do not match'
    ibs.set_image_gps(gid_list, gps_list_orig)
    assert gps_list_orig == ibs.get_image_gps(gid_list), 'gps was not reset to original state'

    eid_list = ibs.get_valid_eids()
    enc_text_list_orig = ibs.get_encounter_enctext(eid_list)
    enc_text_list = [str(x) for x in range(len(eid_list))]
    assert eid_list, 'eid_list is empty'
    print('len eid_list: %d' % len(eid_list))
    ibs.set_encounter_enctext(eid_list, enc_text_list)
    enc_text_list2 = ibs.get_encounter_enctext(eid_list)
    print('enc_text_list = %r' % enc_text_list)
    print('enc_text_list2 = %r' % enc_text_list2)
    assert enc_text_list == enc_text_list2, 'encounter text lists do not match'
    ibs.set_encounter_enctext(eid_list, enc_text_list_orig)
    assert enc_text_list_orig == ibs.get_encounter_enctext(eid_list), 'enc text was not reset'

    """ set annotation names / get_annotation_names """
    aid_list = ibs.get_valid_aids()
    assert aid_list, 'aid_list is empty'
    orig_names = ibs.get_annotation_names(aid_list)
    new_names = ['TESTNAME_' + str(x) for x in range(len(aid_list))]
    ibs.set_annotation_names(aid_list, new_names)
    new_names2 = ibs.get_annotation_names(aid_list)
    assert new_names == new_names2, 'new_names == new_names2 failed!'
    ibs.set_annotation_names(aid_list, orig_names)
    assert orig_names == ibs.get_annotation_names(aid_list)

    """ set annotation species / get annotation species """
    aid_list = ibs.get_valid_aids()
    assert aid_list, 'aid_list is empty'
    orig_species = ibs.get_annotation_species(aid_list)
    new_species = ['TESTSPECIES_' + str(x) for x in range(len(aid_list))]
    ibs.set_annotation_species(aid_list, new_species)
    new_species2 = ibs.get_annotation_species(aid_list)
    assert new_species == new_species2, 'new_species == new_species2 failed!'
    ibs.set_annotation_species(aid_list, orig_species)
    assert orig_species == ibs.get_annotation_species(aid_list)

    return locals()

if __name__ == '__main__':
    import multiprocessing
    multiprocessing.freeze_support()  # For win32
    import ibeis
    # Initialize database
    main_locals = ibeis.main(defaultdb='testdb1', gui=False)
    ibs = main_locals['ibs']
    test_locals = utool.run_test(TEST_IBS_CONTROL, ibs)
    execstr = utool.execstr_dict(test_locals, 'test_locals')
    exec(execstr)
<|endoftext|>


<commit_before>
# Note: Display the list of unsuported semantics

from operator   import itemgetter
from triton     import *

unsuportedSemantics = dict()

def cbefore(instruction):
    if len(instruction.symbolicExpressions) == 0:
        mnemonic = opcodeToString(instruction.opcode)
        if mnemonic in unsuportedSemantics:
            unsuportedSemantics[mnemonic] += 1
        else:
            unsuportedSemantics.update({mnemonic: 1})
    return

def cfini():
    l = unsuportedSemantics.items()
    l.sort(key=itemgetter(1), reverse=True)
    for i in l:
        print '%s: %d' %(i[0].lower(), i[1])
    return

if __name__ == '__main__':
    startAnalysisFromSymbol('main')
    addCallback(cbefore, IDREF.CALLBACK.BEFORE)
    addCallback(cfini, IDREF.CALLBACK.FINI)
    runProgram()

<commit_msg>update unsuported_semantics.py<commit_after>
# Note: Display the list of unsuported semantics

from operator   import itemgetter
from triton     import *

unsuportedSemantics = dict()

def cbefore(instruction):
    if len(instruction.symbolicExpressions) == 0:
        mnemonic = opcodeToString(instruction.opcode)
        if mnemonic in unsuportedSemantics:
            unsuportedSemantics[mnemonic] += 1
        else:
            unsuportedSemantics.update({mnemonic: 1})
    return

def cfini():
    stats = getStats()
    print '============================================================='
    print 'Stats'
    print '============================================================='
    print 'Number of branches:              %d' %(stats['branches'])
    print 'Number of expressions:           %d' %(stats['expressions'])
    print 'Number of unknown expression:    %d' %(stats['unknownExpr'])
    print 'Time of the execution:           %d seconds' %(stats['time'])
    l = unsuportedSemantics.items()
    l.sort(key=itemgetter(1), reverse=True)
    print '============================================================='
    print 'Unsuported Semantics'
    print '============================================================='
    for i in l:
        print '%s: %d' %(i[0].lower(), i[1])
    print '============================================================='
    return

if __name__ == '__main__':
    startAnalysisFromSymbol('main')
    addCallback(cbefore, IDREF.CALLBACK.BEFORE)
    addCallback(cfini, IDREF.CALLBACK.FINI)
    runProgram()

<|endoftext|>


<commit_before>######
#  Create a backup of J2EE Security Roles
#
#  Author:        Christoph Stoettner
#  Mail:          christoph.stoettner@stoeps.de
#  Documentation: http://scripting101.stoeps.de
#
#  Version:       2.0
#  Date:          2014-06-04
#
#  License:       Apache 2.0
#
#

import sys

# Only load commands if not initialized directly (call from menu)
if __name__ == "__main__":
    execfile("ibmcnx/loadCnxApps.py")

path = raw_input( "Please provide a path for your backup files: " )
ibmcnx.loadCnxApps.checkBackupPath( path )

apps = AdminApp.list()
appsList = apps.split( lineSeparator )
for app in appsList:
    filename = path + "/" + app + ".txt"
    print "Backup of %s security roles saved to %s." % app.upper(), filename
    my_file = open( filename, 'w' )
    my_file.write ( AdminApp.view( app, "-MapRolesToUsers" ) )
    my_file.flush
    my_file.close()
<commit_msg>New tests<commit_after>######
#  Create a backup of J2EE Security Roles
#
#  Author:        Christoph Stoettner
#  Mail:          christoph.stoettner@stoeps.de
#  Documentation: http://scripting101.stoeps.de
#
#  Version:       2.0
#  Date:          2014-06-04
#
#  License:       Apache 2.0
#
#

import sys
import ibmcnx.functions

# Only load commands if not initialized directly (call from menu)
if __name__ == "__main__":
    execfile("ibmcnx/loadCnxApps.py")

path = raw_input( "Please provide a path for your backup files: " )
ibmcnx.functions.checkBackupPath( path )

apps = AdminApp.list()
appsList = apps.split( lineSeparator )
for app in appsList:
    filename = path + "/" + app + ".txt"
    print "Backup of %s security roles saved to %s." % app.upper(), filename
    my_file = open( filename, 'w' )
    my_file.write ( AdminApp.view( app, "-MapRolesToUsers" ) )
    my_file.flush
    my_file.close()
<|endoftext|>


<commit_before>import logging
import json
import uuid
from collections import defaultdict

import tornado.web
import tornado.httpclient
from tornado.platform.asyncio import to_asyncio_future
import pymongo
import motor

from iceprod.server.rest import RESTHandler, RESTHandlerSetup, authorization
from iceprod.server.util import nowstr

logger = logging.getLogger('rest.datasets')

def setup(config):
    """
    Setup method for Dataset REST API.

    Sets up any database connections or other prerequisites.

    Args:
        config (dict): an instance of :py:class:`iceprod.server.config`.

    Returns:
        list: Routes for dataset, which can be passed to :py:class:`tornado.web.Application`.
    """
    cfg_rest = config.get('rest',{}).get('datasets',{})
    db_cfg = cfg_rest.get('database',{})

    # add indexes
    db = pymongo.MongoClient(**db_cfg).datasets
    if 'dataset_id_index' not in db.datasets.index_information():
        db.dataset.create_index('dataset_id', name='dataset_id_index', unique=True)

    handler_cfg = RESTHandlerSetup(config)
    handler_cfg.update({
        'database': motor.motor_tornado.MotorClient(**db_cfg).datasets,
        'system_token': config.get('rest',{}).get('system_token',None)
    })

    return [
        (r'/datasets', MultiDatasetHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)', DatasetHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)/description', DatasetDescriptionHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)/status', DatasetStatusHandler, handler_cfg),
        (r'/dataset_summaries/status', DatasetSummariesStatusHandler, handler_cfg),
    ]

class BaseHandler(RESTHandler):
    """
    Base handler for Dataset REST API.
    """
    def initialize(self, database=None, system_token=None, **kwargs):
        super(BaseHandler, self).initialize(**kwargs)
        self.db = database
        self.system_token = system_token

class MultiDatasetHandler(BaseHandler):
    """
    Handle multi-group requests.
    """
    @authorization(roles=['admin']) #TODO: figure out how to do auth for each dataset in the list
    async def get(self):
        """
        Get a dict of datasets.

        Returns:
            dict: {<dataset_id>: metadata}
        """
        ret = {}
        async for row in self.db.datasets.find(projection={'_id':False}):
            k = row['dataset_id']
            ret[k] = row
        self.write(ret)
        self.finish()

    @authorization(roles=['admin','user']) # anyone should be able to create a dataset
    async def post(self):
        """
        Add a dataset.

        Body should contain all necessary fields for a dataset.
        """
        data = json.loads(self.request.body)

        # validate first
        req_fields = {
            'description': str,
            'jobs_submitted': int,
            'tasks_submitted': int,
            'group_id': str
        }
        for k in req_fields:
            if k not in data:
                raise tornado.web.HTTPError(400, reason='missing key: '+k)
            if not isinstance(data[k], req_fields[k]):
                r = 'key {} should be of type {}'.format(k, req_fields[k])
                raise tornado.web.HTTPError(400, reason=r)

        # generate dataset number
        ret = await self.db.settings.find_one_and_update(
            {'name': 'dataset_num'},
            {'$inc': {'num': 1}},
            projection={'num': True, '_id': False},
            upsert=True,
            return_document=pymongo.ReturnDocument.AFTER)
        dataset_num = ret['num']

        # set some fields
        data['dataset_id'] = uuid.uuid1().hex
        data['dataset'] = dataset_num
        data['status'] = 'processing'
        data['start_date'] = nowstr()
        data['username'] = self.auth_data['username']
        if 'priority' not in data:
            data['priority'] = 0
        if 'debug' not in data:
            data['debug'] = False

        # insert
        ret = await self.db.datasets.insert_one(data)

        # set auth rules
        url = self.auth_url+'/auths/'+data['dataset_id']
        http_client = tornado.httpclient.AsyncHTTPClient()
        auth_data = {
            'read_groups':[data['group_id']],
            'write_groups':[data['group_id']],
        }
        await to_asyncio_future(http_client.fetch(url,
                method='PUT', body=json.dumps(auth_data),
                headers={'Authorization': 'bearer '+self.auth_key}))

        # return success
        self.set_status(201)
        self.set_header('Location', '/datasets/'+data['dataset_id'])
        self.write({'result': '/datasets/'+data['dataset_id']})
        self.finish()

class DatasetHandler(BaseHandler):
    """
    Handle dataset requests.
    """
    @authorization(roles=['admin'], attrs=['dataset_id:read'])
    async def get(self, dataset_id):
        """
        Get a dataset.

        Args:
            dataset_id (str): the dataset

        Returns:
            dict: dataset metadata
        """
        ret = await self.db.datasets.find_one({'dataset_id':dataset_id},
                projection={'_id':False})
        if not ret:
            self.send_error(404, reason="Dataset not found")
        else:
            self.write(ret)
            self.finish()

class DatasetDescriptionHandler(BaseHandler):
    """
    Handle dataset description updates.
    """
    @authorization(roles=['admin'], attrs=['dataset_id:write'])
    async def put(self, dataset_id):
        """
        Set a dataset description.

        Args:
            dataset_id (str): the dataset

        Returns:
            dict: empty dict
        """
        data = json.loads(self.request.body)
        if 'description' not in data:
            raise tornado.web.HTTPError(400, reason='missing description')
        elif not isinstance(data['description'],str):
            raise tornado.web.HTTPError(400, reason='bad description')

        ret = await self.db.datasets.find_one_and_update({'dataset_id':dataset_id},
                {'$set':{'description': data['description']}},
                projection=['_id'])
        if not ret:
            self.send_error(404, reason="Dataset not found")
        else:
            self.write({})
            self.finish()

class DatasetStatusHandler(BaseHandler):
    """
    Handle dataset status updates.
    """
    @authorization(roles=['admin'], attrs=['dataset_id:write'])
    async def put(self, dataset_id):
        """
        Set a dataset status.

        Args:
            dataset_id (str): the dataset

        Returns:
            dict: empty dict
        """
        data = json.loads(self.request.body)
        if 'status' not in data:
            raise tornado.web.HTTPError(400, reason='missing status')
        elif data['status'] not in ('processing','suspended','errors','complete'):
            raise tornado.web.HTTPError(400, reason='bad status')

        ret = await self.db.datasets.find_one_and_update({'dataset_id':dataset_id},
                {'$set':{'status': data['status']}},
                projection=['_id'])
        if not ret:
            self.send_error(404, reason="Dataset not found")
        else:
            self.write({})
            self.finish()

class DatasetSummariesStatusHandler(BaseHandler):
    """
    Handle dataset summary grouping by status.
    """
    @authorization(roles=['admin','system']) #TODO: figure out how to do auth for each dataset in the list
    async def get(self):
        """
        Get the dataset summary for all datasets, group by status.

        Returns:
            dict: {<status>: [<dataset_id>,]}
        """
        cursor = self.db.datasets.find(
                projection={'_id':False,'status':True,'dataset_id':True})
        ret = defaultdict(list)
        async for row in cursor:
            ret[row['status']].append(row['dataset_id'])
        self.write(ret)
        self.finish()
<commit_msg>allow any logged in user to view dataset_summaries<commit_after>import logging
import json
import uuid
from collections import defaultdict

import tornado.web
import tornado.httpclient
from tornado.platform.asyncio import to_asyncio_future
import pymongo
import motor

from iceprod.server.rest import RESTHandler, RESTHandlerSetup, authorization
from iceprod.server.util import nowstr

logger = logging.getLogger('rest.datasets')

def setup(config):
    """
    Setup method for Dataset REST API.

    Sets up any database connections or other prerequisites.

    Args:
        config (dict): an instance of :py:class:`iceprod.server.config`.

    Returns:
        list: Routes for dataset, which can be passed to :py:class:`tornado.web.Application`.
    """
    cfg_rest = config.get('rest',{}).get('datasets',{})
    db_cfg = cfg_rest.get('database',{})

    # add indexes
    db = pymongo.MongoClient(**db_cfg).datasets
    if 'dataset_id_index' not in db.datasets.index_information():
        db.dataset.create_index('dataset_id', name='dataset_id_index', unique=True)

    handler_cfg = RESTHandlerSetup(config)
    handler_cfg.update({
        'database': motor.motor_tornado.MotorClient(**db_cfg).datasets,
        'system_token': config.get('rest',{}).get('system_token',None)
    })

    return [
        (r'/datasets', MultiDatasetHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)', DatasetHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)/description', DatasetDescriptionHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)/status', DatasetStatusHandler, handler_cfg),
        (r'/dataset_summaries/status', DatasetSummariesStatusHandler, handler_cfg),
    ]

class BaseHandler(RESTHandler):
    """
    Base handler for Dataset REST API.
    """
    def initialize(self, database=None, system_token=None, **kwargs):
        super(BaseHandler, self).initialize(**kwargs)
        self.db = database
        self.system_token = system_token

class MultiDatasetHandler(BaseHandler):
    """
    Handle multi-group requests.
    """
    @authorization(roles=['admin']) #TODO: figure out how to do auth for each dataset in the list
    async def get(self):
        """
        Get a dict of datasets.

        Returns:
            dict: {<dataset_id>: metadata}
        """
        ret = {}
        async for row in self.db.datasets.find(projection={'_id':False}):
            k = row['dataset_id']
            ret[k] = row
        self.write(ret)
        self.finish()

    @authorization(roles=['admin','user']) # anyone should be able to create a dataset
    async def post(self):
        """
        Add a dataset.

        Body should contain all necessary fields for a dataset.
        """
        data = json.loads(self.request.body)

        # validate first
        req_fields = {
            'description': str,
            'jobs_submitted': int,
            'tasks_submitted': int,
            'group_id': str
        }
        for k in req_fields:
            if k not in data:
                raise tornado.web.HTTPError(400, reason='missing key: '+k)
            if not isinstance(data[k], req_fields[k]):
                r = 'key {} should be of type {}'.format(k, req_fields[k])
                raise tornado.web.HTTPError(400, reason=r)

        # generate dataset number
        ret = await self.db.settings.find_one_and_update(
            {'name': 'dataset_num'},
            {'$inc': {'num': 1}},
            projection={'num': True, '_id': False},
            upsert=True,
            return_document=pymongo.ReturnDocument.AFTER)
        dataset_num = ret['num']

        # set some fields
        data['dataset_id'] = uuid.uuid1().hex
        data['dataset'] = dataset_num
        data['status'] = 'processing'
        data['start_date'] = nowstr()
        data['username'] = self.auth_data['username']
        if 'priority' not in data:
            data['priority'] = 0
        if 'debug' not in data:
            data['debug'] = False

        # insert
        ret = await self.db.datasets.insert_one(data)

        # set auth rules
        url = self.auth_url+'/auths/'+data['dataset_id']
        http_client = tornado.httpclient.AsyncHTTPClient()
        auth_data = {
            'read_groups':[data['group_id']],
            'write_groups':[data['group_id']],
        }
        await to_asyncio_future(http_client.fetch(url,
                method='PUT', body=json.dumps(auth_data),
                headers={'Authorization': 'bearer '+self.auth_key}))

        # return success
        self.set_status(201)
        self.set_header('Location', '/datasets/'+data['dataset_id'])
        self.write({'result': '/datasets/'+data['dataset_id']})
        self.finish()

class DatasetHandler(BaseHandler):
    """
    Handle dataset requests.
    """
    @authorization(roles=['admin'], attrs=['dataset_id:read'])
    async def get(self, dataset_id):
        """
        Get a dataset.

        Args:
            dataset_id (str): the dataset

        Returns:
            dict: dataset metadata
        """
        ret = await self.db.datasets.find_one({'dataset_id':dataset_id},
                projection={'_id':False})
        if not ret:
            self.send_error(404, reason="Dataset not found")
        else:
            self.write(ret)
            self.finish()

class DatasetDescriptionHandler(BaseHandler):
    """
    Handle dataset description updates.
    """
    @authorization(roles=['admin'], attrs=['dataset_id:write'])
    async def put(self, dataset_id):
        """
        Set a dataset description.

        Args:
            dataset_id (str): the dataset

        Returns:
            dict: empty dict
        """
        data = json.loads(self.request.body)
        if 'description' not in data:
            raise tornado.web.HTTPError(400, reason='missing description')
        elif not isinstance(data['description'],str):
            raise tornado.web.HTTPError(400, reason='bad description')

        ret = await self.db.datasets.find_one_and_update({'dataset_id':dataset_id},
                {'$set':{'description': data['description']}},
                projection=['_id'])
        if not ret:
            self.send_error(404, reason="Dataset not found")
        else:
            self.write({})
            self.finish()

class DatasetStatusHandler(BaseHandler):
    """
    Handle dataset status updates.
    """
    @authorization(roles=['admin'], attrs=['dataset_id:write'])
    async def put(self, dataset_id):
        """
        Set a dataset status.

        Args:
            dataset_id (str): the dataset

        Returns:
            dict: empty dict
        """
        data = json.loads(self.request.body)
        if 'status' not in data:
            raise tornado.web.HTTPError(400, reason='missing status')
        elif data['status'] not in ('processing','suspended','errors','complete'):
            raise tornado.web.HTTPError(400, reason='bad status')

        ret = await self.db.datasets.find_one_and_update({'dataset_id':dataset_id},
                {'$set':{'status': data['status']}},
                projection=['_id'])
        if not ret:
            self.send_error(404, reason="Dataset not found")
        else:
            self.write({})
            self.finish()

class DatasetSummariesStatusHandler(BaseHandler):
    """
    Handle dataset summary grouping by status.
    """
    @authorization(roles=['admin','system','user']) #TODO: figure out how to do auth for each dataset in the list
    async def get(self):
        """
        Get the dataset summary for all datasets, group by status.

        Returns:
            dict: {<status>: [<dataset_id>,]}
        """
        cursor = self.db.datasets.find(
                projection={'_id':False,'status':True,'dataset_id':True})
        ret = defaultdict(list)
        async for row in cursor:
            ret[row['status']].append(row['dataset_id'])
        self.write(ret)
        self.finish()
<|endoftext|>


<commit_before>"""
Use the DBLP dataset to recommend experts. Find the optimal parameters.
"""
import gc
import os
import numpy
import logging
import sys
import argparse
from exp.influence2.GraphRanker import GraphRanker
from exp.influence2.RankAggregator import RankAggregator
from exp.influence2.ArnetMinerDataset import ArnetMinerDataset
from apgl.util.Latex import Latex
from apgl.util.Evaluator import Evaluator

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
numpy.set_printoptions(suppress=True, precision=3, linewidth=160)
numpy.random.seed(21)

parser = argparse.ArgumentParser(description='Run reputation evaluation experiments')
parser.add_argument("-r", "--runLDA", action="store_true", help="Run Latent Dirchlet Allocation")
args = parser.parse_args()

averagePrecisionN = 50
similarityCutoff = 0.30
ns = numpy.arange(5, 105, 5)
runLSI = not args.runLDA

dataset = ArnetMinerDataset(runLSI=runLSI)
#dataset.dataFilename = dataset.dataDir + "DBLP-citation-100000.txt"
#dataset.dataFilename = dataset.dataDir + "DBLP-citation-5000000.txt"
dataset.dataFilename = dataset.dataDir + "DBLP-citation-7000000.txt"
dataset.ks = [100, 150, 200, 250, 300, 350]
dataset.overwriteGraph = True
dataset.overwriteModel = True

dataset.overwriteVectoriser = True
dataset.vectoriseDocuments()
dataset.overwriteVectoriser = False

dataset.modelSelection()
dataset.overwriteModel = False

for field in dataset.fields:
    logging.debug("Field = " + field)
    dataset.learnModel()
    relevantExperts = dataset.findSimilarDocuments(field)

    graph, authorIndexer = dataset.coauthorsGraph(field, relevantExperts)
    expertMatches = dataset.matchExperts(relevantExperts, dataset.testExpertDict[field])

    expertMatchesInds = authorIndexer.translate(expertMatches)
    relevantAuthorInds = authorIndexer.translate(relevantExperts)
    assert (numpy.array(relevantAuthorInds) < len(relevantAuthorInds)).all()

    if len(expertMatches) != 0:
        #First compute graph properties
        computeInfluence = False
        graphRanker = GraphRanker(k=100, numRuns=100, computeInfluence=computeInfluence, p=0.05, trainExpertsIdList=expertMatchesInds)
        outputLists = graphRanker.vertexRankings(graph, relevantAuthorInds, [relevantAuthorInds])
        itemList = RankAggregator.generateItemList(outputLists)
        #methodNames = graphRanker.getNames()

        numMethods = len(outputLists)
        precisions = numpy.zeros((len(ns), numMethods))
        averagePrecisions = numpy.zeros(numMethods)

        for i, n in enumerate(ns):
            for j in range(len(outputLists)):
                precisions[i, j] = Evaluator.precisionFromIndLists(expertMatchesInds, outputLists[j][0:n])

        for j in range(len(outputLists)):
            averagePrecisions[j] = Evaluator.averagePrecisionFromLists(expertMatchesInds, outputLists[j][0:averagePrecisionN], averagePrecisionN)

        precisions2 = numpy.c_[numpy.array(ns), precisions]

        logging.debug(Latex.array2DToRows(precisions2))
        logging.debug(Latex.array1DToRow(averagePrecisions))

        resultsFilename = dataset.getResultsDir(field) + "precisions.npz"
        numpy.savez(resultsFilename, precisions, averagePrecisions)
        logging.debug("Saved precisions as " + resultsFilename)

logging.debug("All done!")
<commit_msg>Run on all documents <commit_after>"""
Use the DBLP dataset to recommend experts. Find the optimal parameters.
"""
import gc
import os
import numpy
import logging
import sys
import argparse
from exp.influence2.GraphRanker import GraphRanker
from exp.influence2.RankAggregator import RankAggregator
from exp.influence2.ArnetMinerDataset import ArnetMinerDataset
from apgl.util.Latex import Latex
from apgl.util.Evaluator import Evaluator

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
numpy.set_printoptions(suppress=True, precision=3, linewidth=160)
numpy.random.seed(21)

parser = argparse.ArgumentParser(description='Run reputation evaluation experiments')
parser.add_argument("-r", "--runLDA", action="store_true", help="Run Latent Dirchlet Allocation")
args = parser.parse_args()

averagePrecisionN = 50
similarityCutoff = 0.30
ns = numpy.arange(5, 105, 5)
runLSI = not args.runLDA

dataset = ArnetMinerDataset(runLSI=runLSI)
#dataset.dataFilename = dataset.dataDir + "DBLP-citation-100000.txt"
#dataset.dataFilename = dataset.dataDir + "DBLP-citation-5000000.txt"
#dataset.dataFilename = dataset.dataDir + "DBLP-citation-7000000.txt"
dataset.dataFilename = dataset.dataDir + "DBLP-citation-Feb21.txt"
dataset.ks = [100, 150, 200, 250, 300, 350, 400]
dataset.overwriteGraph = True
dataset.overwriteModel = True

dataset.overwriteVectoriser = True
dataset.vectoriseDocuments()
dataset.overwriteVectoriser = False

dataset.modelSelection()
dataset.overwriteModel = False

for field in dataset.fields:
    logging.debug("Field = " + field)
    dataset.learnModel()
    relevantExperts = dataset.findSimilarDocuments(field)

    graph, authorIndexer = dataset.coauthorsGraph(field, relevantExperts)
    expertMatches = dataset.matchExperts(relevantExperts, dataset.testExpertDict[field])

    expertMatchesInds = authorIndexer.translate(expertMatches)
    relevantAuthorInds = authorIndexer.translate(relevantExperts)
    assert (numpy.array(relevantAuthorInds) < len(relevantAuthorInds)).all()

    if len(expertMatches) != 0:
        #First compute graph properties
        computeInfluence = False
        graphRanker = GraphRanker(k=100, numRuns=100, computeInfluence=computeInfluence, p=0.05, trainExpertsIdList=expertMatchesInds)
        outputLists = graphRanker.vertexRankings(graph, relevantAuthorInds, [relevantAuthorInds])
        itemList = RankAggregator.generateItemList(outputLists)
        #methodNames = graphRanker.getNames()

        numMethods = len(outputLists)
        precisions = numpy.zeros((len(ns), numMethods))
        averagePrecisions = numpy.zeros(numMethods)

        for i, n in enumerate(ns):
            for j in range(len(outputLists)):
                precisions[i, j] = Evaluator.precisionFromIndLists(expertMatchesInds, outputLists[j][0:n])

        for j in range(len(outputLists)):
            averagePrecisions[j] = Evaluator.averagePrecisionFromLists(expertMatchesInds, outputLists[j][0:averagePrecisionN], averagePrecisionN)

        precisions2 = numpy.c_[numpy.array(ns), precisions]

        logging.debug(Latex.array2DToRows(precisions2))
        logging.debug(Latex.array1DToRow(averagePrecisions))

        resultsFilename = dataset.getResultsDir(field) + "precisions.npz"
        numpy.savez(resultsFilename, precisions, averagePrecisions)
        logging.debug("Saved precisions as " + resultsFilename)

logging.debug("All done!")
<|endoftext|>


<commit_before>"""
Use the DBLP dataset to recommend experts. Find the optimal parameters.
"""
import gc
import os
import numpy
import logging
import sys
import argparse
from exp.influence2.GraphRanker import GraphRanker
from exp.influence2.RankAggregator import RankAggregator
from exp.influence2.ArnetMinerDataset import ArnetMinerDataset
from apgl.util.Latex import Latex
from apgl.util.Evaluator import Evaluator

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
numpy.set_printoptions(suppress=True, precision=3, linewidth=160)
numpy.random.seed(21)

parser = argparse.ArgumentParser(description='Run reputation evaluation experiments')
parser.add_argument("-r", "--runLDA", action="store_true", help="Run Latent Dirchlet Allocation")
args = parser.parse_args()

averagePrecisionN = 50
similarityCutoff = 0.30
ns = numpy.arange(5, 105, 5)
runLSI = not args.runLDA

dataset = ArnetMinerDataset(runLSI=runLSI)
#dataset.dataFilename = dataset.dataDir + "DBLP-citation-100000.txt"

dataset.dataFilename = dataset.dataDir + "DBLP-citation-1000000.txt"
#dataset.dataFilename = dataset.dataDir + "DBLP-citation-5000000.txt"
#dataset.dataFilename = dataset.dataDir + "DBLP-citation-7000000.txt"
#dataset.dataFilename = dataset.dataDir + "DBLP-citation-Feb21.txt"
dataset.ks = [300, 350, 400, 450, 500, 550, 600]
dataset.overwriteGraph = True
dataset.overwriteModel = True

dataset.overwriteVectoriser = True
dataset.vectoriseDocuments()
dataset.overwriteVectoriser = False

dataset.modelSelection()
dataset.overwriteModel = False

for field in dataset.fields:
    logging.debug("Field = " + field)
    dataset.learnModel()
    relevantExperts = dataset.findSimilarDocuments(field)

    graph, authorIndexer = dataset.coauthorsGraph(field, relevantExperts)
    expertMatches = dataset.matchExperts(relevantExperts, dataset.testExpertDict[field])

    expertMatchesInds = authorIndexer.translate(expertMatches)
    relevantAuthorInds = authorIndexer.translate(relevantExperts)
    assert (numpy.array(relevantAuthorInds) < len(relevantAuthorInds)).all()

    if len(expertMatches) != 0:
        #First compute graph properties
        computeInfluence = False
        graphRanker = GraphRanker(k=100, numRuns=100, computeInfluence=computeInfluence, p=0.05, trainExpertsIdList=expertMatchesInds)
        outputLists = graphRanker.vertexRankings(graph, relevantAuthorInds, [relevantAuthorInds])
        itemList = RankAggregator.generateItemList(outputLists)
        #methodNames = graphRanker.getNames()

        numMethods = len(outputLists)
        precisions = numpy.zeros((len(ns), numMethods))
        averagePrecisions = numpy.zeros(numMethods)

        for i, n in enumerate(ns):
            for j in range(len(outputLists)):
                precisions[i, j] = Evaluator.precisionFromIndLists(expertMatchesInds, outputLists[j][0:n])

        for j in range(len(outputLists)):
            averagePrecisions[j] = Evaluator.averagePrecisionFromLists(expertMatchesInds, outputLists[j][0:averagePrecisionN], averagePrecisionN)

        precisions2 = numpy.c_[numpy.array(ns), precisions]

        logging.debug(Latex.array2DToRows(precisions2))
        logging.debug(Latex.array1DToRow(averagePrecisions))

        resultsFilename = dataset.getResultsDir(field) + "precisions.npz"
        numpy.savez(resultsFilename, precisions, averagePrecisions)
        logging.debug("Saved precisions as " + resultsFilename)

logging.debug("All done!")
<commit_msg>5000000 lines <commit_after>"""
Use the DBLP dataset to recommend experts. Find the optimal parameters.
"""
import gc
import os
import numpy
import logging
import sys
import argparse
from exp.influence2.GraphRanker import GraphRanker
from exp.influence2.RankAggregator import RankAggregator
from exp.influence2.ArnetMinerDataset import ArnetMinerDataset
from apgl.util.Latex import Latex
from apgl.util.Evaluator import Evaluator

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
numpy.set_printoptions(suppress=True, precision=3, linewidth=160)
numpy.random.seed(21)

parser = argparse.ArgumentParser(description='Run reputation evaluation experiments')
parser.add_argument("-r", "--runLDA", action="store_true", help="Run Latent Dirchlet Allocation")
args = parser.parse_args()

averagePrecisionN = 50
similarityCutoff = 0.30
ns = numpy.arange(5, 105, 5)
runLSI = not args.runLDA

dataset = ArnetMinerDataset(runLSI=runLSI)
#dataset.dataFilename = dataset.dataDir + "DBLP-citation-100000.txt"

#dataset.dataFilename = dataset.dataDir + "DBLP-citation-1000000.txt"
dataset.dataFilename = dataset.dataDir + "DBLP-citation-5000000.txt"
#dataset.dataFilename = dataset.dataDir + "DBLP-citation-7000000.txt"
#dataset.dataFilename = dataset.dataDir + "DBLP-citation-Feb21.txt"
dataset.ks = [300, 350, 400, 450, 500, 550, 600]
dataset.overwriteGraph = True
dataset.overwriteModel = True

dataset.overwriteVectoriser = True
dataset.vectoriseDocuments()
dataset.overwriteVectoriser = False

dataset.modelSelection()
dataset.overwriteModel = False

for field in dataset.fields:
    logging.debug("Field = " + field)
    dataset.learnModel()
    relevantExperts = dataset.findSimilarDocuments(field)

    graph, authorIndexer = dataset.coauthorsGraph(field, relevantExperts)
    expertMatches = dataset.matchExperts(relevantExperts, dataset.testExpertDict[field])

    expertMatchesInds = authorIndexer.translate(expertMatches)
    relevantAuthorInds = authorIndexer.translate(relevantExperts)
    assert (numpy.array(relevantAuthorInds) < len(relevantAuthorInds)).all()

    if len(expertMatches) != 0:
        #First compute graph properties
        computeInfluence = False
        graphRanker = GraphRanker(k=100, numRuns=100, computeInfluence=computeInfluence, p=0.05, trainExpertsIdList=expertMatchesInds)
        outputLists = graphRanker.vertexRankings(graph, relevantAuthorInds, [relevantAuthorInds])
        itemList = RankAggregator.generateItemList(outputLists)
        #methodNames = graphRanker.getNames()

        numMethods = len(outputLists)
        precisions = numpy.zeros((len(ns), numMethods))
        averagePrecisions = numpy.zeros(numMethods)

        for i, n in enumerate(ns):
            for j in range(len(outputLists)):
                precisions[i, j] = Evaluator.precisionFromIndLists(expertMatchesInds, outputLists[j][0:n])

        for j in range(len(outputLists)):
            averagePrecisions[j] = Evaluator.averagePrecisionFromLists(expertMatchesInds, outputLists[j][0:averagePrecisionN], averagePrecisionN)

        precisions2 = numpy.c_[numpy.array(ns), precisions]

        logging.debug(Latex.array2DToRows(precisions2))
        logging.debug(Latex.array1DToRow(averagePrecisions))

        resultsFilename = dataset.getResultsDir(field) + "precisions.npz"
        numpy.savez(resultsFilename, precisions, averagePrecisions)
        logging.debug("Saved precisions as " + resultsFilename)

logging.debug("All done!")
<|endoftext|>


<commit_before>#!/usr/bin/python

## Binary Analysis Tool
## Copyright 2009-2016 Armijn Hemel for Tjaldur Software Governance Solutions
## Licensed under Apache 2.0, see LICENSE file for details

'''This file contains information about how to recognize certain
files, file systems, compression, and so on automatically and which
methods or functions to invoke to unpack these files for further
analysis.'''

## information from:
## 1. /usr/share/magic
## 2. include/linux/magic.h in the Linux kernel sources
## 3. http://www.squashfs-lzma.org/
## 4. http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=364260
## 5. various other places

## This is not the same as the magic database, but just a list of
## identifiers that are used for these file systems, compressed files,etc.
## In BAT a lot more work is done to verify what a file really is, which
## the magic database does not do.

fsmagic = {
            'gzip':             '\x1f\x8b\x08',     # x08 is the only compression method according to RFC 1952
            'compress':         '\x1f\x9d',
            'bz2':              'BZh',
            'rar':              'Rar!\x1a\x07',
            'rarfooter':        '\xc4\x3d\x7b\x00\x40\x07\x00', # http://forensicswiki.org/wiki/RAR#Terminator_.28terminator.29
            'zip':              '\x50\x4b\x03\04',
            'zipend':           '\x50\x4b\x05\06',
            'lrzip':            'LRZI',
            'rzip':             'RZIP',
            'squashfs1':        '\x68\x73\x71\x73', # hsqs -- little endian
            'squashfs2':        '\x73\x71\x73\x68', # sqsh -- big endian
            'squashfs3':        '\x71\x73\x68\x73', # qshs -- little endian
            'squashfs4':        '\x73\x68\x73\x71', # shsq -- big endian
            'squashfs5':        '\x74\x71\x73\x68', # tqsh - used in DD-WRT
            'squashfs6':        '\x68\x73\x71\x74', # hsqt - used in DD-WRT
            'squashfs7':        '\x73\x71\x6c\x7a', # sqlz
            'android-sparse':   '\x3a\xff\x26\xed',
            'lzma_alone':       '\x5d\x00\x00',
            'lzma_alone_alt':   '\x6d\x00\x00',     # used in OpenWrt
            'lzma_alone_alt2':  '\x6c\x00\x00',     # seen in some routers, like ZyXEL NBG5615
            '7z':               '7z\xbc\xaf\x27\x1c',
            'xz':               '\xfd\x37\x7a\x58\x5a\x00',
            'xztrailer':        '\x59\x5a',
            'lzip':             'LZIP',
            'lzop':              '\x89\x4c\x5a\x4f\x00\x0d\x0a\x1a\x0a',
            'lha':              '-lh7-',
            'cramfs_le':        '\x45\x3d\xcd\x28',
            'cramfs_be':        '\x28\xcd\x3d\x45',
            'romfs':            '-rom1fs-',
            'jffs2_le':         '\x85\x19',
            'jffs2_be':         '\x19\x85',
            'ubifs':            '\x31\x18\x10\x06',
            'ubi':              '\x55\x42\x49\x23',
            'rpm':              '\xed\xab\xee\xdb',
            'ext2':             '\x53\xef',        # little endian
            'minix':            '\x8f\x13',        # specific version of Minix v1 file system
            'arj':              '\x60\xea',
            'cab':              'MSCF\x00\x00\x00\x00',    # first four bytes following header are always 0
            'installshield':    'ISc(',
            'pkbac':            'PKBAC',
            'winrar':           'WinRAR',
            'png':              '\x89PNG\x0d\x0a\x1a\x0a',
            'pngtrailer':       '\x00\x00\x00\x00IEND\xae\x42\x60\x82', # length, chunk type and CRC for PNG trailer are always the same
            'cpiotrailer':      'TRAILER!!!',
            'bmp':              'BM',
            'jpeg':             '\xff\xd8',
            'jpegtrailer':      '\xff\xd9',
            'jfif':             'JFIF',
            'gif87':            'GIF87a',
            'gif89':            'GIF89a',
            'ico':              '\x00\x00\x01\x00',
            'riff':             'RIFF',
            'cpio1':            '070701',
            'cpio2':            '070702',
            'cpio3':            '070707',
            'iso9660':          'CD001',
            'swf':              'CWS',
            'pdf':              '%PDF-',
            'pdftrailer':       '%%EOF',
            'ar':               '!<arch>',
            'tar1':             'ustar\x00',
            'tar2':             'ustar\x20',
            'java_serialized':  '\xac\xed\x00',
            'fat12':  		'FAT12',
            'fat16':  		'FAT16',
            'pe':  		'MZ',
            'upx':  		'UPX',
            'java': 		'\xca\xfe\xba\xbe',
            'pack200':		'\xca\xfe\xd0\x0d',
            'dex':		'dex\n', ## Android Dex
            'odex':		'dey\n', ## Android Odex
            'oat':		'oat\n', ## Android OAT
            'otf':		'OTTO',
            'ttf':		'\x00\x01\x00\x00',
            'id3':		'TAG',
            'id3v2':		'ID3',
            'mp4':		'ftyp',
            'ogg':  		'OggS',
            'sqlite3':		'SQLite format 3\x00',
            'u-boot':		'\x27\x05\x19\x56',
            'yaffs2':		'\x03\x00\x00\x00\x01\x00\x00\x00\xff\xff', ## this is not a an official signature, just occuring frequently
            'plf':		'\x50\x4c\x46\x21',
            'chm':		'ITSF\x03\x00\x00\x00\x60\x00\x00\x00\x01\x00\x00\x00',
            'msi':		'\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1', ## not sure this is a correct signature
            'windowsassemblyheader':		'<assembly',
            'windowsassemblytrailer':		'</assembly>',
            'appledouble':	'\x00\x05\x16\x07',
            'mswim':	        'MSWIM\x00\x00\x00',
            'certificate':	'-----BEGIN',
            'androidbackup':	'ANDROID BACKUP\n',
            'woff':		'wOFF',
            'woff2':		'wOF2',
          }

## some offsets can be found after a certain number of bytes, but
## the actual file system or file starts earlier
correction = {
               'ext2':    0x438,
               'minix':   0x410,
               'iso9660': 32769,
               'tar1':    0x101,
               'tar2':    0x101,
               'fat12':   54,
               'fat16':   54,
               'lha':     2,
             }

## collection of markers that should be scanned together
squashtypes = ['squashfs1', 'squashfs2', 'squashfs3', 'squashfs4', 'squashfs5', 'squashfs6']
lzmatypes   = ['lzma_alone', 'lzma_alone_alt', 'lzma_alone_alt2']
cpio        = ['cpio1', 'cpio2', 'cpio3']
gif         = ['gif87', 'gif89']
tar         = ['tar1', 'tar2']
<commit_msg>add magic for xar<commit_after>#!/usr/bin/python

## Binary Analysis Tool
## Copyright 2009-2016 Armijn Hemel for Tjaldur Software Governance Solutions
## Licensed under Apache 2.0, see LICENSE file for details

'''This file contains information about how to recognize certain
files, file systems, compression, and so on automatically and which
methods or functions to invoke to unpack these files for further
analysis.'''

## information from:
## 1. /usr/share/magic
## 2. include/linux/magic.h in the Linux kernel sources
## 3. http://www.squashfs-lzma.org/
## 4. http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=364260
## 5. various other places

## This is not the same as the magic database, but just a list of
## identifiers that are used for these file systems, compressed files,etc.
## In BAT a lot more work is done to verify what a file really is, which
## the magic database does not do.

fsmagic = {
            'gzip':             '\x1f\x8b\x08',     # x08 is the only compression method according to RFC 1952
            'compress':         '\x1f\x9d',
            'bz2':              'BZh',
            'rar':              'Rar!\x1a\x07',
            'rarfooter':        '\xc4\x3d\x7b\x00\x40\x07\x00', # http://forensicswiki.org/wiki/RAR#Terminator_.28terminator.29
            'zip':              '\x50\x4b\x03\04',
            'zipend':           '\x50\x4b\x05\06',
            'lrzip':            'LRZI',
            'rzip':             'RZIP',
            'squashfs1':        '\x68\x73\x71\x73', # hsqs -- little endian
            'squashfs2':        '\x73\x71\x73\x68', # sqsh -- big endian
            'squashfs3':        '\x71\x73\x68\x73', # qshs -- little endian
            'squashfs4':        '\x73\x68\x73\x71', # shsq -- big endian
            'squashfs5':        '\x74\x71\x73\x68', # tqsh - used in DD-WRT
            'squashfs6':        '\x68\x73\x71\x74', # hsqt - used in DD-WRT
            'squashfs7':        '\x73\x71\x6c\x7a', # sqlz
            'android-sparse':   '\x3a\xff\x26\xed',
            'lzma_alone':       '\x5d\x00\x00',
            'lzma_alone_alt':   '\x6d\x00\x00',     # used in OpenWrt
            'lzma_alone_alt2':  '\x6c\x00\x00',     # seen in some routers, like ZyXEL NBG5615
            '7z':               '7z\xbc\xaf\x27\x1c',
            'xz':               '\xfd\x37\x7a\x58\x5a\x00',
            'xztrailer':        '\x59\x5a',
            'lzip':             'LZIP',
            'lzop':              '\x89\x4c\x5a\x4f\x00\x0d\x0a\x1a\x0a',
            'lha':              '-lh7-',
            'cramfs_le':        '\x45\x3d\xcd\x28',
            'cramfs_be':        '\x28\xcd\x3d\x45',
            'romfs':            '-rom1fs-',
            'jffs2_le':         '\x85\x19',
            'jffs2_be':         '\x19\x85',
            'ubifs':            '\x31\x18\x10\x06',
            'ubi':              '\x55\x42\x49\x23',
            'rpm':              '\xed\xab\xee\xdb',
            'ext2':             '\x53\xef',        # little endian
            'minix':            '\x8f\x13',        # specific version of Minix v1 file system
            'arj':              '\x60\xea',
            'cab':              'MSCF\x00\x00\x00\x00',    # first four bytes following header are always 0
            'installshield':    'ISc(',
            'pkbac':            'PKBAC',
            'winrar':           'WinRAR',
            'png':              '\x89PNG\x0d\x0a\x1a\x0a',
            'pngtrailer':       '\x00\x00\x00\x00IEND\xae\x42\x60\x82', # length, chunk type and CRC for PNG trailer are always the same
            'cpiotrailer':      'TRAILER!!!',
            'bmp':              'BM',
            'jpeg':             '\xff\xd8',
            'jpegtrailer':      '\xff\xd9',
            'jfif':             'JFIF',
            'gif87':            'GIF87a',
            'gif89':            'GIF89a',
            'ico':              '\x00\x00\x01\x00',
            'riff':             'RIFF',
            'cpio1':            '070701',
            'cpio2':            '070702',
            'cpio3':            '070707',
            'iso9660':          'CD001',
            'swf':              'CWS',
            'pdf':              '%PDF-',
            'pdftrailer':       '%%EOF',
            'ar':               '!<arch>',
            'tar1':             'ustar\x00',
            'tar2':             'ustar\x20',
            'java_serialized':  '\xac\xed\x00',
            'fat12':  		'FAT12',
            'fat16':  		'FAT16',
            'pe':  		'MZ',
            'upx':  		'UPX',
            'java': 		'\xca\xfe\xba\xbe',
            'pack200':		'\xca\xfe\xd0\x0d',
            'dex':		'dex\n', ## Android Dex
            'odex':		'dey\n', ## Android Odex
            'oat':		'oat\n', ## Android OAT
            'otf':		'OTTO',
            'ttf':		'\x00\x01\x00\x00',
            'id3':		'TAG',
            'id3v2':		'ID3',
            'mp4':		'ftyp',
            'ogg':  		'OggS',
            'sqlite3':		'SQLite format 3\x00',
            'u-boot':		'\x27\x05\x19\x56',
            'yaffs2':		'\x03\x00\x00\x00\x01\x00\x00\x00\xff\xff', ## this is not a an official signature, just occuring frequently
            'plf':		'\x50\x4c\x46\x21',
            'chm':		'ITSF\x03\x00\x00\x00\x60\x00\x00\x00\x01\x00\x00\x00',
            'msi':		'\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1', ## not sure this is a correct signature
            'windowsassemblyheader':		'<assembly',
            'windowsassemblytrailer':		'</assembly>',
            'appledouble':	'\x00\x05\x16\x07',
            'mswim':	        'MSWIM\x00\x00\x00',
            'certificate':	'-----BEGIN',
            'androidbackup':	'ANDROID BACKUP\n',
            'woff':		'wOFF',
            'woff2':		'wOF2',
            'xar':		'\x78\x61\x72\x21',
          }

## some offsets can be found after a certain number of bytes, but
## the actual file system or file starts earlier
correction = {
               'ext2':    0x438,
               'minix':   0x410,
               'iso9660': 32769,
               'tar1':    0x101,
               'tar2':    0x101,
               'fat12':   54,
               'fat16':   54,
               'lha':     2,
             }

## collection of markers that should be scanned together
squashtypes = ['squashfs1', 'squashfs2', 'squashfs3', 'squashfs4', 'squashfs5', 'squashfs6']
lzmatypes   = ['lzma_alone', 'lzma_alone_alt', 'lzma_alone_alt2']
cpio        = ['cpio1', 'cpio2', 'cpio3']
gif         = ['gif87', 'gif89']
tar         = ['tar1', 'tar2']
<|endoftext|>


<commit_before>#!/usr/bin/python

## Binary Analysis Tool
## Copyright 2009-2011 Armijn Hemel for LOCO (LOOHUIS CONSULTING)
## Licensed under Apache 2.0, see LICENSE file for details

'''This file contains information about how to recognize certain
files, file systems, compression, and so on automatically and which
methods or functions to invoke to unpack these files for further
analysis.'''

## information from:
## 1. /usr/share/magic
## 2. include/linux/magic.h in the Linux kernel sources
## 3. http://www.squashfs-lzma.org/
## 4. http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=364260

## basically we are rebuilding the magic database here :(
## (name: identifier)

fsmagic = {
            #'gzip':             '\x1f\x8b\x08',     # this excludes some gzip files!
            'gzip':             '\x1f\x8b',
            'bz2':              'BZh',
            'rar':              'Rar!',
            'zip':              '\x50\x4b\x03\04',
            'squashfs-le':      '\x68\x73\x71\x73', # hsqs
            'squashfs-be':      '\x73\x71\x73\x68', # sqsh
            'squashfs-le-lzma': '\x71\x73\x68\x73', # qshs
            'squashfs-be-lzma': '\x73\x68\x73\x71', # shsq
            'lzma_alone':       '\x5d\x00\x00',
            'xz':               '\xfd\x37\x7a\x58\x5a\x00',
            'xztrailer':        '\x59\x5a',
            'lzip':             'LZIP',
            'lzo':              '\x89\x4c\x5a\x4f\x00\x0d\x0a\x1a\x0a',
            'cramfs':           '\x45\x3d\xcd\x28',
            #'lzma_alone':       '\x5d\x00\x00\x80',
            #'jffs2-le':         '\x85\x19',
            'ubifs':            '\x55\x42\x49\x23',
            'rpm':              '\xed\xab\xee\xdb',
            'ext2':             '\x53\xef',        # little endian
            'arj':              '\x60\xea',
            'cab':              'MSCF',
            'png':              '\x89PNG\x0d\x0a\x1a\x0a',
            'pngtrailer':       'IEND',
            'cpiotrailer':      'TRAILER!!!',
            'jfif':             'JFIF',
            'gif87':            'GIF87a',
            'gif89':            'GIF89a',
            'cpio1':            '070701',
            'cpio2':            '070702',
            'cpio3':            '070707',
          }

## collection of markers that should be scanned together
squashtypes = ['squashfs-le', 'squashfs-be', 'squashfs-le-lzma', 'squashfs-be-lzma']
cpio        = ['cpio1', 'cpio2', 'cpio3']
gif         = ['gif87', 'gif89']
<commit_msg>rename squashfs markers since they did not correctly reflect the actual file system<commit_after>#!/usr/bin/python

## Binary Analysis Tool
## Copyright 2009-2011 Armijn Hemel for LOCO (LOOHUIS CONSULTING)
## Licensed under Apache 2.0, see LICENSE file for details

'''This file contains information about how to recognize certain
files, file systems, compression, and so on automatically and which
methods or functions to invoke to unpack these files for further
analysis.'''

## information from:
## 1. /usr/share/magic
## 2. include/linux/magic.h in the Linux kernel sources
## 3. http://www.squashfs-lzma.org/
## 4. http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=364260

## basically we are rebuilding the magic database here :(
## (name: identifier)

fsmagic = {
            #'gzip':             '\x1f\x8b\x08',     # this excludes some gzip files!
            'gzip':             '\x1f\x8b',
            'bz2':              'BZh',
            'rar':              'Rar!',
            'zip':              '\x50\x4b\x03\04',
            'squashfs1':        '\x68\x73\x71\x73', # hsqs
            'squashfs2':        '\x73\x71\x73\x68', # sqsh
            'squashfs3':        '\x71\x73\x68\x73', # qshs
            'squashfs4':        '\x73\x68\x73\x71', # shsq
            'lzma_alone':       '\x5d\x00\x00',
            'xz':               '\xfd\x37\x7a\x58\x5a\x00',
            'xztrailer':        '\x59\x5a',
            'lzip':             'LZIP',
            'lzo':              '\x89\x4c\x5a\x4f\x00\x0d\x0a\x1a\x0a',
            'cramfs':           '\x45\x3d\xcd\x28',
            #'lzma_alone':       '\x5d\x00\x00\x80',
            #'jffs2-le':         '\x85\x19',
            'ubifs':            '\x55\x42\x49\x23',
            'rpm':              '\xed\xab\xee\xdb',
            'ext2':             '\x53\xef',        # little endian
            'arj':              '\x60\xea',
            'cab':              'MSCF',
            'png':              '\x89PNG\x0d\x0a\x1a\x0a',
            'pngtrailer':       'IEND',
            'cpiotrailer':      'TRAILER!!!',
            'jfif':             'JFIF',
            'gif87':            'GIF87a',
            'gif89':            'GIF89a',
            'cpio1':            '070701',
            'cpio2':            '070702',
            'cpio3':            '070707',
          }

## collection of markers that should be scanned together
squashtypes = ['squashfs1', 'squashfs2', 'squashfs3', 'squashfs4']
cpio        = ['cpio1', 'cpio2', 'cpio3']
gif         = ['gif87', 'gif89']
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# (c) 2012-2013, Timothy Appnel <tim@appnel.com>
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

DOCUMENTATION = '''
---
module: synchronize
version_added: "1.4"
short_description: Uses rsync to make synchronizing file paths in your playbooks quick and easy.
description:
    - This is a wrapper around rsync. Of course you could just use the command action to call rsync yourself, but you also have to add a fair number of boilerplate options and host facts. You still may need to call rsync directly via C(command) or C(shell) depending on your use case. The synchronize action is meant to do common things with C(rsync) easily. It does not provide access to the full power of rsync, but does make most invocations easier to follow.
options:
  src:
    description:
      - Path on the source machine that will be synchronized to the destination; The path can be absolute or relative.
    required: true
  dest:
    description:
      - Path on the destination machine that will be synchronized from the source; The path can be absolute or relative.
    required: true
  dest_port:
    description:
      - Port number for ssh on the destination host. The ansible_ssh_port inventory var takes precedence over this value.
    default: 22
    version_added: "1.5"
  mode:
    description:
      - Specify the direction of the synchronization. In push mode the localhost or delegate is the source; In pull mode the remote host in context is the source.
    required: false
    choices: [ 'push', 'pull' ]
    default: 'push'
  archive:
    description:
      - Mirrors the rsync archive flag, enables recursive, links, perms, times, owner, group flags and -D.
    choices: [ 'yes', 'no' ]
    default: 'yes'
    required: false
  checksum:
    description:
      - Skip based on checksum, rather than mod-time & size; Note that that "archive" option is still enabled by default - the "checksum" option will not disable it.
    choices: [ 'yes', 'no' ]
    default: 'no'
    required: false
    version_added: "1.6"
  compress:
    description:
      - Compress file data during the transfer. In most cases, leave this enabled unless it causes problems.
    choices: [ 'yes', 'no' ]
    default: 'yes'
    required: false
    version_added: "1.7"
  existing_only:
    description:
      - Skip creating new files on receiver.
    choices: [ 'yes', 'no' ]
    default: 'no'
    required: false
    version_added: "1.5"
  delete:
    description:
      - Delete files that don't exist (after transfer, not before) in the C(src) path. This option requires C(recursive=yes).
    choices: [ 'yes', 'no' ]
    default: 'no'
    required: false
  dirs:
    description:
      - Transfer directories without recursing
    choices: [ 'yes', 'no' ]
    default: 'no'
    required: false
  recursive:
    description:
      - Recurse into directories.
    choices: [ 'yes', 'no' ]
    default: the value of the archive option
    required: false
  links:
    description:
      - Copy symlinks as symlinks.
    choices: [ 'yes', 'no' ]
    default: the value of the archive option
    required: false
  copy_links:
    description:
      - Copy symlinks as the item that they point to (the referent) is copied, rather than the symlink.
    choices: [ 'yes', 'no' ]
    default: 'no'
    required: false
  perms:
    description:
      - Preserve permissions.
    choices: [ 'yes', 'no' ]
    default: the value of the archive option
    required: false
  times:
    description:
      - Preserve modification times
    choices: [ 'yes', 'no' ]
    default: the value of the archive option
    required: false
  owner:
    description:
      - Preserve owner (super user only)
    choices: [ 'yes', 'no' ]
    default: the value of the archive option
    required: false
  group:
    description:
      - Preserve group
    choices: [ 'yes', 'no' ]
    default: the value of the archive option
    required: false
  rsync_path:
    description:
      - Specify the rsync command to run on the remote machine. See C(--rsync-path) on the rsync man page.
    required: false
  rsync_timeout:
    description:
      - Specify a --timeout for the rsync command in seconds.
    default: 0
    required: false
  set_remote_user:
    description:
      - put user@ for the remote paths. If you have a custom ssh config to define the remote user for a host
        that does not match the inventory user, you should set this parameter to "no".
    default: yes
  use_ssh_args:
    description:
      - Use the ssh_args specified in ansible.cfg
    default: "yes"
    choices:
      - "yes"
      - "no"
    version_added: "2.0"
  rsync_opts:
    description:
      - Specify additional rsync options by passing in an array.
    default:
    required: false
    version_added: "1.6"
notes:
   - `rsync` must be installed on both the local and remote machine.
   - Inspect the verbose output to validate the destination user/host/path
     are what was expected.
   - The remote user for the dest path will always be the remote_user, not
     the sudo_user.
   - Expect that dest=~/x will be ~<remote_user>/x even if using sudo.
   - To exclude files and directories from being synchronized, you may add
     C(.rsync-filter) files to the source directory.

author: Timothy Appnel
'''

EXAMPLES = '''
# Synchronization of src on the control machine to dest on the remote hosts
synchronize: src=some/relative/path dest=/some/absolute/path

# Synchronization without any --archive options enabled
synchronize: src=some/relative/path dest=/some/absolute/path archive=no

# Synchronization with --archive options enabled except for --recursive
synchronize: src=some/relative/path dest=/some/absolute/path recursive=no

# Synchronization with --archive options enabled except for --times, with --checksum option enabled
synchronize: src=some/relative/path dest=/some/absolute/path checksum=yes times=no

# Synchronization without --archive options enabled except use --links
synchronize: src=some/relative/path dest=/some/absolute/path archive=no links=yes

# Synchronization of two paths both on the control machine
local_action: synchronize src=some/relative/path dest=/some/absolute/path

# Synchronization of src on the inventory host to the dest on the localhost in
pull mode
synchronize: mode=pull src=some/relative/path dest=/some/absolute/path

# Synchronization of src on delegate host to dest on the current inventory host.
# If delegate_to is set to the current inventory host, this can be used to syncronize
# two directories on that host.
synchronize: >
    src=some/relative/path dest=/some/absolute/path
    delegate_to: delegate.host

# Synchronize and delete files in dest on the remote host that are not found in src of localhost.
synchronize: src=some/relative/path dest=/some/absolute/path delete=yes

# Synchronize using an alternate rsync command
synchronize: src=some/relative/path dest=/some/absolute/path rsync_path="sudo rsync"

# Example .rsync-filter file in the source directory
- var       # exclude any path whose last part is 'var'
- /var      # exclude any path starting with 'var' starting at the source directory
+ /var/conf # include /var/conf even though it was previously excluded

# Synchronize passing in extra rsync options
synchronize: src=/tmp/helloworld dest=/var/www/helloword rsync_opts=--no-motd,--exclude=.git
'''

def main():
    module = AnsibleModule(
        argument_spec = dict(
            src = dict(required=True),
            dest = dict(required=True),
            dest_port = dict(default=22),
            delete = dict(default='no', type='bool'),
            private_key = dict(default=None),
            rsync_path = dict(default=None),
            archive = dict(default='yes', type='bool'),
            checksum = dict(default='no', type='bool'),
            compress = dict(default='yes', type='bool'),
            existing_only = dict(default='no', type='bool'),
            dirs  = dict(default='no', type='bool'),
            recursive = dict(type='bool'),
            links = dict(type='bool'),
            copy_links = dict(type='bool'),
            perms = dict(type='bool'),
            times = dict(type='bool'),
            owner = dict(type='bool'),
            group = dict(type='bool'),
            set_remote_user = dict(default='yes', type='bool'),
            rsync_timeout = dict(type='int', default=0),
            rsync_opts = dict(type='list'),
            ssh_args = dict(type='str'),
        ),
        supports_check_mode = True
    )

    source = '"' + module.params['src'] + '"'
    dest = '"' + module.params['dest'] + '"'
    dest_port = module.params['dest_port']
    delete = module.params['delete']
    private_key = module.params['private_key']
    rsync_path = module.params['rsync_path']
    rsync = module.params.get('local_rsync_path', 'rsync')
    rsync_timeout = module.params.get('rsync_timeout', 'rsync_timeout')
    archive = module.params['archive']
    checksum = module.params['checksum']
    compress = module.params['compress']
    existing_only = module.params['existing_only']
    dirs = module.params['dirs']
    # the default of these params depends on the value of archive
    recursive = module.params['recursive']
    links = module.params['links']
    copy_links = module.params['copy_links']
    perms = module.params['perms']
    times = module.params['times']
    owner = module.params['owner']
    group = module.params['group']
    rsync_opts = module.params['rsync_opts']
    ssh_args = module.params['ssh_args']

    cmd = '%s --delay-updates -F' % rsync
    if compress:
        cmd = cmd + ' --compress'
    if rsync_timeout:
        cmd = cmd + ' --timeout=%s' % rsync_timeout
    if module.check_mode:
        cmd = cmd + ' --dry-run'
    if delete:
        cmd = cmd + ' --delete-after'
    if existing_only:
        cmd = cmd + ' --existing'
    if checksum:
        cmd = cmd + ' --checksum'
    if archive:
        cmd = cmd + ' --archive'
        if recursive is False:
            cmd = cmd + ' --no-recursive'
        if links is False:
            cmd = cmd + ' --no-links'
        if copy_links is True:
            cmd = cmd + ' --copy-links'
        if perms is False:
            cmd = cmd + ' --no-perms'
        if times is False:
            cmd = cmd + ' --no-times'
        if owner is False:
            cmd = cmd + ' --no-owner'
        if group is False:
            cmd = cmd + ' --no-group'
    else:
        if recursive is True:
            cmd = cmd + ' --recursive'
        if links is True:
            cmd = cmd + ' --links'
        if copy_links is True:
            cmd = cmd + ' --copy-links'
        if perms is True:
            cmd = cmd + ' --perms'
        if times is True:
            cmd = cmd + ' --times'
        if owner is True:
            cmd = cmd + ' --owner'
        if group is True:
            cmd = cmd + ' --group'
    if dirs:
        cmd = cmd + ' --dirs'
    if private_key is None:
        private_key = ''
    else:
        private_key = '-i '+ private_key

    ssh_opts = '-S none -o StrictHostKeyChecking=no %s' % ssh_args
    if dest_port != 22:
        cmd += " --rsh 'ssh %s %s -o Port=%s'" % (private_key, ssh_opts, dest_port)
    else:
        cmd += " --rsh 'ssh %s %s'" % (private_key, ssh_opts)  # need ssh param

    if rsync_path:
        cmd = cmd + " --rsync-path=%s" % (rsync_path)

    if rsync_opts:
        cmd = cmd + " " +  " ".join(rsync_opts)

    changed_marker = '<<CHANGED>>'
    cmd = cmd + " --out-format='" + changed_marker + "%i %n%L'"

    # expand the paths
    if '@' not in source:
        source = os.path.expanduser(source)
    if '@' not in dest:
        dest = os.path.expanduser(dest)

    cmd = ' '.join([cmd, source, dest])
    cmdstr = cmd
    (rc, out, err) = module.run_command(cmd)
    if rc:
        return module.fail_json(msg=err, rc=rc, cmd=cmdstr)
    else:
        changed = changed_marker in out
        out_clean=out.replace(changed_marker,'')
        out_lines=out_clean.split('\n')
        while '' in out_lines:
            out_lines.remove('')
        return module.exit_json(changed=changed, msg=out_clean,
                                rc=rc, cmd=cmdstr, stdout_lines=out_lines)

# import module snippets
from ansible.module_utils.basic import *

main()

<commit_msg>synchronize: don't add ssh_args to ssh_opts when false or null<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# (c) 2012-2013, Timothy Appnel <tim@appnel.com>
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

DOCUMENTATION = '''
---
module: synchronize
version_added: "1.4"
short_description: Uses rsync to make synchronizing file paths in your playbooks quick and easy.
description:
    - This is a wrapper around rsync. Of course you could just use the command action to call rsync yourself, but you also have to add a fair number of boilerplate options and host facts. You still may need to call rsync directly via C(command) or C(shell) depending on your use case. The synchronize action is meant to do common things with C(rsync) easily. It does not provide access to the full power of rsync, but does make most invocations easier to follow.
options:
  src:
    description:
      - Path on the source machine that will be synchronized to the destination; The path can be absolute or relative.
    required: true
  dest:
    description:
      - Path on the destination machine that will be synchronized from the source; The path can be absolute or relative.
    required: true
  dest_port:
    description:
      - Port number for ssh on the destination host. The ansible_ssh_port inventory var takes precedence over this value.
    default: 22
    version_added: "1.5"
  mode:
    description:
      - Specify the direction of the synchronization. In push mode the localhost or delegate is the source; In pull mode the remote host in context is the source.
    required: false
    choices: [ 'push', 'pull' ]
    default: 'push'
  archive:
    description:
      - Mirrors the rsync archive flag, enables recursive, links, perms, times, owner, group flags and -D.
    choices: [ 'yes', 'no' ]
    default: 'yes'
    required: false
  checksum:
    description:
      - Skip based on checksum, rather than mod-time & size; Note that that "archive" option is still enabled by default - the "checksum" option will not disable it.
    choices: [ 'yes', 'no' ]
    default: 'no'
    required: false
    version_added: "1.6"
  compress:
    description:
      - Compress file data during the transfer. In most cases, leave this enabled unless it causes problems.
    choices: [ 'yes', 'no' ]
    default: 'yes'
    required: false
    version_added: "1.7"
  existing_only:
    description:
      - Skip creating new files on receiver.
    choices: [ 'yes', 'no' ]
    default: 'no'
    required: false
    version_added: "1.5"
  delete:
    description:
      - Delete files that don't exist (after transfer, not before) in the C(src) path. This option requires C(recursive=yes).
    choices: [ 'yes', 'no' ]
    default: 'no'
    required: false
  dirs:
    description:
      - Transfer directories without recursing
    choices: [ 'yes', 'no' ]
    default: 'no'
    required: false
  recursive:
    description:
      - Recurse into directories.
    choices: [ 'yes', 'no' ]
    default: the value of the archive option
    required: false
  links:
    description:
      - Copy symlinks as symlinks.
    choices: [ 'yes', 'no' ]
    default: the value of the archive option
    required: false
  copy_links:
    description:
      - Copy symlinks as the item that they point to (the referent) is copied, rather than the symlink.
    choices: [ 'yes', 'no' ]
    default: 'no'
    required: false
  perms:
    description:
      - Preserve permissions.
    choices: [ 'yes', 'no' ]
    default: the value of the archive option
    required: false
  times:
    description:
      - Preserve modification times
    choices: [ 'yes', 'no' ]
    default: the value of the archive option
    required: false
  owner:
    description:
      - Preserve owner (super user only)
    choices: [ 'yes', 'no' ]
    default: the value of the archive option
    required: false
  group:
    description:
      - Preserve group
    choices: [ 'yes', 'no' ]
    default: the value of the archive option
    required: false
  rsync_path:
    description:
      - Specify the rsync command to run on the remote machine. See C(--rsync-path) on the rsync man page.
    required: false
  rsync_timeout:
    description:
      - Specify a --timeout for the rsync command in seconds.
    default: 0
    required: false
  set_remote_user:
    description:
      - put user@ for the remote paths. If you have a custom ssh config to define the remote user for a host
        that does not match the inventory user, you should set this parameter to "no".
    default: yes
  use_ssh_args:
    description:
      - Use the ssh_args specified in ansible.cfg
    default: "yes"
    choices:
      - "yes"
      - "no"
    version_added: "2.0"
  rsync_opts:
    description:
      - Specify additional rsync options by passing in an array.
    default:
    required: false
    version_added: "1.6"
notes:
   - `rsync` must be installed on both the local and remote machine.
   - Inspect the verbose output to validate the destination user/host/path
     are what was expected.
   - The remote user for the dest path will always be the remote_user, not
     the sudo_user.
   - Expect that dest=~/x will be ~<remote_user>/x even if using sudo.
   - To exclude files and directories from being synchronized, you may add
     C(.rsync-filter) files to the source directory.

author: Timothy Appnel
'''

EXAMPLES = '''
# Synchronization of src on the control machine to dest on the remote hosts
synchronize: src=some/relative/path dest=/some/absolute/path

# Synchronization without any --archive options enabled
synchronize: src=some/relative/path dest=/some/absolute/path archive=no

# Synchronization with --archive options enabled except for --recursive
synchronize: src=some/relative/path dest=/some/absolute/path recursive=no

# Synchronization with --archive options enabled except for --times, with --checksum option enabled
synchronize: src=some/relative/path dest=/some/absolute/path checksum=yes times=no

# Synchronization without --archive options enabled except use --links
synchronize: src=some/relative/path dest=/some/absolute/path archive=no links=yes

# Synchronization of two paths both on the control machine
local_action: synchronize src=some/relative/path dest=/some/absolute/path

# Synchronization of src on the inventory host to the dest on the localhost in
pull mode
synchronize: mode=pull src=some/relative/path dest=/some/absolute/path

# Synchronization of src on delegate host to dest on the current inventory host.
# If delegate_to is set to the current inventory host, this can be used to syncronize
# two directories on that host.
synchronize: >
    src=some/relative/path dest=/some/absolute/path
    delegate_to: delegate.host

# Synchronize and delete files in dest on the remote host that are not found in src of localhost.
synchronize: src=some/relative/path dest=/some/absolute/path delete=yes

# Synchronize using an alternate rsync command
synchronize: src=some/relative/path dest=/some/absolute/path rsync_path="sudo rsync"

# Example .rsync-filter file in the source directory
- var       # exclude any path whose last part is 'var'
- /var      # exclude any path starting with 'var' starting at the source directory
+ /var/conf # include /var/conf even though it was previously excluded

# Synchronize passing in extra rsync options
synchronize: src=/tmp/helloworld dest=/var/www/helloword rsync_opts=--no-motd,--exclude=.git
'''

def main():
    module = AnsibleModule(
        argument_spec = dict(
            src = dict(required=True),
            dest = dict(required=True),
            dest_port = dict(default=22),
            delete = dict(default='no', type='bool'),
            private_key = dict(default=None),
            rsync_path = dict(default=None),
            archive = dict(default='yes', type='bool'),
            checksum = dict(default='no', type='bool'),
            compress = dict(default='yes', type='bool'),
            existing_only = dict(default='no', type='bool'),
            dirs  = dict(default='no', type='bool'),
            recursive = dict(type='bool'),
            links = dict(type='bool'),
            copy_links = dict(type='bool'),
            perms = dict(type='bool'),
            times = dict(type='bool'),
            owner = dict(type='bool'),
            group = dict(type='bool'),
            set_remote_user = dict(default='yes', type='bool'),
            rsync_timeout = dict(type='int', default=0),
            rsync_opts = dict(type='list'),
            ssh_args = dict(type='str'),
        ),
        supports_check_mode = True
    )

    source = '"' + module.params['src'] + '"'
    dest = '"' + module.params['dest'] + '"'
    dest_port = module.params['dest_port']
    delete = module.params['delete']
    private_key = module.params['private_key']
    rsync_path = module.params['rsync_path']
    rsync = module.params.get('local_rsync_path', 'rsync')
    rsync_timeout = module.params.get('rsync_timeout', 'rsync_timeout')
    archive = module.params['archive']
    checksum = module.params['checksum']
    compress = module.params['compress']
    existing_only = module.params['existing_only']
    dirs = module.params['dirs']
    # the default of these params depends on the value of archive
    recursive = module.params['recursive']
    links = module.params['links']
    copy_links = module.params['copy_links']
    perms = module.params['perms']
    times = module.params['times']
    owner = module.params['owner']
    group = module.params['group']
    rsync_opts = module.params['rsync_opts']
    ssh_args = module.params['ssh_args']

    cmd = '%s --delay-updates -F' % rsync
    if compress:
        cmd = cmd + ' --compress'
    if rsync_timeout:
        cmd = cmd + ' --timeout=%s' % rsync_timeout
    if module.check_mode:
        cmd = cmd + ' --dry-run'
    if delete:
        cmd = cmd + ' --delete-after'
    if existing_only:
        cmd = cmd + ' --existing'
    if checksum:
        cmd = cmd + ' --checksum'
    if archive:
        cmd = cmd + ' --archive'
        if recursive is False:
            cmd = cmd + ' --no-recursive'
        if links is False:
            cmd = cmd + ' --no-links'
        if copy_links is True:
            cmd = cmd + ' --copy-links'
        if perms is False:
            cmd = cmd + ' --no-perms'
        if times is False:
            cmd = cmd + ' --no-times'
        if owner is False:
            cmd = cmd + ' --no-owner'
        if group is False:
            cmd = cmd + ' --no-group'
    else:
        if recursive is True:
            cmd = cmd + ' --recursive'
        if links is True:
            cmd = cmd + ' --links'
        if copy_links is True:
            cmd = cmd + ' --copy-links'
        if perms is True:
            cmd = cmd + ' --perms'
        if times is True:
            cmd = cmd + ' --times'
        if owner is True:
            cmd = cmd + ' --owner'
        if group is True:
            cmd = cmd + ' --group'
    if dirs:
        cmd = cmd + ' --dirs'
    if private_key is None:
        private_key = ''
    else:
        private_key = '-i '+ private_key

    if ssh_args:
      ssh_opts = '-S none -o StrictHostKeyChecking=no %s' % ssh_args
    else:
      ssh_opts = '-S none -o StrictHostKeyChecking=no'

    if dest_port != 22:
        cmd += " --rsh 'ssh %s %s -o Port=%s'" % (private_key, ssh_opts, dest_port)
    else:
        cmd += " --rsh 'ssh %s %s'" % (private_key, ssh_opts)  # need ssh param

    if rsync_path:
        cmd = cmd + " --rsync-path=%s" % (rsync_path)

    if rsync_opts:
        cmd = cmd + " " +  " ".join(rsync_opts)

    changed_marker = '<<CHANGED>>'
    cmd = cmd + " --out-format='" + changed_marker + "%i %n%L'"

    # expand the paths
    if '@' not in source:
        source = os.path.expanduser(source)
    if '@' not in dest:
        dest = os.path.expanduser(dest)

    cmd = ' '.join([cmd, source, dest])
    cmdstr = cmd
    (rc, out, err) = module.run_command(cmd)
    if rc:
        return module.fail_json(msg=err, rc=rc, cmd=cmdstr)
    else:
        changed = changed_marker in out
        out_clean=out.replace(changed_marker,'')
        out_lines=out_clean.split('\n')
        while '' in out_lines:
            out_lines.remove('')
        return module.exit_json(changed=changed, msg=out_clean,
                                rc=rc, cmd=cmdstr, stdout_lines=out_lines)

# import module snippets
from ansible.module_utils.basic import *

main()

<|endoftext|>