<commit_before>#!/usr/bin/env python
import os
import argparse
import glob
from klusta_pipeline.maps import port_site
from klusta_pipeline.dataio import load_recordings, load_catlog
from klusta_pipeline.dataio import save_info, save_recording, save_chanlist, save_probe, save_parameters
from klusta_pipeline.utils import get_import_list, validate_merge, realign, calc_weights, do_war, do_car
from klusta_pipeline.probe import get_channel_groups, clean_dead_channels, build_geometries

# assume spike2 export to mat with the following parameters:
# - channel names are "Port_N" where N is the 1-indexed 1401 Port number

def get_args():

    parser = argparse.ArgumentParser(description='Compile Spike2 epoch .mat files into KlustaKwik KWD file.')
    parser.add_argument('rig',type=str,
                       help='defines the rig that the data was collected on')
    parser.add_argument('probe',type=str,
                       help='defines the probe that the data was collected on')
    parser.add_argument('path', default = './', nargs='?',
                       help='directory containing all of the mat files to compile')
    parser.add_argument('dest', default = './', nargs='?',
                       help='destination directory for kwd and other files')
    parser.add_argument('-s','--sampling_rate',dest='fs',type=float, default=None,
                       help='target sampling rate for waveform alignment')
    parser.add_argument('-c','--common_average_ref',dest='car',action='store_true',
                       help='turns on common average referencing')
    parser.add_argument('-w','--weighted',dest='weighted',action='store_true',
                       help='weights channels for common average referencing')
    parser.add_argument('-x','--drop',dest='omit',type=str, default='',
                       help='comma-separate list of channel labels to drop if they exist')

    return parser.parse_args()

def main():
    args = get_args()
    path = os.path.abspath(args.path)
    dest = os.path.abspath(args.dest)

    info = dict(args.__dict__)

    catlog = glob.glob(os.path.join(path,'*.catLog'))[0]
    info['name'] = os.path.split(catlog)[-1].split('.')[0]

    kwd = os.path.join(dest, info['name'] + '.raw.kwd')
    if os.path.exists(kwd):
        raise IOError('%s already exists, please delete or rename and run again' % kwd)

    info['exports'] = load_catlog(catlog)
    import_list = get_import_list(path,info['exports'])
    for item in import_list:
        assert os.path.exists(item), item

    omit = args.omit.split(',')
    mat_data = validate_merge(import_list,omit)

    chans = set(mat_data[0]['chans'])
    for d2 in mat_data[1:]:
        chans = chans.intersection(d2['chans'])
    chans = list(chans)

    for i,m in zip(info['exports'],mat_data):
        i['chans'] = chans

    port_map = port_site[args.rig]
    save_chanlist(dest,chans,port_map)
    save_probe(args.probe,chans,port_map,dest)

    fs = 1.0 / mat_data[0][chans[0]]['interval'] if args.fs is None else args.fs

    info['params'] = {
        'exp': info['name'],
        'fs': fs,
        'nchan': len(chans),
        'probe': args.probe,
    }
    save_parameters(info['params'],dest)
    save_info(dest,info)

    rec_list = []
    # print import_list
    for import_file in import_list:
        recordings = load_recordings(import_file,chans)
        for r in recordings:
            rec = realign(r,chans,args.fs)
            rec['data'] -= rec['data'].mean(axis=0)
            rec_list.append(rec)

    weights = calc_weights(rec_list) if args.weighted else None

    for indx, rec in enumerate(rec_list):
        if args.weighted:
            rec['data'] = do_war(rec['data'],weights)
        elif args.car:
            rec['data'] = do_car(rec['data'])

        save_recording(kwd,rec,indx)

if __name__ == '__main__':
    main()
<commit_msg>make sure we save recording start times<commit_after>#!/usr/bin/env python
import os
import argparse
import glob
from klusta_pipeline.maps import port_site
from klusta_pipeline.dataio import load_recordings, load_catlog
from klusta_pipeline.dataio import save_info, save_recording, save_chanlist, save_probe, save_parameters
from klusta_pipeline.utils import get_import_list, validate_merge, realign, calc_weights, do_war, do_car
from klusta_pipeline.probe import get_channel_groups, clean_dead_channels, build_geometries

# assume spike2 export to mat with the following parameters:
# - channel names are "Port_N" where N is the 1-indexed 1401 Port number

def get_args():

    parser = argparse.ArgumentParser(description='Compile Spike2 epoch .mat files into KlustaKwik KWD file.')
    parser.add_argument('rig',type=str,
                       help='defines the rig that the data was collected on')
    parser.add_argument('probe',type=str,
                       help='defines the probe that the data was collected on')
    parser.add_argument('path', default = './', nargs='?',
                       help='directory containing all of the mat files to compile')
    parser.add_argument('dest', default = './', nargs='?',
                       help='destination directory for kwd and other files')
    parser.add_argument('-s','--sampling_rate',dest='fs',type=float, default=None,
                       help='target sampling rate for waveform alignment')
    parser.add_argument('-c','--common_average_ref',dest='car',action='store_true',
                       help='turns on common average referencing')
    parser.add_argument('-w','--weighted',dest='weighted',action='store_true',
                       help='weights channels for common average referencing')
    parser.add_argument('-x','--drop',dest='omit',type=str, default='',
                       help='comma-separate list of channel labels to drop if they exist')

    return parser.parse_args()

def main():
    args = get_args()
    path = os.path.abspath(args.path)
    dest = os.path.abspath(args.dest)

    info = dict(args.__dict__)

    catlog = glob.glob(os.path.join(path,'*.catLog'))[0]
    info['name'] = os.path.split(catlog)[-1].split('.')[0]

    kwd = os.path.join(dest, info['name'] + '.raw.kwd')
    if os.path.exists(kwd):
        raise IOError('%s already exists, please delete or rename and run again' % kwd)

    info['exports'] = load_catlog(catlog)
    import_list = get_import_list(path,info['exports'])
    for item in import_list:
        assert os.path.exists(item), item

    omit = args.omit.split(',')
    mat_data = validate_merge(import_list,omit)

    chans = set(mat_data[0]['chans'])
    for d2 in mat_data[1:]:
        chans = chans.intersection(d2['chans'])
    chans = list(chans)

    for i,m in zip(info['exports'],mat_data):
        i['chans'] = chans

    port_map = port_site[args.rig]
    save_chanlist(dest,chans,port_map)
    save_probe(args.probe,chans,port_map,dest)

    fs = 1.0 / mat_data[0][chans[0]]['interval'] if args.fs is None else args.fs

    info['params'] = {
        'exp': info['name'],
        'fs': fs,
        'nchan': len(chans),
        'probe': args.probe,
    }
    save_parameters(info['params'],dest)

    rec_list = []
    # print import_list
    for import_file in import_list:
        recordings = load_recordings(import_file,chans)
        for r in recordings:
            rec = realign(r,chans,args.fs)
            rec['data'] -= rec['data'].mean(axis=0)
            rec_list.append(rec)

    info['recordings'] = [{k:v for k,v in rec.items() if k is not 'data'} for rec in rec_list]
    save_info(dest,info)

    weights = calc_weights(rec_list) if args.weighted else None

    for indx, rec in enumerate(rec_list):
        if args.weighted:
            rec['data'] = do_war(rec['data'],weights)
        elif args.car:
            rec['data'] = do_car(rec['data'])

        save_recording(kwd,rec,indx)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import xml.etree.ElementTree as XML

def slack_properties(parser, xml_parent, data):
    """yaml: slack

    Example::

      properties:
        - slack:
            notify-start: true
            notify-success: true
            notify-aborted: true
            notify-notbuilt: true
            notify-unstable: true
            notify-failure: true
            notify-backtonormal: true
    """
    if data is None:
        data = dict()

    notifier = XML.SubElement(
        xml_parent, 'jenkins.plugins.slack.SlackNotifier_-SlackJobProperty')
    notifier.set('plugin', 'slack@1.2')

    for opt, attr in (('notify-start', 'startNotification'),
                      ('notify-success', 'notifySuccess'),
                      ('notify-aborted', 'notifyAborted'),
                      ('notify-notbuilt', 'notifyNotBuilt'),
                      ('notify-unstable', 'notifyUnstable'),
                      ('notify-failure', 'notifyFailure'),
                      ('notify-backtonormal', 'notifyBackToNormal')):
        (XML.SubElement(notifier, attr)
         .text) = data.get(opt, True) and 'true' or 'false'

def slack_publisher(parser, xml_parent, data):
    """yaml: slack

    Example::

      publishers:
        - slack:
            team-domain: example.com
            auth-token: secret
            build-server-url: https://jenkins.example.com
            room: '#jenkins'
    """

    notifier = XML.SubElement(
        xml_parent, 'jenkins.plugins.slack.SlackNotifier')
    notifier.set('plugin', 'slack@1.2')

    for (opt, attr) in (('team-domain', 'teamDomain'),
                        ('auth-token', 'authToken'),
                        ('build-server-url', 'buildServerUrl'),
                        ('room', 'room')):
        XML.SubElement(notifier, attr).text = data.get(opt, '')
<commit_msg>Update for version 1.7 of the slack plugin<commit_after>import xml.etree.ElementTree as XML

def slack_properties(parser, xml_parent, data):
    """yaml: slack

    Example::

      properties:
        - slack:
            notify-start: true
            notify-success: true
            notify-aborted: true
            notify-notbuilt: true
            notify-unstable: true
            notify-failure: true
            notify-backtonormal: true
    """
    if data is None:
        data = dict()

    notifier = XML.SubElement(
        xml_parent, 'jenkins.plugins.slack.SlackNotifier_-SlackJobProperty')
    notifier.set('plugin', 'slack@1.7')

    for opt, attr in (('notify-start', 'startNotification'),
                      ('notify-success', 'notifySuccess'),
                      ('notify-aborted', 'notifyAborted'),
                      ('notify-notbuilt', 'notifyNotBuilt'),
                      ('notify-unstable', 'notifyUnstable'),
                      ('notify-failure', 'notifyFailure'),
                      ('notify-backtonormal', 'notifyBackToNormal'),
                      ('notify-repeatedfailure', 'notifyRepeatedFailure'),
                      ('include-test-summary', 'includeTestSummary'),
                      ('show-commit-list', 'showCommitList')):
        (XML.SubElement(notifier, attr)
         .text) = data.get(opt, True) and 'true' or 'false'

    XML.SubElement(notifier, 'teamDomain').text = data.get('team-domain')
    XML.SubElement(notifier, 'token').text = data.get('token')
    XML.SubElement(notifier, 'room').text = data.get('room')

def slack_publisher(parser, xml_parent, data):
    """yaml: slack

    Example::

      publishers:
        - slack:
            team-domain: example.com
            auth-token: secret
            build-server-url: https://jenkins.example.com
            room: '#jenkins'
    """
    notifier = XML.SubElement(
        xml_parent, 'jenkins.plugins.slack.SlackNotifier')
    notifier.set('plugin', 'slack@1.7')

    for (opt, attr) in (('team-domain', 'teamDomain'),
                        ('auth-token', 'authToken')):
        XML.SubElement(notifier, attr).text = data.get(opt, '')
<|endoftext|>


<commit_before>#!/usr/bin/env python

#  IATI Data Quality, tools for Data QA on IATI-formatted  publications
#  by Mark Brough, Martin Keegan, Ben Webb and Jennifer Smith
#
#  Copyright (C) 2013  Publish What You Fund
#
#  This programme is free software; you may redistribute and/or modify
#  it under the terms of the GNU Affero General Public License v3.0

""" This script is to quickly get started with this tool, by:
        1) creating DB
        2) populating the list of packages from the Registry (will download basic data about all packages)
        3) setting 3 to "active"
"""
#import warnings
#warnings.filterwarnings('error')

import iatidq

from iatidq import models, dqregistry
import iatidq.dqfunctions
import iatidq.dqimporttests
import iatidq.dqdownload
import iatidq.dqcodelists
import iatidq.dqruntests
import iatidq.dqindicators
import iatidq.dqorganisations
import iatidq.dqaggregationtypes
import iatidq.dqtests
import iatidq.dqprocessing
import iatidq.test_level as test_level
import optparse
import sys

which_packages = [
    (u'worldbank-tz', True),
    (u'unops-tz', True),
    (u'dfid-tz', True)
    ]

default_tests_filename="tests/tests.csv"
default_indicator_group_name="pwyf2013"

def refresh(options):
    pkg_names = None
    if options.package_name:
        pkg_names = [options.package_name]
    elif options.minimal_pkgs:
        pkg_names = [i[0] for i in which_packages]

    if pkg_names is not None:
        [ dqregistry.refresh_package_by_name(name) for name in pkg_names ]
    else:
        dqregistry.refresh_packages()

def activate_packages(options):
    dqregistry.activate_packages(which_packages, clear_revision_id=True)

def drop_all(options):
    iatidq.db.drop_all()

def init_db(options):
    iatidq.db.create_all()
    iatidq.dqimporttests.hardcodedTests()

def enroll_tests(options):
    filename = options.enroll_tests.decode()
    result = iatidq.dqimporttests.importTestsFromFile(
        filename=filename,
        level=options.level)
    if not result:
        print "Error importing"

def clear_revisionid(options):
    iatidq.dqfunctions.clear_revisions()

def import_codelists(options):
    iatidq.dqcodelists.importCodelists()

def download(options):
    if options.minimal_pkgs:
        for package_name, _ in which_packages:
            iatidq.dqdownload.run(package_name=package_name)
    else:
        iatidq.dqdownload.run()

def import_indicators(options):
    if options.filename:
        iatidq.dqindicators.importIndicatorsFromFile("pwyf2013",
                                                     options.filename)
    else:
        iatidq.dqindicators.importIndicators()

def import_organisations(options):
    if options.filename:
        iatidq.dqorganisations.importOrganisationPackagesFromFile(options.filename)
    else:
        print "Error: please provide a filename"

def create_aggregation_types(options):
    print "Adding an aggregation type for all data"
    iatidq.dqaggregationtypes.addAggregationType({'name':'All data',
                                                'description': '',
                                                'test_id': None,
                                                'test_result':'1'})
    print "Adding an aggregation type for current data"
    currentdata_test = iatidq.dqtests.test_by_test_name(
        "activity-date[@type='start-planned']/@iso-date or transaction-date/@iso-date (for each transaction) is less than 13 months ago?"
        )
    iatidq.dqaggregationtypes.addAggregationType({'name':'Current data',
                                                'description': '',
                                                'test_id':currentdata_test.id,
                                                'test_result':'1'})

def setup(options):
    print "Creating DB"
    iatidq.db.create_all()
    print "Adding hardcoded tests"
    iatidq.dqimporttests.hardcodedTests()
    print "Importing tests"
    iatidq.dqimporttests.importTestsFromFile(
        default_tests_filename,
        test_level.ACTIVITY)
    print "Importing indicators"
    iatidq.dqindicators.importIndicatorsFromFile(
        default_indicator_group_name,
        default_tests_filename)
    print "Importing indicator descriptions"
    iatidq.dqindicators.importIndicatorDescriptionsFromFile("pwyf2013",
                                                            "tests/indicators.csv")
    print "Importing codelists"
    iatidq.dqcodelists.importCodelists()
    print "Refreshing package data from Registry"
    dqregistry.refresh_packages()
    print "Adding organisations"
    iatidq.dqorganisations.importOrganisationPackagesFromFile("tests/organisations_with_identifiers.csv")
    create_aggregation_types(options)
    print "Setup complete."

def enqueue_test(options):
    assert options.package_name
    assert options.filename

    iatidq.dqruntests.enqueue_package_for_test(options.filename,
                                               options.package_name)
def aggregate_results(options):
    assert options.runtime_id
    assert options.package_id
    iatidq.dqprocessing.aggregate_results(options.runtime_id, options.package_id)

commands = {
    "drop_db": drop_all,
    "init_db": init_db,
    "enroll_tests": enroll_tests,
    "clear_revisionid": clear_revisionid,
    "import_codelists": import_codelists,
    "download": download,
    "import_indicators": import_indicators,
    "import_organisations": import_organisations,
    "setup": setup,
    "enqueue_test": enqueue_test,
    "refresh": refresh,
    "activate_packages": activate_packages,
    "create_aggregation_types": create_aggregation_types,
    "aggregate_results": aggregate_results
}

def main():
    p = optparse.OptionParser()
    p.add_option("--refresh", dest="refresh", action="store_true",
                 help="Refresh")
    p.add_option("--clear-revisionid", dest="clear_revisionid",
                 action="store_true",
                 help="Clear CKAN revision ids")
    p.add_option("--init-db", dest="init_db",
                  action="store_true",
                  help="Initialise DB")
    p.add_option("--drop-db", dest="drop_db",
                  action="store_true",
                  help="Delete DB")
    p.add_option("--enroll-tests", dest="enroll_tests",
                 help="Enroll a CSV file of tests")
    p.add_option("--level", dest="level",
                 type="int",
                 default=1,
                 help="Test level (e.g., 1 == Activity)")
    p.add_option("--minimal-pkgs", dest="minimal_pkgs",
                 action="store_true",
                 default=False,
                 help="Operate on a minimal set of packages")
    p.add_option("--activate-packages", dest="activate_packages",
                 action="store_true",
                 default=False,
                 help="Mark all packages as active")
    p.add_option("--download", dest="download",
                 action="store_true",
                 default=False,
                 help="Download packages")
    p.add_option("--import_codelists", dest="import_codelists",
                 action="store_true",
                 default=False,
                 help="Import codelists")
    p.add_option("--enqueue-test", dest="enqueue_test",
                 action="store_true",
                 default=False,
                 help="Set a package to be tested (with --package)")
    p.add_option("--package", dest="package_name",
                 help="Set name of package to be tested")
    p.add_option("--filename", dest="filename",
                 help="Set filename of data to test")
    p.add_option("--local-folder", dest="local_folder",
                 help="Set local folder where data to test is stored")
    p.add_option("--import-indicators", dest="import_indicators",
                 action="store_true",
                 default=False,
                 help="Import indicators. Will try to assign indicators to existing tests.")
    p.add_option("--import-organisations", dest="import_organisations",
                 action="store_true",
                 default=False,
                 help="Import organisations. Will try to create and assign organisations to existing packages.")
    p.add_option("--create-aggregation-types", dest="create_aggregation_types",
                 action="store_true",
                 default=False,
                 help="Create basic aggregation types.")
    p.add_option("--setup", dest="setup",
                 action="store_true",
                 default=False,
                 help="""Quick setup. Will init db, add tests, add codelists,
                      add indicators, refresh package data from Registry.""")
    p.add_option("--aggregate-results", dest="aggregate_results",
                 action="store_true",
                 default=False,
                 help="Trigger result aggregation")
    p.add_option("--runtime-id", dest="runtime_id",
                 type=int,
                 help="Runtime id (integer)")
    p.add_option("--package-id", dest="package_id",
                 type=int,
                 help="Package id (integer)")

    options, args = p.parse_args()

    for mode, handler in commands.iteritems():
        if getattr(options, mode, None):
            handler(options)
            return

    usage()

def usage():
    print "You need to specify which mode to run under"
    sys.exit(1)

if __name__ == '__main__':
    main()
<commit_msg>reorder option handlers<commit_after>#!/usr/bin/env python

#  IATI Data Quality, tools for Data QA on IATI-formatted  publications
#  by Mark Brough, Martin Keegan, Ben Webb and Jennifer Smith
#
#  Copyright (C) 2013  Publish What You Fund
#
#  This programme is free software; you may redistribute and/or modify
#  it under the terms of the GNU Affero General Public License v3.0

""" This script is to quickly get started with this tool, by:
        1) creating DB
        2) populating the list of packages from the Registry (will download basic data about all packages)
        3) setting 3 to "active"
"""
#import warnings
#warnings.filterwarnings('error')

import iatidq

from iatidq import models, dqregistry
import iatidq.dqfunctions
import iatidq.dqimporttests
import iatidq.dqdownload
import iatidq.dqcodelists
import iatidq.dqruntests
import iatidq.dqindicators
import iatidq.dqorganisations
import iatidq.dqaggregationtypes
import iatidq.dqtests
import iatidq.dqprocessing
import iatidq.test_level as test_level
import optparse
import sys

which_packages = [
    (u'worldbank-tz', True),
    (u'unops-tz', True),
    (u'dfid-tz', True)
    ]

default_tests_filename="tests/tests.csv"
default_indicator_group_name="pwyf2013"

def refresh(options):
    pkg_names = None
    if options.package_name:
        pkg_names = [options.package_name]
    elif options.minimal_pkgs:
        pkg_names = [i[0] for i in which_packages]

    if pkg_names is not None:
        [ dqregistry.refresh_package_by_name(name) for name in pkg_names ]
    else:
        dqregistry.refresh_packages()

def activate_packages(options):
    dqregistry.activate_packages(which_packages, clear_revision_id=True)

def drop_all(options):
    iatidq.db.drop_all()

def init_db(options):
    iatidq.db.create_all()
    iatidq.dqimporttests.hardcodedTests()

def enroll_tests(options):
    filename = options.enroll_tests.decode()
    result = iatidq.dqimporttests.importTestsFromFile(
        filename=filename,
        level=options.level)
    if not result:
        print "Error importing"

def clear_revisionid(options):
    iatidq.dqfunctions.clear_revisions()

def import_codelists(options):
    iatidq.dqcodelists.importCodelists()

def download(options):
    if options.minimal_pkgs:
        for package_name, _ in which_packages:
            iatidq.dqdownload.run(package_name=package_name)
    else:
        iatidq.dqdownload.run()

def import_indicators(options):
    if options.filename:
        iatidq.dqindicators.importIndicatorsFromFile("pwyf2013",
                                                     options.filename)
    else:
        iatidq.dqindicators.importIndicators()

def import_organisations(options):
    if options.filename:
        iatidq.dqorganisations.importOrganisationPackagesFromFile(options.filename)
    else:
        print "Error: please provide a filename"

def create_aggregation_types(options):
    print "Adding an aggregation type for all data"
    iatidq.dqaggregationtypes.addAggregationType({'name':'All data',
                                                'description': '',
                                                'test_id': None,
                                                'test_result':'1'})
    print "Adding an aggregation type for current data"
    currentdata_test = iatidq.dqtests.test_by_test_name(
        "activity-date[@type='start-planned']/@iso-date or transaction-date/@iso-date (for each transaction) is less than 13 months ago?"
        )
    iatidq.dqaggregationtypes.addAggregationType({'name':'Current data',
                                                'description': '',
                                                'test_id':currentdata_test.id,
                                                'test_result':'1'})

def setup(options):
    print "Creating DB"
    iatidq.db.create_all()
    print "Adding hardcoded tests"
    iatidq.dqimporttests.hardcodedTests()
    print "Importing tests"
    iatidq.dqimporttests.importTestsFromFile(
        default_tests_filename,
        test_level.ACTIVITY)
    print "Importing indicators"
    iatidq.dqindicators.importIndicatorsFromFile(
        default_indicator_group_name,
        default_tests_filename)
    print "Importing indicator descriptions"
    iatidq.dqindicators.importIndicatorDescriptionsFromFile("pwyf2013",
                                                            "tests/indicators.csv")
    print "Importing codelists"
    iatidq.dqcodelists.importCodelists()
    print "Refreshing package data from Registry"
    dqregistry.refresh_packages()
    print "Adding organisations"
    iatidq.dqorganisations.importOrganisationPackagesFromFile("tests/organisations_with_identifiers.csv")
    create_aggregation_types(options)
    print "Setup complete."

def enqueue_test(options):
    assert options.package_name
    assert options.filename

    iatidq.dqruntests.enqueue_package_for_test(options.filename,
                                               options.package_name)
def aggregate_results(options):
    assert options.runtime_id
    assert options.package_id
    iatidq.dqprocessing.aggregate_results(options.runtime_id, options.package_id)

commands = {
    "drop_db": drop_all,
    "init_db": init_db,
    "enroll_tests": enroll_tests,
    "clear_revisionid": clear_revisionid,
    "import_codelists": import_codelists,
    "download": download,
    "import_indicators": import_indicators,
    "import_organisations": import_organisations,
    "setup": setup,
    "enqueue_test": enqueue_test,
    "refresh": refresh,
    "activate_packages": activate_packages,
    "create_aggregation_types": create_aggregation_types,
    "aggregate_results": aggregate_results
}

def main():
    p = optparse.OptionParser()
    p.add_option("--refresh", dest="refresh", action="store_true",
                 help="Refresh")
    p.add_option("--clear-revisionid", dest="clear_revisionid",
                 action="store_true",
                 help="Clear CKAN revision ids")
    p.add_option("--init-db", dest="init_db",
                  action="store_true",
                  help="Initialise DB")
    p.add_option("--drop-db", dest="drop_db",
                  action="store_true",
                  help="Delete DB")
    p.add_option("--enroll-tests", dest="enroll_tests",
                 help="Enroll a CSV file of tests")
    p.add_option("--activate-packages", dest="activate_packages",
                 action="store_true",
                 default=False,
                 help="Mark all packages as active")
    p.add_option("--download", dest="download",
                 action="store_true",
                 default=False,
                 help="Download packages")
    p.add_option("--import_codelists", dest="import_codelists",
                 action="store_true",
                 default=False,
                 help="Import codelists")
    p.add_option("--enqueue-test", dest="enqueue_test",
                 action="store_true",
                 default=False,
                 help="Set a package to be tested (with --package)")
    p.add_option("--package", dest="package_name",
                 help="Set name of package to be tested")
    p.add_option("--filename", dest="filename",
                 help="Set filename of data to test")
    p.add_option("--local-folder", dest="local_folder",
                 help="Set local folder where data to test is stored")
    p.add_option("--import-indicators", dest="import_indicators",
                 action="store_true",
                 default=False,
                 help="Import indicators. Will try to assign indicators to existing tests.")
    p.add_option("--import-organisations", dest="import_organisations",
                 action="store_true",
                 default=False,
                 help="Import organisations. Will try to create and assign organisations to existing packages.")
    p.add_option("--create-aggregation-types", dest="create_aggregation_types",
                 action="store_true",
                 default=False,
                 help="Create basic aggregation types.")
    p.add_option("--setup", dest="setup",
                 action="store_true",
                 default=False,
                 help="""Quick setup. Will init db, add tests, add codelists,
                      add indicators, refresh package data from Registry.""")
    p.add_option("--aggregate-results", dest="aggregate_results",
                 action="store_true",
                 default=False,
                 help="Trigger result aggregation")
    p.add_option("--runtime-id", dest="runtime_id",
                 type=int,
                 help="Runtime id (integer)")
    p.add_option("--package-id", dest="package_id",
                 type=int,
                 help="Package id (integer)")
    p.add_option("--level", dest="level",
                 type="int",
                 default=1,
                 help="Test level (e.g., 1 == Activity)")
    p.add_option("--minimal-pkgs", dest="minimal_pkgs",
                 action="store_true",
                 default=False,
                 help="Operate on a minimal set of packages")

    options, args = p.parse_args()

    for mode, handler in commands.iteritems():
        if getattr(options, mode, None):
            handler(options)
            return

    usage()

def usage():
    print "You need to specify which mode to run under"
    sys.exit(1)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from __future__ import print_function
import sys
import httplib2
import os
import io
import apiclient
from ast import literal_eval

import oauth2client
from oauth2client import client
from oauth2client import tools

try:
    import argparse
    flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
    flags = None

# If modifying these scopes, delete your previously saved credentials
# at ~/.credentials/drive-python-quickstart.json
SCOPES = 'https://www.googleapis.com/auth/drive'
CLIENT_SECRET_FILE = 'client_secret_426774337873-0jm29chokpdr9pm0scnla58ck6g1s0r3.apps.googleusercontent.com.json'
APPLICATION_NAME = 'Wdrive'

class GoogleDriveInstance():

    def __init__(self, argv):
        """Get the credentials and set the parameters for sync"""
        """local_path:  (String) The directory that will be local Google Drive folder."""
        """opt_list: (List) The list of IDs that will not be synced with the cloud."""

        self.local_path = os.getcwd()+"/googledrive"
        if not os.path.exists(self.local_path):
            os.makedirs(self.local_path)
        self.credentials = self.get_credentials()
        self.opt_list = self.get_configs()

        try:
            http = self.credentials.authorize(httplib2.Http())
            service = apiclient.discovery.build('drive', 'v3', http=http)
        except httplib2.ServerNotFoundError:
            print ("ServerNotFoundError: Please try again later")

        results = service.files().get(fileId='0B8lhn7ceZT9iYVgzd3lYdE9zSWM').execute()
        self.download_folder(results,self.local_path + '/Documents')

    def get_path(self):
        home_dir = os.path.expanduser('~')
        profile_dir = os.path.join(home_dir, '.credentials')
        if not os.path.exists(profile_dir):
            os.makedirs(profile_dir)
        return profile_dir

    def get_credentials(self):
        """Gets valid user credentials from storage.

        If nothing has been stored, or if the stored credentials are invalid,
        the OAuth2 flow is completed to obtain the new credentials.

        Returns:
            Credentials, the obtained credential.
        """
        credential_dir = self.get_path()
        credential_path = os.path.join(credential_dir,
                                       'Wdrive.json')

        store = oauth2client.file.Storage(credential_path)
        credentials = store.get()
        if not credentials or credentials.invalid:
            flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
            flow.user_agent = APPLICATION_NAME
            if flags:
                credentials = tools.run_flow(flow, store, flags)
            else: # Needed only for compatibility with Python 2.6
                credentials = tools.run(flow, store)
            print('Storing credentials to ' + credential_path)
        return credentials

    def get_configs(self):
        config_dir = self.get_path()
        config_path = os.path.join(config_dir,
                                       'Wdrive.cfg')
        try:
            config = open(config_path, 'r')
            s = config.readline()
            OPTOUTLIST = s.split(",")
        except IOError:
            config = open(config_path, 'w')
            OPTOUTLIST = []
            config.write(",".join(OPTOUTLIST))
            print ("New profile established. Please use\n\t\twdrive ignore\nto opt out folders you don't want to be synced.")

        config.close()

        return OPTOUTLIST

    def opt_out(self,ID):
        config_dir = self.get_path()
        config_path = os.path.join(config_dir,
                                       'Wdrive.cfg')
        config = open(config_path, 'r+b')
        s = config.readline()
        OPTOUTLIST = s.split(",")
        OPTOUTLIST.append(ID)
        self.opt_list = OPTOUTLIST
        if ID not in self.opt_list:
            config.write(","+ID)
        config.close()
        return True

    def makedir_from_path(self, path):
        if not os.path.exists(path):
            os.makedirs(path)
            print ("{0}".format(path))

    def get_files_by_folder(self, folder):
        files = service.files().list(q = "'{0}' in parents".format(folder)).execute()
        files  = files.get(['files'], [])
        return files

    def get_items_by_id(self, fileId):
        return

    def is_folder(self, file):
        return file['mimeType'] == 'application/vnd.google-apps.folder'

    def is_google_doc(self, file):
        return file['mimeType'][0:27] == "application/vnd.google-apps"

    def download_folder(self, folder, base_path = "./"):
        self.makedir_from_path(base_path)
        http = self.credentials.authorize(httplib2.Http())
        service = apiclient.discovery.build('drive','v3',http=http)
        results = service.files().list(q="'{0}' in parents".format(folder['id'])).execute()
        while True:
            file_list = results.get('files',[])
            for file in file_list:
                if file['id'] in self.opt_list:
                    continue
                if self.is_folder(file):
                    self.download_folder(file, base_path + '/{0}'.format(file['name']))
                else:
                    file_path = os.path.join(base_path, file['name'])
                    if self.is_google_doc(file):
                        link_file = service.files().get(fileId = file['id'], fields = "webViewLink").execute()
                        with open("{0}/{1}.desktop".format(base_path,file['name']),'w') as shortcut:
                            shortcut.write("[Desktop Entry]\nEncoding=UTF-8\nName={0}\nURL={1}\nType=Link\nIcon=text-html\nName[en_US]=Google document link".format(file['name'],link_file['webViewLink']))

                    else:
                        request = service.files().get_media(fileId=file['id'])
                        fh = io.FileIO(file_path,'wb')
                        downloader = apiclient.http.MediaIoBaseDownload(fh, request)
                        done = False
                        while done == False:
                            status, done = downloader.next_chunk()
                            print ("File {0} downloaded {1} %.".format( file['name'] ,int(status.progress() * 100)))
                        fh.close()
            if 'nextPageToken' in results:
                 results = service.files().list(pageToken = results['nextPageToken']).execute()
            else:
                break
        return

if __name__ == '__main__':
    drive = GoogleDriveInstance(sys.argv)
<commit_msg>service to attri<commit_after>from __future__ import print_function
import sys
import httplib2
import os
import io
import apiclient
from ast import literal_eval

import oauth2client
from oauth2client import client
from oauth2client import tools

try:
    import argparse
    flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
    flags = None

# If modifying these scopes, delete your previously saved credentials
# at ~/.credentials/drive-python-quickstart.json
SCOPES = 'https://www.googleapis.com/auth/drive'
CLIENT_SECRET_FILE = 'client_secret_426774337873-0jm29chokpdr9pm0scnla58ck6g1s0r3.apps.googleusercontent.com.json'
APPLICATION_NAME = 'Wdrive'

class GoogleDriveInstance():

    def __init__(self, argv):
        """Get the credentials and set the parameters for sync"""
        """local_path:  (String) The directory that will be local Google Drive folder."""
        """opt_list: (List) The list of IDs that will not be synced with the cloud."""

        self.local_path = os.getcwd()+"/googledrive"
        if not os.path.exists(self.local_path):
            os.makedirs(self.local_path)
        self.credentials = self.get_credentials()
        self.opt_list = self.get_configs()

        try:
            http = self.credentials.authorize(httplib2.Http())
            self.service = apiclient.discovery.build('drive', 'v3', http=http)
        except httplib2.ServerNotFoundError:
            print ("ServerNotFoundError: Please try again later")

        results = self.service.files().get(fileId='0B8lhn7ceZT9iYVgzd3lYdE9zSWM').execute()
        self.download_folder(results,self.local_path + '/Documents')

    def get_path(self):
        home_dir = os.path.expanduser('~')
        profile_dir = os.path.join(home_dir, '.credentials')
        if not os.path.exists(profile_dir):
            os.makedirs(profile_dir)
        return profile_dir

    def get_credentials(self):
        """Gets valid user credentials from storage.

        If nothing has been stored, or if the stored credentials are invalid,
        the OAuth2 flow is completed to obtain the new credentials.

        Returns:
            Credentials, the obtained credential.
        """
        credential_dir = self.get_path()
        credential_path = os.path.join(credential_dir,
                                       'Wdrive.json')

        store = oauth2client.file.Storage(credential_path)
        credentials = store.get()
        if not credentials or credentials.invalid:
            flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
            flow.user_agent = APPLICATION_NAME
            if flags:
                credentials = tools.run_flow(flow, store, flags)
            else: # Needed only for compatibility with Python 2.6
                credentials = tools.run(flow, store)
            print('Storing credentials to ' + credential_path)
        return credentials

    def get_configs(self):
        config_dir = self.get_path()
        config_path = os.path.join(config_dir,
                                       'Wdrive.cfg')
        try:
            config = open(config_path, 'r')
            s = config.readline()
            OPTOUTLIST = s.split(",")
        except IOError:
            config = open(config_path, 'w')
            OPTOUTLIST = []
            config.write(",".join(OPTOUTLIST))
            print ("New profile established. Please use\n\t\twdrive ignore\nto opt out folders you don't want to be synced.")

        config.close()

        return OPTOUTLIST

    def opt_out(self,ID):
        config_dir = self.get_path()
        config_path = os.path.join(config_dir,
                                       'Wdrive.cfg')
        config = open(config_path, 'r+b')
        s = config.readline()
        OPTOUTLIST = s.split(",")
        OPTOUTLIST.append(ID)
        self.opt_list = OPTOUTLIST
        if ID not in self.opt_list:
            config.write(","+ID)
        config.close()
        return True

    def makedir_from_path(self, path):
        if not os.path.exists(path):
            os.makedirs(path)
            print ("{0}".format(path))

    def get_files_by_folder(self, folder):
        files = self.service.files().list(q = "'{0}' in parents".format(folder)).execute()
        files  = files.get(['files'], [])
        return files

    def get_items_by_id(self, fileId):
        return

    def is_folder(self, file):
        return file['mimeType'] == 'application/vnd.google-apps.folder'

    def is_google_doc(self, file):
        return file['mimeType'][0:27] == "application/vnd.google-apps"

    def download_folder(self, folder, base_path = "./"):
        self.makedir_from_path(base_path)
        results = self.service.files().list(q="'{0}' in parents".format(folder['id'])).execute()
        while True:
            file_list = results.get('files',[])
            for file in file_list:
                if file['id'] in self.opt_list:
                    continue
                if self.is_folder(file):
                    self.download_folder(file, base_path + '/{0}'.format(file['name']))
                else:
                    file_path = os.path.join(base_path, file['name'])
                    if self.is_google_doc(file):
                        link_file = self.service.files().get(fileId = file['id'], fields = "webViewLink").execute()
                        with open("{0}/{1}.desktop".format(base_path,file['name']),'w') as shortcut:
                            shortcut.write("[Desktop Entry]\nEncoding=UTF-8\nName={0}\nURL={1}\nType=Link\nIcon=text-html\nName[en_US]=Google document link".format(file['name'],link_file['webViewLink']))

                    else:
                        request = self.service.files().get_media(fileId=file['id'])
                        fh = io.FileIO(file_path,'wb')
                        downloader = apiclient.http.MediaIoBaseDownload(fh, request)
                        done = False
                        while done == False:
                            status, done = downloader.next_chunk()
                            print ("File {0} downloaded {1} %.".format( file['name'] ,int(status.progress() * 100)))
                        fh.close()
            if 'nextPageToken' in results:
                 results = self.service.files().list(pageToken = results['nextPageToken']).execute()
            else:
                break
        return

if __name__ == '__main__':
    drive = GoogleDriveInstance(sys.argv)
<|endoftext|>


<commit_before>"""
quilt.Blog

Provide blog capabilities for static site
{: .lead}

1. set `source` to `posts/` along with unique or default quilt and patches
2. as [`QuiltingRoom`](#QuiltingRoom) loops over pages append page data from [`Quilter`](#Quilter) to `Blog`
3. create list of posts in reverse chronological order
4. create group pages for tags and/or categories:
    1. create page that list all groups (e.g. list all tags)
    2. create page for each group with list of associated posts in reverse chronological order
5. generate `atom` feed and/or `rss` feed

project    : quilt
version    : 0.1.1
status     : development
modifydate : 2015-05-13 07:09:00 -0700
createdate : 2015-04-28 06:02:00 -0700
website    : https://github.com/tmthydvnprt/quilt
author     : tmthydvnprt
email      : tmthydvnprt@users.noreply.github.com
maintainer : tmthydvnprt
license    : MIT
copyright  : Copyright 2015, quilt
credits    :

"""

import os
from collections import defaultdict

from quilt.Quilter import Quilter
from quilt.Util import write_file, group_links, reverse_chronological_order, check_local_quilt
from quilt.Constants import ATOMENTRY, ATOMXML
from quilt.Constants import RSSITEM, RSSXML
from quilt.Constants import POST, POSTLIST
from quilt.Constants import GROUP, GROUPLIST, GROUPVARS

#@profile
def create_post_list(name, posts=None, offset=''):
    """create list html from posts"""

    postlist = []
    for post in posts:
        post_url = offset + os.path.basename(post['url'])
        postlist.append(POST % (
            post_url,
            post['title'],
            post['author'],
            post['date'],
            post['summary'].encode('utf8'),
            post_url,
            os.path.join(os.path.dirname(post["url"]), 'tags', 'index.html'),
            group_links(post, "tags"),
            os.path.join(os.path.dirname(post["url"]), 'categories', 'index.html'),
            group_links(post, "categories")
        ))
    return POSTLIST % (name, ''.join(postlist))

class Blog(object):
    """blog object"""

    #@profile
    def __init__(self, name='', source='', output='', quilt_pattern=None, patches=None, config=None):
        """create a blog"""

        self.name = name
        self.source = source
        self.output = output
        self.config = config
        self.quilt_pattern = quilt_pattern
        self.patches = patches
        self.posts = []

    #@profile
    def append(self, post=None):
        """append a post to the blog"""

        self.posts.append(post)
        return self

    #@profile
    def generate_blog_home(self):
        """generate blog index page"""

        # generate reverse chronological order of first x posts
        page_html = GROUPVARS % (
            self.name,
            self.name,
            "quilt"
        )
        page_html = ''.join((page_html, create_post_list(self.name, reverse_chronological_order(self.posts))))

        # stitch blog home page
        page = os.path.join(self.config["posts"], "index.html")

        # check for directory quilt and directory patches?
        quilt, patches = check_local_quilt(page, self.quilt_pattern, self.patches, self.config)
        # stitch the page together
        qultr = Quilter(page, quilt, patches, page_html, self.config)
        qultr.stitch()
        qultr.clean_html()
        qultr.remove_empty()
        qultr.write()
        del qultr

        return self

    #@profile
    def group_by(self, name=''):
        """create group dictionary"""

        groups = defaultdict(list)
        for post in self.posts:
            post_groups = post[name] if isinstance(post[name], list) else [post[name]]
            for group in post_groups:
                groups[group].append(post)
        return groups

    #@profile
    def generate_group_pages(self, name=''):
        """generate tag or category page"""

        # make group folder
        os.makedirs(os.path.join(self.config["posts"], name).replace(self.config["pages"], self.output))

        # create group -> page look up table
        groups = self.group_by(name)

        # create page for each group
        for group, posts in groups.items():
            page_html = create_post_list('%s Posts' % (group.title()), reverse_chronological_order(posts), '../')

            # stitch blog home page
            page = os.path.join(self.config["posts"], name, "%s.html" % group)

            # check for directory quilt and directory patches?
            quilt, patches = check_local_quilt(page, self.quilt_pattern, self.patches, self.config)

            # stitch the page together
            qultr = Quilter(page, quilt, patches, page_html, self.config)
            qultr.stitch()
            qultr.clean_html()
            qultr.remove_empty()
            qultr.write()
            del qultr

        # create index of groups
        grouplist = ''.join([GROUP % (name, "%s.html" % (group), group, len(posts)) for group, posts in groups.items()])
        page_html = GROUPLIST % (name.title(), name, grouplist)

        # stitch blog home page
        page = os.path.join(self.config["posts"], name, "index.html")

        # stitch the page together
        qultr = Quilter(page, self.quilt_pattern, self.patches, page_html, self.config)
        qultr.stitch()
        qultr.clean_html()
        qultr.remove_empty()
        qultr.write()
        del qultr

        return self

    #@profile
    def generate_atom(self):
        """generate atom xml"""

        # generate an atom entry for each blog page
        entries = [ATOMENTRY % (
            post['title'],
            post['url'],
            post['url'],
            self.config['atomid'],
            post['date'],
            post['summary'].encode('utf8'),
            post['content'].encode('utf8'),
            post['author'],
            post['email']
        ) for post in self.posts]

        # generate the atom xml file
        atom_xml = ATOMXML % (
            self.config["name"],
            self.config["posts"],
            self.config["domain"],
            self.config["domain"],
            self.config["atomid"],
            self.config["now"]["iso"],
            ''.join(entries)
        )

        feed_path = os.path.join(self.output, 'feed')
        # make feed/ if need be
        if not os.path.exists(feed_path):
            os.makedirs(feed_path)

        # output new atom files
        write_file(os.path.join(feed_path, 'atom.xml'), atom_xml)
        write_file(os.path.join(feed_path, 'feed.atom'), atom_xml)

        return self

    #@profile
    def generate_rss(self):
        """geneate rss xml"""

        # generate an rss item for each blog page
        items = [RSSITEM % (
            post['title'],
            post['url'],
            self.config['atomid'],
            post['date'],
            post['summary'].encode('utf8')
        ) for post in self.posts]

        # generate the rss xml file
        rss_xml = RSSXML % (
            self.config['name'],
            self.config['posts'],
            self.config['url'],
            self.config['rssid'],
            self.config["now"]["iso"],
            self.config["now"]["iso"],
            self.config['author'],
            ''.join(items)
        )

        feed_path = os.path.join(self.output, 'feed')
        # make feed/ if need be
        if not os.path.exists(feed_path):
            os.makedirs(feed_path)

        # output new atom files
        write_file(os.path.join(feed_path, 'rss.xml'), rss_xml)
        write_file(os.path.join(feed_path, 'feed.rss'), rss_xml)

        return self
<commit_msg>fix keyerror bug<commit_after>"""
quilt.Blog

Provide blog capabilities for static site
{: .lead}

1. set `source` to `posts/` along with unique or default quilt and patches
2. as [`QuiltingRoom`](#QuiltingRoom) loops over pages append page data from [`Quilter`](#Quilter) to `Blog`
3. create list of posts in reverse chronological order
4. create group pages for tags and/or categories:
    1. create page that list all groups (e.g. list all tags)
    2. create page for each group with list of associated posts in reverse chronological order
5. generate `atom` feed and/or `rss` feed

project    : quilt
version    : 0.1.1
status     : development
modifydate : 2015-05-13 07:09:00 -0700
createdate : 2015-04-28 06:02:00 -0700
website    : https://github.com/tmthydvnprt/quilt
author     : tmthydvnprt
email      : tmthydvnprt@users.noreply.github.com
maintainer : tmthydvnprt
license    : MIT
copyright  : Copyright 2015, quilt
credits    :

"""

import os
from collections import defaultdict

from quilt.Quilter import Quilter
from quilt.Util import write_file, group_links, reverse_chronological_order, check_local_quilt
from quilt.Constants import ATOMENTRY, ATOMXML
from quilt.Constants import RSSITEM, RSSXML
from quilt.Constants import POST, POSTLIST
from quilt.Constants import GROUP, GROUPLIST, GROUPVARS

#@profile
def create_post_list(name, posts=None, offset=''):
    """create list html from posts"""

    postlist = []
    for post in posts:
        post_url = offset + os.path.basename(post['url'])
        postlist.append(POST % (
            post_url,
            post['title'],
            post['author'],
            post['date'],
            post['summary'].encode('utf8'),
            post_url,
            os.path.join(os.path.dirname(post["url"]), 'tags', 'index.html'),
            group_links(post, "tags"),
            os.path.join(os.path.dirname(post["url"]), 'categories', 'index.html'),
            group_links(post, "categories")
        ))
    return POSTLIST % (name, ''.join(postlist))

class Blog(object):
    """blog object"""

    #@profile
    def __init__(self, name='', source='', output='', quilt_pattern=None, patches=None, config=None):
        """create a blog"""

        self.name = name
        self.source = source
        self.output = output
        self.config = config
        self.quilt_pattern = quilt_pattern
        self.patches = patches
        self.posts = []

    #@profile
    def append(self, post=None):
        """append a post to the blog"""

        self.posts.append(post)
        return self

    #@profile
    def generate_blog_home(self):
        """generate blog index page"""

        # generate reverse chronological order of first x posts
        page_html = GROUPVARS % (
            self.name,
            self.name,
            "quilt"
        )
        page_html = ''.join((page_html, create_post_list(self.name, reverse_chronological_order(self.posts))))

        # stitch blog home page
        page = os.path.join(self.config["posts"], "index.html")

        # check for directory quilt and directory patches?
        quilt, patches = check_local_quilt(page, self.quilt_pattern, self.patches, self.config)
        # stitch the page together
        qultr = Quilter(page, quilt, patches, page_html, self.config)
        qultr.stitch()
        qultr.clean_html()
        qultr.remove_empty()
        qultr.write()
        del qultr

        return self

    #@profile
    def group_by(self, name=''):
        """create group dictionary"""

        groups = defaultdict(list)
        for post in self.posts:
            post_groups = post[name] if isinstance(post[name], list) else [post[name]]
            for group in post_groups:
                groups[group].append(post)
        return groups

    #@profile
    def generate_group_pages(self, name=''):
        """generate tag or category page"""

        # make group folder
        os.makedirs(os.path.join(self.config["posts"], name).replace(self.config["pages"], self.output))

        # create group -> page look up table
        groups = self.group_by(name)

        # create page for each group
        for group, posts in groups.items():
            page_html = create_post_list('%s Posts' % (group.title()), reverse_chronological_order(posts), '../')

            # stitch blog home page
            page = os.path.join(self.config["posts"], name, "%s.html" % group)

            # check for directory quilt and directory patches?
            quilt, patches = check_local_quilt(page, self.quilt_pattern, self.patches, self.config)

            # stitch the page together
            qultr = Quilter(page, quilt, patches, page_html, self.config)
            qultr.stitch()
            qultr.clean_html()
            qultr.remove_empty()
            qultr.write()
            del qultr

        # create index of groups
        grouplist = ''.join([GROUP % (name, "%s.html" % (group), group, len(posts)) for group, posts in groups.items()])
        page_html = GROUPLIST % (name.title(), name, grouplist)

        # stitch blog home page
        page = os.path.join(self.config["posts"], name, "index.html")

        # stitch the page together
        qultr = Quilter(page, self.quilt_pattern, self.patches, page_html, self.config)
        qultr.stitch()
        qultr.clean_html()
        qultr.remove_empty()
        qultr.write()
        del qultr

        return self

    #@profile
    def generate_atom(self):
        """generate atom xml"""

        # generate an atom entry for each blog page
        entries = [ATOMENTRY % (
            post['title'],
            post['url'],
            post['url'],
            self.config['atomid'],
            post['date'],
            post['summary'].encode('utf8'),
            post['content'].encode('utf8'),
            post['author'],
            post['email']
        ) for post in self.posts]

        # generate the atom xml file
        atom_xml = ATOMXML % (
            self.config["name"],
            self.config["posts"],
            self.config["domain"],
            self.config["domain"],
            self.config["atomid"],
            self.config["now"]["iso"],
            ''.join(entries)
        )

        feed_path = os.path.join(self.output, 'feed')
        # make feed/ if need be
        if not os.path.exists(feed_path):
            os.makedirs(feed_path)

        # output new atom files
        write_file(os.path.join(feed_path, 'atom.xml'), atom_xml)
        write_file(os.path.join(feed_path, 'feed.atom'), atom_xml)

        return self

    #@profile
    def generate_rss(self):
        """geneate rss xml"""

        # generate an rss item for each blog page
        items = [RSSITEM % (
            post['title'],
            post['url'],
            self.config['atomid'],
            post['date'],
            post['summary'].encode('utf8')
        ) for post in self.posts]

        # generate the rss xml file
        rss_xml = RSSXML % (
            self.config['name'],
            self.config['posts'],
            self.config['domain'],
            self.config['rssid'],
            self.config["now"]["iso"],
            self.config["now"]["iso"],
            self.config['author'],
            ''.join(items)
        )

        feed_path = os.path.join(self.output, 'feed')
        # make feed/ if need be
        if not os.path.exists(feed_path):
            os.makedirs(feed_path)

        # output new atom files
        write_file(os.path.join(feed_path, 'rss.xml'), rss_xml)
        write_file(os.path.join(feed_path, 'feed.rss'), rss_xml)

        return self
<|endoftext|>


<commit_before>import hashlib
import os
from base64 import b64encode

import tornado
import tornado.web
import torngithub
from tornado.httputil import url_concat
from torngithub import json_encode

from base_handler import *
from search import *
from user_accounts import *

# BEGIN: read environment variables

if "github_client_id" not in os.environ:
    print("set your github_client_id env variable, and register your app at https://github.com/settings/developers")
    os.environ["github_client_id"] = "gh_client_id"
if "github_client_secret" not in os.environ:
    print("set your github_client_secret env variable, and register your app at https://github.com/settings/developers")
    os.environ["github_client_secret"] = "github_client_secret"

settings = {
    "github_client_id": os.environ["github_client_id"],
    "github_client_secret": os.environ["github_client_secret"]
}

# BEGIN: GitHub repo handlers

class GithubLoginHandler(tornado.web.RequestHandler, torngithub.GithubMixin):
    """ Handle GitHub OAuth. """

    @tornado.gen.coroutine
    def get(self):
        # Heroku does not accurately give self.request.protocol
        if self.request.host == "localhost":
          protocol = "http"
        else:
          protocol = "https"

        # next is the redirect_uri
        redirect_uri = url_concat(protocol
                                  + "://" + self.request.host
                                  + "/oauth",
                                  {"next":
                                   self.get_argument('next', '/')})

        # if we have a code, we have been authorized so we can log in
        if self.get_argument("code", False):
            user = yield self.get_authenticated_user(
                redirect_uri=redirect_uri,
                client_id=settings["github_client_id"],
                client_secret=settings["github_client_secret"],
                code=self.get_argument("code")
            )
            if user:
                self.set_secure_cookie("user", json_encode(user))
                # idempotent operation to make sure GitHub user is in our
                # database
                register_github_user(json_encode(user))
                api_key = str(user["id"])
                hasher = hashlib.sha1()
                hasher.update(api_key.encode('utf-8'))
                api_key = hasher.hexdigest()
                self.set_secure_cookie("api_key", api_key)
            else:
                self.clear_cookie("user")
            self.redirect(self.get_argument("next", "/"))
            return

        # otherwise we need to request an authorization code
        yield self.authorize_redirect(
            redirect_uri=redirect_uri,
            client_id=settings["github_client_id"],
            extra_params={"scope": "repo"})

class GithubLogoutHandler(BaseHandler):
    """ Clear login cookies. """

    def get(self):
        self.clear_cookie("user")
        self.redirect(self.get_argument("next", "/"))

def parse_link(link):
    linkmap = {}
    for s in link.split(","):
        s = s.strip()
        linkmap[s[-5:-1]] = s.split(";")[0].rstrip()[1:-1]
    return linkmap

def get_last_page_num(link):
    if not link:
        return 0
    linkmap = parse_link(link)
    matches = re.search(r"[?&]page=(\d+)", linkmap["last"])
    return int(matches.group(1))

@tornado.gen.coroutine
def get_my_repos(http_client, access_token):
    data = []
    first_page = yield torngithub.github_request(
        http_client, '/user/repos?page=1&per_page=100',
        access_token=access_token)
    # log.info(first_page.headers.get('Link', ''))
    data.extend(first_page.body)
    max_pages = get_last_page_num(first_page.headers.get('Link', ''))

    ress = yield [torngithub.github_request(
        http_client, '/user/repos?per_page=100&page=' + str(i),
        access_token=access_token) for i in range(2, max_pages + 1)]

    for res in ress:
        data.extend(res.body)

    raise tornado.gen.Return(data)

class ReposHandler(BaseHandler, torngithub.GithubMixin):
    """ TODO: We should update the database if there is a discrepency"""
    # @tornado.web.authenticated
    @tornado.web.asynchronous
    @tornado.gen.coroutine
    def get(self):
        try:
            return_list = self.get_argument("list")
        except tornado.web.MissingArgumentError:  # AK: This is hacky.
            return_list = False
        try:
            pmid = self.get_argument("pmid")
        except tornado.web.MissingArgumentError:  # AK: This is again hacky.
            pmid = False

        gh_user = self.__get_current_github_object__()
        if self.get_current_github_access_token():
            # get all repos for an authenticated user
            data = yield get_my_repos(self.get_auth_http_client(),
                                      self.get_current_github_access_token())
            repos = [d for d in data if d["name"].startswith(
                "brainspell-collection")]

            # TODO: Ideally this information would be store in the database
            # this is pretty hacky. I'm checking gathering info for each pmid in
            # each collection. If the user speficified a pmid in the REST call, then
            # check if the pmid exists in the collection

            for repo in repos:
                # this is the name w/out the brainspell-collection in it

                repo["pretty_name"] = repo["name"].replace(
                    "brainspell-collection-", "")

                # get file list content for each repo
                try:
                    content_data = yield torngithub.github_request(
                        self.get_auth_http_client(),
                        '/repos/{owner}/{repo}/contents/{path}'.format(owner=self.get_current_github_username(),
                                                                       repo=repo["name"],
                                                                       # path="{}.json".format(pmid)
                                                                       path=""
                                                                       ),
                        access_token=self.get_current_github_access_token(),
                        method="GET")

                    print(repo["contributors_url"])
                    contrib = yield torngithub.github_request(self.get_auth_http_client(),
                                                              repo["contributors_url"].replace("https://api.github.com", ""),
                                                              access_token=self.get_current_github_access_token(),
                                                              method="GET")
                    repo["contributors"] = contrib["body"]

                    # print(content_data)

                    content = content_data["body"]

                    # extract pmids from content body
                    pmids = [c["name"].replace(".json", "") for c in content]

                    # If we are looking for a certaim pmid, add a tag for if it
                    # exists in the collection
                    if pmid:
                        if pmid in pmids:
                            repo["in_collection"] = True
                        else:
                            repo["in_collection"] = False

                    # get article information from each pmid from the database
                    all_contents = [next(get_article(pmid)) for pmid in pmids]
                except BaseException:  # TODO This is hacky, Empty Repos break the code!
                    all_contents = []
                    repo["contributors"] = {}

                # Convert to a dict the info we want (so it can be JSON
                # serialized later)
                def article_content_dict(cont):
                    return dict(
                        title=cont.title,
                        reference=cont.reference,
                        pmid=cont.pmid)
                repo["contents"] = [
                    article_content_dict(cont) for cont in all_contents]

            # if we want to return a JSON instead or a page render
            if return_list:
                self.write(json_encode(repos))
            else:
                self.render_with_user_info("static/html/github-account.html", {
                    "info": repos
                })

        # if you're not authorized, redirect to oauth
        else:
            self.redirect("/oauth?next=/repos")

class NewRepoHandler(BaseHandler, torngithub.GithubMixin):
    """ Create a new GitHub repo for a collection. """

    # TODO Again update the database
    @tornado.web.asynchronous
    @tornado.gen.coroutine
    def post(self):
        starttime = time.time()
        name = self.get_argument("name")
        desc = self.get_argument("description")
        if not name:
            self.write("")
        else:
            # Update database with new Collection

            new_repo(name, self.get_current_github_username())
            body = {
                "name": "brainspell-collection-{}".format(name),
                "description": desc,
                "homepage": "https://brainspell-neo.herokuapp.com",
                "private": False,
                "has_issues": True,
                "has_projects": True,
                "has_wiki": True
            }
            ress = yield [torngithub.github_request(
                self.get_auth_http_client(), '/user/repos',
                access_token=self.get_current_github_access_token(),
                method="POST",
                body=body)]
            data = []
            for res in ress:
                data.extend(res.body)

            endtime = time.time()
            # print(data)
            return self.redirect("/repos")

class NewFileHandler(BaseHandler, torngithub.GithubMixin):
    # Adds a json file to github repository
    # Add to database as well
    @tornado.web.asynchronous
    @tornado.gen.coroutine
    def post(self):
        starttime = time.time()
        collection = self.get_argument("collection")
        pmid = self.get_argument("pmid")
        print("Collection is {0} and PMID is {1}".format(collection, pmid))
        print(
            "Type of collection is {0} and type of PMID is {1}".format(
                type(collection),
                type(pmid)))
        article = list(get_article(pmid))[0]
        entry = {"pmid": pmid,
                 "title": article.title,
                 "reference": article.reference,
                 "doi": article.doi,
                 "notes": "Here are my notes on this article"}
        content = b64encode(json_encode(entry).encode("utf-8")).decode('utf-8')

        name = collection

        coll_indicator = new_repo(name, self.get_current_github_username())

        if coll_indicator:
            print("Creating collection: " + name)
            body = {
                "name": name,
                "description": "",
                "homepage": "https://brainspell-neo.herokuapp.com",
                "private": False,
                "has_issues": True,
                "has_projects": True,
                "has_wiki": True
            }
            ress = yield [torngithub.github_request(
                self.get_auth_http_client(), '/user/repos',
                access_token=self.get_current_github_access_token(),
                method="POST",
                body=body)]
            data = []
            for res in ress:
                data.extend(res.body)
            print("Created collection")

        add_to_repo(collection, pmid, self.get_current_github_username())
        body = {
            "message": "adding {} to collection".format(pmid),
            "content": content
        }
        ress = yield [torngithub.github_request(
            self.get_auth_http_client(),
            '/repos/{owner}/{repo}/contents/{path}'.format(owner=self.get_current_github_username(),
                                                           repo=collection,
                                                           path="{}.json".format(pmid)),
            access_token=self.get_current_github_access_token(),
            method="PUT",
            body=body)]
        data = []
        for res in ress:
            data.extend(res.body)

        endtime = time.time()

class DeleteFileHandler(BaseHandler, torngithub.GithubMixin):
    # Deletes a JSON file from the User Repo
    @tornado.web.asynchronous
    @tornado.gen.coroutine
    def post(self):
        starttime = time.time()
        collection = self.get_argument("collection")
        pmid = self.get_argument("pmid")
        entry = {"pmid": pmid,
                 "notes": "Here are my notes on this article"}
        content = b64encode(json_encode(entry).encode("utf=8"))
        gh_user = self.__get_current_github_object__()

        remove_from_repo(collection, pmid, self.get_current_github_username())

        body = {
            "message": "deleting {} to collection".format(pmid),
        }

        sha_data = yield [torngithub.github_request(
            self.get_auth_http_client(),
            '/repos/{owner}/{repo}/contents/{path}'.format(owner=self.get_current_github_username(),
                                                           repo=collection,
                                                           path="{}.json".format(pmid)),
            access_token=self.get_current_github_access_token(),
            method="GET")]

        sha = [s["body"]["sha"] for s in sha_data][0]

        ress = yield [torngithub.github_request(
            self.get_auth_http_client(),
            '/repos/{owner}/{repo}/contents/{path}'.format(owner=self.get_current_github_username(),
                                                           repo=collection,
                                                           path="{}.json".format(pmid)),
            access_token=self.get_current_github_access_token(),
            method="DELETE",
            body={"sha": sha, "message": "removing {} from collection".format(pmid)})]

"""
 class BulkNewFileHandler(BaseHandler, torngithub.GithubMixin):
     @tornado.web.asynchronous
     @tornado.gen.coroutine
     def post(self):
         startime = time.time()
         collection = self.get_argument("collection")
         pmids = self.get_argument("pmids")
         pmids = eval(pmids)
         user_info = self.__get_current_github_object__()["login"]
         if collection in next(User.select().where(User.username == user_info).execute()).collections: #If collection exists
             collection = "brainspell-collection-" + collection
             for pmid in pmids:
                 pmid = eval(pmid)
                 article = list(get_article(pmid))[0]
                 entry = {"pmid": pmid,
                         "title": article.title,
                         "reference": article.reference,
                         "doi": article.doi,
                          "notes": "Here are my notes on this article"}
                 content = b64encode(json_encode(entry).encode("utf-8")).decode('utf-8')
                 gh_user = self.__get_current_github_object__()
                 add_to_repo(collection,pmid,self.get_current_github_username())
                 body = {
                     "message": "adding {} to collection".format(pmid),
                     "content": content
                 }
                 ress = yield [
                     torngithub.github_request(
                         self.get_auth_http_client(),
                         '/repos/{owner}/{repo}/contents/{path}'.format(owner=self.get_current_github_username(),
                                                                        repo=collection,
                                                                        path="{}.json".format(pmid)),
                         access_token=self.get_current_github_access_token(),
                         method="PUT",
                         body=body
                     )
                 ]
                 data = []
                 for res in ress:
                     data.extend(res.body)
                 endtime = time.time()
         else:
             print("Your collection doesn't exist")
             return False #TODO: Tell user the collection doesn't exist
"""
<commit_msg>formatting after heroku fix<commit_after>import hashlib
import os
from base64 import b64encode

import tornado
import tornado.web
import torngithub
from tornado.httputil import url_concat
from torngithub import json_encode

from base_handler import *
from search import *
from user_accounts import *

# BEGIN: read environment variables

if "github_client_id" not in os.environ:
    print("set your github_client_id env variable, and register your app at https://github.com/settings/developers")
    os.environ["github_client_id"] = "gh_client_id"
if "github_client_secret" not in os.environ:
    print("set your github_client_secret env variable, and register your app at https://github.com/settings/developers")
    os.environ["github_client_secret"] = "github_client_secret"

settings = {
    "github_client_id": os.environ["github_client_id"],
    "github_client_secret": os.environ["github_client_secret"]
}

# BEGIN: GitHub repo handlers

class GithubLoginHandler(tornado.web.RequestHandler, torngithub.GithubMixin):
    """ Handle GitHub OAuth. """

    @tornado.gen.coroutine
    def get(self):
        # Heroku does not accurately give self.request.protocol
        if self.request.host == "localhost":
            protocol = "http"
        else:
            protocol = "https"

        # next is the redirect_uri
        redirect_uri = url_concat(protocol
                                  + "://" + self.request.host
                                  + "/oauth",
                                  {"next":
                                   self.get_argument('next', '/')})

        # if we have a code, we have been authorized so we can log in
        if self.get_argument("code", False):
            user = yield self.get_authenticated_user(
                redirect_uri=redirect_uri,
                client_id=settings["github_client_id"],
                client_secret=settings["github_client_secret"],
                code=self.get_argument("code")
            )
            if user:
                self.set_secure_cookie("user", json_encode(user))
                # idempotent operation to make sure GitHub user is in our
                # database
                register_github_user(json_encode(user))
                api_key = str(user["id"])
                hasher = hashlib.sha1()
                hasher.update(api_key.encode('utf-8'))
                api_key = hasher.hexdigest()
                self.set_secure_cookie("api_key", api_key)
            else:
                self.clear_cookie("user")
            self.redirect(self.get_argument("next", "/"))
            return

        # otherwise we need to request an authorization code
        yield self.authorize_redirect(
            redirect_uri=redirect_uri,
            client_id=settings["github_client_id"],
            extra_params={"scope": "repo"})

class GithubLogoutHandler(BaseHandler):
    """ Clear login cookies. """

    def get(self):
        self.clear_cookie("user")
        self.redirect(self.get_argument("next", "/"))

def parse_link(link):
    linkmap = {}
    for s in link.split(","):
        s = s.strip()
        linkmap[s[-5:-1]] = s.split(";")[0].rstrip()[1:-1]
    return linkmap

def get_last_page_num(link):
    if not link:
        return 0
    linkmap = parse_link(link)
    matches = re.search(r"[?&]page=(\d+)", linkmap["last"])
    return int(matches.group(1))

@tornado.gen.coroutine
def get_my_repos(http_client, access_token):
    data = []
    first_page = yield torngithub.github_request(
        http_client, '/user/repos?page=1&per_page=100',
        access_token=access_token)
    # log.info(first_page.headers.get('Link', ''))
    data.extend(first_page.body)
    max_pages = get_last_page_num(first_page.headers.get('Link', ''))

    ress = yield [torngithub.github_request(
        http_client, '/user/repos?per_page=100&page=' + str(i),
        access_token=access_token) for i in range(2, max_pages + 1)]

    for res in ress:
        data.extend(res.body)

    raise tornado.gen.Return(data)

class ReposHandler(BaseHandler, torngithub.GithubMixin):
    """ TODO: We should update the database if there is a discrepency"""
    # @tornado.web.authenticated
    @tornado.web.asynchronous
    @tornado.gen.coroutine
    def get(self):
        try:
            return_list = self.get_argument("list")
        except tornado.web.MissingArgumentError:  # AK: This is hacky.
            return_list = False
        try:
            pmid = self.get_argument("pmid")
        except tornado.web.MissingArgumentError:  # AK: This is again hacky.
            pmid = False

        gh_user = self.__get_current_github_object__()
        if self.get_current_github_access_token():
            # get all repos for an authenticated user
            data = yield get_my_repos(self.get_auth_http_client(),
                                      self.get_current_github_access_token())
            repos = [d for d in data if d["name"].startswith(
                "brainspell-collection")]

            # TODO: Ideally this information would be store in the database
            # this is pretty hacky. I'm checking gathering info for each pmid in
            # each collection. If the user speficified a pmid in the REST call, then
            # check if the pmid exists in the collection

            for repo in repos:
                # this is the name w/out the brainspell-collection in it

                repo["pretty_name"] = repo["name"].replace(
                    "brainspell-collection-", "")

                # get file list content for each repo
                try:
                    content_data = yield torngithub.github_request(
                        self.get_auth_http_client(),
                        '/repos/{owner}/{repo}/contents/{path}'.format(owner=self.get_current_github_username(),
                                                                       repo=repo["name"],
                                                                       # path="{}.json".format(pmid)
                                                                       path=""
                                                                       ),
                        access_token=self.get_current_github_access_token(),
                        method="GET")

                    print(repo["contributors_url"])
                    contrib = yield torngithub.github_request(self.get_auth_http_client(),
                                                              repo["contributors_url"].replace("https://api.github.com", ""),
                                                              access_token=self.get_current_github_access_token(),
                                                              method="GET")
                    repo["contributors"] = contrib["body"]

                    # print(content_data)

                    content = content_data["body"]

                    # extract pmids from content body
                    pmids = [c["name"].replace(".json", "") for c in content]

                    # If we are looking for a certaim pmid, add a tag for if it
                    # exists in the collection
                    if pmid:
                        if pmid in pmids:
                            repo["in_collection"] = True
                        else:
                            repo["in_collection"] = False

                    # get article information from each pmid from the database
                    all_contents = [next(get_article(pmid)) for pmid in pmids]
                except BaseException:  # TODO This is hacky, Empty Repos break the code!
                    all_contents = []
                    repo["contributors"] = {}

                # Convert to a dict the info we want (so it can be JSON
                # serialized later)
                def article_content_dict(cont):
                    return dict(
                        title=cont.title,
                        reference=cont.reference,
                        pmid=cont.pmid)
                repo["contents"] = [
                    article_content_dict(cont) for cont in all_contents]

            # if we want to return a JSON instead or a page render
            if return_list:
                self.write(json_encode(repos))
            else:
                self.render_with_user_info("static/html/github-account.html", {
                    "info": repos
                })

        # if you're not authorized, redirect to oauth
        else:
            self.redirect("/oauth?next=/repos")

class NewRepoHandler(BaseHandler, torngithub.GithubMixin):
    """ Create a new GitHub repo for a collection. """

    # TODO Again update the database
    @tornado.web.asynchronous
    @tornado.gen.coroutine
    def post(self):
        starttime = time.time()
        name = self.get_argument("name")
        desc = self.get_argument("description")
        if not name:
            self.write("")
        else:
            # Update database with new Collection

            new_repo(name, self.get_current_github_username())
            body = {
                "name": "brainspell-collection-{}".format(name),
                "description": desc,
                "homepage": "https://brainspell-neo.herokuapp.com",
                "private": False,
                "has_issues": True,
                "has_projects": True,
                "has_wiki": True
            }
            ress = yield [torngithub.github_request(
                self.get_auth_http_client(), '/user/repos',
                access_token=self.get_current_github_access_token(),
                method="POST",
                body=body)]
            data = []
            for res in ress:
                data.extend(res.body)

            endtime = time.time()
            # print(data)
            return self.redirect("/repos")

class NewFileHandler(BaseHandler, torngithub.GithubMixin):
    # Adds a json file to github repository
    # Add to database as well
    @tornado.web.asynchronous
    @tornado.gen.coroutine
    def post(self):
        starttime = time.time()
        collection = self.get_argument("collection")
        pmid = self.get_argument("pmid")
        print("Collection is {0} and PMID is {1}".format(collection, pmid))
        print(
            "Type of collection is {0} and type of PMID is {1}".format(
                type(collection),
                type(pmid)))
        article = list(get_article(pmid))[0]
        entry = {"pmid": pmid,
                 "title": article.title,
                 "reference": article.reference,
                 "doi": article.doi,
                 "notes": "Here are my notes on this article"}
        content = b64encode(json_encode(entry).encode("utf-8")).decode('utf-8')

        name = collection

        coll_indicator = new_repo(name, self.get_current_github_username())

        if coll_indicator:
            print("Creating collection: " + name)
            body = {
                "name": name,
                "description": "",
                "homepage": "https://brainspell-neo.herokuapp.com",
                "private": False,
                "has_issues": True,
                "has_projects": True,
                "has_wiki": True
            }
            ress = yield [torngithub.github_request(
                self.get_auth_http_client(), '/user/repos',
                access_token=self.get_current_github_access_token(),
                method="POST",
                body=body)]
            data = []
            for res in ress:
                data.extend(res.body)
            print("Created collection")

        add_to_repo(collection, pmid, self.get_current_github_username())
        body = {
            "message": "adding {} to collection".format(pmid),
            "content": content
        }
        ress = yield [torngithub.github_request(
            self.get_auth_http_client(),
            '/repos/{owner}/{repo}/contents/{path}'.format(owner=self.get_current_github_username(),
                                                           repo=collection,
                                                           path="{}.json".format(pmid)),
            access_token=self.get_current_github_access_token(),
            method="PUT",
            body=body)]
        data = []
        for res in ress:
            data.extend(res.body)

        endtime = time.time()

class DeleteFileHandler(BaseHandler, torngithub.GithubMixin):
    # Deletes a JSON file from the User Repo
    @tornado.web.asynchronous
    @tornado.gen.coroutine
    def post(self):
        starttime = time.time()
        collection = self.get_argument("collection")
        pmid = self.get_argument("pmid")
        entry = {"pmid": pmid,
                 "notes": "Here are my notes on this article"}
        content = b64encode(json_encode(entry).encode("utf=8"))
        gh_user = self.__get_current_github_object__()

        remove_from_repo(collection, pmid, self.get_current_github_username())

        body = {
            "message": "deleting {} to collection".format(pmid),
        }

        sha_data = yield [torngithub.github_request(
            self.get_auth_http_client(),
            '/repos/{owner}/{repo}/contents/{path}'.format(owner=self.get_current_github_username(),
                                                           repo=collection,
                                                           path="{}.json".format(pmid)),
            access_token=self.get_current_github_access_token(),
            method="GET")]

        sha = [s["body"]["sha"] for s in sha_data][0]

        ress = yield [torngithub.github_request(
            self.get_auth_http_client(),
            '/repos/{owner}/{repo}/contents/{path}'.format(owner=self.get_current_github_username(),
                                                           repo=collection,
                                                           path="{}.json".format(pmid)),
            access_token=self.get_current_github_access_token(),
            method="DELETE",
            body={"sha": sha, "message": "removing {} from collection".format(pmid)})]

"""
 class BulkNewFileHandler(BaseHandler, torngithub.GithubMixin):
     @tornado.web.asynchronous
     @tornado.gen.coroutine
     def post(self):
         startime = time.time()
         collection = self.get_argument("collection")
         pmids = self.get_argument("pmids")
         pmids = eval(pmids)
         user_info = self.__get_current_github_object__()["login"]
         if collection in next(User.select().where(User.username == user_info).execute()).collections: #If collection exists
             collection = "brainspell-collection-" + collection
             for pmid in pmids:
                 pmid = eval(pmid)
                 article = list(get_article(pmid))[0]
                 entry = {"pmid": pmid,
                         "title": article.title,
                         "reference": article.reference,
                         "doi": article.doi,
                          "notes": "Here are my notes on this article"}
                 content = b64encode(json_encode(entry).encode("utf-8")).decode('utf-8')
                 gh_user = self.__get_current_github_object__()
                 add_to_repo(collection,pmid,self.get_current_github_username())
                 body = {
                     "message": "adding {} to collection".format(pmid),
                     "content": content
                 }
                 ress = yield [
                     torngithub.github_request(
                         self.get_auth_http_client(),
                         '/repos/{owner}/{repo}/contents/{path}'.format(owner=self.get_current_github_username(),
                                                                        repo=collection,
                                                                        path="{}.json".format(pmid)),
                         access_token=self.get_current_github_access_token(),
                         method="PUT",
                         body=body
                     )
                 ]
                 data = []
                 for res in ress:
                     data.extend(res.body)
                 endtime = time.time()
         else:
             print("Your collection doesn't exist")
             return False #TODO: Tell user the collection doesn't exist
"""
<|endoftext|>


<commit_before>import os
import argparse

import yaml
from terminaltables import AsciiTable

from .manage import NodeManager

BASE_PATH = os.path.dirname(os.path.realpath(__file__))

with open(os.path.join(BASE_PATH, 'config.yaml'), 'r') as conf_file:
    CONFIG = yaml.load(conf_file)

class StoreNameValuePair(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
        for val in values:
            n, v = val.split('=')
            setattr(namespace, n, v)

def main():

    valid_commands = CONFIG['nodes']['commands']
    valid_commands.extend(CONFIG['services']['commands'])

    parser = argparse.ArgumentParser()
    parser.add_argument('action', metavar='action', type=str, nargs=1, help='Command', choices=valid_commands)
    action_arg = parser.parse_known_args()

    node_manager = NodeManager()

    if action_arg[0].action == ['create']:
        parser.add_argument('name', metavar='name', type=str, nargs=1, help='Node name')
        parser.add_argument('type', metavar='type', type=str, nargs=1, help='Node type')
        parser.add_argument('config', nargs='*', action=StoreNameValuePair, help='Node config options')
        args = parser.parse_args()
        node_manager.node_create(node_name=args.name[0], node_type=args.type[0], **vars(args))

    elif action_arg[0].action == ['delete']:
        parser.add_argument('name', metavar='name', type=str, nargs=1, help='Node name')
        args = parser.parse_args()
        node_manager.node_delete(node_name=args.name[0])

    elif action_arg[0].action == ['show']:
        parser.add_argument('type', metavar='type', type=str, nargs='?', help='Node type')
        args = parser.parse_args()
        if args.type:
            table_data = [
                ['Need to provide:']
            ]
            node_params = node_manager.get_node_params(node_type=args.type)
            for param in node_params:
                table_data.append([param])
            table = AsciiTable(table_data)
            print(table.table)
        else:
            print('Valid node types:')
            table_data = [
                ['Type', 'Config']
            ]
            for type_name, config in node_manager.valid_node_types.items():
                table_data.append([type_name, config])
            table = AsciiTable(table_data)
            print(table.table)

    elif action_arg[0].action == ['list']:
        all_nodes = node_manager.node_list()
        table_data = [
            ['Name', 'Type', 'Services']
        ]
        for node in all_nodes:
            table_data.append([node.name, node.type, ','.join(service for service in node.services)])
        table = AsciiTable(table_data)
        print(table.table)

    else:
        parser.add_argument('name', metavar='name', type=str, nargs=1, help='Node name')
        args = parser.parse_args()
        node = node_manager.node_get(node_name=args.name[0])
        node.command(cmd=action_arg[0].action[0])
<commit_msg>CLI actions and help<commit_after>import os
import argparse

import yaml
from terminaltables import AsciiTable

from .manage import NodeManager

BASE_PATH = os.path.dirname(os.path.realpath(__file__))

with open(os.path.join(BASE_PATH, 'config.yaml'), 'r') as conf_file:
    CONFIG = yaml.load(conf_file)

def main():
    # Get node manager
    node_manager = NodeManager()

    parser = argparse.ArgumentParser(description='Kozinaki compute node manage utility')
    subparsers = parser.add_subparsers(help='Available actions', dest='action')

    # CREATE
    parser_create = subparsers.add_parser(
        'create',
        description='Create compute node for cloud provider',
        help='Create new nova compute node'
    )
    parser_create_subparsers = parser_create.add_subparsers(help='Available cloud providers', dest='type')

    # Create providers subparsers
    for provider_name, config in node_manager.valid_node_types.items():
        parser_create_type = parser_create_subparsers.add_parser(
            provider_name,
            description='Create node in {} cloud'.format(provider_name.upper()),
            help='Create node in {} cloud'.format(provider_name.upper())
        )
        parser_create_type.add_argument('--name', type=str, required=True, help='Compute node name')
        for param in node_manager.get_node_params(provider_name):
            parser_create_type.add_argument('--{}'.format(param), type=str, required=True)

    # DELETE
    parser_delete = subparsers.add_parser(
        'delete',
        description='Delete compute node',
        help='Delete compute node'
    )
    parser_delete.add_argument('--name', type=str, required=True, help='Compute node name')

    # LIST
    parser_node_list = subparsers.add_parser(
        'list',
        description='Show all created compute nodes',
        help='Show all created compute nodes'
    )

    # COMMANDS
    for command in CONFIG['services']['commands']:
        parser_node_command = subparsers.add_parser(
            command,
            description='Pass {} command to all node services'.format(command),
            help='Pass {} command to all node services'.format(command)
        )
        parser_node_command.add_argument('--name', type=str, required=True, help='Compute node name')

    args = parser.parse_args()

    if args.action == 'create':
        node_manager.node_create(node_name=args.name, node_type=args.type, **vars(args))
    elif args.action == 'delete':
        node_manager.node_delete(node_name=args.name)
    elif args.action == 'list':
        table_data = [['Name', 'Type', 'Services']]
        for node in node_manager.node_list():
            table_data.append([node.name, node.type, ','.join(service for service in node.services)])
        table = AsciiTable(table_data)
        print(table.table)
    elif args.action in CONFIG['services']['commands']:
        node = node_manager.node_get(node_name=args.name)
        node.command(cmd=args.action)
<|endoftext|>