<commit_before>#!/usr/bin/python

from oauth2client.client import  OAuth2WebServerFlow, OAuth2Credentials
from oauth2client.file import Storage
from apiclient.discovery import build
from apiclient import errors
import httplib2

import argparse
import sys, os.path
import re
import logging

from operator import itemgetter
from collections import defaultdict
import webbrowser
import yaml, csv, json, pprint
import StringIO
import shelve
import pandas as pd
import datetime

LOGLEVELS = {'DEBUG': logging.DEBUG,
             'INFO': logging.INFO,
             'WARNING': logging.WARNING,
             'ERROR': logging.ERROR,
             'CRITICAL': logging.CRITICAL}

logger = logging.getLogger(__name__)

def default_options():
    defaults = {}
    defaults['store'] = os.path.expanduser('~/.gcat/store')
    defaults['config'] = os.path.expanduser('~/.gcat/config')
    defaults['cache'] = os.path.expanduser('~/.gcat/cache')
    defaults['usecache'] = False
    defaults['redirect_uri'] = 'urn:ietf:wg:oauth:2.0:oob'
    return defaults

def load_config(opts):
    if 'config' in opts:
        try:
            with open(opts['config'], 'r') as f:
                config = yaml.load(f)
                return dict(config)
        except IOError:
            logger.error('Could not find config file at %s', yaml_name)
            sys.exit()
    else:
        return {}

def get_file(title, fmt='dict', **kwargs):
    """
    Simple interface for grabbing a Google Drive file by title.  Retrieves
    file in xlsx format and parses with pandas.ExcelFile
    If keyword argument sheet_name or sheet_id is given, returns only specified sheet.
    The `fmt` keyword argument determines the format of the return value.
    Here is the list of accepted formats and the corresponding return value type:
      * `dict`           : list of dicts (Default).
      * `pandas`         : pandas.DataFrame
      * `list`           : list of lists
      * `pandas_excel`   : Pandas.ExcelFile, (not yet parsed) useful for custom parsing
    For all formats other than `xlrd`, if no sheet name is given, get_file returns
    a dict with sheet names as keys and accordingly formatted rows as values.
    """
    opts = default_options()
    opts['title'] = title
    opts.update((k,v) for k, v in load_config(opts).items() if v is not None)
    opts.update((k,v) for k, v in kwargs.items() if v is not None)
    logger.info('opts:\n%s', pprint.pformat(opts))

    content = get_content(opts)
    wb = pd.ExcelFile(StringIO.StringIO(content))

    if fmt == 'pandas_excel':
        return wb

    try:
        parsed_wb = {sheet_name : wb.parse(sheet_name) for sheet_name in wb.sheet_names}
    except:
        logger.exception('error parsing worksheet using pandas.ExcelFile.parse(sheet_name). '
                         'Consider using pandas_excel and parsing the file yourself to have more control')
    if fmt == 'list':
        fmt_wb = {name : list(df.itertuples()) for name, df in parsed_wb.items()}
    elif fmt == 'dict':
        fmt_wb = {name : [r[1].to_dict() for r in df.iterrows()] for name, df in parsed_wb.items()}
    elif fmt == 'pandas':
        fmt_wb = parsed_wb
    else:
        raise ValueError('unkown format: %s' % fmt)
    if len(fmt_wb) == 1:
        return fmt_wb.values()[0]
    if 'sheet' in opts:
        try:
            return fmt_wb[opts['sheet']]
        except:
            print 'sheet name: `%s` not found in workbook.  sheet_names: %s' % (opts['sheet'], fmt_wb.keys())
            logger.exception('sheet name: %s not found in workbook.  sheet_names: %s', opts['sheet'], fmt_wb.keys())
            raise
    else:
        return fmt_wb

def get_content(opts):
    cache = shelve.open(opts['cache'])
    if opts['usecache'] and opts['title'] in cache:
        content = cache[opts['title']]
    else:
        logger.debug('computing from scratch')
        service = get_service(opts)
        files = service.files()
        try:
            res = files.list().execute()
        except errors.HttpError, error:
            logger.error('An error occurred: %s', exc_info=error)
            raise error

        names = map(itemgetter('title'), res['items'])
        try:
            idx = names.index(opts['title'])
        except ValueError:
            logger.error('file name: %s not in list', opts['title'])
            sys.exit()
        if idx == -1:
            raise ValueError('name %s not found in Google Drive' % opts['title'])
        file = res['items'][idx]
        content = download(service, file)
        cache[opts['title']] = content
    return content

def get_service(opts):
    flow = OAuth2WebServerFlow(client_id=opts['client_id'],
                               client_secret=opts['client_secret'],
                               scope=opts['scope'],
                               redirect_uri=opts['redirect_uri'])

    credentials = get_credentials(flow, opts)

    http = httplib2.Http()
    http = credentials.authorize(http)

    service = build('drive', 'v2', http=http)
    return service

def get_credentials(flow, opts):
    storage = Storage(opts['store'])
    credentials = storage.get()
    if not credentials:
        # get the credentials the hard way
        auth_url = flow.step1_get_authorize_url()
        webbrowser.open(auth_url)
        code = raw_input('go to:\n\n\t%s\n\nand enter in the code displayed:' % auth_url)
        credentials = flow.step2_exchange(code)
        storage.put(credentials)

    #pprint.pprint(json.loads(credentials.to_json()), indent=2)
    if credentials.access_token_expired:
        logger.info('refreshing token')
        refresh_http = httplib2.Http()
        credentials.refresh(refresh_http)
    return credentials

def download(service, file):
    logger.debug('file.viewkeys(): %s', pprint.pformat(file.viewkeys()))
    #download_url = file.get('downloadUrl') # not present for some reason
    #download_url_pdf = file.get('exportLinks')['application/pdf']
    download_url = file.get('exportLinks')['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet']
    #download_url = re.sub('pdf$', 'csv', download_url_pdf)
    #download_url = re.sub('&exportFormat=csv$', '#gid=0&exportFormat=csv', download_url)
    #logger.debug('file.get(\'exportlinks\': %s', file.get('exportLinks'))
    if download_url:
        resp, content = service._http.request(download_url)
        if resp.status == 200:
            logger.debug('Status: %s', resp)
            return content
        else:
            logger.error('An error occurred: %s' % resp)
            return None
    else:
        # The file doesn't have any content stored on Drive.
        logger.error('file does not have any content stored on Drive')
        return None

def merge_config(opts, yaml_name):
    try:
        with open(yaml_name, 'r') as f:
            config = yaml.load(f)
            logger.debug('merging command-line opts with config from file: %s', yaml_name)
            for k, v in config.items():
                if not hasattr(opts,k) or getattr(opts, k) is None:
                    setattr(opts,k,v)
    except IOError:
        logger.error('Could not find config file at %s', yaml_name)
        sys.exit()

class Join(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
        if not isinstance(values, str):
            setattr(namespace, self.dest, ' '.join(values))

def parse_args(**kwopts):
    parser = argparse.ArgumentParser(description='print a google spreadsheet to stdout')
    parser.add_argument('--store',
                        help='location where gcat will store file specific credentials')
    parser.add_argument('--config',
                        help='a yaml file specifying the client_id, client_secret, scope, and redirect_uri')
    parser.add_argument('--client_id',
                        help='google api client id. this can be found at the google api console.  Note that'
                        'you first need to register your client as an "installed application" at the console'
                        ' as well (code.google.com/apis/console)')
    parser.add_argument('--client_secret',
                        help='google api client secret. this can be found at the google api console.  Note that'
                        'you first need to register your client as an "installed application" at the console'
                        ' as well (code.google.com/apis/console)')
    parser.add_argument('--scope',
                        help='list of scopes for which your client is authorized')
    parser.add_argument('--redirect_uri',
                        help='google api redirect URI. this can be found at the google api console under the \"Redirect URI\"'
                        'section.  By default a client if assigned two valid redirect URIs: urn:ietf:wg:oauth:2.0:oob '
                        'and http://localhostl.  use the urn:ietf:wg:oauth:2.0:oob unless you are doing something fancy.'
                        'see https://developers.google.com/accounts/docs/OAuth2InstalledApp for more info')
    parser.add_argument('--title',
                        nargs='+',
                        metavar='title_word',
                        action=Join,
                        required=True,
                        help='the name of the google drive file in question.  If the name has spaces, gcat will do the '
                        ' right thing and join a sequence of command line arguments with space')
    parser.add_argument('--sheet',
                        nargs='+',
                        metavar='sheet_word',
                        action=Join,
                        help='sheet name within the spreadsheet.  If neither --sheet nor --sheet_id arguments are given '
                        'gcat will default to sheet_id=0. If the name has spaces, gcat will do the right thing and join '
                        'a sequence of command line arguments with spaces.')
    parser.add_argument('--cache',
                        help='location in which gcat will store documents when the --usecache flag is given')
    parser.add_argument('--usecache',
                        action='store_true',
                        help='instructs gcat to use the cache located in a file specified by the --cache option')
    args = parser.parse_args()
    return vars(args)

def write_to_stdout(content):
    for line in content:
        print '\t'.join(map(str, line))

def main():
    """
    logging set up
    """
    logger = logging.getLogger(__name__)
    ch = logging.StreamHandler()
    formatter = logging.Formatter('[%(levelname)s]\t[%(filename)s:%(funcName)s:%(lineno)d]\t%(message)s')
    ch.setFormatter(formatter)
    logger.addHandler(ch)
    logger.setLevel(logging.DEBUG)

    content = get_file(as_dict=False, **parse_args())
    write_to_stdout(content)

if __name__ == '__main__':
    main()
<commit_msg>updates dosctring for get_file<commit_after>#!/usr/bin/python

from oauth2client.client import  OAuth2WebServerFlow, OAuth2Credentials
from oauth2client.file import Storage
from apiclient.discovery import build
from apiclient import errors
import httplib2

import argparse
import sys, os.path
import re
import logging

from operator import itemgetter
from collections import defaultdict
import webbrowser
import yaml, csv, json, pprint
import StringIO
import shelve
import pandas as pd
import datetime

LOGLEVELS = {'DEBUG': logging.DEBUG,
             'INFO': logging.INFO,
             'WARNING': logging.WARNING,
             'ERROR': logging.ERROR,
             'CRITICAL': logging.CRITICAL}

logger = logging.getLogger(__name__)

def default_options():
    defaults = {}
    defaults['store'] = os.path.expanduser('~/.gcat/store')
    defaults['config'] = os.path.expanduser('~/.gcat/config')
    defaults['cache'] = os.path.expanduser('~/.gcat/cache')
    defaults['usecache'] = False
    defaults['redirect_uri'] = 'urn:ietf:wg:oauth:2.0:oob'
    return defaults

def load_config(opts):
    if 'config' in opts:
        try:
            with open(opts['config'], 'r') as f:
                config = yaml.load(f)
                return dict(config)
        except IOError:
            logger.error('Could not find config file at %s', yaml_name)
            sys.exit()
    else:
        return {}

def get_file(title, fmt='dict', **kwargs):
    """
    Simple interface for grabbing a Google Drive file by title.  Retrieves
    file in xlsx format and parses with pandas.ExcelFile If keyword argument `sheet` is given,
    returns only specified sheet.
    args:
        title     (str)   : Title of Google Drive document
        fmt       (str)   : Determines the format of the return value.
                            list of accepted formats and the corresponding return value type:
                            * `dict`           : list of dicts (Default).
                            * `pandas`         : pandas.DataFrame
                            * `list`           : list of lists
                            * `pandas_excel`   : Pandas.ExcelFile, (not yet parsed) useful for custom parsing
                            For all formats other than `pandas_excel`, if no sheet name is given and the spreadsheet
                            contains more than one sheet, get_file returns a dict with sheet names as keys and accordingly
                            formatted sheets as values.
    kwargs:
        sheet     (str)   : name of sheet to return
        cache     (str)   : location in which to store the cached contents of files
        usecache  (bool)  : whether to use the cache (default is False), but useful for debugging
        config    (str)   : path from which to read the config file which contains credentials
        store     (str)   : location in which to store file-specific credentials

    """
    opts = default_options()
    opts['title'] = title
    opts.update((k,v) for k, v in load_config(opts).items() if v is not None)
    opts.update((k,v) for k, v in kwargs.items() if v is not None)
    logger.info('opts:\n%s', pprint.pformat(opts))

    content = get_content(opts)
    wb = pd.ExcelFile(StringIO.StringIO(content))

    if fmt == 'pandas_excel':
        return wb

    try:
        parsed_wb = {sheet_name : wb.parse(sheet_name) for sheet_name in wb.sheet_names}
    except:
        logger.exception('error parsing worksheet using pandas.ExcelFile.parse(sheet_name). '
                         'Consider using pandas_excel and parsing the file yourself to have more control')
    if fmt == 'list':
        fmt_wb = {name : list(df.itertuples()) for name, df in parsed_wb.items()}
    elif fmt == 'dict':
        fmt_wb = {name : [r[1].to_dict() for r in df.iterrows()] for name, df in parsed_wb.items()}
    elif fmt == 'pandas':
        fmt_wb = parsed_wb
    else:
        raise ValueError('unkown format: %s' % fmt)
    if len(fmt_wb) == 1:
        return fmt_wb.values()[0]
    if 'sheet' in opts:
        try:
            return fmt_wb[opts['sheet']]
        except:
            print 'sheet name: `%s` not found in workbook.  sheet_names: %s' % (opts['sheet'], fmt_wb.keys())
            logger.exception('sheet name: %s not found in workbook.  sheet_names: %s', opts['sheet'], fmt_wb.keys())
            raise
    else:
        return fmt_wb

def get_content(opts):
    cache = shelve.open(opts['cache'])
    if opts['usecache'] and opts['title'] in cache:
        content = cache[opts['title']]
    else:
        logger.debug('computing from scratch')
        service = get_service(opts)
        files = service.files()
        try:
            res = files.list().execute()
        except errors.HttpError, error:
            logger.error('An error occurred: %s', exc_info=error)
            raise error

        names = map(itemgetter('title'), res['items'])
        try:
            idx = names.index(opts['title'])
        except ValueError:
            logger.error('file name: %s not in list', opts['title'])
            sys.exit()
        if idx == -1:
            raise ValueError('name %s not found in Google Drive' % opts['title'])
        file = res['items'][idx]
        content = download(service, file)
        cache[opts['title']] = content
    return content

def get_service(opts):
    flow = OAuth2WebServerFlow(client_id=opts['client_id'],
                               client_secret=opts['client_secret'],
                               scope=opts['scope'],
                               redirect_uri=opts['redirect_uri'])

    credentials = get_credentials(flow, opts)

    http = httplib2.Http()
    http = credentials.authorize(http)

    service = build('drive', 'v2', http=http)
    return service

def get_credentials(flow, opts):
    storage = Storage(opts['store'])
    credentials = storage.get()
    if not credentials:
        # get the credentials the hard way
        auth_url = flow.step1_get_authorize_url()
        webbrowser.open(auth_url)
        code = raw_input('go to:\n\n\t%s\n\nand enter in the code displayed:' % auth_url)
        credentials = flow.step2_exchange(code)
        storage.put(credentials)

    #pprint.pprint(json.loads(credentials.to_json()), indent=2)
    if credentials.access_token_expired:
        logger.info('refreshing token')
        refresh_http = httplib2.Http()
        credentials.refresh(refresh_http)
    return credentials

def download(service, file):
    logger.debug('file.viewkeys(): %s', pprint.pformat(file.viewkeys()))
    #download_url = file.get('downloadUrl') # not present for some reason
    #download_url_pdf = file.get('exportLinks')['application/pdf']
    download_url = file.get('exportLinks')['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet']
    #download_url = re.sub('pdf$', 'csv', download_url_pdf)
    #download_url = re.sub('&exportFormat=csv$', '#gid=0&exportFormat=csv', download_url)
    #logger.debug('file.get(\'exportlinks\': %s', file.get('exportLinks'))
    if download_url:
        resp, content = service._http.request(download_url)
        if resp.status == 200:
            logger.debug('Status: %s', resp)
            return content
        else:
            logger.error('An error occurred: %s' % resp)
            return None
    else:
        # The file doesn't have any content stored on Drive.
        logger.error('file does not have any content stored on Drive')
        return None

def merge_config(opts, yaml_name):
    try:
        with open(yaml_name, 'r') as f:
            config = yaml.load(f)
            logger.debug('merging command-line opts with config from file: %s', yaml_name)
            for k, v in config.items():
                if not hasattr(opts,k) or getattr(opts, k) is None:
                    setattr(opts,k,v)
    except IOError:
        logger.error('Could not find config file at %s', yaml_name)
        sys.exit()

class Join(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
        if not isinstance(values, str):
            setattr(namespace, self.dest, ' '.join(values))

def parse_args(**kwopts):
    parser = argparse.ArgumentParser(description='print a google spreadsheet to stdout')
    parser.add_argument('--store',
                        help='location where gcat will store file specific credentials')
    parser.add_argument('--config',
                        help='a yaml file specifying the client_id, client_secret, scope, and redirect_uri')
    parser.add_argument('--client_id',
                        help='google api client id. this can be found at the google api console.  Note that'
                        'you first need to register your client as an "installed application" at the console'
                        ' as well (code.google.com/apis/console)')
    parser.add_argument('--client_secret',
                        help='google api client secret. this can be found at the google api console.  Note that'
                        'you first need to register your client as an "installed application" at the console'
                        ' as well (code.google.com/apis/console)')
    parser.add_argument('--scope',
                        help='list of scopes for which your client is authorized')
    parser.add_argument('--redirect_uri',
                        help='google api redirect URI. this can be found at the google api console under the \"Redirect URI\"'
                        'section.  By default a client if assigned two valid redirect URIs: urn:ietf:wg:oauth:2.0:oob '
                        'and http://localhostl.  use the urn:ietf:wg:oauth:2.0:oob unless you are doing something fancy.'
                        'see https://developers.google.com/accounts/docs/OAuth2InstalledApp for more info')
    parser.add_argument('--title',
                        nargs='+',
                        metavar='title_word',
                        action=Join,
                        required=True,
                        help='the name of the google drive file in question.  If the name has spaces, gcat will do the '
                        ' right thing and join a sequence of command line arguments with space')
    parser.add_argument('--sheet',
                        nargs='+',
                        metavar='sheet_word',
                        action=Join,
                        help='sheet name within the spreadsheet.  If neither --sheet nor --sheet_id arguments are given '
                        'gcat will default to sheet_id=0. If the name has spaces, gcat will do the right thing and join '
                        'a sequence of command line arguments with spaces.')
    parser.add_argument('--cache',
                        help='location in which gcat will store documents when the --usecache flag is given')
    parser.add_argument('--usecache',
                        action='store_true',
                        help='instructs gcat to use the cache located in a file specified by the --cache option')
    args = parser.parse_args()
    return vars(args)

def write_to_stdout(content):
    for line in content:
        print '\t'.join(map(str, line))

def main():
    """
    logging set up
    """
    logger = logging.getLogger(__name__)
    ch = logging.StreamHandler()
    formatter = logging.Formatter('[%(levelname)s]\t[%(filename)s:%(funcName)s:%(lineno)d]\t%(message)s')
    ch.setFormatter(formatter)
    logger.addHandler(ch)
    logger.setLevel(logging.DEBUG)

    content = get_file(as_dict=False, **parse_args())
    write_to_stdout(content)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""---------------------------------------------------------------------------
Name:        VoyagerGpToolRunner.py
Purpose:
Author:      VoyagerSearch
Created:     18/12/2013
Copyright:   (c) VoyagerSearch 2013
---------------------------------------------------------------------------"""
import os
import sys
import json
import arcpy

def find(f, seq):
  """Return first item in sequence where f(item) == True."""
  for item in seq:
    if f(item):
      return item

def run_job(json_file):
    """Run a Geoprocessing tool.
    Parse the json for tool name and parameters.
    """
    with open(json_file) as data_file:
        data = json.load(data_file)
        request = data;  # data['request'][0]

        # Load the GP Toolbox
        arcpy.ImportToolbox(os.path.dirname(request['task']))

        # Create the folder if it does not exist.
        if not os.path.exists(data['folder']):
            os.makedirs(data['folder'])

        # Retrieve the list of parameters.
        parameters = request['params']

        # TODO: this script makes assumptions about the *order* of the path elements
        # ideally this should lookup names by key and validate as required
        if os.path.basename(request['task']) == 'ClipData':

            # Retrieve input items to be clipped.
            input_items = find(lambda p: p['name'] == 'input_items', parameters)
            in_data = ';'.join([v['lyr'] if not v['lyr'] == '' else v['path'] for v in input_items['value']])

            # Retrieve clip geometry.
            clip_geom_wkt = find(lambda p: p['name'] == 'clip_geometry', parameters)['wkt']

            # Retieve the coordinate system code.
            sr_code = find(lambda p: p['name'] == 'output_projection', parameters)['value']

            # Retrive the output format type.
            output_format = find(lambda p: p['name'] == 'output_format', parameters)['value']
            if output_format == 'FileGDB':
                output_location = os.path.join(data['folder'], 'output.gdb')
            else:
                output_location = data['folder']

            # Execute task.
            try:
                arcpy.voyager.ClipData(in_data, output_location, clip_geom_wkt, sr_code, output_format)
            except arcpy.ExecuteError:
                sys.stderr.write(arcpy.GetMessages(2))
                sys.exit(1)

        # Success
        sys.exit(0)

if __name__ == '__main__':
    #run_job(r"C:\jason\scripts\cliptask.json")
    run_job(sys.argv[1])

<commit_msg>update the selection query<commit_after>"""---------------------------------------------------------------------------
Name:        VoyagerGpToolRunner.py
Purpose:
Author:      VoyagerSearch
Created:     18/12/2013
Copyright:   (c) VoyagerSearch 2013
---------------------------------------------------------------------------"""
import os
import sys
import json
import arcpy

def find(f, seq):
  """Return first item in sequence where f(item) == True."""
  for item in seq:
    if f(item):
      return item

def run_job(json_file):
    """Run a Geoprocessing tool.
    Parse the json for tool name and parameters.
    """
    with open(json_file) as data_file:
        data = json.load(data_file)
        request = data;  # data['request'][0]

        # Load the GP Toolbox
        arcpy.ImportToolbox(os.path.dirname(request['task']))

        # Create the folder if it does not exist.
        if not os.path.exists(data['folder']):
            os.makedirs(data['folder'])

        # Retrieve the list of parameters.
        parameters = request['params']

        # TODO: this script makes assumptions about the *order* of the path elements
        # ideally this should lookup names by key and validate as required
        if os.path.basename(request['task']) == 'ClipData':

            # Retrieve input items to be clipped.
            input_items = find(lambda p: p['name'] == 'input_items', parameters)
            in_data = ';'.join([v['[lyrFile]'] if not v['[lyrFile]'] == '' else v['path'] for v in input_items['response']['docs'])

            # Retrieve clip geometry.
            clip_geom_wkt = find(lambda p: p['name'] == 'clip_geometry', parameters)['wkt']

            # Retieve the coordinate system code.
            sr_code = find(lambda p: p['name'] == 'output_projection', parameters)['value']

            # Retrive the output format type.
            output_format = find(lambda p: p['name'] == 'output_format', parameters)['value']
            if output_format == 'FileGDB':
                output_location = os.path.join(data['folder'], 'output.gdb')
            else:
                output_location = data['folder']

            # Execute task.
            try:
                arcpy.voyager.ClipData(in_data, output_location, clip_geom_wkt, sr_code, output_format)
            except arcpy.ExecuteError:
                sys.stderr.write(arcpy.GetMessages(2))
                sys.exit(1)

        # Success
        sys.exit(0)

if __name__ == '__main__':
    #run_job(r"C:\jason\scripts\cliptask.json")
    run_job(sys.argv[1])

<|endoftext|>


<commit_before>import collections.abc
import event_model
from sys import maxsize
from functools import partial
import intake
import intake.catalog
import intake.catalog.local
import intake.source.base
import pymongo
import pymongo.errors

from ..core import parse_handler_registry
from ..v2 import Broker

class _Entries(collections.abc.Mapping):
    "Mock the dict interface around a MongoDB query result."
    def __init__(self, catalog):
        self.catalog = catalog

    def _doc_to_entry(self, run_start_doc):

        header_doc = None
        uid = run_start_doc['uid']

        def get_header_field(field):
            nonlocal header_doc
            if header_doc is None:
                header_doc = self.catalog._db.header.find_one(
                                {'run_id': uid}, {'_id': False})
            if field in header_doc:
                if field in ['start', 'stop']:
                    return header_doc[field][0]
                else:
                    return header_doc[field]
            else:
                if field[0:6] == 'count_':
                    return 0
                else:
                    return None

        def get_resource(uid):
            resources = get_header_field('resources')
            for resource in resources:
                if resource['uid'] == uid:
                    return resource
            raise ValueError(f"Could not find Resource with uid={uid}")

        def lookup_resource_for_datum(datum_id):
            """ This method is likely very slow. """
            resources = [resource['uid']
                         for resource in get_header_field('resources')]
            datum_page = self.catalog._db.datum.find_one(
                    {'$and': [{'resource': {'$in': resources}},
                              {'datum_id': datum_id}]},
                    {'_id': False})
            for datum in event_model.unpack_datum_page(datum_page):
                if datum['datum_id'] == datum_id:
                    return datum['resource']
            raise ValueError(f"Could not find Datum with datum_id={datum_id}")

        def get_run_start():
            return run_start_doc

        def get_event_count(descriptor_uid):
            return get_header_field(f'count_{descriptor_uid}')

        entry_metadata = {'start': get_header_field('start'),
                          'stop': get_header_field('stop')}

        args = dict(
            get_run_start=get_run_start,
            get_run_stop=partial(get_header_field, 'stop'),
            get_event_descriptors=partial(
                            get_header_field, 'descriptors'),
            get_event_pages=self.catalog._get_event_pages,
            get_event_count=get_event_count,
            get_resource=get_resource,
            lookup_resource_for_datum=lookup_resource_for_datum,
            get_datum_pages=self.catalog._get_datum_pages,
            filler=self.catalog.filler)
        return intake.catalog.local.LocalCatalogEntry(
            name=run_start_doc['uid'],
            description={},  # TODO
            driver='databroker.core.BlueskyRun',
            direct_access='forbid',  # ???
            args=args,
            cache=None,  # ???
            parameters=[],
            metadata=entry_metadata,
            catalog_dir=None,
            getenv=True,
            getshell=True,
            catalog=self.catalog)

    def __iter__(self):
        cursor = self.catalog._db.header.find(
                    self.catalog._query,
                    {'start.uid': True, '_id': False},
                    sort=[('start.time', pymongo.DESCENDING)])

        for doc in cursor:
            yield doc['start'][0]['uid']

    def __getitem__(self, name):
        # If this came from a client, we might be getting '-1'.
        try:
            N = int(name)
        except ValueError:
            query = {'$and': [self.catalog._query, {'uid': name}]}
            header_doc = self.catalog._db.header.find_one(query)
            if header_doc is None:
                regex_query = {
                    '$and': [self.catalog._query,
                             {'start.uid': {'$regex': f'{name}.*'}}]}
                matches = list(
                    self.catalog._db.header.find(regex_query).limit(10))
                if not matches:
                    raise KeyError(name)
                elif len(matches) == 1:
                    header_doc, = matches
                else:
                    match_list = '\n'.join(doc['uid'] for doc in matches)
                    raise ValueError(
                        f"Multiple matches to partial uid {name!r}. "
                        f"Up to 10 listed here:\n"
                        f"{match_list}")
        else:
            if N < 0:
                # Interpret negative N as "the Nth from last entry".
                query = self.catalog._query
                cursor = (self.catalog._db.header.find(query)
                          .sort('start.time', pymongo.DESCENDING)
                          .skip(-N - 1)
                          .limit(1))
                try:
                    header_doc, = cursor
                except ValueError:
                    raise IndexError(
                        f"Catalog only contains {len(self.catalog)} "
                        f"runs.")
            else:
                # Interpret positive N as
                # "most recent entry with scan_id == N".
                query = {'$and': [self.catalog._query, {'start.scan_id': N}]}
                cursor = (self.catalog._db.header.find(query)
                          .sort('start.time', pymongo.DESCENDING)
                          .limit(1))
                try:
                    header_doc, = cursor
                except ValueError:
                    raise KeyError(f"No run with scan_id={N}")
        if header_doc is None:
            raise KeyError(name)
        return self._doc_to_entry(header_doc['start'][0])

    def __contains__(self, key):
        # Avoid iterating through all entries.
        try:
            self[key]
        except KeyError:
            return False
        else:
            return True

    def __len__(self):
        return len(self.catalog)

class BlueskyMongoCatalog(Broker):
    def __init__(self, datastore_db, *, handler_registry=None, root_map=None,
                 query=None, **kwargs):
        """
        This Catalog is backed by a MongoDB with an embedded data model.

        This embedded data model has three collections: header, event, datum.
        The header collection includes start, stop, descriptor, and resource
        documents. The event_pages are stored in the event colleciton, and
        datum_pages are stored in the datum collection.

        Parameters
        ----------
        datastore_db : pymongo.database.Database or string
            Must be a Database or a URI string that includes a database name.
        handler_registry : dict, optional
            Maps each asset spec to a handler class or a string specifying the
            module name and class name, as in (for example)
            ``{'SOME_SPEC': 'module.submodule.class_name'}``.
        root_map : dict, optional
            Maps resource root paths to different paths.
        query : dict, optional
            MongoDB query. Used internally by the ``search()`` method.
        **kwargs :
            Additional keyword arguments are passed through to the base class,
            Catalog.
        """
        name = 'bluesky-mongo-embedded-catalog'  # noqa

        if isinstance(datastore_db, str):
            self._db = _get_database(datastore_db)
        else:
            self._db = datastore_db

        self._query = query or {}

        if handler_registry is None:
            handler_registry = {}
        parsed_handler_registry = parse_handler_registry(handler_registry)
        self.filler = event_model.Filler(
            parsed_handler_registry, root_map=None, inplace=True)
        super().__init__(**kwargs)

    def _get_event_pages(self, descriptor_uid, skip=0, limit=None):
        if limit is None:
            limit = maxsize

        page_cursor = self._db.event.find(
                            {'$and': [
                                {'descriptor': descriptor_uid},
                                {'last_index': {'$gte': skip}},
                                {'first_index': {'$lte': skip + limit}}]},
                            {'_id': False},
                            sort=[('last_index', pymongo.ASCENDING)])

        return page_cursor

    def _get_datum_pages(self, resource_uid, skip=0, limit=None):
        if limit is None:
            limit = maxsize

        page_cursor = self._db.datum.find(
                            {'$and': [
                                {'resource': resource_uid},
                                {'last_index': {'$gte': skip}},
                                {'first_index': {'$lte': skip + limit}}]},
                            {'_id': False},
                            sort=[('last_index', pymongo.ASCENDING)])

        return page_cursor

    def _make_entries_container(self):
        return _Entries(self)

    def _close(self):
        self._client.close()

    def __len__(self):
        return self._db.header.count_documents(self._query)

    def search(self, query):
        """
        Return a new Catalog with a subset of the entries in this Catalog.

        Parameters
        ----------
        query : dict
            MongoDB query.
        """
        if query:
            query = {f"start.{key}": val for key, val in query.items()}
        if self._query:
            query = {'$and': [self._query, query]}
        cat = type(self)(
            datastore_db=self._db,
            query=query,
            handler_registry=self.filler.handler_registry,
            root_map=self.filler.root_map,
            name='search results',
            getenv=self.getenv,
            getshell=self.getshell,
            auth=self.auth,
            metadata=(self.metadata or {}).copy(),
            storage_options=self.storage_options)
        return cat

def _get_database(uri):
    client = pymongo.MongoClient(uri)
    try:
        # Called with no args, get_database() returns the database
        # specified in the client's uri --- or raises if there was none.
        # There is no public method for checking this in advance, so we
        # just catch the error.
        return client.get_database()
    except pymongo.errors.ConfigurationError as err:
        raise ValueError(
            f"Invalid client: {client} "
            f"Did you forget to include a database?") from err
<commit_msg>Fix typo<commit_after>import collections.abc
import event_model
from sys import maxsize
from functools import partial
import intake
import intake.catalog
import intake.catalog.local
import intake.source.base
import pymongo
import pymongo.errors

from ..core import parse_handler_registry
from ..v2 import Broker

class _Entries(collections.abc.Mapping):
    "Mock the dict interface around a MongoDB query result."
    def __init__(self, catalog):
        self.catalog = catalog

    def _doc_to_entry(self, run_start_doc):

        header_doc = None
        uid = run_start_doc['uid']

        def get_header_field(field):
            nonlocal header_doc
            if header_doc is None:
                header_doc = self.catalog._db.header.find_one(
                                {'run_id': uid}, {'_id': False})
            if field in header_doc:
                if field in ['start', 'stop']:
                    return header_doc[field][0]
                else:
                    return header_doc[field]
            else:
                if field[0:6] == 'count_':
                    return 0
                else:
                    return None

        def get_resource(uid):
            resources = get_header_field('resources')
            for resource in resources:
                if resource['uid'] == uid:
                    return resource
            raise ValueError(f"Could not find Resource with uid={uid}")

        def lookup_resource_for_datum(datum_id):
            """ This method is likely very slow. """
            resources = [resource['uid']
                         for resource in get_header_field('resources')]
            datum_page = self.catalog._db.datum.find_one(
                    {'$and': [{'resource': {'$in': resources}},
                              {'datum_id': datum_id}]},
                    {'_id': False})
            for datum in event_model.unpack_datum_page(datum_page):
                if datum['datum_id'] == datum_id:
                    return datum['resource']
            raise ValueError(f"Could not find Datum with datum_id={datum_id}")

        def get_run_start():
            return run_start_doc

        def get_event_count(descriptor_uid):
            return get_header_field(f'count_{descriptor_uid}')

        entry_metadata = {'start': get_header_field('start'),
                          'stop': get_header_field('stop')}

        args = dict(
            get_run_start=get_run_start,
            get_run_stop=partial(get_header_field, 'stop'),
            get_event_descriptors=partial(
                            get_header_field, 'descriptors'),
            get_event_pages=self.catalog._get_event_pages,
            get_event_count=get_event_count,
            get_resource=get_resource,
            lookup_resource_for_datum=lookup_resource_for_datum,
            get_datum_pages=self.catalog._get_datum_pages,
            filler=self.catalog.filler)
        return intake.catalog.local.LocalCatalogEntry(
            name=run_start_doc['uid'],
            description={},  # TODO
            driver='databroker.core.BlueskyRun',
            direct_access='forbid',  # ???
            args=args,
            cache=None,  # ???
            parameters=[],
            metadata=entry_metadata,
            catalog_dir=None,
            getenv=True,
            getshell=True,
            catalog=self.catalog)

    def __iter__(self):
        cursor = self.catalog._db.header.find(
                    self.catalog._query,
                    {'start.uid': True, '_id': False},
                    sort=[('start.time', pymongo.DESCENDING)])

        for doc in cursor:
            yield doc['start'][0]['uid']

    def __getitem__(self, name):
        # If this came from a client, we might be getting '-1'.
        try:
            N = int(name)
        except ValueError:
            query = {'$and': [self.catalog._query, {'uid': name}]}
            header_doc = self.catalog._db.header.find_one(query)
            if header_doc is None:
                regex_query = {
                    '$and': [self.catalog._query,
                             {'start.uid': {'$regex': f'{name}.*'}}]}
                matches = list(
                    self.catalog._db.header.find(regex_query).limit(10))
                if not matches:
                    raise KeyError(name)
                elif len(matches) == 1:
                    header_doc, = matches
                else:
                    match_list = '\n'.join(doc['uid'] for doc in matches)
                    raise ValueError(
                        f"Multiple matches to partial uid {name!r}. "
                        f"Up to 10 listed here:\n"
                        f"{match_list}")
        else:
            if N < 0:
                # Interpret negative N as "the Nth from last entry".
                query = self.catalog._query
                cursor = (self.catalog._db.header.find(query)
                          .sort('start.time', pymongo.DESCENDING)
                          .skip(-N - 1)
                          .limit(1))
                try:
                    header_doc, = cursor
                except ValueError:
                    raise IndexError(
                        f"Catalog only contains {len(self.catalog)} "
                        f"runs.")
            else:
                # Interpret positive N as
                # "most recent entry with scan_id == N".
                query = {'$and': [self.catalog._query, {'start.scan_id': N}]}
                cursor = (self.catalog._db.header.find(query)
                          .sort('start.time', pymongo.DESCENDING)
                          .limit(1))
                try:
                    header_doc, = cursor
                except ValueError:
                    raise KeyError(f"No run with scan_id={N}")
        if header_doc is None:
            raise KeyError(name)
        return self._doc_to_entry(header_doc['start'][0])

    def __contains__(self, key):
        # Avoid iterating through all entries.
        try:
            self[key]
        except KeyError:
            return False
        else:
            return True

    def __len__(self):
        return len(self.catalog)

class BlueskyMongoCatalog(Broker):
    def __init__(self, datastore_db, *, handler_registry=None, root_map=None,
                 query=None, **kwargs):
        """
        This Catalog is backed by a MongoDB with an embedded data model.

        This embedded data model has three collections: header, event, datum.
        The header collection includes start, stop, descriptor, and resource
        documents. The event_pages are stored in the event colleciton, and
        datum_pages are stored in the datum collection.

        Parameters
        ----------
        datastore_db : pymongo.database.Database or string
            Must be a Database or a URI string that includes a database name.
        handler_registry : dict, optional
            Maps each asset spec to a handler class or a string specifying the
            module name and class name, as in (for example)
            ``{'SOME_SPEC': 'module.submodule.class_name'}``.
        root_map : dict, optional
            Maps resource root paths to different paths.
        query : dict, optional
            MongoDB query. Used internally by the ``search()`` method.
        **kwargs :
            Additional keyword arguments are passed through to the base class,
            Catalog.
        """
        name = 'bluesky-mongo-embedded-catalog'  # noqa

        if isinstance(datastore_db, str):
            self._db = _get_database(datastore_db)
        else:
            self._db = datastore_db

        self._query = query or {}

        if handler_registry is None:
            handler_registry = {}
        parsed_handler_registry = parse_handler_registry(handler_registry)
        self.filler = event_model.Filler(
            parsed_handler_registry, root_map=root_map, inplace=True)
        super().__init__(**kwargs)

    def _get_event_pages(self, descriptor_uid, skip=0, limit=None):
        if limit is None:
            limit = maxsize

        page_cursor = self._db.event.find(
                            {'$and': [
                                {'descriptor': descriptor_uid},
                                {'last_index': {'$gte': skip}},
                                {'first_index': {'$lte': skip + limit}}]},
                            {'_id': False},
                            sort=[('last_index', pymongo.ASCENDING)])

        return page_cursor

    def _get_datum_pages(self, resource_uid, skip=0, limit=None):
        if limit is None:
            limit = maxsize

        page_cursor = self._db.datum.find(
                            {'$and': [
                                {'resource': resource_uid},
                                {'last_index': {'$gte': skip}},
                                {'first_index': {'$lte': skip + limit}}]},
                            {'_id': False},
                            sort=[('last_index', pymongo.ASCENDING)])

        return page_cursor

    def _make_entries_container(self):
        return _Entries(self)

    def _close(self):
        self._client.close()

    def __len__(self):
        return self._db.header.count_documents(self._query)

    def search(self, query):
        """
        Return a new Catalog with a subset of the entries in this Catalog.

        Parameters
        ----------
        query : dict
            MongoDB query.
        """
        if query:
            query = {f"start.{key}": val for key, val in query.items()}
        if self._query:
            query = {'$and': [self._query, query]}
        cat = type(self)(
            datastore_db=self._db,
            query=query,
            handler_registry=self.filler.handler_registry,
            root_map=self.filler.root_map,
            name='search results',
            getenv=self.getenv,
            getshell=self.getshell,
            auth=self.auth,
            metadata=(self.metadata or {}).copy(),
            storage_options=self.storage_options)
        return cat

def _get_database(uri):
    client = pymongo.MongoClient(uri)
    try:
        # Called with no args, get_database() returns the database
        # specified in the client's uri --- or raises if there was none.
        # There is no public method for checking this in advance, so we
        # just catch the error.
        return client.get_database()
    except pymongo.errors.ConfigurationError as err:
        raise ValueError(
            f"Invalid client: {client} "
            f"Did you forget to include a database?") from err
<|endoftext|>


<commit_before>"""
spaceshooter.py
Author: <your name here>
Credit: <list sources used, if any>

Assignment:
Write and submit a program that implements the spacewar game:
https://github.com/HHS-IntroProgramming/Spacewar
"""
"""
tutorial4.py
by E. Dennison
"""
from ggame import App, RectangleAsset, ImageAsset, Sprite, LineStyle, Color, Frame

SCREEN_WIDTH = 640
SCREEN_HEIGHT = 480

class SpaceShip(Sprite):
    """
    Animated space ship
    """
    asset = ImageAsset("images/four_spaceship_by_albertov_with_thrust.png",
        Frame(227,0,292-227,125), 4, 'vertical')

    def __init__(self, position):
        super().__init__(SpaceShip.asset, position)
        self.vx = 1
        self.vy = 1
        self.vr = 0.01
        self.thrust = 0
        self.thrustframe = 1
        SpaceGame.listenKeyEvent("keydown", "space", self.thrustOn)
        SpaceGame.listenKeyEvent("keyup", "space", self.thrustOff)
        self.fxcenter = self.fycenter = 0.5

    def step(self):
        self.x += self.vx
        self.y += self.vy
        self.rotation += self.vr
        if self.thrust == 1:
            self.setImage(self.thrustframe)
            self.thrustframe += 1
            if self.thrustframe == 4:
                self.thrustframe = 1
        else:
            self.setImage(0)

    def thrustOn(self, event):
        self.thrust = 1

    def thrustOff(self, event):
        self.thrust = 0

class SpaceGame(App):
    """
    Tutorial4 space game example.
    """
    def __init__(self, width, height):
        super().__init__(width, height)
        black = Color(0, 1)
        noline = LineStyle(0, black)
        bg_asset = RectangleAsset(width, height, noline, black)
        bg = Sprite(bg_asset, (0,0))
        SpaceShip((100,100))
        SpaceShip((150,150))
        SpaceShip((200,50))

    def step(self):
        for ship in self.getSpritesbyClass(SpaceShip):
            ship.step()

myapp = SpaceGame(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run()<commit_msg>Updated from Brython Server: 3/11/2016 2:27:16 PM<commit_after>"""
spaceshooter.py
Author: <your name here>
Credit: <list sources used, if any>

Assignment:
Write and submit a program that implements the spacewar game:
https://github.com/HHS-IntroProgramming/Spacewar
"""
"""
tutorial4.py
by E. Dennison
"""
from ggame import App, RectangleAsset, ImageAsset, Sprite, LineStyle, Color, Frame

SCREEN_WIDTH = 640
SCREEN_HEIGHT = 480

class SpaceShip(Sprite):
    """
    Animated space ship
    """
    asset = ImageAsset("images/four_spaceship_by_albertov_with_thrust.png",
        Frame(227,0,292-227,125), 4, 'vertical')

    def __init__(self, position):
        super().__init__(SpaceShip.asset, position)
        self.vx = 0
        self.vy = 0
        self.vr = 0.0
        self.thrust = 0
        self.thrustframe = 1
        SpaceGame.listenKeyEvent("keydown", "space", self.thrustOn)
        SpaceGame.listenKeyEvent("keyup", "space", self.thrustOff)
        self.fxcenter = self.fycenter = 0.5

    def step(self):
        self.x += self.vx
        self.y += self.vy
        self.rotation += self.vr
        if self.thrust == 1:
            self.setImage(self.thrustframe)
            self.thrustframe += 1
            if self.thrustframe == 4:
                self.thrustframe = 1
        else:
            self.setImage(0)

    def thrustOn(self, event):
        self.thrust = 1

    def thrustOff(self, event):
        self.thrust = 0

class SpaceGame(App):
    """
    Tutorial4 space game example.
    """
    def __init__(self, width, height):
        super().__init__(width, height)
        black = Color(0, 1)
        noline = LineStyle(0, black)
        bg_asset = RectangleAsset(width, height, noline, black)
        bg = Sprite(bg_asset, (0,0))
        SpaceShip((100,100))
        SpaceShip((150,150))
        SpaceShip((200,50))

    def step(self):
        for ship in self.getSpritesbyClass(SpaceShip):
            ship.step()

myapp = SpaceGame(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run()<|endoftext|>


<commit_before>"""
functions to access the data dictionary in a clearer way
"""

import os
import toolz as tz
from bcbio.utils import file_exists
from bcbio.log import logger
import sys

LOOKUPS = {
    "num_cores": {"keys": ['config', 'algorithm', 'num_cores'],
                  "default": 1},
    "gtf_file": {"keys": ['genome_resources', 'rnaseq', 'transcripts'],
                 "checker": file_exists},
    "work_dir": {"keys": ['dirs', 'work']},
    "sam_ref": {"keys": ["sam_ref"]},
    "lane": {"keys": ["rgnames", "lane"]},
    "cores": {"keys": ["config", "algorithm", "num_cores"], "default": 1},
    "sample_name": {"keys": ['rgnames', 'sample']},
    "strandedness": {"keys": ['config', 'algorithm', 'strandedness'],
                     "default": "unstranded"},
    "work_bam": {"keys": ["work_bam"]},
    "count_file": {"keys": ["count_file"]},
    "combined_counts": {"keys": ["combined_counts"]},
    "annotated_combined_counts": {"keys": ["annotated_combined_counts"]},
    "ref_file": {"keys": ["reference", "fasta", "base"]},
    "dexseq_gff": {"keys": ['genome_resources', 'rnaseq', 'dexseq']},
    "combined_fpkm": {"keys": ['combined_fpkm']},
    "combined_fpkm_isoform": {"keys": ['combined_fpkm_isoform']},
    "fusion_mode": {"keys": ['config', 'algorithm', 'fusion_mode']},
    "dexseq_counts": {"keys": ['dexseq_counts']},
    "description": {"keys": ['description']},
    "aligner": {"keys": ['config', 'algorithm', 'aligner']},
    "quality_format": {"keys": ['config', 'algorithm', 'quality_format'],
                       "default": "standard"},
    "adapters": {"keys": ['config', 'algorithm', 'adapters'],
                 "default": []},
    "qsig_file": {"keys": ['genome_resources', 'variation', 'qsignature'],
                  "checker": file_exists},
    "mixup_check": {"keys": ["config", "algorithm", "mixup_check"],
                    "default": False},
    "cufflinks_dir": {"keys": ['cufflinks_dir']},
    "rsem": {"keys": ["config", "algorithm", "rsem"], "default": False},
    "transcriptome_bam": {"keys": ["transcriptome_bam"]},
    "fpkm_isoform": {"keys": ["fpkm_isoform"]},
    "fpkm": {"keys": ["fpkm"]},
    "assembled_gtf": {"keys": ["assembled_gtf"]},
    "assemble_transcripts": {"keys": ["config", "algorithm", "assemble_transcripts"],
                             "default": False},
    "oncofuse_file": {"keys": ["oncofuse_file"]}
}

def get_dexseq_gff(config, default=None):
    """
    some older versions of the genomes have the DEXseq gff file as
    gff instead of gff3, so this handles that by looking for either one
    """
    dexseq_gff  = tz.get_in(tz.get_in(['dexseq_gff', 'keys'], LOOKUPS, {}),
                            config, None)
    if not dexseq_gff:
        return None
    base_dir = os.path.dirname(get_gtf_file(config))
    base, _ = os.path.splitext(dexseq_gff)
    gff_file = os.path.join(base_dir, base + ".gff")
    if file_exists(gff_file):
        return gff_file
    gtf_file = os.path.join(base_dir, base + ".gff3")
    if file_exists(gtf_file):
        return gtf_file
    else:
        return None

def getter(keys, global_default=None):
    def lookup(config, default=None):
        default = global_default if not default else default
        return tz.get_in(keys, config, default)
    return lookup

def setter(keys, checker):
    def update(config, value):
        if checker and not checker(value):
            logger.error("%s fails check %s." % (value, checker))
            sys.exit(1)
        return tz.update_in(config, keys, lambda x: value, default=value)
    return update

"""
generate the getter and setter functions but don't override any explicitly
defined
"""
_g = globals()
for k, v in LOOKUPS.items():
    keys = v['keys']
    getter_fn = 'get_' + k
    if getter_fn not in _g:
        _g["get_" + k] = getter(keys, v.get('default', None))
    setter_fn = 'set_' + k
    if setter_fn not in _g:
        _g["set_" + k] = setter(keys, v.get('checker', None))
<commit_msg>Add data dictionary iterator.<commit_after>"""
functions to access the data dictionary in a clearer way
"""

import os
import toolz as tz
from bcbio.utils import file_exists
from bcbio.log import logger
import sys

LOOKUPS = {
    "num_cores": {"keys": ['config', 'algorithm', 'num_cores'],
                  "default": 1},
    "gtf_file": {"keys": ['genome_resources', 'rnaseq', 'transcripts'],
                 "checker": file_exists},
    "work_dir": {"keys": ['dirs', 'work']},
    "sam_ref": {"keys": ["sam_ref"]},
    "lane": {"keys": ["rgnames", "lane"]},
    "cores": {"keys": ["config", "algorithm", "num_cores"], "default": 1},
    "sample_name": {"keys": ['rgnames', 'sample']},
    "strandedness": {"keys": ['config', 'algorithm', 'strandedness'],
                     "default": "unstranded"},
    "work_bam": {"keys": ["work_bam"]},
    "count_file": {"keys": ["count_file"]},
    "combined_counts": {"keys": ["combined_counts"]},
    "annotated_combined_counts": {"keys": ["annotated_combined_counts"]},
    "ref_file": {"keys": ["reference", "fasta", "base"]},
    "dexseq_gff": {"keys": ['genome_resources', 'rnaseq', 'dexseq']},
    "combined_fpkm": {"keys": ['combined_fpkm']},
    "combined_fpkm_isoform": {"keys": ['combined_fpkm_isoform']},
    "fusion_mode": {"keys": ['config', 'algorithm', 'fusion_mode']},
    "dexseq_counts": {"keys": ['dexseq_counts']},
    "description": {"keys": ['description']},
    "aligner": {"keys": ['config', 'algorithm', 'aligner']},
    "quality_format": {"keys": ['config', 'algorithm', 'quality_format'],
                       "default": "standard"},
    "adapters": {"keys": ['config', 'algorithm', 'adapters'],
                 "default": []},
    "qsig_file": {"keys": ['genome_resources', 'variation', 'qsignature'],
                  "checker": file_exists},
    "mixup_check": {"keys": ["config", "algorithm", "mixup_check"],
                    "default": False},
    "cufflinks_dir": {"keys": ['cufflinks_dir']},
    "rsem": {"keys": ["config", "algorithm", "rsem"], "default": False},
    "transcriptome_bam": {"keys": ["transcriptome_bam"]},
    "fpkm_isoform": {"keys": ["fpkm_isoform"]},
    "fpkm": {"keys": ["fpkm"]},
    "assembled_gtf": {"keys": ["assembled_gtf"]},
    "assemble_transcripts": {"keys": ["config", "algorithm", "assemble_transcripts"],
                             "default": False},
    "oncofuse_file": {"keys": ["oncofuse_file"]}
}

def get_dexseq_gff(config, default=None):
    """
    some older versions of the genomes have the DEXseq gff file as
    gff instead of gff3, so this handles that by looking for either one
    """
    dexseq_gff  = tz.get_in(tz.get_in(['dexseq_gff', 'keys'], LOOKUPS, {}),
                            config, None)
    if not dexseq_gff:
        return None
    base_dir = os.path.dirname(get_gtf_file(config))
    base, _ = os.path.splitext(dexseq_gff)
    gff_file = os.path.join(base_dir, base + ".gff")
    if file_exists(gff_file):
        return gff_file
    gtf_file = os.path.join(base_dir, base + ".gff3")
    if file_exists(gtf_file):
        return gtf_file
    else:
        return None

def getter(keys, global_default=None):
    def lookup(config, default=None):
        default = global_default if not default else default
        return tz.get_in(keys, config, default)
    return lookup

def setter(keys, checker):
    def update(config, value):
        if checker and not checker(value):
            logger.error("%s fails check %s." % (value, checker))
            sys.exit(1)
        return tz.update_in(config, keys, lambda x: value, default=value)
    return update

"""
generate the getter and setter functions but don't override any explicitly
defined
"""
_g = globals()
for k, v in LOOKUPS.items():
    keys = v['keys']
    getter_fn = 'get_' + k
    if getter_fn not in _g:
        _g["get_" + k] = getter(keys, v.get('default', None))
    setter_fn = 'set_' + k
    if setter_fn not in _g:
        _g["set_" + k] = setter(keys, v.get('checker', None))

def sample_data_iterator(samples):
    """
    for a list of samples, return the data dictionary of each sample
    """
    for sample in samples:
        yield sample[0]
<|endoftext|>


<commit_before>##
#    Copyright (C) 2013 Jessica Tallon & Matt Molyneaux
#
#    This file is part of Inboxen.
#
#    Inboxen is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    Inboxen is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with Inboxen.  If not, see <http://www.gnu.org/licenses/>.
##
from email import encoders
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from email.mime.image import MIMEImage
from email.mime.audio import MIMEAudio
from email.mime.base import MIMEBase
from datetime import datetime
from sys import version_info
from types import UnicodeType

from pytz import utc
from bs4 import BeautifulSoup
from premailer import Premailer

from django.utils.safestring import mark_safe

from website.helper.user import user_profile, null_user
from inboxen.models import Email, Attachment, Alias, Header

# lxml doesn't seem to like WSGI, but premailer seems ok :s
if version_info[:2] == (2, 7):
    PARSER = "html.parser"
else:
    PARSER = "html5lib"

def make_message(email):
    """ makes a python email.message.Message from our Email object """
    msg = MIMEMultipart("alternative")
    # looks to see if a HTML and plaintext is there
    attachments = []
    for attachment in email.attachments.all():
        if attachment.content_type in ["text/plain", "text/html"]:
            attachments.append(attachment)

    if len(attachments) >= 2:
        # we have multiples ones, we should use MIMEMultipart
        for attachment in attachments:
            try:
                gen_type, specific_type = attachment.content_type.split("/", 1)
            except ValueError:
                gen_type, specific_type = "application", "octet-stream"
            msg.attach(MIMEText(attachment.data, specific_type))
    elif attachments and attachments[0].content_type == "text/html":
        part = MIMEText(attachments[0].data, "html", "utf-8")
        msg.attach(part)
    elif attachments and attachments[0].content_type == "text/plain":
        part = MIMEText(attachments[0].data, "plain", "utf-8")
        msg.attach(part)
    else:
        # oh dear, set the body as nothing then
        part = MIMEText('', 'plain', "utf-8")
        msg.attach(part)

    # okay now deal with other attachments
    for attachment in email.attachments.all():
        if attachment in attachments:
            continue # we've already handled it
        # right now deal with it
        try:
            gen_type, specific_type = attachment.content_type.split("/", 1)
        except (AttributeError, IndexError):
            gen_type, specific_type = "application", "octet-stream" # generic
        if gen_type == "audio":
            attach = MIMEAudio(attachment.data, specific_type)
        elif gen_type == "image":
            attach = MIMEImage(attachment.data, specific_type)
        elif gen_type == "text":
            attach = MIMEText(attachment.data, specific_type, "utf-8")
        else:
            attach = MIMEBase(gen_type, specific_type)
            attach.set_payload(attachment.data)
            encoders.encode_base64(attach)

        attach.add_header("Content-Disposition", "attachment", filename=attachment.content_disposition)
        msg.attach(attach)

    # now add the headers
    for header in email.headers.all():
        msg[header.name] = header.data

    if email.body:
        msg["body"] = email.body

    return msg

def clean_html(email):
    """Condense style sheets into style attributes so it doesn't mess with
    Inboxen styles. Also, remove bad tags like <script>
    """

    if type(email) != UnicodeType:
        email = unicode(email, "utf-8")

    # premailer uses lxml, assuming it will accept any old crap
    # and no pretty printing! (we do that later)
    try:
        email = Premailer(email).transform(False)
    except Exception:
        pass
    email = BeautifulSoup(email, PARSER)

    # this doesn't filter out everything
    # TODO: whitelist tags and attributes
    elements = []
    blacklist = ['script','link','head']
    rename_to_div = ['html','body']
    elements.extend(blacklist)
    elements.extend(rename_to_div)

    for elem in email.findAll(elements):
        if elem.name in blacklist:
            elem.extract()
        elif elem.name in rename_to_div:
            elem.name = "div"

    email = email.prettify()

    return mark_safe(email)

def send_email(alias, sender, subject=None, body="", attachments=[]):
    """ Sends an email to an internal alias

    Expects an alias object
    """

    email = Email(
        user=alias.user,
        inbox=alias,
        recieved_date=datetime.now(utc)
    )

    if body:
        email.body = body

    email.save()

    sender = Header(
        name="From",
        data=sender,
    )

    sender.save()

    email.headers.add(sender)

    if subject:
        subject = Header(
            name="Subject",
            data=subject
        )

        subject.save()

        email.headers.add(subject)

    for attachment in attachments:
        email.attachments.add(attachment)

    email.save()

def get_email(user, email_id, preference=None, read=False):
    """ Gets an email based on user preferences and id of the email """
    # does the user want HTML emails?
    # 0 - don't ever give HTML
    # 1 - prefer plain text but if not HTML
    # 2 - prefer HTML
    if None == preference:
        html_preference = user_profile(user).html_preference
    else:
        html_preference = int(preference)

    email_id = int(email_id, 16)

    # quick fix, allows staff to read any message, not just their own
    # not sure if we really want that - M
    if user.is_staff:
        email = Email.objects.get(id=email_id)
    else:
        email = Email.objects.get(id=email_id, user=user)

    message = {
        "date":email.recieved_date
    }

    if read:
        email.read = True
        email.save()

    plain_attachments = email.attachments.filter(content_type="text/plain")
    html_attachments = email.attachments.filter(content_type="text/html")

    message["inbox"] = email.inbox

    if email.body and (html_preference < 2 or not html_attachments.exists()):
        # I think we can give them this?
        # I hope noone sets HTML in the email.body
        message["body"] = email.body
        message["attachments"] = email.attachments.all()
        message["plain"] = True

    if preference < 2 and plain_attachments.exists():
        body = plain_attachments[0]
        message["body"] = body
        message["attachments"] = email.attachments.all()
        message["plain"] = True

    if preference == 0 and html_attachments.exists():
        # we have an html but we don't wanna see
        # we'll strip the tags out...
        message["body"] = ""
        message["attachments"] = email.attachments.all()
        message["plain"] = True

    if html_attachments.exists():
        message["body"] = html_attachments[0].data
        message["attachments"] = email.attachments.all()
        message["plain"] = False

    # handle headers
    for header in email.headers.all():
        if header.name.lower() == "subject":
            message["subject"] = header.data
        elif header.name.lower() == "from":
            message["from"] = header.data

    # ensure subject has been set
    if "subject" not in message:
        message["subject"] = "(No Subject)"

    return message
<commit_msg>Fixed #94<commit_after>##
#    Copyright (C) 2013 Jessica Tallon & Matt Molyneaux
#
#    This file is part of Inboxen.
#
#    Inboxen is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    Inboxen is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with Inboxen.  If not, see <http://www.gnu.org/licenses/>.
##
from email import encoders
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from email.mime.image import MIMEImage
from email.mime.audio import MIMEAudio
from email.mime.base import MIMEBase
from datetime import datetime
from sys import version_info
from types import UnicodeType

from pytz import utc
from bs4 import BeautifulSoup
from premailer import Premailer

from django.utils.safestring import mark_safe

from website.helper.user import user_profile, null_user
from inboxen.models import Email, Attachment, Alias, Header

# lxml doesn't seem to like WSGI, but premailer seems ok :s
if version_info[:2] == (2, 7):
    PARSER = "html.parser"
else:
    PARSER = "html5lib"

def make_message(email):
    """ makes a python email.message.Message from our Email object """
    msg = MIMEMultipart("alternative")
    # looks to see if a HTML and plaintext is there
    attachments = []
    for attachment in email.attachments.all():
        if attachment.content_type in ["text/plain", "text/html"]:
            attachments.append(attachment)

    if len(attachments) >= 2:
        # we have multiples ones, we should use MIMEMultipart
        for attachment in attachments:
            try:
                gen_type, specific_type = attachment.content_type.split("/", 1)
            except ValueError:
                gen_type, specific_type = "application", "octet-stream"
            msg.attach(MIMEText(attachment.data, specific_type))
    elif attachments and attachments[0].content_type == "text/html":
        part = MIMEText(attachments[0].data, "html", "utf-8")
        msg.attach(part)
    elif attachments and attachments[0].content_type == "text/plain":
        part = MIMEText(attachments[0].data, "plain", "utf-8")
        msg.attach(part)
    else:
        # oh dear, set the body as nothing then
        part = MIMEText('', 'plain', "utf-8")
        msg.attach(part)

    # okay now deal with other attachments
    for attachment in email.attachments.all():
        if attachment in attachments:
            continue # we've already handled it
        # right now deal with it
        try:
            gen_type, specific_type = attachment.content_type.split("/", 1)
        except (AttributeError, IndexError):
            gen_type, specific_type = "application", "octet-stream" # generic
        if gen_type == "audio":
            attach = MIMEAudio(attachment.data, specific_type)
        elif gen_type == "image":
            attach = MIMEImage(attachment.data, specific_type)
        elif gen_type == "text":
            attach = MIMEText(attachment.data, specific_type, "utf-8")
        else:
            attach = MIMEBase(gen_type, specific_type)
            attach.set_payload(attachment.data)
            encoders.encode_base64(attach)

        attach.add_header("Content-Disposition", "attachment", filename=attachment.content_disposition)
        msg.attach(attach)

    # now add the headers
    for header in email.headers.all():
        msg[header.name] = header.data

    if email.body:
        msg["body"] = email.body

    return msg

def clean_html(email):
    """Condense style sheets into style attributes so it doesn't mess with
    Inboxen styles. Also, remove bad tags like <script>
    """

    if type(email) != UnicodeType:
        email = unicode(email, "utf-8")

    # premailer uses lxml, assuming it will accept any old crap
    # and no pretty printing! (we do that later)
    try:
        email = Premailer(email).transform(False)
    except Exception:
        pass
    email = BeautifulSoup(email, PARSER)

    # this doesn't filter out everything
    # TODO: whitelist tags and attributes
    elements = []
    blacklist = ['script','link','head']
    rename_to_div = ['html','body']
    elements.extend(blacklist)
    elements.extend(rename_to_div)

    for elem in email.findAll(elements):
        if elem.name in blacklist:
            elem.extract()
        elif elem.name in rename_to_div:
            elem.name = "div"

    email = email.prettify()

    return mark_safe(email)

def send_email(alias, sender, subject=None, body="", attachments=[]):
    """ Sends an email to an internal alias

    Expects an alias object
    """

    email = Email(
        user=alias.user,
        inbox=alias,
        recieved_date=datetime.now(utc)
    )

    if body:
        email.body = body

    email.save()

    sender = Header(
        name="From",
        data=sender,
    )

    sender.save()

    email.headers.add(sender)

    if subject:
        subject = Header(
            name="Subject",
            data=subject
        )

        subject.save()

        email.headers.add(subject)

    for attachment in attachments:
        email.attachments.add(attachment)

    email.save()

def get_email(user, email_id, preference=None, read=False):
    """ Gets an email based on user preferences and id of the email """
    # does the user want HTML emails?
    # 0 - don't ever give HTML
    # 1 - prefer plain text but if not HTML
    # 2 - prefer HTML
    if None == preference:
        html_preference = user_profile(user).html_preference
    else:
        html_preference = int(preference)

    email_id = int(email_id, 16)

    # quick fix, allows staff to read any message, not just their own
    # not sure if we really want that - M
    if user.is_staff:
        email = Email.objects.get(id=email_id)
    else:
        email = Email.objects.get(id=email_id, user=user)

    message = {
        "date":email.recieved_date
    }

    if read:
        email.read = True
        email.save()

    # grab first plaintext and html attachments, if they exist
    try:
        plain_attachment = email.attachments.filter(content_type="text/plain")
        plain_attachemt = plain_attachment[0].data
    except (KeyError, Attachment.DoesNotExist):
        plain_attachemt = None

    try:
        html_attachment = email.attachments.filter(content_type="text/html")
        html_attachment =html_attachment[0].data
    except (KeyError, Attachment.DoesNotExist):
        html_attachment = None

    # grab body content-type, if it has one
    try:
        body_content_type = email.headers.get(name="content-type")
    except Header.DoesNotExist:
        body_content_type = None

    message["inbox"] = email.inbox
    message["attachments"] = email.attachments.all()

    # 0 - never give html
    if html_preference == 0:
        message["plain"] = True
        if not email.body and plain_attachemt:
            message["body"] = plain_attachment
        else:
            message["body"] = email.body

    # 1 - prefer plain
    elif html_preference == 1:
        # prefer whatever is in the body
        if email.body:
            message["body"] = email.body
            message["plain"] = (body_content_type == "text/plain")
        # first plain mime-part
        elif plain_attachment:
            message["body"] = plain_attachment
            message["plain"] = True
        # first html mime-part
        elif html_attachment:
            message["body"] = html_attachment
            message["plain"] = False

    # 2 - prefer HTML
    elif html_preference == 2:
        # prefer html in the body
        if body_content_type == "text/html":
            message["body"] = email.body
            message["plain"] = False
        # first html mime-part
        elif html_attachment:
            message["body"] = html_attachment
            message["plain"] = False
        # fallback to plaintext, prefering body over mime-part
        elif not email.body:
            message["body"] = plain_attachment
            message["plain"] = True
        else:
            message["body"] = email.body
            message["plain"] = True

    # handle headers
    for header in email.headers.all():
        if header.name.lower() == "subject":
            message["subject"] = header.data
        elif header.name.lower() == "from":
            message["from"] = header.data

    # ensure subject has been set
    if "subject" not in message:
        message["subject"] = "(No Subject)"

    return message
<|endoftext|>


<commit_before># Replace the ^] follow tag command with one that expands a word without
# modifying iskeyword.  Setting iskeyword is a lot simpler, but it also
# changes the behavoiour of a lot of vim commands, such as w, which is too
# confusing for me.  This can also do clever things like allow ' in
# identifiers, but strip quotes from haddock references like 'A.B'.
#
# This assumes you are using fast-tags --fully_qualified, since it adds
# full module qualification to qualified tags, after renaming them based on
# the 'import ... as XYZ' lines in this file.
#
# I install it into ~/.vim/py and enable like this:
#
# In global .vimrc:
#
# if has('python')
#     py import sys, os, vim
#     py sys.path.insert(0, os.environ['HOME'] + '/.vim/py')
# endif
#
# In your haskell specific vimrc:
#
# if has('python')
#     py import qualified_tag
#     nnoremap <buffer> <silent> <c-]> :py qualified_tag.tag_word(vim)<cr>
# endif
#
# Or python3 and py3 if you have those.
#
# If you use 'filetype', you can do:
#
# autocmd FileType haskell nnoremap ...

from __future__ import print_function
import sys, re

def tag_word(vim):
    word = get_word(vim)
    qual_to_module = get_qualified_imports(vim.current.buffer)
    tag = guess_tag(qual_to_module, word)
    # If I can't find a qualified target, try it without the qualification.
    # It might be a re-export from another module.
    found = has_target(vim, tag)
    if not found and '.' in tag:
        tag = tag.split('.')[-1]
        found = has_target(vim, tag)
    if found:
        vim.command('tag ' + tag)
    else:
        # I'd use echoerr, but that causes a big messy python traceback.
        vim.command('echohl ErrorMsg | echo %r | echohl None' %
            ('tag not found: ' + tag,))

def has_target(vim, tag):
    return bool(vim.eval('taglist(%r)' % ('^' + tag + '$',)))

def get_word(vim):
    (row, col) = vim.current.window.cursor
    line = vim.current.buffer[row-1]
    return expand_word(line, col)

def expand_word(line, col):
    # This is broken for unicode, but vim gives the col index in bytes, and
    # python gives no way to convert byte index to unicode index, short of
    # decoding myself.  And even if I did, python uses UTF16 so it would
    # still be wrong for unicode above U+FFFF.
    start = col
    while start > 0 and is_keyword(line[start-1]):
        start -= 1
    end = col
    while end < len(line) and is_keyword(line[end]):
        end += 1
    word = line[start:end]
    # Strip punctuation in case it's in a sentence in a comment.
    word = word.strip('.')
    if word and (word[0] == word[-1] == "'" or word[0] == word[-1] == '"'):
        # Make tags on haddock references like 'A.B' work.
        return word[1:-1]
    else:
        return word

def is_keyword(c):
    return c.isalnum() or c in "'._"

def guess_tag(qual_to_module, word):
    if '.' not in word:
        return word
    components = word.split('.')
    qual = '.'.join(components[:-1])
    if qual in qual_to_module:
        return qual_to_module[qual] + '.' + components[-1]
    else:
        return word

def get_qualified_imports(lines):
    imports = get_imports(lines)
    qual_to_module = {}
    for m in re.finditer(r'\b([A-Za-z0-9.]+)\s+as\s+([A-Za-z00-9.]+)$',
            imports, re.MULTILINE):
        qual_to_module[m.group(2)] = m.group(1)
    return qual_to_module

def get_imports(lines):
    imports = []
    in_imports = False
    for line in lines:
        if line.startswith('import'):
            in_imports = True
        if in_imports:
            if finished_imports(line):
                break
            imports.append(line)
    return '\n'.join(imports)

def finished_imports(line):
    # Hacky heuristic to see if I'm past the import block.
    # If it starts with non-space but not 'import', then it's probably
    # not an import or import continuation.
    return line and not line.startswith('import') and not line[0].isspace()

def test():
    fn, tag = sys.argv[1:]
    lines = open(fn).read().split('\n')
    qual_to_module = get_qualified_imports(lines)
    print(qual_to_module)
    print(guess_tag(qual_to_module, tag))

if __name__ == '__main__':
    test()
<commit_msg>handle comments between imports in qualified_tag<commit_after># Replace the ^] follow tag command with one that expands a word without
# modifying iskeyword.  Setting iskeyword is a lot simpler, but it also
# changes the behavoiour of a lot of vim commands, such as w, which is too
# confusing for me.  This can also do clever things like allow ' in
# identifiers, but strip quotes from haddock references like 'A.B'.
#
# This assumes you are using fast-tags --fully_qualified, since it adds
# full module qualification to qualified tags, after renaming them based on
# the 'import ... as XYZ' lines in this file.
#
# I install it into ~/.vim/py and enable like this:
#
# In global .vimrc:
#
# if has('python')
#     py import sys, os, vim
#     py sys.path.insert(0, os.environ['HOME'] + '/.vim/py')
# endif
#
# In your haskell specific vimrc:
#
# if has('python')
#     py import qualified_tag
#     nnoremap <buffer> <silent> <c-]> :py qualified_tag.tag_word(vim)<cr>
# endif
#
# Or python3 and py3 if you have those.
#
# If you use 'filetype', you can do:
#
# autocmd FileType haskell nnoremap ...

from __future__ import print_function
import sys, re

def tag_word(vim):
    word = get_word(vim)
    qual_to_module = get_qualified_imports(vim.current.buffer)
    tag = guess_tag(qual_to_module, word)
    # If I can't find a qualified target, try it without the qualification.
    # It might be a re-export from another module.
    found = has_target(vim, tag)
    if not found and '.' in tag:
        tag = tag.split('.')[-1]
        found = has_target(vim, tag)
    if found:
        vim.command('tag ' + tag)
    else:
        # I'd use echoerr, but that causes a big messy python traceback.
        vim.command('echohl ErrorMsg | echo %r | echohl None' %
            ('tag not found: ' + tag,))

def has_target(vim, tag):
    return bool(vim.eval('taglist(%r)' % ('^' + tag + '$',)))

def get_word(vim):
    (row, col) = vim.current.window.cursor
    line = vim.current.buffer[row-1]
    return expand_word(line, col)

def expand_word(line, col):
    # This is broken for unicode, but vim gives the col index in bytes, and
    # python gives no way to convert byte index to unicode index, short of
    # decoding myself.  And even if I did, python uses UTF16 so it would
    # still be wrong for unicode above U+FFFF.
    start = col
    while start > 0 and is_keyword(line[start-1]):
        start -= 1
    end = col
    while end < len(line) and is_keyword(line[end]):
        end += 1
    word = line[start:end]
    # Strip punctuation in case it's in a sentence in a comment.
    word = word.strip('.')
    if word and (word[0] == word[-1] == "'" or word[0] == word[-1] == '"'):
        # Make tags on haddock references like 'A.B' work.
        return word[1:-1]
    else:
        return word

def is_keyword(c):
    return c.isalnum() or c in "'._"

def guess_tag(qual_to_module, word):
    if '.' not in word:
        return word
    components = word.split('.')
    qual = '.'.join(components[:-1])
    if qual in qual_to_module:
        return qual_to_module[qual] + '.' + components[-1]
    else:
        return word

def get_qualified_imports(lines):
    imports = get_imports(lines)
    qual_to_module = {}
    for m in re.finditer(r'\b([A-Za-z0-9.]+)\s+as\s+([A-Za-z00-9.]+)$',
            imports, re.MULTILINE):
        qual_to_module[m.group(2)] = m.group(1)
    return qual_to_module

def get_imports(lines):
    imports = []
    in_imports = False
    for line in lines:
        if line.startswith('import'):
            in_imports = True
        if in_imports:
            if finished_imports(line):
                break
            imports.append(line)
    return '\n'.join(imports)

def finished_imports(line):
    # Hacky heuristic to see if I'm past the import block.
    # If it starts with non-space but not 'import', then it's probably
    # not an import or import continuation.
    return line and not line.startswith('import') and not line[0].isspace() and not line.startswith('--')

def test():
    fn, tag = sys.argv[1:]
    lines = open(fn).read().split('\n')
    qual_to_module = get_qualified_imports(lines)
    print(qual_to_module)
    print(guess_tag(qual_to_module, tag))

if __name__ == '__main__':
    test()
<|endoftext|>