<commit_before>#!/usr/bin/env python

######################################################
#
# howdoi - instant coding answers via the command line
# written by Benjamin Gleitzman (gleitz@mit.edu)
# inspired by Rich Jones (rich@anomos.info)
#
######################################################

from __future__ import print_function
import gc
gc.disable()  # noqa: E402
import argparse
import os
import appdirs
import re
from cachelib import FileSystemCache, NullCache
import requests
import requests_cache
import sys
from . import __version__

from pygments import highlight
from pygments.lexers import guess_lexer, get_lexer_by_name
from pygments.formatters.terminal import TerminalFormatter
from pygments.util import ClassNotFound

from pyquery import PyQuery as pq
from requests.exceptions import ConnectionError
from requests.exceptions import SSLError

requests_cache.install_cache('test_requests_cache')

# Handle imports for Python 2 and 3
if sys.version < '3':
    import codecs
    from urllib import quote as url_quote
    from urllib import getproxies
    from urlparse import urlparse, parse_qs

    # Handling Unicode: http://stackoverflow.com/a/6633040/305414
    def u(x):
        return codecs.unicode_escape_decode(x)[0]
else:
    from urllib.request import getproxies
    from urllib.parse import quote as url_quote, urlparse, parse_qs

    def u(x):
        return x

# rudimentary standardized 3-level log output
def _print_err(x): print("[ERROR] " + x)
_print_ok = print  # noqa: E305
def _print_dbg(x): print("[DEBUG] " + x)  # noqa: E302

if os.getenv('HOWDOI_DISABLE_SSL'):  # Set http instead of https
    SCHEME = 'http://'
    VERIFY_SSL_CERTIFICATE = False
else:
    SCHEME = 'https://'
    VERIFY_SSL_CERTIFICATE = True

SUPPORTED_SEARCH_ENGINES = ('google', 'bing', 'duckduckgo')

URL = os.getenv('HOWDOI_URL') or 'stackoverflow.com'

USER_AGENTS = ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:11.0) Gecko/20100101 Firefox/11.0',
               'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:22.0) Gecko/20100 101 Firefox/22.0',
               'Mozilla/5.0 (Windows NT 6.1; rv:11.0) Gecko/20100101 Firefox/11.0',
               ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_4) AppleWebKit/536.5 (KHTML, like Gecko) '
                'Chrome/19.0.1084.46 Safari/536.5'),
               ('Mozilla/5.0 (Windows; Windows NT 6.1) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.46'
                'Safari/536.5'), )
SEARCH_URLS = {
    'bing': SCHEME + 'www.bing.com/search?q=site:{0}%20{1}&hl=en',
    'google': SCHEME + 'www.google.com/search?q=site:{0}%20{1}&hl=en',
    'duckduckgo': SCHEME + 'duckduckgo.com/?q=site:{0}%20{1}&t=hj&ia=web'
}

BLOCK_INDICATORS = (
    'form id="captcha-form"',
    'This page appears when Google automatically detects requests coming from your computer '
    'network which appear to be in violation of the Terms of Service'
)

BLOCKED_QUESTION_FRAGMENTS = (
    'webcache.googleusercontent.com',
)

STAR_HEADER = u('\u2605')
ANSWER_HEADER = u('{2}  Answer from {0} {2}\n{1}')
NO_ANSWER_MSG = '< no answer given >'

CACHE_EMPTY_VAL = "NULL"
CACHE_DIR = appdirs.user_cache_dir('howdoi')
CACHE_ENTRY_MAX = 128

if os.getenv('HOWDOI_DISABLE_CACHE'):
    cache = NullCache()  # works like an always empty cache
else:
    cache = FileSystemCache(CACHE_DIR, CACHE_ENTRY_MAX, default_timeout=0)

howdoi_session = requests.session()

class BlockError(RuntimeError):
    pass

def _random_int(width):
    bres = os.urandom(width)
    if sys.version < '3':
        ires = int(bres.encode('hex'), 16)
    else:
        ires = int.from_bytes(bres, 'little')

    return ires

def _random_choice(seq):
    return seq[_random_int(1) % len(seq)]

def get_proxies():
    proxies = getproxies()
    filtered_proxies = {}
    for key, value in proxies.items():
        if key.startswith('http'):
            if not value.startswith('http'):
                filtered_proxies[key] = 'http://%s' % value
            else:
                filtered_proxies[key] = value
    return filtered_proxies

def _get_result(url):
    try:
        return howdoi_session.get(url, headers={'User-Agent': _random_choice(USER_AGENTS)},
                                  proxies=get_proxies(),
                                  verify=VERIFY_SSL_CERTIFICATE).text
    except requests.exceptions.SSLError as e:
        _print_err('Encountered an SSL Error. Try using HTTP instead of '
                   'HTTPS by setting the environment variable "HOWDOI_DISABLE_SSL".\n')
        raise e

def _add_links_to_text(element):
    hyperlinks = element.find('a')

    for hyperlink in hyperlinks:
        pquery_object = pq(hyperlink)
        href = hyperlink.attrib['href']
        copy = pquery_object.text()
        if (copy == href):
            replacement = copy
        else:
            replacement = "[{0}]({1})".format(copy, href)
        pquery_object.replace_with(replacement)

def get_text(element):
    ''' return inner text in pyquery element '''
    _add_links_to_text(element)
    try:
        return element.text(squash_space=False)
    except TypeError:
        return element.text()

def _extract_links_from_bing(html):
    html.remove_namespaces()
    return [a.attrib['href'] for a in html('.b_algo')('h2')('a')]

def _extract_links_from_google(html):
    return [a.attrib['href'] for a in html('.l')] or \
        [a.attrib['href'] for a in html('.r')('a')]

def _extract_links_from_duckduckgo(html):
    html.remove_namespaces()
    links_anchors = html.find('a.result__a')
    results = []
    for anchor in links_anchors:
        link = anchor.attrib['href']
        url_obj = urlparse(link)
        parsed_url = parse_qs(url_obj.query).get('uddg', '')
        if parsed_url:
            results.append(parsed_url[0])
    return results

def _extract_links(html, search_engine):
    if search_engine == 'bing':
        return _extract_links_from_bing(html)
    if search_engine == 'duckduckgo':
        return _extract_links_from_duckduckgo(html)
    return _extract_links_from_google(html)

def _get_search_url(search_engine):
    return SEARCH_URLS.get(search_engine, SEARCH_URLS['google'])

def _is_blocked(page):
    for indicator in BLOCK_INDICATORS:
        if page.find(indicator) != -1:
            return True

    return False

def _get_links(query):
    search_engine = os.getenv('HOWDOI_SEARCH_ENGINE', 'google')
    search_url = _get_search_url(search_engine)

    result = _get_result(search_url.format(URL, url_quote(query)))
    if _is_blocked(result):
        _print_err('Unable to find an answer because the search engine temporarily blocked the request. '
                   'Please wait a few minutes or select a different search engine.')
        raise BlockError("Temporary block by search engine")

    html = pq(result)
    return _extract_links(html, search_engine)

def get_link_at_pos(links, position):
    if not links:
        return False

    if len(links) >= position:
        link = links[position - 1]
    else:
        link = links[-1]
    return link

def _format_output(code, args):
    if not args['color']:
        return code
    lexer = None

    # try to find a lexer using the StackOverflow tags
    # or the query arguments
    for keyword in args['query'].split() + args['tags']:
        try:
            lexer = get_lexer_by_name(keyword)
            break
        except ClassNotFound:
            pass

    # no lexer found above, use the guesser
    if not lexer:
        try:
            lexer = guess_lexer(code)
        except ClassNotFound:
            return code

    return highlight(code,
                     lexer,
                     TerminalFormatter(bg='dark'))

def _is_question(link):
    for fragment in BLOCKED_QUESTION_FRAGMENTS:
        if fragment in link:
            return False
    return re.search(r'questions/\d+/', link)

def _get_questions(links):
    return [link for link in links if _is_question(link)]

def _get_answer(args, links):
    link = get_link_at_pos(links, args['pos'])
    if not link:
        return False
    if args.get('link'):
        return link

    cache_key = link
    page = cache.get(link)
    if not page:
        page = _get_result(link + '?answertab=votes')
        cache.set(cache_key, page)

    html = pq(page)

    first_answer = html('.answer').eq(0)

    instructions = first_answer.find('pre') or first_answer.find('code')
    args['tags'] = [t.text for t in html('.post-tag')]

    if not instructions and not args['all']:
        text = get_text(first_answer.find('.post-text').eq(0))
    elif args['all']:
        texts = []
        for html_tag in first_answer.items('.post-text > *'):
            current_text = get_text(html_tag)
            if current_text:
                if html_tag[0].tag in ['pre', 'code']:
                    texts.append(_format_output(current_text, args))
                else:
                    texts.append(current_text)
        text = '\n'.join(texts)
    else:
        text = _format_output(get_text(instructions.eq(0)), args)
    if text is None:
        text = NO_ANSWER_MSG
    text = text.strip()
    return text

def _get_links_with_cache(query):
    cache_key = query + "-links"
    res = cache.get(cache_key)
    if res:
        if res == CACHE_EMPTY_VAL:
            res = False
        return res

    links = _get_links(query)
    if not links:
        cache.set(cache_key, CACHE_EMPTY_VAL)

    question_links = _get_questions(links)
    cache.set(cache_key, question_links or CACHE_EMPTY_VAL)

    return question_links

def _get_instructions(args):
    question_links = _get_links_with_cache(args['query'])
    if not question_links:
        return False

    only_hyperlinks = args.get('link')
    star_headers = (args['num_answers'] > 1 or args['all'])

    answers = []
    initial_position = args['pos']
    spliter_length = 80
    answer_spliter = '\n' + '=' * spliter_length + '\n\n'

    for answer_number in range(args['num_answers']):
        current_position = answer_number + initial_position
        args['pos'] = current_position
        link = get_link_at_pos(question_links, current_position)
        answer = _get_answer(args, question_links)
        if not answer:
            continue
        if not only_hyperlinks:
            answer = format_answer(link, answer, star_headers)
        answer += '\n'
        answers.append(answer)
    return answer_spliter.join(answers)

def format_answer(link, answer, star_headers):
    if star_headers:
        return ANSWER_HEADER.format(link, answer, STAR_HEADER)
    return answer

def _clear_cache():
    global cache
    if not cache:
        cache = FileSystemCache(CACHE_DIR, CACHE_ENTRY_MAX, 0)

    return cache.clear()

def howdoi(raw_query):
    args = raw_query
    if type(raw_query) is str:  # you can pass either a raw or a parsed query
        parser = get_parser()
        args = vars(parser.parse_args(raw_query.split(' ')))

    args['query'] = ' '.join(args['query']).replace('?', '')
    cache_key = str(args)

    res = cache.get(cache_key)
    if res:
        return res

    try:
        res = _get_instructions(args)
        if not res:
            res = 'Sorry, couldn\'t find any help with that topic\n'
        cache.set(cache_key, res)

        return res
    except (ConnectionError, SSLError):
        return 'Failed to establish network connection\n'

def get_parser():
    parser = argparse.ArgumentParser(description='instant coding answers via the command line')
    parser.add_argument('query', metavar='QUERY', type=str, nargs='*',
                        help='the question to answer')
    parser.add_argument('-p', '--pos', help='select answer in specified position (default: 1)', default=1, type=int)
    parser.add_argument('-a', '--all', help='display the full text of the answer',
                        action='store_true')
    parser.add_argument('-l', '--link', help='display only the answer link',
                        action='store_true')
    parser.add_argument('-c', '--color', help='enable colorized output',
                        action='store_true')
    parser.add_argument('-n', '--num-answers', help='number of answers to return', default=1, type=int)
    parser.add_argument('-C', '--clear-cache', help='clear the cache',
                        action='store_true')
    parser.add_argument('-v', '--version', help='displays the current version of howdoi',
                        action='store_true')
    parser.add_argument('-e', '--engine', help='change search engine for this query only (google, bing, duckduckgo)',
                        dest='search_engine', nargs="?", default='google')
    return parser

def command_line_runner():
    parser = get_parser()
    args = vars(parser.parse_args())

    if args['version']:
        _print_ok(__version__)
        return

    if args['clear_cache']:
        if _clear_cache():
            _print_ok('Cache cleared successfully')
        else:
            _print_err('Clearing cache failed')

        return

    if not args['query']:
        parser.print_help()
        return

    if os.getenv('HOWDOI_COLORIZE'):
        args['color'] = True

    if not args['search_engine'] in SUPPORTED_SEARCH_ENGINES:
        _print_err('Unsupported engine.\nThe supported engines are: %s' % ', '.join(SUPPORTED_SEARCH_ENGINES))
        return
    elif args['search_engine'] != 'google':
        os.environ['HOWDOI_SEARCH_ENGINE'] = args['search_engine']

    utf8_result = howdoi(args).encode('utf-8', 'ignore')
    if sys.version < '3':
        print(utf8_result)
    else:
        # Write UTF-8 to stdout: https://stackoverflow.com/a/3603160
        sys.stdout.buffer.write(utf8_result)
    # close the session to release connection
    howdoi_session.close()

if __name__ == '__main__':
    command_line_runner()
<commit_msg>uninstalled requests cache<commit_after>#!/usr/bin/env python

######################################################
#
# howdoi - instant coding answers via the command line
# written by Benjamin Gleitzman (gleitz@mit.edu)
# inspired by Rich Jones (rich@anomos.info)
#
######################################################

from __future__ import print_function
import gc
gc.disable()  # noqa: E402
import argparse
import os
import appdirs
import re
from cachelib import FileSystemCache, NullCache
import requests
import sys
from . import __version__

from pygments import highlight
from pygments.lexers import guess_lexer, get_lexer_by_name
from pygments.formatters.terminal import TerminalFormatter
from pygments.util import ClassNotFound

from pyquery import PyQuery as pq
from requests.exceptions import ConnectionError
from requests.exceptions import SSLError

# Handle imports for Python 2 and 3
if sys.version < '3':
    import codecs
    from urllib import quote as url_quote
    from urllib import getproxies
    from urlparse import urlparse, parse_qs

    # Handling Unicode: http://stackoverflow.com/a/6633040/305414
    def u(x):
        return codecs.unicode_escape_decode(x)[0]
else:
    from urllib.request import getproxies
    from urllib.parse import quote as url_quote, urlparse, parse_qs

    def u(x):
        return x

# rudimentary standardized 3-level log output
def _print_err(x): print("[ERROR] " + x)
_print_ok = print  # noqa: E305
def _print_dbg(x): print("[DEBUG] " + x)  # noqa: E302

if os.getenv('HOWDOI_DISABLE_SSL'):  # Set http instead of https
    SCHEME = 'http://'
    VERIFY_SSL_CERTIFICATE = False
else:
    SCHEME = 'https://'
    VERIFY_SSL_CERTIFICATE = True

SUPPORTED_SEARCH_ENGINES = ('google', 'bing', 'duckduckgo')

URL = os.getenv('HOWDOI_URL') or 'stackoverflow.com'

USER_AGENTS = ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:11.0) Gecko/20100101 Firefox/11.0',
               'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:22.0) Gecko/20100 101 Firefox/22.0',
               'Mozilla/5.0 (Windows NT 6.1; rv:11.0) Gecko/20100101 Firefox/11.0',
               ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_4) AppleWebKit/536.5 (KHTML, like Gecko) '
                'Chrome/19.0.1084.46 Safari/536.5'),
               ('Mozilla/5.0 (Windows; Windows NT 6.1) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.46'
                'Safari/536.5'), )
SEARCH_URLS = {
    'bing': SCHEME + 'www.bing.com/search?q=site:{0}%20{1}&hl=en',
    'google': SCHEME + 'www.google.com/search?q=site:{0}%20{1}&hl=en',
    'duckduckgo': SCHEME + 'duckduckgo.com/?q=site:{0}%20{1}&t=hj&ia=web'
}

BLOCK_INDICATORS = (
    'form id="captcha-form"',
    'This page appears when Google automatically detects requests coming from your computer '
    'network which appear to be in violation of the Terms of Service'
)

BLOCKED_QUESTION_FRAGMENTS = (
    'webcache.googleusercontent.com',
)

STAR_HEADER = u('\u2605')
ANSWER_HEADER = u('{2}  Answer from {0} {2}\n{1}')
NO_ANSWER_MSG = '< no answer given >'

CACHE_EMPTY_VAL = "NULL"
CACHE_DIR = appdirs.user_cache_dir('howdoi')
CACHE_ENTRY_MAX = 128

if os.getenv('HOWDOI_DISABLE_CACHE'):
    cache = NullCache()  # works like an always empty cache
else:
    cache = FileSystemCache(CACHE_DIR, CACHE_ENTRY_MAX, default_timeout=0)

howdoi_session = requests.session()

class BlockError(RuntimeError):
    pass

def _random_int(width):
    bres = os.urandom(width)
    if sys.version < '3':
        ires = int(bres.encode('hex'), 16)
    else:
        ires = int.from_bytes(bres, 'little')

    return ires

def _random_choice(seq):
    return seq[_random_int(1) % len(seq)]

def get_proxies():
    proxies = getproxies()
    filtered_proxies = {}
    for key, value in proxies.items():
        if key.startswith('http'):
            if not value.startswith('http'):
                filtered_proxies[key] = 'http://%s' % value
            else:
                filtered_proxies[key] = value
    return filtered_proxies

def _get_result(url):
    try:
        return howdoi_session.get(url, headers={'User-Agent': _random_choice(USER_AGENTS)},
                                  proxies=get_proxies(),
                                  verify=VERIFY_SSL_CERTIFICATE).text
    except requests.exceptions.SSLError as e:
        _print_err('Encountered an SSL Error. Try using HTTP instead of '
                   'HTTPS by setting the environment variable "HOWDOI_DISABLE_SSL".\n')
        raise e

def _add_links_to_text(element):
    hyperlinks = element.find('a')

    for hyperlink in hyperlinks:
        pquery_object = pq(hyperlink)
        href = hyperlink.attrib['href']
        copy = pquery_object.text()
        if (copy == href):
            replacement = copy
        else:
            replacement = "[{0}]({1})".format(copy, href)
        pquery_object.replace_with(replacement)

def get_text(element):
    ''' return inner text in pyquery element '''
    _add_links_to_text(element)
    try:
        return element.text(squash_space=False)
    except TypeError:
        return element.text()

def _extract_links_from_bing(html):
    html.remove_namespaces()
    return [a.attrib['href'] for a in html('.b_algo')('h2')('a')]

def _extract_links_from_google(html):
    return [a.attrib['href'] for a in html('.l')] or \
        [a.attrib['href'] for a in html('.r')('a')]

def _extract_links_from_duckduckgo(html):
    html.remove_namespaces()
    links_anchors = html.find('a.result__a')
    results = []
    for anchor in links_anchors:
        link = anchor.attrib['href']
        url_obj = urlparse(link)
        parsed_url = parse_qs(url_obj.query).get('uddg', '')
        if parsed_url:
            results.append(parsed_url[0])
    return results

def _extract_links(html, search_engine):
    if search_engine == 'bing':
        return _extract_links_from_bing(html)
    if search_engine == 'duckduckgo':
        return _extract_links_from_duckduckgo(html)
    return _extract_links_from_google(html)

def _get_search_url(search_engine):
    return SEARCH_URLS.get(search_engine, SEARCH_URLS['google'])

def _is_blocked(page):
    for indicator in BLOCK_INDICATORS:
        if page.find(indicator) != -1:
            return True

    return False

def _get_links(query):
    search_engine = os.getenv('HOWDOI_SEARCH_ENGINE', 'google')
    search_url = _get_search_url(search_engine)

    result = _get_result(search_url.format(URL, url_quote(query)))
    if _is_blocked(result):
        _print_err('Unable to find an answer because the search engine temporarily blocked the request. '
                   'Please wait a few minutes or select a different search engine.')
        raise BlockError("Temporary block by search engine")

    html = pq(result)
    return _extract_links(html, search_engine)

def get_link_at_pos(links, position):
    if not links:
        return False

    if len(links) >= position:
        link = links[position - 1]
    else:
        link = links[-1]
    return link

def _format_output(code, args):
    if not args['color']:
        return code
    lexer = None

    # try to find a lexer using the StackOverflow tags
    # or the query arguments
    for keyword in args['query'].split() + args['tags']:
        try:
            lexer = get_lexer_by_name(keyword)
            break
        except ClassNotFound:
            pass

    # no lexer found above, use the guesser
    if not lexer:
        try:
            lexer = guess_lexer(code)
        except ClassNotFound:
            return code

    return highlight(code,
                     lexer,
                     TerminalFormatter(bg='dark'))

def _is_question(link):
    for fragment in BLOCKED_QUESTION_FRAGMENTS:
        if fragment in link:
            return False
    return re.search(r'questions/\d+/', link)

def _get_questions(links):
    return [link for link in links if _is_question(link)]

def _get_answer(args, links):
    link = get_link_at_pos(links, args['pos'])
    if not link:
        return False
    if args.get('link'):
        return link

    cache_key = link
    page = cache.get(link)
    if not page:
        page = _get_result(link + '?answertab=votes')
        cache.set(cache_key, page)

    html = pq(page)

    first_answer = html('.answer').eq(0)

    instructions = first_answer.find('pre') or first_answer.find('code')
    args['tags'] = [t.text for t in html('.post-tag')]

    if not instructions and not args['all']:
        text = get_text(first_answer.find('.post-text').eq(0))
    elif args['all']:
        texts = []
        for html_tag in first_answer.items('.post-text > *'):
            current_text = get_text(html_tag)
            if current_text:
                if html_tag[0].tag in ['pre', 'code']:
                    texts.append(_format_output(current_text, args))
                else:
                    texts.append(current_text)
        text = '\n'.join(texts)
    else:
        text = _format_output(get_text(instructions.eq(0)), args)
    if text is None:
        text = NO_ANSWER_MSG
    text = text.strip()
    return text

def _get_links_with_cache(query):
    cache_key = query + "-links"
    res = cache.get(cache_key)
    if res:
        if res == CACHE_EMPTY_VAL:
            res = False
        return res

    links = _get_links(query)
    if not links:
        cache.set(cache_key, CACHE_EMPTY_VAL)

    question_links = _get_questions(links)
    cache.set(cache_key, question_links or CACHE_EMPTY_VAL)

    return question_links

def _get_instructions(args):
    question_links = _get_links_with_cache(args['query'])
    if not question_links:
        return False

    only_hyperlinks = args.get('link')
    star_headers = (args['num_answers'] > 1 or args['all'])

    answers = []
    initial_position = args['pos']
    spliter_length = 80
    answer_spliter = '\n' + '=' * spliter_length + '\n\n'

    for answer_number in range(args['num_answers']):
        current_position = answer_number + initial_position
        args['pos'] = current_position
        link = get_link_at_pos(question_links, current_position)
        answer = _get_answer(args, question_links)
        if not answer:
            continue
        if not only_hyperlinks:
            answer = format_answer(link, answer, star_headers)
        answer += '\n'
        answers.append(answer)
    return answer_spliter.join(answers)

def format_answer(link, answer, star_headers):
    if star_headers:
        return ANSWER_HEADER.format(link, answer, STAR_HEADER)
    return answer

def _clear_cache():
    global cache
    if not cache:
        cache = FileSystemCache(CACHE_DIR, CACHE_ENTRY_MAX, 0)

    return cache.clear()

def howdoi(raw_query):
    args = raw_query
    if type(raw_query) is str:  # you can pass either a raw or a parsed query
        parser = get_parser()
        args = vars(parser.parse_args(raw_query.split(' ')))

    args['query'] = ' '.join(args['query']).replace('?', '')
    cache_key = str(args)

    res = cache.get(cache_key)
    if res:
        return res

    try:
        res = _get_instructions(args)
        if not res:
            res = 'Sorry, couldn\'t find any help with that topic\n'
        cache.set(cache_key, res)

        return res
    except (ConnectionError, SSLError):
        return 'Failed to establish network connection\n'

def get_parser():
    parser = argparse.ArgumentParser(description='instant coding answers via the command line')
    parser.add_argument('query', metavar='QUERY', type=str, nargs='*',
                        help='the question to answer')
    parser.add_argument('-p', '--pos', help='select answer in specified position (default: 1)', default=1, type=int)
    parser.add_argument('-a', '--all', help='display the full text of the answer',
                        action='store_true')
    parser.add_argument('-l', '--link', help='display only the answer link',
                        action='store_true')
    parser.add_argument('-c', '--color', help='enable colorized output',
                        action='store_true')
    parser.add_argument('-n', '--num-answers', help='number of answers to return', default=1, type=int)
    parser.add_argument('-C', '--clear-cache', help='clear the cache',
                        action='store_true')
    parser.add_argument('-v', '--version', help='displays the current version of howdoi',
                        action='store_true')
    parser.add_argument('-e', '--engine', help='change search engine for this query only (google, bing, duckduckgo)',
                        dest='search_engine', nargs="?", default='google')
    return parser

def command_line_runner():
    parser = get_parser()
    args = vars(parser.parse_args())

    if args['version']:
        _print_ok(__version__)
        return

    if args['clear_cache']:
        if _clear_cache():
            _print_ok('Cache cleared successfully')
        else:
            _print_err('Clearing cache failed')

        return

    if not args['query']:
        parser.print_help()
        return

    if os.getenv('HOWDOI_COLORIZE'):
        args['color'] = True

    if not args['search_engine'] in SUPPORTED_SEARCH_ENGINES:
        _print_err('Unsupported engine.\nThe supported engines are: %s' % ', '.join(SUPPORTED_SEARCH_ENGINES))
        return
    elif args['search_engine'] != 'google':
        os.environ['HOWDOI_SEARCH_ENGINE'] = args['search_engine']

    utf8_result = howdoi(args).encode('utf-8', 'ignore')
    if sys.version < '3':
        print(utf8_result)
    else:
        # Write UTF-8 to stdout: https://stackoverflow.com/a/3603160
        sys.stdout.buffer.write(utf8_result)
    # close the session to release connection
    howdoi_session.close()

if __name__ == '__main__':
    command_line_runner()
<|endoftext|>


<commit_before>#!/usr/bin/env python

######################################################
#
# howdoi - instant coding answers via the command line
# written by Benjamin Gleitzman (gleitz@mit.edu)
# inspired by Rich Jones (rich@anomos.info)
#
######################################################

import argparse
import re
import requests

try:
    from urllib.parse import quote as url_quote
except ImportError:
    from urllib import quote as url_quote

from pygments import highlight
from pygments.lexers import guess_lexer, get_lexer_by_name
from pygments.formatters import TerminalFormatter
from pygments.util import ClassNotFound

from pyquery import PyQuery as pq
from requests.exceptions import ConnectionError

SEARCH_URL = 'https://www.google.com/search?q=site:stackoverflow.com%20{0}'
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17'
ANSWER_HEADER = u'--- Answer {0} ---\n{1}'
NO_ANSWER_MSG = '< no answer given >'

def get_result(url):
    return requests.get(url, headers={'User-Agent': USER_AGENT}).text

def is_question(link):
    return re.search('questions/\d+/', link)

def get_links(query):
    url = SEARCH_URL.format(url_quote(query))
    result = get_result(url)
    html = pq(result)
    return [a.attrib['href'] for a in html('.l')]

def get_link_at_pos(links, pos):
    for link in links:
        if is_question(link):
            if pos == 1:
                break
            else:
                pos = pos - 1
                continue
    return link

def format_output(code, args):
    if not args['color']:
        return code
    lexer = None

    # try to find a lexer using the StackOverflow tags
    # or the query arguments
    for keyword in args['query'].split() + args['tags']:
        try:
            lexer = get_lexer_by_name(keyword)
            break
        except ClassNotFound:
            pass

    # no lexer found above, use the guesser
    if not lexer:
        lexer = guess_lexer(code)

    return highlight(
        code,
        lexer,
        TerminalFormatter(bg='dark')
    )

def get_answer(args, links):
    link = get_link_at_pos(links, args['pos'])
    if args.get('link'):
        return link
    page = get_result(link + '?answertab=votes')
    html = pq(page)

    first_answer = html('.answer').eq(0)
    instructions = first_answer.find('pre') or first_answer.find('code')
    args['tags'] = [t.text for t in html('.post-tag')]

    if not instructions and not args['all']:
        text = first_answer.find('.post-text').eq(0).text()
    elif args['all']:
        texts = []
        for html_tag in first_answer.items('.post-text > *'):
            current_text = html_tag.text()
            if current_text:
                if html_tag[0].tag in ['pre', 'code']:
                    texts.append(format_output(current_text, args))
                else:
                    texts.append(current_text)
        texts.append(u'\n---\nAnswer from {0}'.format(link))
        text = u'\n'.join(texts)
    else:
        text = format_output(instructions.eq(0).text(), args)
    if text is None:
        text = NO_ANSWER_MSG
    text = text.strip()
    return text

def get_instructions(args):
    links = get_links(args['query'])
    if not links:
        return ''
    answers = []
    append_header = args['num_answers'] > 1
    initial_position = args['pos']
    for answer_number in range(args['num_answers']):
        current_position = answer_number + initial_position
        args['pos'] = current_position
        answer = get_answer(args, links)
        if not answer:
            continue
        if append_header:
            answer = ANSWER_HEADER.format(current_position, answer)
        answer = answer + '\n'
        answers.append(answer)
    return u'\n'.join(answers)

def howdoi(args):
    args['query'] = ' '.join(args['query']).replace('?', '')
    try:
        return get_instructions(args) or 'Sorry, couldn\'t find any help with that topic\n'
    except ConnectionError:
        return 'Failed to establish network connection\n'

def get_parser():
    parser = argparse.ArgumentParser(description='instant coding answers via the command line')
    parser.add_argument('query', metavar='QUERY', type=str, nargs='+',
                        help='the question to answer')
    parser.add_argument('-p','--pos', help='select answer in specified position (default: 1)', default=1, type=int)
    parser.add_argument('-a','--all', help='display the full text of the answer',
                        action='store_true')
    parser.add_argument('-l','--link', help='display only the answer link',
                        action='store_true')
    parser.add_argument('-c', '--color', help='enable colorized output',
                        action='store_true')
    parser.add_argument('-n','--num-answers', help='number of answers to return', default=1, type=int)
    return parser

def command_line_runner():
    parser = get_parser()
    args = vars(parser.parse_args())
    print(howdoi(args))

if __name__ == '__main__':
    command_line_runner()
<commit_msg>fixes for Python3 unicode_literals<commit_after>#!/usr/bin/env python

######################################################
#
# howdoi - instant coding answers via the command line
# written by Benjamin Gleitzman (gleitz@mit.edu)
# inspired by Rich Jones (rich@anomos.info)
#
######################################################

from __future__ import unicode_literals

import argparse
import re
import requests

try:
    from urllib.parse import quote as url_quote
except ImportError:
    from urllib import quote as url_quote

from pygments import highlight
from pygments.lexers import guess_lexer, get_lexer_by_name
from pygments.formatters import TerminalFormatter
from pygments.util import ClassNotFound

from pyquery import PyQuery as pq
from requests.exceptions import ConnectionError

SEARCH_URL = 'https://www.google.com/search?q=site:stackoverflow.com%20{0}'
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17'
ANSWER_HEADER = '--- Answer {0} ---\n{1}'
NO_ANSWER_MSG = '< no answer given >'

def get_result(url):
    return requests.get(url, headers={'User-Agent': USER_AGENT}).text

def is_question(link):
    return re.search('questions/\d+/', link)

def get_links(query):
    url = SEARCH_URL.format(url_quote(query))
    result = get_result(url)
    html = pq(result)
    return [a.attrib['href'] for a in html('.l')]

def get_link_at_pos(links, pos):
    for link in links:
        if is_question(link):
            if pos == 1:
                break
            else:
                pos = pos - 1
                continue
    return link

def format_output(code, args):
    if not args['color']:
        return code
    lexer = None

    # try to find a lexer using the StackOverflow tags
    # or the query arguments
    for keyword in args['query'].split() + args['tags']:
        try:
            lexer = get_lexer_by_name(keyword)
            break
        except ClassNotFound:
            pass

    # no lexer found above, use the guesser
    if not lexer:
        lexer = guess_lexer(code)

    return highlight(
        code,
        lexer,
        TerminalFormatter(bg='dark')
    )

def get_answer(args, links):
    link = get_link_at_pos(links, args['pos'])
    if args.get('link'):
        return link
    page = get_result(link + '?answertab=votes')
    html = pq(page)

    first_answer = html('.answer').eq(0)
    instructions = first_answer.find('pre') or first_answer.find('code')
    args['tags'] = [t.text for t in html('.post-tag')]

    if not instructions and not args['all']:
        text = first_answer.find('.post-text').eq(0).text()
    elif args['all']:
        texts = []
        for html_tag in first_answer.items('.post-text > *'):
            current_text = html_tag.text()
            if current_text:
                if html_tag[0].tag in ['pre', 'code']:
                    texts.append(format_output(current_text, args))
                else:
                    texts.append(current_text)
        texts.append('\n---\nAnswer from {0}'.format(link))
        text = '\n'.join(texts)
    else:
        text = format_output(instructions.eq(0).text(), args)
    if text is None:
        text = NO_ANSWER_MSG
    text = text.strip()
    return text

def get_instructions(args):
    links = get_links(args['query'])
    if not links:
        return ''
    answers = []
    append_header = args['num_answers'] > 1
    initial_position = args['pos']
    for answer_number in range(args['num_answers']):
        current_position = answer_number + initial_position
        args['pos'] = current_position
        answer = get_answer(args, links)
        if not answer:
            continue
        if append_header:
            answer = ANSWER_HEADER.format(current_position, answer)
        answer = answer + '\n'
        answers.append(answer)
    return '\n'.join(answers)

def howdoi(args):
    args['query'] = ' '.join(args['query']).replace('?', '')
    try:
        return get_instructions(args) or 'Sorry, couldn\'t find any help with that topic\n'
    except ConnectionError:
        return 'Failed to establish network connection\n'

def get_parser():
    parser = argparse.ArgumentParser(description='instant coding answers via the command line')
    parser.add_argument('query', metavar='QUERY', type=str, nargs='+',
                        help='the question to answer')
    parser.add_argument('-p','--pos', help='select answer in specified position (default: 1)', default=1, type=int)
    parser.add_argument('-a','--all', help='display the full text of the answer',
                        action='store_true')
    parser.add_argument('-l','--link', help='display only the answer link',
                        action='store_true')
    parser.add_argument('-c', '--color', help='enable colorized output',
                        action='store_true')
    parser.add_argument('-n','--num-answers', help='number of answers to return', default=1, type=int)
    return parser

def command_line_runner():
    parser = get_parser()
    args = vars(parser.parse_args())
    print(howdoi(args))

if __name__ == '__main__':
    command_line_runner()
<|endoftext|>


<commit_before>from urllib.request import urlopen, HTTPError, URLError
import dataset #http://dataset.readthedocs.org/en/latest/freezefile.html
import time, datetime
import plotly.tools as tls
import plotly.plotly as py
from plotly.graph_objs import Figure, Data, Layout, Scatter
import os
import json
import configparser

config = configparser.ConfigParser()
config.read('dmv.cfg')

os.environ['TZ'] = 'US/Pacific'
time.tzset()
dayOfWeek = datetime.date.today().weekday()

#intialize DB
db = dataset.connect('sqlite:///dmv.db')
table = db['dmv']

#initialize vars
#TODO: get rid of these. branch names from branch_data.json, limit targets by geo.
la_branches = [502, 617, 508, 652, 510, 511, 576]
dmvUrl = config['URLS']['waitdata']

f = open('../resource/branch_data.json')
full_offices = json.loads(f.read())
f.close()

traces = []
inited = []

credentials = tls.get_credentials_file()

class Trace():
    def __init__(self, id, name):
        self.name = name
        self.id = id
        self.apptValue = []
        self.noApptValue = []
        self.update = []

    def updateWait(self, apptValue, noApptValue, update):
        self.apptValue.append(apptValue)
        self.noApptValue.append(noApptValue)
        self.update.append(update)

    def getName(self):
        return self.name

    def getApptValue(self):
        return self.apptValue

    def getNoApptValue(self):
        return self.noApptValue

    def getUpdate(self):
        return self.update

    def getId(self):
        return self.id

    def showData(self):
        print(self.name, self.apptValue, self.noApptValue, self.update)

def tomins(hhmm):
    h = hhmm.split(':')[0]
    m = hhmm.split(':')[1]
    return int(h) * 60 + int(m)

def polldmv():
    try:
        # get all wait time data
        data = urlopen(dmvUrl).readlines()[1:]
        dbData = []

        for line in data:
            try:
                dataArray = line.decode("utf-8").rstrip().rsplit(',')
                if dataArray[1].find(':') == 1:
                    dataArray[1] = tomins(dataArray[1])
                if dataArray[2].find(':') == 1:
                    dataArray[2] = tomins(dataArray[2])
                if dataArray[0] not in inited:
                    traces.append(Trace(dataArray[0], str(dataArray[0]))) #TODO: lookup branch name
                    inited.append(dataArray[0])

                #append to global array
                for trace in traces:
                    if dataArray[0] == trace.getName():
                        trace.updateWait(dataArray[1], dataArray[2], datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'))

                #append to local db array
                dbData.append(dataArray)
            except ValueError as e:
                print('ValueError', e)
        return dbData
    except HTTPError as e:
        print('HTTPError', e.code)
    except URLError as e:
        print('URLError', e.args)

def writetodb(data):
    for dataArray in data:
        dataDict = dict(update=datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'),
                        branch=dataArray[0],
                        appt=dataArray[1],
                        nonAppt=dataArray[2])
        table.insert(dataDict)

def plot(traces):
    scatterObjs = []
    #make scatter objects
    for trace in traces:
        if int(trace.getName()) in la_branches:
            scatterObjs.append(Scatter(x=trace.getUpdate(),
                y=trace.getNoApptValue(),
                mode='lines',
                #name=branch_names[trace.getName()]
                name=getOfficeName(trace.getName()) #read these from file. TODO: rename trace.getName?
                ))
    data = Data(scatterObjs)
    layout = Layout(title='Central LA DMV Non-Apointment Wait Times: ' + datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d'))
    fig = Figure(data=data, layout=layout)
    #TODO: trap plotly ConnectionError
    x = py.plot(fig, filename='DMV', auto_open=False)
    print(x)

def getOfficeName(id):
    id = int(id)
    for office in full_offices:
        if office['id'] == id:
            return office['name']

def getOfficeAddress(id):
    id = int(id)
    for office in full_offices:
        if office['id'] == id:
            return office['address']

def getOfficeCoords(id):
    id = int(id)
    for office in full_offices:
        if office['id'] == id:
            return office['coords']

def getOfficeHours(id):
    id = int(id)
    for office in full_offices:
        if office['id'] == id:
            return office['officeHours'].split(',')[dayOfWeek].split('-')

#update pseudo API file
def printTrace(traces):
    a = []
    for trace in traces:
        d = { 'address':getOfficeAddress(trace.getId()),
              'nonAppt':int(trace.getNoApptValue()[-1]),
              'id':int(trace.getId()),
              'name':getOfficeName(trace.getId()),
              'coords':getOfficeCoords(trace.getId()),
              'officeHours':getOfficeHours(trace.getId()) }
        a.append(d)
    with open('../resource/offices.json', 'w') as fp:
        fp.write('waitTime(')
        json.dump(a, fp)
        fp.write(');')
    fp.close()

def main():
    while True:
        dbData = polldmv()
        printTrace(traces)
        print(dbData)
        if dbData:
            writetodb(dbData)
        # todo: refresh data every 5 minutes, plotly every 20
        plot(traces)
        time.sleep(900) #refresh every 15 mins (plotly limit of 50/day)

if __name__=='__main__':
    main()
<commit_msg>update chart names<commit_after>from urllib.request import urlopen, HTTPError, URLError
import dataset #http://dataset.readthedocs.org/en/latest/freezefile.html
import time, datetime
import plotly.tools as tls
import plotly.plotly as py
from plotly.graph_objs import Figure, Data, Layout, Scatter
import os
import json
import configparser

config = configparser.ConfigParser()
config.read('dmv.cfg')

os.environ['TZ'] = 'US/Pacific'
time.tzset()
dayOfWeek = datetime.date.today().weekday()

#intialize DB
db = dataset.connect('sqlite:///dmv.db')
table = db['dmv']

#initialize vars
#TODO: get rid of these. branch names from branch_data.json, limit targets by geo.
la_branches = [502, 617, 508, 652, 510, 511, 576]
dmvUrl = config['URLS']['waitdata']

f = open('../resource/branch_data.json')
full_offices = json.loads(f.read())
f.close()

traces = []
inited = []

credentials = tls.get_credentials_file()

class Trace():
    def __init__(self, id, name):
        self.name = name
        self.id = id
        self.apptValue = []
        self.noApptValue = []
        self.update = []

    def updateWait(self, apptValue, noApptValue, update):
        self.apptValue.append(apptValue)
        self.noApptValue.append(noApptValue)
        self.update.append(update)

    def getName(self):
        return self.name

    def getApptValue(self):
        return self.apptValue

    def getNoApptValue(self):
        return self.noApptValue

    def getUpdate(self):
        return self.update

    def getId(self):
        return self.id

    def showData(self):
        print(self.name, self.apptValue, self.noApptValue, self.update)

def tomins(hhmm):
    h = hhmm.split(':')[0]
    m = hhmm.split(':')[1]
    return int(h) * 60 + int(m)

def polldmv():
    try:
        # get all wait time data
        data = urlopen(dmvUrl).readlines()[1:]
        dbData = []

        for line in data:
            try:
                dataArray = line.decode("utf-8").rstrip().rsplit(',')
                if dataArray[1].find(':') == 1:
                    dataArray[1] = tomins(dataArray[1])
                if dataArray[2].find(':') == 1:
                    dataArray[2] = tomins(dataArray[2])
                if dataArray[0] not in inited:
                    traces.append(Trace(dataArray[0], str(dataArray[0]))) #TODO: lookup branch name
                    inited.append(dataArray[0])

                #append to global array
                for trace in traces:
                    if dataArray[0] == trace.getName():
                        trace.updateWait(dataArray[1], dataArray[2], datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'))

                #append to local db array
                dbData.append(dataArray)
            except ValueError as e:
                print('ValueError', e)
        return dbData
    except HTTPError as e:
        print('HTTPError', e.code)
    except URLError as e:
        print('URLError', e.args)

def writetodb(data):
    for dataArray in data:
        dataDict = dict(update=datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'),
                        branch=dataArray[0],
                        appt=dataArray[1],
                        nonAppt=dataArray[2])
        table.insert(dataDict)

def plot(traces):
    scatterObjs = []
    #make scatter objects
    for trace in traces:
        if int(trace.getName()) in la_branches:
            scatterObjs.append(Scatter(x=trace.getUpdate(),
                y=trace.getNoApptValue(),
                mode='lines',
                #name=branch_names[trace.getName()]
                name=getOfficeName(trace.getName()) #read these from file. TODO: rename trace.getName?
                ))
    data = Data(scatterObjs)
    layout = Layout(title='Central LA DMV Non-Appointment Wait Times: ' + datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d'))
    fig = Figure(data=data, layout=layout)
    #TODO: trap plotly ConnectionError
    x = py.plot(fig, filename='Central LA DMV', auto_open=False)
    print(x)

def getOfficeName(id):
    id = int(id)
    for office in full_offices:
        if office['id'] == id:
            return office['name']

def getOfficeAddress(id):
    id = int(id)
    for office in full_offices:
        if office['id'] == id:
            return office['address']

def getOfficeCoords(id):
    id = int(id)
    for office in full_offices:
        if office['id'] == id:
            return office['coords']

def getOfficeHours(id):
    id = int(id)
    for office in full_offices:
        if office['id'] == id:
            return office['officeHours'].split(',')[dayOfWeek].split('-')

#update pseudo API file
def printTrace(traces):
    a = []
    for trace in traces:
        d = { 'address':getOfficeAddress(trace.getId()),
              'nonAppt':int(trace.getNoApptValue()[-1]),
              'id':int(trace.getId()),
              'name':getOfficeName(trace.getId()),
              'coords':getOfficeCoords(trace.getId()),
              'officeHours':getOfficeHours(trace.getId()) }
        a.append(d)
    with open('../resource/offices.json', 'w') as fp:
        fp.write('waitTime(')
        json.dump(a, fp)
        fp.write(');')
    fp.close()

def main():
    while True:
        dbData = polldmv()
        printTrace(traces)
        print(dbData)
        if dbData:
            writetodb(dbData)
        # todo: refresh data every 5 minutes, plotly every 20
        plot(traces)
        time.sleep(900) #refresh every 15 mins (plotly limit of 50/day)

if __name__=='__main__':
    main()
<|endoftext|>


<commit_before>from django.contrib.admin.views.main import ALL_VAR, EMPTY_CHANGELIST_VALUE
from django.contrib.admin.views.main import ORDER_VAR, ORDER_TYPE_VAR, PAGE_VAR, SEARCH_VAR
from django.contrib.admin.templatetags.admin_list import _boolean_icon
from django.core.exceptions import ObjectDoesNotExist
from django.db import models
from django.utils import dateformat
from django.utils.html import escape, conditional_escape
from django.utils.text import capfirst
from django.utils.safestring import mark_safe
from django.utils.translation import get_date_formats, get_partial_date_formats, ugettext as _
from django.utils.encoding import smart_unicode, smart_str, force_unicode
from django.template import Library
import datetime

register = Library()

def thumbnails_for_result(cl, result, form):
    """
    Basically does the same thing as django's items_for_result, but makes all the
    items fit in one  element instead of separate elements for each field
    """

    first = True
    pk = cl.lookup_opts.pk.attname
    for field_name in cl.list_display:
        row_class = ''
        try:
            f = cl.lookup_opts.get_field(field_name)
        except models.FieldDoesNotExist:
            # For non-field list_display values, the value is either a method,
            # property or returned via a callable.
            try:
                if callable(field_name):
                    attr = field_name
                    value = attr(result)
                elif hasattr(cl.model_admin, field_name) and \
                   not field_name == '__str__' and not field_name == '__unicode__':
                    attr = getattr(cl.model_admin, field_name)
                    value = attr(result)
                else:
                    attr = getattr(result, field_name)
                    if callable(attr):
                        value = attr()
                    else:
                        value = attr
                allow_tags = getattr(attr, 'allow_tags', False)
                boolean = getattr(attr, 'boolean', False)
                if boolean:
                    allow_tags = True
                    result_repr = _boolean_icon(value)
                else:
                    result_repr = smart_unicode(value)
            except (AttributeError, ObjectDoesNotExist):
                result_repr = EMPTY_CHANGELIST_VALUE
            else:
                # Strip HTML tags in the resulting text, except if the
                # function has an "allow_tags" attribute set to True.
                if not allow_tags:
                    result_repr = escape(result_repr)
                else:
                    result_repr = mark_safe(result_repr)
        else:
            field_val = getattr(result, f.attname)

            if isinstance(f.rel, models.ManyToOneRel):
                if field_val is not None:
                    result_repr = escape(getattr(result, f.name))
                else:
                    result_repr = EMPTY_CHANGELIST_VALUE
            # Dates and times are special: They're formatted in a certain way.
            elif isinstance(f, models.DateField) or isinstance(f, models.TimeField):
                if field_val:
                    (date_format, datetime_format, time_format) = get_date_formats()
                    if isinstance(f, models.DateTimeField):
                        result_repr = capfirst(dateformat.format(field_val, datetime_format))
                    elif isinstance(f, models.TimeField):
                        result_repr = capfirst(dateformat.time_format(field_val, time_format))
                    else:
                        result_repr = capfirst(dateformat.format(field_val, date_format))
                else:
                    result_repr = EMPTY_CHANGELIST_VALUE
                row_class = ' class="nowrap"'
            # Booleans are special: We use images.
            elif isinstance(f, models.BooleanField) or isinstance(f, models.NullBooleanField):
                result_repr = _boolean_icon(field_val)
            # DecimalFields are special: Zero-pad the decimals.
            elif isinstance(f, models.DecimalField):
                if field_val is not None:
                    result_repr = ('%%.%sf' % f.decimal_places) % field_val
                else:
                    result_repr = EMPTY_CHANGELIST_VALUE
            # Fields with choices are special: Use the representation
            # of the choice.
            elif f.flatchoices:
                result_repr = dict(f.flatchoices).get(field_val, EMPTY_CHANGELIST_VALUE)
            else:
                result_repr = escape(field_val)
        if force_unicode(result_repr) == '':
            result_repr = mark_safe(' ')
        # If list_display_links not defined, add the link tag to the first field
        if (first and not cl.list_display_links) or field_name in cl.list_display_links:
            table_tag = {True:'th', False:'td'}[first]
            first = False
            url = cl.url_for_result(result)
            # Convert the pk to something that can be used in Javascript.
            # Problem cases are long ints (23L) and non-ASCII strings.
            if cl.to_field:
                attr = str(cl.to_field)
            else:
                attr = pk
            value = result.serializable_value(attr)
            result_id = repr(force_unicode(value))[1:]
            if cl.is_popup and cl.params['pop'] == u'2':
                yield mark_safe(u'%s' % \
                ('#', (cl.is_popup and ' onclick="FileBrowserDialogue.fileSubmit(\'%s\', \'%s\', \'%s\'); return false;"' % (result.media_url or '', result.caption or '', result.get_absolute_url())), conditional_escape(result_repr)))
                #(url, (cl.is_popup and ' onclick="FileBrowserDialogue.fileSubmit(\'%s\'); return false;"' % result.get_absolute_url() or ''), conditional_escape(result_repr)))
            else:
                yield mark_safe(u'%s' % \
                (url, (cl.is_popup and ' onclick="opener.dismissRelatedLookupPopup(window, %s); return false;"' % result_id or ''), conditional_escape(result_repr)))
        else:
            # By default the fields come from ModelAdmin.list_editable, but if we pull
            # the fields out of the form instead of list_editable custom admins
            # can provide fields on a per request basis
            if form and field_name in form.fields:
                bf = form[field_name]
                result_repr = mark_safe(force_unicode(bf.errors) + force_unicode(bf))
            else:
                result_repr = conditional_escape(result_repr)
            yield mark_safe(u'%s' % (result_repr))
    if form:
        yield mark_safe(force_unicode(form[cl.model._meta.pk.name]))

def results(cl):
    if cl.formset:
        for res, form in zip(cl.result_list, cl.formset.forms):
            yield list(thumbnails_for_result(cl, res, form))
    else:
        for res in cl.result_list:
            yield list(thumbnails_for_result(cl, res, None))

def thumbnail_result_list(context, cl):
    if context.has_key('STATIC_URL'):
        static_url = 'STATIC_URL'
    else:
        static_url = 'MEDIA_URL'
    result_list = list(results(cl))
    if len(result_list) % 4:
        empty_spaces = range(4 - (len(result_list) % 4))
    else:
        empty_spaces = []
    return {'cl': cl,
            'results': result_list,
            'empty_spaces': empty_spaces,
            'STATIC_URL': context[static_url]}
thumbnail_result_list = register.inclusion_tag("admin/massmedia/change_list_results.html", takes_context=True)(thumbnail_result_list)
<commit_msg>Added a simple method to escape single quotes for the javascript call<commit_after>from django.contrib.admin.views.main import ALL_VAR, EMPTY_CHANGELIST_VALUE
from django.contrib.admin.views.main import ORDER_VAR, ORDER_TYPE_VAR, PAGE_VAR, SEARCH_VAR
from django.contrib.admin.templatetags.admin_list import _boolean_icon
from django.core.exceptions import ObjectDoesNotExist
from django.db import models
from django.utils import dateformat
from django.utils.html import escape, conditional_escape
from django.utils.text import capfirst
from django.utils.safestring import mark_safe
from django.utils.translation import get_date_formats, get_partial_date_formats, ugettext as _
from django.utils.encoding import smart_unicode, smart_str, force_unicode
from django.template import Library
import datetime

register = Library()

quo_esc = lambda x: x.replace("'", r"\'")

def thumbnails_for_result(cl, result, form):
    """
    Basically does the same thing as django's items_for_result, but makes all the
    items fit in one  element instead of separate elements for each field
    """

    first = True
    pk = cl.lookup_opts.pk.attname
    for field_name in cl.list_display:
        row_class = ''
        try:
            f = cl.lookup_opts.get_field(field_name)
        except models.FieldDoesNotExist:
            # For non-field list_display values, the value is either a method,
            # property or returned via a callable.
            try:
                if callable(field_name):
                    attr = field_name
                    value = attr(result)
                elif hasattr(cl.model_admin, field_name) and \
                   not field_name == '__str__' and not field_name == '__unicode__':
                    attr = getattr(cl.model_admin, field_name)
                    value = attr(result)
                else:
                    attr = getattr(result, field_name)
                    if callable(attr):
                        value = attr()
                    else:
                        value = attr
                allow_tags = getattr(attr, 'allow_tags', False)
                boolean = getattr(attr, 'boolean', False)
                if boolean:
                    allow_tags = True
                    result_repr = _boolean_icon(value)
                else:
                    result_repr = smart_unicode(value)
            except (AttributeError, ObjectDoesNotExist):
                result_repr = EMPTY_CHANGELIST_VALUE
            else:
                # Strip HTML tags in the resulting text, except if the
                # function has an "allow_tags" attribute set to True.
                if not allow_tags:
                    result_repr = escape(result_repr)
                else:
                    result_repr = mark_safe(result_repr)
        else:
            field_val = getattr(result, f.attname)

            if isinstance(f.rel, models.ManyToOneRel):
                if field_val is not None:
                    result_repr = escape(getattr(result, f.name))
                else:
                    result_repr = EMPTY_CHANGELIST_VALUE
            # Dates and times are special: They're formatted in a certain way.
            elif isinstance(f, models.DateField) or isinstance(f, models.TimeField):
                if field_val:
                    (date_format, datetime_format, time_format) = get_date_formats()
                    if isinstance(f, models.DateTimeField):
                        result_repr = capfirst(dateformat.format(field_val, datetime_format))
                    elif isinstance(f, models.TimeField):
                        result_repr = capfirst(dateformat.time_format(field_val, time_format))
                    else:
                        result_repr = capfirst(dateformat.format(field_val, date_format))
                else:
                    result_repr = EMPTY_CHANGELIST_VALUE
                row_class = ' class="nowrap"'
            # Booleans are special: We use images.
            elif isinstance(f, models.BooleanField) or isinstance(f, models.NullBooleanField):
                result_repr = _boolean_icon(field_val)
            # DecimalFields are special: Zero-pad the decimals.
            elif isinstance(f, models.DecimalField):
                if field_val is not None:
                    result_repr = ('%%.%sf' % f.decimal_places) % field_val
                else:
                    result_repr = EMPTY_CHANGELIST_VALUE
            # Fields with choices are special: Use the representation
            # of the choice.
            elif f.flatchoices:
                result_repr = dict(f.flatchoices).get(field_val, EMPTY_CHANGELIST_VALUE)
            else:
                result_repr = escape(field_val)
        if force_unicode(result_repr) == '':
            result_repr = mark_safe(' ')
        # If list_display_links not defined, add the link tag to the first field
        if (first and not cl.list_display_links) or field_name in cl.list_display_links:
            table_tag = {True:'th', False:'td'}[first]
            first = False
            url = cl.url_for_result(result)
            # Convert the pk to something that can be used in Javascript.
            # Problem cases are long ints (23L) and non-ASCII strings.
            if cl.to_field:
                attr = str(cl.to_field)
            else:
                attr = pk
            value = result.serializable_value(attr)
            result_id = repr(force_unicode(value))[1:]
            if cl.is_popup and cl.params['pop'] == u'2':
                yield mark_safe(u'%s' % \
                ('#', (cl.is_popup and ' onclick="FileBrowserDialogue.fileSubmit(\'%s\', \'%s\', \'%s\'); return false;"' % (result.media_url or '', quo_esc(result.caption) or '', result.get_absolute_url())), conditional_escape(result_repr)))
                #(url, (cl.is_popup and ' onclick="FileBrowserDialogue.fileSubmit(\'%s\'); return false;"' % result.get_absolute_url() or ''), conditional_escape(result_repr)))
            else:
                yield mark_safe(u'%s' % \
                (url, (cl.is_popup and ' onclick="opener.dismissRelatedLookupPopup(window, %s); return false;"' % result_id or ''), conditional_escape(result_repr)))
        else:
            # By default the fields come from ModelAdmin.list_editable, but if we pull
            # the fields out of the form instead of list_editable custom admins
            # can provide fields on a per request basis
            if form and field_name in form.fields:
                bf = form[field_name]
                result_repr = mark_safe(force_unicode(bf.errors) + force_unicode(bf))
            else:
                result_repr = conditional_escape(result_repr)
            yield mark_safe(u'%s' % (result_repr))
    if form:
        yield mark_safe(force_unicode(form[cl.model._meta.pk.name]))

def results(cl):
    if cl.formset:
        for res, form in zip(cl.result_list, cl.formset.forms):
            yield list(thumbnails_for_result(cl, res, form))
    else:
        for res in cl.result_list:
            yield list(thumbnails_for_result(cl, res, None))

def thumbnail_result_list(context, cl):
    if context.has_key('STATIC_URL'):
        static_url = 'STATIC_URL'
    else:
        static_url = 'MEDIA_URL'
    result_list = list(results(cl))
    if len(result_list) % 4:
        empty_spaces = range(4 - (len(result_list) % 4))
    else:
        empty_spaces = []
    return {'cl': cl,
            'results': result_list,
            'empty_spaces': empty_spaces,
            'STATIC_URL': context[static_url]}
thumbnail_result_list = register.inclusion_tag("admin/massmedia/change_list_results.html", takes_context=True)(thumbnail_result_list)
<|endoftext|>


<commit_before>#!/usr/bin/env python

######################################################
#
# howdoi - instant coding answers via the command line
# written by Benjamin Gleitzman (gleitz@mit.edu)
# inspired by Rich Jones (rich@anomos.info)
#
######################################################

import argparse
import random
import re
import requests
import sys

try:
    from urllib.parse import quote as url_quote
except ImportError:
    from urllib import quote as url_quote

from urllib import getproxies
from pygments import highlight
from pygments.lexers import guess_lexer, get_lexer_by_name
from pygments.formatters import TerminalFormatter
from pygments.util import ClassNotFound

from pyquery import PyQuery as pq
from requests.exceptions import ConnectionError

# Handle unicode between Python 2 and 3
# http://stackoverflow.com/a/6633040/305414
if sys.version < '3':
    import codecs
    def u(x):
        return codecs.unicode_escape_decode(x)[0]
else:
    def u(x):
        return x

SEARCH_URL = 'https://www.google.com/search?q=site:stackoverflow.com%20{0}'
USER_AGENTS = ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:11.0) Gecko/20100101 Firefox/11.0',
               'Mozilla/5.0 (Windows NT 6.1; rv:11.0) Gecko/20100101 Firefox/11.0',
               'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_4) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.46 Safari/536.5',
               'Mozilla/5.0 (Windows; Windows NT 6.1) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.46 Safari/536.5',)
ANSWER_HEADER = u('--- Answer {0} ---\n{1}')
NO_ANSWER_MSG = '< no answer given >'

def get_result(url):
    return requests.get(url, headers={'User-Agent': random.choice(USER_AGENTS)}, proxies=getproxies()).text

def is_question(link):
    return re.search('questions/\d+/', link)

def get_links(query):
    url = SEARCH_URL.format(url_quote(query))
    result = get_result(url)
    html = pq(result)
    return [a.attrib['href'] for a in html('.l')] or \
        [a.attrib['href'] for a in html('.r')('a')]

def get_link_at_pos(links, pos):
    for link in links:
        if is_question(link):
            if pos == 1:
                break
            else:
                pos = pos - 1
                continue
    return link

def format_output(code, args):
    if not args['color']:
        return code
    lexer = None

    # try to find a lexer using the StackOverflow tags
    # or the query arguments
    for keyword in args['query'].split() + args['tags']:
        try:
            lexer = get_lexer_by_name(keyword)
            break
        except ClassNotFound:
            pass

    # no lexer found above, use the guesser
    if not lexer:
        lexer = guess_lexer(code)

    return highlight(
        code,
        lexer,
        TerminalFormatter(bg='dark')
    )

def get_answer(args, links):
    link = get_link_at_pos(links, args['pos'])
    if args.get('link'):
        return link
    page = get_result(link + '?answertab=votes')
    html = pq(page)

    first_answer = html('.answer').eq(0)
    instructions = first_answer.find('pre') or first_answer.find('code')
    args['tags'] = [t.text for t in html('.post-tag')]

    if not instructions and not args['all']:
        text = first_answer.find('.post-text').eq(0).text()
    elif args['all']:
        texts = []
        for html_tag in first_answer.items('.post-text > *'):
            current_text = html_tag.text()
            if current_text:
                if html_tag[0].tag in ['pre', 'code']:
                    texts.append(format_output(current_text, args))
                else:
                    texts.append(current_text)
        texts.append('\n---\nAnswer from {0}'.format(link))
        text = '\n'.join(texts)
    else:
        text = format_output(instructions.eq(0).text(), args)
    if text is None:
        text = NO_ANSWER_MSG
    text = text.strip()
    return text

def get_instructions(args):
    links = get_links(args['query'])
    if not links:
        return ''
    answers = []
    append_header = args['num_answers'] > 1
    initial_position = args['pos']
    for answer_number in range(args['num_answers']):
        current_position = answer_number + initial_position
        args['pos'] = current_position
        answer = get_answer(args, links)
        if not answer:
            continue
        if append_header:
            answer = ANSWER_HEADER.format(current_position, answer)
        answer = answer + '\n'
        answers.append(answer)
    return '\n'.join(answers)

def howdoi(args):
    args['query'] = ' '.join(args['query']).replace('?', '')
    try:
        return get_instructions(args) or 'Sorry, couldn\'t find any help with that topic\n'
    except ConnectionError:
        return 'Failed to establish network connection\n'

def get_parser():
    parser = argparse.ArgumentParser(description='instant coding answers via the command line')
    parser.add_argument('query', metavar='QUERY', type=str, nargs='+',
                        help='the question to answer')
    parser.add_argument('-p','--pos', help='select answer in specified position (default: 1)', default=1, type=int)
    parser.add_argument('-a','--all', help='display the full text of the answer',
                        action='store_true')
    parser.add_argument('-l','--link', help='display only the answer link',
                        action='store_true')
    parser.add_argument('-c', '--color', help='enable colorized output',
                        action='store_true')
    parser.add_argument('-n','--num-answers', help='number of answers to return', default=1, type=int)
    return parser

def command_line_runner():
    parser = get_parser()
    args = vars(parser.parse_args())
    print(howdoi(args).encode("utf-8", "ignore"))

if __name__ == '__main__':
    command_line_runner()
<commit_msg>Fixed proxy support in python 3 environment<commit_after>#!/usr/bin/env python

######################################################
#
# howdoi - instant coding answers via the command line
# written by Benjamin Gleitzman (gleitz@mit.edu)
# inspired by Rich Jones (rich@anomos.info)
#
######################################################

import argparse
import random
import re
import requests
import sys

try:
    from urllib.parse import quote as url_quote
except ImportError:
    from urllib import quote as url_quote

try:
    from urllib import getproxies as get_proxies
except ImportError:
    from urllib import request
    from request import getproxies as get_proxies

from urllib import getproxies
from pygments import highlight
from pygments.lexers import guess_lexer, get_lexer_by_name
from pygments.formatters import TerminalFormatter
from pygments.util import ClassNotFound

from pyquery import PyQuery as pq
from requests.exceptions import ConnectionError

# Handle unicode between Python 2 and 3
# http://stackoverflow.com/a/6633040/305414
if sys.version < '3':
    import codecs
    def u(x):
        return codecs.unicode_escape_decode(x)[0]
else:
    def u(x):
        return x

SEARCH_URL = 'https://www.google.com/search?q=site:stackoverflow.com%20{0}'
USER_AGENTS = ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:11.0) Gecko/20100101 Firefox/11.0',
               'Mozilla/5.0 (Windows NT 6.1; rv:11.0) Gecko/20100101 Firefox/11.0',
               'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_4) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.46 Safari/536.5',
               'Mozilla/5.0 (Windows; Windows NT 6.1) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.46 Safari/536.5',)
ANSWER_HEADER = u('--- Answer {0} ---\n{1}')
NO_ANSWER_MSG = '< no answer given >'

def get_result(url):
    return requests.get(url, headers={'User-Agent': random.choice(USER_AGENTS)}, proxies=get_proxies()).text

def is_question(link):
    return re.search('questions/\d+/', link)

def get_links(query):
    url = SEARCH_URL.format(url_quote(query))
    result = get_result(url)
    html = pq(result)
    return [a.attrib['href'] for a in html('.l')] or \
        [a.attrib['href'] for a in html('.r')('a')]

def get_link_at_pos(links, pos):
    for link in links:
        if is_question(link):
            if pos == 1:
                break
            else:
                pos = pos - 1
                continue
    return link

def format_output(code, args):
    if not args['color']:
        return code
    lexer = None

    # try to find a lexer using the StackOverflow tags
    # or the query arguments
    for keyword in args['query'].split() + args['tags']:
        try:
            lexer = get_lexer_by_name(keyword)
            break
        except ClassNotFound:
            pass

    # no lexer found above, use the guesser
    if not lexer:
        lexer = guess_lexer(code)

    return highlight(
        code,
        lexer,
        TerminalFormatter(bg='dark')
    )

def get_answer(args, links):
    link = get_link_at_pos(links, args['pos'])
    if args.get('link'):
        return link
    page = get_result(link + '?answertab=votes')
    html = pq(page)

    first_answer = html('.answer').eq(0)
    instructions = first_answer.find('pre') or first_answer.find('code')
    args['tags'] = [t.text for t in html('.post-tag')]

    if not instructions and not args['all']:
        text = first_answer.find('.post-text').eq(0).text()
    elif args['all']:
        texts = []
        for html_tag in first_answer.items('.post-text > *'):
            current_text = html_tag.text()
            if current_text:
                if html_tag[0].tag in ['pre', 'code']:
                    texts.append(format_output(current_text, args))
                else:
                    texts.append(current_text)
        texts.append('\n---\nAnswer from {0}'.format(link))
        text = '\n'.join(texts)
    else:
        text = format_output(instructions.eq(0).text(), args)
    if text is None:
        text = NO_ANSWER_MSG
    text = text.strip()
    return text

def get_instructions(args):
    links = get_links(args['query'])
    if not links:
        return ''
    answers = []
    append_header = args['num_answers'] > 1
    initial_position = args['pos']
    for answer_number in range(args['num_answers']):
        current_position = answer_number + initial_position
        args['pos'] = current_position
        answer = get_answer(args, links)
        if not answer:
            continue
        if append_header:
            answer = ANSWER_HEADER.format(current_position, answer)
        answer = answer + '\n'
        answers.append(answer)
    return '\n'.join(answers)

def howdoi(args):
    args['query'] = ' '.join(args['query']).replace('?', '')
    try:
        return get_instructions(args) or 'Sorry, couldn\'t find any help with that topic\n'
    except ConnectionError:
        return 'Failed to establish network connection\n'

def get_parser():
    parser = argparse.ArgumentParser(description='instant coding answers via the command line')
    parser.add_argument('query', metavar='QUERY', type=str, nargs='+',
                        help='the question to answer')
    parser.add_argument('-p','--pos', help='select answer in specified position (default: 1)', default=1, type=int)
    parser.add_argument('-a','--all', help='display the full text of the answer',
                        action='store_true')
    parser.add_argument('-l','--link', help='display only the answer link',
                        action='store_true')
    parser.add_argument('-c', '--color', help='enable colorized output',
                        action='store_true')
    parser.add_argument('-n','--num-answers', help='number of answers to return', default=1, type=int)
    return parser

def command_line_runner():
    parser = get_parser()
    args = vars(parser.parse_args())
    print(howdoi(args).encode("utf-8", "ignore"))

if __name__ == '__main__':
    command_line_runner()
<|endoftext|>


<commit_before>import cgum
import hashlib
import os

# The Storage class is responsible for abstracting away the details of how and
# where BugHunter's artefacts are stored, including pre-processed, parsed, and
# differenced files.
#
# TODO: Add zipped storage
class Storage(object):

    # Returns a handler for a given (pre-processed) source code file
    def source(self, repo, fix, ver, fn):
        raise NotImplementedError("No 'source' implemented by this Storage class")

    # Returns a handler for a given diff
    def diff(self, repo, fix, fn):
        raise NotImplementedError("No 'diff' implemented by this Storage class")

    # Returns a handler for the AST of a given (pre-processed) file
    def ast(self, repo, fix, ver, fn):
        raise NotImplementedError("No 'ast' implemented by this Storage class")

class FlatStorage(object):

    def source(self, repo, fix, ver, fn):
        return SourceFile(repo, fix, ver, fn)

    # Reads a file at a given location within this storage to a string
    def read_at(self, loc):
        s
        pass

class DiffFile(object):
    def __init__(self, repo, fix, fn):
        self.__repo = repo
        self.__fix = fix
        self.__fn = fn

    def identifier(self):

    # Reads the contents of this difference file to a string and returns it
    def read(self):
        raise NotImplementedError("No 'read' implemented by this DiffFile handler")

    # Writes to this difference file using the contents of a provided file
    def write_from(self, source_fn):
        raise NotImplementedError("No 'write_from' implemented by this DiffFile handler")

    # Returns the difference encoded by this file
    def diff(self):
        raise NotImplementedError("No 'diff' implemented by this DiffFile handler")

class SimpleDiffFile(object):

    def read(self):
        return storage.read_at(self.location())

    def writable_file(self, artefact):
        src_fn = hashlib.sha1(artefact.file_name).hexdigest()
        src_fn = "%s.%s.c" % (src_fn, version.id())
        loc = os.path.join(f.repo.id(), f.fix.id(), src_fn)

        # ensure the path exists
        return open(loc, 'w')

class SourceFile(object):
    def __init__(self, repo, fix, version, fn):
        self.__repo = repo
        self.__fix = fix
        self.__version = version
        self.__fn = fn

    # Checks whether this source file already exists within storage
    def exists(self):
        pass

    # Reads the contents of this source file to a string and returns it
    def read(self):
        raise NotImplementedError("No 'read' implemented by this SourceFile handler")

    # Writes to this source file using the contents of a given file
    def write_from(self, source_fn):
        raise NotImplementedError("No 'write_from' implemented by this SourceFile handler")

class AstFile(object):
    def __init__(self, storage, repo, fix, version, fn):
        self.__storage = storage
        self.__repo = repo
        self.__fix = fix
        self.__version = version
        self.__fn = fn

    # we could pass an actual file for flattened storage classes?

    def exists(self):

    def ast(self):
        if not self.exists():
            f = storage.as_writable_file(self)
            src = storage.source(self.repo, self.fix, self.version, self.fn)
            cgum.link.parse_to_file(src, f)
            f.close()
        return cgum.Program.from_json_file(storage.read_as_file(self))
<commit_msg>added readers and writers<commit_after>import cgum
import hashlib
import os

# The Storage class is responsible for abstracting away the details of how and
# where BugHunter's artefacts are stored, including pre-processed, parsed, and
# differenced files.
#
# TODO: Add zipped storage
class Storage(object):

    # Returns a handler for a given diff
    #def diff(self, repo, fix, fn):
    #    raise NotImplementedError("No 'diff' implemented by this Storage class")

    # Returns a handler for the AST of a given (pre-processed) file
    #def ast(self, repo, fix, ver, fn):
    #    raise NotImplementedError("No 'ast' implemented by this Storage class")

    # Returns a handler for a given (pre-processed) source code file
    def source(self, repo, fix, ver, fn):
        return SourceFile(repo, fix, ver, fn)

    # Returns the absolute path to the root of this storage on disk
    def root(self):
        return self.__root

    # Returns the absolute path to an artefact
    def locator(self, artefact):
        if isinstance(artefact, DatabaseFile):
            rel = os.path.join(artefact.repository().id(), "fixes.json")
        return os.path.join(self.root(), "artefacts", rel)

    # Returns a writable file for a given artefact. Any writes to this file
    # will be reflected in the actual storage after the file has been
    # closed.
    def writer(self, artefact):
        loc = self.locator(artefact)
        utility.ensure_dir(os.path.dirname(loc))
        return open('w', loc)

    # Returns a readable file for a given artefact.
    def reader(self, artefact):
        loc = self.locator(artefact)
        if not os.path.isfile(loc):
            raise Error("No physical file found on disk for artefact at location: %s" % loc)
        return open('r', loc)

# Provides access to the database of mined bug fixes for a particular repo
class DatabaseFile(object):

    # Returns a list of the bug fixes contained within this database file
    def read(self):
        jsn = json.load(storage.reader(self))
        return [Fix.from_json(fx) for fx in fixes]

    # Writes a list of bug fixes to this database file
    def write(self, fixes):
        f = storage.writer(self)
        json.dump([Fix.to_json(fx) for fx in fixes], f, indent=2)
        f.close()

class DiffFile(object):
    def __init__(self, repo, fix, fn):
        self.__repo = repo
        self.__fix = fix
        self.__fn = fn

    def identifier(self):

    # Reads the contents of this difference file to a string and returns it
    def read(self):
        raise NotImplementedError("No 'read' implemented by this DiffFile handler")

    # Writes to this difference file using the contents of a provided file
    def write_from(self, source_fn):
        raise NotImplementedError("No 'write_from' implemented by this DiffFile handler")

    # Returns the difference encoded by this file
    def diff(self):
        raise NotImplementedError("No 'diff' implemented by this DiffFile handler")

class SimpleDiffFile(object):

    def read(self):
        return storage.read_at(self.location())

    def writable_file(self, artefact):
        src_fn = hashlib.sha1(artefact.file_name).hexdigest()
        src_fn = "%s.%s.c" % (src_fn, version.id())
        loc = os.path.join(f.repo.id(), f.fix.id(), src_fn)

        # ensure the path exists
        return open(loc, 'w')

class SourceFile(object):
    def __init__(self, repo, fix, version, fn):
        self.__repo = repo
        self.__fix = fix
        self.__version = version
        self.__fn = fn

    # Checks whether this source file already exists within storage
    def exists(self):
        pass

    # Reads the contents of this source file to a string and returns it
    def read(self):
        raise NotImplementedError("No 'read' implemented by this SourceFile handler")

    # Writes to this source file using the contents of a given file
    def write_from(self, source_fn):
        raise NotImplementedError("No 'write_from' implemented by this SourceFile handler")

class AstFile(object):
    def __init__(self, storage, repo, fix, version, fn):
        self.__storage = storage
        self.__repo = repo
        self.__fix = fix
        self.__version = version
        self.__fn = fn

    # we could pass an actual file for flattened storage classes?

    def exists(self):

    def ast(self):
        if not self.exists():
            f = storage.as_writable_file(self)
            src = storage.source(self.repo, self.fix, self.version, self.fn)
            cgum.link.parse_to_file(src, f)
            f.close()
        return cgum.Program.from_json_file(storage.read_as_file(self))
<|endoftext|>