<commit_before>"""
    A sample base plugin. Do not forget to add an init file in plugins.d
"""
from cement.core import handler, controller
from plugins import BasePlugin
import common

class Example(BasePlugin):
    # a file containing a list of plugins, one per line.
    plugins_file = "plugins/drupal/wordlists/plugins_3500"
    # the location of the plugins. If there are multiple locations, such as in
    # drupal, you can specify a list. First %s will be replaced with the site
    # url and the second will be replaced with the module name.
    plugins_base_url = ["%ssites/all/modules/%s/",
            "%ssites/default/modules/%s/"]
    # same as above
    themes_file = "plugins/drupal/wordlists/themes_1250"
    themes_base_url = ["%ssites/all/themes/%s/",
            "%ssites/default/themes/%s/"]
    # a URL which commonly responds with forbidden (most frequently folders, but
    # could be something else.)
    forbidden_url = "misc/"
    # a URL which you know results in a 200 OK. If item is a list, then all
    # items are tested to see if any responds with OK.
    regular_file_url = "misc/drupal.js"
    # a file which always exists in the modules.
    module_common_file = "README.txt"

    # a list of tuples that contain on the index 0 a url, and on 1 a description
    # to be shown to the user if the URL replies with 200 found
    interesting_urls = [("CHANGELOG", "This CMS' default changelog 200 OK.")]

    # A list of interesting files which are normally present in modules,
    # relative to the module's folder.
    interesting_module_urls = [
        ('CHANGELOG.txt', 'Changelog file'),
    ]

    # a file which validates against common/versions.xsd for version
    # fingerprinting.
    versions_file = "plugins/drupal/versions.xml"

    class Meta:
        label = 'example'

    @controller.expose(help='example scanner')
    def example(self):
        self.plugin_init()

    def update_version_check(self):
        """
            @return True if new versions of this software have been found.
        """

    def update_version(self):
        """
            This function needs to return an updated VersionsFile, which will
                be written to disk. There are APIs on common.update_api which can
                be used.
            @return updated VersionsFile
        """

    # there is a plethora of functions to override in BasePlugin and BaseInternalPlugin.

def load():
    handler.register(Example)

<commit_msg>Improve documentation.<commit_after>"""
    A sample base plugin. Do not forget to add an init file in plugins.d
"""
from cement.core import handler, controller
from plugins import BasePlugin
import common

class Example(BasePlugin):
    """
        Remember to create in dscan/plugins/<plugin_name>/ three files:

            - plugins.txt
            - themes.txt
            - versions.xml
    """

    # the location of the plugins. If there are multiple locations, such as in
    # drupal, you can specify a list. First %s will be replaced with the site
    # url and the second will be replaced with the module name.
    plugins_base_url = ["%ssites/all/modules/%s/",
            "%ssites/default/modules/%s/"]
    themes_base_url = ["%ssites/all/themes/%s/",
            "%ssites/default/themes/%s/"]

    # A URL which is known to usually be a folder.
    forbidden_url = "misc/"
    # a URL which you know results in a 200 OK. If item is a list, then all
    # items are tested to see if any responds with OK.
    regular_file_url = "misc/drupal.js"
    # a file which commonly exists in the modules.
    module_common_file = "README.txt"
    # Major branches which are supported by the vendor. This is used for
    # updating.
    update_majors = ['6','7','8']

    # a list of tuples that contain on the index 0 a url, and on 1 a description
    # to be shown to the user if the URL replies with 200 found
    interesting_urls = [("CHANGELOG", "This CMS' default changelog.")]

    # A list of interesting files which are normally present in modules,
    # relative to the module's folder.
    interesting_module_urls = [
        ('CHANGELOG.txt', 'Changelog file'),
    ]

    class Meta:
        # The label is important, choose the CMS name in lowercase.
        label = 'example'

    # This function is the entry point for the CMS.
    @controller.expose(help='example scanner')
    def example(self):
        self.plugin_init()

    # The four functions below get called when ./droopescan update is called.
    # They can make use of dscan.common.update_api in order to update the
    # versions.xml and the *.txt files.
    #
    # See drupal and silverstripe for examples.
    def update_version_check(self):
        """
            @return True if new versions of this software have been found.
        """

    def update_version(self):
        """
            This function needs to return an updated VersionsFile, which will
            be written to disk. There are APIs on common.update_api which can
            be used.
            @return updated VersionsFile
        """

    def update_plugins_check(self):
        return ua.update_modules_check(self)

    def update_plugins(self):
        """
            This function needs to return two updated lists containing plugins
            and themes. There are APIs on common.update_api which can be used.
            @return (plugins, themes)
        """

def load():
    handler.register(Example)

<|endoftext|>


<commit_before>''' This script generates a fries standoff file format by building on top of
nxml2txt output'''

import argparse
import itertools as it

nil = 'N/A'

def span_contains(parent, child):
    ''' Returns wether parent contains child '''

    return True if parent[0] <= child[0] and parent[1] >= child[1] else False

def get_text(txt, start, end, citations):
    ''' Returns a string with the text within start-end removing all the text within any of the ranges in citations '''

    span = (start, end)

    staging = [s for s in citations if span_contains(span, s)]

    # Calculate the ranges to include

    for s in staging:
        s = list(s)
        snippet = txt[s[0]:s[1]]

        if snippet[0] != '[':
            if txt[s[0]-1] == '[':
                s[0] -= 1

        if snippet[-1] != ']':
            if txt[s[1]] == ']':
                s[1] += 1

        txt = txt[:s[0]] + ' ' * (s[1] - s[0]) + txt[s[1]:]

    return txt[start:end+1]

def parse_args():
    parser = argparse.ArgumentParser(description='parse nxml file and dump sections')
    parser.add_argument('--no-citations', action='store_true', dest='no_citations', help='replace citations with spaces')
    parser.add_argument('--no-header', dest='header', action='store_false', help="don't include column header")
    parser.add_argument('textfile', help='text file')
    parser.add_argument('standoff', help='standoff file')
    return parser.parse_args()

if __name__ == '__main__':
    args = parse_args()

    # Read the text file
    with open(args.textfile) as f:
        txt = f.read()

    # Read the standoff file
    with open(args.standoff) as f:
        soff = f.readlines()

    # print column headers if required
    if args.header:
        print 'chunk_id\tsection_id\tname\tis_title\ttext'

    keep = ('article-title', 'abstract', 'sec', 'title', 'fig', 'p', 'supplementary-material', 'ref-list', 'ref')
    remove = ('xref')

    # Compute the spans of the citations to remove them from the text
    citations = []

    if args.no_citations:
        for line in (s for s in soff if s.split('\t', 2)[1].split()[0] in remove):
            # Parse the line
            _, t_r, __ = line.split('\t', 2)

            _, start, end = t_r.split(' ')

            start, end = int(start), int(end)

            if start != end:
                citations.append((start, end))

    # Keep only the lines that correspond to a tag in 'keep'
    entries = (s for s in soff if s.split('\t', 2)[1].split()[0] in keep)

    secs = [] # Stack of sections
    title_seen = False
    sm = fig = 1

    for ix, e in enumerate(entries):

        # Parse the line
        _, t_r, __, attrs = e.split('\t')
        tag, start, end = t_r.split(' ')
        start = int(start)
        end = int(end)

        if tag in ('article-title', 'abstract'):

            # This is to enforce only one "article-title" (the actual title of the paper tag per file.
            # There are other instance of the tag in the references

            if tag == 'article-title':
                if title_seen:
                    continue
                else:
                    title_seen = True

            print '%i\t%s\t%s\t%i\t%s' % (ix, nil, tag, 1 if tag == 'article-title' else 0, get_text(txt, start, end, citations))

        elif tag in ('sec', 'fig', 'supplementary-material', 'ref-list'):
            # For our purposes, sections and figures are equivalent

            # Parse the attributes
            attrs = {k:v.strip('"') for k, v in ((s.split('=') for s in attrs.split()))}

            # Figure out the section id
            if tag == 'sec':
                sec = attrs['id'] if 'id' in attrs else nil
                sec_norm = attrs['sec-type'] if 'sec-type' in attrs else nil
            elif tag == 'fig':
                sec = 'fig-%i' % fig
                sec_norm = sec
                fig += 1
            elif tag == 'supplementary-material':
                sec = 'supm-%i' % sm
                sec_norm = sec
                sm += 1
            elif tag == 'ref-list':
                sec = 'references'
                sec_norm = sec
            else:
                raise Exception("I shouldn't be here")

            # Add this section to the sections stack
            secs.append((sec, sec_norm))

            # Remember the current span
            current_span = (start, end)

        elif tag in ('title', 'p', 'ref'):

            # Only if we are in a section or image, otherwise this could be part of the references or metadata
            if len(secs) > 0:

                if span_contains(current_span, (start, end)):
                    # If the current section's span contains this element
                    sec, sec_norm = secs[-1]
                else:
                    # Pop the current section to return to the previous onse
                    secs.pop()
                    if len(secs) > 0:
                        sec, sec_norm = secs[-1]
                    else:
                        # If there are no more sections in the stack, this element may not be useful information
                        continue

                text = get_text(txt, start, end, citations)

                if tag == 'ref':
                    text = text.replace('\n', ' ')

                print '%i\t%s\t%s\t%i\t%s' % (ix, sec, sec_norm, 1 if tag == 'title' else 0, text)
<commit_msg>Replaces new line characters with whitespaces<commit_after>''' This script generates a fries standoff file format by building on top of
nxml2txt output'''

import argparse
import itertools as it

nil = 'N/A'

def span_contains(parent, child):
    ''' Returns wether parent contains child '''

    return True if parent[0] <= child[0] and parent[1] >= child[1] else False

def get_text(txt, start, end, citations):
    ''' Returns a string with the text within start-end removing all the text within any of the ranges in citations '''

    span = (start, end)

    staging = [s for s in citations if span_contains(span, s)]

    # Calculate the ranges to include

    for s in staging:
        s = list(s)
        snippet = txt[s[0]:s[1]]

        if snippet[0] != '[':
            if txt[s[0]-1] == '[':
                s[0] -= 1

        if snippet[-1] != ']':
            if txt[s[1]] == ']':
                s[1] += 1

        txt = txt[:s[0]] + ' ' * (s[1] - s[0]) + txt[s[1]:]

    # These lines replaces the new-line characters for spaces
    ret = txt[start:end+1]
    ret = ret.replace('\n', ' ')

    return ret

def parse_args():
    parser = argparse.ArgumentParser(description='parse nxml file and dump sections')
    parser.add_argument('--no-citations', action='store_true', dest='no_citations', help='replace citations with spaces')
    parser.add_argument('--no-header', dest='header', action='store_false', help="don't include column header")
    parser.add_argument('textfile', help='text file')
    parser.add_argument('standoff', help='standoff file')
    return parser.parse_args()

if __name__ == '__main__':
    args = parse_args()

    # Read the text file
    with open(args.textfile) as f:
        txt = f.read()

    # Read the standoff file
    with open(args.standoff) as f:
        soff = f.readlines()

    # print column headers if required
    if args.header:
        print 'chunk_id\tsection_id\tname\tis_title\ttext'

    keep = ('article-title', 'abstract', 'sec', 'title', 'fig', 'p', 'supplementary-material', 'ref-list', 'ref')
    remove = ('xref')

    # Compute the spans of the citations to remove them from the text
    citations = []

    if args.no_citations:
        for line in (s for s in soff if s.split('\t', 2)[1].split()[0] in remove):
            # Parse the line
            _, t_r, __ = line.split('\t', 2)

            _, start, end = t_r.split(' ')

            start, end = int(start), int(end)

            if start != end:
                citations.append((start, end))

    # Keep only the lines that correspond to a tag in 'keep'
    entries = (s for s in soff if s.split('\t', 2)[1].split()[0] in keep)

    secs = [] # Stack of sections
    title_seen = False
    sm = fig = 1

    for ix, e in enumerate(entries):

        # Parse the line
        _, t_r, __, attrs = e.split('\t')
        tag, start, end = t_r.split(' ')
        start = int(start)
        end = int(end)

        if tag in ('article-title', 'abstract'):

            # This is to enforce only one "article-title" (the actual title of the paper tag per file.
            # There are other instance of the tag in the references

            if tag == 'article-title':
                if title_seen:
                    continue
                else:
                    title_seen = True

            print '%i\t%s\t%s\t%i\t%s' % (ix, nil, tag, 1 if tag == 'article-title' else 0, get_text(txt, start, end, citations))

        elif tag in ('sec', 'fig', 'supplementary-material', 'ref-list'):
            # For our purposes, sections and figures are equivalent

            # Parse the attributes
            attrs = {k:v.strip('"') for k, v in ((s.split('=') for s in attrs.split()))}

            # Figure out the section id
            if tag == 'sec':
                sec = attrs['id'] if 'id' in attrs else nil
                sec_norm = attrs['sec-type'] if 'sec-type' in attrs else nil
            elif tag == 'fig':
                sec = 'fig-%i' % fig
                sec_norm = sec
                fig += 1
            elif tag == 'supplementary-material':
                sec = 'supm-%i' % sm
                sec_norm = sec
                sm += 1
            elif tag == 'ref-list':
                sec = 'references'
                sec_norm = sec
            else:
                raise Exception("I shouldn't be here")

            # Add this section to the sections stack
            secs.append((sec, sec_norm))

            # Remember the current span
            current_span = (start, end)

        elif tag in ('title', 'p', 'ref'):

            # Only if we are in a section or image, otherwise this could be part of the references or metadata
            if len(secs) > 0:

                if span_contains(current_span, (start, end)):
                    # If the current section's span contains this element
                    sec, sec_norm = secs[-1]
                else:
                    # Pop the current section to return to the previous onse
                    secs.pop()
                    if len(secs) > 0:
                        sec, sec_norm = secs[-1]
                    else:
                        # If there are no more sections in the stack, this element may not be useful information
                        continue

                text = get_text(txt, start, end, citations)

                if tag == 'ref':
                    text = text.replace('\n', ' ')

                print '%i\t%s\t%s\t%i\t%s' % (ix, sec, sec_norm, 1 if tag == 'title' else 0, text)
<|endoftext|>


<commit_before>"""python wrapper for the duck duck go zero-click api"""

import urllib
import urllib2
import json as j

def search(query, **kwargs):
    useragent = 'py-ddg'
    params = {
        'q': query,
        'format': 'json',
        'pretty': '1',
        'no_redirect': '1',
        'no_html': '1',
        'skip_disambig': '1',
        }
    params.update(kwargs)
    enc_params = urllib.urlencode(params)
    url = 'http://api.duckduckgo.com/?' + enc_params

    try:
        request = urllib2.Request(url, headers={'User-Agent': useragent})
        response = urllib2.urlopen(request)
        json = j.loads(response.read())
        response.close()
        return Results(json)
    except urllib2.HTTPError, err:
        print 'HTTPError = ' + str(err.code)
    except urllib2.URLError, err:
        print 'URLError = ' + str(err.reason)
    except urllib2.HTTPException:
        print 'HTTPException'
    except Exception:
        import traceback
        print traceback.format_exec()
    import sys
    sys.exit("Error attempting to search " + url)

class Results(object):
    def __init__(self, json):
        self.json = j.dumps(json, indent=2)
        self.type = {'A': 'article', 'D': 'disambiguation',
                     'C': 'category', 'N': 'name',
                     'E': 'exclusive', '': 'nothing'}[json['Type']]
        self.answer = Answer(json)
        self.result = Result(json.get('Results', None))
        self.abstract = Abstract(json)
        self.definition = Definition(json)
        self.redirect = Redirect(json)

class Result(object):
    def __init__(self, json):
        self.html = json[0].get('Result', '') if json else ''
        self.text = json[0].get('Text', '') if json else ''
        self.url = json[0].get('FirstURL', '') if json else ''

class Abstract(object):
    def __init__(self, json):
        self.html = json['Abstract']
        self.text = json['AbstractText']
        self.url = json['AbstractURL']
        self.source = json['AbstractSource']
        self.heading = json['Heading']

class Answer(object):
    def __init__(self, json):
        self.text = json['Answer']
        self.type = json['AnswerType']
        self.url = None

class Definition(object):
    def __init__(self, json):
        self.text = json['Definition']
        self.url = json['DefinitionURL']
        self.source = json['DefinitionSource']

class Redirect(object):
    def __init__(self, json):
        self.url = json['Redirect']
<commit_msg>Updated the exception handling code<commit_after>"""python wrapper for the duck duck go zero-click api"""

import urllib
import urllib2
import json as jsonlib

def search(query, **kwargs):
    useragent = 'py-ddg'
    params = {
        'q': query,
        'format': 'json',
        'pretty': '1',
        'no_redirect': '1',
        'no_html': '1',
        'skip_disambig': '1',
        }
    params.update(kwargs)
    enc_params = urllib.urlencode(params)
    url = 'http://api.duckduckgo.com/?' + enc_params

    try:
        request = urllib2.Request(url, headers={'User-Agent': useragent})
        response = urllib2.urlopen(request)
        json = jsonlib.loads(response.read())
        response.close()
        return Results(json)
    except urllib2.HTTPError, err:
        print 'HTTPError = ' + str(err.code)
    except urllib2.URLError, err:
        print 'URLError = ' + str(err.reason)
    except Exception:
        print 'Unhandled exception'
        raise
    import sys
    sys.exit("Unable to process " + query + " please try again.")

class Results(object):
    def __init__(self, json):
        self.json = jsonlib.dumps(json, indent=2)
        self.type = {'A': 'article', 'D': 'disambiguation',
                     'C': 'category', 'N': 'name',
                     'E': 'exclusive', '': 'nothing'}[json['Type']]
        self.answer = Answer(json)
        self.result = Result(json.get('Results', None))
        self.abstract = Abstract(json)
        self.definition = Definition(json)
        self.redirect = Redirect(json)

class Result(object):
    def __init__(self, json):
        self.html = json[0].get('Result', '') if json else ''
        self.text = json[0].get('Text', '') if json else ''
        self.url = json[0].get('FirstURL', '') if json else ''

class Abstract(object):
    def __init__(self, json):
        self.html = json['Abstract']
        self.text = json['AbstractText']
        self.url = json['AbstractURL']
        self.source = json['AbstractSource']
        self.heading = json['Heading']

class Answer(object):
    def __init__(self, json):
        self.text = json['Answer']
        self.type = json['AnswerType']
        self.url = None

class Definition(object):
    def __init__(self, json):
        self.text = json['Definition']
        self.url = json['DefinitionURL']
        self.source = json['DefinitionSource']

class Redirect(object):
    def __init__(self, json):
        self.url = json['Redirect']
<|endoftext|>


<commit_before>import pysolr
import json
import re
from datetime import datetime
import argparse
import os
import requests
from kafka import KafkaConsumer, KafkaProducer

def scroll_and_get_results(solr, q, **kwargs):
    results = []
    start = 0
    rows = 100
    hits = 100
    while start < hits:
        result = solr.search(q=q, rows=rows, start=start, **kwargs)
        hits = result.hits
        print "GEtting {} / {}".format(start, hits)
        if hits == 0:
            break
        results += result.docs
        start +=rows
    return results

def get_parent(parent_id, solr):
    fl = ['content_type', 'id', 'crawler', 'raw_content', '*_hd', 'fetch_timestamp', 'url']
    fq = ['status:"FETCHED"', "id:"+parent_id]
    q = "*:*"
    results = solr.search(q=q, **{'fl':fl, 'fq':fq})
    if results.hits is not 1:
        print 'Got more than one hit for parent id {}, exiting...'.format(parent_id)
        exit(1)
    return results.docs[0]

def get_all_objects_for_parent(parent_id, solr):
    fl = ['content_type', 'id', 'crawler', 'raw_content', '*_hd', 'fetch_timestamp', 'url',
          'relative_path']
    fq = ['status:"FETCHED"', '!content_type:text*',
          'parent:{}'.format(parent_id)]
    q = "*:*"
    print "GEtting objects for  {}".format(parent_id)
    return scroll_and_get_results(solr, q, **{'fq':fq, 'fl':fl})

def get_parent_id(solr):
    fq = ['status:"FETCHED"', 'content_type:text/html']
    q = "*:*"
    print "Getting parent ids"
    return scroll_and_get_results(solr, q, **{'fq':fq, 'fl':'id'})

def get_headers(doc):
    headers = {}
    for key in doc:
        match = re.match('(.*)(_[a-z])_hd', key)
        if match is not None:
            headers[match.group(1).encode('utf-8')] = doc.get(key)
    return {'response_headers':headers}

def prepare_objects_list(objects):
    objects_list = []
    for o in objects:
        obj_json = {}
        obj_json['obj_original_url'] = o.get('url')
        obj_json['obj_stored_url'] = o.get('relative_path')
        obj_json['content_type'] = o.get('content_type')
        obj_json['timestamp_crawl'] = o.get('fetch_timestamp')
        obj_json.update(get_headers(o))
        objects_list.append(obj_json)
    return objects_list

def prepare_cdrjson(parent, objects):
    team = "JPL"
    version = 3.1
    timestamp = datetime.utcnow().isoformat() + 'Z'
    cdr_json = {}
    cdr_json['_id'] = parent.get('id')
    cdr_json['content_type'] = parent.get('content_type')
    cdr_json['crawler'] = parent.get('crawler')
    cdr_json['objects'] = prepare_objects_list(objects)
    cdr_json['raw_content'] = parent.get('raw_content')
    cdr_json.update(get_headers(parent))
    cdr_json['team'] = team
    cdr_json['timestamp_crawl'] = parent.get('fetch_timestamp')
    cdr_json['timestamp_index'] = timestamp
    cdr_json['url'] = parent.get('url')
    cdr_json['version'] = version
    return cdr_json

def prepare_es_header(index, doc_type, id):
    header = {"index":{"_index":index, "_type":doc_type, "_id":id}}
    return header

def upload_to_elasticsearch(data, es_url):
    print("Uploading to elasticsearch: {}".format(es_url))
    r = requests.post(es_url + "/_bulk", data=data)
    print(r.text)

def main(config, is_es_format=False):
    solr_url = "{}/{}".format(config['solr'], config['core'])
    solr = pysolr.Solr(solr_url)

    print("Starting output dump")

    parent_ids = get_parent_id(solr)
    dump_file = os.path.join("/projects/sce/data/dumper", "crawl-data-dump.jsonl")
    if os.path.exists(dump_file):
        os.rename(dump_file, "{}.old".format(dump_file))
    print "Found {} parents".format(len(parent_ids))
    total = len(parent_ids)
    with open(dump_file, 'w') as fw:
        for parent_id in parent_ids:
            parent_doc = get_parent(parent_id['id'], solr)
            objects = get_all_objects_for_parent(parent_id['id'], solr)
            cdr_doc = prepare_cdrjson(parent_doc, objects)
            # print json.dumps(cdr_doc)
            try:

                if (is_es_format):
                    header = prepare_es_header(config["es_index"], config["es_doctype"], cdr_doc["_id"])
                    fw.write(json.dumps(header, encoding='utf-8') + "\n")
                # Writing to file
                del cdr_doc["_id"]
                fw.write(json.dumps(cdr_doc, encoding='utf-8') + "\n")
                total -= 1
                print "Remaining {}".format(total)
            except:
                print "Error writing {}".format(parent_id)
    print("Export complete")
    # if (is_es_format):
    #     upload_to_elasticsearch(open(dump_file, 'r'), config["es_url"])

def produce_to_kafka(config):
    producer = KafkaProducer(bootstrap_servers=config.get("kafka_brokers"),
                             value_serializer=lambda m: json.dumps(m).encode('ascii'),
                             security_protocol='SSL',
                             ssl_cafile=config.get("kafka_cafile"),
                             ssl_certfile=config.get("kafka_certfile"),
                             ssl_keyfile=config.get("kafka_keyfile"),
                             ssl_check_hostname=False)
    topic = config.get("kafka_topic")
    dump_file = os.path.join("/projects/sce/data/dumper", "crawl-data-dump.jsonl")
    with open(dump_file, 'r') as f:
        _id = ""
        for line in f:
            line = line.strip("\n")
            data = json.loads(line)
            if "index" in data:
                _id = data.get("index").get("_id")
                continue
            data.update({"_id":_id})
            producer.send(topic, data)

def test_kafka(config):
    consumer = KafkaConsumer(config.get("kafka_topic"),
                             bootstrap_servers=config.get("kafka_brokers"),
                             security_protocol='SSL',
                             ssl_cafile=config.get("kafka_cafile"),
                             ssl_certfile=config.get("kafka_certfile"),
                             ssl_keyfile=config.get("kafka_keyfile"),
                             ssl_check_hostname=False,
                             auto_offset_reset='earliest',
                             group_id=None)

    print consumer.poll(max_records=10, timeout_ms=10000)
    print ('Start consuming')
    for message in consumer:
    # message value and key are raw bytes -- decode if necessary!
    # e.g., for unicode: `message.value.decode('utf-8')`
        print ("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition,
                                          message.offset, message.key,
                                          message.value))

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    default = os.path.join(os.path.dirname(os.path.realpath(__file__)),"dumper.config")
    parser.add_argument("--config_file", help="JSON Configuration file", default=default)
    parser.add_argument("-es", help="Output file in ES bulk upload format", default=False, action='store_true')
    parser.add_argument("--es_url", help="Elasticsearch endpoint URL")
    parser.add_argument("--es_index", help="Elasticsearch index")
    parser.add_argument("--es_doctype", help="Elasticsearch doc type")
    parser.add_argument("--kafka_brokers", help="A list of kafka brokers")
    parser.add_argument("--kafka_topic")
    parser.add_argument("--kafka_keyfile")
    parser.add_argument("--kafka_certfile")
    parser.add_argument("--kafka_cafile")
    parser.add_argument("--kafka", default=False)
    args = parser.parse_args()
    config = {}
    with open(args.config_file, 'r') as f:
        config = json.load(f)
        # if (args.es):
        #     config["es_url"] = args.es_url
        #     config["es_index"] = args.es_index
        #     config["es_doctype"] = args.es_doctype

    main(config, args.es)
    if args.kafka:
        produce_to_kafka(config)
<commit_msg>Update loggin of producer<commit_after>import pysolr
import json
import re
from datetime import datetime
import argparse
import os
import requests
from kafka import KafkaConsumer, KafkaProducer

def scroll_and_get_results(solr, q, **kwargs):
    results = []
    start = 0
    rows = 100
    hits = 100
    while start < hits:
        result = solr.search(q=q, rows=rows, start=start, **kwargs)
        hits = result.hits
        print "GEtting {} / {}".format(start, hits)
        if hits == 0:
            break
        results += result.docs
        start +=rows
    return results

def get_parent(parent_id, solr):
    fl = ['content_type', 'id', 'crawler', 'raw_content', '*_hd', 'fetch_timestamp', 'url']
    fq = ['status:"FETCHED"', "id:"+parent_id]
    q = "*:*"
    results = solr.search(q=q, **{'fl':fl, 'fq':fq})
    if results.hits is not 1:
        print 'Got more than one hit for parent id {}, exiting...'.format(parent_id)
        exit(1)
    return results.docs[0]

def get_all_objects_for_parent(parent_id, solr):
    fl = ['content_type', 'id', 'crawler', 'raw_content', '*_hd', 'fetch_timestamp', 'url',
          'relative_path']
    fq = ['status:"FETCHED"', '!content_type:text*',
          'parent:{}'.format(parent_id)]
    q = "*:*"
    print "GEtting objects for  {}".format(parent_id)
    return scroll_and_get_results(solr, q, **{'fq':fq, 'fl':fl})

def get_parent_id(solr):
    fq = ['status:"FETCHED"', 'content_type:text/html']
    q = "*:*"
    print "Getting parent ids"
    return scroll_and_get_results(solr, q, **{'fq':fq, 'fl':'id'})

def get_headers(doc):
    headers = {}
    for key in doc:
        match = re.match('(.*)(_[a-z])_hd', key)
        if match is not None:
            headers[match.group(1).encode('utf-8')] = doc.get(key)
    return {'response_headers':headers}

def prepare_objects_list(objects):
    objects_list = []
    for o in objects:
        obj_json = {}
        obj_json['obj_original_url'] = o.get('url')
        obj_json['obj_stored_url'] = o.get('relative_path')
        obj_json['content_type'] = o.get('content_type')
        obj_json['timestamp_crawl'] = o.get('fetch_timestamp')
        obj_json.update(get_headers(o))
        objects_list.append(obj_json)
    return objects_list

def prepare_cdrjson(parent, objects):
    team = "JPL"
    version = 3.1
    timestamp = datetime.utcnow().isoformat() + 'Z'
    cdr_json = {}
    cdr_json['_id'] = parent.get('id')
    cdr_json['content_type'] = parent.get('content_type')
    cdr_json['crawler'] = parent.get('crawler')
    cdr_json['objects'] = prepare_objects_list(objects)
    cdr_json['raw_content'] = parent.get('raw_content')
    cdr_json.update(get_headers(parent))
    cdr_json['team'] = team
    cdr_json['timestamp_crawl'] = parent.get('fetch_timestamp')
    cdr_json['timestamp_index'] = timestamp
    cdr_json['url'] = parent.get('url')
    cdr_json['version'] = version
    return cdr_json

def prepare_es_header(index, doc_type, id):
    header = {"index":{"_index":index, "_type":doc_type, "_id":id}}
    return header

def upload_to_elasticsearch(data, es_url):
    print("Uploading to elasticsearch: {}".format(es_url))
    r = requests.post(es_url + "/_bulk", data=data)
    print(r.text)

def main(config, is_es_format=False):
    solr_url = "{}/{}".format(config['solr'], config['core'])
    solr = pysolr.Solr(solr_url)

    print("Starting output dump")

    parent_ids = get_parent_id(solr)
    dump_file = os.path.join("/projects/sce/data/dumper", "crawl-data-dump.jsonl")
    if os.path.exists(dump_file):
        os.rename(dump_file, "{}.old".format(dump_file))
    print "Found {} parents".format(len(parent_ids))
    total = len(parent_ids)
    with open(dump_file, 'w') as fw:
        for parent_id in parent_ids:
            parent_doc = get_parent(parent_id['id'], solr)
            objects = get_all_objects_for_parent(parent_id['id'], solr)
            cdr_doc = prepare_cdrjson(parent_doc, objects)
            # print json.dumps(cdr_doc)
            try:

                if (is_es_format):
                    header = prepare_es_header(config["es_index"], config["es_doctype"], cdr_doc["_id"])
                    fw.write(json.dumps(header, encoding='utf-8') + "\n")
                # Writing to file
                del cdr_doc["_id"]
                fw.write(json.dumps(cdr_doc, encoding='utf-8') + "\n")
                total -= 1
                print "Remaining {}".format(total)
            except:
                print "Error writing {}".format(parent_id)
    print("Export complete")
    # if (is_es_format):
    #     upload_to_elasticsearch(open(dump_file, 'r'), config["es_url"])

def produce_to_kafka(config):
    producer = KafkaProducer(bootstrap_servers=config.get("kafka_brokers"),
                             value_serializer=lambda m: json.dumps(m).encode('ascii'),
                             security_protocol='SSL',
                             ssl_cafile=config.get("kafka_cafile"),
                             ssl_certfile=config.get("kafka_certfile"),
                             ssl_keyfile=config.get("kafka_keyfile"),
                             ssl_check_hostname=False)
    topic = config.get("kafka_topic")
    dump_file = os.path.join("/projects/sce/data/dumper", "crawl-data-dump.jsonl")
    with open(dump_file, 'r') as f:
        _id = ""
        for line in f:
            line = line.strip("\n")
            data = json.loads(line)
            if "index" in data:
                _id = data.get("index").get("_id")
                continue
            data.update({"_id":_id})
            response =  producer.send(topic, data)
            print "Posted {} : {}".format(_id, response.is_done)

def test_kafka(config):
    consumer = KafkaConsumer(config.get("kafka_topic"),
                             bootstrap_servers=config.get("kafka_brokers"),
                             security_protocol='SSL',
                             ssl_cafile=config.get("kafka_cafile"),
                             ssl_certfile=config.get("kafka_certfile"),
                             ssl_keyfile=config.get("kafka_keyfile"),
                             ssl_check_hostname=False,
                             auto_offset_reset='earliest',
                             group_id=None)

    print consumer.poll(max_records=10, timeout_ms=10000)
    print ('Start consuming')
    for message in consumer:
    # message value and key are raw bytes -- decode if necessary!
    # e.g., for unicode: `message.value.decode('utf-8')`
        print ("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition,
                                          message.offset, message.key,
                                          message.value))

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    default = os.path.join(os.path.dirname(os.path.realpath(__file__)),"dumper.config")
    parser.add_argument("--config_file", help="JSON Configuration file", default=default)
    parser.add_argument("-es", help="Output file in ES bulk upload format", default=False, action='store_true')
    parser.add_argument("--es_url", help="Elasticsearch endpoint URL")
    parser.add_argument("--es_index", help="Elasticsearch index")
    parser.add_argument("--es_doctype", help="Elasticsearch doc type")
    parser.add_argument("--kafka_brokers", help="A list of kafka brokers")
    parser.add_argument("--kafka_topic")
    parser.add_argument("--kafka_keyfile")
    parser.add_argument("--kafka_certfile")
    parser.add_argument("--kafka_cafile")
    parser.add_argument("--kafka", default=False)
    args = parser.parse_args()
    config = {}
    with open(args.config_file, 'r') as f:
        config = json.load(f)
        # if (args.es):
        #     config["es_url"] = args.es_url
        #     config["es_index"] = args.es_index
        #     config["es_doctype"] = args.es_doctype

    main(config, args.es)
    if args.kafka:
        produce_to_kafka(config)
<|endoftext|>


<commit_before>from django.conf.urls import patterns, include, url
from django.contrib import admin
from django.contrib.auth import views as auth_views

from core import views

admin.autodiscover()

urlpatterns = patterns('',
                       url(r'^$', views.index),
                       url(r'^login/?$', views.login),
                       url(r'^main/$', views.tracker),
                       url(r'^logout/?$', views.logout_view),
                       url(r'^reporting/(?P<ryear>\d{4})/(?P<rmonth>\d+)/$', views.reporting),
                       url(r'^ajax/$', views.ajax),
                       url(r'^overview/$', views.edit),
                       url(r'^password_change/$',
                           auth_views.password_change,
                           {'template_name': 'password_change_form.html'},
                           name='password_change'),
                       url(r'^password_change_done/$',
                           auth_views.password_change_done,
                           {'template_name': 'password_change_done.html'},
                           name='password_change_done'),
                       url(r'^(?P<drafttype>[a-z]{4})/(?P<dnumber>\d+)/(?P<language>[A-Z]{2})/$', views.draft),
                       url(r'^reminders/(?P<year>\d{4})/(?P<week>\d+)/$', views.tracking_calendar),
                       url(r'^admin/', include(admin.site.urls)),
                       url(r'^password_reset/$', auth_views.password_reset,
                           {'from_email': 'admin@dunnings.com'},
                           name="password_reset"),
                       url(r'^password_reset_done/$', auth_views.password_reset_done,
                           name="password_reset_done"),
                       url(r'password_reset_confirm/(?P<uidb64>[0-9A-Za-z]+)\/(?P<token>.+)/$', auth_views.password_reset_confirm,
                           name="password_reset_confirm"),
                       url(r'^password_reset_complete/$', auth_views.password_reset_complete,
                           name="password_reset_complete"),
)
<commit_msg>Added 2 urls for pdf modification/generation.<commit_after>from django.conf.urls import patterns, include, url
from django.contrib import admin
from django.contrib.auth import views as auth_views

from core import views

admin.autodiscover()

urlpatterns = patterns('',
                       url(r'^$', views.index),
                       url(r'^login/?$', views.login),
                       url(r'^main/$', views.tracker),
                       url(r'^logout/?$', views.logout_view),
                       url(r'^reporting/(?P<ryear>\d{4})/(?P<rmonth>\d+)/$', views.reporting),
                       url(r'^ajax/$', views.ajax),
                       url(r'^overview/$', views.edit),
                       url(r'^pdf_get/(?P<file_name>.+)/$', views.serve_pdf),
                       url(r'^pdfgen/$', views.pdfgeneration),
                       url(r'^password_change/$',
                           auth_views.password_change,
                           {'template_name': 'password_change_form.html'},
                           name='password_change'),
                       url(r'^password_change_done/$',
                           auth_views.password_change_done,
                           {'template_name': 'password_change_done.html'},
                           name='password_change_done'),
                       url(r'^(?P<drafttype>[a-z]{4})/(?P<dnumber>\d+)/(?P<language>[A-Z]{2})/$', views.draft),
                       url(r'^reminders/(?P<year>\d{4})/(?P<week>\d+)/$', views.tracking_calendar),
                       url(r'^admin/', include(admin.site.urls)),
                       url(r'^password_reset/$', auth_views.password_reset,
                           {'from_email': 'admin@dunnings.com'},
                           name="password_reset"),
                       url(r'^password_reset_done/$', auth_views.password_reset_done,
                           name="password_reset_done"),
                       url(r'password_reset_confirm/(?P<uidb64>[0-9A-Za-z]+)\/(?P<token>.+)/$', auth_views.password_reset_confirm,
                           name="password_reset_confirm"),
                       url(r'^password_reset_complete/$', auth_views.password_reset_complete,
                           name="password_reset_complete"),
)
<|endoftext|>


<commit_before># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import logging
import string
import sys

if sys.version_info < (2, 7):
    import functools

    @functools.wraps(logging.Formatter.format)
    def new_format(self, record, string=string,
                   old_format=logging.Formatter.format):
        if string.find is None:
            string.find = str.find
        return old_format(self, record)
    logging.Formatter.format = new_format

class ColorFormatter(logging.Formatter):
    LEVEL_COLORS = {
        logging.DEBUG: '\033[00;32m',  # GREEN
        logging.INFO: '\033[00;36m',  # CYAN
        logging.WARN: '\033[01;33m',  # BOLD YELLOW
        logging.ERROR: '\033[01;31m',  # BOLD RED
        logging.CRITICAL: '\033[01;31m',  # BOLD RED
    }

    def format(self, record, Formatter=logging.Formatter):
        # Hack to get last log line instead of exception on 2.6
        res = Formatter.format(self, record)  # old-style class on 2.6
        return self.LEVEL_COLORS[record.levelno] + res + '\033[m'

def set_console_formatter(**formatter_kwargs):
    formatter_kwargs.setdefault(
        'fmt', '%(asctime)s %(levelname)-8s %(name)-15s %(message)s')
    formatter_kwargs.setdefault('datefmt', '%Y-%m-%d %H:%M:%S')
    root_logger = logging.getLogger('')
    for handler in root_logger.handlers:
        if handler.__class__ is logging.StreamHandler:  # Skip subclasses
            console_handler = handler
            break
    else:
        return  # Didn't find any StreamHandlers there
    formatter = ColorFormatter(**formatter_kwargs)
    console_handler.setFormatter(formatter)

def silence_iso8601():
    iso8601_logger = logging.getLogger('iso8601')
    iso8601_logger.setLevel(logging.INFO)

class UrllibFilter(logging.Filter):
    def filter(self, record):
        if record.msg.startswith('Starting new HTTP connection'):
            record.levelno = logging.DEBUG
        return True

def lower_urllib_level():
    urllib_logger = logging.getLogger('urllib3.connectionpool')
    urllib_logger.addFilter(UrllibFilter())
<commit_msg>Colorize output on tty only<commit_after># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import logging
import string
import sys

if sys.version_info < (2, 7):
    import functools

    @functools.wraps(logging.Formatter.format)
    def new_format(self, record, string=string,
                   old_format=logging.Formatter.format):
        if string.find is None:
            string.find = str.find
        return old_format(self, record)
    logging.Formatter.format = new_format

class ColorFormatter(logging.Formatter):
    LEVEL_COLORS = {
        logging.DEBUG: '\033[00;32m',  # GREEN
        logging.INFO: '\033[00;36m',  # CYAN
        logging.WARN: '\033[01;33m',  # BOLD YELLOW
        logging.ERROR: '\033[01;31m',  # BOLD RED
        logging.CRITICAL: '\033[01;31m',  # BOLD RED
    }

    def format(self, record, Formatter=logging.Formatter):
        # Hack to get last log line instead of exception on 2.6
        res = Formatter.format(self, record)  # old-style class on 2.6
        return self.LEVEL_COLORS[record.levelno] + res + '\033[m'

def set_console_formatter(**formatter_kwargs):
    formatter_kwargs.setdefault(
        'fmt', '%(asctime)s %(levelname)-8s %(name)-15s %(message)s')
    formatter_kwargs.setdefault('datefmt', '%Y-%m-%d %H:%M:%S')
    root_logger = logging.getLogger('')
    for handler in root_logger.handlers:
        if handler.__class__ is logging.StreamHandler:  # Skip subclasses
            console_handler = handler
            # Skip if not a tty (default ssh, redirect, ...)
            isatty = getattr(handler.stream, 'isatty', None)
            if isatty is None or not isatty():
                continue
            break
    else:
        return  # Didn't find any suitable StreamHandlers there
    formatter = ColorFormatter(**formatter_kwargs)
    console_handler.setFormatter(formatter)

def silence_iso8601():
    iso8601_logger = logging.getLogger('iso8601')
    iso8601_logger.setLevel(logging.INFO)

class UrllibFilter(logging.Filter):
    def filter(self, record):
        if record.msg.startswith('Starting new HTTP connection'):
            record.levelno = logging.DEBUG
        return True

def lower_urllib_level():
    urllib_logger = logging.getLogger('urllib3.connectionpool')
    urllib_logger.addFilter(UrllibFilter())
<|endoftext|>


<commit_before>from dvc.command.base import CmdBase, DvcLock
from dvc.logger import Logger

import dvc.data_cloud as cloud

class CmdDataSync(CmdBase):
    def __init__(self, settings):
        super(CmdDataSync, self).__init__(settings)

    def run(self):
        with DvcLock(self.is_locker, self.git):
            self.cloud.sync(self.parsed_args.targets, self.parsed_args.jobs)

class CmdDataPull(CmdBase):
    def __init__(self, settings):
        super(CmdDataPull, self).__init__(settings)

    def run(self):
        with DvcLock(self.is_locker, self.git):
            self.cloud.pull(self.parsed_args.targets, self.parsed_args.jobs)

class CmdDataPush(CmdBase):
    def __init__(self, settings):
        super(CmdDataPush, self).__init__(settings)

    def run(self):
        with DvcLock(self.is_locker, self.git):
            self.cloud.push(self.parsed_args.targets, self.parsed_args.jobs)

class CmdDataStatus(CmdBase):
    def __init__(self, settings):
        super(CmdDataStatus, self).__init__(settings)

    def _show(self, status):
        for s in status:
            target, ret = s

            if ret == cloud.STATUS_UNKNOWN or ret == cloud.STATUS_OK:
                continue

            prefix_map = {
                cloud.STATUS_DELETED  : 'deleted: ',
                cloud.STATUS_MODIFIED : 'modified:',
                cloud.STATUS_NEW      : 'new file:',
            }

            Logger.info('\t{}\t{}'.format(prefix_map[ret], target.resolved_cache.dvc))

    def run(self):
        with DvcLock(self.is_locker, self.git):
            status = self.cloud.status(self.parsed_args.targets, self.parsed_args.jobs)

            if len(list(status)) != len(self.parsed_args.targets):
                return 1

            self._show(status)

            return 0
<commit_msg>status: don't return 1 if len(statuses) != len(targets)<commit_after>from dvc.command.base import CmdBase, DvcLock
from dvc.logger import Logger

import dvc.data_cloud as cloud

class CmdDataSync(CmdBase):
    def __init__(self, settings):
        super(CmdDataSync, self).__init__(settings)

    def run(self):
        with DvcLock(self.is_locker, self.git):
            self.cloud.sync(self.parsed_args.targets, self.parsed_args.jobs)

class CmdDataPull(CmdBase):
    def __init__(self, settings):
        super(CmdDataPull, self).__init__(settings)

    def run(self):
        with DvcLock(self.is_locker, self.git):
            self.cloud.pull(self.parsed_args.targets, self.parsed_args.jobs)

class CmdDataPush(CmdBase):
    def __init__(self, settings):
        super(CmdDataPush, self).__init__(settings)

    def run(self):
        with DvcLock(self.is_locker, self.git):
            self.cloud.push(self.parsed_args.targets, self.parsed_args.jobs)

class CmdDataStatus(CmdBase):
    def __init__(self, settings):
        super(CmdDataStatus, self).__init__(settings)

    def _show(self, status):
        for s in status:
            target, ret = s

            if ret == cloud.STATUS_UNKNOWN or ret == cloud.STATUS_OK:
                continue

            prefix_map = {
                cloud.STATUS_DELETED  : 'deleted: ',
                cloud.STATUS_MODIFIED : 'modified:',
                cloud.STATUS_NEW      : 'new file:',
            }

            Logger.info('\t{}\t{}'.format(prefix_map[ret], target.resolved_cache.dvc))

    def run(self):
        with DvcLock(self.is_locker, self.git):
            status = self.cloud.status(self.parsed_args.targets, self.parsed_args.jobs)

            self._show(status)

            return 0
<|endoftext|>


<commit_before>"""
Provides functionality for validation of the data-types specified
for odml
"""

import sys
self = sys.modules[__name__].__dict__

from datetime import datetime, date, time

types = ['string', 'int', 'text', 'float', 'URL', 'datetime', 'boolean', 'date', 'binary', 'person', 'time']

def valid_type(dtype):
    """
    checks if *dtype* is a valid type
    """
    if dtype in types: return True
    if dtype is None: return True
    if dtype.endswith("-tuple"):
        try:
            count = int(dtype[:-6])
            return True
        except ValueError:
            pass
    return False

def validate(string, dtype):
    """
    checks if:

     * *dtype* is a valid type
     * *string* is a valid expression of type *dtype*
    """
    try:
        if not dtype in types:
            if dtype.endswith("-tuple"):
                count = int(dtype[:-6])
                #try to parse it
                get_tuple(string, strict=True, count=count)
                return True
            return False
        #try to parse it
        self.get(dtype+"_get", str_get)(string)
    except: #any error, this type ain't valid
        return False

def get(string, dtype=None):
    """
    convert *string* to the corresponding *dtype*
    """
    if not dtype: return str_get(string)
    if dtype.endswith("-tuple"): # special case, as the count-number is included in the type-name
        return get_tuple(string)
    return self.get(dtype+"_get", str_get)(string)

def set(value, dtype=None):
    """
    serialize a *value* of type *dtype* to a unicode string
    """
    if not dtype: return str_set(value)
    if dtype.endswith("-tuple"):
        return set_tuple(string)
    return self.get(dtype+"_set", str_set)(value)

def int_get(string):
    if not string: return 0
    return int(float(string))

def float_get(string):
    if not string: return 0.0
    return float(string)

def str_get(string):
    return unicode(string)

def str_set(value):
    return unicode(value)

def time_get(string):
    if not string: return None
    return datetime.strptime(string, '%H:%M:%S').time()

def time_set(value):
    if not value: return None
    return value.isoformat()

def date_get(string):
    if not string: return None
    return datetime.strptime(string, '%Y-%m-%d').date()

date_set = time_set

def datetime_get(string):
    if not string: return None
    return datetime.strptime(string, '%Y-%m-%d %H:%M:%S')

def datetime_set(value):
    if not value: return None
    return value.isoformat(' ')

def boolean_get(string):
    truth = ["true", "t", "1"] # be kind, spec only accepts True / False
    if string in truth: return True
    false = ["false", "f", "0"]
    if string in false: return False
    raise ValueError("Cannot interpret '%s' as boolean" % string)

def binary_get(string):
    "base64decode the data"
    return base64.b64decode(string)

def binary_set(value):
    "base64encode the data"
    return base64.b64encode(string)

<commit_msg>types: implement tuple-methods and fix base64<commit_after>"""
Provides functionality for validation of the data-types specified
for odml
"""

import sys
self = sys.modules[__name__].__dict__

from datetime import datetime, date, time
import base64

types = ['string', 'int', 'text', 'float', 'URL', 'datetime', 'boolean', 'date', 'binary', 'person', 'time']

def valid_type(dtype):
    """
    checks if *dtype* is a valid type
    """
    if dtype in types: return True
    if dtype is None: return True
    if dtype.endswith("-tuple"):
        try:
            count = int(dtype[:-6])
            return True
        except ValueError:
            pass
    return False

def validate(string, dtype):
    """
    checks if:

     * *dtype* is a valid type
     * *string* is a valid expression of type *dtype*
    """
    try:
        if not dtype in types:
            if dtype.endswith("-tuple"):
                count = int(dtype[:-6])
                #try to parse it
                tuple_get(string, count=count)
                return True
            return False
        #try to parse it
        self.get(dtype+"_get", str_get)(string)
    except: #any error, this type ain't valid
        return False

def get(string, dtype=None):
    """
    convert *string* to the corresponding *dtype*
    """
    if not dtype: return str_get(string)
    if dtype.endswith("-tuple"): # special case, as the count-number is included in the type-name
        return tuple_get(string)
    return self.get(dtype+"_get", str_get)(string)

def set(value, dtype=None):
    """
    serialize a *value* of type *dtype* to a unicode string
    """
    if not dtype: return str_set(value)
    if dtype.endswith("-tuple"):
        return tuple_set(value)
    return self.get(dtype+"_set", str_set)(value)

def int_get(string):
    if not string: return 0
    return int(float(string))

def float_get(string):
    if not string: return 0.0
    return float(string)

def str_get(string):
    return unicode(string)

def str_set(value):
    return unicode(value)

def time_get(string):
    if not string: return None
    return datetime.strptime(string, '%H:%M:%S').time()

def time_set(value):
    if not value: return None
    return value.isoformat()

def date_get(string):
    if not string: return None
    return datetime.strptime(string, '%Y-%m-%d').date()

date_set = time_set

def datetime_get(string):
    if not string: return None
    return datetime.strptime(string, '%Y-%m-%d %H:%M:%S')

def datetime_set(value):
    if not value: return None
    return value.isoformat(' ')

def boolean_get(string):
    if not string: return None
    string = string.lower()
    truth = ["true", "t", "1"] # be kind, spec only accepts True / False
    if string in truth: return True
    false = ["false", "f", "0"]
    if string in false: return False
    raise ValueError("Cannot interpret '%s' as boolean" % string)

def boolean_set(value):
    if value is None: return None
    return str(value)

def tuple_get(string, count=None):
    """
    parse a tuple string like "(1024;768)" and return strings of the elements
    """
    if not string: return None
    string = string.strip()
    assert string.startswith("(") and string.endswith(")")
    string = string[1:-1]
    res = string.split(";")
    if count is not None: # be strict
        assert len(res) == count
    return res

def tuple_set(value):
    if not value: return None
    return "(%s)" % ";".join(value)

def binary_get(string):
    "base64decode the data"
    if not string: return None
    return base64.b64decode(string)

def binary_set(value):
    "base64encode the data"
    if not value: return None
    return base64.b64encode(value)

<|endoftext|>


<commit_before>"""
Django settings for scuole project.

For more information on this file, see
https://docs.djangoproject.com/en/1.8/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.8/ref/settings/
"""

from __future__ import absolute_import, unicode_literals
from os import environ, path

import dj_database_url

from django.core.exceptions import ImproperlyConfigured

###########
# HELPERS #
###########

def env(setting, default=None):
    """ Get the environment setting or return exception """
    try:
        return environ[setting]
    except KeyError:
        if default is not None:
            return default
        else:
            error_msg = ('The {} env variable was not found '
                         'and no default was set!').format(setting)
            raise ImproperlyConfigured(error_msg)

######################
# PATH CONFIGURATION #
######################

# This file
BASE_FILE = path.abspath(__file__)

# Absolute filesystem path to the project directory
ROOT_DIR = path.abspath(path.join(path.dirname(BASE_FILE), '..', '..'))

# Absolute filesystem path to the Django project's app folder
APPS_DIR = path.join(ROOT_DIR, 'scuole')

#######################
# DEBUG CONFIGURATION #
#######################

DEBUG = False

TEMPLATE_DEBUG = DEBUG

#####################
# APP CONFIGURATION #
#####################

DJANGO_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
)

THIRD_PARTY_APPS = (

)

LOCAL_APPS = (
    '.scuole.counties',
    'scuole.states',
)

# https://docs.djangoproject.com/en/1.8/ref/settings/#installed-apps
INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS

############################
# MIDDLEWARE CONFIGURATION #
############################
# https://docs.djangoproject.com/en/1.8/topics/http/middleware/

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.middleware.security.SecurityMiddleware',
)

##########################
# TEMPLATE CONFIGURATION #
##########################

# https://docs.djangoproject.com/en/1.8/topics/templates/#configuration
TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            path.join(APPS_DIR, 'templates'),
        ],
        'OPTIONS': {
            'context_processors': [
                'django.contrib.auth.context_processors.auth',
                'django.template.context_processors.debug',
                'django.template.context_processors.i18n',
                'django.template.context_processors.media',
                'django.template.context_processors.static',
                'django.template.context_processors.tz',
                'django.contrib.messages.context_processors.messages',
            ],
            'loaders': [
                'django.template.loaders.filesystem.Loader',
                'django.template.loaders.app_directories.Loader',
            ],
        },
    },
]

##########################
# DATABASE CONFIGURATION #
##########################

# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
DATABASES = {
    'default':  dj_database_url.config(default='postgres://localhost/scuole')
}

DATABASES['default']['ATOMIC_REQUESTS'] = True

#############################
# STATIC FILE CONFIGURATION #
#############################

# https://docs.djangoproject.com/en/1.8/ref/settings/#static-root
STATIC_ROOT = path.join(APPS_DIR, 'assets')

# https://docs.djangoproject.com/en/1.8/ref/settings/#static-url
STATIC_URL = '/static/'

# https://docs.djangoproject.com/en/1.8/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS
STATICFILES_DIRS = (
    path.join(APPS_DIR, 'static'),
)

# https://docs.djangoproject.com/en/1.8/ref/contrib/staticfiles/#staticfiles-finders
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
)

############################
# MEDIA FILE CONFIGURATION #
############################

# https://docs.djangoproject.com/en/1.8/ref/settings/#media-root
MEDIA_ROOT = path.join(APPS_DIR, 'media')

# https://docs.djangoproject.com/en/1.8/ref/settings/#media-url
MEDIA_URL = '/media/'

#####################
# URL CONFIGURATION #
#####################

ROOT_URLCONF = 'config.urls'

# https://docs.djangoproject.com/en/1.8/ref/settings/#wsgi-application
WSGI_APPLICATION = 'config.wsgi.application'

#########################
# GENERAL CONFIGURATION #
#########################

# https://docs.djangoproject.com/en/1.8/ref/settings/#language-code
LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

# https://docs.djangoproject.com/en/1.8/ref/settings/#use-i18n
USE_I18N = True

# https://docs.djangoproject.com/en/1.8/ref/settings/#use-l10n
USE_L10N = True

# https://docs.djangoproject.com/en/1.8/ref/settings/#use-tz
USE_TZ = True
<commit_msg>scuole data folder bath<commit_after>"""
Django settings for scuole project.

For more information on this file, see
https://docs.djangoproject.com/en/1.8/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.8/ref/settings/
"""

from __future__ import absolute_import, unicode_literals
from os import environ, path

import dj_database_url

from django.core.exceptions import ImproperlyConfigured

###########
# HELPERS #
###########

def env(setting, default=None):
    """ Get the environment setting or return exception """
    try:
        return environ[setting]
    except KeyError:
        if default is not None:
            return default
        else:
            error_msg = ('The {} env variable was not found '
                         'and no default was set!').format(setting)
            raise ImproperlyConfigured(error_msg)

######################
# PATH CONFIGURATION #
######################

# This file
BASE_FILE = path.abspath(__file__)

# Absolute filesystem path to the project directory
ROOT_DIR = path.abspath(path.join(path.dirname(BASE_FILE), '..', '..'))

# Absolute filesystem path to the Django project's app folder
APPS_DIR = path.join(ROOT_DIR, 'scuole')

#######################
# DEBUG CONFIGURATION #
#######################

DEBUG = False

TEMPLATE_DEBUG = DEBUG

#####################
# APP CONFIGURATION #
#####################

DJANGO_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
)

THIRD_PARTY_APPS = (

)

LOCAL_APPS = (
    'scuole.counties',
    'scuole.states',
)

# https://docs.djangoproject.com/en/1.8/ref/settings/#installed-apps
INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS

############################
# MIDDLEWARE CONFIGURATION #
############################
# https://docs.djangoproject.com/en/1.8/topics/http/middleware/

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.middleware.security.SecurityMiddleware',
)

##########################
# TEMPLATE CONFIGURATION #
##########################

# https://docs.djangoproject.com/en/1.8/topics/templates/#configuration
TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            path.join(APPS_DIR, 'templates'),
        ],
        'OPTIONS': {
            'context_processors': [
                'django.contrib.auth.context_processors.auth',
                'django.template.context_processors.debug',
                'django.template.context_processors.i18n',
                'django.template.context_processors.media',
                'django.template.context_processors.static',
                'django.template.context_processors.tz',
                'django.contrib.messages.context_processors.messages',
            ],
            'loaders': [
                'django.template.loaders.filesystem.Loader',
                'django.template.loaders.app_directories.Loader',
            ],
        },
    },
]

##########################
# DATABASE CONFIGURATION #
##########################

# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
DATABASES = {
    'default':  dj_database_url.config(default='postgres://localhost/scuole')
}

DATABASES['default']['ATOMIC_REQUESTS'] = True

#############################
# STATIC FILE CONFIGURATION #
#############################

# https://docs.djangoproject.com/en/1.8/ref/settings/#static-root
STATIC_ROOT = path.join(APPS_DIR, 'assets')

# https://docs.djangoproject.com/en/1.8/ref/settings/#static-url
STATIC_URL = '/static/'

# https://docs.djangoproject.com/en/1.8/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS
STATICFILES_DIRS = (
    path.join(APPS_DIR, 'static'),
)

# https://docs.djangoproject.com/en/1.8/ref/contrib/staticfiles/#staticfiles-finders
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
)

############################
# MEDIA FILE CONFIGURATION #
############################

# https://docs.djangoproject.com/en/1.8/ref/settings/#media-root
MEDIA_ROOT = path.join(APPS_DIR, 'media')

# https://docs.djangoproject.com/en/1.8/ref/settings/#media-url
MEDIA_URL = '/media/'

#####################
# URL CONFIGURATION #
#####################

ROOT_URLCONF = 'config.urls'

# https://docs.djangoproject.com/en/1.8/ref/settings/#wsgi-application
WSGI_APPLICATION = 'config.wsgi.application'

#########################
# GENERAL CONFIGURATION #
#########################

# https://docs.djangoproject.com/en/1.8/ref/settings/#language-code
LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

# https://docs.djangoproject.com/en/1.8/ref/settings/#use-i18n
USE_I18N = True

# https://docs.djangoproject.com/en/1.8/ref/settings/#use-l10n
USE_L10N = True

# https://docs.djangoproject.com/en/1.8/ref/settings/#use-tz
USE_TZ = True

#########################
# GENERAL CONFIGURATION #
#########################

DATA_FOLDER = env('DATA_FOLDER', path.join(ROOT_DIR, '../scuole-data'))
<|endoftext|>


<commit_before># -*- coding: utf-8  -*-
#
# Copyright (C) 2009-2012 Ben Kurtovic <ben.kurtovic@verizon.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import imp

from earwigbot.irc import IRCConnection, RC

__all__ = ["Watcher"]

class Watcher(IRCConnection):
    """
    **EarwigBot: IRC Watcher Component**

    The IRC watcher runs on a wiki recent-changes server and listens for
    edits. Users cannot interact with this part of the bot. When an event
    occurs, we run it through some rules stored in our config, which can result
    in wiki bot tasks being started or messages being sent to channels on the
    IRC frontend.
    """

    def __init__(self, bot):
        self.bot = bot
        cf = bot.config.irc["watcher"]
        base = super(Watcher, self)
        base.__init__(cf["host"], cf["port"], cf["nick"], cf["ident"],
                      cf["realname"], bot.logger.getChild("watcher"))
        self._prepare_process_hook()
        self._connect()

    def __repr__(self):
        """Return the canonical string representation of the Watcher."""
        res = "Watcher(host={0!r}, port={1!r}, nick={2!r}, ident={3!r}, realname={4!r}, bot={5!r})"
        return res.format(self.host, self.port, self.nick, self.ident,
                          self.realname, self.bot)

    def __str__(self):
        """Return a nice string representation of the Watcher."""
        res = "<Watcher {0}!{1} at {2}:{3}>"
        return res.format(self.nick, self.ident, self.host, self.port)

    def _process_message(self, line):
        """Process a single message from IRC."""
        if line[1] == "PRIVMSG":
            chan = line[2]

            # Ignore messages originating from channels not in our list, to
            # prevent someone PMing us false data:
            if chan not in self.bot.config.irc["watcher"]["channels"]:
                return

            msg = " ".join(line[3:])[1:]
            rc = RC(chan, msg)  # New RC object to store this event's data
            rc.parse()  # Parse a message into pagenames, usernames, etc.
            self._process_rc_event(rc)

        # When we've finished starting up, join all watcher channels:
        elif line[1] == "376":
            for chan in self.bot.config.irc["watcher"]["channels"]:
                self.join(chan)

    def _prepare_process_hook(self):
        """Create our RC event process hook from information in config.

        This will get put in the function self._process_hook, which takes the
        Bot object and an RC object and returns a list of frontend channels to
        report this event to.
        """
        # Set a default RC process hook that does nothing:
        self._process_hook = lambda rc: ()
        try:
            rules = self.bot.config.data["rules"]
        except KeyError:
            return
        module = imp.new_module("_rc_event_processing_rules")
        path = self.bot.config.path
        try:
            exec compile(rules, path, "exec") in module.__dict__
        except Exception:
            e = "Could not compile config file's RC event rules:"
            self.logger.exception(e)
            return
        self._process_hook_module = module
        try:
            self._process_hook = module.process
        except AttributeError:
            e = "RC event rules compiled correctly, but no process(bot, rc) function was found"
            self.logger.error(e)
            return

    def _process_rc_event(self, rc):
        """Process a recent change event from IRC (or, an RC object).

        The actual processing is configurable, so we don't have that hard-coded
        here. We simply call our process hook (self._process_hook), created by
        self._prepare_process_hook() from information in the "rules" section of
        our config.
        """
        chans = self._process_hook(self.bot, rc)
        with self.bot.component_lock:
            frontend = self.bot.frontend
            if chans and frontend and not frontend.is_stopped():
                pretty = rc.prettify()
                for chan in chans:
                    frontend.say(chan, pretty[:500 - len(chan)])
<commit_msg>Move watcher rules to a dedicated file instead of inside config.<commit_after># -*- coding: utf-8  -*-
#
# Copyright (C) 2009-2012 Ben Kurtovic <ben.kurtovic@verizon.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import imp
import os

from earwigbot.irc import IRCConnection, RC

__all__ = ["Watcher"]

class Watcher(IRCConnection):
    """
    **EarwigBot: IRC Watcher Component**

    The IRC watcher runs on a wiki recent-changes server and listens for
    edits. Users cannot interact with this part of the bot. When an event
    occurs, we run it through some rules stored in our working directory under
    :file:`rules.py`, which can result in wiki bot tasks being started or
    messages being sent to channels on the IRC frontend.
    """

    def __init__(self, bot):
        self.bot = bot
        cf = bot.config.irc["watcher"]
        base = super(Watcher, self)
        base.__init__(cf["host"], cf["port"], cf["nick"], cf["ident"],
                      cf["realname"], bot.logger.getChild("watcher"))
        self._prepare_process_hook()
        self._connect()

    def __repr__(self):
        """Return the canonical string representation of the Watcher."""
        res = "Watcher(host={0!r}, port={1!r}, nick={2!r}, ident={3!r}, realname={4!r}, bot={5!r})"
        return res.format(self.host, self.port, self.nick, self.ident,
                          self.realname, self.bot)

    def __str__(self):
        """Return a nice string representation of the Watcher."""
        res = "<Watcher {0}!{1} at {2}:{3}>"
        return res.format(self.nick, self.ident, self.host, self.port)

    def _process_message(self, line):
        """Process a single message from IRC."""
        if line[1] == "PRIVMSG":
            chan = line[2]

            # Ignore messages originating from channels not in our list, to
            # prevent someone PMing us false data:
            if chan not in self.bot.config.irc["watcher"]["channels"]:
                return

            msg = " ".join(line[3:])[1:]
            rc = RC(chan, msg)  # New RC object to store this event's data
            rc.parse()  # Parse a message into pagenames, usernames, etc.
            self._process_rc_event(rc)

        # When we've finished starting up, join all watcher channels:
        elif line[1] == "376":
            for chan in self.bot.config.irc["watcher"]["channels"]:
                self.join(chan)

    def _prepare_process_hook(self):
        """Create our RC event process hook from information in rules.py.

        This will get put in the function self._process_hook, which takes the
        Bot object and an RC object and returns a list of frontend channels to
        report this event to.
        """
        # Set a default RC process hook that does nothing:
        self._process_hook = lambda bot, rc: ()

        path = os.path.join(self.bot.config.root_dir, "rules.py")
        f, path, desc = imp.find_module("rules", [path])
        try:
            module = imp.load_module(name, f, path, desc)
        except Exception:
            return
        finally:
            f.close()

        self._process_hook_module = module
        try:
            self._process_hook = module.process
        except AttributeError:
            e = "RC event rules imported correctly, but no process(bot, rc) function was found"
            self.logger.error(e)
            return

    def _process_rc_event(self, rc):
        """Process a recent change event from IRC (or, an RC object).

        The actual processing is configurable, so we don't have that hard-coded
        here. We simply call our process hook (self._process_hook), created by
        self._prepare_process_hook() from information in the "rules" section of
        our config.
        """
        chans = self._process_hook(self.bot, rc)
        with self.bot.component_lock:
            frontend = self.bot.frontend
            if chans and frontend and not frontend.is_stopped():
                pretty = rc.prettify()
                for chan in chans:
                    frontend.say(chan, pretty[:500 - len(chan)])
<|endoftext|>


<commit_before>import os
import unittest

from grep import grep

# Tests adapted from `problem-specifications//canonical-data.json` @ v1.0.0

ILIADFILENAME = 'iliad.txt'
ILIADCONTENTS = '''Achilles sing, O Goddess! Peleus' son;
His wrath pernicious, who ten thousand woes
Caused to Achaia's host, sent many a soul
Illustrious into Ades premature,
And Heroes gave (so stood the will of Jove)
To dogs and to all ravening fowls a prey,
When fierce dispute had separated once
The noble Chief Achilles from the son
Of Atreus, Agamemnon, King of men.
'''

MIDSUMMERNIGHTFILENAME = 'midsummer-night.txt'
MIDSUMMERNIGHTCONTENTS = '''I do entreat your grace to pardon me.
I know not by what power I am made bold,
Nor how it may concern my modesty,
In such a presence here to plead my thoughts;
But I beseech your grace that I may know
The worst that may befall me in this case,
If I refuse to wed Demetrius.
'''

PARADISELOSTFILENAME = 'paradise-lost.txt'
PARADISELOSTCONTENTS = '''Of Mans First Disobedience, and the Fruit
Of that Forbidden Tree, whose mortal tast
Brought Death into the World, and all our woe,
With loss of Eden, till one greater Man
Restore us, and regain the blissful Seat,
Sing Heav'nly Muse, that on the secret top
Of Oreb, or of Sinai, didst inspire
That Shepherd, who first taught the chosen Seed
'''

def remove_file(file_name):
    try:
        os.remove(file_name)
    except OSError:
        pass

def create_file(name, contents):
    with open(name, 'w') as f:
        f.write(contents)

class GrepTest(unittest.TestCase):
    @classmethod
    def setUpClass(self):
        create_file(ILIADFILENAME, ILIADCONTENTS)
        create_file(MIDSUMMERNIGHTFILENAME, MIDSUMMERNIGHTCONTENTS)
        create_file(PARADISELOSTFILENAME, PARADISELOSTCONTENTS)

    @classmethod
    def tearDownClass(self):
        remove_file(ILIADFILENAME)
        remove_file(MIDSUMMERNIGHTFILENAME)
        remove_file(PARADISELOSTFILENAME)

    def test_one_file_one_match_no_flags(self):
        self.assertMultiLineEqual(
            grep("Agamemnon", [ILIADFILENAME]),
            "Of Atreus, Agamemnon, King of men.\n")

    def test_one_file_one_match_print_line_numbers_flag(self):
        self.assertMultiLineEqual(
            grep("Forbidden", [PARADISELOSTFILENAME], "-n"),
            "2:Of that Forbidden Tree, whose mortal tast\n")

    def test_one_file_one_match_case_insensitive_flag(self):
        self.assertMultiLineEqual(
            grep("FORBIDDEN", [PARADISELOSTFILENAME], "-i"),
            "Of that Forbidden Tree, whose mortal tast\n")

    def test_one_file_one_match_print_file_names_flag(self):
        self.assertMultiLineEqual(
            grep("Forbidden", [PARADISELOSTFILENAME], "-l"),
            PARADISELOSTFILENAME + '\n')

    def test_one_file_one_match_match_entire_lines_flag(self):
        self.assertMultiLineEqual(
            grep("With loss of Eden, till one greater Man",
                 [PARADISELOSTFILENAME], "-x"),
            "With loss of Eden, till one greater Man\n")

    def test_one_file_one_match_multiple_flags(self):
        self.assertMultiLineEqual(
            grep("OF ATREUS, Agamemnon, KIng of MEN.", [ILIADFILENAME],
                 "-n -i -x"), "9:Of Atreus, Agamemnon, King of men.\n")

    def test_one_file_several_matches_no_flags(self):
        self.assertMultiLineEqual(
            grep("may", [MIDSUMMERNIGHTFILENAME]),
            ("Nor how it may concern my modesty,\n"
             "But I beseech your grace that I may know\n"
             "The worst that may befall me in this case,\n"))

    def test_one_file_several_matches_print_line_numbers_flag(self):
        self.assertMultiLineEqual(
            grep("may", [MIDSUMMERNIGHTFILENAME], "-n"),
            ("3:Nor how it may concern my modesty,\n"
             "5:But I beseech your grace that I may know\n"
             "6:The worst that may befall me in this case,\n"))

    def test_one_file_several_matches_match_entire_lines_flag(self):
        self.assertMultiLineEqual(
            grep("may", [MIDSUMMERNIGHTFILENAME], "-x"), "")

    def test_one_file_several_matches_case_insensitive_flag(self):
        self.assertMultiLineEqual(
            grep("ACHILLES", [ILIADFILENAME], "-i"),
            ("Achilles sing, O Goddess! Peleus' son;\n"
             "The noble Chief Achilles from the son\n"))

    def test_one_file_several_matches_inverted_flag(self):
        self.assertMultiLineEqual(
            grep("Of", [PARADISELOSTFILENAME], "-v"),
            ("Brought Death into the World, and all our woe,\n"
             "With loss of Eden, till one greater Man\n"
             "Restore us, and regain the blissful Seat,\n"
             "Sing Heav'nly Muse, that on the secret top\n"
             "That Shepherd, who first taught the chosen Seed\n"))

    def test_one_file_no_matches_various_flags(self):
        self.assertMultiLineEqual(
            grep("Gandalf", [ILIADFILENAME], "-n -l -x -i"), "")

    def test_multiple_files_one_match_no_flags(self):
        self.assertMultiLineEqual(
            grep(
                "Agamemnon",
                [ILIADFILENAME, MIDSUMMERNIGHTFILENAME, PARADISELOSTFILENAME]),
            "iliad.txt:Of Atreus, Agamemnon, King of men.\n")

    def test_multiple_files_several_matches_no_flags(self):
        self.assertMultiLineEqual(
            grep("may",
                 ["iliad.txt", "midsummer-night.txt", "paradise-lost.txt"]),
            ("midsummer-night.txt:Nor how it may concern my modesty,\n"
             "midsummer-night.txt:But I beseech your grace that I may know\n"
             "midsummer-night.txt:The worst that may befall me in this case,\n"
             ))

    def test_multiple_files_several_matches_print_line_numbers_flag(self):
        self.assertMultiLineEqual(
            grep("that",
                 ["iliad.txt", "midsummer-night.txt", "paradise-lost.txt"],
                 "-n"),
            ("midsummer-night.txt:5:But I beseech your grace that I may know\n"
             "midsummer-night.txt:6:The worst that may befall me in this case,"
             "\nparadise-lost.txt:2:Of that Forbidden Tree, whose mortal tast"
             "\nparadise-lost.txt:6:Sing Heav'nly Muse, that on the secret top"
             "\n"))

    def test_multiple_files_one_match_print_file_names_flag(self):
        self.assertMultiLineEqual(
            grep("who",
                 ["iliad.txt", "midsummer-night.txt", "paradise-lost.txt"],
                 "-l"), ILIADFILENAME + '\n' + PARADISELOSTFILENAME + '\n')

    def test_multiple_files_several_matches_case_insensitive_flag(self):
        self.assertMultiLineEqual(
            grep("TO",
                 ["iliad.txt", "midsummer-night.txt", "paradise-lost.txt"],
                 "-i"),
            ("iliad.txt:Caused to Achaia's host, sent many a soul\n"
             "iliad.txt:Illustrious into Ades premature,\n"
             "iliad.txt:And Heroes gave (so stood the will of Jove)\n"
             "iliad.txt:To dogs and to all ravening fowls a prey,\n"
             "midsummer-night.txt:I do entreat your grace to pardon me.\n"
             "midsummer-night.txt:In such a presence here to plead my thoughts"
             ";\nmidsummer-night.txt:If I refuse to wed Demetrius.\n"
             "paradise-lost.txt:Brought Death into the World, and all our woe,"
             "\nparadise-lost.txt:Restore us, and regain the blissful Seat,\n"
             "paradise-lost.txt:Sing Heav'nly Muse, that on the secret top\n"))

    def test_multiple_files_several_matches_inverted_flag(self):
        self.assertMultiLineEqual(
            grep(
                "a", ["iliad.txt", "midsummer-night.txt", "paradise-lost.txt"],
                "-v"), ("iliad.txt:Achilles sing, O Goddess! Peleus' son;\n"
                        "iliad.txt:The noble Chief Achilles from the son\n"
                        "midsummer-night.txt:If I refuse to wed Demetrius.\n"))

    def test_multiple_files_one_match_match_entire_lines_flag(self):
        self.assertMultiLineEqual(
            grep("But I beseech your grace that I may know",
                 ["iliad.txt", "midsummer-night.txt", "paradise-lost.txt"],
                 "-x"),
            "midsummer-night.txt:But I beseech your grace that I may know\n")

    def test_multiple_files_one_match_multiple_flags(self):
        self.assertMultiLineEqual(
            grep("WITH LOSS OF EDEN, TILL ONE GREATER MAN",
                 ["iliad.txt", "midsummer-night.txt", "paradise-lost.txt"],
                 "-n -i -x"),
            "paradise-lost.txt:4:With loss of Eden, till one greater Man\n")

    def test_multiple_files_no_matches_various_flags(self):
        self.assertMultiLineEqual(
            grep("Frodo",
                 ["iliad.txt", "midsummer-night.txt", "paradise-lost.txt"],
                 "-n -l -x -i"), "")

if __name__ == '__main__':
    unittest.main()
<commit_msg>grep: replace usage of real files (#1159)<commit_after>import unittest
try:
    import builtins
except ImportError:
    import __builtin__ as builtins

from grep import grep

# Tests adapted from `problem-specifications//canonical-data.json` @ v1.0.0

ILIADFILENAME = 'iliad.txt'
ILIADCONTENTS = '''Achilles sing, O Goddess! Peleus' son;
His wrath pernicious, who ten thousand woes
Caused to Achaia's host, sent many a soul
Illustrious into Ades premature,
And Heroes gave (so stood the will of Jove)
To dogs and to all ravening fowls a prey,
When fierce dispute had separated once
The noble Chief Achilles from the son
Of Atreus, Agamemnon, King of men.'''

MIDSUMMERNIGHTFILENAME = 'midsummer-night.txt'
MIDSUMMERNIGHTCONTENTS = '''I do entreat your grace to pardon me.
I know not by what power I am made bold,
Nor how it may concern my modesty,
In such a presence here to plead my thoughts;
But I beseech your grace that I may know
The worst that may befall me in this case,
If I refuse to wed Demetrius.'''

PARADISELOSTFILENAME = 'paradise-lost.txt'
PARADISELOSTCONTENTS = '''Of Mans First Disobedience, and the Fruit
Of that Forbidden Tree, whose mortal tast
Brought Death into the World, and all our woe,
With loss of Eden, till one greater Man
Restore us, and regain the blissful Seat,
Sing Heav'nly Muse, that on the secret top
Of Oreb, or of Sinai, didst inspire
That Shepherd, who first taught the chosen Seed'''
FILENAMES = [
    ILIADFILENAME,
    MIDSUMMERNIGHTFILENAME,
    PARADISELOSTFILENAME,
]
FILES = {}

class File(object):
    def __init__(self, name=''):
        self.name = name
        self.contents = ''

    def read(self):
        return self.contents

    def readlines(self):
        return [line + '\n' for line in self.read().split('\n') if line]

    def write(self, data):
        self.contents += data

    def __enter__(self):
        return self

    def __exit__(self, *args):
        pass

# Store builtin definition of open()
builtins.oldopen = builtins.open

def open(name, mode='r', *args, **kwargs):
    # if name is a mocked file name, lookup corresponding mocked file
    if name in FILENAMES:
        if mode == 'w' or name not in FILES:
            FILES[name] = File(name)
        return FILES[name]
    # else call builtin open()
    else:
        return builtins.oldopen(name, mode, *args, **kwargs)

builtins.open = open

# remove mocked file contents
def remove_file(file_name):
    del FILES[file_name]

def create_file(name, contents):
    with open(name, 'w') as f:
        f.write(contents)

class GrepTest(unittest.TestCase):
    @classmethod
    def setUpClass(self):
        # Override builtin open() with mock-file-enabled one
        builtins.open = open
        create_file(ILIADFILENAME, ILIADCONTENTS)
        create_file(MIDSUMMERNIGHTFILENAME, MIDSUMMERNIGHTCONTENTS)
        create_file(PARADISELOSTFILENAME, PARADISELOSTCONTENTS)

    @classmethod
    def tearDownClass(self):
        remove_file(ILIADFILENAME)
        remove_file(MIDSUMMERNIGHTFILENAME)
        remove_file(PARADISELOSTFILENAME)
        # Restore builtin open()
        builtins.open = builtins.oldopen

    def test_one_file_one_match_no_flags(self):
        self.assertMultiLineEqual(
            grep("Agamemnon", [ILIADFILENAME]),
            "Of Atreus, Agamemnon, King of men.\n"
        )

    def test_one_file_one_match_print_line_numbers_flag(self):
        self.assertMultiLineEqual(
            grep("Forbidden", [PARADISELOSTFILENAME], "-n"),
            "2:Of that Forbidden Tree, whose mortal tast\n"
        )

    def test_one_file_one_match_case_insensitive_flag(self):
        self.assertMultiLineEqual(
            grep("FORBIDDEN", [PARADISELOSTFILENAME], "-i"),
            "Of that Forbidden Tree, whose mortal tast\n"
        )

    def test_one_file_one_match_print_file_names_flag(self):
        self.assertMultiLineEqual(
            grep("Forbidden", [PARADISELOSTFILENAME], "-l"),
            PARADISELOSTFILENAME + '\n'
        )

    def test_one_file_one_match_match_entire_lines_flag(self):
        self.assertMultiLineEqual(
            grep("With loss of Eden, till one greater Man",
                 [PARADISELOSTFILENAME], "-x"),
            "With loss of Eden, till one greater Man\n"
        )

    def test_one_file_one_match_multiple_flags(self):
        self.assertMultiLineEqual(
            grep(
                "OF ATREUS, Agamemnon, KIng of MEN.",
                [ILIADFILENAME],
                "-n -i -x"
            ),
            "9:Of Atreus, Agamemnon, King of men.\n"
        )

    def test_one_file_several_matches_no_flags(self):
        self.assertMultiLineEqual(
            grep("may", [MIDSUMMERNIGHTFILENAME]),
            "Nor how it may concern my modesty,\n"
            "But I beseech your grace that I may know\n"
            "The worst that may befall me in this case,\n"
        )

    def test_one_file_several_matches_print_line_numbers_flag(self):
        self.assertMultiLineEqual(
            grep("may", [MIDSUMMERNIGHTFILENAME], "-n"),
            "3:Nor how it may concern my modesty,\n"
            "5:But I beseech your grace that I may know\n"
            "6:The worst that may befall me in this case,\n"
        )

    def test_one_file_several_matches_match_entire_lines_flag(self):
        self.assertMultiLineEqual(
            grep("may", [MIDSUMMERNIGHTFILENAME], "-x"),
            ""
        )

    def test_one_file_several_matches_case_insensitive_flag(self):
        self.assertMultiLineEqual(
            grep("ACHILLES", [ILIADFILENAME], "-i"),
            "Achilles sing, O Goddess! Peleus' son;\n"
            "The noble Chief Achilles from the son\n"
        )

    def test_one_file_several_matches_inverted_flag(self):
        self.assertMultiLineEqual(
            grep("Of", [PARADISELOSTFILENAME], "-v"),
            "Brought Death into the World, and all our woe,\n"
            "With loss of Eden, till one greater Man\n"
            "Restore us, and regain the blissful Seat,\n"
            "Sing Heav'nly Muse, that on the secret top\n"
            "That Shepherd, who first taught the chosen Seed\n"
        )

    def test_one_file_no_matches_various_flags(self):
        self.assertMultiLineEqual(
            grep("Gandalf", [ILIADFILENAME], "-n -l -x -i"),
            ""
        )

    def test_multiple_files_one_match_no_flags(self):
        self.assertMultiLineEqual(
            grep("Agamemnon", FILENAMES),
            "iliad.txt:Of Atreus, Agamemnon, King of men.\n"
        )

    def test_multiple_files_several_matches_no_flags(self):
        self.assertMultiLineEqual(
            grep("may", FILENAMES),
            "midsummer-night.txt:Nor how it may concern my modesty,\n"
            "midsummer-night.txt:But I beseech your grace that I may know\n"
            "midsummer-night.txt:The worst that may befall me in this case,\n"
        )

    def test_multiple_files_several_matches_print_line_numbers_flag(self):
        expected = (
            "midsummer-night.txt:5:But I beseech your grace that I may know\n"
            "midsummer-night.txt:6:The worst that may befall me in this case,"
            "\nparadise-lost.txt:2:Of that Forbidden Tree, whose mortal tast\n"
            "paradise-lost.txt:6:Sing Heav'nly Muse, that on the secret top\n"
        )
        self.assertMultiLineEqual(
            grep("that", FILENAMES, "-n"),
            expected
        )

    def test_multiple_files_one_match_print_file_names_flag(self):
        self.assertMultiLineEqual(
            grep("who", FILENAMES, "-l"),
            ILIADFILENAME + '\n' + PARADISELOSTFILENAME + '\n')

    def test_multiple_files_several_matches_case_insensitive_flag(self):
        expected = (
            "iliad.txt:Caused to Achaia's host, sent many a soul\n"
            "iliad.txt:Illustrious into Ades premature,\n"
            "iliad.txt:And Heroes gave (so stood the will of Jove)\n"
            "iliad.txt:To dogs and to all ravening fowls a prey,\n"
            "midsummer-night.txt:I do entreat your grace to pardon me.\n"
            "midsummer-night.txt:In such a presence here to plead my thoughts;"
            "\nmidsummer-night.txt:If I refuse to wed Demetrius.\n"
            "paradise-lost.txt:Brought Death into the World, and all our woe,"
            "\nparadise-lost.txt:Restore us, and regain the blissful Seat,\n"
            "paradise-lost.txt:Sing Heav'nly Muse, that on the secret top\n"
        )
        self.assertMultiLineEqual(
            grep("TO", FILENAMES, "-i"),
            expected
        )

    def test_multiple_files_several_matches_inverted_flag(self):
        self.assertMultiLineEqual(
            grep("a", FILENAMES, "-v"),
            "iliad.txt:Achilles sing, O Goddess! Peleus' son;\n"
            "iliad.txt:The noble Chief Achilles from the son\n"
            "midsummer-night.txt:If I refuse to wed Demetrius.\n"
        )

    def test_multiple_files_one_match_match_entire_lines_flag(self):
        self.assertMultiLineEqual(
            grep("But I beseech your grace that I may know",
                 FILENAMES, "-x"),
            "midsummer-night.txt:But I beseech your grace that I may know\n")

    def test_multiple_files_one_match_multiple_flags(self):
        self.assertMultiLineEqual(
            grep("WITH LOSS OF EDEN, TILL ONE GREATER MAN",
                 FILENAMES, "-n -i -x"),
            "paradise-lost.txt:4:With loss of Eden, till one greater Man\n")

    def test_multiple_files_no_matches_various_flags(self):
        self.assertMultiLineEqual(
            grep("Frodo", FILENAMES, "-n -l -x -i"),
            ""
        )

if __name__ == '__main__':
    unittest.main()
<|endoftext|>