<commit_before>'''
Given a list of candidate public domain book titles and authors attempt to find the best matching
OpenLibrary records with full text available on Internet Archive.
'''

import codecs
import requests
import requests_cache
from time import clock,sleep

RATE = 2.0

requests_cache.install_cache('openlibrary')

count = 0

def search_open_library(author,title, language):
    title = title.split(':')[0].strip() # main title only
    author = author.strip()
    payload = {'has_fulltext' : 'true', 'title': title} #, 'language': language}
    if author:
        payload.update({'author': author})
    response = requests.get('http://openlibrary.org/search.json',params=payload)
    if response.status_code == 200:
        docs = response.json()['docs']
        print '\n%d OpenLibrary works found for %s by %s:' % (len(docs),title,author)
        return docs
    else:
        print 'Request failed',response.status_code,payload
        return []

def check_language(lang,doc):
    if 'language' in doc:
        if  lang in doc['language']:
            return True
        else:
            print 'Non-English version ',doc['key'],doc['language']
    else:
        # TODO:
        #aid = doc['ocaid']
        #marc_url = 'https://archive.org/download/%s/%s_marc.xml' % (aid,aid)
        # download MARC
        # check for 'English' in 240$l

        print 'Unknown language ',doc['key']
        return False

def all_editions(doc):
    '''
    Merge the contents of the two IA editions fields
    (one may be a superset of the other, but lets just be safe)
    '''
    editions = set()
    for k in ('ia_loaded_id','ia'):
        if k in doc:
            editions.update(doc[k])
    print len(editions), ' editions'
    return editions

def get_ia_edition(iaid):
    '''Get JSON for an edition using its IA identifier.  Follows non-HTTP OpenLibrary redirect records '''
    edition_url = 'http://openlibrary.org/books/ia:%s.json' % iaid
    edition = requests.get(edition_url).json()
    if 'type' in edition and 'key' in edition['type'] and edition['type']['key'] == '/type/redirect':
        edition_url =  'http://openlibrary.org%s.json' % edition['location']
        edition = requests.get(edition_url).json()
    return edition

def test_file_availability(iaid,suffix):
    '''
    Test whether a file in the given format is available for an Internet Archive ID.
    Follows redirects if necessary.

    If the "suffix" parameter doesnt contain a period, one will be prepended.
    This allows both "epub" and "_files.xml" style suffixes.
    '''
    #files = requests.get('http://archive.org/download/%s/%s_files.xml' % (iaid,iaid))
    if suffix.find('.') < 0:
        suffix = '.'+suffix
    epub_url = 'http://archive.org/download/%s/%s%s' % (iaid,iaid,suffix)
    epub = requests.head(epub_url)
    if epub.status_code == 302:
        #print 'Redirecting to ',epub.headers['location']
        epub = requests.head(epub.headers['location'])
    if epub.status_code == 200:
        return epub_url
    elif epub.status_code != 403:
        print 'HTTP error (%d) fetching %s for %s :from %s ',(epub.status_code,suffix,iaid,epub_url)
    return None

def rate_wait(time):
    last = now()
    while True:
        yield()

def main():
    with codecs.open('../data/SCCL-classics-candidates-from-editions.tsv','w',encoding='utf-8') as output:
        count = 0
        for line in codecs.open('../data/SCCL-classics-edition-titles.tsv', encoding='utf8'):
            count += 1
            title,author = line.rstrip('\n').split('\t')
            docs = search_open_library(author, title, 'eng')
            if not docs:
                # Output a blank record so we know it got no matches
                print 'No matches for %s by %s' % (title, author)
                output.write('\t'.join([title, author])+'\n')
            for doc in docs:
                key = doc['key']
                #if not 'public_scan_b' in doc or doc['public_scan_b']: # not reliable
                if check_language('eng',doc):
                    nonIA = 0
                    for e in all_editions(doc):
                        last = clock()
                        edition = get_ia_edition(e)
                        if 'ocaid' in edition:
                            ia = edition['ocaid']
                            key = edition['key']
                            date = edition['publish_date'] if 'publish_date' in edition else ''
                            epub_url = test_file_availability(ia,'epub')
                            if epub_url:
                                ol_edition_url = 'http://openlibrary.org' + key
                                print '\t'.join([ date, ol_edition_url, epub_url])
                                output.write('\t'.join([title, author, date, ol_edition_url, epub_url])+'\n')
                        else:
                            nonIA += 1
                            print 'OL edition record unexpectedly missing "ocaid" key',edition
                        elapsed = last-clock()
                        last = clock()
                        if elapsed > (1.0/RATE):
                            sleep((1.0/RATE)-elapsed)
                    # print 'Editions with no IA equiv = %d' % nonIA

    print count,title,author

main()
<commit_msg>Final version for Nov 2013 run. Add caching, throttling, stricter selection criteria.<commit_after>'''
Given a list of candidate public domain book titles and authors attempt to find the best matching
OpenLibrary records with full text available on Internet Archive.
'''

import codecs
import xml.etree.ElementTree  as ET
import pymarc
from requests import ConnectionError
import requests_cache
import time

CACHE_DIR = '../cache/'
DATA_DIR = '../data/'
RATE = 2.0 # requests/second
count = 0

requests_cache.install_cache('openlibrary')
session = requests_cache.CachedSession()

def make_throttle_hook(timeout=1.0):
    """
    Returns a response hook function which sleeps for `timeout` seconds if
    response is not cached
    """
    def hook(response, **kwargs):
        if not getattr(response, 'from_cache', False):
            time.sleep(timeout)
        return response
    return hook

session.hooks = {'response': make_throttle_hook(1.0/RATE)}

def search_open_library(author,title, language):
    result = []
    author = author.strip()
    author = ','.join(author.split(',')[0:2])
    if author and author[-1] == '.':
        author = author[0:-1]
    payload = {'has_fulltext' : 'true', 'title': title} #, 'language': language}
    if author:
        payload.update({'author': author})
    response = session.get('http://openlibrary.org/search.json',params=payload)
    if response.status_code == 200:
        docs = response.json()['docs']
        result = docs
    else:
        print 'Request failed',response.status_code,payload
    return result

def check_language(lang,doc):
    if 'language' in doc:
        if  lang in doc['language']:
            return True
        else:
            print 'Non-English version ',doc['key'],doc['language']
    else:
        # TODO:
        #aid = doc['ocaid']
        #marc_url = 'https://archive.org/download/%s/%s_marc.xml' % (aid,aid)
        # download MARC
        # check for 'English' in 240$l

        print 'Unknown language ',doc['key']
        return False

def all_editions(doc):
    '''
    Merge the contents of the two IA editions fields
    (one may be a superset of the other, but lets just be safe)
    '''
    editions = set()
    for k in ('ia_loaded_id','ia'):
        if k in doc:
            editions.update(doc[k])
    print len(editions), ' editions'
    return editions

def get_json(url):
    response = session.get(url)
    if response.status_code == 200:
        edition = response.json()
        return edition
    else:
        print 'Failed to get JSON for %s - status code %d' % (url,response.status_code)

def get_ia_edition(iaid):
    '''Get JSON for an edition using its IA identifier.  Follows non-HTTP OpenLibrary redirect records '''
    edition_url = 'http://openlibrary.org/books/ia:%s.json' % iaid
    edition = get_json(edition_url)
    if edition and 'type' in edition and 'key' in edition['type'] and edition['type']['key'] == '/type/redirect':
        edition_url =  'http://openlibrary.org%s.json' % edition['location']
        edition = get_json(edition_url)
    return edition

def get_file(iaid,suffix,body=False):
    '''
    Test whether a file in the given format is available for an Internet Archive ID.
    Follows redirects if necessary.

    If the "suffix" parameter doesn't contain a period, one will be prepended.
    This allows both "epub" and "_files.xml" style suffixes.

    if body=False (default), the content will not be fetched.  Redirects are followed and the
    final URL is returned.  If this is false, the actual content will be fetched and returned.
    '''
    #files = session.get('http://archive.org/download/%s/%s_files.xml' % (iaid,iaid))
    if suffix.find('.') < 0:
        suffix = '.'+suffix
    url = 'http://archive.org/download/%s/%s%s' % (iaid,iaid,suffix)

    try:
        if not body:
            epub = session.head(url)
        else:
            epub = session.get(url)

        if epub.status_code == 302:
            url = epub.headers['location']
            #print 'Redirecting to ',url
            if not body:
                epub = session.head(url)
            else:
                epub = session.get(url)
        if epub.status_code == 200:
            return epub if body else url
        elif epub.status_code != 403:
            print 'HTTP error (%d) fetching %s for %s from %s ' % (epub.status_code,suffix,iaid,url)
    except ConnectionError as e:
        print 'Error fetching URL: ',url,e
    return None

def rate_wait(time):
    last = now()
    while True:
        yield()

def merge(base, added):
    keys = [b['key'] for b in base]
    base += [a for a in added if not a['key'] in keys]
    return base

def publicdomain(date):
    if not date:
        return False
    try:
        return int(date) < 1923
    except ValueError:
        return False

def get_files(ia):
    suffix = '_files.xml'
    filename = CACHE_DIR + ia + suffix
    root = None

    # check cache
    try:
        with file(filename, 'r') as cachefile:
            root = ET.parse(cachefile).getroot()
    except IOError:
        files_xml =  get_file(ia,suffix,body=True)
        if files_xml and files_xml.status_code == 200:
            with file(filename, 'w') as output:
                output.write(files_xml.content)
            root = ET.fromstring(files_xml.content)
    if root is not None:
        files = [f.get('name') for f in root.findall('file')]
        return files

def find_file(files,suffix):
    if files and suffix:
        for f in files:
            if f[-len(suffix):len(f)] == suffix:
                return f

def main():
    with codecs.open(DATA_DIR+'SCCL-classics-ebook-candidates.tsv','w',encoding='utf-8') as output:
        count = 0
        for line in codecs.open(DATA_DIR+'SCCL-classics-edition-author-work.tsv', encoding='utf-8'):
            count += 1
            if count == 1:
                continue # skip header line
            title,author,work_title = line.rstrip('\n').split('\t')
            title = title.split(':')[0].strip() # main title only
            docs = search_open_library(author, title, 'eng')
            print '\n%d OpenLibrary works found for %s by %s:' % (len(docs),title,author)

            if work_title and work_title != title:
                before = len(docs)
                docs = merge(docs,search_open_library(author, title, 'eng'))
                added = len(docs) - before
                if added:
                    print 'Added %d new search results' % added

            if not docs:
                # Output a blank record so we know it got no matches
                print 'No matches for %s by %s' % (title, author)
                output.write('\t'.join([title, author])+'\n')
            for doc in docs:
                key = doc['key']
                #if not 'public_scan_b' in doc or doc['public_scan_b']: # not reliable
                if True or check_language('eng',doc):
                    nonIA = 0
                    for e in all_editions(doc):
                        edition = get_ia_edition(e)
                        if edition and 'ocaid' in edition:
                            ia = edition['ocaid']
                            key = edition['key']
                            date = edition['publish_date'] if 'publish_date' in edition else ''
                            if publicdomain(date):
                                #print date
                                files = get_files(ia)
                                # ePub is not listed typically
                                #epub_url = find_file(files,'epub')
                                abbyy_url = find_file(files,'_abbyy.gz')
                                marc_url = find_file(files,'_meta.mrc')
                                if abbyy_url and marc_url:
                                    response = get_file(ia,'_meta.mrc',body=True)
                                    if response and response.status_code == 200:
                                        try:
                                            marcfile = pymarc.MARCReader(response.content)
                                            marc = marcfile.next()
                                            field = marc['008'].value()
                                            yr = field[7:11]
                                            lang = field[35:38]
                                            print yr, lang
                                            ol_edition_url = 'http://openlibrary.org' + key
                                            #print '\t'.join([ date, ol_edition_url, epub_url])
                                            if lang == 'eng':
                                                # We have a winner! - write it to our output file.
                                                output.write('\t'.join([title, author, date, ol_edition_url])+'\n')
                                            else:
                                                print 'Skipping lang: ',lang
                                        except:
                                            print 'Failed to parse MARC record ',marc_url
                                    else:
                                        print 'No MARC record ',ia,key
                                else:
                                    print 'Skipping ',ia,key,abbyy_url,marc_url
                        else:
                            nonIA += 1
                            print 'OL edition record unexpectedly missing "ocaid" key',edition
                    # print 'Editions with no IA equiv = %d' % nonIA

    print count,title,author

main()
<|endoftext|>


<commit_before>import re
import os
import sys
import parse
import time
import itertools
import datetime
import logging
from IPython.parallel import Client
import requests
import zipfile
import cStringIO as StringIO
from BeautifulSoup import BeautifulSoup as bs
import lib.alchemy as alchemy

sys.path.append('lib')
from config_parser import get_config_options

logfile = "./" + 'xml-parsing.log'
logging.basicConfig(filename=logfile, level=logging.DEBUG)

# table bookkeeping for the parse script

def get_year_list(yearstring):
    """
    Given a [yearstring] of forms
    year1
    year1-year2
    year1,year2,year3
    year1-year2,year3-year4
    Expands into a list of year integers, and returns
    """
    years = []
    for subset in yearstring.split(','):
        if subset == 'latest':
            years.append('latest')
            continue
        sublist = subset.split('-')
        start = int(sublist[0])
        end = int(sublist[1])+1 if len(sublist) > 1 else start+1
        years.extend(range(start,end))
    return years

def generate_download_list(years):
    """
    Given the year string from the configuration file, return
    a list of urls to be downloaded
    """
    if not years: return []
    urls = []
    url = requests.get('https://www.google.com/googlebooks/uspto-patents-grants-text.html')
    soup = bs(url.content)
    years = get_year_list(years)

    # latest file link
    if 'latest' in years:
        a = soup.h3.findNext('h3').findPrevious('a')
        urls.append(a['href'])
        years.remove('latest')
    # get year links
    for year in years:
        header = soup.find('h3', {'id': str(year)})
        a = header.findNext()
        while a.name != 'h3':
            urls.append(a['href'])
            a = a.findNext()
    return urls

def download_files():
    """
    [downloaddir]: string representing base download directory. Will download
    files to this directory in folders named for each year
    Returns: False if files were not downloaded or if there was some error,
    True otherwise
    """
    import os
    import requests
    import zipfile
    import cStringIO as StringIO
    if not (downloaddir and urls): return False
    complete = True
    print 'downloading to',downloaddir
    for url in urls:
        filename = url.split('/')[-1].replace('zip','xml')
        if filename in os.listdir(downloaddir):
            print 'already have',filename
            continue
        print 'downloading',url
        try:
            r = requests.get(url)
            z = zipfile.ZipFile(StringIO.StringIO(r.content))
            print 'unzipping',filename
            z.extractall(downloaddir)
        except:
            print 'ERROR: downloading or unzipping',filename
            complete = False
            continue
    return complete

def connect_client():
    """
    Loops for a minute until the Client connects to the local ipcluster
    """
    start=time.time()
    print 'Client connecting...'
    while time.time()-start < 300:
        try:
            c = Client()
            dview = c[:]
            break
        except:
            time.sleep(2)
            continue
    return dview

def run_parse():
    import parse
    import time
    import sys
    import itertools
    import lib.alchemy as alchemy
    xmls = parse.parse_files(files)
    return parse.parse_patents(xmls)

# TODO: these don't work
def run_clean(process_config):
    if process_config['clean']:
        print 'Running clean...'
        import clean

def run_consolidate(process_config):
    if process_config['consolidate']:
        print 'Running consolidate...'
        import consolidate

if __name__=='__main__':
    s = datetime.datetime.now()
    # accepts path to configuration file as command line option
    process_config, parse_config = get_config_options(sys.argv[1])

    # connect to ipcluster and get config options
    dview = connect_client()
    dview.block=True
    dview['process_config'] = process_config
    dview['parse_config'] = parse_config

    # download the files to be parsed
    urls = generate_download_list(parse_config['years'])
    dview.scatter('urls', urls)
    # check download directory
    downloaddir = parse_config['downloaddir']
    if downloaddir and not os.path.exists(downloaddir):
        os.makedirs(downloaddir)
    dview['downloaddir'] = parse_config['downloaddir']
    dview.apply(download_files)
    print 'Downloaded files:',parse_config['years']
    f = datetime.datetime.now()
    print 'Finished downloading in {0}'.format(str(f-s))

    # find files
    print "Starting parse on {0} on directory {1}".format(str(datetime.datetime.today()),parse_config['datadir'])
    files = parse.list_files(parse_config['datadir'],parse_config['dataregex'])
    dview.scatter('files',files)
    print "Found {2} files matching {0} in directory {1}".format(parse_config['dataregex'], parse_config['datadir'], len(files))

    # run parse and commit SQL
    print 'Running parse...'
    patobjects = list(itertools.chain.from_iterable(dview.apply(run_parse)))
    parse.database_commit(patobjects)
    f = datetime.datetime.now()
    print 'Finished parsing in {0}'.format(str(f-s))

    # run extra phases if needed, then move output files
    run_clean(process_config)
    run_consolidate(process_config)
    parse.move_tables(process_config['outputdir'])
<commit_msg>adjust start.py interface to work better with sqlalchemy<commit_after>import re
import os
import sys
import parse
import time
import itertools
import datetime
import logging
from IPython.parallel import Client
import requests
import zipfile
import cStringIO as StringIO
from BeautifulSoup import BeautifulSoup as bs
import lib.alchemy as alchemy

sys.path.append('lib')
from config_parser import get_config_options

logfile = "./" + 'xml-parsing.log'
logging.basicConfig(filename=logfile, level=logging.DEBUG)

# table bookkeeping for the parse script

def get_year_list(yearstring):
    """
    Given a [yearstring] of forms
    year1
    year1-year2
    year1,year2,year3
    year1-year2,year3-year4
    Expands into a list of year integers, and returns
    """
    years = []
    for subset in yearstring.split(','):
        if subset == 'latest':
            years.append('latest')
            continue
        sublist = subset.split('-')
        start = int(sublist[0])
        end = int(sublist[1])+1 if len(sublist) > 1 else start+1
        years.extend(range(start,end))
    return years

def generate_download_list(years):
    """
    Given the year string from the configuration file, return
    a list of urls to be downloaded
    """
    if not years: return []
    urls = []
    url = requests.get('https://www.google.com/googlebooks/uspto-patents-grants-text.html')
    soup = bs(url.content)
    years = get_year_list(years)

    # latest file link
    if 'latest' in years:
        a = soup.h3.findNext('h3').findPrevious('a')
        urls.append(a['href'])
        years.remove('latest')
    # get year links
    for year in years:
        header = soup.find('h3', {'id': str(year)})
        a = header.findNext()
        while a.name != 'h3':
            urls.append(a['href'])
            a = a.findNext()
    return urls

def download_files():
    """
    [downloaddir]: string representing base download directory. Will download
    files to this directory in folders named for each year
    Returns: False if files were not downloaded or if there was some error,
    True otherwise
    """
    import os
    import requests
    import zipfile
    import cStringIO as StringIO
    if not (downloaddir and urls): return False
    complete = True
    print 'downloading to',downloaddir
    for url in urls:
        filename = url.split('/')[-1].replace('zip','xml')
        if filename in os.listdir(downloaddir):
            print 'already have',filename
            continue
        print 'downloading',url
        try:
            r = requests.get(url)
            z = zipfile.ZipFile(StringIO.StringIO(r.content))
            print 'unzipping',filename
            z.extractall(downloaddir)
        except:
            print 'ERROR: downloading or unzipping',filename
            complete = False
            continue
    return complete

def connect_client():
    """
    Loops for a minute until the Client connects to the local ipcluster
    """
    start=time.time()
    print 'Client connecting...'
    while time.time()-start < 300:
        try:
            c = Client()
            dview = c[:]
            break
        except:
            time.sleep(2)
            continue
    return c, dview

def run_parse():
    import parse
    import time
    import sys
    import itertools
    import lib.alchemy as alchemy
    import logging
    logfile = "./" + 'xml-parsing.log'
    logging.basicConfig(filename=logfile, level=logging.DEBUG)
    xmls = parse.parse_files(files)
    if xmls:
        return parse.parse_patents(xmls)
    return []

def run_clean(process_config):
    if process_config['clean']:
        print 'Running clean...'
        import clean

def run_consolidate(process_config):
    if process_config['consolidate']:
        print 'Running consolidate...'
        import consolidate

if __name__=='__main__':
    s = datetime.datetime.now()
    # accepts path to configuration file as command line option
    process_config, parse_config = get_config_options(sys.argv[1])

    # connect to ipcluster and get config options
    c, dview = connect_client()
    dview.block=True
    dview['process_config'] = process_config
    dview['parse_config'] = parse_config

    # download the files to be parsed
    urls = generate_download_list(parse_config['years'])
    dview.scatter('urls', urls)
    # check download directory
    downloaddir = parse_config['downloaddir']
    if downloaddir and not os.path.exists(downloaddir):
        os.makedirs(downloaddir)
    dview['downloaddir'] = parse_config['downloaddir']
    dview.apply(download_files)
    print 'Downloaded files:',parse_config['years']
    f = datetime.datetime.now()
    print 'Finished downloading in {0}'.format(str(f-s))

    # find files
    print "Starting parse on {0} on directory {1}".format(str(datetime.datetime.today()),parse_config['datadir'])
    files = parse.list_files(parse_config['datadir'],parse_config['dataregex'])
    dview.scatter('files',files)
    print "Found {2} files matching {0} in directory {1}".format(parse_config['dataregex'], parse_config['datadir'], len(files))

    # run parse and commit SQL
    print 'Running parse...'
    patobjects = itertools.chain.from_iterable(dview.apply(run_parse))
    del dview
    parse.database_commit(patobjects)
    f = datetime.datetime.now()
    print 'Finished parsing in {0}'.format(str(f-s))

    # run extra phases if needed, then move output files
    run_clean(process_config)
    run_consolidate(process_config)
    parse.move_tables(process_config['outputdir'])
<|endoftext|>


<commit_before>import re
import sys
import parse
import time
import itertools
import datetime
from IPython.parallel import Client

sys.path.append('lib')
from patSQL import *
from config_parser import get_config_options

# table bookkeeping for the parse script

assignee_table = AssigneeSQL()
citation_table = CitationSQL()
class_table = ClassSQL()
inventor_table = InventorSQL()
patent_table = PatentSQL()
patdesc_table = PatdescSQL()
lawyer_table = LawyerSQL()
sciref_table = ScirefSQL()
usreldoc_table = UsreldocSQL()

xmlclasses = [AssigneeXML, CitationXML, ClassXML, InventorXML, \
              PatentXML, PatdescXML, LawyerXML, ScirefXML, UsreldocXML]

def connect_client():
    """
    Loops for a minute until the Client connects to the local ipcluster
    """
    start=time.time()
    print 'Client connecting...'
    while time.time()-start < 300:
        try:
            c = Client()
            dview = c[:]
            break
        except:
            time.sleep(5)
            continue
    return dview

def run_parse():
    import parse
    import time
    import sys
    import itertools
    parsed_xmls = parse.parallel_parse(files)
    parsed_grants = parse.parse_patent(parsed_xmls)
    parse.build_tables(parsed_grants)
    return parse.get_inserts()

# TODO: these don't work
def run_clean(process_config):
    if process_config['clean']:
        print 'Running clean...'
        execfile('clean.py')

def run_consolidate(process_config):
    if process_config['consolidate']:
        print 'Running consolidate...'
        execfile('consolidate.py')

if __name__=='__main__':
    s = datetime.datetime.now()
    # accepts path to configuration file as command line option
    process_config, parse_config = get_config_options(sys.argv[1])

    # download the files to be parsed
    download_files(parse_config['downloaddir'], parse_config['years'])

    # find files
    print "Starting parse on {0} on directory {1}".format(str(datetime.datetime.today()),parse_config['datadir'])
    files = parse.list_files(parse_config['datadir'],parse_config['dataregex'])
    print "Found {2} files matching {0} in directory {1}".format(parse_config['dataregex'], parse_config['datadir'], len(files))

    # connect to ipcluster and get config options
    dview = connect_client()
    dview.block=True
    dview.scatter('files',files)
    dview['process_config'] = process_config
    dview['parse_config'] = parse_config

    # run parse and commit SQL
    print 'Running parse...'
    inserts = list(itertools.chain.from_iterable(dview.apply(run_parse)))
    parse.commit_tables(inserts)
    f = datetime.datetime.now()
    print 'Finished parsing in {0}'.format(str(f-s))

    # run extra phases if needed, then move output files
    run_clean(process_config)
    run_consolidate(process_config)
    parse.move_tables(process_config['outputdir'])
<commit_msg>add imports to start.py<commit_after>import re
import os
import sys
import parse
import time
import itertools
import datetime
from IPython.parallel import Client
import urllib2
from BeautifulSoup import BeautifulSoup as bs

sys.path.append('lib')
from patSQL import *
from config_parser import get_config_options

# table bookkeeping for the parse script

assignee_table = AssigneeSQL()
citation_table = CitationSQL()
class_table = ClassSQL()
inventor_table = InventorSQL()
patent_table = PatentSQL()
patdesc_table = PatdescSQL()
lawyer_table = LawyerSQL()
sciref_table = ScirefSQL()
usreldoc_table = UsreldocSQL()

xmlclasses = [AssigneeXML, CitationXML, ClassXML, InventorXML, \
              PatentXML, PatdescXML, LawyerXML, ScirefXML, UsreldocXML]

def connect_client():
    """
    Loops for a minute until the Client connects to the local ipcluster
    """
    start=time.time()
    print 'Client connecting...'
    while time.time()-start < 300:
        try:
            c = Client()
            dview = c[:]
            break
        except:
            time.sleep(5)
            continue
    return dview

def run_parse():
    import parse
    import time
    import sys
    import itertools
    parsed_xmls = parse.parallel_parse(files)
    parsed_grants = parse.parse_patent(parsed_xmls)
    parse.build_tables(parsed_grants)
    return parse.get_inserts()

# TODO: these don't work
def run_clean(process_config):
    if process_config['clean']:
        print 'Running clean...'
        execfile('clean.py')

def run_consolidate(process_config):
    if process_config['consolidate']:
        print 'Running consolidate...'
        execfile('consolidate.py')

if __name__=='__main__':
    s = datetime.datetime.now()
    # accepts path to configuration file as command line option
    process_config, parse_config = get_config_options(sys.argv[1])

    # download the files to be parsed
    download_files(parse_config['downloaddir'], parse_config['years'])

    # find files
    print "Starting parse on {0} on directory {1}".format(str(datetime.datetime.today()),parse_config['datadir'])
    files = parse.list_files(parse_config['datadir'],parse_config['dataregex'])
    print "Found {2} files matching {0} in directory {1}".format(parse_config['dataregex'], parse_config['datadir'], len(files))

    # connect to ipcluster and get config options
    dview = connect_client()
    dview.block=True
    dview.scatter('files',files)
    dview['process_config'] = process_config
    dview['parse_config'] = parse_config

    # run parse and commit SQL
    print 'Running parse...'
    inserts = list(itertools.chain.from_iterable(dview.apply(run_parse)))
    parse.commit_tables(inserts)
    f = datetime.datetime.now()
    print 'Finished parsing in {0}'.format(str(f-s))

    # run extra phases if needed, then move output files
    run_clean(process_config)
    run_consolidate(process_config)
    parse.move_tables(process_config['outputdir'])
<|endoftext|>


<commit_before>#!/usr/bin/env python

import ctypes
import os
import SimpleHTTPServer
import socket
import SocketServer
import subprocess
import sys

# Create Handler for HTTP Requests (This is so it works for all languages)
# *ALL* HTML requests will directed to "exploit/index.html"
# *ALL* JS requests will be directed to like named files in "exploit/"
class Handler(SimpleHTTPServer.SimpleHTTPRequestHandler):
    def do_GET(self):
        try:
            if self.path.endswith('.html'):
                f = open('exploit' + os.sep + 'index.html')
                self.send_response(200)
                self.send_header('Content-type', 'text/html')
                self.end_headers()
                self.wfile.write(f.read())
                f.close()
            if self.path.endswith('.js'):
                f = open('exploit' + os.sep + self.path.rsplit('/', 1)[-1])
                self.send_response(200)
                self.send_header('Content-type', 'application/javascript')
                self.end_headers()
                self.wfile.write(f.read())
                f.close()
        except IOError:
            self.send_error(404, 'File Not Found')

def main():
    try:
        root = os.getuid() == 0
    except AttributeError:
        root = ctypes.windll.shell32.IsUserAnAdmin() != 0
    if root is False:
        sys.exit('>> This must be run by root as it requires port 53 & 80')

    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    try:
        s.connect(('10.255.255.255', 1))
        lan = s.getsockname()[0]
    except:
        lan = '127.0.0.1'
    finally:
        s.close()
    print('>> Your DNS IP is ' + lan)

    with open('dns.conf', 'w') as f:
        f.write('A manuals.playstation.net ' + lan)

    dns = subprocess.Popen(['python', 'FakeDns/fakedns.py', '-c', 'dns.conf'])

    try:
        httpd = SocketServer.TCPServer(('', 80), Handler)
    except socket.error:
        dns.kill()
        sys.exit('>> Port 80 already in use')
    try:
        print('>> Starting HTTP Server...')
        httpd.serve_forever()
    except KeyboardInterrupt:
        httpd.server_close()
        dns.kill()
        sys.exit()

if __name__ == '__main__':
    main()
<commit_msg>More Cleanup Busywork<commit_after>#!/usr/bin/env python
"""Easy PS4 Exploit Hosting by Al-Azif
   Source: https://github.com/Al-Azif/ps4-exploit-host
"""

# IMPORTS
import ctypes
import os
import SimpleHTTPServer
import socket
import SocketServer
import subprocess
import sys

# GLOBAL VARS
DNS_LOC = 'dns.conf'
EXPLOIT_LOC = 'exploit'

class Handler(SimpleHTTPServer.SimpleHTTPRequestHandler):
    """Create Handler for HTTP Requests (This is so it works for all languages)

       *ALL* HTML requests will directed to "exploit/index.html"
       *ALL* JS requests will be directed to like named files in "exploit/"
    """
    def do_GET(self):
        try:
            if self.path.endswith('.html'):
                f = open(EXPLOIT_LOC + os.sep + 'index.html')
                self.send_response(200)
                self.send_header('Content-type', 'text/html')
                self.end_headers()
                self.wfile.write(f.read())
                f.close()
            if self.path.endswith('.js'):
                f = open(EXPLOIT_LOC + os.sep + self.path.rsplit(os.sep, 1)[-1])
                self.send_response(200)
                self.send_header('Content-type', 'application/javascript')
                self.end_headers()
                self.wfile.write(f.read())
                f.close()
        except IOError:
            self.send_error(404, 'File Not Found')

def main():
    """The main logic"""
    try:
        root = os.getuid() == 0
    except AttributeError:
        root = ctypes.windll.shell32.IsUserAnAdmin() != 0
    if root is False:
        sys.exit('>> This must be run by root as it requires port 53 & 80')

    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    try:
        s.connect(('10.255.255.255', 1))
        lan = s.getsockname()[0]
    except socket.error:
        lan = '127.0.0.1'
    finally:
        s.close()
    print '>> Your DNS IP is ' + lan

    with open(DNS_LOC, 'w') as f:
        f.write('A manuals.playstation.net ' + lan)

    dns = subprocess.Popen(['python', 'FakeDns' + os.sep + 'fakedns.py', '-c', DNS_LOC])

    try:
        httpd = SocketServer.TCPServer(('', 80), Handler)
    except socket.error:
        dns.kill()
        sys.exit('>> Port 80 already in use')
    try:
        print '>> Starting HTTP Server...'
        httpd.serve_forever()
    except KeyboardInterrupt:
        httpd.server_close()
        dns.kill()
        sys.exit()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""This is a Raspberry network scanner application"""

import subprocess
import time
import re
from Node import Node
from listener import *
import threading
import logging

__credits__ = [""]
__version__ = "2.0.0"
__maintainer__ = "Michael Oyibo"
__email__ = "wi13b051@technikum-wien.at"
__status__ = "Development"
__Group__ = "Database"
__GroupMembers__ = "['Michael', 'Andreas Wenzl', 'Linda']"

def initialize():
    """This function initializes the application"""

    #Gather information about the current device and display it
    thisNode = Node(fetch_IP_address(), fetch_MAC_address(), fetch_broadcast_address(), scanHosts())
    print("My IP is {} and MAC is {}\n".format(thisNode.getIPAddress, thisNode.getMACAddress))

    #Register this device as a listener for incoming messages
    listener = Listener(thisNode)

    #Election of a master node
    begin_election_process(thisNode, listener)

    #Initialises threads for scanning devices and messaging
    scanner_thread = threading.Thread(target=network_scanner, args=(thisNode,listener))
    listener_thread = threading.Thread(target=network_listener, args=(listener,))

    #start scanning and listener threads
    scanner_thread.start()
    listener_thread.start()

def network_listener(listener):
    '''Thread to forever listen and send messages'''
    listener.client_listener()

def network_scanner(thisNode, listener):
    '''Thread to forever loop scanning the network'''
    while True:
        try:
            print("Scanning for new hosts............\n")
            newHostScans = scanHosts()
        except subprocess.CalledProcessError as e:
            print('nmam error')
            time.sleep(60*5)
            continue

        """ Looping through the newly scanned hosts"""
        #Check if each host in the new list is in our current list
        for host in newHostScans:
            if host not in thisNode.getCurrentHosts:
                thisNode.add_node(host)

        """Ping all addresses on our list to check if they are alive or dead"""
        for host in thisNode.getCurrentHosts:
            if command_executor(host[0]) == '':
                thisNode.remove_node(host)

        '''Check if master is alive/dead then raise election requests'''
        if not thisNode.getMasterNode:
            begin_election_process(thisNode, listener)
        else:
            if command_executor(thisNode.getMasterNode[0]) == '':
            #We kickoff election contest
                begin_election_process(thisNode, listener)

        if command_executor(thisNode.getIPAddress) == '':
            thisNode.update_ip_address(fetch_IP_address())
            begin_election_process(thisNode, listener)

        time.sleep(5)

        if thisNode.getMasterNode:
            print("Current Master is Node with IP: {} ,  MAC: {}, Vendor: {}\n\n".format(thisNode.getMasterNode[0],thisNode.getMasterNode[1],thisNode.getMasterNode[2]))
        print("Current live hosts are {}\n\n".format(thisNode.getCurrentHosts))

def begin_election_process(thisNode, listener):
    '''The Bully Election Algorithm begins'''
    #We kickoff election contest
    contest_election_thread = threading.Thread(target=contest_election, args=(thisNode,listener))
    contest_election_thread.start()

def contest_election(thisNode, listener):
    #A list of contestable nodes in the network i.e Nodes with higher last IP octet on the network
    contestableHosts = list(filter(lambda x: int(x[0].split('.')[3]) > int(thisNode.getIPAddress.split('.')[3]), thisNode.getCurrentHosts))

    process_election(contestableHosts, listener, thisNode)

def process_election(contestableHosts, listener, thisNode):
    #Sending a request for election if at least one node with higher last octet is available
    if len(contestableHosts) >= 1:
        for host in contestableHosts:
            propagate_election_thread = threading.Thread(target=listener.send_contest_request, args=("election", (host[0], 4242)))
            propagate_election_thread.start()
    else:
        #update master and broadcast information no other node with higher last IP octet or is the only available node
        thisNode.update_master_node(thisNode.getIPAddress.split('.')[3])

        broadcast_master_to_all = threading.Thread(target=listener.broadcast_master, args=("master".encode("utf-8"),))
        broadcast_master_to_all.start()

def scanHosts(): #We will scan for hosts every 5 seconds
    """This function scans for hosts on the network"""
    nmap3 = subprocess.check_output("sudo nmap -sP 192.168.1.0/24", shell=True).decode('utf-8')
    matches = re.findall("([\d]+.[\d]+.[\d]+.[\d]+)\nHost is up .*.\nMAC Address: ([0-9A-F:.]*) \((.*)\)\n",nmap3)
    listOfLiveHost = list(filter(lambda x: x[1].startswith('B8:27:EB'), matches))
    return listOfLiveHost

def fetch_MAC_address():
    """This function returns MAC address"""
    return subprocess.check_output("ifconfig | awk '/eth0/ { print toupper($5) }'", shell=True).decode('utf-8')

def fetch_IP_address():
    """This function returns IP address"""
    return re.findall("([\d]+.[\d]+.[\d]+.[\d]+)",subprocess.check_output("hostname -I", shell=True).decode('utf-8'))[0]

def command_executor(ipAddress):
    """This function pings a node to check if dead or alive"""
    return subprocess.check_output("nmap "+ ipAddress + " | awk '/Host is up/{ print $1, $2, $3 }'", shell=True).decode('utf-8')

def fetch_broadcast_address():
    """This function fetches the broadcast address of the current host"""
    return subprocess.check_output("ifconfig | awk '/Bcast:/ {split($3,arr,\":\"); print arr[2]}'", shell=True).decode('utf-8')

if __name__== '__main__':
    if subprocess.check_output("nmap | awk '/nmap -v -A/{print $4}'", shell=True).decode('utf-8') == '':
        print("Updating packages and installing nmap..........\n\n")
        subprocess.call("(sudo apt-get update -y)", shell=True)
        subprocess.call("(sudo apt-get install nmap -y)", shell=True)

    initialize()
<commit_msg>added space<commit_after>#!/usr/bin/env python3

"""This is a Raspberry network scanner application"""

import subprocess
import time
import re
from Node import Node
from listener import *
import threading
import logging

__credits__ = [""]
__version__ = "2.0.0"
__maintainer__ = "Michael Oyibo"
__email__ = "wi13b051@technikum-wien.at"
__status__ = "Development"
__Group__ = "Database"
__GroupMembers__ = "['Michael', 'Andreas Wenzl', 'Linda']"

def initialize():
    """This function initializes the application"""

    #Gather information about the current device and display it
    thisNode = Node(fetch_IP_address(), fetch_MAC_address(), fetch_broadcast_address(), scanHosts())
    print("My IP is {} and MAC is {}\n".format(thisNode.getIPAddress, thisNode.getMACAddress))

    #Register this device as a listener for incoming messages
    listener = Listener(thisNode)

    #Election of a master node
    begin_election_process(thisNode, listener)

    #Initialises threads for scanning devices and messaging
    scanner_thread = threading.Thread(target=network_scanner, args=(thisNode,listener))
    listener_thread = threading.Thread(target=network_listener, args=(listener,))

    #start scanning and listener threads
    scanner_thread.start()
    listener_thread.start()

def network_listener(listener):
    '''Thread to forever listen and send messages'''
    listener.client_listener()

def network_scanner(thisNode, listener):
    '''Thread to forever loop scanning the network'''
    while True:
        try:
            print("Scanning for new hosts............\n")
            newHostScans = scanHosts()
        except subprocess.CalledProcessError as e:
            print('nmam error')
            time.sleep(60*5)
            continue

        """ Looping through the newly scanned hosts"""
        #Check if each host in the new list is in our current list
        for host in newHostScans:
            if host not in thisNode.getCurrentHosts:
                thisNode.add_node(host)

        """Ping all addresses on our list to check if they are alive or dead"""
        for host in thisNode.getCurrentHosts:
            if command_executor(host[0]) == '':
                thisNode.remove_node(host)

        '''Check if master is alive/dead then raise election requests'''
        if not thisNode.getMasterNode:
            begin_election_process(thisNode, listener)
        else:
            if command_executor(thisNode.getMasterNode[0]) == '':
            #We kickoff election contest
                begin_election_process(thisNode, listener)

        if command_executor(thisNode.getIPAddress) == '':
            thisNode.update_ip_address(fetch_IP_address())
            begin_election_process(thisNode, listener)

        time.sleep(5)

        if thisNode.getMasterNode:
            print("Current Master is Node with IP: {} ,  MAC: {}, Vendor: {}\n\n".format(thisNode.getMasterNode[0],thisNode.getMasterNode[1],thisNode.getMasterNode[2]))
        print("Current live hosts are {}\n\n".format(thisNode.getCurrentHosts))

def begin_election_process(thisNode, listener):
    '''The Bully Election Algorithm begins'''
    #We kickoff election contest
    contest_election_thread = threading.Thread(target=contest_election, args=(thisNode,listener))
    contest_election_thread.start()

def contest_election(thisNode, listener):
    #A list of contestable nodes in the network i.e Nodes with higher last IP octet on the network
    contestableHosts = list(filter(lambda x: int(x[0].split('.')[3]) > int(thisNode.getIPAddress.split('.')[3]), thisNode.getCurrentHosts))

    process_election(contestableHosts, listener, thisNode)

def process_election(contestableHosts, listener, thisNode):
    #Sending a request for election if at least one node with higher last octet is available
    if len(contestableHosts) >= 1:
        for host in contestableHosts:
            propagate_election_thread = threading.Thread(target=listener.send_contest_request, args=("election", (host[0], 4242)))
            propagate_election_thread.start()
    else:
        #update master and broadcast information no other node with higher last IP octet or is the only available node
        thisNode.update_master_node(thisNode.getIPAddress.split('.')[3])

        broadcast_master_to_all = threading.Thread(target=listener.broadcast_master, args=("master".encode("utf-8"),))
        broadcast_master_to_all.start()

def scanHosts(): #We will scan for hosts every 5 seconds
    """This function scans for hosts on the network"""
    nmap3 = subprocess.check_output("sudo nmap -sP 192.168.1.0/24", shell=True).decode('utf-8')
    matches = re.findall("([\d]+.[\d]+.[\d]+.[\d]+)\nHost is up .*.\nMAC Address: ([0-9A-F:.]*) \((.*)\)\n",nmap3)
    listOfLiveHost = list(filter(lambda x: x[1].startswith('B8:27:EB'), matches))
    return listOfLiveHost

def fetch_MAC_address():
    """This function returns MAC address"""
    return subprocess.check_output("ifconfig | awk '/eth0/ { print toupper($5) }'", shell=True).decode('utf-8')

def fetch_IP_address():
    """This function returns IP address"""
    return re.findall("([\d]+.[\d]+.[\d]+.[\d]+)",subprocess.check_output("hostname -I", shell=True).decode('utf-8'))[0]

def command_executor(ipAddress):
    """This function pings a node to check if dead or alive"""
    return subprocess.check_output("nmap "+ ipAddress + " | awk '/Host is up/{ print $1, $2, $3 }'", shell=True).decode('utf-8')

def fetch_broadcast_address():
    """This function fetches the broadcast address of the current host"""
    return subprocess.check_output("ifconfig | awk '/Bcast:/ {split($3,arr,\":\"); print arr[2]}'", shell=True).decode('utf-8')

if __name__== '__main__':

    if subprocess.check_output("nmap | awk '/nmap -v -A/{print $4}'", shell=True).decode('utf-8') == '':
        print("Updating packages and installing nmap..........\n\n")
        subprocess.call("(sudo apt-get update -y)", shell=True)
        subprocess.call("(sudo apt-get install nmap -y)", shell=True)

    initialize()
<|endoftext|>


<commit_before>from sklearn import metrics
import matplotlib.pyplot as plt

class StatList(list):

    def __init__(self, complete):
        self.theme = [item['theme'] for item in complete]
        self.classe = [item['class'] for item in complete]
        self.themeclasse = ['{0}_{1}'.format(item['theme'], item['class']) for item in complete]

    def add(self, k):
        stat = Stat(k, self.theme, self.classe, self.themeclasse)
        self.append(stat)
        return stat

    def plot(self):
        plots = []
        plots.extend(
        [
            {
               "ylabel": "Creation Time",
                "xlabel": "K",
                "dataX": [],
                "dataY": []
            },
            {
               "ylabel": "Execution Time",
                "xlabel": "K",
                "dataX": [],
                "dataY": []
            },
            {
                "ylabel": "Total Time",
                "xlabel": "K",
                "dataX": [],
                "dataY": []
            },
            {
                "ylabel": "Indice Rand ajustado com relacao a Classe",
                "xlabel": "K",
                "dataX": [],
                "dataY": []
            },
            {
                "ylabel": "Indice Rand ajustado com relacao ao Tema",
                "xlabel": "K",
                "dataX": [],
                "dataY": []
            },
            {
                "ylabel": "Indice Rand ajustado com relacao ao Tema+Classe",
                "xlabel": "K",
                "dataX": [],
                "dataY": []
            },{
                "ylabel": "Indice Silhueta com relacao a base inicial",
                "xlabel": "K",
                "dataX": [],
                "dataY": []
            },
            {
                "ylabel": "Indice Variancia Intra-cluster",
                "xlabel": "K",
                "dataX": [],
                "dataY": []
            }])

        for stat in self:
            plots[0]["dataX"].append(stat.k)
            plots[0]["dataY"].append(stat.creationTime)
            plots[1]["dataX"].append(stat.k)
            plots[1]["dataY"].append(stat.executionTime)
            plots[2]["dataX"].append(stat.k)
            plots[2]["dataY"].append(stat.totalTime)
            plots[3]["dataX"].append(stat.k)
            plots[3]["dataY"].append(stat.adjusted_rand_score_classe)
            plots[4]["dataX"].append(stat.k)
            plots[4]["dataY"].append(stat.adjusted_rand_score_theme)
            plots[5]["dataX"].append(stat.k)
            plots[5]["dataY"].append(stat.adjusted_rand_score_themeclasse)
            plots[6]["dataX"].append(stat.k)
            plots[6]["dataY"].append(stat.silhouette_score_)
            plots[7]["dataX"].append(stat.k)
            plots[7]["dataY"].append(stat.index_intracluster_variance_)
        for p in plots:
            plt.figure()
            plt.ylabel(p["ylabel"])
            plt.xlabel(p["xlabel"])
            plt.plot(p["dataX"], p["dataY"])
            print(p["dataX"])
            print(p)
        #times
        plt.figure()
        plt.ylabel("Time")
        plt.xlabel("k")
        for plot in plots[0:3]:
            plt.plot(plot["dataX"], plot["dataY"], label=plot["ylabel"])

class Stat:
    beginCreationTime = None
    endCreationTime = None
    creationTime = None
    beginExecutionTime = None
    endExecutionTime = None
    executionTime = None
    totalTime=None
    adjusted_rand_score_classe = None
    adjusted_rand_score_theme = None
    adjusted_rand_score_themeclasse = None
    silhouette_score_ = None
    index_intracluster_variance_ = None
    theme = None
    classe = None
    themeclasse = None
    k = None

    def __init__(self, k, theme, classe, themeclasse):
        self.theme = theme
        self.classe = classe
        self.themeclasse = themeclasse
        self.k = k

    def calc(self, data, particao, centroids):
        self.creationTime = self.endCreationTime - self.beginCreationTime
        self.executionTime = self.endExecutionTime - self.beginExecutionTime
        self.totalTime= self.creationTime + self.executionTime
        self.adjusted_rand_score_classe = metrics.adjusted_rand_score(self.classe, particao)
        self.adjusted_rand_score_theme = metrics.adjusted_rand_score(self.theme , particao)
        self.adjusted_rand_score_themeclasse = metrics.adjusted_rand_score(self.themeclasse, particao)

        try:
            self.silhouette_score_ = metrics.silhouette_score(data, particao)
        except:
            pass

        try:
            self.index_intracluster_variance_ = self.index_intracluster_variance(data, particao, centroids)
        except:
            pass

    def index_intracluster_variance(data, labels, centroids):
        soma = 0
        n = len(labels)
        for idx, item in enumerate(centroids):
            soma += sum(metrics.pairwise.pairwise_distances(np.array(data)[labels == idx], np.array(item).reshape(1, -1), metric='euclidean'))

        return (int(soma)/n)**(1/2)

    def toString(self):
        strstats = '"Tempo Criacao Modelo";"%.2fs"\n' % self.creationTime
        strstats += '"Tempo Execucao Modelo";"%.2fs"\n' % self.executionTime
        strstats += '"Tempo Total Modelo";"%.2fs"\n' % self.totalTime
        strstats += '"Indice Rand ajustado com relacao a Classe";"{0}"\n'.format(self.adjusted_rand_score_classe)
        strstats += '"Indice Rand ajustado com relacao ao Tema";"{0}"\n'.format(self.adjusted_rand_score_theme)
        strstats += '"Indice Rand ajustado com relacao ao Tema+Classe";"{0}"\n'.format(self.adjusted_rand_score_themeclasse)

        try:
            strstats += '"Indice Silhueta com relacao a base inicial";"{0}"\n'.format(self.silhouette_score_)
        except:
            pass

        try:
            strstats += '"Indice Variancia Intra-cluster";"{0}"\n'.format(self.index_intracluster_variance_)
        except:
            pass
        return strstats
    def toStringChart(self):
        strstats = 'Tempo Criacao Modelo: %.2fs\n' % self.creationTime
        strstats += 'Tempo Execucao Modelo: %.2fs\n' % self.executionTime
        strstats += 'Tempo Total Modelo: %.2fs\n' % self.totalTime
        strstats += 'Indice Rand ajustado com relacao a Classe: {0}\n'.format(self.adjusted_rand_score_classe)
        strstats += 'Indice Rand ajustado com relacao ao Tema: {0}\n'.format(self.adjusted_rand_score_theme)
        strstats += 'Indice Rand ajustado com relacao ao Tema+Classe: {0}\n'.format(self.adjusted_rand_score_themeclasse)
        try:
            strstats += "Indice Silhueta com relacao a base inicial: {0}\n".format(self.silhouette_score_)
        except:
            pass

        try:
            strstats += "Indice Variancia Intra-cluster: {0}\n".format(self.index_intracluster_variance_)
        except:
            pass
        return strstats

def main():
    print("Finished")

if __name__ == '__main__':
    main()<commit_msg>Corridigido calculo da variancia Intra-Cluster<commit_after>from sklearn import metrics
import matplotlib.pyplot as plt
import numpy as np

class StatList(list):

    def __init__(self, complete):
        self.theme = [item['theme'] for item in complete]
        self.classe = [item['class'] for item in complete]
        self.themeclasse = ['{0}_{1}'.format(item['theme'], item['class']) for item in complete]

    def add(self, k):
        stat = Stat(k, self.theme, self.classe, self.themeclasse)
        self.append(stat)
        return stat

    def plot(self):
        plots = []
        plots.extend(
        [
            {
               "ylabel": "Creation Time",
                "xlabel": "K",
                "dataX": [],
                "dataY": []
            },
            {
               "ylabel": "Execution Time",
                "xlabel": "K",
                "dataX": [],
                "dataY": []
            },
            {
                "ylabel": "Total Time",
                "xlabel": "K",
                "dataX": [],
                "dataY": []
            },
            {
                "ylabel": "Indice Rand ajustado com relacao a Classe",
                "xlabel": "K",
                "dataX": [],
                "dataY": []
            },
            {
                "ylabel": "Indice Rand ajustado com relacao ao Tema",
                "xlabel": "K",
                "dataX": [],
                "dataY": []
            },
            {
                "ylabel": "Indice Rand ajustado com relacao ao Tema+Classe",
                "xlabel": "K",
                "dataX": [],
                "dataY": []
            },{
                "ylabel": "Indice Silhueta com relacao a base inicial",
                "xlabel": "K",
                "dataX": [],
                "dataY": []
            },
            {
                "ylabel": "Indice Variancia Intra-cluster",
                "xlabel": "K",
                "dataX": [],
                "dataY": []
            }])

        for stat in self:
            plots[0]["dataX"].append(stat.k)
            plots[0]["dataY"].append(stat.creationTime)
            plots[1]["dataX"].append(stat.k)
            plots[1]["dataY"].append(stat.executionTime)
            plots[2]["dataX"].append(stat.k)
            plots[2]["dataY"].append(stat.totalTime)
            plots[3]["dataX"].append(stat.k)
            plots[3]["dataY"].append(stat.adjusted_rand_score_classe)
            plots[4]["dataX"].append(stat.k)
            plots[4]["dataY"].append(stat.adjusted_rand_score_theme)
            plots[5]["dataX"].append(stat.k)
            plots[5]["dataY"].append(stat.adjusted_rand_score_themeclasse)
            plots[6]["dataX"].append(stat.k)
            plots[6]["dataY"].append(stat.silhouette_score_)
            plots[7]["dataX"].append(stat.k)
            plots[7]["dataY"].append(stat.index_intracluster_variance_)
        for p in plots:
            plt.figure()
            plt.ylabel(p["ylabel"])
            plt.xlabel(p["xlabel"])
            plt.plot(p["dataX"], p["dataY"])
            print(p["dataX"])
            print(p)
        #times
        plt.figure()
        plt.ylabel("Time")
        plt.xlabel("k")
        for plot in plots[0:3]:
            plt.plot(plot["dataX"], plot["dataY"], label=plot["ylabel"])

class Stat:
    beginCreationTime = None
    endCreationTime = None
    creationTime = None
    beginExecutionTime = None
    endExecutionTime = None
    executionTime = None
    totalTime=None
    adjusted_rand_score_classe = None
    adjusted_rand_score_theme = None
    adjusted_rand_score_themeclasse = None
    silhouette_score_ = None
    index_intracluster_variance_ = None
    theme = None
    classe = None
    themeclasse = None
    k = None

    def __init__(self, k, theme, classe, themeclasse):
        self.theme = theme
        self.classe = classe
        self.themeclasse = themeclasse
        self.k = k

    def calc(self, data, labels, centroids):
        self.creationTime = self.endCreationTime - self.beginCreationTime
        self.executionTime = self.endExecutionTime - self.beginExecutionTime
        self.totalTime= self.creationTime + self.executionTime
        self.adjusted_rand_score_classe = metrics.adjusted_rand_score(self.classe, labels)
        self.adjusted_rand_score_theme = metrics.adjusted_rand_score(self.theme , labels)
        self.adjusted_rand_score_themeclasse = metrics.adjusted_rand_score(self.themeclasse, labels)

        try:
            self.silhouette_score_ = metrics.silhouette_score(data, labels)
        except:
            pass

        try:
            self.index_intracluster_variance_ = self.index_intracluster_variance(data, labels, centroids)
        except:
            pass

    def index_intracluster_variance(self, data, labels, centroids):
        sum_distances = 0
        n = len(labels)
        for idx, item in enumerate(centroids):
            sum_distances += sum(metrics.pairwise.pairwise_distances(np.array(data)[labels == idx], np.array(item).reshape(1, -1), metric='euclidean'))

        return (int(sum_distances)/n)**(1/2)

    def toString(self):
        strstats = '"Tempo Criacao Modelo";"%.2fs"\n' % self.creationTime
        strstats += '"Tempo Execucao Modelo";"%.2fs"\n' % self.executionTime
        strstats += '"Tempo Total Modelo";"%.2fs"\n' % self.totalTime
        strstats += '"Indice Rand ajustado com relacao a Classe";"{0}"\n'.format(self.adjusted_rand_score_classe)
        strstats += '"Indice Rand ajustado com relacao ao Tema";"{0}"\n'.format(self.adjusted_rand_score_theme)
        strstats += '"Indice Rand ajustado com relacao ao Tema+Classe";"{0}"\n'.format(self.adjusted_rand_score_themeclasse)
        strstats += '"Indice Silhueta com relacao a base inicial";"{0}"\n'.format(self.silhouette_score_)
        strstats += '"Indice Variancia Intra-cluster";"{0}"\n'.format(self.index_intracluster_variance_)

        return strstats

    def toStringChart(self):
        strstats = 'Tempo Criacao Modelo: %.2fs\n' % self.creationTime
        strstats += 'Tempo Execucao Modelo: %.2fs\n' % self.executionTime
        strstats += 'Tempo Total Modelo: %.2fs\n' % self.totalTime
        strstats += 'Indice Rand ajustado com relacao a Classe: {0}\n'.format(self.adjusted_rand_score_classe)
        strstats += 'Indice Rand ajustado com relacao ao Tema: {0}\n'.format(self.adjusted_rand_score_theme)
        strstats += 'Indice Rand ajustado com relacao ao Tema+Classe: {0}\n'.format(self.adjusted_rand_score_themeclasse)
        try:
            strstats += "Indice Silhueta com relacao a base inicial: {0}\n".format(self.silhouette_score_)
        except:
            pass

        try:
            strstats += "Indice Variancia Intra-cluster: {0}\n".format(self.index_intracluster_variance_)
        except:
            pass
        return strstats

def main():
    print("Finished")

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

""" subst - Replace PATTERN with REPLACE in many files.
    http://mysz.github.io/subst
    Author: Marcin Sztolcman (marcin@urzenia.net)

    Get help with: subst --help
    Information about version: subst --version
"""

from __future__ import print_function, unicode_literals

import argparse
import os, os.path
import re
import shutil
import sys
import tempfile
import textwrap

from pprint import pprint, pformat  # pylint: disable-msg=unused-import

__version__ = '0.2'

DEFAULT_BACKUP_EXTENSION = 'bak'

class SubstException(Exception):
    """ Exception raised when there is some error.
    """

    pass

class ParserException(SubstException):
    """ Exception raised when pattern given by user has errors.
    """

    pass

def show_version():
    """ Show version info and exit.
    """

    msg('{0}: version {1}'.format(os.path.basename(sys.argv[0]), __version__))
    sys.exit(0)

def errmsg(message, indent=0, end=None):
    """ Display error message.

        Prints always to stderr.
    """

    print((' ' * indent * 4) + message, file=sys.stderr, end=end)

def msg(message, indent=0, end=None):
    """ Display message.
    """

    print((' ' * indent * 4) + message, end=end)

def debug(message, indent=0, end=None):
    """ Display debug message.

        Prints always to stderr.
    """

    print((' ' * indent * 4) + message, file=sys.stderr, end=end)

def _parse_args__get_ext(args):
    """ Find extension for backup files.

        Extension in args is supposed to not have leading dot.
    """
    if args.no_backup:
        return ''

    if args.ext is None or len(args.ext) == 0:
        return '.' + DEFAULT_BACKUP_EXTENSION

    return '.' + args.ext

def _parse_args__eval_replacement(repl):
    """ Compile replace argument as valid Python code and return
        function which can be passed to re.sub or re.subn functions.
    """
    def _(match):  # pylint: disable-msg=missing-docstring
        return eval (repl, { '__builtins__': __builtins__ }, { 'm': match })

    return _

def _parse_args__pattern(args):  # pylint: disable-msg=too-many-branches
    """ Read arguments from argparse.ArgumentParser instance, and
        parse it to find correct values for arguments:
            * pattern
            * replace
            * count
        If is provided --pattern_and_replace argument, then parse
        it and return data from it.

        In other case just return data from argparse.ArgumentParser
        instance.

        Reads also string argument, and apply it to pattern.

        Returned pattern is compiled (see: re.compile).
    """

    flags = 0

    if args.ignore_case:
        flags |= re.IGNORECASE

    if args.pattern_dot_all:
        flags |= re.DOTALL

    if args.pattern_verbose:
        flags |= re.VERBOSE

    if args.pattern_multiline:
        flags |= re.MULTILINE

    if args.pattern is not None and args.replace is not None:
        if args.string:
            pattern = re.escape(args.pattern)
        else:
            pattern = args.pattern

        return re.compile(pattern, flags), args.replace, args.count or 0

    elif args.pattern_and_replace is not None:
        pat = args.pattern_and_replace

        try:
            # pattern must begin with 's'
            if not pat.startswith('s'):
                raise ParserException('Bad pattern specified: {0}'.format(args.pattern_and_replace))
            pat = pat[1:]

            # parse pattern
            if pat.startswith('('):
                pattern, replace, _flags = pat[1:].split(')(', 2)
            elif pat.startswith('{'):
                pattern, replace, _flags = pat[1:].split('}{', 2)
            elif pat.startswith('['):
                pattern, replace, _flags = pat[1:].split('][', 2)
            elif pat.startswith('<'):
                pattern, replace, _flags = pat[1:].split('><', 2)
            else:
                delim = pat[0]

                pat = pat[1:]
                pattern, replace, _flags = pat.split(delim, 2)

            # pattern can be suffixed with g (as global)
            if 'g' in _flags:
                count = 0
                pat = pat[:-1]
            else:
                count = args.count if args.count is not None else 1

            if 'i' in _flags:
                flags |= re.IGNORECASE
            if 'x' in _flags:
                flags |= re.VERBOSE
            if 's' in _flags:
                flags |= re.DOTALL
            if 'm' in _flags:
                flags |= re.MULTILINE

        except ValueError:
            raise ParserException('Bad pattern specified: {0}'.format(args.pattern_and_replace))

        return re.compile(pattern, flags), replace, count
    else:
        raise ParserException('Bad pattern specified: {0}'.format(args.pattern_and_replace))

def wrap_text(txt):
    """ Make custom wrapper for passed text.

        Splits given text for lines, and for every line apply custom
        textwrap.TextWrapper settings, then return reformatted string.
    """
    _wrap = textwrap.TextWrapper(
        width = 72,
        expand_tabs = True,
        replace_whitespace = False,
        drop_whitespace = True,
        subsequent_indent = '  ',
    )
    txt = [ _wrap.fill(line) for line in txt.splitlines() ]
    return "\n".join(txt)

def parse_args(args):
    """ Parse arguments passed to script, validate it, compile if needed and return.
    """

    p = argparse.ArgumentParser(  # pylint: disable-msg=invalid-name
        description='Replace PATTERN with REPLACE in many files.',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=wrap_text("Miscellaneous notes:\n"
            "* regular expressions engine used here is PCRE, dialect from Python\n"
            "* is required to pass either --pattern and -replace, or --pattern_and_replace argument\n"
            "* if pattern passed to --pattern_and_replace has /g modifier, it overwrites --count value\n"
            "* if neither /g modifier nor --count argument is passed, assume that --count is equal 1\n"
            "* if only --count is given, this value is used\n"
            "* if --eval-replace is given, --replace must be valid Python code, where can be used m variable."
                "m holds MatchObject instance (see: http://http://docs.python.org/2/library/re.html#match-objects, "
                "for example:\n"
            "    --eval-replace --replace 'm.group(1).lower()'\n"
            "* regular expressions with non linear search read whole file to yours computer memory - if file size is "
                "bigger then you have memory in your computer, it fails\n"
            "* parsing expression passed to --pattern_and_replace argument is very simple - if you use / as delimiter, "
                "then in your expression can\'t be used this character anymore. If you need to use same character as "
                "delimiter and in expression, then better use --pattern and --replace argument\n"
#             "* "
            "\n"
            "Security notes:\n"
            "* be carefull with --eval-replace argument. When it's given, value passed to --replace is eval-ed, so any "
                         "not safe code will be executed!\n"
            "\n"
            "Author:\n"
            "Marcin Sztolcman <marcin@urzenia.net> // http://urzenia.net\n"
            "\n"
            "HomePage:\n"
            "http://mysz.github.io/subst/"
        ),
    )

    p.add_argument('-p', '--pattern', type=str,
                   help='pattern to replace for. Supersede --pattern_and_replace. Required if --replace is specified.')
    p.add_argument('-r', '--replace', type=str,
                   help='replacement. Supersede --pattern_and_replace. Required if --pattern is specified.')
    p.add_argument('--eval-replace', dest='eval', action='store_true',
                   help='if specified, make eval data from --replace(should be valid Python code). Ignored with '
                   '--pattern_and_replace argument.')
    p.add_argument('-t', '--string', type=bool,
                   help='if specified, treats --pattern as string, not as regular expression. Ignored with '
                   '--pattern_and_replace argument.')
    p.add_argument('-s', '--pattern_and_replace', metavar='"s/PAT/REP/gixsm"', type=str,
                   help='pattern and replacement in one: s/pattern/replace/g(pattern is always regular expression, /g '
                   'is optional and stands for --count=0, /i == --ignore-case, /s == --pattern-dot-all, /m == --pattern-multiline).')
    p.add_argument('-c', '--count', type=int,
                   help='make COUNT replacements for every file(0 make unlimited changes, default).')
    p.add_argument('-l', '--linear', action='store_true',
                   help='apply pattern for every line separately. Without this flag whole file is read into memory.')
    p.add_argument('-i', '--ignore-case', dest='ignore_case', action='store_true',
                   help='ignore case of characters when matching')
    p.add_argument('--pattern-dot-all', dest='pattern_dot_all', action='store_true',
                   help='with this flag, dot(.) character in pattern match also new line character (see: '
                   'http://docs.python.org/2/library/re.html#re.DOTALL).')
    p.add_argument('--pattern-verbose', dest='pattern_verbose', action='store_true',
                   help='with this flag pattern can be passed as verbose(see: http://docs.python.org/2/library/re.html#re.VERBOSE).')
    p.add_argument('--pattern-multiline', dest='pattern_multiline', action='store_true',
                   help='with this flag pattern can be passed as multiline(see: http://docs.python.org/2/library/re.html#re.MULTILINE).')
    p.add_argument('-b', '--no-backup', dest='no_backup', action='store_true',
                   help='disable creating backup of modified files.')
    p.add_argument('-e', '--backup-extension', dest='ext', default=DEFAULT_BACKUP_EXTENSION, type=str,
                   help='extension for backuped files(ignore if no backup is created), without leading dot. Defaults to: "bak".')
    p.add_argument('--stdin', action='store_true',
                   help='read data from STDIN(implies --stdout)')
    p.add_argument('--stdout', action='store_true',
                   help='output data to STDOUT instead of change files in-place(implies --no-backup)')
    p.add_argument('--verbose', action='store_true',
                   help='show files and how many replacements was done')
    p.add_argument('--debug', action='store_true',
                   help='show more infos')
    p.add_argument('-v', '--version', action='store_true',
                   help='show version and exit')
    p.add_argument('files', nargs='*', type=str,
                   help='file to parse.')

    args = p.parse_args()

    if args.version:
        return args

    if not args.files or args.files[0] == '-':
        args.stdin = True

    if args.stdin:
        args.stdout = True

    if args.stdout:
        args.no_backup = True

    if \
            (args.pattern is None and args.replace is None and args.pattern_and_replace is None) or \
            (args.pattern is None and args.replace is not None) or \
            (args.pattern is not None and args.replace is None):
        p.error('must be provided --pattern and --replace options, or --pattern_and_replace.')

    try:
        args.ext = _parse_args__get_ext(args)
        args.pattern, args.replace, args.count = _parse_args__pattern(args)
        if args.eval:
            args.replace = _parse_args__eval_replacement (args.replace)
    except ParserException as ex:
        p.error(ex)

    return args

def replace_linear(src, dst, pattern, replace, count):
    """ Read data from 'src' line by line, replace some data from
        regular expression in 'pattern' with data in 'replace',
        write it to 'dst', and return quantity of replaces.
    """
    ret = 0
    for line in src:
        if count == 0 or ret < count:
            line, rest_count = pattern.subn(replace, line, max(0, count - ret))
            ret += rest_count
        dst.write(line)
    return ret

def replace_global(src, dst, pattern, replace, count):
    """ Read whole file from 'src', replace some data from
        regular expression in 'pattern' with data in 'replace',
        write it to 'dst', and return quantity of replaces.
    """
    data = src.read()
    data, ret = pattern.subn(replace, data, count)
    dst.write(data)
    return ret

def _process_file__make_backup(path, backup_ext):
    """ Create backup of file: copy it new extension.

        Returns path to backup file.
    """

    root = os.path.dirname(path)
    backup_path = os.path.join(root, path + backup_ext)

    if os.path.exists(backup_path):
        raise SubstException('Backup path: "{0}" for file "{1}" already exists, file omited'.format(backup_path, path))

    try:
        shutil.copy2(path, backup_path)
    except (shutil.Error, IOError) as ex:
        raise SubstException('Cannot create backup for "{0}": {1}'.format(path, ex))

    return backup_path

def _process_file__handle(src_path, dst_fh, cfg, replace_func):
    """ Read data from `src_path`, replace data with `replace_func` and
        save it to `dst_fh`.
    """

    with open(src_path, 'r') as fh_src:
        cnt = replace_func(fh_src, dst_fh, cfg.pattern, cfg.replace, cfg.count)
        if cfg.verbose or cfg.debug:
            debug('{0} replacements'.format(cnt), 1)

def _process_file__regular(src_path, cfg, replace_func):
    """ Read data from `src_path`, replace data with `replace_func` and
        save it.

        It's safe operation - first save data to temporary file, and then try to rename
        new file to old one.
    """

    tmp_fh, tmp_path = tempfile.mkstemp()
    tmp_fh = os.fdopen(tmp_fh, 'w')

    _process_file__handle(src_path, tmp_fh, cfg, replace_func)

    try:
        os.rename(tmp_path, src_path)
    except OSError as ex:
        raise SubstException('Error replacing "{0}" with "{1}": {2}'.format(src_path, tmp_path, ex))
    else:
        if cfg.debug:
            debug('moved temporary file to original', 1)

    try:
        tmp_fh.close()
    except:  # pylint: disable-msg=bare-except
        pass

def process_file(path, replace_func, cfg):
    """ Process single file: open, read, make backup and replace data.
    """

    if cfg.verbose or cfg.debug:
        debug(path)

    if not os.path.exists(path):
        raise SubstException('Path "{0}" doesn\'t exists'.format(path))

    if not os.path.isfile(path) or os.path.islink(path):
        raise SubstException('Path "{0}" is not a regular file'.format(path))

    if not cfg.no_backup:
        backup_path = _process_file__make_backup(path, cfg.ext)

        if cfg.debug:
            debug('created backup file: "{0}"'.format(backup_path), 1)

    try:
        if cfg.stdout:
            _process_file__handle(path, sys.stdout, cfg, replace_func)
        else:
            _process_file__regular(path, cfg, replace_func)
    except SubstException as ex:
        errmsg(ex)

def main():
    """ Run tool: parse input arguments, read data, replace and save or display.
    """
    args = parse_args(sys.argv[1:])

    if args.version:
        show_version()

    if args.linear:
        replace_func = replace_linear
    else:
        replace_func = replace_global

    if args.stdin:
        replace_func(sys.stdin, sys.stdout, args.pattern, args.replace, args.count)

    else:
        for path in args.files:
            try:
                process_file(path, replace_func, args)
            except SubstException as ex:
                errmsg(ex.message, int(args.verbose or args.debug))

if __name__ == '__main__':
    main()
<commit_msg>_parse_args__get_ext: simplified condition<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

""" subst - Replace PATTERN with REPLACE in many files.
    http://mysz.github.io/subst
    Author: Marcin Sztolcman (marcin@urzenia.net)

    Get help with: subst --help
    Information about version: subst --version
"""

from __future__ import print_function, unicode_literals

import argparse
import os, os.path
import re
import shutil
import sys
import tempfile
import textwrap

from pprint import pprint, pformat  # pylint: disable-msg=unused-import

__version__ = '0.2'

DEFAULT_BACKUP_EXTENSION = 'bak'

class SubstException(Exception):
    """ Exception raised when there is some error.
    """

    pass

class ParserException(SubstException):
    """ Exception raised when pattern given by user has errors.
    """

    pass

def show_version():
    """ Show version info and exit.
    """

    msg('{0}: version {1}'.format(os.path.basename(sys.argv[0]), __version__))
    sys.exit(0)

def errmsg(message, indent=0, end=None):
    """ Display error message.

        Prints always to stderr.
    """

    print((' ' * indent * 4) + message, file=sys.stderr, end=end)

def msg(message, indent=0, end=None):
    """ Display message.
    """

    print((' ' * indent * 4) + message, end=end)

def debug(message, indent=0, end=None):
    """ Display debug message.

        Prints always to stderr.
    """

    print((' ' * indent * 4) + message, file=sys.stderr, end=end)

def _parse_args__get_ext(args):
    """ Find extension for backup files.

        Extension in args is supposed to not have leading dot.
    """
    if args.no_backup:
        return ''

    if not args.ext:
        return '.' + DEFAULT_BACKUP_EXTENSION

    return '.' + args.ext

def _parse_args__eval_replacement(repl):
    """ Compile replace argument as valid Python code and return
        function which can be passed to re.sub or re.subn functions.
    """
    def _(match):  # pylint: disable-msg=missing-docstring
        return eval (repl, { '__builtins__': __builtins__ }, { 'm': match })

    return _

def _parse_args__pattern(args):  # pylint: disable-msg=too-many-branches
    """ Read arguments from argparse.ArgumentParser instance, and
        parse it to find correct values for arguments:
            * pattern
            * replace
            * count
        If is provided --pattern_and_replace argument, then parse
        it and return data from it.

        In other case just return data from argparse.ArgumentParser
        instance.

        Reads also string argument, and apply it to pattern.

        Returned pattern is compiled (see: re.compile).
    """

    flags = 0

    if args.ignore_case:
        flags |= re.IGNORECASE

    if args.pattern_dot_all:
        flags |= re.DOTALL

    if args.pattern_verbose:
        flags |= re.VERBOSE

    if args.pattern_multiline:
        flags |= re.MULTILINE

    if args.pattern is not None and args.replace is not None:
        if args.string:
            pattern = re.escape(args.pattern)
        else:
            pattern = args.pattern

        return re.compile(pattern, flags), args.replace, args.count or 0

    elif args.pattern_and_replace is not None:
        pat = args.pattern_and_replace

        try:
            # pattern must begin with 's'
            if not pat.startswith('s'):
                raise ParserException('Bad pattern specified: {0}'.format(args.pattern_and_replace))
            pat = pat[1:]

            # parse pattern
            if pat.startswith('('):
                pattern, replace, _flags = pat[1:].split(')(', 2)
            elif pat.startswith('{'):
                pattern, replace, _flags = pat[1:].split('}{', 2)
            elif pat.startswith('['):
                pattern, replace, _flags = pat[1:].split('][', 2)
            elif pat.startswith('<'):
                pattern, replace, _flags = pat[1:].split('><', 2)
            else:
                delim = pat[0]

                pat = pat[1:]
                pattern, replace, _flags = pat.split(delim, 2)

            # pattern can be suffixed with g (as global)
            if 'g' in _flags:
                count = 0
                pat = pat[:-1]
            else:
                count = args.count if args.count is not None else 1

            if 'i' in _flags:
                flags |= re.IGNORECASE
            if 'x' in _flags:
                flags |= re.VERBOSE
            if 's' in _flags:
                flags |= re.DOTALL
            if 'm' in _flags:
                flags |= re.MULTILINE

        except ValueError:
            raise ParserException('Bad pattern specified: {0}'.format(args.pattern_and_replace))

        return re.compile(pattern, flags), replace, count
    else:
        raise ParserException('Bad pattern specified: {0}'.format(args.pattern_and_replace))

def wrap_text(txt):
    """ Make custom wrapper for passed text.

        Splits given text for lines, and for every line apply custom
        textwrap.TextWrapper settings, then return reformatted string.
    """
    _wrap = textwrap.TextWrapper(
        width = 72,
        expand_tabs = True,
        replace_whitespace = False,
        drop_whitespace = True,
        subsequent_indent = '  ',
    )
    txt = [ _wrap.fill(line) for line in txt.splitlines() ]
    return "\n".join(txt)

def parse_args(args):
    """ Parse arguments passed to script, validate it, compile if needed and return.
    """

    p = argparse.ArgumentParser(  # pylint: disable-msg=invalid-name
        description='Replace PATTERN with REPLACE in many files.',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=wrap_text("Miscellaneous notes:\n"
            "* regular expressions engine used here is PCRE, dialect from Python\n"
            "* is required to pass either --pattern and -replace, or --pattern_and_replace argument\n"
            "* if pattern passed to --pattern_and_replace has /g modifier, it overwrites --count value\n"
            "* if neither /g modifier nor --count argument is passed, assume that --count is equal 1\n"
            "* if only --count is given, this value is used\n"
            "* if --eval-replace is given, --replace must be valid Python code, where can be used m variable."
                "m holds MatchObject instance (see: http://http://docs.python.org/2/library/re.html#match-objects, "
                "for example:\n"
            "    --eval-replace --replace 'm.group(1).lower()'\n"
            "* regular expressions with non linear search read whole file to yours computer memory - if file size is "
                "bigger then you have memory in your computer, it fails\n"
            "* parsing expression passed to --pattern_and_replace argument is very simple - if you use / as delimiter, "
                "then in your expression can\'t be used this character anymore. If you need to use same character as "
                "delimiter and in expression, then better use --pattern and --replace argument\n"
#             "* "
            "\n"
            "Security notes:\n"
            "* be carefull with --eval-replace argument. When it's given, value passed to --replace is eval-ed, so any "
                         "not safe code will be executed!\n"
            "\n"
            "Author:\n"
            "Marcin Sztolcman <marcin@urzenia.net> // http://urzenia.net\n"
            "\n"
            "HomePage:\n"
            "http://mysz.github.io/subst/"
        ),
    )

    p.add_argument('-p', '--pattern', type=str,
                   help='pattern to replace for. Supersede --pattern_and_replace. Required if --replace is specified.')
    p.add_argument('-r', '--replace', type=str,
                   help='replacement. Supersede --pattern_and_replace. Required if --pattern is specified.')
    p.add_argument('--eval-replace', dest='eval', action='store_true',
                   help='if specified, make eval data from --replace(should be valid Python code). Ignored with '
                   '--pattern_and_replace argument.')
    p.add_argument('-t', '--string', type=bool,
                   help='if specified, treats --pattern as string, not as regular expression. Ignored with '
                   '--pattern_and_replace argument.')
    p.add_argument('-s', '--pattern_and_replace', metavar='"s/PAT/REP/gixsm"', type=str,
                   help='pattern and replacement in one: s/pattern/replace/g(pattern is always regular expression, /g '
                   'is optional and stands for --count=0, /i == --ignore-case, /s == --pattern-dot-all, /m == --pattern-multiline).')
    p.add_argument('-c', '--count', type=int,
                   help='make COUNT replacements for every file(0 make unlimited changes, default).')
    p.add_argument('-l', '--linear', action='store_true',
                   help='apply pattern for every line separately. Without this flag whole file is read into memory.')
    p.add_argument('-i', '--ignore-case', dest='ignore_case', action='store_true',
                   help='ignore case of characters when matching')
    p.add_argument('--pattern-dot-all', dest='pattern_dot_all', action='store_true',
                   help='with this flag, dot(.) character in pattern match also new line character (see: '
                   'http://docs.python.org/2/library/re.html#re.DOTALL).')
    p.add_argument('--pattern-verbose', dest='pattern_verbose', action='store_true',
                   help='with this flag pattern can be passed as verbose(see: http://docs.python.org/2/library/re.html#re.VERBOSE).')
    p.add_argument('--pattern-multiline', dest='pattern_multiline', action='store_true',
                   help='with this flag pattern can be passed as multiline(see: http://docs.python.org/2/library/re.html#re.MULTILINE).')
    p.add_argument('-b', '--no-backup', dest='no_backup', action='store_true',
                   help='disable creating backup of modified files.')
    p.add_argument('-e', '--backup-extension', dest='ext', default=DEFAULT_BACKUP_EXTENSION, type=str,
                   help='extension for backuped files(ignore if no backup is created), without leading dot. Defaults to: "bak".')
    p.add_argument('--stdin', action='store_true',
                   help='read data from STDIN(implies --stdout)')
    p.add_argument('--stdout', action='store_true',
                   help='output data to STDOUT instead of change files in-place(implies --no-backup)')
    p.add_argument('--verbose', action='store_true',
                   help='show files and how many replacements was done')
    p.add_argument('--debug', action='store_true',
                   help='show more infos')
    p.add_argument('-v', '--version', action='store_true',
                   help='show version and exit')
    p.add_argument('files', nargs='*', type=str,
                   help='file to parse.')

    args = p.parse_args()

    if args.version:
        return args

    if not args.files or args.files[0] == '-':
        args.stdin = True

    if args.stdin:
        args.stdout = True

    if args.stdout:
        args.no_backup = True

    if \
            (args.pattern is None and args.replace is None and args.pattern_and_replace is None) or \
            (args.pattern is None and args.replace is not None) or \
            (args.pattern is not None and args.replace is None):
        p.error('must be provided --pattern and --replace options, or --pattern_and_replace.')

    try:
        args.ext = _parse_args__get_ext(args)
        args.pattern, args.replace, args.count = _parse_args__pattern(args)
        if args.eval:
            args.replace = _parse_args__eval_replacement (args.replace)
    except ParserException as ex:
        p.error(ex)

    return args

def replace_linear(src, dst, pattern, replace, count):
    """ Read data from 'src' line by line, replace some data from
        regular expression in 'pattern' with data in 'replace',
        write it to 'dst', and return quantity of replaces.
    """
    ret = 0
    for line in src:
        if count == 0 or ret < count:
            line, rest_count = pattern.subn(replace, line, max(0, count - ret))
            ret += rest_count
        dst.write(line)
    return ret

def replace_global(src, dst, pattern, replace, count):
    """ Read whole file from 'src', replace some data from
        regular expression in 'pattern' with data in 'replace',
        write it to 'dst', and return quantity of replaces.
    """
    data = src.read()
    data, ret = pattern.subn(replace, data, count)
    dst.write(data)
    return ret

def _process_file__make_backup(path, backup_ext):
    """ Create backup of file: copy it new extension.

        Returns path to backup file.
    """

    root = os.path.dirname(path)
    backup_path = os.path.join(root, path + backup_ext)

    if os.path.exists(backup_path):
        raise SubstException('Backup path: "{0}" for file "{1}" already exists, file omited'.format(backup_path, path))

    try:
        shutil.copy2(path, backup_path)
    except (shutil.Error, IOError) as ex:
        raise SubstException('Cannot create backup for "{0}": {1}'.format(path, ex))

    return backup_path

def _process_file__handle(src_path, dst_fh, cfg, replace_func):
    """ Read data from `src_path`, replace data with `replace_func` and
        save it to `dst_fh`.
    """

    with open(src_path, 'r') as fh_src:
        cnt = replace_func(fh_src, dst_fh, cfg.pattern, cfg.replace, cfg.count)
        if cfg.verbose or cfg.debug:
            debug('{0} replacements'.format(cnt), 1)

def _process_file__regular(src_path, cfg, replace_func):
    """ Read data from `src_path`, replace data with `replace_func` and
        save it.

        It's safe operation - first save data to temporary file, and then try to rename
        new file to old one.
    """

    tmp_fh, tmp_path = tempfile.mkstemp()
    tmp_fh = os.fdopen(tmp_fh, 'w')

    _process_file__handle(src_path, tmp_fh, cfg, replace_func)

    try:
        os.rename(tmp_path, src_path)
    except OSError as ex:
        raise SubstException('Error replacing "{0}" with "{1}": {2}'.format(src_path, tmp_path, ex))
    else:
        if cfg.debug:
            debug('moved temporary file to original', 1)

    try:
        tmp_fh.close()
    except:  # pylint: disable-msg=bare-except
        pass

def process_file(path, replace_func, cfg):
    """ Process single file: open, read, make backup and replace data.
    """

    if cfg.verbose or cfg.debug:
        debug(path)

    if not os.path.exists(path):
        raise SubstException('Path "{0}" doesn\'t exists'.format(path))

    if not os.path.isfile(path) or os.path.islink(path):
        raise SubstException('Path "{0}" is not a regular file'.format(path))

    if not cfg.no_backup:
        backup_path = _process_file__make_backup(path, cfg.ext)

        if cfg.debug:
            debug('created backup file: "{0}"'.format(backup_path), 1)

    try:
        if cfg.stdout:
            _process_file__handle(path, sys.stdout, cfg, replace_func)
        else:
            _process_file__regular(path, cfg, replace_func)
    except SubstException as ex:
        errmsg(ex)

def main():
    """ Run tool: parse input arguments, read data, replace and save or display.
    """
    args = parse_args(sys.argv[1:])

    if args.version:
        show_version()

    if args.linear:
        replace_func = replace_linear
    else:
        replace_func = replace_global

    if args.stdin:
        replace_func(sys.stdin, sys.stdout, args.pattern, args.replace, args.count)

    else:
        for path in args.files:
            try:
                process_file(path, replace_func, args)
            except SubstException as ex:
                errmsg(ex.message, int(args.verbose or args.debug))

if __name__ == '__main__':
    main()
<|endoftext|>