<commit_before>from deploy import lazy_file, pipe
import ftplib
import os
import paramiko
import tarfile
import urllib
import urlparse
import sh
import socket
import stat
import threading

class location( object ):
    def __init__( self, url ):
        if isinstance( url, basestring ):
            url = urlparse.urlparse( url )
        self.url = url

    def source( self ):
        pass

    def destination( self, files ):
        pass

class local( location ):
    def source( self, base_path = '', recursive = False ):
        cur_path = os.path.join( self.url.path, base_path )

        for path in os.listdir( cur_path ):
            rel_path = os.path.join( base_path, path )
            full_path = os.path.join( cur_path, path )

            if os.path.isfile( full_path ):
                yield lazy_file( full_path ).rename( rel_path )
            elif os.path.isdir( full_path ) and recursive:
                for f in self.source( rel_path, True ):
                    yield f

    def destination( self, files ):
        for f in files:
            dfn = os.path.join( self.url.path, f.name )

            # Ensure dest paths exist
            dpath = os.path.dirname( dfn )
            if not os.path.exists( dpath ):
                os.makedirs( dpath )

            if not f.delete:
                print 'local:%s' % dfn
                with open( dfn, 'wb' ) as dest:
                    while True:
                        chunk = f.read()
                        if chunk:
                            dest.write( chunk )
                        else:
                            break
            else:
                print 'local DELETE', dfn
                try:
                    os.remove( dfn )
                except OSError:
                    pass

class remote( location ):
    def __init__( self, url ):
        super( remote, self ).__init__( url )

        self.con = None

    def connect( self ):
        pass

    def destination( self, files ):
        for f in files:
            if not self.con:
                self.connect()

            if f.delete:
                self.rm( f )
            else:
                self.put( f )

class BadPassiveFTP( ftplib.FTP ):
    def makepasv(self):
        host, port = ftplib.FTP.makepasv( self )

        return socket.gethostbyname( self.host ), port

class ftp( remote ):
    def __init__( self, url, bad_passive_server = False ):
        super( ftp, self ).__init__( url )
        self.bad_passive_server = bad_passive_server

    def connect( self ):
        if self.bad_passive_server:
            self.con = ftplib.FTP()
        else:
            self.con = BadPassiveFTP()

        self.con.connect( self.url.hostname, self.url.port )
        self.con.login( urllib.unquote( self.url.username ), urllib.unquote( self.url.password ) )

    def _remote_path( self, f ):
        return os.path.join( self.url.path, os.path.dirname( f.name ) )

    def source( self, base_path = '', recursive = False ):
        if not self.con:
            self.connect()

        cur_path = os.path.join( self.url.path, base_path )

        try:
            paths = self.con.nlst( cur_path )
        except ftplib.error_perm:
            return

        if paths == [ cur_path ]:
            yield self.get( cur_path ).rename( base_path )
            return

        for path in paths:
            if path in [ '.', '..' ]:
                continue

            rel_path = os.path.join( base_path, path )

            for f in self.source( rel_path, recursive ):
                yield f

    def get( self, path ):
        p = pipe( path )
        self.con.cwd( self._remote_path( p ) )

        def run():
            p.need_data.wait()
            self.con.retrbinary( 'RETR %s' % os.path.basename( path ), p.w.write )
            p.w.write( None )

        threading.Thread( target = run ).start()

        return p.r

    def put( self, f ):
        print f.name

        fpath = self._remote_path( f )

        try:
            self.con.cwd( fpath )
        except ftplib.error_perm as e:
            if e.message.startswith( '550' ):
                self.mkdirs( fpath )
                self.con.cwd( fpath )

        print '%s:%s' % ( self.url.hostname, f.name )
        self.con.storbinary( 'STOR %s' % os.path.basename( f.name ), f )

    def rm( self, f ):
        try:
            self.con.cwd( _remote_path( f ) )
        except Exception, e:
            print 'FTP-ERROR: Could not change directory to', os.path.dirname( f.name )
            print e
            print 'FTP-ERROR: Could not delete', f.name
            return

        try:
            print '%s DELETE %s' % ( self.url.hostname, f.name )
            self.con.delete( os.path.basename( f.name ) )
        except Exception, e:
            print 'FTP-ERROR: Could not delete', f.name
            print e

    def mkdirs( self, path ):
        if path.startswith( '/' ):
            path = path[1:]

        if not path:
            return

        self.con.cwd( '/' )

        last_existed = True
        for p in path.split( os.sep ):
            if not last_existed or p not in self.con.nlst():
                self.con.mkd( p )
                last_existed = False
            self.con.cwd( p )

class ssh( remote ):
    def connect( self ):
        self.sshcli = paramiko.SSHClient()
        self.sshcli.set_missing_host_key_policy( paramiko.AutoAddPolicy() )
        self.sshcli.connect( self.url.hostname, port = self.url.port )
        self.con = self.sshcli.open_sftp()

        try:
            self.con.chdir( self.url.path )
        except IOError:
            self.mkdirs( self.url.path )
            self.con.chdir( self.url.path )

    def source( self, base_path = '', recursive = False ):
        if not self.con:
            self.connect()

        cur_path = os.path.join( self.url.path, base_path )

        for file_attr in self.con.listdir_attr( cur_path ):
            rel_path = os.path.join( base_path, file_attr.filename )
            full_path = os.path.join( cur_path, file_attr.filename )

            if stat.S_ISDIR( file_attr.st_mode ) and recursive:
                for f in self.source( rel_path, recursive ):
                    yield f
            else:
                yield self.get( full_path ).rename( rel_path )

    def get( self, path ):
        p = pipe( path )

        def run():
            p.need_data.wait()
            self.con.getfo( path, p.w, callback = self.report_progress )
            p.w.write( None )

        threading.Thread( target = run ).start()

        return p.r

    def put( self, f ):
        print '%s:%s' % ( self.url.hostname, f.name )
        try:
            self.con.putfo( f, f.name, callback = self.report_progress )
        except IOError:
            # Most likely that dir does not exist, create and retry
            # TODO: Causes infinite recursion on permission denied
            self.mkdirs( os.path.dirname( f.name ) )
            self.put( f )

    def report_progress( self, a, b ):
        print a, b

    def rm( self, f ):
        print '%s DELETE %s' % ( self.url.hostname, f.name )
        try:
            self.con.remove( f.name )
        except IOError:
            # Most likely file does not exist, no need to remove it then
            pass

    def mkdirs( self, path ):
        cur_path = ''
        last_existed = True

        for p in path.split( os.sep ):
            cur_path = os.path.join( cur_path, p )

            if last_existed:
                try:
                    self.con.stat( cur_path )
                    continue
                except IOError:
                    pass

            print 'creating', cur_path
            self.con.mkdir( cur_path )

class git( local ):
    def __init__( self, url = '' ):
        super( git, self ).__init__( url )

    def get_ref_commit( self, ref = 'HEAD' ):
        return str( sh.git( 'rev-parse', ref ) ).strip()

    def source( self, from_commit = None, to_commit = None ):
        if from_commit is not None:
            if to_commit is not None:
                files = sh.git.diff( '--name-status', '--no-renames', '--color=never', from_commit, to_commit, _iter = True, _tty_out = False )
            else:
                files = sh.git.diff( '--name-status', '--no-renames', '--color=never', from_commit, _iter = True, _tty_out = False )

            all_files = False
        else:
            files = sh.git( "ls-files", _iter = True, _tty_out = False )
            all_files = True

        for l in files:
            mode = 'A'
            if not all_files:
                mode, fname = l.strip().split( '\t' )
            else:
                fname = l.strip()

            f = lazy_file( fname )
            if mode == 'D':
                f.delete = True

            yield f

class tar( location ):
    def __init__( self, f ):
        self.f = f

    def source( self ):
        tar = tarfile.open( fileobj = self.f, mode = 'r|*' )

        for tarinfo in tar:
            yield tar.extractfile( tarinfo )

    def destination( self, files ):
        tar = tarfile.open( fileobj = self.f, mode = 'w|gz' )

        for f in files:
            f.read(0)
            tarinfo = tarfile.TarInfo( f.name )
            tarinfo.size = f.size
            tar.addfile( tarinfo, f )

class zip( location ):
    # Use zip
    # https://docs.python.org/2/library/zip.html
    pass
<commit_msg>use read_all in local loc<commit_after>from deploy import lazy_file, pipe, read_all
import ftplib
import os
import paramiko
import tarfile
import urllib
import urlparse
import sh
import socket
import stat
import threading

class location( object ):
    def __init__( self, url ):
        if isinstance( url, basestring ):
            url = urlparse.urlparse( url )
        self.url = url

    def source( self ):
        pass

    def destination( self, files ):
        pass

class local( location ):
    def source( self, base_path = '', recursive = False ):
        cur_path = os.path.join( self.url.path, base_path )

        for path in os.listdir( cur_path ):
            rel_path = os.path.join( base_path, path )
            full_path = os.path.join( cur_path, path )

            if os.path.isfile( full_path ):
                yield lazy_file( full_path ).rename( rel_path )
            elif os.path.isdir( full_path ) and recursive:
                for f in self.source( rel_path, True ):
                    yield f

    def destination( self, files ):
        for f in files:
            dfn = os.path.join( self.url.path, f.name )

            # Ensure dest paths exist
            dpath = os.path.dirname( dfn )
            if not os.path.exists( dpath ):
                os.makedirs( dpath )

            if not f.delete:
                print 'local:%s' % dfn
                with open( dfn, 'wb' ) as dest:
                    read_all( f, dest.write )
            else:
                print 'local DELETE', dfn
                try:
                    os.remove( dfn )
                except OSError:
                    pass

class remote( location ):
    def __init__( self, url ):
        super( remote, self ).__init__( url )

        self.con = None

    def connect( self ):
        pass

    def destination( self, files ):
        for f in files:
            if not self.con:
                self.connect()

            if f.delete:
                self.rm( f )
            else:
                self.put( f )

class BadPassiveFTP( ftplib.FTP ):
    def makepasv(self):
        host, port = ftplib.FTP.makepasv( self )

        return socket.gethostbyname( self.host ), port

class ftp( remote ):
    def __init__( self, url, bad_passive_server = False ):
        super( ftp, self ).__init__( url )
        self.bad_passive_server = bad_passive_server

    def connect( self ):
        if self.bad_passive_server:
            self.con = ftplib.FTP()
        else:
            self.con = BadPassiveFTP()

        self.con.connect( self.url.hostname, self.url.port )
        self.con.login( urllib.unquote( self.url.username ), urllib.unquote( self.url.password ) )

    def _remote_path( self, f ):
        return os.path.join( self.url.path, os.path.dirname( f.name ) )

    def source( self, base_path = '', recursive = False ):
        if not self.con:
            self.connect()

        cur_path = os.path.join( self.url.path, base_path )

        try:
            paths = self.con.nlst( cur_path )
        except ftplib.error_perm:
            return

        if paths == [ cur_path ]:
            yield self.get( cur_path ).rename( base_path )
            return

        for path in paths:
            if path in [ '.', '..' ]:
                continue

            rel_path = os.path.join( base_path, path )

            for f in self.source( rel_path, recursive ):
                yield f

    def get( self, path ):
        p = pipe( path )
        self.con.cwd( self._remote_path( p ) )

        def run():
            p.need_data.wait()
            self.con.retrbinary( 'RETR %s' % os.path.basename( path ), p.w.write )
            p.w.write( None )

        threading.Thread( target = run ).start()

        return p.r

    def put( self, f ):
        print f.name

        fpath = self._remote_path( f )

        try:
            self.con.cwd( fpath )
        except ftplib.error_perm as e:
            if e.message.startswith( '550' ):
                self.mkdirs( fpath )
                self.con.cwd( fpath )

        print '%s:%s' % ( self.url.hostname, f.name )
        self.con.storbinary( 'STOR %s' % os.path.basename( f.name ), f )

    def rm( self, f ):
        try:
            self.con.cwd( _remote_path( f ) )
        except Exception, e:
            print 'FTP-ERROR: Could not change directory to', os.path.dirname( f.name )
            print e
            print 'FTP-ERROR: Could not delete', f.name
            return

        try:
            print '%s DELETE %s' % ( self.url.hostname, f.name )
            self.con.delete( os.path.basename( f.name ) )
        except Exception, e:
            print 'FTP-ERROR: Could not delete', f.name
            print e

    def mkdirs( self, path ):
        if path.startswith( '/' ):
            path = path[1:]

        if not path:
            return

        self.con.cwd( '/' )

        last_existed = True
        for p in path.split( os.sep ):
            if not last_existed or p not in self.con.nlst():
                self.con.mkd( p )
                last_existed = False
            self.con.cwd( p )

class ssh( remote ):
    def connect( self ):
        self.sshcli = paramiko.SSHClient()
        self.sshcli.set_missing_host_key_policy( paramiko.AutoAddPolicy() )
        self.sshcli.connect( self.url.hostname, port = self.url.port )
        self.con = self.sshcli.open_sftp()

        try:
            self.con.chdir( self.url.path )
        except IOError:
            self.mkdirs( self.url.path )
            self.con.chdir( self.url.path )

    def source( self, base_path = '', recursive = False ):
        if not self.con:
            self.connect()

        cur_path = os.path.join( self.url.path, base_path )

        for file_attr in self.con.listdir_attr( cur_path ):
            rel_path = os.path.join( base_path, file_attr.filename )
            full_path = os.path.join( cur_path, file_attr.filename )

            if stat.S_ISDIR( file_attr.st_mode ) and recursive:
                for f in self.source( rel_path, recursive ):
                    yield f
            else:
                yield self.get( full_path ).rename( rel_path )

    def get( self, path ):
        p = pipe( path )

        def run():
            p.need_data.wait()
            self.con.getfo( path, p.w, callback = self.report_progress )
            p.w.write( None )

        threading.Thread( target = run ).start()

        return p.r

    def put( self, f ):
        print '%s:%s' % ( self.url.hostname, f.name )
        try:
            self.con.putfo( f, f.name, callback = self.report_progress )
        except IOError:
            # Most likely that dir does not exist, create and retry
            # TODO: Causes infinite recursion on permission denied
            self.mkdirs( os.path.dirname( f.name ) )
            self.put( f )

    def report_progress( self, a, b ):
        print a, b

    def rm( self, f ):
        print '%s DELETE %s' % ( self.url.hostname, f.name )
        try:
            self.con.remove( f.name )
        except IOError:
            # Most likely file does not exist, no need to remove it then
            pass

    def mkdirs( self, path ):
        cur_path = ''
        last_existed = True

        for p in path.split( os.sep ):
            cur_path = os.path.join( cur_path, p )

            if last_existed:
                try:
                    self.con.stat( cur_path )
                    continue
                except IOError:
                    pass

            print 'creating', cur_path
            self.con.mkdir( cur_path )

class git( local ):
    def __init__( self, url = '' ):
        super( git, self ).__init__( url )

    def get_ref_commit( self, ref = 'HEAD' ):
        return str( sh.git( 'rev-parse', ref ) ).strip()

    def source( self, from_commit = None, to_commit = None ):
        if from_commit is not None:
            if to_commit is not None:
                files = sh.git.diff( '--name-status', '--no-renames', '--color=never', from_commit, to_commit, _iter = True, _tty_out = False )
            else:
                files = sh.git.diff( '--name-status', '--no-renames', '--color=never', from_commit, _iter = True, _tty_out = False )

            all_files = False
        else:
            files = sh.git( "ls-files", _iter = True, _tty_out = False )
            all_files = True

        for l in files:
            mode = 'A'
            if not all_files:
                mode, fname = l.strip().split( '\t' )
            else:
                fname = l.strip()

            f = lazy_file( fname )
            if mode == 'D':
                f.delete = True

            yield f

class tar( location ):
    def __init__( self, f ):
        self.f = f

    def source( self ):
        tar = tarfile.open( fileobj = self.f, mode = 'r|*' )

        for tarinfo in tar:
            yield tar.extractfile( tarinfo )

    def destination( self, files ):
        tar = tarfile.open( fileobj = self.f, mode = 'w|gz' )

        for f in files:
            f.read(0)
            tarinfo = tarfile.TarInfo( f.name )
            tarinfo.size = f.size
            tar.addfile( tarinfo, f )

class zip( location ):
    # Use zip
    # https://docs.python.org/2/library/zip.html
    pass
<|endoftext|>


<commit_before>import re
import sys
from datetime import datetime

# import code; code.interact(local=dict(globals(), **locals()))

def parse(filename, processors):
    'Process a given log file with provided processors'
    log_re = re.compile(r"""
        (?P<ip>[.:0-9a-fA-F]{7,45})
        \s
        (?P<identd>-|.+?)
        \s
        (?P<userid>-|.+?)
        \s
        \[(?P<timestamp>.+?)\]
        \s
        "(?P<uri>[A-Z]+?\ /.*?\ HTTP/1.\d)"
        \s
        (?P<status_code>\d+?)
        \s
        (?P<size_bytes>\d+?)
        \s
        "(?P<referer>.*?)"
        \s
        "(?P<user_agent>.*?)"
        \s
        (?P<duration_microseconds>\d+)
    """, re.VERBOSE)

    # Scan through a string, looking for any location where this RE matches.
    match = re.compile(log_re).match
    matches = (match(line) for line in open(filename, 'r'))

    matched_tuples = (match.groupdict() for match in matches if match)

    for matched in matched_tuples:
        for processor in processors:
            processor.process(matched)

class LineCount(object):
    def __init__(self):
        self.lines = 0;

    def process(self, matches):
        self.lines += 1

    def print_result(self):
        print "Number of lines parsed:", self.lines

class LogDuration(object):
    def __init__(self):
        self.earliest_timestamp = None;
        self.latest_timestamp = None;

    def process(self, matches):
        timestamp_string = matches['timestamp'][0:-6] # Strip the trailing timezone
        timestamp = datetime.strptime(timestamp_string, '%d/%b/%Y:%H:%M:%S')
        if self.earliest_timestamp == None or timestamp < self.earliest_timestamp:
            self.earliest_timestamp = timestamp

        if self.latest_timestamp == None or timestamp > self.latest_timestamp:
            self.latest_timestamp = timestamp

    def print_result(self):
        duration = self.latest_timestamp - self.earliest_timestamp
        print "Duration of log file", duration

class MostRequestedPage(object):
    def __init__(self): pass

    def process(self, matches): pass

    def print_result(self):
        print "Most requested page:"

class MostFrequentVisitor(object):
    def __init__(self): pass

    def process(self, matches): pass

    def print_result(self):
        print "Most frequent visitor:"

class MinPageLoadTime(object):
    def __init__(self):
        self.min_page_load_time = None

    def process(self, matches):
        duration_microseconds = int(matches['duration_microseconds'])
        if self.min_page_load_time == None \
                or duration_microseconds < self.min_page_load_time:
            self.min_page_load_time = duration_microseconds

    def print_result(self):
        print "Min page load time:", self.min_page_load_time

class AveragePageLoadTime(object):
    def __init__(self):
        self.total_page_load_time = 0
        self.total_pages_loaded = 0

    def process(self, matches):
        self.total_page_load_time += int(matches['duration_microseconds'])
        self.total_pages_loaded += 1

    def print_result(self):
        average_load_time = float(self.total_page_load_time) / self.total_pages_loaded

        print "Average page load time:", average_load_time

class MaxPageLoadTime(object):
    def __init__(self):
        self.max_page_load_time = None

    def process(self, matches):
        duration_microseconds = int(matches['duration_microseconds'])
        if self.max_page_load_time == None \
                or duration_microseconds > self.max_page_load_time:
            self.max_page_load_time = duration_microseconds

    def print_result(self):
        print "Max page load time:", self.max_page_load_time

class NumberOfErrors(object):
    def __init__(self):
        self.total_errors = 0

    def process(self, matches):
        status_code = int(matches['status_code'])
        if status_code >= 400:
            self.total_errors += 1;

    def print_result(self):
        print "Number of errors:", self.total_errors

class TotalDataTransfered(object):
    def __init__(self):
        self.total_data = 0

    def process(self, matches):
        self.total_data += int(matches['size_bytes'])

    def print_result(self):
        print "Total data transferred:", self.total_data

def main():
    if len(sys.argv) < 2:
        print "You didn't pass a filename to parse!"
        sys.exit(1)
    filename = sys.argv[1]
    processors = [
            LineCount(),
            LogDuration(),
            MostRequestedPage(),
            MostFrequentVisitor(),
            MinPageLoadTime(),
            AveragePageLoadTime(),
            MaxPageLoadTime(),
            NumberOfErrors(),
            TotalDataTransfered()
            ]
    parse(filename, processors)
    for processor in processors:
        processor.print_result()

if __name__ == '__main__':
    main()
<commit_msg>Implement MostFrequentVisitor<commit_after>import re
import sys
from datetime import datetime

# import code; code.interact(local=dict(globals(), **locals()))

def parse(filename, processors):
    'Process a given log file with provided processors'
    log_re = re.compile(r"""
        (?P<ip>[.:0-9a-fA-F]{7,45})
        \s
        (?P<identd>-|.+?)
        \s
        (?P<userid>-|.+?)
        \s
        \[(?P<timestamp>.+?)\]
        \s
        "(?P<uri>[A-Z]+?\ /.*?\ HTTP/1.\d)"
        \s
        (?P<status_code>\d+?)
        \s
        (?P<size_bytes>\d+?)
        \s
        "(?P<referer>.*?)"
        \s
        "(?P<user_agent>.*?)"
        \s
        (?P<duration_microseconds>\d+)
    """, re.VERBOSE)

    # Scan through a string, looking for any location where this RE matches.
    match = re.compile(log_re).match
    matches = (match(line) for line in open(filename, 'r'))

    matched_tuples = (match.groupdict() for match in matches if match)

    for matched in matched_tuples:
        for processor in processors:
            processor.process(matched)

class LineCount(object):
    def __init__(self):
        self.lines = 0;

    def process(self, matches):
        self.lines += 1

    def print_result(self):
        print "Number of lines parsed:", self.lines

class LogDuration(object):
    def __init__(self):
        self.earliest_timestamp = None;
        self.latest_timestamp = None;

    def process(self, matches):
        timestamp_string = matches['timestamp'][0:-6] # Strip the trailing timezone
        timestamp = datetime.strptime(timestamp_string, '%d/%b/%Y:%H:%M:%S')
        if self.earliest_timestamp == None or timestamp < self.earliest_timestamp:
            self.earliest_timestamp = timestamp

        if self.latest_timestamp == None or timestamp > self.latest_timestamp:
            self.latest_timestamp = timestamp

    def print_result(self):
        duration = self.latest_timestamp - self.earliest_timestamp
        print "Duration of log file", duration

class MostRequestedPage(object):
    def __init__(self): pass

    def process(self, matches): pass

    def print_result(self):
        print "Most requested page:"

class MostFrequentVisitor(object):
    def __init__(self):
        self.visitors = {}
        self.most_frequent_visitor = None

    def process(self, matches):
        visitor = matches['ip']
        if visitor not in self.visitors:
            self.visitors[visitor] = 0

        self.visitors[visitor] += 1

        if self.most_frequent_visitor == None \
                or self.visitors[self.most_frequent_visitor] < self.visitors[visitor]:
            self.most_frequent_visitor = visitor

    def print_result(self):
        print "Most frequent visitor:", self.most_frequent_visitor

class MinPageLoadTime(object):
    def __init__(self):
        self.min_page_load_time = None

    def process(self, matches):
        duration_microseconds = int(matches['duration_microseconds'])
        if self.min_page_load_time == None \
                or duration_microseconds < self.min_page_load_time:
            self.min_page_load_time = duration_microseconds

    def print_result(self):
        print "Min page load time:", self.min_page_load_time

class AveragePageLoadTime(object):
    def __init__(self):
        self.total_page_load_time = 0
        self.total_pages_loaded = 0

    def process(self, matches):
        self.total_page_load_time += int(matches['duration_microseconds'])
        self.total_pages_loaded += 1

    def print_result(self):
        average_load_time = float(self.total_page_load_time) / self.total_pages_loaded

        print "Average page load time:", average_load_time

class MaxPageLoadTime(object):
    def __init__(self):
        self.max_page_load_time = None

    def process(self, matches):
        duration_microseconds = int(matches['duration_microseconds'])
        if self.max_page_load_time == None \
                or duration_microseconds > self.max_page_load_time:
            self.max_page_load_time = duration_microseconds

    def print_result(self):
        print "Max page load time:", self.max_page_load_time

class NumberOfErrors(object):
    def __init__(self):
        self.total_errors = 0

    def process(self, matches):
        status_code = int(matches['status_code'])
        if status_code >= 400:
            self.total_errors += 1;

    def print_result(self):
        print "Number of errors:", self.total_errors

class TotalDataTransfered(object):
    def __init__(self):
        self.total_data = 0

    def process(self, matches):
        self.total_data += int(matches['size_bytes'])

    def print_result(self):
        print "Total data transferred:", self.total_data

def main():
    if len(sys.argv) < 2:
        print "You didn't pass a filename to parse!"
        sys.exit(1)
    filename = sys.argv[1]
    processors = [
            LineCount(),
            LogDuration(),
            MostRequestedPage(),
            MostFrequentVisitor(),
            MinPageLoadTime(),
            AveragePageLoadTime(),
            MaxPageLoadTime(),
            NumberOfErrors(),
            TotalDataTransfered()
            ]
    parse(filename, processors)
    for processor in processors:
        processor.print_result()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

#       This program is free software; you can redistribute it and/or modify
#       it under the terms of the GNU General Public License as published by
#       the Free Software Foundation; either version 3 of the License, or
#       (at your option) any later version.
#
#       This program is distributed in the hope that it will be useful,
#       but WITHOUT ANY WARRANTY; without even the implied warranty of
#       MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#       GNU General Public License for more details.
#
#       You should have received a copy of the GNU General Public License
#       along with this program; if not, write to the Free Software
#       Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#       MA 02110-1301, USA.

from sys import argv
import re
import time
import locale
import os

try:
    from mod_python import apache
except:
    pass

class chatlog:
    def __init__(self, textlog, language, plain=False):
        """
        Input a text/plain chatlog from zweipktfkt and get out HTML5 goodness.
        """

        # precompile regular expressions
        hosts_re = re.compile(r'(^[0-9]{2}:[0-9]{2} [^<][^ (]*) \(.*@.*\) (has (joined|quit|left))')
        chars_re = re.compile(u'[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F]')
        uri_patterns = [ r'''((?<=\()\b[A-Za-z][A-Za-z0-9\+\.\-]*:([A-Za-z0-9\.\-_~:/\?#\[\]@!\$&'\(\)\*\+,;=]|%[A-Fa-f0-9]{2})+(?=\)))''', r'''((?<=<)\b[A-Za-z][A-Za-z0-9\+\.\-]*:([A-Za-z0-9\.\-_~:/\?#\[\]@!\$&'\(\)\*\+,;=]|%[A-Fa-f0-9]{2})+(?=>))''', r'''(?<!\()\b([A-Za-z][A-Za-z0-9\+\.\-]*:([A-Za-z0-9\.\-_~:/\?#\[\]@!\$&'\(\)\*\+,;=]|%[A-Fa-f0-9]{2})+)''', ]
        uri_res = [re.compile(p) for p in uri_patterns]

        self.log = ""

        firstline = textlog.split("\n")[0]
        try:
            locale.setlocale(locale.LC_ALL, "en_US.utf-8")
            datetime = time.strptime(firstline[15:])
        except ValueError:
            locale.setlocale(locale.LC_ALL, "de_DE.utf-8")
            datetime = time.strptime(firstline[15:])
        locale.setlocale(locale.LC_ALL, "de_DE.utf-8")

        if not plain:
            self.log += '''<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html>

    <title>Log für ''' + time.strftime("%A, den %d. %B %Y", datetime) + '''</title>
    <link rel="stylesheet" href="css/tango.css" title="Defaul-Stil" type="text/css"/>

Plaintext

    <input type="checkbox" id="compact" onclick="save()"/> Compact

'''

        lastlineid = 1

        for lineid, line in enumerate(textlog.split("\n")):

            # remove hosts
            line = hosts_re.sub(r'\1 \2', line)

            # remove erroneous spaces
            try:
                if line[6:8] == "< ":
                    line = line[:7] + line[8:]
            except IndexError:
                pass

            # replace xml chars
            if not plain:
                line = line.replace("&","&")
                line = line.replace("<","<")
                line = line.replace(">",">")
                line = line.replace("'","'")
                line = line.replace("\"",""")

            # input is mixed utf-8 and latin-1
            try:
                line = unicode(line,'utf-8','strict')
            except UnicodeDecodeError:
                line = unicode(line,'latin-1','strict')

            line = line.encode('utf-8')
            if not plain:
                # WTF is this supposed to do?
                line, count = chars_re.subn('',line)

            if not plain:
                # markup log time, add anchors, mark non-speak lines
                try:
                    int(line[:2])
                    int(line[3:5])
                    line = '' + line[:5] + '' + line[5:]

                    if line[32:36] == "<":
                        line = '#' + line + ''
                    else:
                        line = '#' + line + ''

                    lastlineid = lineid
                except ValueError:
                    pass

                # markup links
                uri_replacement = r'''\1'''

                for p in uri_res:
                    line, nsubs = p.subn(uri_replacement, line)
                    if nsubs > 0: break     # only use first matching pattern

            self.log += line + ("\n" if plain else "
\n")

            if (not plain) and line == textlog.split("\n")[-1:][0]:
                link = '''⤓'''
                self.log = self.log.replace("", link)

        if not plain:
            self.log += """
"""

    def __str__(self):
        return self.log

class DirectoryListing:
    def __init__(self, path, language):
        self.listing = '''<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html>

    <title>Logs</title>
    <link rel="stylesheet" href="css/tango.css" title="Defaul-Stil" type="text/css"/>

'''

        files = [l for l in os.listdir(path) if l[-4:] == ".log"]
        files.sort()
        files.reverse()
        for f in files:
            self.listing += "" + f[:-4] + ""
            self.listing += " (plain)"
            self.listing += "
\n"

        #self.listing += """
Change password

        self.listing += """
"""

    def __str__(self):
        return self.listing

if __name__ == '__main__':
    infile = argv[1]
    if os.path.isdir(infile):
        outfile = os.path.join(infile, "index.html")
        g = open(outfile,"w")
        g.write(str(DirectoryListing(infile, "de")))
        g.close()
    else:
        outfile = infile + ".xhtml"

        f = open(infile, "r")
        input = f.read()
        f.close()

        nicelog = str(chatlog(input, "de"))
        stats = DirectoryStatistics.FileStatistics(input)
        print str(stats)

        g = open(outfile, "w")
        g.write(nicelog)
        g.close()

<commit_msg>do not filter hosts from output<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

#       This program is free software; you can redistribute it and/or modify
#       it under the terms of the GNU General Public License as published by
#       the Free Software Foundation; either version 3 of the License, or
#       (at your option) any later version.
#
#       This program is distributed in the hope that it will be useful,
#       but WITHOUT ANY WARRANTY; without even the implied warranty of
#       MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#       GNU General Public License for more details.
#
#       You should have received a copy of the GNU General Public License
#       along with this program; if not, write to the Free Software
#       Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#       MA 02110-1301, USA.

from sys import argv
import re
import time
import locale
import os

try:
    from mod_python import apache
except:
    pass

class chatlog:
    def __init__(self, textlog, language, plain=False):
        """
        Input a text/plain chatlog from zweipktfkt and get out HTML5 goodness.
        """

        # precompile regular expressions
        chars_re = re.compile(u'[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F]')
        uri_patterns = [ r'''((?<=\()\b[A-Za-z][A-Za-z0-9\+\.\-]*:([A-Za-z0-9\.\-_~:/\?#\[\]@!\$&'\(\)\*\+,;=]|%[A-Fa-f0-9]{2})+(?=\)))''', r'''((?<=<)\b[A-Za-z][A-Za-z0-9\+\.\-]*:([A-Za-z0-9\.\-_~:/\?#\[\]@!\$&'\(\)\*\+,;=]|%[A-Fa-f0-9]{2})+(?=>))''', r'''(?<!\()\b([A-Za-z][A-Za-z0-9\+\.\-]*:([A-Za-z0-9\.\-_~:/\?#\[\]@!\$&'\(\)\*\+,;=]|%[A-Fa-f0-9]{2})+)''', ]
        uri_res = [re.compile(p) for p in uri_patterns]

        self.log = ""

        firstline = textlog.split("\n")[0]
        try:
            locale.setlocale(locale.LC_ALL, "en_US.utf-8")
            datetime = time.strptime(firstline[15:])
        except ValueError:
            locale.setlocale(locale.LC_ALL, "de_DE.utf-8")
            datetime = time.strptime(firstline[15:])
        locale.setlocale(locale.LC_ALL, "de_DE.utf-8")

        if not plain:
            self.log += '''<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html>

    <title>Log für ''' + time.strftime("%A, den %d. %B %Y", datetime) + '''</title>
    <link rel="stylesheet" href="css/tango.css" title="Defaul-Stil" type="text/css"/>

Plaintext

    <input type="checkbox" id="compact" onclick="save()"/> Compact

'''

        lastlineid = 1

        for lineid, line in enumerate(textlog.split("\n")):

            # remove erroneous spaces
            try:
                if line[6:8] == "< ":
                    line = line[:7] + line[8:]
            except IndexError:
                pass

            # replace xml chars
            if not plain:
                line = line.replace("&","&")
                line = line.replace("<","<")
                line = line.replace(">",">")
                line = line.replace("'","'")
                line = line.replace("\"",""")

            # input is mixed utf-8 and latin-1
            try:
                line = unicode(line,'utf-8','strict')
            except UnicodeDecodeError:
                line = unicode(line,'latin-1','strict')

            line = line.encode('utf-8')
            if not plain:
                # WTF is this supposed to do?
                line, count = chars_re.subn('',line)

            if not plain:
                # markup log time, add anchors, mark non-speak lines
                try:
                    int(line[:2])
                    int(line[3:5])
                    line = '' + line[:5] + '' + line[5:]

                    if line[32:36] == "<":
                        line = '#' + line + ''
                    else:
                        line = '#' + line + ''

                    lastlineid = lineid
                except ValueError:
                    pass

                # markup links
                uri_replacement = r'''\1'''

                for p in uri_res:
                    line, nsubs = p.subn(uri_replacement, line)
                    if nsubs > 0: break     # only use first matching pattern

            self.log += line + ("\n" if plain else "
\n")

            if (not plain) and line == textlog.split("\n")[-1:][0]:
                link = '''⤓'''
                self.log = self.log.replace("", link)

        if not plain:
            self.log += """
"""

    def __str__(self):
        return self.log

class DirectoryListing:
    def __init__(self, path, language):
        self.listing = '''<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html>

    <title>Logs</title>
    <link rel="stylesheet" href="css/tango.css" title="Defaul-Stil" type="text/css"/>

'''

        files = [l for l in os.listdir(path) if l[-4:] == ".log"]
        files.sort()
        files.reverse()
        for f in files:
            self.listing += "" + f[:-4] + ""
            self.listing += " (plain)"
            self.listing += "
\n"

        #self.listing += """
Change password

        self.listing += """
"""

    def __str__(self):
        return self.listing

if __name__ == '__main__':
    infile = argv[1]
    if os.path.isdir(infile):
        outfile = os.path.join(infile, "index.html")
        g = open(outfile,"w")
        g.write(str(DirectoryListing(infile, "de")))
        g.close()
    else:
        outfile = infile + ".xhtml"

        f = open(infile, "r")
        input = f.read()
        f.close()

        nicelog = str(chatlog(input, "de"))
        stats = DirectoryStatistics.FileStatistics(input)
        print str(stats)

        g = open(outfile, "w")
        g.write(nicelog)
        g.close()

<|endoftext|>


<commit_before>from __future__ import division

from django.shortcuts import render_to_response,  get_object_or_404, redirect
from django.template.context import RequestContext, Context
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.contrib.auth.models import User

from django.core.mail import send_mail
from django.template.loader import render_to_string

import stripe

from models import *
from django.conf import settings

from views_utils import *

def processStripe(request, customer = None):
    args = {}
    stripe.api_key = settings.STRIPE_SECRET_KEY
    token = request.POST['stripeToken']
    amount = request.POST['cc-amount']
    if customer is None:
        customer = stripe.Customer.create(
            source=token,
            description="New Customer"
        )
    try:
        #Save New customer to user
        if request.user.is_authenticated():
            if not BillingInfo.objects.filter(user=request.user).exists():
                billing = BillingInfo(user=request.user, stripe=customer.id)
                billing.save()
            else:
                request.user.billinginfo.stripe = customer.id
                request.user.billinginfo.save()
        order = processProducts(request, request.session.get('cart', []))
        try:
            processEmail(order)
        except:
            #Set a method to handle errors
            pass
        charge = stripe.Charge.create(
            amount=int(order.total), # amount in cents, again
            currency="usd",
            customer=customer.id,
        )
        args['cc_result'] = "Your card was successfully charged $" + str(float(order.total))
    except stripe.error.CardError, e:
        # The card has been declined
        args['cc_result'] = e
    return args

def processEmail(order):
    d = Context({ 'order': order })
    msg_plain = render_to_string('email/notification.txt', d)
    msg_html = render_to_string('email/notification.html', d)
    send_mail(
        'We have a new order!',
        msg_plain,
        'admin@code.coop',
        ["bjageman@code.coop"],
        html_message=msg_html,
    )
    if order.shipping_email is not None:
        msg_plain = render_to_string('email/order_confirm.txt', d)
        msg_html = render_to_string('email/order_confirm.html', d)
        send_mail(
            'Thanks for the order!',
            msg_plain,
            'admin@code.coop',
            [order.shipping_email],
            html_message=msg_html,
        )

def processProducts(request, cart):
    products = setProducts(request.session.get('cart', []))
    user = request.user
    if request.user.is_authenticated():
        address_text = str(user.shipping.name + " " + user.shipping.address + " " + user.shipping.city + ", " + user.shipping.state + " " + user.shipping.postal_code)
        order = Order(user=user, shipping_address=address_text, shipping_email=user.shipping.email)
    else:
        address_text = str(request.session['shipping']['name'] + " " + request.session['shipping']['address'] + " " + request.session['shipping']['city'] + ", " + request.session['shipping']['state'] + " " + request.session['shipping']['postal_code'])
        order = Order(shipping_address=address_text, shipping_email=request.session['shipping']['email'])
    order.save()
    #Edit the quantity of the products in stock
    for item in products:
        quantity = item['session']['quantity']
        #Verify that the product does not go beyond the stock limit
        if not (item['product'].preorder or item['product'].status() == "unlimited"):
            quantity = item['product'].set_limit(quantity)
        if item['accessory'] is not None:
            if not (item['accessory'].preorder or item['accessory'].status() == "unlimited"):
                quantity = item['accessory'].set_limit(quantity)
            if item['accessory'].stock is not None:
                item['accessory'].stock = item['accessory'].stock - quantity
            item['accessory'].save()
        if item['product'].stock is not None:
            item['product'].stock = item['product'].stock - quantity
        addProductToOrder(order, item['product'], item['accessory'], item['option'], quantity)
        item['product'].save()
    order.save()
    if 'coupon' in request.session:
        coupon = Coupon.objects.get(code=request.session['coupon']['code'])
        #This will be need to be changed once product associations are implemented
        order.coupon = coupon.code + ": " + str(coupon.amount_off)
        order.discount = coupon.amount_off
        coupon.times_redeemed = coupon.times_redeemed + 1
        del request.session['coupon']
        coupon.save()
    order.save()
    del request.session['cart']
    return order

def totalProductOrder(product, accessory):
    if accessory is not None:
        return product.total() + accessory.total()
    else:
        return product.total()

def addProductToOrder(order, product, accessory, option, quantity):
    productText = str(product.pk) + ") " + product.title
    accessory_title = ""
    option_title = ""
    if accessory is not None:
        accessory_title = accessory.title
        productText = productText + " w/ " + accessory.title
    if option is not None:
        option_title = option.title
        productText = productText + "(" + option.title +")"
    productOrder = ProductOrder(
            order=order,
            product_log=productText,
            accessories=accessory_title,
            options=option_title,
            quantity=quantity,
            price=totalProductOrder(product, accessory)
        )
    productOrder.save()

def charge(request):
    args = {}
    if request.user.is_authenticated():
        args["shipping"] = get_object_or_404(Shipping, user=request.user)
    elif 'shipping' in request.session:
        args["shipping"] = request.session['shipping']
    else:
        return HttpResponseRedirect( reverse('store:checkout') )
    args = processStripe(request)
    return render_to_response('store/order-complete.html', RequestContext(request,args))
<commit_msg>some bug fixes<commit_after>from __future__ import division

from django.shortcuts import render_to_response,  get_object_or_404, redirect
from django.template.context import RequestContext, Context
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.contrib.auth.models import User

from django.core.mail import send_mail
from django.template.loader import render_to_string

import stripe

from models import *
from django.conf import settings

from views_utils import *

def processStripe(request, customer = None):
    args = {}
    stripe.api_key = settings.STRIPE_SECRET_KEY
    token = request.POST['stripeToken']
    amount = request.POST['cc-amount']
    if customer is None:
        customer = stripe.Customer.create(
            source=token,
            description="New Customer"
        )
    try:
        #Save New customer to user
        if request.user.is_authenticated():
            if not BillingInfo.objects.filter(user=request.user).exists():
                billing = BillingInfo(user=request.user, stripe=customer.id)
                billing.save()
            else:
                request.user.billinginfo.stripe = customer.id
                request.user.billinginfo.save()
        order = processProducts(request, request.session.get('cart', []))
        try:
            processEmail(order)
        except:
            #Set a method to handle errors
            pass
        charge_amount = int(order.total * 100)
        charge = stripe.Charge.create(
            amount=charge_amount, # amount in cents, again
            currency="usd",
            customer=customer.id,
        )
        args['cc_result'] = "Your card was successfully charged $%.2f" % (charge_amount / 100)
    except stripe.error.CardError, e:
        # The card has been declined
        args['cc_result'] = e
    return args

def processEmail(order):
    d = Context({ 'order': order })
    msg_plain = render_to_string('email/notification.txt', d)
    msg_html = render_to_string('email/notification.html', d)
    send_mail(
        'We have a new order!',
        msg_plain,
        'admin@code.coop',
        ["bjageman@code.coop"],
        html_message=msg_html,
    )
    if order.shipping_email is not None:
        msg_plain = render_to_string('email/order_confirm.txt', d)
        msg_html = render_to_string('email/order_confirm.html', d)
        send_mail(
            'Thanks for the order!',
            msg_plain,
            'admin@code.coop',
            [order.shipping_email],
            html_message=msg_html,
        )

def processProducts(request, cart):
    products = setProducts(request.session.get('cart', []))
    user = request.user
    if request.user.is_authenticated():
        address_text = str(user.shipping.name + " " + user.shipping.address + " " + user.shipping.city + ", " + user.shipping.state + " " + user.shipping.postal_code)
        order = Order(user=user, shipping_address=address_text, shipping_email=user.shipping.email)
    else:
        address_text = str(request.session['shipping']['name'] + " " + request.session['shipping']['address'] + " " + request.session['shipping']['city'] + ", " + request.session['shipping']['state'] + " " + request.session['shipping']['postal_code'])
        order = Order(shipping_address=address_text, shipping_email=request.session['shipping']['email'])
    order.save()
    #Edit the quantity of the products in stock
    for item in products:
        quantity = item['session']['quantity']
        #Verify that the product does not go beyond the stock limit
        if not (item['product'].preorder or item['product'].status() == "unlimited"):
            quantity = item['product'].set_limit(quantity)
        if item['accessory'] is not None:
            if not (item['accessory'].preorder or item['accessory'].status() == "unlimited"):
                quantity = item['accessory'].set_limit(quantity)
            if item['accessory'].stock is not None:
                item['accessory'].stock = item['accessory'].stock - quantity
            item['accessory'].save()
        if item['product'].stock is not None:
            item['product'].stock = item['product'].stock - quantity
        addProductToOrder(order, item['product'], item['accessory'], item['option'], quantity)
        item['product'].save()
    order.save()
    if 'coupon' in request.session:
        coupon = Coupon.objects.get(code=request.session['coupon']['code'])
        #This will be need to be changed once product associations are implemented
        order.coupon = coupon.code + ": " + str(coupon.amount_off)
        order.discount = coupon.amount_off
        coupon.times_redeemed = coupon.times_redeemed + 1
        del request.session['coupon']
        coupon.save()
    order.save()
    del request.session['cart']
    return order

def totalProductOrder(product, accessory):
    if accessory is not None:
        return product.total() + accessory.total()
    else:
        return product.total()

def addProductToOrder(order, product, accessory, option, quantity):
    productText = str(product.pk) + ") " + product.title
    accessory_title = ""
    option_title = ""
    if accessory is not None:
        accessory_title = accessory.title
        productText = productText + " w/ " + accessory.title
    if option is not None:
        option_title = option.title
        productText = productText + "(" + option.title +")"
    productOrder = ProductOrder(
            order=order,
            product_log=productText,
            accessories=accessory_title,
            options=option_title,
            quantity=quantity,
            price=totalProductOrder(product, accessory)
        )
    productOrder.save()

def charge(request):
    args = {}
    if request.user.is_authenticated():
        args["shipping"] = get_object_or_404(Shipping, user=request.user)
    elif 'shipping' in request.session:
        args["shipping"] = request.session['shipping']
    else:
        return HttpResponseRedirect( reverse('store:checkout') )
    args = processStripe(request)
    return render_to_response('store/order-complete.html', RequestContext(request,args))
<|endoftext|>


<commit_before>import mechanize
import re
import sys
import urllib
import HTMLParser
h = HTMLParser.HTMLParser()
voter_state = "LA"
source = "State"

#cleaning FIPS for use

fips_all = open("C:\Users\pkoms\Documents\TurboVote\Scraping\\fips.txt").read()

fips_data_re = re.compile(".+?LA.+?\n")
fips_number_re = re.compile("\d+")
fips_names_re = re.compile("(.+?)\t")

fips_data = fips_data_re.findall(fips_all)

fips_numbers = []
fips_names = []

for fip in fips_data:
	fips_numbers.append(fips_number_re.findall(fip)[0])
	fips_names.append(fips_names_re.findall(fip)[0])

result = [("authority_name", "first_name", "last_name", "parish_name", "fips",
    "street", "city", "address_state", "zip_code",
    "po_street", "po_city", "po_state", "po_zip_code",
    "reg_authority_name", "reg_first", "reg_last",
    "reg_street", "reg_city", "reg_state", "reg_zip_code",
    "reg_po_street", "reg_po_city", "reg_po_state", "reg_po_zip_code",
    "reg_phone", "reg_fax", "reg_email", "reg_website", "reg_hours",
    "phone", "fax", "email", "website", "hours", "voter_state", "source", "review")]

file_path = "C:\Users\pkoms\Documents\TurboVote\Scraping\louisiana-clerks.html"

# url = "https://voterportal.sos.la.gov/registration/registrarframed.aspx"

# br = mechanize.Browser()
# br.set_handle_robots(False) # ignore robots
# content = br.open(url).read()
# output = open(file_path,"w")
# output.write(content)
# output.close()

data = open(file_path).read()

print data

county_data_re = re.compile("div class=.regList.+?
([^>]+?) Parish")

county_data = county_data_re.findall(data)

for county in county_data:
	print "++++++++++++++++++++++++++++++++"
	# print county
	print "--------------------------------"

	county_name_all = county_name_re.findall(county)
	if county_name_all:
		county_name = county_name_all[0]
	else:
		i = 1
		while not county_name:
			county_name_all = county_name_re.findall(county_data[county_data.index(county)-i])
			if county_name_all:
				county_name = county_name_all[0]
			i = i + 1

	print county_name<commit_msg>Fixing louisiana.py<commit_after>import mechanize
import re
import sys
import urllib
import HTMLParser
import os
h = HTMLParser.HTMLParser()
voter_state = "LA"
source = "State"

cdir = os.path.dirname(os.path.abspath(__file__))

#cleaning FIPS for use

fips_all = open(os.path.join(cdir, "fips.txt")).read()

fips_data_re = re.compile(".+?LA.+?\n")
fips_number_re = re.compile("\d+")
fips_names_re = re.compile("(.+?)\t")

fips_data = fips_data_re.findall(fips_all)

fips_numbers = []
fips_names = []

for fip in fips_data:
	fips_numbers.append(fips_number_re.findall(fip)[0])
	fips_names.append(fips_names_re.findall(fip)[0])

result = [("authority_name", "first_name", "last_name", "parish_name", "fips",
    "street", "city", "address_state", "zip_code",
    "po_street", "po_city", "po_state", "po_zip_code",
    "reg_authority_name", "reg_first", "reg_last",
    "reg_street", "reg_city", "reg_state", "reg_zip_code",
    "reg_po_street", "reg_po_city", "reg_po_state", "reg_po_zip_code",
    "reg_phone", "reg_fax", "reg_email", "reg_website", "reg_hours",
    "phone", "fax", "email", "website", "hours", "voter_state", "source", "review")]

file_path = os.path.join(cdir, "louisiana-clerks.html")

if not os.path.isfile(file_path):
	url = "https://voterportal.sos.la.gov/registration/registrarframed.aspx"

	br = mechanize.Browser()
	br.set_handle_robots(False) # ignore robots
	content = br.open(url).read()
	output = open(file_path,"w")
	output.write(content)
	output.close()

data = open(file_path).read()

print data

county_data_re = re.compile("div class=.regList.+?
([^>]+?) Parish")

county_data = county_data_re.findall(data)

for county in county_data:
	print "++++++++++++++++++++++++++++++++"
	# print county
	print "--------------------------------"

	county_name_all = county_name_re.findall(county)
	if county_name_all:
		county_name = county_name_all[0]
	else:
		i = 1
		while not county_name:
			county_name_all = county_name_re.findall(county_data[county_data.index(county)-i])
			if county_name_all:
				county_name = county_name_all[0]
			i = i + 1

	print county_name<|endoftext|>


<commit_before>#!/usr/bin/env python3

import random

from operator import itemgetter
from collections import defaultdict

from lsh.model import LSHItem
from lsh.readers import LSHReader
from lsh.writers import LSHWriter

"""

KNNQuery needs to run in two modes:
    INDEX   items are given to the object for storage
    QUERY   given a LSHItem id, return the k-nearest neighbours in the index, with counts populated

"""

class KNNQuery(object):
    def __init__(self, perm_num, perm_length, sig_length, window_size, n_hashes, prefix_length):
        self.generate_perms(perm_num, perm_length, sig_length)
        self.window_size = window_size
        self.items = []
        self.n_hashes = n_hashes
        self.prefix_length = prefix_length

    def find_neighbours(self, query_item, k):
        candidates = defaultdict(int)
        for perm in self.perms:
            for i in range(self.n_hashes): # TODO: make this global, or parsed into query object etc
                buckets = defaultdict(list)
                query_prefix = ""
                for item in self.items:
                    #rep = ''
                    #for bit_index in perm:
                    #    rep += str(int(item.signature[bit_index]))
                    h = item.signature.hashes[i]
                    h.lrotate(perm)
                    prefix = h.get_prefix(self.prefix_length)
                    buckets[prefix].append(item)
                    if item.id == query_item.id:
                        query_prefix = prefix
                self.add_candidates(buckets, candidates, query_item, query_prefix)
        sorted_candidates = sorted(candidates.items(), key=itemgetter(1), reverse=True)
        return sorted_candidates[:k]

    def add_candidates(self, buckets, candidates, query_item, query_prefix):
        ncandidates = self.window_size * 2
        nadded = 0
        for item in buckets[query_prefix]:
            if query_item.id != item.id:
                candidates[item] += 1
                nadded += 1
        prefix_index = 0
        while nadded <= ncandidates and prefix_index < self.prefix_length:
            prefix = query_prefix ^ (1 << prefix_index)
            for item in buckets[prefix]:
                if query_item.id != item.id:
                    candidates[item] += 1
                    nadded += 1
            prefix_index += 1

    def add_items_to_index(self, filelike):
        r = LSHReader()
        for item in r.process_file(filelike):
            self.add_item_to_index(item)

    def add_item_to_index(self, item):
        self.items.append(item)

    def generate_perms(self, perm_num, perm_length, sig_length):
        self.perm_length = perm_length
        self.perms = []
        ints = list(range(sig_length))
        random.shuffle(ints)
        total_rot = 0
        for r in ints:
            if not total_rot + r % sig_length:
                continue
            total_rot += r
            self.perms.append(r)
            if len(self.perms) == perm_length:
                break

if __name__ == '__main__':
    TESTFILE = './schwa-py-lsh/test/test_data_1000.txt'
    q = KNNQuery(perm_num=10, perm_length=5, sig_length=100, window_size=10)
    q.add_items_to_index(open(TESTFILE, 'r'))
    for item in q.items:
        print(str(item))
        print('\n'.join(str(n) for n in q.find_neighbours(item, 10)))
        print()
<commit_msg> * fixing a cython reference issue that caused hash to be right-shifted to oblivion<commit_after>#!/usr/bin/env python3

import random

from copy import deepcopy
from operator import itemgetter
from collections import defaultdict

from lsh.model import LSHItem
from lsh.readers import LSHReader
from lsh.writers import LSHWriter

"""

KNNQuery needs to run in two modes:
    INDEX   items are given to the object for storage
    QUERY   given a LSHItem id, return the k-nearest neighbours in the index, with counts populated

"""

class KNNQuery(object):
    def __init__(self, perm_num, perm_length, sig_length, window_size, n_hashes, prefix_length):
        self.generate_perms(perm_num, perm_length, sig_length)
        self.window_size = window_size
        self.items = []
        self.sig_length = sig_length
        self.n_hashes = n_hashes
        self.prefix_length = prefix_length

    def find_neighbours(self, query_item, k):
        candidates = defaultdict(int)
        for perm in self.perms:
            for i in range(self.n_hashes): # TODO: make this global, or parsed into query object etc
                buckets = defaultdict(list)
                query_prefix = ""
                for item in self.items:
                    #rep = ''
                    #for bit_index in perm:
                    #    rep += str(int(item.signature[bit_index]))
                    h = item.signature.hashes[i]
                    h.lrotate(perm)
                    prefix = deepcopy(h.get_data())
                    prefix = prefix >> (self.sig_length - self.prefix_length)
                    buckets[prefix].append(item)
                    if item.id == query_item.id:
                        query_prefix = prefix
                self.add_candidates(buckets, candidates, query_item, query_prefix)
        sorted_candidates = sorted(candidates.items(), key=itemgetter(1), reverse=True)
        return sorted_candidates[:k]

    def add_candidates(self, buckets, candidates, query_item, query_prefix):
        ncandidates = self.window_size * 2
        nadded = 0
        for item in buckets[query_prefix]:
            if query_item.id != item.id:
                candidates[item] += 1
                nadded += 1
        prefix_index = 0
        while nadded <= ncandidates and prefix_index < self.prefix_length:
            prefix = query_prefix ^ (1 << prefix_index)
            for item in buckets[prefix]:
                if query_item.id != item.id:
                    candidates[item] += 1
                    nadded += 1
            prefix_index += 1

    def add_items_to_index(self, filelike):
        r = LSHReader()
        for item in r.process_file(filelike):
            self.add_item_to_index(item)

    def add_item_to_index(self, item):
        self.items.append(item)

    def generate_perms(self, perm_num, perm_length, sig_length):
        self.perm_length = perm_length
        self.perms = []
        ints = list(range(sig_length))
        random.shuffle(ints)
        total_rot = 0
        for r in ints:
            if not total_rot + r % sig_length:
                continue
            total_rot += r
            self.perms.append(r)
            if len(self.perms) == perm_length:
                break

if __name__ == '__main__':
    TESTFILE = './schwa-py-lsh/test/test_data_1000.txt'
    q = KNNQuery(perm_num=10, perm_length=5, sig_length=100, window_size=10)
    q.add_items_to_index(open(TESTFILE, 'r'))
    for item in q.items:
        print(str(item))
        print('\n'.join(str(n) for n in q.find_neighbours(item, 10)))
        print()
<|endoftext|>


<commit_before># Copyright 2015-2016 HyperBit developers

import appdirs
import asyncio
import os
import sqlite3

from hyperbit import config, crypto, inventory, message, network, objscanner, objtypes, wallet, worker, packet

class Core(object):
    def __init__(self):
        user_config_dir = appdirs.user_config_dir('hyperbit', '')
        os.makedirs(user_config_dir, 0o700, exist_ok=True)
        self._db = sqlite3.connect(os.path.join(user_config_dir, 'hyperbit.sqlite3'))
        self._db.execute('pragma synchronous = off')
        self._db.execute('pragma locking_mode = exclusive')

        self._db.execute('create table if not exists config (id unique, value)')
        self.inv = inventory.Inventory(self._db)
        self.peers = network.PeerManager(self, self._db, self.inv)
        self.inv.on_add_object.append(self.peers.send_inv)
        self.wal = wallet.Wallet(self._db)
        self.list = message.ThreadList(self._db)
        self.scanner = objscanner.Scanner(self._db, self.inv, self.wal)
        self.worker = worker.Worker(self._db)
        self.worker.on_object_done.append(self.inv.add_object)
        self.wal.on_add_identity.append(self.scan_identity)
        self.inv.on_add_object.append(self.scan_object)
        self.scanner.on_scan_item.append(self.do_scan)

    @asyncio.coroutine
    def _save(self):
        while True:
            self._db.commit()
            yield from asyncio.sleep(1)

    def get_config(self, key, default=None):
        return self._db.execute('select coalesce(min(value), ?) from config where id = ?', (default, key)).fetchone()[0]

    def set_config(self, key, value):
        self._db.execute('insert or replace into config (id, value) values (?, ?)', (key, value))

    @asyncio.coroutine
    def run(self):
        asyncio.get_event_loop().create_task(self._save())
        yield from self.peers.run()

    def scan_object(self, object):
        self.scanner.scan(object.hash, None)
        for identity in self.wal.identities:
            self.scanner.scan(object.hash, identity)

    def scan_identity(self, identity):
        for hash in self.inv.get_hashes():
            self.scanner.scan(hash, identity)

    def do_scan_msg_1(self, object, identity):
        try:
            decrypted = identity.decrypt(object.payload)
            msg = objtypes.MsgData.from_bytes(decrypted)
        except:
            pass
        else:
            if msg.encoding in [objtypes.Encoding.trivial, objtypes.Encoding.simple]:
                simple = objtypes.SimpleMessage.from_bytes(msg.message)
                channel = wallet.Address(4, config.NETWORK_STREAM, msg.ripe).to_bytes()
                creator = wallet.Address(4, config.NETWORK_STREAM, crypto.to_ripe(msg.verkey, msg.enckey)).to_bytes()
                reply = simple.subject[0:4] == 'Re: '
                if reply:
                    simple.subject = simple.subject[4:]

                for t in self.list.threads:
                    if t.channel == channel and t.subject == simple.subject:
                        thread = t
                        if not reply:
                            thread.creator = creator
                        break
                else:
                    if not reply:
                        thread = self.list.new_thread(channel, creator, simple.subject)
                    else:
                        thread = self.list.new_thread(channel, b'', simple.subject)
                if len(thread.longest) < len(simple.body):
                    thread.longest = simple.body
                if not reply:
                    for c in thread.comments:
                        if c.parent_text == '' and c.text == simple.body:
                            comment = c
                            comment.creator = creator
                            break
                    else:
                        comment = thread.new_comment('', creator, simple.body)
                else:
                    bodies = reversed(simple.body.split('\n'+54*'-'+'\n'))
                    parent_text = ''
                    for i, body in enumerate(bodies):
                        for c in thread.comments:
                            if c.text == body:
                                comment = c
                                comment.parent_text = parent_text
                                break
                        else:
                            comment = thread.new_comment(parent_text, b'', body)
                        parent_text = body
                    comment.creator = creator

    def do_scan(self, object, identity):
        if identity is None:
            return
        if object is None:
            return
        if object.type == objtypes.Type.msg and object.version == 1:
            self.do_scan_msg_1(object, identity)

    def send_message(self, src, dst, encoding, message):
        object = packet.Object(
            nonce=0,
            expires=int(time.time() + 4*24*60*60 + crypto.randint(-60*60, 60*60)),
            type=objtypes.Type.msg,
            version=1,
            stream=config.NETWORK_STREAM,
            payload=b'',
        )
        msg = objtypes.MsgData(
            addrver=4, stream=config.NETWORK_STREAM, behavior=0,
            verkey=src.profile.verkey, enckey=src.profile.enckey,
            trials=config.NETWORK_TRIALS, extra=config.NETWORK_EXTRA,
            ripe=dst.address.ripe,
            encoding=message.encoding, message=message.to_bytes(),
            ack=b'', signature=b''
        )
        msg.sign(src.sigkey, object)
        object.payload = crypto.encrypt(dst.enckey, msg.to_bytes())

        self.worker.add_object(object, config.NETWORK_TRIALS, config.NETWORK_EXTRA, int(time.time()))

<commit_msg>Fix error<commit_after># Copyright 2015-2016 HyperBit developers

import appdirs
import asyncio
import os
import sqlite3
import time

from hyperbit import config, crypto, inventory, message, network, objscanner, objtypes, wallet, worker, packet

class Core(object):
    def __init__(self):
        user_config_dir = appdirs.user_config_dir('hyperbit', '')
        os.makedirs(user_config_dir, 0o700, exist_ok=True)
        self._db = sqlite3.connect(os.path.join(user_config_dir, 'hyperbit.sqlite3'))
        self._db.execute('pragma synchronous = off')
        self._db.execute('pragma locking_mode = exclusive')

        self._db.execute('create table if not exists config (id unique, value)')
        self.inv = inventory.Inventory(self._db)
        self.peers = network.PeerManager(self, self._db, self.inv)
        self.inv.on_add_object.append(self.peers.send_inv)
        self.wal = wallet.Wallet(self._db)
        self.list = message.ThreadList(self._db)
        self.scanner = objscanner.Scanner(self._db, self.inv, self.wal)
        self.worker = worker.Worker(self._db)
        self.worker.on_object_done.append(self.inv.add_object)
        self.wal.on_add_identity.append(self.scan_identity)
        self.inv.on_add_object.append(self.scan_object)
        self.scanner.on_scan_item.append(self.do_scan)

    @asyncio.coroutine
    def _save(self):
        while True:
            self._db.commit()
            yield from asyncio.sleep(1)

    def get_config(self, key, default=None):
        return self._db.execute('select coalesce(min(value), ?) from config where id = ?', (default, key)).fetchone()[0]

    def set_config(self, key, value):
        self._db.execute('insert or replace into config (id, value) values (?, ?)', (key, value))

    @asyncio.coroutine
    def run(self):
        asyncio.get_event_loop().create_task(self._save())
        yield from self.peers.run()

    def scan_object(self, object):
        self.scanner.scan(object.hash, None)
        for identity in self.wal.identities:
            self.scanner.scan(object.hash, identity)

    def scan_identity(self, identity):
        for hash in self.inv.get_hashes():
            self.scanner.scan(hash, identity)

    def do_scan_msg_1(self, object, identity):
        try:
            decrypted = identity.decrypt(object.payload)
            msg = objtypes.MsgData.from_bytes(decrypted)
        except:
            pass
        else:
            if msg.encoding in [objtypes.Encoding.trivial, objtypes.Encoding.simple]:
                simple = objtypes.SimpleMessage.from_bytes(msg.message)
                channel = wallet.Address(4, config.NETWORK_STREAM, msg.ripe).to_bytes()
                creator = wallet.Address(4, config.NETWORK_STREAM, crypto.to_ripe(msg.verkey, msg.enckey)).to_bytes()
                reply = simple.subject[0:4] == 'Re: '
                if reply:
                    simple.subject = simple.subject[4:]

                for t in self.list.threads:
                    if t.channel == channel and t.subject == simple.subject:
                        thread = t
                        if not reply:
                            thread.creator = creator
                        break
                else:
                    if not reply:
                        thread = self.list.new_thread(channel, creator, simple.subject)
                    else:
                        thread = self.list.new_thread(channel, b'', simple.subject)
                if len(thread.longest) < len(simple.body):
                    thread.longest = simple.body
                if not reply:
                    for c in thread.comments:
                        if c.parent_text == '' and c.text == simple.body:
                            comment = c
                            comment.creator = creator
                            break
                    else:
                        comment = thread.new_comment('', creator, simple.body)
                else:
                    bodies = reversed(simple.body.split('\n'+54*'-'+'\n'))
                    parent_text = ''
                    for i, body in enumerate(bodies):
                        for c in thread.comments:
                            if c.text == body:
                                comment = c
                                comment.parent_text = parent_text
                                break
                        else:
                            comment = thread.new_comment(parent_text, b'', body)
                        parent_text = body
                    comment.creator = creator

    def do_scan(self, object, identity):
        if identity is None:
            return
        if object is None:
            return
        if object.type == objtypes.Type.msg and object.version == 1:
            self.do_scan_msg_1(object, identity)

    def send_message(self, src, dst, message):
        object = packet.Object(
            nonce=0,
            expires=int(time.time() + 4*24*60*60 + crypto.randint(-60*60, 60*60)),
            type=objtypes.Type.msg,
            version=1,
            stream=config.NETWORK_STREAM,
            payload=b'',
        )
        msg = objtypes.MsgData(
            addrver=4, stream=config.NETWORK_STREAM, behavior=0,
            verkey=src.profile.verkey, enckey=src.profile.enckey,
            trials=config.NETWORK_TRIALS, extra=config.NETWORK_EXTRA,
            ripe=dst.address.ripe,
            encoding=message.encoding, message=message.to_bytes(),
            ack=b'', signature=b''
        )
        msg.sign(src.sigkey, object)
        object.payload = crypto.encrypt(dst.enckey, msg.to_bytes())

        self.worker.add_object(object, config.NETWORK_TRIALS, config.NETWORK_EXTRA, int(time.time()))

<|endoftext|>