<commit_before># DEMO:
#   rye run rfs.py *.py -- -port=9988 -root=/etc  &
#   rye run fu.py *.py  -- --rfs=127.0.0.1:9988 find /There/localhost:9988/java

from go import os
from go import path/filepath

from . import A
from . import flag
from . import afs
from . import rfs

J = filepath.Join

def Test1(args):
  fs = rfs.RfsClient(RFS.X, rfs.KEYNAME, rfs.KEY)

  vec = fs.ListDir('vga')
  say 'ListDir', vec
  assert len(vec) > 2
  assert len(vec[1]) == 4

  buf, eof = fs.ReadAt('motd', 8, 0)
  say 'ReadAt', buf, eof
  assert len(buf) == 8
  assert not eof

  buf, eof = fs.ReadAt('motd', 12, 8)
  say 'ReadAt', buf, eof
  assert len(buf) == 12
  assert not eof

  print "OK"

BS = 512 # Block Size

def cat1(path, out):
  fd = afs.Open(path)
  #say 'YYY cat1', fd
  with defer fd.Close():
    while True:
      #say 'YYY cat1 fd.Read', fd
      buf, eof = fd.Read(BS)
      #say 'YYY cat1 fd.Read', buf, eof
      if buf:
        ### aphid.WrapWrite(out, buf)  # Writes fully.
        n = len(buf)
        while n > 0:
          c = out.Write(buf)
          buf = buf[c:]
          n -= c
          #say 'YYYYYYYY', n, c
      #say 'YYY called aphid.WrapWrite', len(buf), buf
      if eof:
        break
    pass
  pass

def Cat(args):
  out = None
  if CREATE.X:
    must not APPEND.X
    out = afs.Create(CREATE.X)
  if APPEND.X:
    must not CREATE.X
    out = afs.Append(APPEND.X)
  if not out:
    out = afs.Append('/Std/out')
  with defer out.Close():
    for arg in args:
      cat1(arg, out)
  pass

def FindFiles(args):
  for a in args:
    for name, isDir, mtime, sz in FindFiles1(a):
      print '%s %v %d %d' % (name, isDir, mtime, sz)

def FindFiles1(top):
  #say '<<<', top
  try:
    d = afs.Open(top)
  except as ex:
    A.Err('fu find: Cannot Open (%s): %q\n' % (ex, top))
    A.SetExitStatus(2)
    return
  with defer d.Close():
    try:
      for name, isDir, mtime, sz in d.List():
        #say 'FFFFFF', name, isDir, mtime, sz
        jname = J(top, name)
        #say 'FFFFFFJ', jname, isDir, mtime, sz
        if isDir:
          for x in FindFiles1(jname):
            yield x
        else:
          yield jname, isDir, mtime, sz
    except as ex:
      A.Err('fu find: Cannot List (%s): %q' % (ex, top))
      A.SetExitStatus(2)
      return

def Sync(args):
  if len(args) != 2:
    A.Err('fu sync: To perform a sync, we need a destination and a path.')
    A.SetExitStatus(2)
    return

  Sync1(args[0], args[1])

# rye run fu.py *.py -- sync /Here/static_test_src /Here/static_test_dst
def Sync1(source, dest):
  src_files = FindFiles1(source)
  dst_files = FindFiles1(dest)

  dest_dict = {}

  # I want to convert my list to a dict of tuples
  # Use the relative path as the key
  for path, isDir, mtime, size in dst_files:
    rel_dest_path = path[len(dest):]
    dest_dict[rel_dest_path] = (isDir, mtime, size)

  for path, isDir, mtime, size in src_files:
    rel_src_path = path[len(source):]
    if dest_dict.get(rel_src_path) == None:
      say rel_src_path, 'need to create this'

Ensemble = { 'test1': Test1, 'cat': Cat, 'find': FindFiles, 'sync': Sync }

CREATE = flag.String('create', '', 'Create output file with "cat" command.')
APPEND = flag.String('append', '', 'Append output file with "cat" command.')
RFS = flag.String('rfs', 'localhost:9876', 'Location of rfs server')

def main(args):
  args = flag.Munch(args)

  if not len(args):
    say 'Expected an Emsemble argument:', [k for k in Ensemble]
    os.Exit(13)

  cmd, args = args[0], args[1:]
  f = Ensemble.get(cmd)
  if not f:
    A.Err("Available commands: %v" % sorted(Ensemble.keys()))
    A.Fatal("No such command: %q" % cmd)
    os.Exit(11)

  f(args)
  A.Exit(0)
<commit_msg>Can continue lines after open groupings.<commit_after># DEMO:
#   rye run rfs.py *.py -- -port=9988 -root=/etc  &
#   rye run fu.py *.py  -- --rfs=127.0.0.1:9988 find /There/localhost:9988/java

from go import os
from go import path/filepath

from . import A
from . import flag
from . import afs
from . import rfs

J = filepath.Join

def Test1(args):
  fs = rfs.RfsClient(RFS.X, rfs.KEYNAME, rfs.KEY)

  vec = fs.ListDir('vga')
  say 'ListDir', vec
  assert len(vec) > 2
  assert len(vec[1]) == 4

  buf, eof = fs.ReadAt('motd', 8, 0)
  say 'ReadAt', buf, eof
  assert len(buf) == 8
  assert not eof

  buf, eof = fs.ReadAt('motd', 12, 8)
  say 'ReadAt', buf, eof
  assert len(buf) == 12
  assert not eof

  print "OK"

BS = 512 # Block Size

def cat1(path, out):
  fd = afs.Open(path)
  #say 'YYY cat1', fd
  with defer fd.Close():
    while True:
      #say 'YYY cat1 fd.Read', fd
      buf, eof = fd.Read(BS)
      #say 'YYY cat1 fd.Read', buf, eof
      if buf:
        ### aphid.WrapWrite(out, buf)  # Writes fully.
        n = len(buf)
        while n > 0:
          c = out.Write(buf)
          buf = buf[c:]
          n -= c
          #say 'YYYYYYYY', n, c
      #say 'YYY called aphid.WrapWrite', len(buf), buf
      if eof:
        break
    pass
  pass

def Cat(args):
  out = None
  if CREATE.X:
    must not APPEND.X
    out = afs.Create(CREATE.X)
  if APPEND.X:
    must not CREATE.X
    out = afs.Append(APPEND.X)
  if not out:
    out = afs.Append('/Std/out')
  with defer out.Close():
    for arg in args:
      cat1(arg, out)
  pass

def FindFiles(args):
  for a in args:
    for name, isDir, mtime, sz in FindFiles1(a):
      print '%s %v %d %d' % (name, isDir, mtime, sz)

def FindFiles1(top):
  #say '<<<', top
  try:
    d = afs.Open(top)
  except as ex:
    A.Err('fu find: Cannot Open (%s): %q\n' % (ex, top))
    A.SetExitStatus(2)
    return
  with defer d.Close():
    try:
      for name, isDir, mtime, sz in d.List():
        #say 'FFFFFF', name, isDir, mtime, sz
        jname = J(top, name)
        #say 'FFFFFFJ', jname, isDir, mtime, sz
        if isDir:
          for x in FindFiles1(jname):
            yield x
        else:
          yield jname, isDir, mtime, sz
    except as ex:
      A.Err('fu find: Cannot List (%s): %q' % (ex, top))
      A.SetExitStatus(2)
      return

def Sync(args):
  if len(args) != 2:
    A.Err('fu sync: To perform a sync, we need a destination and a path.')
    A.SetExitStatus(2)
    return

  Sync1(args[0], args[1])

# rye run fu.py *.py -- sync /Here/static_test_src /Here/static_test_dst
def Sync1(source, dest):
  src_files = FindFiles1(source)
  dst_files = FindFiles1(dest)

  dest_dict = {}

  # I want to convert my list to a dict of tuples
  # Use the relative path as the key
  for path, isDir, mtime, size in dst_files:
    rel_dest_path = path[len(dest):]
    dest_dict[rel_dest_path] = (isDir, mtime, size)

  for path, isDir, mtime, size in src_files:
    rel_src_path = path[len(source):]
    if dest_dict.get(rel_src_path) == None:
      say rel_src_path, 'need to create this'

Ensemble = {
    'test1': Test1,
    'cat': Cat,
    'find': FindFiles,
    'sync': Sync
}

CREATE = flag.String('create', '',
    'Create output file with "cat" command.')
APPEND = flag.String('append', '',
    'Append output file with "cat" command.')
RFS = flag.String('rfs', 'localhost:9876',
    'Location of rfs server')

def main(args):
  args = flag.Munch(args)

  if not len(args):
    say 'Expected an Emsemble argument:', [k for k in Ensemble]
    os.Exit(13)

  cmd, args = args[0], args[1:]
  f = Ensemble.get(cmd)
  if not f:
    A.Err("Available commands: %v" % sorted(Ensemble.keys()))
    A.Fatal("No such command: %q" % cmd)
    os.Exit(11)

  f(args)
  A.Exit(0)
<|endoftext|>


<commit_before>from __future__ import print_function
import sys
import os
import pprint
from getopt import getopt
import json
import pickle
import traceback
import marshal
import time

import yara
from yara import scan

"""
Scanner command line interface.

[mjdorma@gmail.com]
"""

__help__ = """
NAME yara-ctypes Scan files or processes against YARA rules

SYNOPSIS
    yara-ctypes [OPTIONS]... [FILE(s)|PID(s)]...

DESCRIPTION

Scanner control:
    --mode=default mode is stdin or file
        mode options [file|chunk|proc|stdin]
        file  -queues file paths found in FILE(s)
        chunk -queues chunks of data read from files found in FILE(s)
        proc  -enqueues PID(s) for process scanning
        stdin -read from the stdin stream.

    --execute-pool=%s
        size of the execute pool used in the Scanner instance

    --execute-type=[thread|process]
        choose to scan using a process pool or a thread pool

    --chunk-size=%s
        size of data in bytes to read and enqueue from data read from a stream

    --chunk-overlap=%s
        percentage from 0 - 99 %% of data to be reprocesses on block boundaries

    --chunk-read-max=
        set the maximum chunks to read from a stream

    --readhead-limit=%s
        maximum number of bytes to read ahead when reading data from a stream

File control:
    --recurse-dirs
        recurse directories specified in FILE(s)

    --exclude-filesize-lt=
        exclude files which are less then this value

    --exclude-filesize-gt=
        exclude files which are greater then then value

    --path-end-include=[end1,end2,end3, ...]
        path endings inclusion filter (comma separate list)

    --path-end-exclude=[end1,end2,end3, ...]
        path endings exclusion filter (comma separate list)

    --path-contains-exclude=[str1,str2,str3, ...]
        exclude path's that contain str (comma separate list)

    --path-contains-include=[str1,str2,str3, ...]
        include path's that contain str (comma separate list)

Rules control:
  rule:
    --fast
        fast matching mode

    -d <identifier>=<value>
        define external variable.

  namespace:
    --list
        list available YARA namespaces

    -w, --whitelist=
        whitelist of comma separated YARA namespaces to include in scan

    -b, --blacklist=
        blacklist of comma separated YARA namespaces to exclude from scan

    --root=(env[YARA_RULES] or <pkg>/yara/rules/)
        set the YARA_RULES path (path to the root of the rules directory)

  yarafile:
    -r, --rule=
        Use the rule file specified by this input argument and ignore the
        YARA namespaces

Scan output control:
    --fmt=dict
        output format [dict|pprint|json|pickle|marshal]
    -o
        outfile path -> redirect stdout results to outfile

    -t  [tag1,tag2,tag3, ...]
        print matches that contain specific tags and filter out the rest

    -i  [ident1,ident2,ident3, ...]
        print matches that contain specific identifiers and filter out the rest

    --simple
        print the filepath and the rule which was hit

    --error-log=
        write scan errors out to error-log

Other:
    --version
        print the version of yara-ctypes

    -h, --help
        print this help
""" % (scan.DEFAULT_EXECUTE_POOL,
       scan.DEFAULT_STREAM_CHUNK_SIZE,
       scan.DEFAULT_STREAM_CHUNK_OVERLAP,
       scan.DEFAULT_STREAM_READAHEAD_LIMIT)

def match_filter(tags_filter, idents_filter, res):
    if tags_filter is not None:
        new_res = {}
        for ns, matches in res.iteritems():
            for match in matches:
                if tags_filter.intersection(match['tags']):
                    mlist = new_res.get(ns, [])
                    mlist.append(match)
                    new_res[ns] = mlist
        res = new_res
    if idents_filter is not None:
        new_res = {}
        for ns, matches in res.iteritems():
            for match in matches:
                idents = [s['identifier'] for s in match['strings']]
                if idents_filter.intersection(idents):
                    mlist = new_res.get(ns, [])
                    mlist.append(match)
                    new_res[ns] = mlist
        res = new_res
    return res

STATUS = "\
scanned: %-8s matches: %-4s errors: %-4s scan queue: %-8s res queue: %-7s"
def clear_status(scanner):
    status = STATUS % (scanner.scanned, scanner.matches,
                            scanner.errors, scanner.sq_size, scanner.rq_size)
    sys.stderr.write("\b" * len(status))

def print_status(scanner):
    status = STATUS % (scanner.scanned, scanner.matches,
                            scanner.errors, scanner.sq_size, scanner.rq_size)
    sys.stderr.write("\b" * len(status) + status)

def run_scanner(scanner,
                out_stream=None,
                out_stream_fmt=str,
                err_stream=None,
                output_simple=False,
                tags_filter=None,
                idents_filter=None
            ):
    if out_stream is None:
        out_stream = sys.stdout
    stime = time.time()
    try:
        for arg, res in scanner:
            print_status(scanner)
            if not res:
                continue

            if type(res) is dict:
                if out_stream == sys.stdout:
                    clear_status(scanner)
                res = match_filter(tags_filter, idents_filter, res)
                if not res:
                    continue
                if output_simple:
                    out_stream.write("%s:" % arg)
                    for namespace, hits in res.items():
                        for hit in hits:
                            out_stream.write(" %s.%s" % (namespace,
                                                         hit['rule']))
                    out_stream.write("\n")
                else:
                    try:
                        formatted_res = out_stream_fmt(res)
                    except Exception as exc:
                        exc.error = "Failed to render res\n%s\n%s" % (\
                                        res, traceback.format_exc())
                        raise
                    print("<scan arg='%s'>" % arg, file=out_stream)
                    print(formatted_res, file=out_stream)
                    print("</scan>", file=out_stream)
                    out_stream.flush()
            else:
                if err_stream is not None:
                    print("<scan arg='%s'>%s</scan>" % (arg, res),
                                                        file=err_stream)
    finally:
        print_status(scanner)
        sys.stderr.write("\nwaiting scanner ... ")
        scanner.quit.set()
        scanner.join()
        print("scan completed after %0.02fs." % (time.time()-stime),
                file=sys.stderr)

def build_rules(fast_match=False,
                   rules_rootpath=yara.YARA_RULES_ROOT,
                   whitelist=[],
                   blacklist=[],
                   rule_filepath=None,
                   externals={}):
    try:
        if rule_filepath is None:
            return yara.load_rules(rules_rootpath=rules_rootpath,
                                   blacklist=blacklist,
                                   whitelist=whitelist,
                                   includes=True,
                                   externals=externals,
                                   fast_match=fast_match)
        else:
            return yara.compile(filepath=rule_filepath,
                                   includes=True,
                                   externals=externals,
                                   fast_match=fast_match)
    except yara.YaraSyntaxError as err:
        print("Failed to load rules with the following error(s):\n%s" % \
                "\n".join([e for _,_,e in err.errors]), file=sys.stderr)

        if rule_filepath is None:
            blacklist = set()
            for f, _, _ in err.errors:
                f = os.path.splitext(f[len(rules_rootpath)+1:])[0]
                blacklist.add(f.replace(os.path.sep, '.'))
            print("\nYou could blacklist the erroneous rules using:",
                    file=sys.stderr)
            print(" --blacklist=%s" % ",".join(blacklist), file=sys.stderr)
    return None

def main(args):
    try:
        opts, args = getopt(args, 'hw:b:t:o:i:d:r:', ['help', 'version',
            'list',
            'mode=',
            'execute-pool=', 'execute-mode=',
            'rule=',
            'root=',
            'whitelist=',
            'blacklist=',
            'fast',
            'fmt=',
            'simple',
            'error-log=',
            'recurse-dirs',
            'exclude-filesize-lt=', 'exclude-filesize-gt=',
            'path-end-exclude=', 'path-end-include=',
            'path-contains-exclude=', 'path-contains-include=',
            'chunk-size=', 'chunk-overlap=', 'readahead-limit=',
            'chunk-read-max=',
        ])
    except Exception as exc:
        print("Getopt error: %s" % (exc), file=sys.stderr)
        return -1

    rules_kwargs = {}
    scanner_kwargs = {}
    scanner_kwargs['args'] = args

    if not args and sys.stdin.isatty():
        ScannerClass = scan.StdinScanner
    else:
        ScannerClass = scan.PathScanner

    list_rules = False
    run_scan_kwargs = {}

    for opt, arg in opts:
        if opt in ['-h', '--help']:
            print(__help__)
            return 0
        elif opt in ['--version']:
            print("yara-ctypes version %s" % yara.__version__)
            return 0
        elif opt in ['--list']:
            list_rules = True

        elif opt in ['-o']:
            run_scan_kwargs['out_stream'] = open(arg, 'wb')
        elif opt in ['--error-log']:
            run_scan_kwargs['err_stream'] = open(arg, 'wb')
        elif opt in ['--simple']:
            run_scan_kwargs['output_simple'] = True
        elif opt in ['-t']:
            run_scan_kwargs['tags_filter'] = set(arg.split(','))
        elif opt in ['-i']:
            run_scan_kwargs['idents_filter'] = arg
        elif opt in ['--fmt']:
            if arg == 'pickle':
                out_stream_fmt = pickle.dumps
            elif arg == 'json':
                out_stream_fmt = lambda a: json.dumps(a, ensure_ascii=False,
                                                check_circular=False, indent=4)
            elif arg == 'pprint':
                out_stream_fmt = pprint.pformat
            elif arg == 'marshal':
                out_stream_fmt = marshal.dumps
            elif arg == 'dict':
                out_stream_fmt = str
            else:
                print("unknown output format %s" % arg, file=sys.stderr)
                return -1
            run_scan_kwargs['out_stream_fmt'] = out_stream_fmt

        elif opt in ['-r', '--rule']:
            if not os.path.exists(arg):
                print("rule path '%s' does not exist" % arg, file=sys.stderr)
                return -1
            rules_kwargs['rule_filepath'] = arg
        elif opt in ['-w', '--whitelist']:
            rules_kwargs['whitelist'] = arg.split(',')
        elif opt in ['b', '--blacklist']:
            rules_kwargs['blacklist'] = arg.split(',')
        elif opt in ['--root']:
            if not os.path.exists(arg):
                print("root path '%s' does not exist" % arg, file=sys.stderr)
                return -1
            rules_kwargs['rules_rootpath'] = os.path.abspath(arg)
        elif opt in ['-d']:
            try:
                if 'externals' not in rules_kwargs:
                    rules_kwargs['externals'] = {}
                rules_kwargs['externals'].update(eval("dict(%s)" % arg))
            except SyntaxError:
                print("external '%s' syntax error" % arg, file=sys.stderr)
                return -1
        elif opt in ['--fast']:
            rules_kwargs['fast_match'] = True
        elif opt in ['--exclude-filesize-lt']:
            try:
                scanner_kwargs['filesize_lt'] = int(arg)
            except ValueError:
                print("param '%s' was not an int" % (arg), file=sys.stderr)
                return -1
        elif opt in ['--exclude-filesize-gt']:
            try:
                scanner_kwargs['filesize_gt'] = int(arg)
            except ValueError:
                print("param '%s' was not an int" % (arg), file=sys.stderr)
                return -1
        elif opt in ['--path-end-include']:
            scanner_kwargs['path_end_include'] = arg.split(',')
        elif opt in ['--path-end-exclude']:
            scanner_kwargs['path_end_exclude'] = arg.split(',')
        elif opt in ['--path-contains-include']:
            scanner_kwargs['path_contains_include'] = arg.split(',')
        elif opt in ['--path-contains-exclude']:
            scanner_kwargs['path_contains_exclude'] = arg.split(',')
        elif opt in ['--recurse-dirs']:
            scanner_kwargs['recurse_dirs'] = True
        elif opt in ['--readahead-limit']:
            if ScannerClass != scan.StdinScanner:
                ScannerClass = scan.FileChunkScanner
            try:
                scanner_kwargs['stream_read_ahead_limit'] = int(arg)
            except ValueError:
                print("param '%s' was not an int" % (arg), file=sys.stderr)
                return -1
        elif opt in ['--chunk-read-max']:
            if ScannerClass != scan.StdinScanner:
                ScannerClass = scan.FileChunkScanner
            try:
                chunk_read_max = int(arg)
            except ValueError:
                print("param '%s' was not an int" % (arg), file=sys.stderr)
                return -1
            if chunk_read_max < 1:
                print("chunk-read-max value must be greater than 0",
                        file=sys.stderr)
                return -1
            scanner_kwargs['stream_chunk_read_max'] = chunk_read_max
        elif opt in ['--chunk-overlap']:
            if ScannerClass != scan.StdinScanner:
                ScannerClass = scan.FileChunkScanner
            try:
                chunk_overlap = int(arg)
            except ValueError:
                print("param '%s' was not an int" % (arg), file=sys.stderr)
                return -1
            if chunk_overlap < 0 or chunk_overlap > 99:
                print("chunk-overlap value must be between 0 - 99",
                        file=sys.stderr)
                return -1
            scanner_kwargs['stream_chunk_overlap'] = chunk_overlap
        elif opt in ['--chunk-size']:
            if ScannerClass != scan.StdinScanner:
                ScannerClass = scan.FileChunkScanner
            try:
                scanner_kwargs['stream_chunk_size'] = int(arg)
            except ValueError:
                print("param '%s' was not an int" % (arg), file=sys.stderr)
                return -1
        elif opt in ['--execute-mode']:
            if arg == 'thread':
                scanner_kwargs['execute_mode'] = scan.EXECUTE_THREAD
            elif arg == 'process':
                scanner_kwargs['execute_mode'] = scan.EXECUTE_PROCESS
            else:
                print("unknown execute mode '%s'" % arg)
                return -1
        elif opt in ['--execute-pool']:
            try:
                scanner_kwargs['execute_pool'] = int(arg)
            except ValueError:
                print("param '%s' was not an int" % (arg), file=sys.stderr)
                return -1
            if scanner_kwargs['execute_pool'] < 1:
                print("--execute-pool value can not be lower than 1",
                        file=sys.stderr)
                return -1
        elif opt in ['--mode']:
            if arg == 'file':
                ScannerClass = scan.PathScanner
            elif arg == 'chunk':
                ScannerClass = scan.FileChunkScanner
            elif arg == 'proc':
                ScannerClass = scan.PidScanner
            elif arg == 'stdin':
                if not sys.stdin.isatty():
                    print("No stdin available for stdin mode to function")
                ScannerClass = scan.StdinScanner
            else:
                print("unknown mode %s" % arg, file=sys.stderr)
                return -1

    #build rules object
    rules = build_rules(**rules_kwargs)
    if rules is None:
        return -1

    #print out configured rules
    if list_rules == True:
        print(rules)
        return 0

    #build scanner object
    try:
        print("Building %s" % ScannerClass.__name__, file=sys.stderr)
        for k, v in scanner_kwargs.items():
            if k == 'args':
                continue
            print("   %s=%s" % (k, repr(v)), file=sys.stderr)
        scanner = ScannerClass(rules=rules, **scanner_kwargs)
    except Exception as exc:
        print("Failed to build Scanner with error: %s" % \
                    traceback.format_exc(), file=sys.stderr)
        return -1

    #run scanner
    try:
        print("Run scanner", file=sys.stderr)
        for k, v in run_scan_kwargs.items():
            print("   %s=%s" % (k, repr(v)), file=sys.stderr)
        run_scanner(scanner, **run_scan_kwargs)
    except KeyboardInterrupt:
        pass

    return 0

entry = lambda : sys.exit(main(sys.argv[1:]))
if __name__ == "__main__":
    entry()
<commit_msg>make sure stdout and stderr are flushed after each scan and error....  change execute-mode to execute-type<commit_after>from __future__ import print_function
import sys
import os
import pprint
from getopt import getopt
import json
import pickle
import traceback
import marshal
import time

import yara
from yara import scan

"""
Scanner command line interface.

[mjdorma@gmail.com]
"""

__help__ = """
NAME yara-ctypes Scan files or processes against YARA rules

SYNOPSIS
    yara-ctypes [OPTIONS]... [FILE(s)|PID(s)]...

DESCRIPTION

Scanner control:
    --mode=default mode is stdin or file
        mode options [file|chunk|proc|stdin]
        file  -queues file paths found in FILE(s)
        chunk -queues chunks of data read from files found in FILE(s)
        proc  -enqueues PID(s) for process scanning
        stdin -read from the stdin stream.

    --execute-pool=%s
        size of the execute pool used in the Scanner instance

    --execute-type=[thread|process]
        choose to scan using a process pool or a thread pool

    --chunk-size=%s
        size of data in bytes to read and enqueue from data read from a stream

    --chunk-overlap=%s
        percentage from 0 - 99 %% of data to be reprocesses on block boundaries

    --chunk-read-max=
        set the maximum chunks to read from a stream

    --readhead-limit=%s
        maximum number of bytes to read ahead when reading data from a stream

File control:
    --recurse-dirs
        recurse directories specified in FILE(s)

    --exclude-filesize-lt=
        exclude files which are less then this value

    --exclude-filesize-gt=
        exclude files which are greater then then value

    --path-end-include=[end1,end2,end3, ...]
        path endings inclusion filter (comma separate list)

    --path-end-exclude=[end1,end2,end3, ...]
        path endings exclusion filter (comma separate list)

    --path-contains-exclude=[str1,str2,str3, ...]
        exclude path's that contain str (comma separate list)

    --path-contains-include=[str1,str2,str3, ...]
        include path's that contain str (comma separate list)

Rules control:
  rule:
    --fast
        fast matching mode

    -d <identifier>=<value>
        define external variable.

  namespace:
    --list
        list available YARA namespaces

    -w, --whitelist=
        whitelist of comma separated YARA namespaces to include in scan

    -b, --blacklist=
        blacklist of comma separated YARA namespaces to exclude from scan

    --root=(env[YARA_RULES] or <pkg>/yara/rules/)
        set the YARA_RULES path (path to the root of the rules directory)

  yarafile:
    -r, --rule=
        Use the rule file specified by this input argument and ignore the
        YARA namespaces

Scan output control:
    --fmt=dict
        output format [dict|pprint|json|pickle|marshal]
    -o
        outfile path -> redirect stdout results to outfile

    -t  [tag1,tag2,tag3, ...]
        print matches that contain specific tags and filter out the rest

    -i  [ident1,ident2,ident3, ...]
        print matches that contain specific identifiers and filter out the rest

    --simple
        print the filepath and the rule which was hit

    --error-log=
        write scan errors out to error-log

Other:
    --version
        print the version of yara-ctypes

    -h, --help
        print this help
""" % (scan.DEFAULT_EXECUTE_POOL,
       scan.DEFAULT_STREAM_CHUNK_SIZE,
       scan.DEFAULT_STREAM_CHUNK_OVERLAP,
       scan.DEFAULT_STREAM_READAHEAD_LIMIT)

def match_filter(tags_filter, idents_filter, res):
    if tags_filter is not None:
        new_res = {}
        for ns, matches in res.iteritems():
            for match in matches:
                if tags_filter.intersection(match['tags']):
                    mlist = new_res.get(ns, [])
                    mlist.append(match)
                    new_res[ns] = mlist
        res = new_res
    if idents_filter is not None:
        new_res = {}
        for ns, matches in res.iteritems():
            for match in matches:
                idents = [s['identifier'] for s in match['strings']]
                if idents_filter.intersection(idents):
                    mlist = new_res.get(ns, [])
                    mlist.append(match)
                    new_res[ns] = mlist
        res = new_res
    return res

STATUS = "\
scanned: %-8s matches: %-4s errors: %-4s scan queue: %-8s res queue: %-7s"
def clear_status(scanner):
    status = STATUS % (scanner.scanned, scanner.matches,
                            scanner.errors, scanner.sq_size, scanner.rq_size)
    sys.stderr.write("\b" * len(status))

def print_status(scanner):
    status = STATUS % (scanner.scanned, scanner.matches,
                            scanner.errors, scanner.sq_size, scanner.rq_size)
    sys.stderr.write("\b" * len(status) + status)

def run_scanner(scanner,
                out_stream=None,
                out_stream_fmt=str,
                err_stream=None,
                output_simple=False,
                tags_filter=None,
                idents_filter=None
            ):
    if out_stream is None:
        out_stream = sys.stdout
    stime = time.time()
    try:
        for arg, res in scanner:
            print_status(scanner)
            if not res:
                continue

            if type(res) is dict:
                if out_stream == sys.stdout:
                    clear_status(scanner)
                res = match_filter(tags_filter, idents_filter, res)
                if not res:
                    continue
                if output_simple:
                    out_stream.write("%s:" % arg)
                    for namespace, hits in res.items():
                        for hit in hits:
                            out_stream.write(" %s.%s" % (namespace,
                                                         hit['rule']))
                    out_stream.write("\n")
                else:
                    try:
                        formatted_res = out_stream_fmt(res)
                    except Exception as exc:
                        exc.error = "Failed to render res\n%s\n%s" % (\
                                        res, traceback.format_exc())
                        raise
                    print("<scan arg='%s'>" % arg, file=out_stream)
                    print(formatted_res, file=out_stream)
                    print("</scan>", file=out_stream)
                out_stream.flush()
            else:
                if err_stream is not None:
                    print("<scan arg='%s'>%s</scan>" % (arg, res),
                                                        file=err_stream)
                    err_stream.flush()
    finally:
        print_status(scanner)
        sys.stderr.write("\nwaiting scanner ... ")
        scanner.quit.set()
        scanner.join()
        print("scan completed after %0.02fs." % (time.time()-stime),
                file=sys.stderr)

def build_rules(fast_match=False,
                   rules_rootpath=yara.YARA_RULES_ROOT,
                   whitelist=[],
                   blacklist=[],
                   rule_filepath=None,
                   externals={}):
    try:
        if rule_filepath is None:
            return yara.load_rules(rules_rootpath=rules_rootpath,
                                   blacklist=blacklist,
                                   whitelist=whitelist,
                                   includes=True,
                                   externals=externals,
                                   fast_match=fast_match)
        else:
            return yara.compile(filepath=rule_filepath,
                                   includes=True,
                                   externals=externals,
                                   fast_match=fast_match)
    except yara.YaraSyntaxError as err:
        print("Failed to load rules with the following error(s):\n%s" % \
                "\n".join([e for _,_,e in err.errors]), file=sys.stderr)

        if rule_filepath is None:
            blacklist = set()
            for f, _, _ in err.errors:
                f = os.path.splitext(f[len(rules_rootpath)+1:])[0]
                blacklist.add(f.replace(os.path.sep, '.'))
            print("\nYou could blacklist the erroneous rules using:",
                    file=sys.stderr)
            print(" --blacklist=%s" % ",".join(blacklist), file=sys.stderr)
    return None

def main(args):
    try:
        opts, args = getopt(args, 'hw:b:t:o:i:d:r:', ['help', 'version',
            'list',
            'mode=',
            'execute-pool=', 'execute-type=',
            'rule=',
            'root=',
            'whitelist=',
            'blacklist=',
            'fast',
            'fmt=',
            'simple',
            'error-log=',
            'recurse-dirs',
            'exclude-filesize-lt=', 'exclude-filesize-gt=',
            'path-end-exclude=', 'path-end-include=',
            'path-contains-exclude=', 'path-contains-include=',
            'chunk-size=', 'chunk-overlap=', 'readahead-limit=',
            'chunk-read-max=',
        ])
    except Exception as exc:
        print("Getopt error: %s" % (exc), file=sys.stderr)
        return -1

    rules_kwargs = {}
    scanner_kwargs = {}
    scanner_kwargs['args'] = args

    if not args and sys.stdin.isatty():
        ScannerClass = scan.StdinScanner
    else:
        ScannerClass = scan.PathScanner

    list_rules = False
    run_scan_kwargs = {}

    for opt, arg in opts:
        if opt in ['-h', '--help']:
            print(__help__)
            return 0
        elif opt in ['--version']:
            print("yara-ctypes version %s" % yara.__version__)
            return 0
        elif opt in ['--list']:
            list_rules = True

        elif opt in ['-o']:
            run_scan_kwargs['out_stream'] = open(arg, 'wb')
        elif opt in ['--error-log']:
            run_scan_kwargs['err_stream'] = open(arg, 'wb')
        elif opt in ['--simple']:
            run_scan_kwargs['output_simple'] = True
        elif opt in ['-t']:
            run_scan_kwargs['tags_filter'] = set(arg.split(','))
        elif opt in ['-i']:
            run_scan_kwargs['idents_filter'] = arg
        elif opt in ['--fmt']:
            if arg == 'pickle':
                out_stream_fmt = pickle.dumps
            elif arg == 'json':
                out_stream_fmt = lambda a: json.dumps(a, ensure_ascii=False,
                                                check_circular=False, indent=4)
            elif arg == 'pprint':
                out_stream_fmt = pprint.pformat
            elif arg == 'marshal':
                out_stream_fmt = marshal.dumps
            elif arg == 'dict':
                out_stream_fmt = str
            else:
                print("unknown output format %s" % arg, file=sys.stderr)
                return -1
            run_scan_kwargs['out_stream_fmt'] = out_stream_fmt

        elif opt in ['-r', '--rule']:
            if not os.path.exists(arg):
                print("rule path '%s' does not exist" % arg, file=sys.stderr)
                return -1
            rules_kwargs['rule_filepath'] = arg
        elif opt in ['-w', '--whitelist']:
            rules_kwargs['whitelist'] = arg.split(',')
        elif opt in ['b', '--blacklist']:
            rules_kwargs['blacklist'] = arg.split(',')
        elif opt in ['--root']:
            if not os.path.exists(arg):
                print("root path '%s' does not exist" % arg, file=sys.stderr)
                return -1
            rules_kwargs['rules_rootpath'] = os.path.abspath(arg)
        elif opt in ['-d']:
            try:
                if 'externals' not in rules_kwargs:
                    rules_kwargs['externals'] = {}
                rules_kwargs['externals'].update(eval("dict(%s)" % arg))
            except SyntaxError:
                print("external '%s' syntax error" % arg, file=sys.stderr)
                return -1
        elif opt in ['--fast']:
            rules_kwargs['fast_match'] = True
        elif opt in ['--exclude-filesize-lt']:
            try:
                scanner_kwargs['filesize_lt'] = int(arg)
            except ValueError:
                print("param '%s' was not an int" % (arg), file=sys.stderr)
                return -1
        elif opt in ['--exclude-filesize-gt']:
            try:
                scanner_kwargs['filesize_gt'] = int(arg)
            except ValueError:
                print("param '%s' was not an int" % (arg), file=sys.stderr)
                return -1
        elif opt in ['--path-end-include']:
            scanner_kwargs['path_end_include'] = arg.split(',')
        elif opt in ['--path-end-exclude']:
            scanner_kwargs['path_end_exclude'] = arg.split(',')
        elif opt in ['--path-contains-include']:
            scanner_kwargs['path_contains_include'] = arg.split(',')
        elif opt in ['--path-contains-exclude']:
            scanner_kwargs['path_contains_exclude'] = arg.split(',')
        elif opt in ['--recurse-dirs']:
            scanner_kwargs['recurse_dirs'] = True
        elif opt in ['--readahead-limit']:
            if ScannerClass != scan.StdinScanner:
                ScannerClass = scan.FileChunkScanner
            try:
                scanner_kwargs['stream_read_ahead_limit'] = int(arg)
            except ValueError:
                print("param '%s' was not an int" % (arg), file=sys.stderr)
                return -1
        elif opt in ['--chunk-read-max']:
            if ScannerClass != scan.StdinScanner:
                ScannerClass = scan.FileChunkScanner
            try:
                chunk_read_max = int(arg)
            except ValueError:
                print("param '%s' was not an int" % (arg), file=sys.stderr)
                return -1
            if chunk_read_max < 1:
                print("chunk-read-max value must be greater than 0",
                        file=sys.stderr)
                return -1
            scanner_kwargs['stream_chunk_read_max'] = chunk_read_max
        elif opt in ['--chunk-overlap']:
            if ScannerClass != scan.StdinScanner:
                ScannerClass = scan.FileChunkScanner
            try:
                chunk_overlap = int(arg)
            except ValueError:
                print("param '%s' was not an int" % (arg), file=sys.stderr)
                return -1
            if chunk_overlap < 0 or chunk_overlap > 99:
                print("chunk-overlap value must be between 0 - 99",
                        file=sys.stderr)
                return -1
            scanner_kwargs['stream_chunk_overlap'] = chunk_overlap
        elif opt in ['--chunk-size']:
            if ScannerClass != scan.StdinScanner:
                ScannerClass = scan.FileChunkScanner
            try:
                scanner_kwargs['stream_chunk_size'] = int(arg)
            except ValueError:
                print("param '%s' was not an int" % (arg), file=sys.stderr)
                return -1
        elif opt in ['--execute-type']:
            if arg == 'thread':
                scanner_kwargs['execute_type'] = scan.EXECUTE_THREAD
            elif arg == 'process':
                scanner_kwargs['execute_type'] = scan.EXECUTE_PROCESS
            else:
                print("unknown execute type '%s'" % arg)
                return -1
        elif opt in ['--execute-pool']:
            try:
                scanner_kwargs['execute_pool'] = int(arg)
            except ValueError:
                print("param '%s' was not an int" % (arg), file=sys.stderr)
                return -1
            if scanner_kwargs['execute_pool'] < 1:
                print("--execute-pool value can not be lower than 1",
                        file=sys.stderr)
                return -1
        elif opt in ['--mode']:
            if arg == 'file':
                ScannerClass = scan.PathScanner
            elif arg == 'chunk':
                ScannerClass = scan.FileChunkScanner
            elif arg == 'proc':
                ScannerClass = scan.PidScanner
            elif arg == 'stdin':
                if not sys.stdin.isatty():
                    print("No stdin available for stdin mode to function")
                ScannerClass = scan.StdinScanner
            else:
                print("unknown mode %s" % arg, file=sys.stderr)
                return -1

    #build rules object
    rules = build_rules(**rules_kwargs)
    if rules is None:
        return -1

    #print out configured rules
    if list_rules == True:
        print(rules)
        return 0

    #build scanner object
    try:
        print("Building %s" % ScannerClass.__name__, file=sys.stderr)
        for k, v in scanner_kwargs.items():
            if k == 'args':
                continue
            print("   %s=%s" % (k, repr(v)), file=sys.stderr)
        scanner = ScannerClass(rules=rules, **scanner_kwargs)
    except Exception as exc:
        print("Failed to build Scanner with error: %s" % \
                    traceback.format_exc(), file=sys.stderr)
        return -1

    #run scanner
    try:
        print("Run scanner", file=sys.stderr)
        for k, v in run_scan_kwargs.items():
            print("   %s=%s" % (k, repr(v)), file=sys.stderr)
        run_scanner(scanner, **run_scan_kwargs)
    except KeyboardInterrupt:
        pass

    return 0

entry = lambda : sys.exit(main(sys.argv[1:]))
if __name__ == "__main__":
    entry()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# A Simple Application for handling yay0 files

from tkinter import *
import tkinter.filedialog as filedialog
from PIL import Image, ImageTk
import  os, logging
import frontend

class Application(Frame):
    def __init__(self, master=None):
        Frame.__init__(self, master)
        self.master = master
        self.appName = "Application Utility"
        self.infilename = ''
        self.tmpfilename = "game_over.256x32.png"
        self.log = logging.getLogger(self.appName)
        self.prepare()
    def prepare(self):
        self.master.title(self.appName)
        self.pack(fill=BOTH, expand=1)
        quitButton = Button(self, text="Decode",
            command=self.image_decode)
        quitButton.place(x=0, y=0)
        self.textLabel = Label(self, text="Ready!")
        self.textLabel.place(x=0, y=50)
        self.imgLabel = Label(self, image=None)
        self.imgLabel.place(x=0, y=100)

        mainMenu = Menu(self.master)
        self.master.config(menu=mainMenu)

        fileMenu = Menu(mainMenu)
        fileMenu.add_command(label="Open...", command=self.open_file)
        fileMenu.add_command(label="Save PNG", command = self.save_file)
        fileMenu.add_command(label="Exit", command=self.app_exit)
        mainMenu.add_cascade(label="File", menu=fileMenu)

        viewMenu = Menu(mainMenu)
        viewMenu.add_command(label="Show Image", command=self.show_image)
        viewMenu.add_command(label="Show Text", command=self.show_text)
        mainMenu.add_cascade(label="View", menu=viewMenu)
    def image_decode(self):
        self.tmpfilename = ''
        ext = frontend.parseFilename(self.infilename)[-1]
        # If file extension ends in y it's pixel data only with a
        # separate palette file containing palette information.
        if 'y' == ext[-1]:
            self.tmpfilename = frontend.processMultiFileImage(self.infilename)
        else:
            self.tmpfilename = frontend.processSingleFileImage(self.infilename)
        if(self.tmpfilename == ''):
            self.textLabel['text'] = "Image Decode Failed!"
        else:
            self.textLabel['text'] = "Image Decoded Successfully!"
    def open_file(self):
        self.infilename = filedialog.askopenfilename()
        self.log.info(("Selected: %s" % self.infilename))
        self.textLabel['text'] = "Opened:"+self.infilename
    def save_file(self):
        self.outfilename = frontend.getPngFileName(self.infilename)
        os.rename(self.tmpfilename, self.outfilename)
        self.textLabel['text'] = "Saved:" + self.outfilename
    def app_exit(self):
        exit()
    def show_image(self):
        load = Image.open(self.tmpfilename) # TODO: Modify this !!!
        render = ImageTk.PhotoImage(load)
        # self.imgLabel = Label(self, image=render)
        self.imgLabel.configure(image=render)
        self.imgLabel.image = render
        # self.imgLabel.configure(image=render)
        # self.imgLabel.place(x=0, y=100)
    def show_text(self):
        self.textLabel['text'] = "Image Displayed!"

logging.basicConfig(level=logging.INFO)
root = Tk()
root.geometry("400x400")
app = Application(root)
root.mainloop()
<commit_msg>Update app.py to open GUI only if invoked as main<commit_after>#!/usr/bin/env python3

# A Simple Application for handling yay0 files

from tkinter import *
import tkinter.filedialog as filedialog
from PIL import Image, ImageTk
import  os, logging
import frontend

class Application(Frame):
    def __init__(self, master=None):
        Frame.__init__(self, master)
        self.master = master
        self.appName = "Application Utility"
        self.infilename = ''
        self.tmpfilename = "game_over.256x32.png"
        self.log = logging.getLogger(self.appName)
        self.prepare()
    def prepare(self):
        self.master.title(self.appName)
        self.pack(fill=BOTH, expand=1)
        quitButton = Button(self, text="Decode",
            command=self.image_decode)
        quitButton.place(x=0, y=0)
        self.textLabel = Label(self, text="Ready!")
        self.textLabel.place(x=0, y=50)
        self.imgLabel = Label(self, image=None)
        self.imgLabel.place(x=0, y=100)

        mainMenu = Menu(self.master)
        self.master.config(menu=mainMenu)

        fileMenu = Menu(mainMenu)
        fileMenu.add_command(label="Open...", command=self.open_file)
        fileMenu.add_command(label="Save PNG", command = self.save_file)
        fileMenu.add_command(label="Exit", command=self.app_exit)
        mainMenu.add_cascade(label="File", menu=fileMenu)

        viewMenu = Menu(mainMenu)
        viewMenu.add_command(label="Show Image", command=self.show_image)
        viewMenu.add_command(label="Show Text", command=self.show_text)
        mainMenu.add_cascade(label="View", menu=viewMenu)
    def image_decode(self):
        self.tmpfilename = ''
        ext = frontend.parseFilename(self.infilename)[-1]
        # If file extension ends in y it's pixel data only with a
        # separate palette file containing palette information.
        if 'y' == ext[-1]:
            self.tmpfilename = frontend.processMultiFileImage(self.infilename)
        else:
            self.tmpfilename = frontend.processSingleFileImage(self.infilename)
        if(self.tmpfilename == ''):
            self.textLabel['text'] = "Image Decode Failed!"
        else:
            self.textLabel['text'] = "Image Decoded Successfully!"
    def open_file(self):
        self.infilename = filedialog.askopenfilename()
        self.log.info(("Selected: %s" % self.infilename))
        self.textLabel['text'] = "Opened:"+self.infilename
    def save_file(self):
        self.outfilename = frontend.getPngFileName(self.infilename)
        os.rename(self.tmpfilename, self.outfilename)
        self.textLabel['text'] = "Saved:" + self.outfilename
    def app_exit(self):
        exit()
    def show_image(self):
        load = Image.open(self.tmpfilename) # TODO: Modify this !!!
        render = ImageTk.PhotoImage(load)
        # self.imgLabel = Label(self, image=render)
        self.imgLabel.configure(image=render)
        self.imgLabel.image = render
        # self.imgLabel.configure(image=render)
        # self.imgLabel.place(x=0, y=100)
    def show_text(self):
        self.textLabel['text'] = "Image Displayed!"

def main():
    logging.basicConfig(level=logging.INFO)
    root = Tk()
    root.geometry("400x400")
    app = Application(root)
    root.mainloop()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from django.db import models
from transmeta import TransMeta
from django.utils.translation import ugettext_lazy as _
from questionnaire import QuestionChoices
import re
from utils import split_numal
from django.utils import simplejson as json
from parsers import parse_checks, ParseException
from django.conf import settings

_numre = re.compile("(\d+)([a-z]+)", re.I)

class Subject(models.Model):
    STATE_CHOICES = [
        ("active", _("Active")),
        ("inactive", _("Inactive")),
        # Can be changed from elsewhere with
        # Subject.STATE_CHOICES[:] = [ ('blah', 'Blah') ]
    ]
    state = models.CharField(max_length=16, default="inactive",
        choices = STATE_CHOICES, verbose_name=_('State'))
    surname = models.CharField(max_length=64, blank=True, null=True,
        verbose_name=_('Surname'))
    givenname = models.CharField(max_length=64, blank=True, null=True,
        verbose_name=_('Given name'))
    email = models.EmailField(null=True, blank=True, verbose_name=_('Email'))
    gender = models.CharField(max_length=8, default="unset", blank=True,
        verbose_name=_('Gender'),
        choices = ( ("unset", _("Unset")),
                    ("male", _("Male")),
                    ("female", _("Female")),
        )
    )
    nextrun = models.DateField(verbose_name=_('Next Run'), blank=True, null=True)
    formtype = models.CharField(max_length=16, default='email',
        verbose_name = _('Form Type'),
        choices = (
            ("email", _("Subject receives emails")),
            ("paperform", _("Subject is sent paper form"),))
    )
    language = models.CharField(max_length=2, default=settings.LANGUAGE_CODE,
        verbose_name = _('Language'), choices = settings.LANGUAGES)

    def __unicode__(self):
        return u'%s, %s (%s)' % (self.surname, self.givenname, self.email)

    def next_runid(self):
        "Return the string form of the runid for the upcoming run"
        return str(self.nextrun.year)

    def last_run(self):
        "Returns the last completed run or None"
        try:
            query = RunInfoHistory.objects.filter(subject=self)
            return query.order_by('-completed')[0]
        except IndexError:
            return None

    def history(self):
        return RunInfoHistory.objects.filter(subject=self).order_by('runid')

    def pending(self):
        return RunInfo.objects.filter(subject=self).order_by('runid')

class Questionnaire(models.Model):
    name = models.CharField(max_length=128)
    redirect_url = models.CharField(max_length=128, help_text="URL to redirect to when Questionnaire is complete. Macros: $SUBJECTID, $RUNID, $LANG", default="/static/complete.html")

    def __unicode__(self):
        return self.name

    def questionsets(self):
        if not hasattr(self, "__qscache"):
            self.__qscache = \
              QuestionSet.objects.filter(questionnaire=self).order_by('sortid')
        return self.__qscache

    class Meta:
        permissions = (
            ("export", "Can export questionnaire answers"),
            ("management", "Management Tools")
        )

class QuestionSet(models.Model):
    __metaclass__ = TransMeta

    "Which questions to display on a question page"
    questionnaire = models.ForeignKey(Questionnaire)
    sortid = models.IntegerField() # used to decide which order to display in
    heading = models.CharField(max_length=64)
    checks = models.CharField(max_length=128, blank=True,
        help_text = """Current options are 'femaleonly' or 'maleonly' and shownif="QuestionNumber,Answer" which takes the same format as <tt>requiredif</tt> for questions.""")
    text = models.TextField(help_text="This is interpreted as Textile: http://hobix.com/textile/quick.html")

    def questions(self):
        if not hasattr(self, "__qcache"):
            self.__qcache = list(Question.objects.filter(questionset=self).order_by('number'))
            self.__qcache.sort()
        return self.__qcache

    def next(self):
        qs = self.questionnaire.questionsets()
        retnext = False
        for q in qs:
            if retnext:
                return q
            if q == self:
                retnext = True
        return None

    def prev(self):
        qs = self.questionnaire.questionsets()
        last = None
        for q in qs:
            if q == self:
                return last
            last = q

    def is_last(self):
        try:
            return self.questionnaire.questionsets()[-1] == self
        except NameError:
            # should only occur if not yet saved
            return True

    def is_first(self):
        try:
            return self.questionnaire.questionsets()[0] == self
        except NameError:
            # should only occur if not yet saved
            return True

    def __unicode__(self):
        return u'%s: %s' % (self.questionnaire.name, self.heading)

    class Meta:
        translate = ('text',)

class RunInfo(models.Model):
    "Store the active/waiting questionnaire runs here"
    subject = models.ForeignKey(Subject)
    random = models.CharField(max_length=32) # probably a randomized md5sum
    runid = models.CharField(max_length=32)
    # questionset should be set to the first QuestionSet initially, and to null on completion
    # ... although the RunInfo entry should be deleted then anyway.
    questionset = models.ForeignKey(QuestionSet, blank=True, null=True) # or straight int?
    emailcount = models.IntegerField(default=0)

    created = models.DateTimeField(auto_now_add=True)
    emailsent = models.DateTimeField(null=True, blank=True)

    lastemailerror = models.CharField(max_length=64, null=True, blank=True)

    state = models.CharField(max_length=16, null=True, blank=True)
    cookies = models.TextField(null=True, blank=True)

    tags = models.TextField(
            blank=True,
            help_text=u"Tags active on this run, separated by commas"
        )

    skipped = models.TextField(
            blank=True,
            help_text=u"A comma sepearted list of questions to skip"
        )

    def save(self):
        self.random = (self.random or '').lower()
        super(RunInfo, self).save()

    def set_cookie(self, key, value):
        "runinfo.set_cookie(key, value). If value is None, delete cookie"
        key = key.lower().strip()
        cookies = self.get_cookiedict()
        if type(value) not in (int, str, unicode, type(None)):
            raise Exception("Can only store cookies of type integer or string")
        if value is None:
            if key in cookies:
                del cookies[key]
        else:
            if type(value) == 'int':
                value=str(value)
            cookies[key] = value
        cstr = json.dumps(cookies)
        self.cookies=cstr
        self.save()
        self.__cookiecache = cookies

    def get_cookie(self, key, default=None):
        if not self.cookies:
            return default
        d = self.get_cookiedict()
        return d.get(key.lower().strip(), default)

    def get_cookiedict(self):
        if not self.cookies:
            return {}
        if not hasattr(self, '__cookiecache'):
            self.__cookiecache = json.loads(self.cookies)
        return self.__cookiecache

    def __unicode__(self):
        return "%s: %s, %s" % (self.runid, self.subject.surname, self.subject.givenname)

    class Meta:
        verbose_name_plural = 'Run Info'

class RunInfoHistory(models.Model):
    subject = models.ForeignKey(Subject)
    runid = models.CharField(max_length=32)
    completed = models.DateField()
    tags = models.TextField(
            blank=True,
            help_text=u"Tags used on this run, separated by commas"
        )
    skipped = models.TextField(
            blank=True,
            help_text=u"A comma sepearted list of questions skipped by this run"
        )
    questionnaire = models.ForeignKey(Questionnaire)

    def __unicode__(self):
        return "%s: %s on %s" % (self.runid, self.subject, self.completed)

    def answers(self):
        "Returns the query for the answers."
        return Answer.objects.filter(subject=self.subject, runid=self.runid)

    class Meta:
        verbose_name_plural = 'Run Info History'

class Question(models.Model):
    __metaclass__ = TransMeta

    questionset = models.ForeignKey(QuestionSet)
    number = models.CharField(max_length=8, help_text=
        "eg. <tt>1</tt>, <tt>2a</tt>, <tt>2b</tt>, <tt>3c</tt>
 "
        "Number is also used for ordering questions.")
    text = models.TextField(blank=True)
    type = models.CharField(u"Type of question", max_length=32,
        choices = QuestionChoices,
        help_text = u"Determines the means of answering the question. " \
        "An open question gives the user a single-line textfield, " \
        "multiple-choice gives the user a number of choices he/she can " \
        "choose from. If a question is multiple-choice, enter the choices " \
        "this user can choose from below'.")
    extra = models.CharField(u"Extra information", max_length=128, blank=True, null=True, help_text=u"Extra information (use  on question type)")
    checks = models.CharField(u"Additional checks", max_length=128, blank=True,
        null=True, help_text="Additional checks to be performed for this "
        "value (space separated)

"
        "For text fields, <tt>required</tt> is a valid check.
"
        "For yes/no choice, <tt>required</tt>, <tt>required-yes</tt>, "
        "and <tt>required-no</tt> are valid.

"
        "If this question is required only if another question's answer is "
        'something specific, use <tt>requiredif="QuestionNumber,Value"</tt> '
        'or <tt>requiredif="QuestionNumber,!Value"</tt> for anything but '
        "a specific value.  "
        "You may also combine tests appearing in <tt>requiredif</tt> "
        "by joining them with the words <tt>and</tt> or <tt>or</tt>, "
        'eg. <tt>requiredif="Q1,A or Q2,B"</tt>')
    footer = models.TextField(u"Footer", help_text="Footer rendered below the question interpreted as textile", blank=True)

    def questionnaire(self):
        return self.questionset.questionnaire

    def getcheckdict(self):
        """getcheckdict returns a dictionary of the values in self.checks"""
        if(hasattr(self, '__checkdict_cached')):
            return self.__checkdict_cached
        try:
            self.__checkdict_cached = d = parse_checks(self.sameas().checks or '')
        except ParseException:
            raise Exception("Error Parsing Checks for Question %s: %s" % (
                self.number, self.sameas().checks))
        return d

    def __unicode__(self):
        return u'{%s} (%s) %s' % (unicode(self.questionset), self.number, self.text)

    def sameas(self):
        if self.type == 'sameas':
            try:
                self.__sameas = res = getattr(self, "__sameas",
                    Question.objects.get(number=self.checks,
                        questionset__questionnaire=self.questionset.questionnaire))
                return res
            except Question.DoesNotExist:
                return Question(type='comment') # replace with something benign
        return self

    def display_number(self):
        "Return either the number alone or the non-number part of the question number indented"
        m = _numre.match(self.number)
        if m:
            sub = m.group(2)
            return "   " + sub
        return self.number

    def choices(self):
        if self.type == 'sameas':
            return self.sameas().choices()
        res = Choice.objects.filter(question=self).order_by('sortid')
        return res

    def is_custom(self):
        return "custom" == self.sameas().type

    def get_type(self):
        "Get the type name, treating sameas and custom specially"
        t = self.sameas().type
        if t == 'custom':
            cd = self.sameas().getcheckdict()
            if 'type' not in cd:
                raise Exception("When using custom types, you must have type=<name> in the additional checks field")
            return cd.get('type')
        return t

    def questioninclude(self):
        return "questionnaire/" + self.get_type() + ".html"

    def __cmp__(a, b):
        anum, astr = split_numal(a.number)
        bnum, bstr = split_numal(b.number)
        cmpnum = cmp(anum, bnum)
        return cmpnum or cmp(astr, bstr)

    class Meta:
        translate = ('text', 'extra', 'footer')

class Choice(models.Model):
    __metaclass__ = TransMeta

    question = models.ForeignKey(Question)
    sortid = models.IntegerField()
    value = models.CharField(u"Short Value", max_length=64)
    text = models.CharField(u"Choice Text", max_length=200)

    def __unicode__(self):
        return u'(%s) %d. %s' % (self.question.number, self.sortid, self.text)

    class Meta:
        translate = ('text',)

class Answer(models.Model):
    subject = models.ForeignKey(Subject, help_text = u'The user who supplied this answer')
    question = models.ForeignKey(Question, help_text = u"The question that this is an answer to")
    runid = models.CharField(u'RunID', help_text = u"The RunID (ie. year)", max_length=32)
    answer = models.TextField()

    def __unicode__(self):
        return "Answer(%s: %s, %s)" % (self.question.number, self.subject.surname, self.subject.givenname)

    def choice_str(self, secondary = False):
        choice_string = ""
        choices = self.question.get_choices()

        for choice in choices:
            for split_answer in self.split_answer():
                if str(split_answer) == choice.value:
                    choice_string += str(choice.text) + " "

    def split_answer(self):
        """
        Decode stored answer value and return as a list of choices.
        Any freeform value will be returned in a list as the last item.

        Calling code should be tolerant of freeform answers outside
        of additional [] if data has been stored in plain text format
        """
        try:
            return json.loads(self.answer)
        except ValueError:
            # this was likely saved as plain text, try to guess what the
            # value(s) were
            if 'multiple' in self.question.type:
                return self.answer.split('; ')
            else:
                return [self.answer]

    def check_answer(self):
        "Confirm that the supplied answer matches what we expect"
        return True<commit_msg>Adds ability to store floats in the cookiedict<commit_after>from django.db import models
from transmeta import TransMeta
from django.utils.translation import ugettext_lazy as _
from questionnaire import QuestionChoices
import re
from utils import split_numal
from django.utils import simplejson as json
from parsers import parse_checks, ParseException
from django.conf import settings

_numre = re.compile("(\d+)([a-z]+)", re.I)

class Subject(models.Model):
    STATE_CHOICES = [
        ("active", _("Active")),
        ("inactive", _("Inactive")),
        # Can be changed from elsewhere with
        # Subject.STATE_CHOICES[:] = [ ('blah', 'Blah') ]
    ]
    state = models.CharField(max_length=16, default="inactive",
        choices = STATE_CHOICES, verbose_name=_('State'))
    surname = models.CharField(max_length=64, blank=True, null=True,
        verbose_name=_('Surname'))
    givenname = models.CharField(max_length=64, blank=True, null=True,
        verbose_name=_('Given name'))
    email = models.EmailField(null=True, blank=True, verbose_name=_('Email'))
    gender = models.CharField(max_length=8, default="unset", blank=True,
        verbose_name=_('Gender'),
        choices = ( ("unset", _("Unset")),
                    ("male", _("Male")),
                    ("female", _("Female")),
        )
    )
    nextrun = models.DateField(verbose_name=_('Next Run'), blank=True, null=True)
    formtype = models.CharField(max_length=16, default='email',
        verbose_name = _('Form Type'),
        choices = (
            ("email", _("Subject receives emails")),
            ("paperform", _("Subject is sent paper form"),))
    )
    language = models.CharField(max_length=2, default=settings.LANGUAGE_CODE,
        verbose_name = _('Language'), choices = settings.LANGUAGES)

    def __unicode__(self):
        return u'%s, %s (%s)' % (self.surname, self.givenname, self.email)

    def next_runid(self):
        "Return the string form of the runid for the upcoming run"
        return str(self.nextrun.year)

    def last_run(self):
        "Returns the last completed run or None"
        try:
            query = RunInfoHistory.objects.filter(subject=self)
            return query.order_by('-completed')[0]
        except IndexError:
            return None

    def history(self):
        return RunInfoHistory.objects.filter(subject=self).order_by('runid')

    def pending(self):
        return RunInfo.objects.filter(subject=self).order_by('runid')

class Questionnaire(models.Model):
    name = models.CharField(max_length=128)
    redirect_url = models.CharField(max_length=128, help_text="URL to redirect to when Questionnaire is complete. Macros: $SUBJECTID, $RUNID, $LANG", default="/static/complete.html")

    def __unicode__(self):
        return self.name

    def questionsets(self):
        if not hasattr(self, "__qscache"):
            self.__qscache = \
              QuestionSet.objects.filter(questionnaire=self).order_by('sortid')
        return self.__qscache

    class Meta:
        permissions = (
            ("export", "Can export questionnaire answers"),
            ("management", "Management Tools")
        )

class QuestionSet(models.Model):
    __metaclass__ = TransMeta

    "Which questions to display on a question page"
    questionnaire = models.ForeignKey(Questionnaire)
    sortid = models.IntegerField() # used to decide which order to display in
    heading = models.CharField(max_length=64)
    checks = models.CharField(max_length=128, blank=True,
        help_text = """Current options are 'femaleonly' or 'maleonly' and shownif="QuestionNumber,Answer" which takes the same format as <tt>requiredif</tt> for questions.""")
    text = models.TextField(help_text="This is interpreted as Textile: http://hobix.com/textile/quick.html")

    def questions(self):
        if not hasattr(self, "__qcache"):
            self.__qcache = list(Question.objects.filter(questionset=self).order_by('number'))
            self.__qcache.sort()
        return self.__qcache

    def next(self):
        qs = self.questionnaire.questionsets()
        retnext = False
        for q in qs:
            if retnext:
                return q
            if q == self:
                retnext = True
        return None

    def prev(self):
        qs = self.questionnaire.questionsets()
        last = None
        for q in qs:
            if q == self:
                return last
            last = q

    def is_last(self):
        try:
            return self.questionnaire.questionsets()[-1] == self
        except NameError:
            # should only occur if not yet saved
            return True

    def is_first(self):
        try:
            return self.questionnaire.questionsets()[0] == self
        except NameError:
            # should only occur if not yet saved
            return True

    def __unicode__(self):
        return u'%s: %s' % (self.questionnaire.name, self.heading)

    class Meta:
        translate = ('text',)

class RunInfo(models.Model):
    "Store the active/waiting questionnaire runs here"
    subject = models.ForeignKey(Subject)
    random = models.CharField(max_length=32) # probably a randomized md5sum
    runid = models.CharField(max_length=32)
    # questionset should be set to the first QuestionSet initially, and to null on completion
    # ... although the RunInfo entry should be deleted then anyway.
    questionset = models.ForeignKey(QuestionSet, blank=True, null=True) # or straight int?
    emailcount = models.IntegerField(default=0)

    created = models.DateTimeField(auto_now_add=True)
    emailsent = models.DateTimeField(null=True, blank=True)

    lastemailerror = models.CharField(max_length=64, null=True, blank=True)

    state = models.CharField(max_length=16, null=True, blank=True)
    cookies = models.TextField(null=True, blank=True)

    tags = models.TextField(
            blank=True,
            help_text=u"Tags active on this run, separated by commas"
        )

    skipped = models.TextField(
            blank=True,
            help_text=u"A comma sepearted list of questions to skip"
        )

    def save(self):
        self.random = (self.random or '').lower()
        super(RunInfo, self).save()

    def set_cookie(self, key, value):
        "runinfo.set_cookie(key, value). If value is None, delete cookie"
        key = key.lower().strip()
        cookies = self.get_cookiedict()
        if type(value) not in (int, float, str, unicode, type(None)):
            raise Exception("Can only store cookies of type integer or string")
        if value is None:
            if key in cookies:
                del cookies[key]
        else:
            if type(value) in ('int', 'float'):
                value=str(value)
            cookies[key] = value
        cstr = json.dumps(cookies)
        self.cookies=cstr
        self.save()
        self.__cookiecache = cookies

    def get_cookie(self, key, default=None):
        if not self.cookies:
            return default
        d = self.get_cookiedict()
        return d.get(key.lower().strip(), default)

    def get_cookiedict(self):
        if not self.cookies:
            return {}
        if not hasattr(self, '__cookiecache'):
            self.__cookiecache = json.loads(self.cookies)
        return self.__cookiecache

    def __unicode__(self):
        return "%s: %s, %s" % (self.runid, self.subject.surname, self.subject.givenname)

    class Meta:
        verbose_name_plural = 'Run Info'

class RunInfoHistory(models.Model):
    subject = models.ForeignKey(Subject)
    runid = models.CharField(max_length=32)
    completed = models.DateField()
    tags = models.TextField(
            blank=True,
            help_text=u"Tags used on this run, separated by commas"
        )
    skipped = models.TextField(
            blank=True,
            help_text=u"A comma sepearted list of questions skipped by this run"
        )
    questionnaire = models.ForeignKey(Questionnaire)

    def __unicode__(self):
        return "%s: %s on %s" % (self.runid, self.subject, self.completed)

    def answers(self):
        "Returns the query for the answers."
        return Answer.objects.filter(subject=self.subject, runid=self.runid)

    class Meta:
        verbose_name_plural = 'Run Info History'

class Question(models.Model):
    __metaclass__ = TransMeta

    questionset = models.ForeignKey(QuestionSet)
    number = models.CharField(max_length=8, help_text=
        "eg. <tt>1</tt>, <tt>2a</tt>, <tt>2b</tt>, <tt>3c</tt>
 "
        "Number is also used for ordering questions.")
    text = models.TextField(blank=True)
    type = models.CharField(u"Type of question", max_length=32,
        choices = QuestionChoices,
        help_text = u"Determines the means of answering the question. " \
        "An open question gives the user a single-line textfield, " \
        "multiple-choice gives the user a number of choices he/she can " \
        "choose from. If a question is multiple-choice, enter the choices " \
        "this user can choose from below'.")
    extra = models.CharField(u"Extra information", max_length=128, blank=True, null=True, help_text=u"Extra information (use  on question type)")
    checks = models.CharField(u"Additional checks", max_length=128, blank=True,
        null=True, help_text="Additional checks to be performed for this "
        "value (space separated)

"
        "For text fields, <tt>required</tt> is a valid check.
"
        "For yes/no choice, <tt>required</tt>, <tt>required-yes</tt>, "
        "and <tt>required-no</tt> are valid.

"
        "If this question is required only if another question's answer is "
        'something specific, use <tt>requiredif="QuestionNumber,Value"</tt> '
        'or <tt>requiredif="QuestionNumber,!Value"</tt> for anything but '
        "a specific value.  "
        "You may also combine tests appearing in <tt>requiredif</tt> "
        "by joining them with the words <tt>and</tt> or <tt>or</tt>, "
        'eg. <tt>requiredif="Q1,A or Q2,B"</tt>')
    footer = models.TextField(u"Footer", help_text="Footer rendered below the question interpreted as textile", blank=True)

    def questionnaire(self):
        return self.questionset.questionnaire

    def getcheckdict(self):
        """getcheckdict returns a dictionary of the values in self.checks"""
        if(hasattr(self, '__checkdict_cached')):
            return self.__checkdict_cached
        try:
            self.__checkdict_cached = d = parse_checks(self.sameas().checks or '')
        except ParseException:
            raise Exception("Error Parsing Checks for Question %s: %s" % (
                self.number, self.sameas().checks))
        return d

    def __unicode__(self):
        return u'{%s} (%s) %s' % (unicode(self.questionset), self.number, self.text)

    def sameas(self):
        if self.type == 'sameas':
            try:
                self.__sameas = res = getattr(self, "__sameas",
                    Question.objects.get(number=self.checks,
                        questionset__questionnaire=self.questionset.questionnaire))
                return res
            except Question.DoesNotExist:
                return Question(type='comment') # replace with something benign
        return self

    def display_number(self):
        "Return either the number alone or the non-number part of the question number indented"
        m = _numre.match(self.number)
        if m:
            sub = m.group(2)
            return "   " + sub
        return self.number

    def choices(self):
        if self.type == 'sameas':
            return self.sameas().choices()
        res = Choice.objects.filter(question=self).order_by('sortid')
        return res

    def is_custom(self):
        return "custom" == self.sameas().type

    def get_type(self):
        "Get the type name, treating sameas and custom specially"
        t = self.sameas().type
        if t == 'custom':
            cd = self.sameas().getcheckdict()
            if 'type' not in cd:
                raise Exception("When using custom types, you must have type=<name> in the additional checks field")
            return cd.get('type')
        return t

    def questioninclude(self):
        return "questionnaire/" + self.get_type() + ".html"

    def __cmp__(a, b):
        anum, astr = split_numal(a.number)
        bnum, bstr = split_numal(b.number)
        cmpnum = cmp(anum, bnum)
        return cmpnum or cmp(astr, bstr)

    class Meta:
        translate = ('text', 'extra', 'footer')

class Choice(models.Model):
    __metaclass__ = TransMeta

    question = models.ForeignKey(Question)
    sortid = models.IntegerField()
    value = models.CharField(u"Short Value", max_length=64)
    text = models.CharField(u"Choice Text", max_length=200)

    def __unicode__(self):
        return u'(%s) %d. %s' % (self.question.number, self.sortid, self.text)

    class Meta:
        translate = ('text',)

class Answer(models.Model):
    subject = models.ForeignKey(Subject, help_text = u'The user who supplied this answer')
    question = models.ForeignKey(Question, help_text = u"The question that this is an answer to")
    runid = models.CharField(u'RunID', help_text = u"The RunID (ie. year)", max_length=32)
    answer = models.TextField()

    def __unicode__(self):
        return "Answer(%s: %s, %s)" % (self.question.number, self.subject.surname, self.subject.givenname)

    def choice_str(self, secondary = False):
        choice_string = ""
        choices = self.question.get_choices()

        for choice in choices:
            for split_answer in self.split_answer():
                if str(split_answer) == choice.value:
                    choice_string += str(choice.text) + " "

    def split_answer(self):
        """
        Decode stored answer value and return as a list of choices.
        Any freeform value will be returned in a list as the last item.

        Calling code should be tolerant of freeform answers outside
        of additional [] if data has been stored in plain text format
        """
        try:
            return json.loads(self.answer)
        except ValueError:
            # this was likely saved as plain text, try to guess what the
            # value(s) were
            if 'multiple' in self.question.type:
                return self.answer.split('; ')
            else:
                return [self.answer]

    def check_answer(self):
        "Confirm that the supplied answer matches what we expect"
        return True<|endoftext|>


<commit_before>import ctypes
import datetime
import errno

library = ctypes.CDLL("libybinlogp.so.1", use_errno=True)

class EventStruct(ctypes.Structure):
	_fields_ = [("timestamp", ctypes.c_uint32),
			("type_code", ctypes.c_uint8),
			("server_id", ctypes.c_uint32),
			("length", ctypes.c_uint32),
			("next_position", ctypes.c_uint32),
			("flags", ctypes.c_uint16),
			("data", ctypes.c_void_p),
			("offset", ctypes.c_uint64)]

	_pack_ = 1

class Event(object):
	def __init__(self, event_type, timestamp):
		self.event_type = event_type
		self.time = datetime.datetime.fromtimestamp(timestamp)
		self.data = None

	def __str__(self):
		return "%s at %s%s" % (self.event_type, self.time, ": " + str(self.data) if self.data is not None else "")

class QueryEventStruct(ctypes.Structure):
	_fields_ = [("thread_id", ctypes.c_uint32),
			("query_time", ctypes.c_uint32),
			("db_name_len", ctypes.c_uint8),
			("error_code", ctypes.c_uint16),
			("status_var_len", ctypes.c_uint16),
			("statement", ctypes.c_char_p),
			("statement_len", ctypes.c_size_t),
			("status_var", ctypes.c_char_p),
			("db_name", ctypes.c_char_p)]

class QueryEvent(object):
	def __init__(self, db_name, statement, query_time):
		self.db_name = db_name
		self.statement = statement
		self.query_time = query_time

	def __str__(self):
		return "Query(db=%s, statement=%s, query_time=%d)" % (self.db_name, self.statement, self.query_time)

class RotateEventStruct(ctypes.Structure):
	_fields_ = [("next_position", ctypes.c_uint64),
			("file_name", ctypes.c_char_p),
			("file_name_len", ctypes.c_size_t)]

class RotateEvent(object):
	def __init__(self, next_position, file_name):
		self.next_position = next_position
		self.file_name = file_name

	def __str__(self):
		return "Rotate(next file=%s, next_position=%d)" % (self.file_name, self.next_position)

class XIDEventStruct(ctypes.Structure):
	_fields_ = [("id", ctypes.c_uint64)]

class XIDEvent(object):
	def __init__(self, xid):
		self.xid = xid

	def __str__(self):
		return "COMMIT xid %d" % self.xid

_init_bp = library.ybp_get_binlog_parser
_init_bp.argtypes = [ctypes.c_int]
_init_bp.restype = ctypes.c_void_p

_get_event = library.ybp_get_event
_get_event.argtypes = []
_get_event.restype = ctypes.POINTER(EventStruct)

_next_event = library.ybp_next_event
_next_event.argtypes = [ctypes.c_void_p, ctypes.POINTER(EventStruct)]
_next_event.restype = ctypes.c_int

_reset_event = library.ybp_reset_event
_reset_event.argtypes = [ctypes.POINTER(EventStruct)]
_reset_event.restype = None

_dispose_event = library.ybp_dispose_event
_dispose_event.argtypes = [ctypes.POINTER(EventStruct)]
_dispose_event.restype = None

_dispose_bp = library.ybp_dispose_binlog_parser
_dispose_bp.argtypes = [ctypes.c_void_p]
_dispose_bp.restype = None

_update_bp = library.ybp_update_bp
_update_bp.argtypes = [ctypes.c_void_p]
_update_bp.restype = None

_event_type = library.ybp_event_type
_event_type.argtypes = [ctypes.POINTER(EventStruct)]
_event_type.restype = ctypes.c_char_p

_event_to_safe_qe = library.ybp_event_to_safe_qe
_event_to_safe_qe.argtypes = [ctypes.POINTER(EventStruct)]
_event_to_safe_qe.restype = ctypes.POINTER(QueryEventStruct)

_dispose_safe_qe = library.ybp_dispose_safe_qe
_dispose_safe_qe.argtype = [ctypes.c_void_p]
_dispose_safe_qe.qestype = None

_event_to_safe_re = library.ybp_event_to_safe_re
_event_to_safe_re.argtypes = [ctypes.POINTER(EventStruct)]
_event_to_safe_re.restype = ctypes.POINTER(RotateEventStruct)

_dispose_safe_re = library.ybp_dispose_safe_re
_dispose_safe_re.argtype = [ctypes.c_void_p]
_dispose_safe_re.restype = None

_event_to_safe_xe = library.ybp_event_to_safe_xe
_event_to_safe_xe.argtypes = [ctypes.POINTER(EventStruct)]
_event_to_safe_xe.restype = ctypes.POINTER(XIDEventStruct)

_dispose_safe_xe = library.ybp_dispose_safe_xe
_dispose_safe_xe.argtype = [ctypes.c_void_p]
_dispose_safe_xe.restype = None

class YBinlogPError(Exception):
	def __init__(self, errno):
		self.errno = errno

	def __repr__(self):
		return "NextEventError(%s)" % errno.errorcode.get(self.errno, "Unknown")

	def __str__(self):
		return repr(self)

class NextEventError(YBinlogPError):
	pass

class YBinlogP(object):
	def __init__(self, filename, always_update=False):
		"""Construct a YBinlogP.

		Arguments:
			filename: A binlog filename
			always_update: if True, stat the file before doing anything
				interesting
		"""
		self.filename = filename
		self.file = open(filename, 'r')
		self.binlog_parser_handle = _init_bp(self.file.fileno())
		self.event_buffer = _get_event();
		self.always_update = always_update

	def _get_next_event(self):
		last = _next_event(self.binlog_parser_handle, self.event_buffer)
		if last < 0:
			raise NextEventError(ctypes.get_errno())
		event = None
		et = _event_type(self.event_buffer)
		base_event = Event(et, self.event_buffer.contents.timestamp)
		if et == "QUERY_EVENT":
			query_event = _event_to_safe_qe(self.event_buffer)
			base_event.data = QueryEvent(query_event.contents.db_name, query_event.contents.statement, query_event.contents.query_time)
			_dispose_safe_qe(query_event)

		elif et == "ROTATE_EVENT":
			rotate_event = _event_to_safe_re(self.event_buffer)
			base_event.data = RotateEvent(rotate_event.contents.next_position, rotate_event.contents.file_name)
			_dispose_safe_re(rotate_event)

		elif et == "XID_EVENT":
			xid_event = _event_to_safe_xe(self.event_buffer)
			base_event.data = XIDEvent(xid_event.contents.id)
			_dispose_safe_xe(xid_event)
		return (base_event, last == 0)

	def clean_up(self):
		"""Clean up some things that are allocated in C-land. Attempting to
		use this object after calling this method will break."""
		_dispose_bp(self.binlog_parser_handle)
		self.binlog_parser_handle = None
		_dispose_event(self.event_buffer)
		self.event_buffer = None

	def update(self):
		_update_bp(self.binlog_parser_handle)

	def __iter__(self):
		while True:
			if self.always_update:
				bp.update()
			(event, last) = self._get_next_event()
			yield event
			if last:
				raise StopIteration

	def rewind(self):
		"""Reset this bp to point to the beginning of the file"""
		pass

if __name__ == '__main__':
	bp = YBinlogP("mysql-bin.000018")
	for event in bp:
		print event
	bp.clean_up()
<commit_msg>more python stuff<commit_after>import ctypes
import datetime
import errno

library = ctypes.CDLL("libybinlogp.so.1", use_errno=True)

class EventStruct(ctypes.Structure):
	_fields_ = [("timestamp", ctypes.c_uint32),
			("type_code", ctypes.c_uint8),
			("server_id", ctypes.c_uint32),
			("length", ctypes.c_uint32),
			("next_position", ctypes.c_uint32),
			("flags", ctypes.c_uint16),
			("data", ctypes.c_void_p),
			("offset", ctypes.c_uint64)]

	_pack_ = 1

class Event(object):
	def __init__(self, event_type, timestamp):
		self.event_type = event_type
		self.time = datetime.datetime.fromtimestamp(timestamp)
		self.data = None

	def __str__(self):
		return "%s at %s%s" % (self.event_type, self.time, ": " + str(self.data) if self.data is not None else "")

class QueryEventStruct(ctypes.Structure):
	_fields_ = [("thread_id", ctypes.c_uint32),
			("query_time", ctypes.c_uint32),
			("db_name_len", ctypes.c_uint8),
			("error_code", ctypes.c_uint16),
			("status_var_len", ctypes.c_uint16),
			("statement", ctypes.c_char_p),
			("statement_len", ctypes.c_size_t),
			("status_var", ctypes.c_char_p),
			("db_name", ctypes.c_char_p)]

class QueryEvent(object):
	def __init__(self, db_name, statement, query_time):
		self.db_name = db_name
		self.statement = statement
		self.query_time = query_time

	def __str__(self):
		return "Query(db='%s', statement='%s', query_time=%d)" % (self.db_name, self.statement, self.query_time)

class RotateEventStruct(ctypes.Structure):
	_fields_ = [("next_position", ctypes.c_uint64),
			("file_name", ctypes.c_char_p),
			("file_name_len", ctypes.c_size_t)]

class RotateEvent(object):
	def __init__(self, next_position, file_name):
		self.next_position = next_position
		self.file_name = file_name

	def __str__(self):
		return "Rotate(next file=%s, next_position=%d)" % (self.file_name, self.next_position)

class XIDEventStruct(ctypes.Structure):
	_fields_ = [("id", ctypes.c_uint64)]

class XIDEvent(object):
	def __init__(self, xid):
		self.xid = xid

	def __str__(self):
		return "COMMIT xid %d" % self.xid

_init_bp = library.ybp_get_binlog_parser
_init_bp.argtypes = [ctypes.c_int]
_init_bp.restype = ctypes.c_void_p

_get_event = library.ybp_get_event
_get_event.argtypes = []
_get_event.restype = ctypes.POINTER(EventStruct)

_next_event = library.ybp_next_event
_next_event.argtypes = [ctypes.c_void_p, ctypes.POINTER(EventStruct)]
_next_event.restype = ctypes.c_int

_reset_event = library.ybp_reset_event
_reset_event.argtypes = [ctypes.POINTER(EventStruct)]
_reset_event.restype = None

_dispose_event = library.ybp_dispose_event
_dispose_event.argtypes = [ctypes.POINTER(EventStruct)]
_dispose_event.restype = None

_dispose_bp = library.ybp_dispose_binlog_parser
_dispose_bp.argtypes = [ctypes.c_void_p]
_dispose_bp.restype = None

_update_bp = library.ybp_update_bp
_update_bp.argtypes = [ctypes.c_void_p]
_update_bp.restype = None

_event_type = library.ybp_event_type
_event_type.argtypes = [ctypes.POINTER(EventStruct)]
_event_type.restype = ctypes.c_char_p

_event_to_safe_qe = library.ybp_event_to_safe_qe
_event_to_safe_qe.argtypes = [ctypes.POINTER(EventStruct)]
_event_to_safe_qe.restype = ctypes.POINTER(QueryEventStruct)

_dispose_safe_qe = library.ybp_dispose_safe_qe
_dispose_safe_qe.argtype = [ctypes.POINTER(QueryEventStruct)]
_dispose_safe_qe.qestype = None

_event_to_safe_re = library.ybp_event_to_safe_re
_event_to_safe_re.argtypes = [ctypes.POINTER(EventStruct)]
_event_to_safe_re.restype = ctypes.POINTER(RotateEventStruct)

_dispose_safe_re = library.ybp_dispose_safe_re
_dispose_safe_re.argtype = [ctypes.POINTER(RotateEventStruct)]
_dispose_safe_re.restype = None

_event_to_safe_xe = library.ybp_event_to_safe_xe
_event_to_safe_xe.argtypes = [ctypes.POINTER(EventStruct)]
_event_to_safe_xe.restype = ctypes.POINTER(XIDEventStruct)

_dispose_safe_xe = library.ybp_dispose_safe_xe
_dispose_safe_xe.argtype = [ctypes.POINTER(XIDEventStruct)]
_dispose_safe_xe.restype = None

_rewind_bp = library.ybp_rewind_bp
_rewind_bp.argtypes = [ctypes.c_void_p, ctypes.c_ulong]
_rewind_bp.restype = None

class YBinlogPError(Exception):
	def __init__(self, errno):
		self.errno = errno

	def __repr__(self):
		return "NextEventError(%s)" % errno.errorcode.get(self.errno, "Unknown")

	def __str__(self):
		return repr(self)

class NextEventError(YBinlogPError):
	pass

class YBinlogP(object):
	def __init__(self, filename, always_update=False):
		"""Construct a YBinlogP.

		Arguments:
			filename: A binlog filename
			always_update: if True, stat the file before doing anything
				interesting
		"""
		self.filename = filename
		self.file = open(filename, 'r')
		self.binlog_parser_handle = _init_bp(self.file.fileno())
		self.event_buffer = _get_event();
		self.always_update = always_update

	def _get_next_event(self):
		last = _next_event(self.binlog_parser_handle, self.event_buffer)
		if last < 0:
			raise NextEventError(ctypes.get_errno())
		event = None
		et = _event_type(self.event_buffer)
		base_event = Event(et, self.event_buffer.contents.timestamp)
		if et == "QUERY_EVENT":
			query_event = _event_to_safe_qe(self.event_buffer)
			base_event.data = QueryEvent(query_event.contents.db_name, query_event.contents.statement, query_event.contents.query_time)
			_dispose_safe_qe(query_event)

		elif et == "ROTATE_EVENT":
			rotate_event = _event_to_safe_re(self.event_buffer)
			base_event.data = RotateEvent(rotate_event.contents.next_position, rotate_event.contents.file_name)
			_dispose_safe_re(rotate_event)

		elif et == "XID_EVENT":
			xid_event = _event_to_safe_xe(self.event_buffer)
			base_event.data = XIDEvent(xid_event.contents.id)
			_dispose_safe_xe(xid_event)
		return (base_event, last == 0)

	def clean_up(self):
		"""Clean up some things that are allocated in C-land. Attempting to
		use this object after calling this method will break."""
		_dispose_bp(self.binlog_parser_handle)
		self.binlog_parser_handle = None
		_dispose_event(self.event_buffer)
		self.event_buffer = None

	def update(self):
		_update_bp(self.binlog_parser_handle)

	def __iter__(self):
		while True:
			if self.always_update:
				bp.update()
			(event, last) = self._get_next_event()
			yield event
			if last:
				raise StopIteration

	def rewind(self, offset):
		"""Reset this bp to point to the beginning of the file"""
		_rewind_bp(self.binlog_parser_handle, offset)

if __name__ == '__main__':
	bp = YBinlogP("mysql-bin.000017")
	for i,event in enumerate(bp):
		if i >= 10000:
			break
		print event
	bp.clean_up()
<|endoftext|>