<commit_before>from Bio import SeqIO
import numpy as np
import pandas as pd
from mutation_creator import Mutation_Creator
from mutation_tracker import Mutation_Tracker
import logging
from probabilities_config import structural_variations_probabilities, snv_probabilities

class Mutation_Orchestrator:
    """ Mutation_Orchestrator is a class that operates on a genome to make a mutation.
        It chooses the type, length, and start of the mutation probabilistically, using
        the probabilities defined in probabilities_config. It stores a record of the mutations
        in the Mutation_Tracker class, and modifies the genome string using Mutation_Creator"""
    def __init__(self):
        self.tracker = Mutation_Tracker()
        self.creator = Mutation_Creator()
        self.structural_variations = {
        'deletion': self.orchestrate_deletion,
        'translocation': self.orchestrate_translocation,
        'duplication' : self.orchestrate_duplication,
        'inversion' : self.orchestrate_inversion,
        'insertion' : self.orchestrate_insertion
        }

        self.logger = logging.basicConfig(filename='example.log',level=logging.DEBUG)

    def snv_fast(self, genome, number):
        chroms = self.pick_chromosomes(genome, number)
        ### assume for normal bases
        new_bases = np.random.choice(list(snv_probabilities.keys()), number, snv_probabilities.values())
        for i in range(number):
            start = self.get_location_on_sequence(genome[chroms[i]])
            genome[chroms[i]] = self.creator.create_snv(genome[chroms[i]], start, new_bases[i])
            logging.info('Added base {} at position {} in chrom {}'.format(new_bases[i], str(start), chroms[i]))
        return genome

    def pick_chromosomes(self, genome, number=1, replace=True):
        relative_lengths = np.array([len(genome[x]) for x in genome])
        probabilities = relative_lengths / float(relative_lengths.sum())
        chroms = np.random.choice(list(genome.keys()), number, replace=replace ,p=probabilities.tolist())
        return chroms

    def get_location_on_sequence(self, seq, distribution='uniform'):
        if distribution == 'uniform':
            # Don't select a location with a N
            while True:
                location = np.random.randint(len(seq))
                if seq[location] != 'N':
                    return location
        else:
            raise NotImplementedError("Only Uniform is implemented!")

    # Default to being a big deletion, but p=0.6 makes it a small deletion

    def orchestrate_deletion(self, genome, distribution='uniform', p=0.001):
        chrom = self.pick_chromosomes(genome)[0]
        start = self.get_location_on_sequence(genome[chrom])
        end = start + self.get_event_length()
        self.tracker.create_deletion(chrom, start, end)
        #genome[chrom] = self.creator.create_deletion(genome[chrom], start, end)
        logging.info('Orchestrated deletion from {} to {} in chrom {}'.format(start, end, chrom))

    def orchestrate_translocation(self, genome, distribution='uniform'):
        if len(genome) == 1:
            print('No translocations allowed: genome too small')
            return
        (chrom_source, chrom_target) = self.pick_chromosomes(genome, number = 2, replace = False)
        start_source = self.get_location_on_sequence(genome[chrom_source])
        start_target = self.get_location_on_sequence(genome[chrom_target])
        source_event_length = self.get_event_length(p=0.001)
        target_event_length = self.get_event_length(p=0.001)
        end_source = start_source+source_event_length
        end_target = start_target+target_event_length
        new_seq_source = genome[chrom_target][start_target:end_target]
        new_seq_target = genome[chrom_source][start_source:end_source]
        self.tracker.create_translocation(chrom_source, chrom_target, start_source,
                        start_target, end_source, end_target, new_seq_source, new_seq_target)
        logging.info('Orchestrated translocation at position {} of length {} on chrom {} to position {} of length {} on chrom {}'.format(
            start_source, source_event_length, chrom_source, start_target, target_event_length, chrom_target))

    # Models exponential decay, discretely, within a 1-10 range.
    # Expected value of event is 1/p
    def get_event_length(self, p=0.6, number = 1):
        z = np.random.geometric(p, size=number)
        return z[0]

    # Duplication currently only goes one direction (forward)
    # Creates a variable amount of duplications (num_duplications, drawn from geometric dist)
    def orchestrate_duplication(self, genome, distribution='uniform'):
        chrom = self.pick_chromosomes(genome, number = 1)[0]
        start = self.get_location_on_sequence(genome[chrom])
        end = start + self.get_event_length(p=0.001)
        num_duplications = self.get_event_length(p=0.6) # exponential ranging from 1 to 10
        new_seq = str(genome[chrom][start:end]) * num_duplications
        self.tracker.create_insertion(chrom, start, new_seq,
             name='duplication (times {})'.format(num_duplications))
        logging.info('Orchestrated duplication at position {} to {} on chrom {}'.format(start, end, chrom))

    def orchestrate_inversion(self, genome, distribution='uniform'):
        chrom = self.pick_chromosomes(genome, number = 1)[0]
        start = self.get_location_on_sequence(genome[chrom])
        end = start + self.get_event_length(p=0.001)
        self.tracker.create_inversion(chrom, start, end)
        logging.info('Orchestrated inversion at position {} to {} on chrom {}'.format(start, end, chrom))

    def orchestrate_insertion(self, genome, distribution='uniform', p=0.001):
        chrom = self.pick_chromosomes(genome, number = 1)[0]
        start = self.get_location_on_sequence(genome[chrom])
        event_length = self.get_event_length(p)
        new_seq_start = self.get_location_on_sequence(genome[chrom])
        new_seq_end = new_seq_start + event_length
        new_seq = genome[chrom][new_seq_start:new_seq_end]
        self.tracker.create_insertion(chrom, start, new_seq)
        logging.info('Orchestrated insertion at position {} on chrom {} adding bases from position {} to {}'.format(start,
         chrom, new_seq_start, new_seq_end))

    def generate_structural_variations(self, genome, number):
        variations = np.random.choice(list(structural_variations_probabilities.keys()),
                number, structural_variations_probabilities.values())
        for variation in variations:
            self.structural_variations[variation](genome)

    # Create small insertions and small deletions
    def generate_indels(self, genome, number):
        variations = np.random.choice(list(['insertion', 'deletion']), number)
        for variation in variations:
            self.structural_variations[variation](genome, p=0.6)

    # Actually collapses the list of changes
    def generate_fasta_and_bed(self, genome):
        (genome, log_data_frame) =  self.tracker.collapse_list(genome)
        return (genome, self.bed_correct(log_data_frame))

    # We need to store insertions as same start and end. Let's correct that when outputting bed files
    def bed_correct(self, df):
        same_vals = df[df['start'] == df['end']]
        df.ix[same_vals.index, 'end'] +=1
        return df<commit_msg>remove unnecessary import<commit_after>from Bio import SeqIO
import numpy as np
from mutation_creator import Mutation_Creator
from mutation_tracker import Mutation_Tracker
import logging
from probabilities_config import structural_variations_probabilities, snv_probabilities

class Mutation_Orchestrator:
    """ Mutation_Orchestrator is a class that operates on a genome to make a mutation.
        It chooses the type, length, and start of the mutation probabilistically, using
        the probabilities defined in probabilities_config. It stores a record of the mutations
        in the Mutation_Tracker class, and modifies the genome string using Mutation_Creator"""
    def __init__(self):
        self.tracker = Mutation_Tracker()
        self.creator = Mutation_Creator()
        self.structural_variations = {
        'deletion': self.orchestrate_deletion,
        'translocation': self.orchestrate_translocation,
        'duplication' : self.orchestrate_duplication,
        'inversion' : self.orchestrate_inversion,
        'insertion' : self.orchestrate_insertion
        }

        self.logger = logging.basicConfig(filename='example.log',level=logging.DEBUG)

    def snv_fast(self, genome, number):
        chroms = self.pick_chromosomes(genome, number)
        ### assume for normal bases
        new_bases = np.random.choice(list(snv_probabilities.keys()), number, snv_probabilities.values())
        for i in range(number):
            start = self.get_location_on_sequence(genome[chroms[i]])
            genome[chroms[i]] = self.creator.create_snv(genome[chroms[i]], start, new_bases[i])
            logging.info('Added base {} at position {} in chrom {}'.format(new_bases[i], str(start), chroms[i]))
        return genome

    def pick_chromosomes(self, genome, number=1, replace=True):
        relative_lengths = np.array([len(genome[x]) for x in genome])
        probabilities = relative_lengths / float(relative_lengths.sum())
        chroms = np.random.choice(list(genome.keys()), number, replace=replace ,p=probabilities.tolist())
        return chroms

    def get_location_on_sequence(self, seq, distribution='uniform'):
        if distribution == 'uniform':
            # Don't select a location with a N
            while True:
                location = np.random.randint(len(seq))
                if seq[location] != 'N':
                    return location
        else:
            raise NotImplementedError("Only Uniform is implemented!")

    # Default to being a big deletion, but p=0.6 makes it a small deletion

    def orchestrate_deletion(self, genome, distribution='uniform', p=0.001):
        chrom = self.pick_chromosomes(genome)[0]
        start = self.get_location_on_sequence(genome[chrom])
        end = start + self.get_event_length()
        self.tracker.create_deletion(chrom, start, end)
        #genome[chrom] = self.creator.create_deletion(genome[chrom], start, end)
        logging.info('Orchestrated deletion from {} to {} in chrom {}'.format(start, end, chrom))

    def orchestrate_translocation(self, genome, distribution='uniform'):
        if len(genome) == 1:
            print('No translocations allowed: genome too small')
            return
        (chrom_source, chrom_target) = self.pick_chromosomes(genome, number = 2, replace = False)
        start_source = self.get_location_on_sequence(genome[chrom_source])
        start_target = self.get_location_on_sequence(genome[chrom_target])
        source_event_length = self.get_event_length(p=0.001)
        target_event_length = self.get_event_length(p=0.001)
        end_source = start_source+source_event_length
        end_target = start_target+target_event_length
        new_seq_source = genome[chrom_target][start_target:end_target]
        new_seq_target = genome[chrom_source][start_source:end_source]
        self.tracker.create_translocation(chrom_source, chrom_target, start_source,
                        start_target, end_source, end_target, new_seq_source, new_seq_target)
        logging.info('Orchestrated translocation at position {} of length {} on chrom {} to position {} of length {} on chrom {}'.format(
            start_source, source_event_length, chrom_source, start_target, target_event_length, chrom_target))

    # Models exponential decay, discretely, within a 1-10 range.
    # Expected value of event is 1/p
    def get_event_length(self, p=0.6, number = 1):
        z = np.random.geometric(p, size=number)
        return z[0]

    # Duplication currently only goes one direction (forward)
    # Creates a variable amount of duplications (num_duplications, drawn from geometric dist)
    def orchestrate_duplication(self, genome, distribution='uniform'):
        chrom = self.pick_chromosomes(genome, number = 1)[0]
        start = self.get_location_on_sequence(genome[chrom])
        end = start + self.get_event_length(p=0.001)
        num_duplications = self.get_event_length(p=0.6) # exponential ranging from 1 to 10
        new_seq = str(genome[chrom][start:end]) * num_duplications
        self.tracker.create_insertion(chrom, start, new_seq,
             name='duplication (times {})'.format(num_duplications))
        logging.info('Orchestrated duplication at position {} to {} on chrom {}'.format(start, end, chrom))

    def orchestrate_inversion(self, genome, distribution='uniform'):
        chrom = self.pick_chromosomes(genome, number = 1)[0]
        start = self.get_location_on_sequence(genome[chrom])
        end = start + self.get_event_length(p=0.001)
        self.tracker.create_inversion(chrom, start, end)
        logging.info('Orchestrated inversion at position {} to {} on chrom {}'.format(start, end, chrom))

    def orchestrate_insertion(self, genome, distribution='uniform', p=0.001):
        chrom = self.pick_chromosomes(genome, number = 1)[0]
        start = self.get_location_on_sequence(genome[chrom])
        event_length = self.get_event_length(p)
        new_seq_start = self.get_location_on_sequence(genome[chrom])
        new_seq_end = new_seq_start + event_length
        new_seq = genome[chrom][new_seq_start:new_seq_end]
        self.tracker.create_insertion(chrom, start, new_seq)
        logging.info('Orchestrated insertion at position {} on chrom {} adding bases from position {} to {}'.format(start,
         chrom, new_seq_start, new_seq_end))

    def generate_structural_variations(self, genome, number):
        variations = np.random.choice(list(structural_variations_probabilities.keys()),
                number, structural_variations_probabilities.values())
        for variation in variations:
            self.structural_variations[variation](genome)

    # Create small insertions and small deletions
    def generate_indels(self, genome, number):
        variations = np.random.choice(list(['insertion', 'deletion']), number)
        for variation in variations:
            self.structural_variations[variation](genome, p=0.6)

    # Actually collapses the list of changes
    def generate_fasta_and_bed(self, genome):
        (genome, log_data_frame) =  self.tracker.collapse_list(genome)
        return (genome, self.bed_correct(log_data_frame))

    # We need to store insertions as same start and end. Let's correct that when outputting bed files
    def bed_correct(self, df):
        same_vals = df[df['start'] == df['end']]
        df.ix[same_vals.index, 'end'] +=1
        return df<|endoftext|>


<commit_before>"""
Notify windows.

Copyright (c) 2013 - 2016 Isaac Muse <isaacmuse@gmail.com>
License: MIT
"""
from __future__ import unicode_literals
import traceback
import winsound
import ctypes
import ctypes.wintypes as wintypes
import os
import platform

__all__ = ("get_notify", "alert", "setup", "windows_icons", "destroy")

if ctypes.sizeof(ctypes.c_long) == ctypes.sizeof(ctypes.c_void_p):
    WPARAM = ctypes.c_ulong
    LPARAM = ctypes.c_long
    LRESULT = ctypes.c_long
elif ctypes.sizeof(ctypes.c_longlong) == ctypes.sizeof(ctypes.c_void_p):
    WPARAM = ctypes.c_ulonglong
    LPARAM = ctypes.c_longlong
    LRESULT = ctypes.c_longlong
HANDLE = ctypes.c_void_p
WNDPROCTYPE = WNDPROC = ctypes.CFUNCTYPE(LRESULT, HANDLE, ctypes.c_uint, WPARAM, LPARAM)

WM_DESTROY = 2
IMAGE_ICON = 1
LR_LOADFROMFILE = 16
LR_DEFAULTSIZE = 64
IDI_APPLICATION = 1
WS_OVERLAPPED = 0
WS_SYSMENU = 524288
CW_USEDEFAULT = -2147483648
WM_USER = 1024

NIM_ADD = 0
NIM_MODIFY = 1
NIM_DELETE = 2
NIM_SETVERSION = 4
NIF_ICON = 2
NIF_MESSAGE = 1
NIF_TIP = 4
NIIF_INFO = 1
NIIF_WARNING = 2
NIIF_ERROR = 3
NIF_INFO = 16
NIF_SHOWTIP = 0x80

class WndClassEx(ctypes.Structure):
    """WNDCLASSEX structure."""

    _fields_ = [
        ("cbSize", ctypes.c_uint),
        ("style", ctypes.c_uint),
        ("lpfnWndProc", WNDPROCTYPE),
        ("cbClsExtra", ctypes.c_int),
        ("cbWndExtra", ctypes.c_int),
        ("hInstance", HANDLE),
        ("hIcon", HANDLE),
        ("hCursor", HANDLE),
        ("hbrBackground", HANDLE),
        ("lpszMenuName", wintypes.LPCWSTR),
        ("lpszClassName", wintypes.LPCWSTR),
        ("hIconSm", HANDLE)
    ]

class NotifyIconData(ctypes.Structure):
    """NOTIFYICONDATA structure."""

    _fields_ = [
        ("cbSize", ctypes.c_uint),
        ("hWnd", HANDLE),
        ("uID", ctypes.c_uint),
        ("uFlags", ctypes.c_uint),
        ("uCallbackMessage", ctypes.c_uint),
        ("hIcon", HANDLE),
        ("szTip", ctypes.c_wchar * 128),
        ("dwState", ctypes.c_uint),
        ("dwStateMask", ctypes.c_uint),
        ("szInfo", ctypes.c_wchar * 256),
        ("uVersion", ctypes.c_uint),
        ("szInfoTitle", ctypes.c_wchar * 64),
        ("dwInfoFlags", ctypes.c_uint),
        ("guidItem", ctypes.c_char * 16),
        ("hBalloonIcon", HANDLE),
    ]

class NotifyIconDataV3(ctypes.Structure):
    """NOTIFYICONDATA_V3 structure."""

    _fields_ = NotifyIconData._fields_[:-1]  # noqa

class Options(object):
    """Notification options."""

    notify = None
    instance = None

    @classmethod
    def clear(cls):
        """Clear."""

        cls.notify = None
        cls.instance = None

def alert(sound=None):
    """Play an alert sound for the OS."""

    snd = sound if sound is not None else "*"
    winsound.PlaySound(snd, winsound.SND_ALIAS)

class WinNotifyLevel(object):
    """Windows notification level."""

    ICON_INFORMATION = 0x01
    ICON_WARNING = 0x02
    ICON_ERROR = 0x04

windows_icons = {
    "Info": WinNotifyLevel.ICON_INFORMATION,
    "Warning": WinNotifyLevel.ICON_WARNING,
    "Error": WinNotifyLevel.ICON_ERROR
}

def notify_win_fallback(title, message, sound, icon, fallback):
    """Notify win calls the fallback."""

    fallback(title, message, sound)

class WindowsNotify(object):
    """Windows notification class."""

    window_handle = None
    taskbar_icon = None
    wc = None

    def __init__(self, app_name, icon, tooltip=None):
        """
        Create the taskbar for the application and register it.

        Show nothing by default until called.
        """

        def winproc(hwnd, msg, wparam, lparam):
            """Winproc funciton to handle events."""

            if msg == WM_USER + 20:
                self.OnTaskbarNotify(hwnd, msg, wparam, lparam)
                return 0
            elif msg == WM_DESTROY:
                self.OnDestroy(hwnd, msg, wparam, lparam)
                return 0
            return hwnd

        self.tooltip = tooltip
        self.is_xp = platform.release().lower() == 'xp'
        self.visible = False
        self.app_name = app_name

        # Register window class
        wc = WndClassEx()
        self.hinst = wc.hInstance = ctypes.windll.kernel32.GetModuleHandleW(None)
        wc.cbSize = ctypes.sizeof(wc)
        wc.lpszClassName = ctypes.c_wchar_p(app_name + "Taskbar")
        wc.lpfnWndProc = WNDPROCTYPE(winproc)
        wc.style = 0
        wc.cbClsExtra = 0
        wc.cbWndExtra = 0
        wc.hIcon = 0
        wc.hCursor = 0
        wc.hbrBackground = 0

        if WindowsNotify.wc is not None:
            self._destroy_window()
            ctypes.windll.user32.UnregisterClassW(wc.lpszClassName, None)
            WindowsNotify.wc = wc
        ctypes.windll.user32.RegisterClassExW(ctypes.byref(wc))
        WindowsNotify.wc = wc

        self._create_window(wc.lpszClassName)

        self.hicon = self.get_icon(icon)

    def get_icon(self, icon):
        """
        Get icon.

        Try to load the given icon from the path given,
        else default to generic application icon from the OS.
        """

        if WindowsNotify.taskbar_icon is not None:
            ctypes.windll.user32.DestroyIcon(wintypes.HICON(WindowsNotify.taskbar_icon))
            WindowsNotify.taskbar_icon = None

        icon_flags = LR_LOADFROMFILE | LR_DEFAULTSIZE
        try:
            assert icon is not None
            hicon = ctypes.windll.user32.LoadImageW(
                self.hinst, icon,
                IMAGE_ICON,
                0, 0, icon_flags
            )
        except Exception:
            hicon = ctypes.windll.user32.LoadIconA(0, IDI_APPLICATION)
        WindowsNotify.taskbar_icon = hicon

        return hicon

    def show_notification(self, title, msg, sound, icon, fallback):
        """
        Attemp to show notifications.

        Provide fallback for consistency with other notifyicatin methods.
        """

        try:
            self._show_notification(title, msg, sound, icon)
        except Exception:
            print(traceback.format_exc())
            fallback(title, msg, sound)

    def _create_window(self, classname):
        """Create the Window."""

        style = WS_OVERLAPPED | WS_SYSMENU
        self.hwnd = ctypes.windll.user32.CreateWindowExW(
            0, classname, classname, style,
            0, 0, CW_USEDEFAULT, CW_USEDEFAULT,
            0, 0, self.hinst, None
        )
        WindowsNotify.window_handle = self.hwnd
        ctypes.windll.user32.UpdateWindow(self.hwnd)

    def _destroy_window(self):
        """Destroy the window."""

        if WindowsNotify.window_handle:
            ctypes.windll.user32.DestroyWindow(WindowsNotify.window_handle)
            WindowsNotify.window_handle = None
            self.hwnd = None

    def _show_notification(self, title, msg, sound, icon):
        """Call windows API to show notification."""

        icon_level = 0
        if icon & WinNotifyLevel.ICON_INFORMATION:
            icon_level |= NIIF_INFO
        elif icon & WinNotifyLevel.ICON_WARNING:
            icon_level |= NIIF_WARNING
        elif icon & WinNotifyLevel.ICON_ERROR:
            icon_level |= NIIF_ERROR

        res = NotifyIconDataV3() if self.is_xp else NotifyIconData()
        res.cbSize = ctypes.sizeof(res)
        res.hWnd = self.hwnd
        res.uID = 0
        res.uFlags = NIF_INFO | NIF_ICON | NIF_MESSAGE | NIF_TIP | NIF_SHOWTIP
        res.uCallbackMessage = WM_USER + 20
        res.hIcon = self.hicon
        res.szTip = self.app_name[:128]
        res.uVersion = 3 if self.is_xp else 4
        res.szInfo = msg[:256]
        res.szInfoTitle = title[:64]
        res.dwInfoFlags = icon_level

        self.show_icon(res)

        if sound:
            alert()

    def OnTaskbarNotify(self, hwnd, msg, wparam, lparam):
        """When recieving the dismiss code for the notification, hide the icon."""

        if lparam == 1028:
            self.hide_icon()

    def show_icon(self, res):
        """Display the taskbar icon."""

        if not ctypes.windll.shell32.Shell_NotifyIconW(NIM_MODIFY, ctypes.byref(res)):
            tres = NotifyIconDataV3() if self.is_xp else NotifyIconData()
            tres.cbSize = ctypes.sizeof(res)
            tres.hWnd = self.hwnd
            tres.uID = 0
            tres.uFlags = NIF_ICON | NIF_MESSAGE | NIF_TIP
            tres.uCallbackMessage = WM_USER + 20
            tres.hIcon = self.hicon
            tres.szTip = self.app_name[:128]
            tres.uVersion = 3 if self.is_xp else 4
            ctypes.windll.shell32.Shell_NotifyIconW(NIM_ADD, ctypes.byref(tres))
            ctypes.windll.shell32.Shell_NotifyIconW(NIM_SETVERSION , ctypes.byref(tres))
            ctypes.windll.shell32.Shell_NotifyIconW(NIM_MODIFY, ctypes.byref(res))

        self.visible = True
        self.hide_icon()

    def hide_icon(self):
        """Hide icon."""

        if self.visible:
            res = NotifyIconDataV3() if self.is_xp else NotifyIconData()
            res.cbSize = ctypes.sizeof(res)
            res.hWnd = self.hwnd
            res.uID = 0
            res.uFlags = 0
            res.uVersion = 3 if self.is_xp else 4

            ctypes.windll.shell32.Shell_NotifyIconW(NIM_DELETE, ctypes.byref(res))
            ctypes.windll.user32.UpdateWindow(self.hwnd)
        self.visible = False

    def OnDestroy(self, hwnd, msg, wparam, lparam):
        """Remove icon and notification."""

        self.hide_icon()

    def destroy(self):

        self.hide_icon()
        self._destroy_window()

@staticmethod
def NotifyWin(title, msg, sound, icon, fallback):
    """Notify for windows."""

    Options.instance.show_notification(title, msg, sound, icon, fallback)

def setup(app_name, icon, *args):
    """Setup."""

    try:
        assert(icon is not None and os.path.exists(icon))
    except Exception:
        icon = None

    Options.instance = WindowsNotify(app_name, icon, app_name)
    Options.notify = NotifyWin

def destroy():
    """Destroy."""

    if Options.instance is not None:
        Options.instance.destroy()

    Options.clear()
    Options.notify = notify_win_fallback

def get_notify():
    """Get the notification."""

    return Options.notify

Options.notify = notify_win_fallback
<commit_msg>Remove callback and fix lint<commit_after>"""
Notify windows.

Copyright (c) 2013 - 2016 Isaac Muse <isaacmuse@gmail.com>
License: MIT
"""
from __future__ import unicode_literals
import traceback
import winsound
import ctypes
import ctypes.wintypes as wintypes
import os
import platform

__all__ = ("get_notify", "alert", "setup", "windows_icons", "destroy")

if ctypes.sizeof(ctypes.c_long) == ctypes.sizeof(ctypes.c_void_p):
    WPARAM = ctypes.c_ulong
    LPARAM = ctypes.c_long
    LRESULT = ctypes.c_long
elif ctypes.sizeof(ctypes.c_longlong) == ctypes.sizeof(ctypes.c_void_p):
    WPARAM = ctypes.c_ulonglong
    LPARAM = ctypes.c_longlong
    LRESULT = ctypes.c_longlong
HANDLE = ctypes.c_void_p
WNDPROCTYPE = WNDPROC = ctypes.CFUNCTYPE(LRESULT, HANDLE, ctypes.c_uint, WPARAM, LPARAM)

WM_DESTROY = 2
IMAGE_ICON = 1
LR_LOADFROMFILE = 16
LR_DEFAULTSIZE = 64
IDI_APPLICATION = 1
WS_OVERLAPPED = 0
WS_SYSMENU = 524288
CW_USEDEFAULT = -2147483648
WM_USER = 1024

NIM_ADD = 0
NIM_MODIFY = 1
NIM_DELETE = 2
NIM_SETVERSION = 4
NIF_ICON = 2
NIF_MESSAGE = 1
NIF_TIP = 4
NIIF_INFO = 1
NIIF_WARNING = 2
NIIF_ERROR = 3
NIF_INFO = 16
NIF_SHOWTIP = 0x80

class WndClassEx(ctypes.Structure):
    """WNDCLASSEX structure."""

    _fields_ = [
        ("cbSize", ctypes.c_uint),
        ("style", ctypes.c_uint),
        ("lpfnWndProc", WNDPROCTYPE),
        ("cbClsExtra", ctypes.c_int),
        ("cbWndExtra", ctypes.c_int),
        ("hInstance", HANDLE),
        ("hIcon", HANDLE),
        ("hCursor", HANDLE),
        ("hbrBackground", HANDLE),
        ("lpszMenuName", wintypes.LPCWSTR),
        ("lpszClassName", wintypes.LPCWSTR),
        ("hIconSm", HANDLE)
    ]

class NotifyIconData(ctypes.Structure):
    """NOTIFYICONDATA structure."""

    _fields_ = [
        ("cbSize", ctypes.c_uint),
        ("hWnd", HANDLE),
        ("uID", ctypes.c_uint),
        ("uFlags", ctypes.c_uint),
        ("uCallbackMessage", ctypes.c_uint),
        ("hIcon", HANDLE),
        ("szTip", ctypes.c_wchar * 128),
        ("dwState", ctypes.c_uint),
        ("dwStateMask", ctypes.c_uint),
        ("szInfo", ctypes.c_wchar * 256),
        ("uVersion", ctypes.c_uint),
        ("szInfoTitle", ctypes.c_wchar * 64),
        ("dwInfoFlags", ctypes.c_uint),
        ("guidItem", ctypes.c_char * 16),
        ("hBalloonIcon", HANDLE),
    ]

class NotifyIconDataV3(ctypes.Structure):
    """NOTIFYICONDATA_V3 structure."""

    _fields_ = NotifyIconData._fields_[:-1]  # noqa

class Options(object):
    """Notification options."""

    notify = None
    instance = None

    @classmethod
    def clear(cls):
        """Clear."""

        cls.notify = None
        cls.instance = None

def alert(sound=None):
    """Play an alert sound for the OS."""

    snd = sound if sound is not None else "*"
    winsound.PlaySound(snd, winsound.SND_ALIAS)

class WinNotifyLevel(object):
    """Windows notification level."""

    ICON_INFORMATION = 0x01
    ICON_WARNING = 0x02
    ICON_ERROR = 0x04

windows_icons = {
    "Info": WinNotifyLevel.ICON_INFORMATION,
    "Warning": WinNotifyLevel.ICON_WARNING,
    "Error": WinNotifyLevel.ICON_ERROR
}

def notify_win_fallback(title, message, sound, icon, fallback):
    """Notify win calls the fallback."""

    fallback(title, message, sound)

class WindowsNotify(object):
    """Windows notification class."""

    window_handle = None
    taskbar_icon = None
    wc = None

    def __init__(self, app_name, icon, tooltip=None):
        """
        Create the taskbar for the application and register it.

        Show nothing by default until called.
        """

        def winproc(hwnd, msg, wparam, lparam):
            """Winproc funciton to handle events."""

            return hwnd

        self.tooltip = tooltip
        self.is_xp = platform.release().lower() == 'xp'
        self.visible = False
        self.app_name = app_name

        # Register window class
        wc = WndClassEx()
        self.hinst = wc.hInstance = ctypes.windll.kernel32.GetModuleHandleW(None)
        wc.cbSize = ctypes.sizeof(wc)
        wc.lpszClassName = ctypes.c_wchar_p(app_name + "Taskbar")
        wc.lpfnWndProc = WNDPROCTYPE(winproc)
        wc.style = 0
        wc.cbClsExtra = 0
        wc.cbWndExtra = 0
        wc.hIcon = 0
        wc.hCursor = 0
        wc.hbrBackground = 0

        if WindowsNotify.wc is not None:
            self._destroy_window()
            ctypes.windll.user32.UnregisterClassW(wc.lpszClassName, None)
            WindowsNotify.wc = wc
        ctypes.windll.user32.RegisterClassExW(ctypes.byref(wc))
        WindowsNotify.wc = wc

        self._create_window(wc.lpszClassName)

        self.hicon = self.get_icon(icon)

    def get_icon(self, icon):
        """
        Get icon.

        Try to load the given icon from the path given,
        else default to generic application icon from the OS.
        """

        if WindowsNotify.taskbar_icon is not None:
            ctypes.windll.user32.DestroyIcon(wintypes.HICON(WindowsNotify.taskbar_icon))
            WindowsNotify.taskbar_icon = None

        icon_flags = LR_LOADFROMFILE | LR_DEFAULTSIZE
        try:
            assert icon is not None
            hicon = ctypes.windll.user32.LoadImageW(
                self.hinst, icon,
                IMAGE_ICON,
                0, 0, icon_flags
            )
        except Exception:
            hicon = ctypes.windll.user32.LoadIconA(0, IDI_APPLICATION)
        WindowsNotify.taskbar_icon = hicon

        return hicon

    def show_notification(self, title, msg, sound, icon, fallback):
        """
        Attemp to show notifications.

        Provide fallback for consistency with other notifyicatin methods.
        """

        try:
            self._show_notification(title, msg, sound, icon)
        except Exception:
            print(traceback.format_exc())
            fallback(title, msg, sound)

    def _create_window(self, classname):
        """Create the Window."""

        style = WS_OVERLAPPED | WS_SYSMENU
        self.hwnd = ctypes.windll.user32.CreateWindowExW(
            0, classname, classname, style,
            0, 0, CW_USEDEFAULT, CW_USEDEFAULT,
            0, 0, self.hinst, None
        )
        WindowsNotify.window_handle = self.hwnd
        ctypes.windll.user32.UpdateWindow(self.hwnd)

    def _destroy_window(self):
        """Destroy the window."""

        if WindowsNotify.window_handle:
            ctypes.windll.user32.DestroyWindow(WindowsNotify.window_handle)
            WindowsNotify.window_handle = None
            self.hwnd = None

    def _show_notification(self, title, msg, sound, icon):
        """Call windows API to show notification."""

        icon_level = 0
        if icon & WinNotifyLevel.ICON_INFORMATION:
            icon_level |= NIIF_INFO
        elif icon & WinNotifyLevel.ICON_WARNING:
            icon_level |= NIIF_WARNING
        elif icon & WinNotifyLevel.ICON_ERROR:
            icon_level |= NIIF_ERROR

        res = NotifyIconDataV3() if self.is_xp else NotifyIconData()
        res.cbSize = ctypes.sizeof(res)
        res.hWnd = self.hwnd
        res.uID = 0
        res.uFlags = NIF_INFO | NIF_ICON | NIF_MESSAGE | NIF_TIP | NIF_SHOWTIP
        res.uCallbackMessage = WM_USER + 20
        res.hIcon = self.hicon
        res.szTip = self.app_name[:128]
        res.uVersion = 3 if self.is_xp else 4
        res.szInfo = msg[:256]
        res.szInfoTitle = title[:64]
        res.dwInfoFlags = icon_level

        self.show_icon(res)

        if sound:
            alert()

    def show_icon(self, res):
        """Display the taskbar icon."""

        if not ctypes.windll.shell32.Shell_NotifyIconW(NIM_MODIFY, ctypes.byref(res)):
            tres = NotifyIconDataV3() if self.is_xp else NotifyIconData()
            tres.cbSize = ctypes.sizeof(res)
            tres.hWnd = self.hwnd
            tres.uID = 0
            tres.uFlags = NIF_ICON | NIF_MESSAGE | NIF_TIP
            tres.uCallbackMessage = WM_USER + 20
            tres.hIcon = self.hicon
            tres.szTip = self.app_name[:128]
            tres.uVersion = 3 if self.is_xp else 4
            ctypes.windll.shell32.Shell_NotifyIconW(NIM_ADD, ctypes.byref(tres))
            ctypes.windll.shell32.Shell_NotifyIconW(NIM_SETVERSION, ctypes.byref(tres))
            ctypes.windll.shell32.Shell_NotifyIconW(NIM_MODIFY, ctypes.byref(res))

        self.visible = True
        self.hide_icon()

    def hide_icon(self):
        """Hide icon."""

        if self.visible:
            res = NotifyIconDataV3() if self.is_xp else NotifyIconData()
            res.cbSize = ctypes.sizeof(res)
            res.hWnd = self.hwnd
            res.uID = 0
            res.uFlags = 0
            res.uVersion = 3 if self.is_xp else 4

            ctypes.windll.shell32.Shell_NotifyIconW(NIM_DELETE, ctypes.byref(res))
            ctypes.windll.user32.UpdateWindow(self.hwnd)
        self.visible = False

    def destroy(self):

        self.hide_icon()
        self._destroy_window()

@staticmethod
def NotifyWin(title, msg, sound, icon, fallback):
    """Notify for windows."""

    Options.instance.show_notification(title, msg, sound, icon, fallback)

def setup(app_name, icon, *args):
    """Setup."""

    try:
        assert(icon is not None and os.path.exists(icon))
    except Exception:
        icon = None

    Options.instance = WindowsNotify(app_name, icon, app_name)
    Options.notify = NotifyWin

def destroy():
    """Destroy."""

    if Options.instance is not None:
        Options.instance.destroy()

    Options.clear()
    Options.notify = notify_win_fallback

def get_notify():
    """Get the notification."""

    return Options.notify

Options.notify = notify_win_fallback
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import argparse
import json
import imp
import sys
import types

from enum import Enum
from pprint import pformat
from collections import namedtuple, deque
from pathlib import PurePath, Path

import requests
import yaml

# Kept as a way to safely do .get() but allow a None reference
_NULL_OBJECT = object();

def load_config(path):
   with path.open('rb') as fi:
       file_bytes = fi.read()
       config = yaml.load(file_bytes.decode('utf-8'))

   return config

def create_request_class(name, args, defaults):
   signature = deque()

   # Modify the parameters of the signature such that those
   # with defaults follow those without
   for arg in args:
       if arg in defaults.keys():
           signature.append(arg)
       else:
           signature.appendleft(arg)

   signature = tuple(signature)
   newclass = namedtuple('{}Request'.format(name), signature)

   default_values = []

   for arg, value in sorted(defaults.items(), key=lambda x: signature.index(x[0])):
       try:
           index = signature.index(arg)
       except ValueError:
           raise RuntimeError('Not able to find argument: {}'.format(arg))

       default_values.append(value)

   newclass.__new__.__defaults__ = tuple(default_values)
   return newclass

# def create_response_class(name, args):
#    newclass = namedtuple('{}Response'.format(name), args)
#    newclass.__new__.__defaults__ = tuple([None] * len(args))

#    return newclass

def create_api_definition(api, defn, baseurl):
   defaults = dict([(k, v['default']) for k,v in defn['args'].items()
       if v is not None and isinstance(v, dict) and v.get('default', _NULL_OBJECT) is not _NULL_OBJECT])

   request = create_request_class(api, defn['args'].keys(), defaults)
#   response = create_response_class(api, defn['returns'])
   newclass = namedtuple('{}Resource'.format(api), ('name', 'url', 'action', 'request_class'))
   return newclass(api, baseurl + defn['url'], defn['action'], request)

def create_service_module(service_name, apis):
   service_module = imp.new_module(service_name.lower())

   for api in apis:
      setattr(service_module, api.name, api)

   setattr(service_module, 'apis', apis)

   sys.modules[service_name.lower()] = service_module
   return service_module

class BaseResponse(dict):
    def __init__(self, *args, **kwargs):
        super(BaseResponse, self).__init__(*args, **kwargs)
        self.__dict__ = self

HTTPVerb = Enum('HttpVerb',
    ('POST', 'PUT', 'PATCH', 'DELETE', 'GET', 'OPTIONS', 'HEAD'))

class RestClient(object):
   """
   """

   def __init__(self, apis, **reqargs):

       self.request_by_name = dict([ (a.name, a) for a in apis])
       self.reqargs = reqargs
       print(self.reqargs)

       for api in apis:

           method_name = api.action.lower() + api.name.title()

           def create_new_api_call_func(api_defn):
               # Scopes some local context in which we can build
               # request functions with reflection that primed with
               # some static parameters.
               def api_call_func(self, **kwargs):

                   try:
                       action_name = api_defn.action.upper()
                       verb = HTTPVerb[action_name]
                   except KeyError:
                       raise RuntimeError('{} is not a valid http verb'.format(api_defn.action))

                   if verb == HTTPVerb.GET:
                       request = api_defn.request_class(**kwargs)
                       params = dict([ (k,v) for k,v in zip(request._fields, request) if v is not None ])
                       params['_expand'] = 'instance,environments'
                       raw_response = requests.get(api.url, params=params, **self.reqargs)
                   else:
                       raise RuntimeError('{} is not a handled http verb'.format(verb))

                   if raw_response.status_code != 200:
                       raw_response.raise_for_status()

                   response = raw_response.json(object_hook=lambda obj: BaseResponse(obj))
                   return response

               api_call_func.__doc__ = "{}\nParameters:\n  {}".format(
                   method_name, '\n  '.join(api_defn.request_class._fields))

               return api_call_func

           client_func = create_new_api_call_func(api)
           client_func.__name__ = method_name
           setattr(self, method_name, types.MethodType(client_func, self))

def load_service(config_path):
   service_config = load_config(config_path)

   apis = []
   for api, defn in service_config['apis'].items():
       api_def= create_api_definition(api, defn, service_config['base_url'])
       apis.append(api_def)

   service_module = create_service_module(service_config['name'], apis)
   return service_module

def parse_args():
   parser = argparse.ArgumentParser("gen rest")
   parser.add_argument('--config', '-c', required=True)
   parser.add_argument('--cert')
   parser.add_argument('--no-verify', action='store_false', dest='verify')

   return parser.parse_args()

def main():

   args = parse_args()
   config_path = Path(args.config)
   service_module = load_service(config_path)

   print(service_module)
   print(dir(service_module))

   cert = args.cert
   verify = args.verify

   client = RestClient(service_module.apis, cert=cert, verify=verify)

   #print(service_module.Places)
   #print(dir(client))
   #result = client.getPlaces(key='XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', location='47.6097,-122.3331', radius=10000)
   #import random
   #r1 = random.choice(result.results)
   # print(r1.get('name'))
   # print(r1)
   # print(r1.rating)
   # print(dir(r1))
   # print(r1.geometry.location.lat)

if __name__ == '__main__':
   main()
<commit_msg>major update and cleanup to rekt, adding post support<commit_after># -*- coding: utf-8 -*-
"""
rekt
====

rekt is a wrapper around the requests library that makes generic rest
operations less painful. I was frustrated with the implementation of
many service speicific rest wrappers especially for google apis, this
library is meant to be generic and dynamic enough by templating the common
client code for most rest services.
"""
import argparse
import json
import imp
import sys
import types

from enum import Enum
from pprint import pformat
from collections import namedtuple, deque
from pathlib import PurePath, Path

import requests
import yaml

# Kept as a way to safely do .get() but allow a None reference
_NULL_OBJECT = object();

#: Enum defining all HTTP verbs
HTTPVerb = Enum('HTTPVerb',
    ( 'DELETE', 'GET', 'HEAD' 'OPTIONS', 'PATCH', 'POST', 'PUT', ))

_RESOURCE_NAME_FMT = '{}Resource'
_RESOURCE_ATTRIBUTES = ('name', 'url', 'action', 'request_class')
_REQUEST_NAME_FMT = '{}Request'

class DynamicResponse(dict):
    """
    """
    # Recipe for allowing . attribute access on a dictionary from
    # http://stackoverflow.com/questions/4984647/accessing-dict-keys-like-an-attribute-in-python
    def __init__(self, *args, **kwargs):
        super(DynamicResponse, self).__init__(*args, **kwargs)
        self.__dict__ = self

class RestClient(object):
   """
   Class for convenience off of which we will dynamically create
   the rest client
   """
   pass

def create_request_class(name, args, defaults):
   """
   """
   signature = deque()

   # Modify the parameters of the signature such that those
   # with defaults follow those without
   for arg in args:
       if arg in defaults.keys():
           signature.append(arg)
       else:
           signature.appendleft(arg)

   signature = tuple(signature)
   newclass = namedtuple(_REQUEST_NAME_FMT.format(name), signature)

   default_values = []

   for arg, value in sorted(defaults.items(), key=lambda x: signature.index(x[0])):
       try:
           index = signature.index(arg)
       except ValueError:
           raise RuntimeError('Not able to find argument: {}'.format(arg))

       default_values.append(value)

   newclass.__new__.__defaults__ = tuple(default_values)
   return newclass

def create_api_definition(api, defn, baseurl):
   defaults = dict([(k, v['default']) for k,v in defn['args'].items()
       if v is not None
       and isinstance(v, dict)
       and v.get('default', _NULL_OBJECT) is not _NULL_OBJECT])

   request = create_request_class(api, defn['args'].keys(), defaults)
   newclass = namedtuple(_RESOURCE_NAME_FMT.format(api), _RESOURCE_ATTRIBUTES)

   return newclass(api, baseurl + defn['url'], defn['action'], request)

def create_api_call_func(api):
   """
   From an api definition object create the related api call method
   that will validate the arguments for the api call and then
   dynamicaly dispatch the request to the appropriate requests module
   convenience method for the specific HTTP verb .
   """

   # Scopes some local context in which we can build
   # request functions with reflection that primed with
   # some static parameters.
   def api_call_func(self, **kwargs):

      try:
         action_name = api.action.upper()
         verb = HTTPVerb[action_name]
      except KeyError:
         raise RuntimeError('{} is not a valid http verb'.format(api.action))

      request = api.request_class(**kwargs)
      params = dict([ (k,v) for k,v in zip(request._fields, request) if v is not None ])

      if HTTPVerb.GET == verb:
         raw_response = requests.get(api.url, params=params, **self.reqargs)

      elif HTTPVerb.POST == verb:
         raw_response = requests.post(api.url, data=params, **self.reqargs)

      else:
         raise RuntimeError('{} is not a handled http verb'.format(verb))

      if raw_response.status_code != 200:
         raw_response.raise_for_status()

      # The object hook will convert all dictionaries from the json
      # objects in the response to a . attribute access
      response = raw_response.json(object_hook=lambda obj: DynamicResponse(obj))
      return response

   method_name = api.action.lower() + api.name.title()

   api_call_func.__name__ = method_name
   api_call_func.__doc__ = "{}\nParameters:\n  {}".format(
      method_name, '\n  '.join(api.request_class._fields))

   return api_call_func

def create_rest_client(name, apis, BaseClass=RestClient):
    """
    Generate the api call functions and attach them to the generated
    RestClient subclass with the name <Service>Client.
    """

    api_funcs = [create_api_call_func(api) for api in apis]
    api_mapper = dict([ (f.__name__, f) for f in api_funcs ])

    # Adapted from :
    # http://stackoverflow.com/questions/15247075/how-can-i-dynamically-create-derived-classes-from-a-base-class
    def __init__(self, **reqargs):
        BaseClass.__init__(self)
        setattr(self, 'reqargs', reqargs)

    api_mapper['__init__'] =  __init__

    ClientClass = type('{}Client'.format(name), (BaseClass,), api_mapper)
    return ClientClass

def create_service_module(service_name, apis):
   """
   Dynamically creates a module named defined by the PEP-8 version of
   the string contained in service_name (from the YAML config). This
   module will contain a Client class, a Call Factory, and list of API
   definition objects.
   """
   service_module = imp.new_module(service_name.lower())

   for api in apis:
      setattr(service_module, api.__class__.__name__, api)

   ClientClass = create_rest_client(service_name, apis)

   setattr(service_module, 'resources', apis)
   setattr(service_module, 'Client', ClientClass)

   sys.modules[service_name.lower()] = service_module
   return service_module

def load_service(config_path):
   """
   Load a restful service specified by some YAML file at config_path.

   :param config_path: A pathlib Path object that points to the yaml
       config
   :returns: A python module containing a Client class, call factory,
       and the definition of each of the APIs defined by the config.
   """
   service_config = load_config(config_path)

   apis = []
   for api, defn in service_config['apis'].items():
       api_def= create_api_definition(api, defn, service_config['base_url'])
       apis.append(api_def)

   service_module = create_service_module(service_config['name'], apis)
   return service_module

def load_config(path):
   """
   Loads a yaml configuration.

   :param path: a pathlib Path object pointing to the configuration
   """
   with path.open('rb') as fi:
       file_bytes = fi.read()
       config = yaml.load(file_bytes.decode('utf-8'))

   return config

def parse_args():
   parser = argparse.ArgumentParser("gen rest")
   parser.add_argument('--config', '-c', required=True)
   parser.add_argument('--cert')
   parser.add_argument('--no-verify', action='store_false', dest='verify')
   parser.add_argument('--key')

   return parser.parse_args()

def main():
   args = parse_args()
   config_path = Path(args.config)
   service_module = load_service(config_path)

   print(service_module)
   print(dir(service_module))
   print(dir(service_module.Client))

   cert = args.cert
   verify = args.verify
   key = args.key

   client = service_module.Client(cert=cert, verify=verify)
   result = client.getPlaces(key=key, location='47.6097,-122.3331', keyword='bar', radius=10000)

   import random
   r1 = random.choice(result.results)
   print(r1.get('name'))
   details = client.getDetails(key=key, placeid=r1.place_id)
   print(details.keys())
   print(details.result.keys())
   print(details.result.opening_hours)

   # print(r1)
   # print(r1.rating)
   # print(dir(r1))
   # print(r1.geometry.location.lat)

if __name__ == '__main__':
   main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""This program will read keyed JSON Lines file (such as
adjudicated_modeled_live_eyehair_100.kjsonl), process it with CRF++, and print
detected attributes as a keyed JSON file, formatted to Karma's liking. The
keys in the input file will be passed through to the output file, but the
text and tokens will not.

"""

import argparse
import codecs
import sys
import crf_sentences as crfs
import crf_features as crff
import CRFPP
import json

def applyCrfKj(sentences, featlist, model, debug=False, statistics=False):
    """This is a generater for result phrases.  The primary input, 'sentences', is an iterator.  This model of an interator input and iterator results should work well with Spark."""

    # Create a CrfFeatures object.  This class provides a lot of services, but we'll use only a few.
    crfFeatures = crff.CrfFeatures(featlist)

    # Create a CRF++ processor.
    tagger = CRFPP.Tagger("-m " + model)

    # Clear the statistics:
    sentenceCount = 0 # Number of input "sentences" -- e.g., ads
    tokenCount = 0    # Number of input tokens -- words, punctuation, whatever
    phraseCount = 0   # Number of output phrases

    for sentence in sentences:
        tokens = sentence.getAllTokens()
        if debug:
            print "len(tokens)=%d" % len(tokens)
            sentenceCount += 1
            tokenCount += len(tokens)

        fc = crfFeatures.featurizeSentence(tokens)
        if debug:
            print "len(fc)=%d" % len(fc)
        tagger.clear()
        for idx, token in enumerate(tokens):
            features = fc[idx]
            if debug:
                print "token#%d (%s) has %d features" % (idx, token, len(features))
            tf = token + ' ' + ' '.join(features)
            tagger.add(tf.encode('utf-8'))
        tagger.parse()
        # tagger.size() returns the number of tokens that were added.
        # tagger.xsize() returns the number of features plus 1 (for the token).
        if debug:
            print "size=%d" % tagger.size()
            print "xsize=%d" % tagger.xsize()
            print "ysize=%d" % tagger.ysize()
            print "dsize=%d" % tagger.dsize()
            print "vlevel=%d" % tagger.vlevel()
            print "nbest=%d" % tagger.nbest()

        ntokens = tagger.size()
        if ntokens != len(tokens):
            print "received %d tokens , expected %d" % (ntokens, len(tokens))
        nfeatures = tagger.xsize()

        # Accumulate interesting tags.  We'll use a dictionary to a list of tokens.
        # We'll use the tag name as the key to the dictionary.  Would it be significantly
        # faster to use the tag index, instead?
        currentTagName = None
        tags = { }
        for tokenIdx in range(0, tagger.size()):
            if debug:
                for featureIdx in range (0, nfeatures):
                    print "x(%d, %d)=%s" % (tokenIdx, featureIdx, tagger.x(tokenIdx, featureIdx))
            # tagger.x(tokenIdx, 0) is the original token
            # tagger.y(tokenIdx) is the index of the tag assigned to that token.
            # tagger.yname(tagger.y(tokenIdx)) is the tag assigned to that token.
            #
            # Assume that tagger.y(tokenIdx) != 0 iff something interesting was found.
            tagIdx = tagger.y(tokenIdx)
            if debug:
                print "%s %s %d" % (tagger.x(tokenIdx, 0), tagger.yname(tagIdx), tagIdx)
            if tagIdx != 0:
                tagName = tagger.yname(tagIdx)

                if tagName != currentTagName:
                    if currentTagName != None:
                        yield sentence.getKey() + '\t' + json.dumps(tags, indent=None)
                        phraseCount += 1
                        tags.clear()
                    currentTagName = tagName

                if tagName not in tags:
                    tags[tagName] = []
                tags[tagName].append(tagger.x(tokenIdx, 0))
            else:
                if currentTagName != None:
                    yield sentence.getKey() + '\t' + json.dumps(tags, indent=None)
                    phraseCount += 1
                    tags.clear()
                    currentTagName = None

        # Write out any remaining tags (boundary case):
        if currentTagName != None:
            yield sentence.getKey() + '\t' + json.dumps(tags, indent=None)
            phraseCount += 1
            tags.clear()
            currentTagName = None

    if statistics:
        print "input:  %d sentences, %d tokens" % (sentenceCount, tokenCount)
        print "output: %d phrases" % phraseCount

def main(argv=None):
    '''this is called if run from command line'''
    parser = argparse.ArgumentParser()
    parser.add_argument('-d','--debug', help="Optionallly give debugging feedback.", required=False, action='store_true')
    parser.add_argument('-f','--featlist', help="Required input file with features to be extracted, one feature entry per line.", required=True)
    parser.add_argument('-i','--input', help="Required input file with Web scraping sentences in keyed JSON Lines format.", required=True)
    parser.add_argument('-m','--model', help="Required input model file.", required=True)
    parser.add_argument('-o','--output', help="Optional output file of phrases in keyed JSON Lines format.", required=False)
    parser.add_argument('-s','--statistics', help="Optionally report use statistics.", required=False, action='store_true')
    args = parser.parse_args()

    outfile = sys.stdout
    if args.output != None:
        outfile = codecs.open(args.output, 'wb', 'utf-8')

    # Read the Web scrapings as keyed JSON Lines:
    sentences = crfs.CrfSentencesFromKeyedJsonLinesFile(args.input)

    for result in applyCrfKj(sentences, args.featlist, args.model, args.debug, args.statistics):
        outfile.write(result + '\n')

    if args.output != None:
        outfile.close()

# call main() if this is run as standalone
if __name__ == "__main__":
    sys.exit(main())
<commit_msg>Improved some documentation.  Added a tagged token count ot the statistics.<commit_after>#!/usr/bin/env python

"""This program will read keyed JSON Lines file (such as
adjudicated_modeled_live_eyehair_100.kjsonl), process it with CRF++, and print
detected attributes as a keyed JSON file, formatted to Karma's liking. The
keys in the input file will be passed through to the output file, but the
text and tokens will not.

"""

import argparse
import codecs
import sys
import crf_sentences as crfs
import crf_features as crff
import CRFPP
import json

def applyCrfKj(sentences, featureListFilePath, modelFilePath, debug=False, statistics=False):
    """Apply CRF++ to a sequence of "sentences", generating tagged phrases as
output.  0 to N tagged phrases will generated as output for each input
sentence.

The primary input, 'sentences', is an iterator, which is used without internal
buffering.  This function is a generator, which is equivalent to producing an
iterator as output.  This paradigm of an interator input, iterator results,
and no internal buffering should work well with Spark.

featureListFilePath is the path to the word and phrase-list control file used
by crf_features.

modelFilePath is the path to a trained CRF++ model.

debug, when True, causes the code to emit helpful debugging information on
standard output.

statistics, when True, emits a count of input sentences and tokens, and a
count of output phrases, when done.

CRF++ is written in C++ code, which is accessed by a Python wrapper.  The
CRF++ code must be installed in the Python interpreter.  If Spark is used to
distribute processing among multiple systems, CRF++ must be installed on the
Python interpreter used by Spark on each system.

Opportunities for further optimization: The 'debug' and 'ststistics' code
support could be removed.  crf_features could be restructured to isolate its
core routines.  crf_sentences defines a CrfSentence object with getter
methods; these could be interpolated into this code for efficiency, but that
might reduce maintainability.

    """

    def result(sentence, tags):
        return sentence.getKey() + '\t' + json.dumps(tags, indent=None)

    # Create a CrfFeatures object.  This class provides a lot of services, but we'll use only a few.
    crfFeatures = crff.CrfFeatures(featureListFilePath)

    # Create a CRF++ processor object:
    tagger = CRFPP.Tagger("-m " + modelFilePath)

    # Clear the statistics:
    sentenceCount = 0     # Number of input "sentences" -- e.g., ads
    tokenCount = 0        # Number of input tokens -- words, punctuation, whatever
    taggedPhraseCount = 0 # Number of tagged output phrases
    taggedTokenCount = 0  # Number of tagged output tokens

    for sentence in sentences:
        sentenceCount += 1
        tokens = sentence.getAllTokens()
        tokenCount += len(tokens)
        if debug:
            print "len(tokens)=%d" % len(tokens)

        fc = crfFeatures.featurizeSentence(tokens)
        if debug:
            print "len(fc)=%d" % len(fc)
        tagger.clear()
        for idx, token in enumerate(tokens):
            features = fc[idx]
            if debug:
                print "token#%d (%s) has %d features" % (idx, token, len(features))
            tf = token + ' ' + ' '.join(features)
            tagger.add(tf.encode('utf-8'))
        tagger.parse()
        # tagger.size() returns the number of tokens that were added.
        # tagger.xsize() returns the number of features plus 1 (for the token).
        if debug:
            print "size=%d" % tagger.size()
            print "xsize=%d" % tagger.xsize()
            print "ysize=%d" % tagger.ysize()
            print "dsize=%d" % tagger.dsize()
            print "vlevel=%d" % tagger.vlevel()
            print "nbest=%d" % tagger.nbest()

        ntokens = tagger.size()
        if ntokens != len(tokens):
            print "received %d tokens , expected %d" % (ntokens, len(tokens))
        nfeatures = tagger.xsize()

        # Accumulate interesting tags.  We'll use a dictionary to a list of tokens.
        # We'll use the tag name as the key to the dictionary.  Would it be significantly
        # faster to use the tag index, instead?
        currentTagName = None
        tags = { }
        for tokenIdx in range(0, tagger.size()):
            if debug:
                for featureIdx in range (0, nfeatures):
                    print "x(%d, %d)=%s" % (tokenIdx, featureIdx, tagger.x(tokenIdx, featureIdx))
            # tagger.x(tokenIdx, 0) is the original token
            # tagger.y(tokenIdx) is the index of the tag assigned to that token.
            # tagger.yname(tagger.y(tokenIdx)) is the tag assigned to that token.
            #
            # Assume that tagger.y(tokenIdx) != 0 iff something interesting was found.
            tagIdx = tagger.y(tokenIdx)
            if debug:
                print "%s %s %d" % (tagger.x(tokenIdx, 0), tagger.yname(tagIdx), tagIdx)
            if tagIdx != 0:
                tagName = tagger.yname(tagIdx)

                if tagName != currentTagName:
                    if currentTagName != None:
                        yield result(sentence, tags)
                        taggedPhraseCount += 1
                        tags.clear()
                    currentTagName = tagName

                if tagName not in tags:
                    tags[tagName] = []
                tags[tagName].append(tagger.x(tokenIdx, 0))
                taggedTokenCount += 1
            else:
                if currentTagName != None:
                    yield result(sentence, tags)
                    taggedPhraseCount += 1
                    taggedTokenCount += len(tags)
                    tags.clear()
                    currentTagName = None

        # Write out any remaining tags (boundary case):
        if currentTagName != None:
            yield result(sentence, tags)
            taggedPhraseCount += 1
            taggedTokenCount += len(tags)
            tags.clear()
            currentTagName = None

    if statistics:
        print "input:  %d sentences, %d tokens" % (sentenceCount, tokenCount)
        print "output: %d phrases, %d tokens" % (taggedPhraseCount, taggedTokenCount)

def main(argv=None):
    '''this is called if run from command line'''
    parser = argparse.ArgumentParser()
    parser.add_argument('-d','--debug', help="Optionallly give debugging feedback.", required=False, action='store_true')
    parser.add_argument('-f','--featlist', help="Required input file with features to be extracted, one feature entry per line.", required=True)
    parser.add_argument('-i','--input', help="Required input file with Web scraping sentences in keyed JSON Lines format.", required=True)
    parser.add_argument('-m','--model', help="Required input model file.", required=True)
    parser.add_argument('-o','--output', help="Optional output file of phrases in keyed JSON Lines format.", required=False)
    parser.add_argument('-s','--statistics', help="Optionally report use statistics.", required=False, action='store_true')
    args = parser.parse_args()

    outfile = sys.stdout
    if args.output != None:
        outfile = codecs.open(args.output, 'wb', 'utf-8')

    # Read the Web scrapings as keyed JSON Lines:
    sentences = crfs.CrfSentencesFromKeyedJsonLinesFile(args.input)

    for result in applyCrfKj(sentences, args.featlist, args.model, args.debug, args.statistics):
        outfile.write(result + '\n')

    if args.output != None:
        outfile.close()

# call main() if this is run as standalone
if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>


<commit_before>
"""
    Implements a new custom Distutils command for handling library
    configuration.

    The "configure" command here doesn't directly affect things like
    config.pxi; rather, it exists to provide a set of attributes that are
    used by the build_ext replacement in setup_build.py.

    Options from the command line and environment variables are stored
    between invocations in a pickle file.  This allows configuring the library
    once and e.g. calling "build" and "test" without recompiling everything
    or explicitly providing the same options every time.

    This module also contains the auto-detection logic for figuring out
    the currently installed HDF5 version.
"""

from distutils.cmd import Command
import os
import os.path as op
import sys
import pickle

def loadpickle():
    """ Load settings dict from the pickle file """
    try:
        with open('h5config.pkl','rb') as f:
            cfg = pickle.load(f)
        if not isinstance(cfg, dict): raise TypeError
    except Exception:
        return {}
    return cfg

def savepickle(dct):
    """ Save settings dict to the pickle file """
    with open('h5config.pkl','wb') as f:
        pickle.dump(dct, f, protocol=0)

def validate_version(s):
    """ Ensure that s contains an X.Y.Z format version string, or ValueError.
    """
    try:
        tpl = tuple(int(x) for x in s.split('.'))
        if len(tpl) != 3: raise ValueError
    except Exception:
        raise ValueError("HDF5 version string must be in X.Y.Z format")

class EnvironmentOptions(object):

    """
        Convenience class representing the current environment variables.
    """

    def __init__(self):
        self.hdf5 = os.environ.get('HDF5_DIR')
        self.hdf5_version = os.environ.get('HDF5_VERSION')
        if self.hdf5_version is not None:
            validate_version(self.hdf5_version)

class configure(Command):

    """
        Configure build options for h5py: custom path to HDF5, version of
        the HDF5 library, and whether MPI is enabled.

        Options come from the following sources, in order of priority:

        1. Current command-line options
        2. Old command-line options
        3. Current environment variables
        4. Old environment variables
        5. Autodetection

        When options change, the rebuild_required attribute is set, and
        may only be reset by calling reset_rebuild().  The custom build_ext
        command does this.s
    """

    description = "Configure h5py build options"

    user_options = [('hdf5=', 'h', 'Custom path to HDF5'),
                    ('hdf5-version=', '5', 'HDF5 version "X.Y.Z"'),
                    ('mpi', 'm', 'Enable MPI building'),
                    ('reset', 'r', 'Reset config options') ]

    def initialize_options(self):
        self.hdf5 = None
        self.hdf5_version = None
        self.mpi = None
        self.reset = None

    def finalize_options(self):
        if self.hdf5_version is not None:
            validate_version(self.hdf5_version)

    def reset_rebuild(self):
        """ Mark this configuration as built """
        dct = loadpickle()
        dct['rebuild'] = False
        savepickle(dct)

    def run(self):
        """ Distutils calls this when the command is run """

        env = EnvironmentOptions()

        # Step 1: determine if settings have changed and update cache

        oldsettings = {} if self.reset else loadpickle()
        dct = oldsettings.copy()

        # Only update settings which have actually been specified this
        # round; ignore the others (which have value None).
        if self.hdf5 is not None:
            dct['cmd_hdf5'] = self.hdf5
        if env.hdf5 is not None:
            dct['env_hdf5'] = env.hdf5
        if self.hdf5_version is not None:
            dct['cmd_hdf5_version'] = self.hdf5_version
        if env.hdf5_version is not None:
            dct['env_hdf5_version'] = env.hdf5_version
        if self.mpi is not None:
            dct['cmd_mpi'] = self.mpi

        self.rebuild_required = dct.get('rebuild') or dct != oldsettings

        # Corner case: rebuild if options reset, but only if they previously
        # had non-default values (to handle multiple resets in a row)
        if self.reset and any(loadpickle().values()):
            self.rebuild_required = True

        dct['rebuild'] = self.rebuild_required

        savepickle(dct)

        # Step 2: update public config attributes according to priority rules

        if self.hdf5 is None:
            self.hdf5 = oldsettings.get('cmd_hdf5')
        if self.hdf5 is None:
            self.hdf5 = env.hdf5
        if self.hdf5 is None:
            self.hdf5 = oldsettings.get('env_hdf5')

        if self.hdf5_version is None:
            self.hdf5_version = oldsettings.get('cmd_hdf5_version')
        if self.hdf5_version is None:
            self.hdf5_version = env.hdf5_version
        if self.hdf5_version is None:
            self.hdf5_version = oldsettings.get('env_hdf5_version')
        if self.hdf5_version is None:
            self.hdf5_version = autodetect_version(op.join(self.hdf5, 'lib') if self.hdf5 is not None else None)

        if self.mpi is None:
            self.mpi = oldsettings.get('cmd_mpi')

        # Step 3: print the resulting configuration to stdout

        print('*' * 80)
        print(' ' * 23 + "Summary of the h5py configuration")
        print('')
        print("    Path to HDF5: " + repr(self.hdf5))
        print("    HDF5 Version: " + repr(self.hdf5_version))
        print("     MPI Enabled: " + repr(bool(self.mpi)))
        print("Rebuild Required: " + repr(bool(self.rebuild_required)))
        print('')
        print('*' * 80)

def autodetect_version(libdir=None):
    """
    Detect the current version of HDF5, and return X.Y.Z version string.

    If the version can't be determined, prints error information to stderr
    and returns the lowest supported version.

    libdir: optional place to search for libhdf5.so.
    """

    import os
    import sys
    import os.path as op
    import re
    import ctypes
    from ctypes import byref

    if sys.platform.startswith('win'):
        regexp = re.compile('^hdf5.dll$')
    elif sys.platform.startswith('darwin'):
        regexp = re.compile(r'^libhdf5.dylib')
    else:
        regexp = re.compile(r'^libhdf5.so')

    try:
        path = None

        libdirs = [] if libdir is None else [libdir]
        libdirs += ['/usr/local/lib', '/opt/local/lib']
        for d in libdirs:
            try:
                candidates = [x for x in os.listdir(d) if regexp.match(x)]
                if len(candidates) != 0:
                    candidates.sort(key=lambda x: len(x))   # Prefer libfoo.so to libfoo.so.X.Y.Z
                    path = op.abspath(op.join(d, candidates[0]))
            except Exception:
                pass   # We skip invalid entries, because that's what the C compiler does

        if path is None:
            path = "libhdf5.so"

        lib = ctypes.cdll.LoadLibrary(path)

        major = ctypes.c_uint()
        minor = ctypes.c_uint()
        release = ctypes.c_uint()

        lib.H5get_libversion(byref(major), byref(minor), byref(release))

        vers = (int(major.value), int(minor.value), int(release.value))
        vers = "%s.%s.%s" % vers
        print("Autodetected HDF5 version " + vers)

        return vers

    except Exception as e:

        sys.stderr.write("Autodetection skipped [")
        sys.stderr.write(str(e))
        sys.stderr.write("]\n")
        return '1.8.4'
<commit_msg>Refactor autodetection<commit_after>
"""
    Implements a new custom Distutils command for handling library
    configuration.

    The "configure" command here doesn't directly affect things like
    config.pxi; rather, it exists to provide a set of attributes that are
    used by the build_ext replacement in setup_build.py.

    Options from the command line and environment variables are stored
    between invocations in a pickle file.  This allows configuring the library
    once and e.g. calling "build" and "test" without recompiling everything
    or explicitly providing the same options every time.

    This module also contains the auto-detection logic for figuring out
    the currently installed HDF5 version.
"""

from distutils.cmd import Command
import os
import os.path as op
import sys
import pickle

def loadpickle():
    """ Load settings dict from the pickle file """
    try:
        with open('h5config.pkl','rb') as f:
            cfg = pickle.load(f)
        if not isinstance(cfg, dict): raise TypeError
    except Exception:
        return {}
    return cfg

def savepickle(dct):
    """ Save settings dict to the pickle file """
    with open('h5config.pkl','wb') as f:
        pickle.dump(dct, f, protocol=0)

def validate_version(s):
    """ Ensure that s contains an X.Y.Z format version string, or ValueError.
    """
    try:
        tpl = tuple(int(x) for x in s.split('.'))
        if len(tpl) != 3: raise ValueError
    except Exception:
        raise ValueError("HDF5 version string must be in X.Y.Z format")

class EnvironmentOptions(object):

    """
        Convenience class representing the current environment variables.
    """

    def __init__(self):
        self.hdf5 = os.environ.get('HDF5_DIR')
        self.hdf5_version = os.environ.get('HDF5_VERSION')
        if self.hdf5_version is not None:
            validate_version(self.hdf5_version)

class configure(Command):

    """
        Configure build options for h5py: custom path to HDF5, version of
        the HDF5 library, and whether MPI is enabled.

        Options come from the following sources, in order of priority:

        1. Current command-line options
        2. Old command-line options
        3. Current environment variables
        4. Old environment variables
        5. Autodetection

        When options change, the rebuild_required attribute is set, and
        may only be reset by calling reset_rebuild().  The custom build_ext
        command does this.s
    """

    description = "Configure h5py build options"

    user_options = [('hdf5=', 'h', 'Custom path to HDF5'),
                    ('hdf5-version=', '5', 'HDF5 version "X.Y.Z"'),
                    ('mpi', 'm', 'Enable MPI building'),
                    ('reset', 'r', 'Reset config options') ]

    def initialize_options(self):
        self.hdf5 = None
        self.hdf5_version = None
        self.mpi = None
        self.reset = None

    def finalize_options(self):
        if self.hdf5_version is not None:
            validate_version(self.hdf5_version)

    def reset_rebuild(self):
        """ Mark this configuration as built """
        dct = loadpickle()
        dct['rebuild'] = False
        savepickle(dct)

    def run(self):
        """ Distutils calls this when the command is run """

        env = EnvironmentOptions()

        # Step 1: determine if settings have changed and update cache

        oldsettings = {} if self.reset else loadpickle()
        dct = oldsettings.copy()

        # Only update settings which have actually been specified this
        # round; ignore the others (which have value None).
        if self.hdf5 is not None:
            dct['cmd_hdf5'] = self.hdf5
        if env.hdf5 is not None:
            dct['env_hdf5'] = env.hdf5
        if self.hdf5_version is not None:
            dct['cmd_hdf5_version'] = self.hdf5_version
        if env.hdf5_version is not None:
            dct['env_hdf5_version'] = env.hdf5_version
        if self.mpi is not None:
            dct['cmd_mpi'] = self.mpi

        self.rebuild_required = dct.get('rebuild') or dct != oldsettings

        # Corner case: rebuild if options reset, but only if they previously
        # had non-default values (to handle multiple resets in a row)
        if self.reset and any(loadpickle().values()):
            self.rebuild_required = True

        dct['rebuild'] = self.rebuild_required

        savepickle(dct)

        # Step 2: update public config attributes according to priority rules

        if self.hdf5 is None:
            self.hdf5 = oldsettings.get('cmd_hdf5')
        if self.hdf5 is None:
            self.hdf5 = env.hdf5
        if self.hdf5 is None:
            self.hdf5 = oldsettings.get('env_hdf5')

        if self.hdf5_version is None:
            self.hdf5_version = oldsettings.get('cmd_hdf5_version')
        if self.hdf5_version is None:
            self.hdf5_version = env.hdf5_version
        if self.hdf5_version is None:
            self.hdf5_version = oldsettings.get('env_hdf5_version')
        if self.hdf5_version is None:
            try:
                self.hdf5_version = autodetect_version(self.hdf5)
                print("Autodetected HDF5 %s" % self.hdf5_version)
            except Exception as e:
                sys.stderr.write("Autodetection skipped [%s]\n" % e)
                self.hdf5_version = '1.8.4'

        if self.mpi is None:
            self.mpi = oldsettings.get('cmd_mpi')

        # Step 3: print the resulting configuration to stdout

        print('*' * 80)
        print(' ' * 23 + "Summary of the h5py configuration")
        print('')
        print("    Path to HDF5: " + repr(self.hdf5))
        print("    HDF5 Version: " + repr(self.hdf5_version))
        print("     MPI Enabled: " + repr(bool(self.mpi)))
        print("Rebuild Required: " + repr(bool(self.rebuild_required)))
        print('')
        print('*' * 80)

def autodetect_version(hdf5_dir=None):
    """
    Detect the current version of HDF5, and return X.Y.Z version string.

    Intended for Unix-ish platforms (Linux, OS X, BSD).
    Does not support Windows. Raises an exception if anything goes wrong.

    hdf5_dir: optional HDF5 install directory to look in (containing "lib")
    """

    import os
    import sys
    import os.path as op
    import re
    import ctypes
    from ctypes import byref

    if sys.platform.startswith('darwin'):
        regexp = re.compile(r'^libhdf5.dylib')
    else:
        regexp = re.compile(r'^libhdf5.so')

    libdirs = ['/usr/local/lib', '/opt/local/lib']
    if hdf5_dir is not None:
        libdirs.insert(0, op.join(hdf5_dir, 'lib'))

    path = None
    for d in libdirs:
        try:
            candidates = [x for x in os.listdir(d) if regexp.match(x)]
        except Exception:
            continue   # Skip invalid entries

        if len(candidates) != 0:
            candidates.sort(key=lambda x: len(x))   # Prefer libfoo.so to libfoo.so.X.Y.Z
            path = op.abspath(op.join(d, candidates[0]))
            break

    if path is None:
        path = "libhdf5.so"

    lib = ctypes.cdll.LoadLibrary(path)

    major = ctypes.c_uint()
    minor = ctypes.c_uint()
    release = ctypes.c_uint()

    lib.H5get_libversion(byref(major), byref(minor), byref(release))

    return "{}.{}.{}".format(int(major.value), int(minor.value), int(release.value))
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""This program will read keyed JSON Lines file (such as
adjudicated_modeled_live_eyehair_100.kjsonl), process it with CRF++, and print
detected attributes as a keyed JSON file, formatted to Karma's liking. The
keys in the input file will be passed through to the output file, but the
text and tokens will not.

"""

import argparse
import codecs
import sys
import crf_sentences as crfs
import crf_features as crff
import CRFPP
import json

def applyCrfKj(sentences, featureListFilePath, modelFilePath, debug=False, statistics=False):
    """Apply CRF++ to a sequence of "sentences", generating tagged phrases as
output.  0 to N tagged phrases will generated as output for each input
sentence.

The primary input, 'sentences', is an iterator, which is used without internal
buffering.  This function is a generator, which is equivalent to producing an
iterator as output.  This paradigm of an interator input, iterator results,
and no internal buffering should work well with Spark.

featureListFilePath is the path to the word and phrase-list control file used
by crf_features.

modelFilePath is the path to a trained CRF++ model.

debug, when True, causes the code to emit helpful debugging information on
standard output.

statistics, when True, emits a count of input sentences and tokens, and a
count of output phrases, when done.

CRF++ is written in C++ code, which is accessed by a Python wrapper.  The
CRF++ code must be installed in the Python interpreter.  If Spark is used to
distribute processing among multiple systems, CRF++ must be installed on the
Python interpreter used by Spark on each system.

Opportunities for further optimization: The 'debug' and 'ststistics' code
support could be removed.  crf_features could be restructured to isolate its
core routines.  crf_sentences defines a CrfSentence object with getter
methods; these could be interpolated into this code for efficiency, but that
might reduce maintainability.

    """

    def result(sentence, tags):
        """Format the result as keyed Json Lines."""
        return sentence.getKey() + '\t' + json.dumps(tags, indent=None)

    # Create a CrfFeatures object.  This class provides a lot of services, but we'll use only a few.
    crfFeatures = crff.CrfFeatures(featureListFilePath)

    # Create a CRF++ processor object:
    tagger = CRFPP.Tagger("-m " + modelFilePath)

    # Clear the statistics:
    sentenceCount = 0     # Number of input "sentences" -- e.g., ads
    tokenCount = 0        # Number of input tokens -- words, punctuation, whatever
    taggedPhraseCount = 0 # Number of tagged output phrases
    taggedTokenCount = 0  # Number of tagged output tokens

    for sentence in sentences:
        sentenceCount += 1
        tokens = sentence.getAllTokens()
        tokenCount += len(tokens)
        if debug:
            print "len(tokens)=%d" % len(tokens)

        fc = crfFeatures.featurizeSentence(tokens)
        if debug:
            print "len(fc)=%d" % len(fc)
        tagger.clear()
        for idx, token in enumerate(tokens):
            features = fc[idx]
            if debug:
                print "token#%d (%s) has %d features" % (idx, token, len(features))
            tf = token + ' ' + ' '.join(features)
            tagger.add(tf.encode('utf-8'))
        tagger.parse()
        # tagger.size() returns the number of tokens that were added.
        # tagger.xsize() returns the number of features plus 1 (for the token).
        if debug:
            print "size=%d" % tagger.size()
            print "xsize=%d" % tagger.xsize()
            print "ysize=%d" % tagger.ysize()
            print "dsize=%d" % tagger.dsize()
            print "vlevel=%d" % tagger.vlevel()
            print "nbest=%d" % tagger.nbest()

        ntokens = tagger.size()
        if ntokens != len(tokens):
            print "received %d tokens , expected %d" % (ntokens, len(tokens))
        nfeatures = tagger.xsize()

        # Accumulate interesting tags.  We'll use a dictionary to a list of tokens.
        # We'll use the tag name as the key to the dictionary.  Would it be significantly
        # faster to use the tag index, instead?
        currentTagName = None
        tags = { }
        for tokenIdx in range(0, tagger.size()):
            if debug:
                for featureIdx in range (0, nfeatures):
                    print "x(%d, %d)=%s" % (tokenIdx, featureIdx, tagger.x(tokenIdx, featureIdx))
            # tagger.x(tokenIdx, 0) is the original token
            # tagger.y(tokenIdx) is the index of the tag assigned to that token.
            # tagger.yname(tagger.y(tokenIdx)) is the tag assigned to that token.
            #
            # Assume that tagger.y(tokenIdx) != 0 iff something interesting was found.
            tagIdx = tagger.y(tokenIdx)
            tagName = tagger.yname(tagIdx)
            if debug:
                print "%s %s %d" % (tagger.x(tokenIdx, 0), tagger.yname(tagIdx), tagIdx)
            if tagName != "O":
                if tagName != currentTagName:
                    if currentTagName != None:
                        yield result(sentence, tags)
                        taggedPhraseCount += 1
                        tags.clear()
                    currentTagName = tagName

                if tagName not in tags:
                    tags[tagName] = []
                tags[tagName].append(tagger.x(tokenIdx, 0))
                taggedTokenCount += 1
            else:
                if currentTagName != None:
                    yield result(sentence, tags)
                    taggedPhraseCount += 1
                    taggedTokenCount += len(tags)
                    tags.clear()
                    currentTagName = None

        # Write out any remaining tags (boundary case):
        if currentTagName != None:
            yield result(sentence, tags)
            taggedPhraseCount += 1
            taggedTokenCount += len(tags)
            tags.clear()
            currentTagName = None

    if statistics:
        print "input:  %d sentences, %d tokens" % (sentenceCount, tokenCount)
        print "output: %d phrases, %d tokens" % (taggedPhraseCount, taggedTokenCount)

def main(argv=None):
    '''this is called if run from command line'''
    parser = argparse.ArgumentParser()
    parser.add_argument('-d','--debug', help="Optionallly give debugging feedback.", required=False, action='store_true')
    parser.add_argument('-f','--featlist', help="Required input file with features to be extracted, one feature entry per line.", required=True)
    parser.add_argument('-i','--input', help="Required input file with Web scraping sentences in keyed JSON Lines format.", required=True)
    parser.add_argument('-m','--model', help="Required input model file.", required=True)
    parser.add_argument('-o','--output', help="Optional output file of phrases in keyed JSON Lines format.", required=False)
    parser.add_argument('-s','--statistics', help="Optionally report use statistics.", required=False, action='store_true')
    args = parser.parse_args()

    outfile = sys.stdout
    if args.output != None:
        outfile = codecs.open(args.output, 'wb', 'utf-8')

    # Read the Web scrapings as keyed JSON Lines:
    sentences = crfs.CrfSentencesFromKeyedJsonLinesFile(args.input)

    for result in applyCrfKj(sentences, args.featlist, args.model, args.debug, args.statistics):
        outfile.write(result + '\n')

    if args.output != None:
        outfile.close()

# call main() if this is run as standalone
if __name__ == "__main__":
    sys.exit(main())
<commit_msg>Update the comment on what makes a tag special.<commit_after>#!/usr/bin/env python

"""This program will read keyed JSON Lines file (such as
adjudicated_modeled_live_eyehair_100.kjsonl), process it with CRF++, and print
detected attributes as a keyed JSON file, formatted to Karma's liking. The
keys in the input file will be passed through to the output file, but the
text and tokens will not.

"""

import argparse
import codecs
import sys
import crf_sentences as crfs
import crf_features as crff
import CRFPP
import json

def applyCrfKj(sentences, featureListFilePath, modelFilePath, debug=False, statistics=False):
    """Apply CRF++ to a sequence of "sentences", generating tagged phrases as
output.  0 to N tagged phrases will generated as output for each input
sentence.

The primary input, 'sentences', is an iterator, which is used without internal
buffering.  This function is a generator, which is equivalent to producing an
iterator as output.  This paradigm of an interator input, iterator results,
and no internal buffering should work well with Spark.

featureListFilePath is the path to the word and phrase-list control file used
by crf_features.

modelFilePath is the path to a trained CRF++ model.

debug, when True, causes the code to emit helpful debugging information on
standard output.

statistics, when True, emits a count of input sentences and tokens, and a
count of output phrases, when done.

CRF++ is written in C++ code, which is accessed by a Python wrapper.  The
CRF++ code must be installed in the Python interpreter.  If Spark is used to
distribute processing among multiple systems, CRF++ must be installed on the
Python interpreter used by Spark on each system.

Opportunities for further optimization: The 'debug' and 'ststistics' code
support could be removed.  crf_features could be restructured to isolate its
core routines.  crf_sentences defines a CrfSentence object with getter
methods; these could be interpolated into this code for efficiency, but that
might reduce maintainability.

    """

    def result(sentence, tags):
        """Format the result as keyed Json Lines."""
        return sentence.getKey() + '\t' + json.dumps(tags, indent=None)

    # Create a CrfFeatures object.  This class provides a lot of services, but we'll use only a few.
    crfFeatures = crff.CrfFeatures(featureListFilePath)

    # Create a CRF++ processor object:
    tagger = CRFPP.Tagger("-m " + modelFilePath)

    # Clear the statistics:
    sentenceCount = 0     # Number of input "sentences" -- e.g., ads
    tokenCount = 0        # Number of input tokens -- words, punctuation, whatever
    taggedPhraseCount = 0 # Number of tagged output phrases
    taggedTokenCount = 0  # Number of tagged output tokens

    for sentence in sentences:
        sentenceCount += 1
        tokens = sentence.getAllTokens()
        tokenCount += len(tokens)
        if debug:
            print "len(tokens)=%d" % len(tokens)

        fc = crfFeatures.featurizeSentence(tokens)
        if debug:
            print "len(fc)=%d" % len(fc)
        tagger.clear()
        for idx, token in enumerate(tokens):
            features = fc[idx]
            if debug:
                print "token#%d (%s) has %d features" % (idx, token, len(features))
            tf = token + ' ' + ' '.join(features)
            tagger.add(tf.encode('utf-8'))
        tagger.parse()
        # tagger.size() returns the number of tokens that were added.
        # tagger.xsize() returns the number of features plus 1 (for the token).
        if debug:
            print "size=%d" % tagger.size()
            print "xsize=%d" % tagger.xsize()
            print "ysize=%d" % tagger.ysize()
            print "dsize=%d" % tagger.dsize()
            print "vlevel=%d" % tagger.vlevel()
            print "nbest=%d" % tagger.nbest()

        ntokens = tagger.size()
        if ntokens != len(tokens):
            print "received %d tokens , expected %d" % (ntokens, len(tokens))
        nfeatures = tagger.xsize()

        # Accumulate interesting tags.  We'll use a dictionary to a list of tokens.
        # We'll use the tag name as the key to the dictionary.  Would it be significantly
        # faster to use the tag index, instead?
        currentTagName = None
        tags = { }
        for tokenIdx in range(0, tagger.size()):
            if debug:
                for featureIdx in range (0, nfeatures):
                    print "x(%d, %d)=%s" % (tokenIdx, featureIdx, tagger.x(tokenIdx, featureIdx))
            # tagger.x(tokenIdx, 0) is the original token
            # tagger.y(tokenIdx) is the index of the tag assigned to that token.
            # tagger.yname(tagger.y(tokenIdx)) is the tag assigned to that token.
            #
            # Assume that tagger.yname(tokenIdx) != "O" iff something interesting was found.
            tagIdx = tagger.y(tokenIdx)
            tagName = tagger.yname(tagIdx)
            if debug:
                print "%s %s %d" % (tagger.x(tokenIdx, 0), tagger.yname(tagIdx), tagIdx)
            if tagName != "O":
                if tagName != currentTagName:
                    if currentTagName != None:
                        yield result(sentence, tags)
                        taggedPhraseCount += 1
                        tags.clear()
                    currentTagName = tagName

                if tagName not in tags:
                    tags[tagName] = []
                tags[tagName].append(tagger.x(tokenIdx, 0))
                taggedTokenCount += 1
            else:
                if currentTagName != None:
                    yield result(sentence, tags)
                    taggedPhraseCount += 1
                    taggedTokenCount += len(tags)
                    tags.clear()
                    currentTagName = None

        # Write out any remaining tags (boundary case):
        if currentTagName != None:
            yield result(sentence, tags)
            taggedPhraseCount += 1
            taggedTokenCount += len(tags)
            tags.clear()
            currentTagName = None

    if statistics:
        print "input:  %d sentences, %d tokens" % (sentenceCount, tokenCount)
        print "output: %d phrases, %d tokens" % (taggedPhraseCount, taggedTokenCount)

def main(argv=None):
    '''this is called if run from command line'''
    parser = argparse.ArgumentParser()
    parser.add_argument('-d','--debug', help="Optionallly give debugging feedback.", required=False, action='store_true')
    parser.add_argument('-f','--featlist', help="Required input file with features to be extracted, one feature entry per line.", required=True)
    parser.add_argument('-i','--input', help="Required input file with Web scraping sentences in keyed JSON Lines format.", required=True)
    parser.add_argument('-m','--model', help="Required input model file.", required=True)
    parser.add_argument('-o','--output', help="Optional output file of phrases in keyed JSON Lines format.", required=False)
    parser.add_argument('-s','--statistics', help="Optionally report use statistics.", required=False, action='store_true')
    args = parser.parse_args()

    outfile = sys.stdout
    if args.output != None:
        outfile = codecs.open(args.output, 'wb', 'utf-8')

    # Read the Web scrapings as keyed JSON Lines:
    sentences = crfs.CrfSentencesFromKeyedJsonLinesFile(args.input)

    for result in applyCrfKj(sentences, args.featlist, args.model, args.debug, args.statistics):
        outfile.write(result + '\n')

    if args.output != None:
        outfile.close()

# call main() if this is run as standalone
if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>