<commit_before># coding: utf-8
# Copyright (c) Pymatgen Development Team.
# Distributed under the terms of the MIT License.

from __future__ import division, unicode_literals

import logging

from monty.json import MSONable

from pymatgen.core.structure import Molecule
from pymatgen.analysis.graphs import build_MoleculeGraph
from pymatgen.analysis.local_env import OpenBabelNN
import networkx as nx

__author__ = "Samuel Blau"
__copyright__ = "Copyright 2018, The Materials Project"
__version__ = "1.0"
__maintainer__ = "Samuel Blau"
__email__ = "samblau1@gmail.com"
__status__ = "Alpha"
__date__ = "7/26/18"

logger = logging.getLogger(__name__)

class BondDissociationEnergies(MSONable):

    def __init__(self, molecule_entry, fragment_entries):
        """
        Standard constructor for bond dissociation energies.

        Args:

        Get list of all bonds in target molecule
        Build molecule graph of target molecule
        Loop through bonds aka edges and attempt to split into disconnected subgraphs
            On success:
                Search through fragments: find entries with initial OR final molecules which are isomorphic
                Should be three entires per fragment: original charge, OC+1, OC-1
                If there are too few entries:
                    If one of the fragments is an H atom, you should only find two. H+ should be set (to what value???)
                    If fragment matches initial, but not final structure:
                        Just accept the rearrangement as part of the energetic cost of dissociation?
                        At the very least, flag this structure change in the eventual entry
                    If fragment matches final, but not initial structure:
                        Ignore unless no other fragment matches either?
                    If fragment is unstable:
                        Find its subfragments? Is that even sensical?
                        Just pass?
                    If fragment truly cannot be found, aka has not been calculated or calculate failed to complete:
                        Tell user to calculate it and pass
                If there are too many entries:
                    Get rid of duplicates
                        Going to be annoying given time stamps, rounding errors, etc
                    Remove those whose initial rems don't conform to current defaults - Happens before passing
                    If still more than three, for now, isolate differences, report to user, and grab most recent
                    Just take more negative?
                Grab energies of each fragment at all three charges
                If target molecule is neutral:
                    Calculate E_frag1(0)  + E_frag2(0)
                    Calculate E_frag1(+1) + E_frag2(-1)
                    Calculate E_frag1(-1) + E_frag2(+1)
                If target molecule is +1:
                    Calculate E_frag1(0)  + E_frag2(+1)
                    Calculate E_frag1(+1) + E_frag2(0)
                    Calculate E_frag1(+2) + E_frag2(-1)?
                    Calculate E_frag1(-1) + E_frag2(+2)?
                If target molecule is -1:
                    Calculate E_frag1(0)  + E_frag2(-1)
                    Calculate E_frag1(-1) + E_frag2(0)
                    Calculate E_frag1(-2) + E_frag2(+1)?
                    Calculate E_frag1(+1) + E_frag2(-2)?
                Don't limit - just make final charge add to original charge
                Save E_molecule - calculated values associated with bond indices & charges in some data structure
            On failure:
                Track bonds that do not break to yield subgraphs as ring bonds
        Make all possible pairs of ring bonds
        Loop through ring bond pairs and attempt to split into disconnected subgraphs
            Same success/failure procedure as above. Should be wrapped in a function.
        Resolve individual rings
        Attempt to solve for single ring bond dissociation energies, careful with charge

        """

        self.molecule_entry = molecule_entry
        self.fragment_entries = fragment_entries
        self.bond_dissociation_energies = []
        self.done_frag_pairs = []
        self.ring_bonds = []
        self.bad_pairs = []
        self.mol_graph = build_MoleculeGraph(Molecule.from_dict(molecule_entry["output"]["optimized_molecule"]),
                                             strategy=OpenBabelNN,
                                             reorder=False,
                                             extend_structure=False)
        for bond in self.mol_graph.graph.edges:
            bonds = [(bond[0],bond[1])]
            self.fragment_and_process(bonds)
        self.bond_pairs = []
        for ii,bond in enumerate(self.ring_bonds):
            for jj in range(ii+1,len(self.ring_bonds)):
                bond_pair = [bond, self.ring_bonds[jj]]
                self.bond_pairs += [bond_pair]
        if len(self.bond_pairs) > 0:
            print("Ring bonds detected!")
        for bond_pair in self.bond_pairs:
            self.fragment_and_process(bond_pair)
        self.solve_ring_bonds()

    def fragment_and_process(self, bonds):
        try:
            frags = self.mol_graph.split_molecule_subgraphs(bonds,allow_reverse=True)
            frag_success = True
        except RuntimeError:
            if len(bonds) == 1:
                self.ring_bonds += bonds
            elif len(bonds) == 2:
                self.bad_pairs += bonds
            else:
                print('No reason to try and break more than two bonds at once! Exiting...')
                raise ValueError
            frag_success = False
        if frag_success:
            frags_done = False
            for frag_pair in self.done_frag_pairs:
                if not frags_done:
                    if frag_pair[0].isomorphic_to(frags[0]):
                        if frag_pair[1].isomorphic_to(frags[1]):
                            frags_done = True
                    elif frag_pair[1].isomorphic_to(frags[0]):
                        if frag_pair[0].isomorphic_to(frags[1]):
                            frags_done = True
            if not frags_done:
                self.done_frag_pairs += [frags]
                frag1_entries = self.search_fragment_entries(frags[0])
                frag2_entries = self.search_fragment_entries(frags[1])
                for frag1 in frag1_entries[0]:
                    for frag2 in frag2_entries[0]:
                        if frag1["calcs_reversed"][-1]["input"]["rem"]["method"] == frag2["calcs_reversed"][-1]["input"]["rem"]["method"] and frag1["calcs_reversed"][-1]["input"]["rem"]["basis"] == frag2["calcs_reversed"][-1]["input"]["rem"]["basis"]: # Add PCM check?
                            if frag1["output"]["initial_molecule"]["charge"] + frag2["output"]["initial_molecule"]["charge"] == self.molecule_entry["output"]["optimized_molecule"]["charge"] and self.molecule_entry["calcs_reversed"][-1]["input"]["rem"]["basis"] == frag1["calcs_reversed"][-1]["input"]["rem"]["basis"] and self.molecule_entry["calcs_reversed"][-1]["input"]["rem"]["method"] == frag1["calcs_reversed"][-1]["input"]["rem"]["method"]:
                                coords = nx.get_node_attributes(self.mol_graph.graph, "coords")
                                specie = nx.get_node_attributes(self.mol_graph.graph, "specie")
                                if frag1["output"]["final_energy"] == "unstable" or frag2["output"]["final_energy"] == "unstable":
                                    new_entry = ["unstable", bonds, specie[bonds[0][0]], specie[bonds[0][1]], coords[bonds[0][0]], coords[bonds[0][1]], Molecule.from_dict(frag1["output"]["initial_molecule"]).composition.reduced_formula, frag1["output"]["initial_molecule"]["charge"], frag1["output"]["final_energy"], Molecule.from_dict(frag2["output"]["initial_molecule"]).composition.reduced_formula, frag2["output"]["initial_molecule"]["charge"], frag2["output"]["final_energy"]]
                                else:
                                    new_entry = [self.molecule_entry["output"]["final_energy"] - (frag1["output"]["final_energy"] + frag2["output"]["final_energy"]), bonds, specie[bonds[0][0]], specie[bonds[0][1]], coords[bonds[0][0]], coords[bonds[0][1]], Molecule.from_dict(frag1["output"]["initial_molecule"]).composition.reduced_formula, frag1["output"]["initial_molecule"]["charge"], frag1["output"]["final_energy"], Molecule.from_dict(frag2["output"]["initial_molecule"]).composition.reduced_formula, frag2["output"]["initial_molecule"]["charge"], frag2["output"]["final_energy"]]
                                self.bond_dissociation_energies += [new_entry]

    def search_fragment_entries(self, frag):
        entries = []
        initial_entries = []
        final_entries = []
        for entry in self.fragment_entries:
            initial = build_MoleculeGraph(Molecule.from_dict(entry["input"]["initial_molecule"]),
                                          strategy=OpenBabelNN,
                                          reorder=False,
                                          extend_structure=False)
            final = build_MoleculeGraph(Molecule.from_dict(entry["output"]["initial_molecule"]),
                                        strategy=OpenBabelNN,
                                        reorder=False,
                                        extend_structure=False)
            if frag.isomorphic_to(initial) and frag.isomorphic_to(final):
                entries += [entry]
            elif frag.isomorphic_to(initial):
                entries += [entry]
                initial_entries += [entry]
            elif frag.isomorphic_to(final):
                final_entries += [entry]
        return [entries, initial_entries, final_entries]

    def solve_ring_bonds(self):
        pass
<commit_msg>Bit more efficient<commit_after># coding: utf-8
# Copyright (c) Pymatgen Development Team.
# Distributed under the terms of the MIT License.

from __future__ import division, unicode_literals

import logging

from monty.json import MSONable

from pymatgen.core.structure import Molecule
from pymatgen.analysis.graphs import build_MoleculeGraph
from pymatgen.analysis.local_env import OpenBabelNN
import networkx as nx

__author__ = "Samuel Blau"
__copyright__ = "Copyright 2018, The Materials Project"
__version__ = "1.0"
__maintainer__ = "Samuel Blau"
__email__ = "samblau1@gmail.com"
__status__ = "Alpha"
__date__ = "7/26/18"

logger = logging.getLogger(__name__)

class BondDissociationEnergies(MSONable):

    def __init__(self, molecule_entry, fragment_entries):
        """
        Standard constructor for bond dissociation energies.

        Args:

        Get list of all bonds in target molecule
        Build molecule graph of target molecule
        Loop through bonds aka edges and attempt to split into disconnected subgraphs
            On success:
                Search through fragments: find entries with initial OR final molecules which are isomorphic
                Should be three entires per fragment: original charge, OC+1, OC-1
                If there are too few entries:
                    If one of the fragments is an H atom, you should only find two. H+ should be set (to what value???)
                    If fragment matches initial, but not final structure:
                        Just accept the rearrangement as part of the energetic cost of dissociation?
                        At the very least, flag this structure change in the eventual entry
                    If fragment matches final, but not initial structure:
                        Ignore unless no other fragment matches either?
                    If fragment is unstable:
                        Find its subfragments? Is that even sensical?
                        Just pass?
                    If fragment truly cannot be found, aka has not been calculated or calculate failed to complete:
                        Tell user to calculate it and pass
                If there are too many entries:
                    Get rid of duplicates
                        Going to be annoying given time stamps, rounding errors, etc
                    Remove those whose initial rems don't conform to current defaults - Happens before passing
                    If still more than three, for now, isolate differences, report to user, and grab most recent
                    Just take more negative?
                Grab energies of each fragment at all three charges
                If target molecule is neutral:
                    Calculate E_frag1(0)  + E_frag2(0)
                    Calculate E_frag1(+1) + E_frag2(-1)
                    Calculate E_frag1(-1) + E_frag2(+1)
                If target molecule is +1:
                    Calculate E_frag1(0)  + E_frag2(+1)
                    Calculate E_frag1(+1) + E_frag2(0)
                    Calculate E_frag1(+2) + E_frag2(-1)?
                    Calculate E_frag1(-1) + E_frag2(+2)?
                If target molecule is -1:
                    Calculate E_frag1(0)  + E_frag2(-1)
                    Calculate E_frag1(-1) + E_frag2(0)
                    Calculate E_frag1(-2) + E_frag2(+1)?
                    Calculate E_frag1(+1) + E_frag2(-2)?
                Don't limit - just make final charge add to original charge
                Save E_molecule - calculated values associated with bond indices & charges in some data structure
            On failure:
                Track bonds that do not break to yield subgraphs as ring bonds
        Make all possible pairs of ring bonds
        Loop through ring bond pairs and attempt to split into disconnected subgraphs
            Same success/failure procedure as above. Should be wrapped in a function.
        Resolve individual rings
        Attempt to solve for single ring bond dissociation energies, careful with charge

        """

        self.molecule_entry = molecule_entry
        self.fragment_entries = fragment_entries
        self.bond_dissociation_energies = []
        self.done_frag_pairs = []
        self.ring_bonds = []
        self.bad_pairs = []
        self.mol_graph = build_MoleculeGraph(Molecule.from_dict(molecule_entry["output"]["optimized_molecule"]),
                                             strategy=OpenBabelNN,
                                             reorder=False,
                                             extend_structure=False)
        for bond in self.mol_graph.graph.edges:
            bonds = [(bond[0],bond[1])]
            self.fragment_and_process(bonds)
        self.bond_pairs = []
        for ii,bond in enumerate(self.ring_bonds):
            for jj in range(ii+1,len(self.ring_bonds)):
                bond_pair = [bond, self.ring_bonds[jj]]
                self.bond_pairs += [bond_pair]
        if len(self.bond_pairs) > 0:
            print("Ring bonds detected!")
        for bond_pair in self.bond_pairs:
            self.fragment_and_process(bond_pair)
        self.solve_ring_bonds()

    def fragment_and_process(self, bonds):
        try:
            frags = self.mol_graph.split_molecule_subgraphs(bonds,allow_reverse=True)
            frag_success = True
        except RuntimeError:
            if len(bonds) == 1:
                self.ring_bonds += bonds
            elif len(bonds) == 2:
                self.bad_pairs += bonds
            else:
                print('No reason to try and break more than two bonds at once! Exiting...')
                raise ValueError
            frag_success = False
        if frag_success:
            frags_done = False
            for frag_pair in self.done_frag_pairs:
                if frag_pair[0].isomorphic_to(frags[0]):
                    if frag_pair[1].isomorphic_to(frags[1]):
                        frags_done = True
                        break
                elif frag_pair[1].isomorphic_to(frags[0]):
                    if frag_pair[0].isomorphic_to(frags[1]):
                        frags_done = True
                        break
            if not frags_done:
                self.done_frag_pairs += [frags]
                frag1_entries = self.search_fragment_entries(frags[0])
                frag2_entries = self.search_fragment_entries(frags[1])
                for frag1 in frag1_entries[0]:
                    for frag2 in frag2_entries[0]:
                        if frag1["calcs_reversed"][-1]["input"]["rem"]["method"] == frag2["calcs_reversed"][-1]["input"]["rem"]["method"] and frag1["calcs_reversed"][-1]["input"]["rem"]["basis"] == frag2["calcs_reversed"][-1]["input"]["rem"]["basis"]: # Add PCM check?
                            if frag1["output"]["initial_molecule"]["charge"] + frag2["output"]["initial_molecule"]["charge"] == self.molecule_entry["output"]["optimized_molecule"]["charge"] and self.molecule_entry["calcs_reversed"][-1]["input"]["rem"]["basis"] == frag1["calcs_reversed"][-1]["input"]["rem"]["basis"] and self.molecule_entry["calcs_reversed"][-1]["input"]["rem"]["method"] == frag1["calcs_reversed"][-1]["input"]["rem"]["method"]:
                                coords = nx.get_node_attributes(self.mol_graph.graph, "coords")
                                specie = nx.get_node_attributes(self.mol_graph.graph, "specie")
                                if frag1["output"]["final_energy"] == "unstable" or frag2["output"]["final_energy"] == "unstable":
                                    new_entry = ["unstable", bonds, specie[bonds[0][0]], specie[bonds[0][1]], coords[bonds[0][0]], coords[bonds[0][1]], Molecule.from_dict(frag1["output"]["initial_molecule"]).composition.reduced_formula, frag1["output"]["initial_molecule"]["charge"], frag1["output"]["final_energy"], Molecule.from_dict(frag2["output"]["initial_molecule"]).composition.reduced_formula, frag2["output"]["initial_molecule"]["charge"], frag2["output"]["final_energy"]]
                                else:
                                    new_entry = [self.molecule_entry["output"]["final_energy"] - (frag1["output"]["final_energy"] + frag2["output"]["final_energy"]), bonds, specie[bonds[0][0]], specie[bonds[0][1]], coords[bonds[0][0]], coords[bonds[0][1]], Molecule.from_dict(frag1["output"]["initial_molecule"]).composition.reduced_formula, frag1["output"]["initial_molecule"]["charge"], frag1["output"]["final_energy"], Molecule.from_dict(frag2["output"]["initial_molecule"]).composition.reduced_formula, frag2["output"]["initial_molecule"]["charge"], frag2["output"]["final_energy"]]
                                self.bond_dissociation_energies += [new_entry]

    def search_fragment_entries(self, frag):
        entries = []
        initial_entries = []
        final_entries = []
        for entry in self.fragment_entries:
            initial = build_MoleculeGraph(Molecule.from_dict(entry["input"]["initial_molecule"]),
                                          strategy=OpenBabelNN,
                                          reorder=False,
                                          extend_structure=False)
            final = build_MoleculeGraph(Molecule.from_dict(entry["output"]["initial_molecule"]),
                                        strategy=OpenBabelNN,
                                        reorder=False,
                                        extend_structure=False)
            if frag.isomorphic_to(initial) and frag.isomorphic_to(final):
                entries += [entry]
            elif frag.isomorphic_to(initial):
                entries += [entry]
                initial_entries += [entry]
            elif frag.isomorphic_to(final):
                final_entries += [entry]
        return [entries, initial_entries, final_entries]

    def solve_ring_bonds(self):
        pass
<|endoftext|>


<commit_before>#!/usr/bin/python

import re
import sys

from bs4 import BeautifulSoup
from urllib.request import urlopen

SCP_SERIES_REGEX = re.compile(r'/scp-[0-9]{3,4}')

def count_slots(url):
    with urlopen(url) as response:
        data = response.read()

    empty_slots = 0
    total_slots = 0
    scps = set()

    soup = BeautifulSoup(data, 'html.parser')
    for link in soup.select('.series a'):
        href = link.get('href')
        if href is None or SCP_SERIES_REGEX.match(href) is None:
            continue

        if href in scps:
            continue

        scps.add(href)
        total_slots += 1
        if link.get('class') == ['newpage']:
            empty_slots += 1

    return empty_slots, total_slots

def get_series_url(number):
    if number == 1:
        return 'http://www.scp-wiki.net/scp-series'

    return f'http://www.scp-wiki.net/scp-series-{series}'

if __name__ == '__main__':
    if len(sys.argv) > 1:
        series = int(sys.argv[1])
    else:
        series = 5

    url = get_series_url(series)
    print(f'Counting slots in {url}...')
    empty, total = count_slots(url)
    print(f'{empty} / {total} slots empty ({empty/total*100:.2f}%)')
<commit_msg>Script is python3 only.<commit_after>#!/usr/bin/python3

import re
import sys

from bs4 import BeautifulSoup
from urllib.request import urlopen

SCP_SERIES_REGEX = re.compile(r'/scp-[0-9]{3,4}')

def count_slots(url):
    with urlopen(url) as response:
        data = response.read()

    empty_slots = 0
    total_slots = 0
    scps = set()

    soup = BeautifulSoup(data, 'html.parser')
    for link in soup.select('.series a'):
        href = link.get('href')
        if href is None or SCP_SERIES_REGEX.match(href) is None:
            continue

        if href in scps:
            continue

        scps.add(href)
        total_slots += 1
        if link.get('class') == ['newpage']:
            empty_slots += 1

    return empty_slots, total_slots

def get_series_url(number):
    if number == 1:
        return 'http://www.scp-wiki.net/scp-series'

    return f'http://www.scp-wiki.net/scp-series-{series}'

if __name__ == '__main__':
    if len(sys.argv) > 1:
        series = int(sys.argv[1])
    else:
        series = 5

    url = get_series_url(series)
    print(f'Counting slots in {url}...')
    empty, total = count_slots(url)
    print(f'{empty} / {total} slots empty ({empty/total*100:.2f}%)')
<|endoftext|>


<commit_before>#!/usr/bin/env python

# train.py
# constructs the Theano optimization and trains a learning model,
# optionally by initializing the network with pre-trained weights.

# our own imports
import utils
import model

# module imports
import time
import theano
import numpy as np
import cPickle as pickle

from lasagne import layers

from os.path import join, abspath
import random

def train(data_file, labels_file, weights_file, pretrained_weights_file=None):

    def augmentation(Xb, yb):
        # label_map = {
        #     0:  4,
        #     1:  5,
        #     2:  6,
        #     3:  7,
        #     8:  12,
        #     9:  13,
        #     10: 14,
        #     11: 15,
        # }
        label_map = { x: x + 4 for x in range(0, 4) + range(8, 12) }
        # Apply inverse
        for key in label_map.keys():
            label = label_map[key]
            label_map[label] = key
        # Map
        points, channels, height, width = Xb.shape
        for index in range(points):
            if random.uniform(0.0, 1.0) <= 0.5:
                Xb[index] = Xb[index, :, ::-1]
                yb[index] = label_map[yb[index]]
        return Xb, yb

    def learning_rate_update(x):
        return x / 10.0

    def learning_rate_shock(x):
        return min(learning_rate, x * 10.0)

    # Training parameters
    center         = True
    learning_rate  = 0.03
    momentum       = 0.9
    batch_size     = 128
    patience       = 15
    max_epochs     = patience * 10
    # regularization = None
    regularization = 0.0001
    output_dims    = 16    # the number of outputs from the softmax layer (# classes)

    ######################################################################################

    # Load the data
    print('\n[data] loading data...')
    data, labels = utils.load(data_file, labels_file)
    # print('[load] adding channels...')
    # data = utils.add_channels(data)
    print('[data]     X.shape = %r' % (data.shape,))
    print('[data]     X.dtype = %r' % (data.dtype,))
    print('[data]     y.shape = %r' % (labels.shape,))
    print('[data]     y.dtype = %r' % (labels.dtype,))

    # utils.show_image_from_data(data)

    # Split the dataset into training and validation
    print('[data] creating train, validation datasaets...')
    dataset = utils.train_test_split(data, labels, eval_size=0.2)
    X_train, y_train, X_valid, y_valid = dataset

    # Center the data by subtracting the mean
    if center:
        print('[data] applying data centering...')
        center_mean = np.mean(X_train, axis=0)
        # center_std  = np.std(X_train, axis=0)
        center_std  = 255.0
    else:
        center_mean = 0.0
        center_std  = 255.0

    # Build and print the model
    print('\n[model] building model...')
    input_cases, input_channels, input_height, input_width = data.shape
    output_layer = model.build_model(batch_size, input_width, input_height,
                                     input_channels, output_dims)
    utils.print_layer_info(output_layer)

    # Load the pretrained model if specified
    if pretrained_weights_file is not None:
        print('[model] loading pretrained weights from %s' % (pretrained_weights_file))
        with open(pretrained_weights_file, 'rb') as pfile:
            pretrained_weights = pickle.load(pfile)
            layers.set_all_param_values(output_layer, pretrained_weights)

    # Create the Theano primitives
    print('[model] creating Theano primitives...')
    learning_rate_ = theano.shared(utils.float32(learning_rate))
    all_iters = utils.create_iter_funcs(learning_rate_, momentum, output_layer, regularization=regularization)
    train_iter, valid_iter, predict_iter = all_iters

    # Begin training the neural network
    print('\n[train] starting training at %s with learning rate %.9f' % (utils.get_current_time(), learning_rate, ))
    utils.print_header_columns()
    epoch, best_weights, best_accuracy, best_epoch = 0, None, 0.0, 0
    best_train_loss, best_valid_loss, best_valid_accuracy = np.inf, np.inf, 0.0
    try:
        while True:
            try:
                # Reset the loses for the batch
                train_losses, valid_losses, valid_accuracies = [], [], []

                t0 = time.time()
                # compute the loss over all training batches
                for Xb, yb in utils.batch_iterator(X_train, y_train, batch_size,
                                                   center_mean, center_std,
                                                   rand=True, augment=augmentation):
                    batch_train_loss = train_iter(Xb, yb)
                    train_losses.append(batch_train_loss)

                # compute the loss over all validation batches
                for Xb, yb in utils.batch_iterator(X_valid, y_valid, batch_size,
                                                   center_mean, center_std):
                    batch_valid_loss, batch_accuracy = valid_iter(Xb, yb)
                    valid_losses.append(batch_valid_loss)
                    valid_accuracies.append(batch_accuracy)

                # compute the loss over all validation batches
                for Xb, yb in utils.batch_iterator(X_valid, y_valid, batch_size,
                                                   center_mean, center_std):
                    batch_predict_proba, batch_pred = predict_iter(Xb, yb)
                    print(batch_pred)
                    print(yb)
                    print('--------------')

                # estimate the loss over all batches
                avg_train_loss = np.mean(train_losses)
                avg_valid_loss = np.mean(valid_losses)
                avg_valid_accuracy = np.mean(valid_accuracies)

                # If the training loss is nan, the training has diverged
                if np.isnan(avg_train_loss):
                    print('\n[train] training diverged\n')
                    break

                # Running tab for what the best model
                if avg_train_loss < best_train_loss:
                    best_train_loss = avg_train_loss
                if avg_valid_loss < best_valid_loss:
                    best_valid_loss = avg_valid_loss
                    best_epoch = epoch
                    best_weights = layers.get_all_param_values(output_layer)
                    best_accuracy = avg_valid_accuracy
                if avg_valid_accuracy > best_valid_accuracy:
                    best_valid_accuracy = avg_valid_accuracy

                # Learning rate schedule update
                if epoch >= best_epoch + patience:
                    best_epoch = epoch
                    utils.set_learning_rate(learning_rate_, learning_rate_update)

                # Increment the epoch
                epoch += 1
                utils.print_epoch_info(avg_valid_loss, best_valid_loss, avg_valid_accuracy,
                                       best_valid_accuracy, avg_train_loss, best_train_loss,
                                       epoch, time.time() - t0)

                # Break on max epochs
                if epoch >= max_epochs:
                    print('\n[train] maximum number of epochs reached\n')
                    break
            except KeyboardInterrupt:
                # We have caught the Keyboard Interrupt, figure out what resolution mode
                print('\n[train] Caught CRTL+C')
                resolution = ''
                while not (resolution.isdigit() and int(resolution) in [1, 2, 3]):
                    print('\n[train] What do you want to do?')
                    print('[train]     1 - Shock weights')
                    print('[train]     2 - Save best weights')
                    print('[train]     3 - Stop network training')
                    resolution = raw_input('[train] Resolution: ')
                resolution = int(resolution)
                # We have a resolution
                if resolution == 1:
                    # Shock the weights of the network
                    utils.shock_network(output_layer)
                    best_epoch = epoch
                    utils.set_learning_rate(learning_rate_, learning_rate_shock)
                elif resolution == 2:
                    # Save the weights of the network
                    utils.save_best_model(best_weights, best_accuracy, weights_file)
                else:
                    # Terminate the network training
                    raise KeyboardInterrupt
    except KeyboardInterrupt:
        print('\n\n[train] ...stopping network training\n')

    # Save the best network
    utils.save_best_model(best_weights, best_accuracy, weights_file)

if __name__ == '__main__':
    project_name            = 'viewpoint'
    root                    = abspath(join('..', 'data'))
    train_data_file         = join(root, 'numpy', project_name, 'X.npy')
    train_labels_file       = join(root, 'numpy', project_name, 'y.npy')
    weights_file            = join(root, 'nets', 'ibeis_cnn_weights.pickle')
    pretrained_weights_file = join(root, 'nets', 'pretrained_weights.pickle')

    train(train_data_file, train_labels_file, weights_file)
    #train(train_data_file, train_labels_file, weights_file, pretrained_weights_file)
<commit_msg>Small updates<commit_after>#!/usr/bin/env python

# train.py
# constructs the Theano optimization and trains a learning model,
# optionally by initializing the network with pre-trained weights.

# our own imports
import utils
import model

# module imports
import time
import theano
import numpy as np
import cPickle as pickle

from lasagne import layers

from os.path import join, abspath
import random

def train(data_file, labels_file, weights_file, pretrained_weights_file=None):

    def augmentation(Xb, yb):
        # label_map = {
        #     0:  4,
        #     1:  5,
        #     2:  6,
        #     3:  7,
        #     8:  12,
        #     9:  13,
        #     10: 14,
        #     11: 15,
        # }
        label_map = { x: x + 4 for x in range(0, 4) + range(8, 12) }
        # Apply inverse
        for key in label_map.keys():
            label = label_map[key]
            label_map[label] = key
        # Map
        points, channels, height, width = Xb.shape
        for index in range(points):
            if random.uniform(0.0, 1.0) <= 0.5:
                Xb[index] = Xb[index, :, ::-1]
                yb[index] = label_map[yb[index]]
        return Xb, yb

    def learning_rate_update(x):
        return x / 10.0

    def learning_rate_shock(x):
        return min(learning_rate, x * 10.0)

    # Training parameters
    center         = True
    learning_rate  = 0.03
    momentum       = 0.9
    batch_size     = 128
    patience       = 15
    max_epochs     = patience * 10
    # regularization = None
    regularization = 0.0001
    output_dims    = 16    # the number of outputs from the softmax layer (# classes)

    ######################################################################################

    # Load the data
    print('\n[data] loading data...')
    data, labels = utils.load(data_file, labels_file)
    # print('[load] adding channels...')
    # data = utils.add_channels(data)
    print('[data]     X.shape = %r' % (data.shape,))
    print('[data]     X.dtype = %r' % (data.dtype,))
    print('[data]     y.shape = %r' % (labels.shape,))
    print('[data]     y.dtype = %r' % (labels.dtype,))

    # utils.show_image_from_data(data)

    # Split the dataset into training and validation
    print('[data] creating train, validation datasaets...')
    dataset = utils.train_test_split(data, labels, eval_size=0.2)
    X_train, y_train, X_valid, y_valid = dataset

    # Center the data by subtracting the mean
    if center:
        print('[data] applying data centering...')
        center_mean = np.mean(X_train, axis=0)
        # center_std  = np.std(X_train, axis=0)
        center_std  = 255.0
    else:
        center_mean = 0.0
        center_std  = 255.0

    # Build and print the model
    print('\n[model] building model...')
    input_cases, input_channels, input_height, input_width = data.shape
    output_layer = model.build_model(batch_size, input_width, input_height,
                                     input_channels, output_dims)
    utils.print_layer_info(output_layer)

    # Load the pretrained model if specified
    if pretrained_weights_file is not None:
        print('[model] loading pretrained weights from %s' % (pretrained_weights_file))
        with open(pretrained_weights_file, 'rb') as pfile:
            pretrained_weights = pickle.load(pfile)
            layers.set_all_param_values(output_layer, pretrained_weights)

    # Create the Theano primitives
    print('[model] creating Theano primitives...')
    learning_rate_ = theano.shared(utils.float32(learning_rate))
    all_iters = utils.create_iter_funcs(learning_rate_, momentum, output_layer, regularization=regularization)
    train_iter, valid_iter, predict_iter = all_iters

    # Begin training the neural network
    print('\n[train] starting training at %s with learning rate %.9f' % (utils.get_current_time(), learning_rate, ))
    utils.print_header_columns()
    epoch, best_weights, best_accuracy, best_epoch = 0, None, 0.0, 0
    best_train_loss, best_valid_loss, best_valid_accuracy = np.inf, np.inf, 0.0
    try:
        while True:
            try:
                # Reset the loses for the batch
                train_losses, valid_losses, valid_accuracies = [], [], []

                t0 = time.time()
                # compute the loss over all training batches
                for Xb, yb in utils.batch_iterator(X_train, y_train, batch_size,
                                                   center_mean, center_std,
                                                   rand=True, augment=augmentation):
                    batch_train_loss = train_iter(Xb, yb)
                    train_losses.append(batch_train_loss)

                # compute the loss over all validation batches
                for Xb, yb in utils.batch_iterator(X_valid, y_valid, batch_size,
                                                   center_mean, center_std):
                    batch_valid_loss, batch_accuracy = valid_iter(Xb, yb)
                    valid_losses.append(batch_valid_loss)
                    valid_accuracies.append(batch_accuracy)

                # compute the loss over all validation batches
                for Xb, yb in utils.batch_iterator(X_valid, y_valid, batch_size,
                                                   center_mean, center_std):
                    batch_predict_proba, batch_pred = predict_iter(Xb)
                    print(batch_pred)
                    print(yb)
                    print('--------------')

                # estimate the loss over all batches
                avg_train_loss = np.mean(train_losses)
                avg_valid_loss = np.mean(valid_losses)
                avg_valid_accuracy = np.mean(valid_accuracies)

                # If the training loss is nan, the training has diverged
                if np.isnan(avg_train_loss):
                    print('\n[train] training diverged\n')
                    break

                # Running tab for what the best model
                if avg_train_loss < best_train_loss:
                    best_train_loss = avg_train_loss
                if avg_valid_loss < best_valid_loss:
                    best_valid_loss = avg_valid_loss
                    best_epoch = epoch
                    best_weights = layers.get_all_param_values(output_layer)
                    best_accuracy = avg_valid_accuracy
                if avg_valid_accuracy > best_valid_accuracy:
                    best_valid_accuracy = avg_valid_accuracy

                # Learning rate schedule update
                if epoch >= best_epoch + patience:
                    best_epoch = epoch
                    utils.set_learning_rate(learning_rate_, learning_rate_update)

                # Increment the epoch
                epoch += 1
                utils.print_epoch_info(avg_valid_loss, best_valid_loss, avg_valid_accuracy,
                                       best_valid_accuracy, avg_train_loss, best_train_loss,
                                       epoch, time.time() - t0)

                # Break on max epochs
                if epoch >= max_epochs:
                    print('\n[train] maximum number of epochs reached\n')
                    break
            except KeyboardInterrupt:
                # We have caught the Keyboard Interrupt, figure out what resolution mode
                print('\n[train] Caught CRTL+C')
                resolution = ''
                while not (resolution.isdigit() and int(resolution) in [1, 2, 3]):
                    print('\n[train] What do you want to do?')
                    print('[train]     1 - Shock weights')
                    print('[train]     2 - Save best weights')
                    print('[train]     3 - Stop network training')
                    resolution = raw_input('[train] Resolution: ')
                resolution = int(resolution)
                # We have a resolution
                if resolution == 1:
                    # Shock the weights of the network
                    utils.shock_network(output_layer)
                    best_epoch = epoch
                    utils.set_learning_rate(learning_rate_, learning_rate_shock)
                elif resolution == 2:
                    # Save the weights of the network
                    utils.save_best_model(best_weights, best_accuracy, weights_file)
                else:
                    # Terminate the network training
                    raise KeyboardInterrupt
    except KeyboardInterrupt:
        print('\n\n[train] ...stopping network training\n')

    # Save the best network
    utils.save_best_model(best_weights, best_accuracy, weights_file)

if __name__ == '__main__':
    project_name            = 'viewpoint'
    root                    = abspath(join('..', 'data'))
    train_data_file         = join(root, 'numpy', project_name, 'X.npy')
    train_labels_file       = join(root, 'numpy', project_name, 'y.npy')
    weights_file            = join(root, 'nets', 'ibeis_cnn_weights.pickle')
    pretrained_weights_file = join(root, 'nets', 'pretrained_weights.pickle')

    train(train_data_file, train_labels_file, weights_file)
    #train(train_data_file, train_labels_file, weights_file, pretrained_weights_file)
<|endoftext|>


<commit_before>import argparse
import baseline
import sys
from baseline.embeddings import load_embeddings_overlay
from eight_mile.utils import read_config_stream
from baseline.pytorch.embeddings import *
from baseline.embeddings import *
from baseline.vectorizers import *
from baseline.vectorizers import BPEVectorizer1D, WordpieceVectorizer1D
from mead.utils import convert_path, parse_extra_args, configure_logger, parse_and_merge_overrides, read_config_file_or_json, index_by_label
import pandas as pd
logger = logging.getLogger(__file__)
DEFAULT_SETTINGS_LOC = 'config/mead-settings.json'
DEFAULT_DATASETS_LOC = 'config/datasets.json'
DEFAULT_LOGGING_LOC = 'config/logging.json'
DEFAULT_EMBEDDINGS_LOC = 'config/embeddings.json'
DEFAULT_VECTORIZERS_LOC = 'config/vecs.json'

parser = argparse.ArgumentParser(description='Encode a sentence as an embedding')
parser.add_argument('--subword_model_file', help='Subword model file')
parser.add_argument('--nctx', default=256, type=int)
parser.add_argument('--batchsz', default=20, type=int)
parser.add_argument('--vec_id', default='bert-base-uncased', help='Reference to a specific embedding type')
parser.add_argument('--embed_id', default='bert-base-uncased-pooled', help='What type of embeddings to use')
parser.add_argument('--file', required=True)
parser.add_argument('--column', required=True)
parser.add_argument('--output', default='embeddings.npz')
parser.add_argument('--pool', help='Should a reduction be applied on the embeddings?', choices=['cls', 'max', 'mean'], default='cls')
parser.add_argument('--max_len1d', type=int, default=100)
parser.add_argument('--embeddings', help='index of embeddings: local file, remote URL or mead-ml/hub ref', type=convert_path)
parser.add_argument('--vecs', help='index of vectorizers: local file, remote URL or hub mead-ml/ref', type=convert_path)
parser.add_argument('--cuda', type=baseline.str2bool, default=True)
args = parser.parse_args()

args.embeddings = convert_path(DEFAULT_EMBEDDINGS_LOC) if args.embeddings is None else args.embeddings
args.embeddings = read_config_stream(args.embeddings)

args.vecs = convert_path(DEFAULT_VECTORIZERS_LOC) if args.vecs is None else args.vecs

vecs_index = read_config_stream(args.vecs)
vecs_set = index_by_label(vecs_index)
vec_params = vecs_set[args.vec_id]
vec_params['mxlen'] = args.nctx

if 'transform_fn' in vec_params and isinstance(vec_params['transform_fn'], str):
    vec_params['transform_fn'] = eval(vec_params['transform_fn'])

vectorizer = create_vectorizer(**vec_params)

embeddings_index = read_config_stream(args.embeddings)
embeddings_set = index_by_label(embeddings_index)
embeddings_params = embeddings_set[args.embed_id]
embeddings = load_embeddings_overlay(embeddings_set, embeddings_params, vectorizer.vocab)

vocabs = {'x': embeddings['vocab']}
embedder = embeddings['embeddings'].cpu()
if args.cuda:
    embedder = embedder.cuda()

def chunks(lst, n):
    """Yield successive n-sized chunks from lst."""
    for i in range(0, len(lst), n):
        yield lst[i:i + n]

df = pd.read_csv(args.file)[args.column]
batches = []
as_list = df.tolist()
num_batches = math.ceil(len(as_list) / args.batchsz)
pg = baseline.create_progress_bar(num_batches, name='tqdm')
for i, batch in enumerate(chunks(as_list, args.batchsz)):
    pg.update()
    with torch.no_grad():
        vecs = []
        for line in batch:
            vec, l = vectorizer.run(line.split(), vocabs['x'])
            vecs.append(vec)
        vecs = torch.tensor(np.stack(vecs))
        if args.cuda:
            vecs = vecs.cuda()
        embedding = embedder(vecs)
        batches += embedding.cpu().numpy().tolist()

np.savez(args.output, embeddings=batches, text=as_list)
<commit_msg>make reader more flexible<commit_after>import argparse
import baseline
import sys
from baseline.embeddings import load_embeddings_overlay
from eight_mile.utils import read_config_stream
from baseline.pytorch.embeddings import *
from baseline.embeddings import *
from baseline.vectorizers import *
from baseline.vectorizers import BPEVectorizer1D, WordpieceVectorizer1D
from mead.utils import convert_path, parse_extra_args, configure_logger, parse_and_merge_overrides, read_config_file_or_json, index_by_label
import pandas as pd
logger = logging.getLogger(__file__)
DEFAULT_SETTINGS_LOC = 'config/mead-settings.json'
DEFAULT_DATASETS_LOC = 'config/datasets.json'
DEFAULT_LOGGING_LOC = 'config/logging.json'
DEFAULT_EMBEDDINGS_LOC = 'config/embeddings.json'
DEFAULT_VECTORIZERS_LOC = 'config/vecs.json'

parser = argparse.ArgumentParser(description='Encode a sentence as an embedding')
parser.add_argument('--subword_model_file', help='Subword model file')
parser.add_argument('--nctx', default=256, type=int)
parser.add_argument('--batchsz', default=20, type=int)
parser.add_argument('--vec_id', default='bert-base-uncased', help='Reference to a specific embedding type')
parser.add_argument('--embed_id', default='bert-base-uncased-pooled', help='What type of embeddings to use')
parser.add_argument('--file', required=True)
parser.add_argument('--column', required=True)
parser.add_argument('--output', default='embeddings.npz')
parser.add_argument('--pool', help='Should a reduction be applied on the embeddings?', choices=['cls', 'max', 'mean'], default='cls')
parser.add_argument('--max_len1d', type=int, default=100)
parser.add_argument('--embeddings', help='index of embeddings: local file, remote URL or mead-ml/hub ref', type=convert_path)
parser.add_argument('--vecs', help='index of vectorizers: local file, remote URL or hub mead-ml/ref', type=convert_path)
parser.add_argument('--cuda', type=baseline.str2bool, default=True)
parser.add_argument('--has_header', type=baseline.str2bool, default=True)
parser.add_argument('--sep', default='\t')

args = parser.parse_args()

args.embeddings = convert_path(DEFAULT_EMBEDDINGS_LOC) if args.embeddings is None else args.embeddings
args.embeddings = read_config_stream(args.embeddings)

args.vecs = convert_path(DEFAULT_VECTORIZERS_LOC) if args.vecs is None else args.vecs

vecs_index = read_config_stream(args.vecs)
vecs_set = index_by_label(vecs_index)
vec_params = vecs_set[args.vec_id]
vec_params['mxlen'] = args.nctx

if 'transform_fn' in vec_params and isinstance(vec_params['transform_fn'], str):
    vec_params['transform_fn'] = eval(vec_params['transform_fn'])

vectorizer = create_vectorizer(**vec_params)

embeddings_index = read_config_stream(args.embeddings)
embeddings_set = index_by_label(embeddings_index)
embeddings_params = embeddings_set[args.embed_id]
embeddings = load_embeddings_overlay(embeddings_set, embeddings_params, vectorizer.vocab)

vocabs = {'x': embeddings['vocab']}
embedder = embeddings['embeddings'].cpu()
if args.cuda:
    embedder = embedder.cuda()

def chunks(lst, n):
    """Yield successive n-sized chunks from lst."""
    for i in range(0, len(lst), n):
        yield lst[i:i + n]

df = pd.read_csv(args.file, header='infer' if args.has_header else None, sep=args.sep)
if not args.has_header:
    col = df[int(args.column)]
else:
    col = df[args.column]
batches = []
as_list = col.tolist()
num_batches = math.ceil(len(as_list) / args.batchsz)
pg = baseline.create_progress_bar(num_batches, name='tqdm')
for i, batch in enumerate(chunks(as_list, args.batchsz)):
    pg.update()
    with torch.no_grad():
        vecs = []
        for line in batch:
            vec, l = vectorizer.run(line.split(), vocabs['x'])
            vecs.append(vec)
        vecs = torch.tensor(np.stack(vecs))
        if args.cuda:
            vecs = vecs.cuda()
        embedding = embedder(vecs)
        batches += embedding.cpu().numpy().tolist()

np.savez(args.output, embeddings=batches, text=as_list)
<|endoftext|>


<commit_before><commit_msg>Implement stdin as INFILE<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# The pam components have been modified to be salty and have been taken from
# the pam module under this licence:
# (c) 2007 Chris AtLee <chris@atlee.ca>
# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license.php
'''
Authenticate against PAM

Provides an authenticate function that will allow the caller to authenticate
a user against the Pluggable Authentication Modules (PAM) on the system.

Implemented using ctypes, so no compilation is necessary.

.. note:: PAM authentication will not work for the ``root`` user.

    The Python interface to PAM does not support authenticating as ``root``.

'''

# Import python libs
from ctypes import CDLL, POINTER, Structure, CFUNCTYPE, cast, pointer, sizeof
from ctypes import c_void_p, c_uint, c_char_p, c_char, c_int
from ctypes.util import find_library

# Import Salt libs
from salt.utils import get_group_list

LIBPAM = CDLL(find_library('pam'))
LIBC = CDLL(find_library('c'))

CALLOC = LIBC.calloc
CALLOC.restype = c_void_p
CALLOC.argtypes = [c_uint, c_uint]

STRDUP = LIBC.strdup
STRDUP.argstypes = [c_char_p]
STRDUP.restype = POINTER(c_char)  # NOT c_char_p !!!!

# Various constants
PAM_PROMPT_ECHO_OFF = 1
PAM_PROMPT_ECHO_ON = 2
PAM_ERROR_MSG = 3
PAM_TEXT_INFO = 4

class PamHandle(Structure):
    '''
    Wrapper class for pam_handle_t
    '''
    _fields_ = [
            ('handle', c_void_p)
            ]

    def __init__(self):
        Structure.__init__(self)
        self.handle = 0

class PamMessage(Structure):
    '''
    Wrapper class for pam_message structure
    '''
    _fields_ = [
            ("msg_style", c_int),
            ("msg", c_char_p),
            ]

    def __repr__(self):
        return "<PamMessage %i '%s'>" % (self.msg_style, self.msg)

class PamResponse(Structure):
    '''
    Wrapper class for pam_response structure
    '''
    _fields_ = [
            ('resp', c_char_p),
            ('resp_retcode', c_int),
            ]

    def __repr__(self):
        return "<PamResponse %i '%s'>" % (self.resp_retcode, self.resp)

CONV_FUNC = CFUNCTYPE(c_int,
        c_int, POINTER(POINTER(PamMessage)),
               POINTER(POINTER(PamResponse)), c_void_p)

class PamConv(Structure):
    '''
    Wrapper class for pam_conv structure
    '''
    _fields_ = [
            ('conv', CONV_FUNC),
            ('appdata_ptr', c_void_p)
            ]

try:
    PAM_START = LIBPAM.pam_start
    PAM_START.restype = c_int
    PAM_START.argtypes = [c_char_p, c_char_p, POINTER(PamConv),
            POINTER(PamHandle)]

    PAM_AUTHENTICATE = LIBPAM.pam_authenticate
    PAM_AUTHENTICATE.restype = c_int
    PAM_AUTHENTICATE.argtypes = [PamHandle, c_int]
except Exception:
    HAS_PAM = False
else:
    HAS_PAM = True

def __virtual__():
    '''
    Only load on Linux systems
    '''
    return HAS_PAM

def authenticate(username, password, service='login'):
    '''
    Returns True if the given username and password authenticate for the
    given service.  Returns False otherwise

    ``username``: the username to authenticate

    ``password``: the password in plain text

    ``service``: the PAM service to authenticate against.
                 Defaults to 'login'
    '''
    @CONV_FUNC
    def my_conv(n_messages, messages, p_response, app_data):
        '''
        Simple conversation function that responds to any
        prompt where the echo is off with the supplied password
        '''
        # Create an array of n_messages response objects
        addr = CALLOC(n_messages, sizeof(PamResponse))
        p_response[0] = cast(addr, POINTER(PamResponse))
        for i in range(n_messages):
            if messages[i].contents.msg_style == PAM_PROMPT_ECHO_OFF:
                pw_copy = STRDUP(str(password))
                p_response.contents[i].resp = cast(pw_copy, c_char_p)
                p_response.contents[i].resp_retcode = 0
        return 0

    handle = PamHandle()
    conv = PamConv(my_conv, 0)
    retval = PAM_START(service, username, pointer(conv), pointer(handle))

    if retval != 0:
        # TODO: This is not an authentication error, something
        # has gone wrong starting up PAM
        return False

    retval = PAM_AUTHENTICATE(handle, 0)
    return retval == 0

def auth(username, password, **kwargs):
    '''
    Authenticate via pam
    '''
    return authenticate(username, password, kwargs.get('service', 'login'))

def groups(username, *args, **kwargs):
    '''
    Retreive groups for a given user for this auth provider

    Uses system groups
    '''
    return get_group_list(username)
<commit_msg>Replace string substitution with string formatting<commit_after># -*- coding: utf-8 -*-
# The pam components have been modified to be salty and have been taken from
# the pam module under this licence:
# (c) 2007 Chris AtLee <chris@atlee.ca>
# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license.php
'''
Authenticate against PAM

Provides an authenticate function that will allow the caller to authenticate
a user against the Pluggable Authentication Modules (PAM) on the system.

Implemented using ctypes, so no compilation is necessary.

.. note:: PAM authentication will not work for the ``root`` user.

    The Python interface to PAM does not support authenticating as ``root``.

'''

# Import python libs
from ctypes import CDLL, POINTER, Structure, CFUNCTYPE, cast, pointer, sizeof
from ctypes import c_void_p, c_uint, c_char_p, c_char, c_int
from ctypes.util import find_library

# Import Salt libs
from salt.utils import get_group_list

LIBPAM = CDLL(find_library('pam'))
LIBC = CDLL(find_library('c'))

CALLOC = LIBC.calloc
CALLOC.restype = c_void_p
CALLOC.argtypes = [c_uint, c_uint]

STRDUP = LIBC.strdup
STRDUP.argstypes = [c_char_p]
STRDUP.restype = POINTER(c_char)  # NOT c_char_p !!!!

# Various constants
PAM_PROMPT_ECHO_OFF = 1
PAM_PROMPT_ECHO_ON = 2
PAM_ERROR_MSG = 3
PAM_TEXT_INFO = 4

class PamHandle(Structure):
    '''
    Wrapper class for pam_handle_t
    '''
    _fields_ = [
            ('handle', c_void_p)
            ]

    def __init__(self):
        Structure.__init__(self)
        self.handle = 0

class PamMessage(Structure):
    '''
    Wrapper class for pam_message structure
    '''
    _fields_ = [
            ("msg_style", c_int),
            ("msg", c_char_p),
            ]

    def __repr__(self):
        return '<PamMessage {0} {1!r}>'.format(self.msg_style, self.msg)

class PamResponse(Structure):
    '''
    Wrapper class for pam_response structure
    '''
    _fields_ = [
            ('resp', c_char_p),
            ('resp_retcode', c_int),
            ]

    def __repr__(self):
        return '<PamResponse {0} {1!r}>'.format(self.resp_retcode, self.resp)

CONV_FUNC = CFUNCTYPE(c_int,
        c_int, POINTER(POINTER(PamMessage)),
               POINTER(POINTER(PamResponse)), c_void_p)

class PamConv(Structure):
    '''
    Wrapper class for pam_conv structure
    '''
    _fields_ = [
            ('conv', CONV_FUNC),
            ('appdata_ptr', c_void_p)
            ]

try:
    PAM_START = LIBPAM.pam_start
    PAM_START.restype = c_int
    PAM_START.argtypes = [c_char_p, c_char_p, POINTER(PamConv),
            POINTER(PamHandle)]

    PAM_AUTHENTICATE = LIBPAM.pam_authenticate
    PAM_AUTHENTICATE.restype = c_int
    PAM_AUTHENTICATE.argtypes = [PamHandle, c_int]
except Exception:
    HAS_PAM = False
else:
    HAS_PAM = True

def __virtual__():
    '''
    Only load on Linux systems
    '''
    return HAS_PAM

def authenticate(username, password, service='login'):
    '''
    Returns True if the given username and password authenticate for the
    given service.  Returns False otherwise

    ``username``: the username to authenticate

    ``password``: the password in plain text

    ``service``: the PAM service to authenticate against.
                 Defaults to 'login'
    '''
    @CONV_FUNC
    def my_conv(n_messages, messages, p_response, app_data):
        '''
        Simple conversation function that responds to any
        prompt where the echo is off with the supplied password
        '''
        # Create an array of n_messages response objects
        addr = CALLOC(n_messages, sizeof(PamResponse))
        p_response[0] = cast(addr, POINTER(PamResponse))
        for i in range(n_messages):
            if messages[i].contents.msg_style == PAM_PROMPT_ECHO_OFF:
                pw_copy = STRDUP(str(password))
                p_response.contents[i].resp = cast(pw_copy, c_char_p)
                p_response.contents[i].resp_retcode = 0
        return 0

    handle = PamHandle()
    conv = PamConv(my_conv, 0)
    retval = PAM_START(service, username, pointer(conv), pointer(handle))

    if retval != 0:
        # TODO: This is not an authentication error, something
        # has gone wrong starting up PAM
        return False

    retval = PAM_AUTHENTICATE(handle, 0)
    return retval == 0

def auth(username, password, **kwargs):
    '''
    Authenticate via pam
    '''
    return authenticate(username, password, kwargs.get('service', 'login'))

def groups(username, *args, **kwargs):
    '''
    Retreive groups for a given user for this auth provider

    Uses system groups
    '''
    return get_group_list(username)
<|endoftext|>


<commit_before>from six.moves import range
from cloudvolume.threaded_queue import ThreadedQueue
from functools import partial

def test_threading():
  execution_count = 1000
  executions = []

  def reset_executions():
    return [ False for _ in range(execution_count) ]

  def addone(idnum, should_be_none):
    executions[idnum] = True
    assert should_be_none is None

  executions = reset_executions()

  with ThreadedQueue(n_threads=1) as tq:
    for idnum in range(execution_count):
      fn = partial(addone, idnum)
      tq.put(fn)
  assert all(executions)

  executions = reset_executions()
  tq = ThreadedQueue(n_threads=40)
  for idnum in range(execution_count):
    fn = partial(addone, idnum)
    tq.put(fn)
  tq.wait().kill_threads()
  assert tq.processed == execution_count
  assert all(executions)

  # Base class with 0 threads on with statement will never terminate
  try:
    with ThreadedQueue(n_threads=0) as tq:
      assert False
  except ValueError:
    assert True
  except Exception:
    assert False

def test_derived_class():

  def what_fun(should_be_fun):
    assert should_be_fun == 'fun'

  class DerivedThreadedQueue(ThreadedQueue):
    def _initialize_interface(self):
      return 'fun'

  with DerivedThreadedQueue(n_threads=1) as tq:
    for _ in range(1000):
      tq.put(what_fun)

    tq.wait()
    assert tq.processed == 1000

  # shouldn't crash w/ 0 threads because it's a derived class
  with DerivedThreadedQueue(n_threads=0) as tq:
    pass

def test_threads_die():
  tq = ThreadedQueue(n_threads=40)
  assert tq.are_threads_alive()
  tq.kill_threads()
  assert not tq.are_threads_alive()

  tq = ThreadedQueue(n_threads=0)
  assert not tq.are_threads_alive()

  with ThreadedQueue(n_threads=40) as tq:
    threads = tq._threads

  assert not any(map(lambda t: t.isAlive(), threads))

def test_thread_exceptions():

  def diediedie(interface):
    raise NotImplementedError("Not implemented at all.")

  tq = ThreadedQueue(n_threads=40)
  for _ in range(1000):
    tq.put(diediedie)

  try:
    tq.wait()
  except NotImplementedError:
    pass

<commit_msg>changing spelling of isAlive>is_alive (#300)<commit_after>from six.moves import range
from cloudvolume.threaded_queue import ThreadedQueue
from functools import partial

def test_threading():
  execution_count = 1000
  executions = []

  def reset_executions():
    return [ False for _ in range(execution_count) ]

  def addone(idnum, should_be_none):
    executions[idnum] = True
    assert should_be_none is None

  executions = reset_executions()

  with ThreadedQueue(n_threads=1) as tq:
    for idnum in range(execution_count):
      fn = partial(addone, idnum)
      tq.put(fn)
  assert all(executions)

  executions = reset_executions()
  tq = ThreadedQueue(n_threads=40)
  for idnum in range(execution_count):
    fn = partial(addone, idnum)
    tq.put(fn)
  tq.wait().kill_threads()
  assert tq.processed == execution_count
  assert all(executions)

  # Base class with 0 threads on with statement will never terminate
  try:
    with ThreadedQueue(n_threads=0) as tq:
      assert False
  except ValueError:
    assert True
  except Exception:
    assert False

def test_derived_class():

  def what_fun(should_be_fun):
    assert should_be_fun == 'fun'

  class DerivedThreadedQueue(ThreadedQueue):
    def _initialize_interface(self):
      return 'fun'

  with DerivedThreadedQueue(n_threads=1) as tq:
    for _ in range(1000):
      tq.put(what_fun)

    tq.wait()
    assert tq.processed == 1000

  # shouldn't crash w/ 0 threads because it's a derived class
  with DerivedThreadedQueue(n_threads=0) as tq:
    pass

def test_threads_die():
  tq = ThreadedQueue(n_threads=40)
  assert tq.are_threads_alive()
  tq.kill_threads()
  assert not tq.are_threads_alive()

  tq = ThreadedQueue(n_threads=0)
  assert not tq.are_threads_alive()

  with ThreadedQueue(n_threads=40) as tq:
    threads = tq._threads

  assert not any(map(lambda t: t.is_alive(), threads))

def test_thread_exceptions():

  def diediedie(interface):
    raise NotImplementedError("Not implemented at all.")

  tq = ThreadedQueue(n_threads=40)
  for _ in range(1000):
    tq.put(diediedie)

  try:
    tq.wait()
  except NotImplementedError:
    pass

<|endoftext|>


<commit_before>"""Create table of contents from multiple files

Usage:

    pandoc --number-sections --file-scope --toc -s *.md | pandoc -f html -o toc.html -F fixtoc.py -M files:"*.md"
"""

import glob
import panflute as pf

def prepare(doc):
    # Create file list
    fns = doc.get_metadata('files')
    pf.debug('-' * 64)
    pf.debug('Expanding pattern:', fns)
    fns = glob.glob(fns)
    pf.debug('Files:', fns)
    fns = [fn.replace('.md', '.html') for fn in fns]
    doc.fns = fns
    pf.debug('-' * 64)

    # Clear all contents except TOC
    doc.content.list = doc.content.list[:1]

def action(e, doc):
    if isinstance(e, pf.Div) and e.identifier=='TOC':
        e.walk(fix_links)

def fix_links(e, doc):
    if isinstance(e, pf.Link):
        e.url = doc.fns[0] + e.url
    elif isinstance(e, pf.ListItem) and isinstance(e.parent.parent, pf.Div):
        doc.fns.pop(0)  # Switch to the next filename

if __name__ == '__main__':
    pf.run_filter(action, prepare=prepare)
<commit_msg>Update fixtoc.py<commit_after>"""Create table of contents from multiple files

Usage:

    pandoc --number-sections --file-scope --toc -s *.md | pandoc -s -f html -o toc.html -F fixtoc.py -M files:"*.md"
"""

import glob
import panflute as pf

def prepare(doc):
    # Create file list
    fns = doc.get_metadata('files')
    pf.debug('-' * 64)
    pf.debug('Expanding pattern:', fns)
    fns = glob.glob(fns)
    pf.debug('Files:', fns)
    fns = [fn.replace('.md', '.html') for fn in fns]
    doc.fns = fns
    pf.debug('-' * 64)

    # Clear all contents except TOC
    doc.content.list = doc.content.list[:1]

def action(e, doc):
    if isinstance(e, pf.Div) and e.identifier=='TOC':
        e.walk(fix_links)

def fix_links(e, doc):
    if isinstance(e, pf.Link):
        e.url = doc.fns[0] + e.url
    elif isinstance(e, pf.ListItem) and isinstance(e.parent.parent, pf.Div):
        doc.fns.pop(0)  # Switch to the next filename

if __name__ == '__main__':
    pf.run_filter(action, prepare=prepare)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from flask import Blueprint, render_template, g, request
from dataviva.apps.general.views import get_locale
from dataviva.api.secex.services import Product as ProductService
from dataviva.api.secex.services import ProductTradePartners as ProductTradePartnersService
from dataviva.api.secex.services import ProductMunicipalities as ProductMunicipalitiesService
from dataviva.api.secex.services import ProductLocations as ProductLocationsService

mod = Blueprint('product', __name__,
                template_folder='templates',
                url_prefix='/<lang_code>/product',
                static_folder='static')

@mod.url_value_preprocessor
def pull_lang_code(endpoint, values):
    g.locale = values.pop('lang_code')

@mod.url_defaults
def add_language_code(endpoint, values):
    values.setdefault('lang_code', get_locale())

@mod.route('/<product_id>')
def index(product_id):

    bra_id = request.args.get('bra_id')

    #None database fields must be treated (/05?bra_id=2ce020008)
    #and templates with no data shall be omitted...

    #Vars to do tests:
    #section (depth == 2)
    #positon (depth == 6)
    #brazil (bra_id == None)
    #product_id = #05 #052601
    #bra_id = #None #4mg #4mg01 #4mg0000 #4mg010206 #2ce020008

    context = {
        'background_image':'static/img/bg-profile-location.jpg',
        'portrait':'static/img/mineric_product.jpg',
        'desc_general': 'Sample Text',
        'desc_international_trade': 'Sample Text',
        'desc_economic_opp': 'Sample Text'
    }

    context['bra_id'] = bra_id
    if bra_id:
        context['bra_id_len'] = len(bra_id)
    context['depth'] = len(product_id)

    header = {}
    body = {}

    trade_partners_service = ProductTradePartnersService(product_id=product_id, bra_id=bra_id)
    municipalities_service = ProductMunicipalitiesService(product_id=product_id, bra_id=bra_id)

    if len(product_id) == 6:
            product_service = ProductService(product_id=product_id)
            header['pci'] = product_service.complexity()

    if bra_id:
        product_service = ProductLocationsService(product_id=product_id, bra_id=bra_id)

        body['destination_name_export'] = trade_partners_service.destination_with_more_exports()
        body['destination_export_value'] = trade_partners_service.highest_export_value()
        body['origin_name_import'] = trade_partners_service.origin_with_more_imports()
        body['origin_import_value'] = trade_partners_service.highest_import_value()

        if len(product_id) == 6:
            header['rca_wld'] = product_service.rca_wld()
            header['distance_wld'] = product_service.distance_wld()
            header['opp_gain_wld'] = product_service.opp_gain_wld()

        if len(bra_id) != 9:
            body['municipality_name_export'] = municipalities_service.municipality_with_more_exports()
            body['municipality_export_value'] = municipalities_service.highest_export_value()
            body['municipality_name_import'] = municipalities_service.municipality_with_more_imports()
            body['municipality_import_value'] = municipalities_service.highest_import_value()

    else:
        product_service = ProductService(product_id=product_id)

        body['municipality_name_export'] = municipalities_service.municipality_with_more_exports()
        body['municipality_export_value'] = municipalities_service.highest_export_value()
        body['municipality_name_import'] = municipalities_service.municipality_with_more_imports()
        body['municipality_import_value'] = municipalities_service.highest_import_value()
        body['destination_name_export'] = trade_partners_service.destination_with_more_exports()
        body['destination_export_value'] = trade_partners_service.highest_export_value()
        body['origin_name_import'] = trade_partners_service.origin_with_more_imports()
        body['origin_import_value'] = trade_partners_service.highest_import_value()

    header['name'] = product_service.product_name()
    header['year'] = product_service.year()
    header['trade_balance'] = product_service.trade_balance()
    header['export_val'] = product_service.total_exported()
    header['export_net_weight'] = product_service.unity_weight_export_price()
    header['import_val'] = product_service.total_imported()
    header['import_net_weight'] = product_service.unity_weight_import_price()

    return render_template('product/index.html', body_class='perfil-estado', header=header, body=body, context=context)<commit_msg>Adjusting name method to get product complexity.<commit_after># -*- coding: utf-8 -*-
from flask import Blueprint, render_template, g, request
from dataviva.apps.general.views import get_locale
from dataviva.api.secex.services import Product as ProductService
from dataviva.api.secex.services import ProductTradePartners as ProductTradePartnersService
from dataviva.api.secex.services import ProductMunicipalities as ProductMunicipalitiesService
from dataviva.api.secex.services import ProductLocations as ProductLocationsService

mod = Blueprint('product', __name__,
                template_folder='templates',
                url_prefix='/<lang_code>/product',
                static_folder='static')

@mod.url_value_preprocessor
def pull_lang_code(endpoint, values):
    g.locale = values.pop('lang_code')

@mod.url_defaults
def add_language_code(endpoint, values):
    values.setdefault('lang_code', get_locale())

@mod.route('/<product_id>')
def index(product_id):

    bra_id = request.args.get('bra_id')

    #None database fields must be treated (/05?bra_id=2ce020008)
    #and templates with no data shall be omitted...

    #Vars to do tests:
    #section (depth == 2)
    #positon (depth == 6)
    #brazil (bra_id == None)
    #product_id = #05 #052601
    #bra_id = #None #4mg #4mg01 #4mg0000 #4mg010206 #2ce020008

    context = {
        'background_image':'static/img/bg-profile-location.jpg',
        'portrait':'static/img/mineric_product.jpg',
        'desc_general': 'Sample Text',
        'desc_international_trade': 'Sample Text',
        'desc_economic_opp': 'Sample Text'
    }

    context['bra_id'] = bra_id
    if bra_id:
        context['bra_id_len'] = len(bra_id)
    context['depth'] = len(product_id)

    header = {}
    body = {}

    trade_partners_service = ProductTradePartnersService(product_id=product_id, bra_id=bra_id)
    municipalities_service = ProductMunicipalitiesService(product_id=product_id, bra_id=bra_id)

    if len(product_id) == 6:
            product_service = ProductService(product_id=product_id)
            header['pci'] = product_service.product_complexity()

    if bra_id:
        product_service = ProductLocationsService(product_id=product_id, bra_id=bra_id)

        body['destination_name_export'] = trade_partners_service.destination_with_more_exports()
        body['destination_export_value'] = trade_partners_service.highest_export_value()
        body['origin_name_import'] = trade_partners_service.origin_with_more_imports()
        body['origin_import_value'] = trade_partners_service.highest_import_value()

        if len(product_id) == 6:
            header['rca_wld'] = product_service.rca_wld()
            header['distance_wld'] = product_service.distance_wld()
            header['opp_gain_wld'] = product_service.opp_gain_wld()

        if len(bra_id) != 9:
            body['municipality_name_export'] = municipalities_service.municipality_with_more_exports()
            body['municipality_export_value'] = municipalities_service.highest_export_value()
            body['municipality_name_import'] = municipalities_service.municipality_with_more_imports()
            body['municipality_import_value'] = municipalities_service.highest_import_value()

    else:
        product_service = ProductService(product_id=product_id)

        body['municipality_name_export'] = municipalities_service.municipality_with_more_exports()
        body['municipality_export_value'] = municipalities_service.highest_export_value()
        body['municipality_name_import'] = municipalities_service.municipality_with_more_imports()
        body['municipality_import_value'] = municipalities_service.highest_import_value()
        body['destination_name_export'] = trade_partners_service.destination_with_more_exports()
        body['destination_export_value'] = trade_partners_service.highest_export_value()
        body['origin_name_import'] = trade_partners_service.origin_with_more_imports()
        body['origin_import_value'] = trade_partners_service.highest_import_value()

    header['name'] = product_service.product_name()
    header['year'] = product_service.year()
    header['trade_balance'] = product_service.trade_balance()
    header['export_val'] = product_service.total_exported()
    header['export_net_weight'] = product_service.unity_weight_export_price()
    header['import_val'] = product_service.total_imported()
    header['import_net_weight'] = product_service.unity_weight_import_price()

    return render_template('product/index.html', body_class='perfil-estado', header=header, body=body, context=context)<|endoftext|>


<commit_before>import shelve
import pdb
import re
try:
    import cPickle as pickle
except:
    import pickle
from nltk.corpus import stopwords
from Commons import *

HASH1 = 'Hash#1.shelve'

#word_data = dict()
#sense_data = dict()

def word_processing(wd_filename):
    #word_data = dict()
    word_data = Shelveopen(wd_filename, writeback=True)
    hash1 = Shelveopen(HASH1)
    hash2 = Shelveopen('Hash#2.shelve')
    count = 0
    #try:
    for w in hash1.keys():
        print w
        word_data[w] = dict()
        word_data[w]['W2S'] = list()
        word_ = hash1[w]
        #word_data[w]['D2S'] = dict()
        for i in word_.Noun_synsets() + word_.Verb_synsets() \
            + word_.Adj_synsets() + word_.Adv_synsets():
            if i in hash2:
                if w in hash2[i].lemma_names():
                    freq = hash2[i].lemma_count()[hash2[i].lemma_names().index(w)]
                else:
                    print '&&&&&&&&&&&&&&&&word not found in lemma_names&&&&&&&&&&&&&&'
                    try:
                        word_hash = Shelveopen('Word#.shelve')
                    except:
                        print 'cannot open word_hash'
                    w_mod = word_hash[w]
                    print w_mod
                    print '**************************************************'
                    Shelveclose(word_hash)
                    try:
                        freq = hash2[i].lemma_count()[hash2[i].lemma_names().index(w_mod)]
                    except:
                        '''word_hash does not yet support all kinds of variations eg. 'gripes' -> 'gripe'; ignore for now'''
                        pass

                total_freq = 10
                word_data[w]['W2S'].append((i,freq,total_freq))
            else:
                print 'word_processing : ' + str(i) + ' not found in hash2'

        count += 1
    '''
    except TypeError as e:
        print 'TypeError : ' + str(e)
    except KeyError as e:
        print 'KeyError : ' + str(e)
    except Exception as e:
        print e
    '''
    Shelveclose(hash1)
    Shelveclose(hash2)
    print 'word_processing() - Words processed : ' + str(count)
    #pickle.dump( word_data, open( wd_filename, "wb" ) )
    Shelveclose(word_data)

def get_words(input_str,size_):
    '''
        strip words from sentence acc to value of 'size_'
        check word existence in wordnet data
    '''
    hash1 = Shelveopen(HASH1)
    input_tokens = [x.lower() for x in input_str.split()]
    if size_ == 1:
        for tkn in input_tokens:
            if not hash1.has_key(tkn):
                input_tokens.remove(tkn)
        return input_tokens
    if size_ == 2:
        tokens = list()
        for i in range(0,len(input_tokens)-1):
            val = str(input_tokens[i:i+size_][0]) + str('_') + str(input_tokens[i:i+size_][1])
            if hash1.has_key(val):
                tokens.append(val)
        return tokens

def number_non_noise_words(synset_defn):
    '''
        return the number of non noise words in synset_defn
    '''
    '''
    cnt = 0
    for word in synset_defn.split():
        if word not in stopwords.words('english'):
            cnt += 1
    return cnt
    '''
    return len(synset_defn.split())

def count_words(word, input_str):
    if '_' in word:
        new_word = word.replace('_', ' ')
        return sum(1 for _ in re.finditer(r'\b%s\b' % re.escape(new_word), input_str))
    else:
        return sum(1 for _ in re.finditer(r'\b%s\b' % re.escape(word), input_str))

def sense_to_word_processing(sd_filename,string,synset_name,tokens,word_type):
    '''
        sense - definition words
        sense - example words, processing
    '''
    sense_data = Shelveopen(sd_filename)
    hash1 = Shelveopen(HASH1)
    for tkn in tokens:
        if(word_type == 'definition'):
            sense_data[synset_name]['S2D'].append((tkn, count_words(tkn,string), number_non_noise_words(string)))
        elif(word_type == 'example'):
            sense_data[synset_name]['S2E'].append((tkn, count_words(tkn,string), number_non_noise_words(string)))
    Shelveclose(hash1)
    Shelveclose(sense_data)

def remove_overlap_words(tokens_1, tokens_2):
    final = list()
    flag = 0
    for tkn in tokens_1:
        for tkn2 in tokens_2:
            if tkn in tkn2:
                flag = 1
                break
        if flag == 0:
            final.append(tkn)
        flag = 0
    return final

def defn_word_to_sense_processing(synset_defn,synset,wd_filename,tokens_1,tokens_2):
    hash1 = Shelveopen(HASH1)
    #word_data = pickle.load( open(wd_filename,'rb'))
    word_data = Shelveopen(wd_filename)
    for word in tokens_1+tokens_2:
        if hash1.has_key(word):
            #-1 => total number of occurences of word in all nodes in the graph; compute ONLY THIS after all ops are done
            try:
                if word_data[word]:
                    word_data[word]['D2S'] = dict()
                    word_data[word]['D2S'][synset] = count_words(word, synset_defn)
                    if count_words(word, synset_defn) == 0:
                        print word + str(" : ") + synset_defn
                        print 'zero'
                else:
                    word_data[word] = dict()
                    word_data[word]['D2S'] = dict()
                    word_data[word]['D2S'][synset] = count_words(word, synset_defn)
                    if count_words(word, synset_defn) == 0:
                        print word + str(" : ") + synset_defn
                        print 'zero'
            except KeyError as e:
                print 'Defn_word_to_sense - KeyError - Word: ' + word + ' : not found in dict word_data'
            except Exception as e:
                if word not in word_data:
                    print 'Defn_word_to_sense -  Word - ' + word + ' : not found in dict word_data'
                else:
                    print 'defn_word_to_sense_processing : ' + str(e)
        else:
            #put to logs
            pass
    #pickle.dump( word_data, open( wd_filename, "wb" ) )
    Shelveclose(word_data)
    Shelveclose(hash1)

def update_words_frequency(tokens_1,tokens_2):
    '''
        word_freq - shelve dict
    '''
    word_freq = Shelveopen('word_freq.shelve')
    for word in tokens_1+tokens_2:
        if word in word_freq:
            v = word_freq[word] #Do not change; necessary to assign values to shelve variables
            v += 1
            word_freq[word] = v
        else:
            word_freq[word] = 1
    Shelveclose(word_freq)

def handle_error(sense_data,sd_filename):
    print 'something seriously f*cked up'
    #pickle.dump( sense_data, open( 'sense_data.pkl', "wb" ) )
    #pickle.dump( sense_data, open( sd_filename, "wb" ) )
    Shelveclose(sense_data)

def sense_processing(sd_filename,wd_filename):
    #sense_data = dict()
    #sense_data = Shelveopen(sd_filename)
    hash2 = Shelveopen('Hash#2.shelve')
    #try:
    for s in hash2.keys()[:300]:
        sense_data = Shelveopen(sd_filename)
        print s
        sense_data[s] = dict()
        synset_ = hash2[s]

        synset_defn = synset_.definition()

        #print 'synset_defn = ' + str(synset_defn)

        sense_data[s]['S2W'] = list()
        for i in synset_.lemma_names():
            sense_data[s]['S2W'].append(i)

        #TODO - remove non noise words in definition of sense before creating the 'sense-defn word' connections
        sense_data[s]['S2D'] = list()
        Shelveclose(sense_data)
        tokens_2 = get_words(synset_defn,2)
        if tokens_2:
            sense_to_word_processing(sd_filename=sd_filename,string=synset_defn,synset_name=s,tokens=tokens_2,word_type='definition')

        tokens_1 = get_words(synset_defn,1)
        tokens_1 = remove_overlap_words(tokens_1,tokens_2)

        if tokens_1:
            sense_to_word_processing(sd_filename=sd_filename,string=synset_defn,synset_name=s,tokens=tokens_1,word_type='definition')#change this - pass both tokens_1 and tokens_2 together
        #print "\t" + str(s) + str(" : ") + str('***************  sense_to_defn_word done  ************************')

        #update_words_frequency(tokens_1,tokens_2)###
        #if tokens_1 or tokens_2:
            #print '1'
        #    defn_word_to_sense_processing(synset_defn,s,wd_filename,tokens_1,tokens_2)
        #print "\t" + str(s) + str(" : ") + str('***************  defn_word_to_sense done  ************************')
        #print '2'
        sense_data = Shelveopen(sd_filename)
        sense_data[s]['S2E'] = list()
        Shelveclose(sense_data)
        if synset_.examples():
            #print 'inside examples'
            for example in synset_.examples():
                tokens_1 = get_words(example,1)
                sense_to_word_processing(sd_filename=sd_filename,string=example,synset_name=s,tokens=tokens_1,word_type='example')

        #print "\t" + str(s) + str(" : ") + str('***************  sense_to_example_word done  ************************')
        sense_data = Shelveopen(sd_filename)
        sense_data[s]['Hyponym'] = synset_.hyponyms()
        sense_data[s]['Hypernym'] = synset_.hypernyms()
        sense_data[s]['Holonym'] = synset_.holonyms()
        sense_data[s]['Meronym'] = synset_.meronyms()
        sense_data[s]['Entailment'] = synset_.entailments()
        sense_data[s]['Similar'] = synset_.similar_tos()
        Shelveclose(sense_data)
        #print "\t" + str(s) + str(" : ") + str('*************** sense-sense relations done  ************************')
    #except Exception as e:
    #    print "Exception : " + str(e)
    #    handle_error(sense_data,sd_filename)
    #pickle.dump( sense_data, open( sd_filename, "wb" ) )
    Shelveclose(hash2)
    #Shelveclose(sense_data)

def hotfix(wd_filename):
    word_data = Shelveopen(wd_filename)
    word_data_hotfix = Shelveopen('Shelves/Hash#3_hotfix.shelve')
    for i in word_data:
        if word_data[i].has_key('D2S'):
            word_data_hotfix[i] = word_data[i]

    Shelveclose(word_data)
    Shelveclose(word_data_hotfix)

if __name__ == "__main__":

    #pdb.set_trace()
    #word_processing(wd_filename='Hash#3.shelve')
    sense_processing(sd_filename='Hash#4.shelve', wd_filename='Hash#3.shelve')
    #hotfix(wd_filename='Hash#3.shelve')
<commit_msg>Minor fix<commit_after>import shelve
import pdb
import re
try:
    import cPickle as pickle
except:
    import pickle
from nltk.corpus import stopwords
from Commons import *

HASH1 = 'Hash#1.shelve'

#word_data = dict()
#sense_data = dict()

def word_processing(wd_filename):
    #word_data = dict()
    word_data = Shelveopen(wd_filename)
    hash1 = Shelveopen(HASH1)
    hash2 = Shelveopen('Hash#2.shelve')
    count = 0
    #try:
    for w in hash1.keys():
        print w
        word_data[w] = dict()
        word_data[w]['W2S'] = list()
        word_ = hash1[w]
        #word_data[w]['D2S'] = dict()
        for i in word_.Noun_synsets() + word_.Verb_synsets() \
            + word_.Adj_synsets() + word_.Adv_synsets():
            if i in hash2:
                if w in hash2[i].lemma_names():
                    freq = hash2[i].lemma_count()[hash2[i].lemma_names().index(w)]
                else:
                    print '&&&&&&&&&&&&&&&&word not found in lemma_names&&&&&&&&&&&&&&'
                    try:
                        word_hash = Shelveopen('Word#.shelve')
                    except:
                        print 'cannot open word_hash'
                    w_mod = word_hash[w]
                    print w_mod
                    print '**************************************************'
                    Shelveclose(word_hash)
                    try:
                        freq = hash2[i].lemma_count()[hash2[i].lemma_names().index(w_mod)]
                    except:
                        '''word_hash does not yet support all kinds of variations eg. 'gripes' -> 'gripe'; ignore for now'''
                        pass

                total_freq = 10
                word_data[w]['W2S'].append((i,freq,total_freq))
            else:
                print 'word_processing : ' + str(i) + ' not found in hash2'

        count += 1
    '''
    except TypeError as e:
        print 'TypeError : ' + str(e)
    except KeyError as e:
        print 'KeyError : ' + str(e)
    except Exception as e:
        print e
    '''
    Shelveclose(hash1)
    Shelveclose(hash2)
    print 'word_processing() - Words processed : ' + str(count)
    #pickle.dump( word_data, open( wd_filename, "wb" ) )
    Shelveclose(word_data)

def get_words(input_str,size_):
    '''
        strip words from sentence acc to value of 'size_'
        check word existence in wordnet data
    '''
    hash1 = Shelveopen(HASH1)
    input_tokens = [x.lower() for x in input_str.split()]
    if size_ == 1:
        for tkn in input_tokens:
            if not hash1.has_key(tkn):
                input_tokens.remove(tkn)
        return input_tokens
    if size_ == 2:
        tokens = list()
        for i in range(0,len(input_tokens)-1):
            val = str(input_tokens[i:i+size_][0]) + str('_') + str(input_tokens[i:i+size_][1])
            if hash1.has_key(val):
                tokens.append(val)
        return tokens

def number_non_noise_words(synset_defn):
    '''
        return the number of non noise words in synset_defn
    '''
    '''
    cnt = 0
    for word in synset_defn.split():
        if word not in stopwords.words('english'):
            cnt += 1
    return cnt
    '''
    return len(synset_defn.split())

def count_words(word, input_str):
    if '_' in word:
        new_word = word.replace('_', ' ')
        return sum(1 for _ in re.finditer(r'\b%s\b' % re.escape(new_word), input_str))
    else:
        return sum(1 for _ in re.finditer(r'\b%s\b' % re.escape(word), input_str))

def sense_to_word_processing(sd_filename,string,synset_name,tokens,word_type):
    '''
        sense - definition words
        sense - example words, processing
    '''
    sense_data = Shelveopen(sd_filename)
    hash1 = Shelveopen(HASH1)
    for tkn in tokens:
        if(word_type == 'definition'):
            sense_data[synset_name]['S2D'].append((tkn, count_words(tkn,string), number_non_noise_words(string)))
        elif(word_type == 'example'):
            sense_data[synset_name]['S2E'].append((tkn, count_words(tkn,string), number_non_noise_words(string)))
    Shelveclose(hash1)
    Shelveclose(sense_data)

def remove_overlap_words(tokens_1, tokens_2):
    final = list()
    flag = 0
    for tkn in tokens_1:
        for tkn2 in tokens_2:
            if tkn in tkn2:
                flag = 1
                break
        if flag == 0:
            final.append(tkn)
        flag = 0
    return final

def defn_word_to_sense_processing(synset_defn,synset,wd_filename,tokens_1,tokens_2):
    hash1 = Shelveopen(HASH1)
    #word_data = pickle.load( open(wd_filename,'rb'))
    word_data = Shelveopen(wd_filename)
    for word in tokens_1+tokens_2:
        if hash1.has_key(word):
            #-1 => total number of occurences of word in all nodes in the graph; compute ONLY THIS after all ops are done
            try:
                if word_data[word]:
                    word_data[word]['D2S'] = dict()
                    word_data[word]['D2S'][synset] = count_words(word, synset_defn)
                    if count_words(word, synset_defn) == 0:
                        print word + str(" : ") + synset_defn
                        print 'zero'
                else:
                    word_data[word] = dict()
                    word_data[word]['D2S'] = dict()
                    word_data[word]['D2S'][synset] = count_words(word, synset_defn)
                    if count_words(word, synset_defn) == 0:
                        print word + str(" : ") + synset_defn
                        print 'zero'
            except KeyError as e:
                print 'Defn_word_to_sense - KeyError - Word: ' + word + ' : not found in dict word_data'
            except Exception as e:
                if word not in word_data:
                    print 'Defn_word_to_sense -  Word - ' + word + ' : not found in dict word_data'
                else:
                    print 'defn_word_to_sense_processing : ' + str(e)
        else:
            #put to logs
            pass
    #pickle.dump( word_data, open( wd_filename, "wb" ) )
    Shelveclose(word_data)
    Shelveclose(hash1)

def update_words_frequency(tokens_1,tokens_2):
    '''
        word_freq - shelve dict
    '''
    word_freq = Shelveopen('word_freq.shelve')
    for word in tokens_1+tokens_2:
        if word in word_freq:
            v = word_freq[word] #Do not change; necessary to assign values to shelve variables
            v += 1
            word_freq[word] = v
        else:
            word_freq[word] = 1
    Shelveclose(word_freq)

def handle_error(sense_data,sd_filename):
    print 'something seriously f*cked up'
    #pickle.dump( sense_data, open( 'sense_data.pkl', "wb" ) )
    #pickle.dump( sense_data, open( sd_filename, "wb" ) )
    Shelveclose(sense_data)

def sense_processing(sd_filename,wd_filename):
    #sense_data = dict()
    #sense_data = Shelveopen(sd_filename)
    hash2 = Shelveopen('Hash#2.shelve')
    #try:
    for s in hash2.keys()[:300]:
        sense_data = Shelveopen(sd_filename)
        print s
        sense_data[s] = dict()
        synset_ = hash2[s]

        synset_defn = synset_.definition()

        #print 'synset_defn = ' + str(synset_defn)

        sense_data[s]['S2W'] = list()
        for i in synset_.lemma_names():
            sense_data[s]['S2W'].append(i)

        #TODO - remove non noise words in definition of sense before creating the 'sense-defn word' connections
        sense_data[s]['S2D'] = list()
        Shelveclose(sense_data)
        tokens_2 = get_words(synset_defn,2)
        if tokens_2:
            sense_to_word_processing(sd_filename=sd_filename,string=synset_defn,synset_name=s,tokens=tokens_2,word_type='definition')

        tokens_1 = get_words(synset_defn,1)
        tokens_1 = remove_overlap_words(tokens_1,tokens_2)

        if tokens_1:
            sense_to_word_processing(sd_filename=sd_filename,string=synset_defn,synset_name=s,tokens=tokens_1,word_type='definition')#change this - pass both tokens_1 and tokens_2 together
        #print "\t" + str(s) + str(" : ") + str('***************  sense_to_defn_word done  ************************')

        #update_words_frequency(tokens_1,tokens_2)###
        #if tokens_1 or tokens_2:
            #print '1'
        #    defn_word_to_sense_processing(synset_defn,s,wd_filename,tokens_1,tokens_2)
        #print "\t" + str(s) + str(" : ") + str('***************  defn_word_to_sense done  ************************')
        #print '2'
        sense_data = Shelveopen(sd_filename)
        sense_data[s]['S2E'] = list()
        Shelveclose(sense_data)
        if synset_.examples():
            #print 'inside examples'
            for example in synset_.examples():
                tokens_1 = get_words(example,1)
                sense_to_word_processing(sd_filename=sd_filename,string=example,synset_name=s,tokens=tokens_1,word_type='example')

        #print "\t" + str(s) + str(" : ") + str('***************  sense_to_example_word done  ************************')
        sense_data = Shelveopen(sd_filename)
        sense_data[s]['Hyponym'] = synset_.hyponyms()
        sense_data[s]['Hypernym'] = synset_.hypernyms()
        sense_data[s]['Holonym'] = synset_.holonyms()
        sense_data[s]['Meronym'] = synset_.meronyms()
        sense_data[s]['Entailment'] = synset_.entailments()
        sense_data[s]['Similar'] = synset_.similar_tos()
        Shelveclose(sense_data)
        #print "\t" + str(s) + str(" : ") + str('*************** sense-sense relations done  ************************')
    #except Exception as e:
    #    print "Exception : " + str(e)
    #    handle_error(sense_data,sd_filename)
    #pickle.dump( sense_data, open( sd_filename, "wb" ) )
    Shelveclose(hash2)
    #Shelveclose(sense_data)

def hotfix(wd_filename):
    word_data = Shelveopen(wd_filename)
    word_data_hotfix = Shelveopen('Hash#3_hotfix.shelve')
    for i in word_data:
        if word_data[i].has_key('D2S'):
            word_data_hotfix[i] = word_data[i]

    Shelveclose(word_data)
    Shelveclose(word_data_hotfix)

if __name__ == "__main__":

    #pdb.set_trace()
    #word_processing(wd_filename='Hash#3.shelve')
    sense_processing(sd_filename='Hash#4.shelve', wd_filename='Hash#3.shelve')
    #hotfix(wd_filename='Hash#3.shelve')
<|endoftext|>