<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Copyright 2016 Guenter Bartsch
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#
# nlp dictionaries, predicate terms, numpy model transformation
#

import os
import sys
import logging

import numpy as np

import codecs

from speech_tokenizer import tokenize

from sqlalchemy.orm import sessionmaker
import model

import tensorflow as tf

from tensorflow.models.rnn.translate import seq2seq_model
from tensorflow.models.rnn.translate.data_utils import _PAD, PAD_ID, _GO, GO_ID, _EOS, EOS_ID, _UNK, UNK_ID

OUT_DICT_FN  = 'data/dst/nlp_out_dict.csv'
IN_DICT_FN   = 'data/dst/nlp_in_dict.csv'

CKPT_FN      = 'data/dst/nlp_model.ckpt'

# We use a number of buckets and pad to the closest one for efficiency.
# See seq2seq_model.Seq2SeqModel for details of how they work.
# BUCKETS = [(5, 10), (10, 15), (20, 25), (40, 50)]
BUCKETS = [(7, 4), (14, 8)]

# number of LSTM layers : 1/2/3
NUM_LAYERS = 1
# typical options : 128, 256, 512, 1024
LAYER_SIZE = 128

BATCH_SIZE = 64

LEARNING_RATE              = 0.5
LEARNING_RATE_DECAY_FACTOR = 0.99
MAX_GRADIENT_NORM          = 5.0
USE_LSTM                   = False
NUM_SAMPLES                = 32
FORWARD_ONLY               = False

class NLPModel(object):

    def __init__(self, session, input_dim = 16 ):
        self.session   = session
        self.input_dim = input_dim

    def compute_input_hist(self):

        hist = {}

        for dr in self.session.query(model.DiscourseRound).all():

            tokens = tokenize (dr.inp_tokenized)

            if not (len(tokens) in hist):
                hist[len(tokens)] = 0

            hist[len(tokens)] += 1

        return hist

    def compute_output_hist(self):

        hist = {}

        for dr in self.session.query(model.DiscourseRound).all():

            preds = dr.response.split(';')

            if not (len(preds) in hist):
                hist[len(preds)] = 0

            hist[len(preds)] += 1

        return hist

    def compute_dicts(self):

        # build input and output dicts

        self.input_dict  = {_PAD : PAD_ID, _GO : GO_ID, _EOS : EOS_ID, _UNK : UNK_ID}
        self.output_dict = {_PAD : PAD_ID, _GO : GO_ID, _EOS : EOS_ID, _UNK : UNK_ID}

        self.input_max_len  = 0
        self.output_max_len = 0

        self.num_segments = 0

        for dr in self.session.query(model.DiscourseRound).all():

            # input

            tokens = tokenize (dr.inp_tokenized)

            l = len(tokens)

            if l > self.input_max_len:
                self.input_max_len = l

            i = 0
            for token in tokens:

                if not token in self.input_dict:
                    self.input_dict[token] = len(self.input_dict)

            # output

            preds = dr.response.split(';')
            l = len(preds) + 1 # +1 to account for _EOS token

            if l > self.output_max_len:
                self.output_max_len = l

            i = 0
            for pred in preds:
                if not pred in self.output_dict:
                    self.output_dict[pred] = len(self.output_dict)

            self.num_segments += 1

        logging.info ('dicts done. input: %d enties, input_max_len=%d. output: %d enties, input_max_len=%d.  num_segments: %d' %
                      (len(self.input_dict), self.input_max_len, len(self.output_dict), self.output_max_len, self.num_segments))

    def save_dicts(self):

        with open(IN_DICT_FN, 'w') as f:

            f.write("%d\n" % self.input_max_len)

            for k in sorted(self.input_dict):

                f.write((u"%d;%s\n" % (self.input_dict[k], k)).encode('utf8'))

        logging.info ('%s written.', IN_DICT_FN)

        with open(OUT_DICT_FN, 'w') as f:

            f.write("%d\n" % self.output_max_len)

            for k in sorted(self.output_dict):

                f.write((u"%d;%s\n" % (self.output_dict[k], k)).encode('utf8'))

        logging.info ('%s written.', OUT_DICT_FN)

    def load_dicts(self):

        with open(IN_DICT_FN, 'r') as f:

            self.input_max_len = int(f.readline().rstrip())

            self.input_dict = {}

            while True:
                line = f.readline()
                if not line:
                    break

                line = line.lstrip().rstrip()

                parts = line.split(';')

                self.input_dict[parts[1]] = int(parts[0])

        logging.info ('%s read, %d entries, output_max_len=%d.' % (IN_DICT_FN, len(self.input_dict), self.input_max_len))

        with open(OUT_DICT_FN, 'r') as f:

            self.output_max_len = int(f.readline().rstrip())

            self.output_dict = {}

            while True:
                line = f.readline()
                if not line:
                    break

                line = line.lstrip().rstrip()

                parts = line.split(';')

                self.output_dict[parts[1]] = int(parts[0])

        logging.info ('%s read, %d entries, output_max_len=%d.' % (OUT_DICT_FN, len(self.output_dict), self.output_max_len))

    def compute_x(self, txt):

        tokens = tokenize(txt)

        return map(lambda token: self.input_dict[token] if token in self.input_dict else UNK_ID, tokens)

        # x = np.zeros(self.input_max_len, np.int32)
        #l = len(tokens)
        #i = 0
        #for token in tokens:
        #    x[self.input_max_len - l + i] = self.input_dict[token] if token in self.input_dict else 0
        #    i += 1

        #return x

    def compute_y(self, response):

        preds = map(lambda pred: self.output_dict[pred] if pred in self.output_dict else UNK_ID, response.split(';'))

        preds.append(EOS_ID)

        return preds

        # y = np.zeros((self.output_max_len, len(self.output_dict)), np.float32)

        # preds = response.split(';')

        # l = len(preds)
        # i = 0
        # for pred in preds:
        #     out_idx = self.output_dict[pred] if pred in self.output_dict else 0
        #     y[self.output_max_len - l + i, out_idx] = 1.0
        #     i += 1

        # return y

    def create_tf_model(self, tf_session, layer_size                 = LAYER_SIZE,
                                          num_layers                 = NUM_LAYERS,
                                          max_gradient_norm          = MAX_GRADIENT_NORM,
                                          batch_size                 = BATCH_SIZE,
                                          learning_rate              = LEARNING_RATE,
                                          learning_rate_decay_factor = LEARNING_RATE_DECAY_FACTOR,
                                          use_lstm                   = USE_LSTM,
                                          num_samples                = NUM_SAMPLES,
                                          forward_only               = FORWARD_ONLY):

        logging.info("creating seq2seq model: %d layers of %d units." % (num_layers, layer_size))

        print len(self.input_dict), len(self.output_dict), BUCKETS, layer_size, num_layers, max_gradient_norm, batch_size, learning_rate, learning_rate_decay_factor, num_samples, forward_only

        # 20000 20000 [(5, 10), (10, 15), (20, 25), (40, 50)] 128 1 5.0 64 0.5 0.99 True
        #   103    59 [(7, 4), (14, 8)]                       128 1 5.0 64 0.5 0.99 32 True

        self.model = seq2seq_model.Seq2SeqModel( len(self.input_dict),
                                                 len(self.output_dict),
                                                 BUCKETS,
                                                 layer_size,
                                                 num_layers,
                                                 max_gradient_norm,
                                                 batch_size,
                                                 learning_rate,
                                                 learning_rate_decay_factor,
                                                 num_samples=num_samples,
                                                 forward_only=forward_only)

        tf_session.run(tf.initialize_all_variables())

        return self.model

    def save_model (self, tf_session, fn=CKPT_FN):
        # self.model.saver.save(tf_session, fn, global_step=self.model.global_step)
        self.model.saver.save(tf_session, fn)
        logging.info("model saved to %s ." % fn)

    def load_model(self, tf_session, fn=CKPT_FN):
        self.model.saver.restore(tf_session, fn)
        logging.info("model restored from %s ." % fn)

    # def __len__(self):
    #     return len(self.segments)

    # def __getitem__(self, key):
    #     return self.segments[key]

    # def __iter__(self):
    #     return iter(sorted(self.segments))
<commit_msg>update to newer tf api<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Copyright 2016 Guenter Bartsch
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#
# nlp dictionaries, predicate terms, numpy model transformation
#

import os
import sys
import logging

import numpy as np

import codecs

from speech_tokenizer import tokenize

from sqlalchemy.orm import sessionmaker
import model

import tensorflow as tf

from tensorflow.models.rnn.translate import seq2seq_model
from tensorflow.models.rnn.translate.data_utils import _PAD, PAD_ID, _GO, GO_ID, _EOS, EOS_ID, _UNK, UNK_ID

OUT_DICT_FN  = 'data/dst/nlp_out_dict.csv'
IN_DICT_FN   = 'data/dst/nlp_in_dict.csv'

CKPT_FN      = 'data/dst/nlp_model.ckpt'

# We use a number of buckets and pad to the closest one for efficiency.
# See seq2seq_model.Seq2SeqModel for details of how they work.
# BUCKETS = [(5, 10), (10, 15), (20, 25), (40, 50)]
BUCKETS = [(7, 4), (14, 8)]

# number of LSTM layers : 1/2/3
NUM_LAYERS = 1
# typical options : 128, 256, 512, 1024
LAYER_SIZE = 128

BATCH_SIZE = 64

LEARNING_RATE              = 0.5
LEARNING_RATE_DECAY_FACTOR = 0.99
MAX_GRADIENT_NORM          = 5.0
USE_LSTM                   = False
NUM_SAMPLES                = 32
FORWARD_ONLY               = False

class NLPModel(object):

    def __init__(self, session, input_dim = 16 ):
        self.session   = session
        self.input_dim = input_dim

    def compute_input_hist(self):

        hist = {}

        for dr in self.session.query(model.DiscourseRound).all():

            tokens = tokenize (dr.inp_tokenized)

            if not (len(tokens) in hist):
                hist[len(tokens)] = 0

            hist[len(tokens)] += 1

        return hist

    def compute_output_hist(self):

        hist = {}

        for dr in self.session.query(model.DiscourseRound).all():

            preds = dr.response.split(';')

            if not (len(preds) in hist):
                hist[len(preds)] = 0

            hist[len(preds)] += 1

        return hist

    def compute_dicts(self):

        # build input and output dicts

        self.input_dict  = {_PAD : PAD_ID, _GO : GO_ID, _EOS : EOS_ID, _UNK : UNK_ID}
        self.output_dict = {_PAD : PAD_ID, _GO : GO_ID, _EOS : EOS_ID, _UNK : UNK_ID}

        self.input_max_len  = 0
        self.output_max_len = 0

        self.num_segments = 0

        for dr in self.session.query(model.DiscourseRound).all():

            # input

            tokens = tokenize (dr.inp_tokenized)

            l = len(tokens)

            if l > self.input_max_len:
                self.input_max_len = l

            i = 0
            for token in tokens:

                if not token in self.input_dict:
                    self.input_dict[token] = len(self.input_dict)

            # output

            preds = dr.response.split(';')
            l = len(preds) + 1 # +1 to account for _EOS token

            if l > self.output_max_len:
                self.output_max_len = l

            i = 0
            for pred in preds:
                if not pred in self.output_dict:
                    self.output_dict[pred] = len(self.output_dict)

            self.num_segments += 1

        logging.info ('dicts done. input: %d enties, input_max_len=%d. output: %d enties, input_max_len=%d.  num_segments: %d' %
                      (len(self.input_dict), self.input_max_len, len(self.output_dict), self.output_max_len, self.num_segments))

    def save_dicts(self):

        with open(IN_DICT_FN, 'w') as f:

            f.write("%d\n" % self.input_max_len)

            for k in sorted(self.input_dict):

                f.write((u"%d;%s\n" % (self.input_dict[k], k)).encode('utf8'))

        logging.info ('%s written.', IN_DICT_FN)

        with open(OUT_DICT_FN, 'w') as f:

            f.write("%d\n" % self.output_max_len)

            for k in sorted(self.output_dict):

                f.write((u"%d;%s\n" % (self.output_dict[k], k)).encode('utf8'))

        logging.info ('%s written.', OUT_DICT_FN)

    def load_dicts(self):

        with open(IN_DICT_FN, 'r') as f:

            self.input_max_len = int(f.readline().rstrip())

            self.input_dict = {}

            while True:
                line = f.readline()
                if not line:
                    break

                line = line.lstrip().rstrip()

                parts = line.split(';')

                self.input_dict[parts[1]] = int(parts[0])

        logging.info ('%s read, %d entries, output_max_len=%d.' % (IN_DICT_FN, len(self.input_dict), self.input_max_len))

        with open(OUT_DICT_FN, 'r') as f:

            self.output_max_len = int(f.readline().rstrip())

            self.output_dict = {}

            while True:
                line = f.readline()
                if not line:
                    break

                line = line.lstrip().rstrip()

                parts = line.split(';')

                self.output_dict[parts[1]] = int(parts[0])

        logging.info ('%s read, %d entries, output_max_len=%d.' % (OUT_DICT_FN, len(self.output_dict), self.output_max_len))

    def compute_x(self, txt):

        tokens = tokenize(txt)

        return map(lambda token: self.input_dict[token] if token in self.input_dict else UNK_ID, tokens)

        # x = np.zeros(self.input_max_len, np.int32)
        #l = len(tokens)
        #i = 0
        #for token in tokens:
        #    x[self.input_max_len - l + i] = self.input_dict[token] if token in self.input_dict else 0
        #    i += 1

        #return x

    def compute_y(self, response):

        preds = map(lambda pred: self.output_dict[pred] if pred in self.output_dict else UNK_ID, response.split(';'))

        preds.append(EOS_ID)

        return preds

        # y = np.zeros((self.output_max_len, len(self.output_dict)), np.float32)

        # preds = response.split(';')

        # l = len(preds)
        # i = 0
        # for pred in preds:
        #     out_idx = self.output_dict[pred] if pred in self.output_dict else 0
        #     y[self.output_max_len - l + i, out_idx] = 1.0
        #     i += 1

        # return y

    def create_tf_model(self, tf_session, layer_size                 = LAYER_SIZE,
                                          num_layers                 = NUM_LAYERS,
                                          max_gradient_norm          = MAX_GRADIENT_NORM,
                                          batch_size                 = BATCH_SIZE,
                                          learning_rate              = LEARNING_RATE,
                                          learning_rate_decay_factor = LEARNING_RATE_DECAY_FACTOR,
                                          use_lstm                   = USE_LSTM,
                                          num_samples                = NUM_SAMPLES,
                                          forward_only               = FORWARD_ONLY):

        logging.info("creating seq2seq model: %d layers of %d units." % (num_layers, layer_size))

        print len(self.input_dict), len(self.output_dict), BUCKETS, layer_size, num_layers, max_gradient_norm, batch_size, learning_rate, learning_rate_decay_factor, num_samples, forward_only

        # 20000 20000 [(5, 10), (10, 15), (20, 25), (40, 50)] 128 1 5.0 64 0.5 0.99 True
        #   103    59 [(7, 4), (14, 8)]                       128 1 5.0 64 0.5 0.99 32 True

        self.model = seq2seq_model.Seq2SeqModel( len(self.input_dict),
                                                 len(self.output_dict),
                                                 BUCKETS,
                                                 layer_size,
                                                 num_layers,
                                                 max_gradient_norm,
                                                 batch_size,
                                                 learning_rate,
                                                 learning_rate_decay_factor,
                                                 num_samples=num_samples,
                                                 forward_only=forward_only)
        init = tf.global_variables_initializer()
        tf_session.run(init)

        return self.model

    def save_model (self, tf_session, fn=CKPT_FN):
        # self.model.saver.save(tf_session, fn, global_step=self.model.global_step)
        self.model.saver.save(tf_session, fn)
        logging.info("model saved to %s ." % fn)

    def load_model(self, tf_session, fn=CKPT_FN):
        self.model.saver.restore(tf_session, fn)
        logging.info("model restored from %s ." % fn)

    # def __len__(self):
    #     return len(self.segments)

    # def __getitem__(self, key):
    #     return self.segments[key]

    # def __iter__(self):
    #     return iter(sorted(self.segments))
<|endoftext|>


<commit_before>#!/usr/bin/env python2
import socket
import ttt
import logging
import sys
import select
import random

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
handler = logging.FileHandler('server.log')
handler.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - ' +
                              '%(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)

class Session(object):
    ''' Keeps track of game and its two participating client sockets.'''
    def __init__(self, firstclient, sessiontype):
        self.players = {ttt.BSTATES['P1']:firstclient}
        self.game = ttt.TicTacToeGame(start=random.choice([ttt.BSTATES['P1'],
                                                           ttt.BSTATES['P2']]))
        # 'a' for ai, 'p' for person
        self.sessiontype = sessiontype
        if sessiontype == 'a':
            self.players[ttt.BSTATES['P2']] = 'computer'

    def add_player(self, player2):
        ''' returns True if there was room for a second player (adding
            a second player succeeded)
        '''
        if self.players.get(ttt.BSTATES['P2']):
            return False
        else:
            self.players[ttt.BSTATES['P2']] = player2

        return True

    # For now, if a player leaves a gamesession, that gamesession is
    # concluded
    # and his partner is automatically put in a new empty gamesession.

class Server:
    def __init__(self):
        self.readables = []
        self.writables = []
        self.allsockets = []
        # all initiated games
        self.sessions = []
        # which clients belong to which games
        self.sessiondict = {}
        # a queue of messages?
        # dictionary of sock : reuqests, dictionary  sock: responses?

    def run(self):
        listenersock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        #prevent the "already in use error"
        listenersock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        listenersock.setblocking(False)
        # TODO: bind raises an exception if port is already in use
        listenersock.bind((HOST, PORT))

        listenersock.listen(5)
        print "Server on"

        self.readables = [listenersock, sys.stdin]
        self.allsockets = [listenersock]

        while True:
            # wait for at least one of the sockets to be ready
            #import time
            #time.sleep(10)
            (ready_rds, ready_wrs,
                exceptional) = select.select(self.readables,
                                             self.writables,
                                             self.allsockets, 60)

            # Process incoming data
            for r in ready_rds:
                # accept a client connection
                if r is listenersock:
                    clientsock, clientaddr = listenersock.accept()
                    clientsock.setblocking(False)
                    for iolist in [self.readables, self.writables,
                                   self.allsockets]:
                        iolist.append(clientsock)
                    print 'Connected to ' + str(clientaddr)
                    #logger.info('Connected to ' + str(clientaddr))
                    #self.assign_to_session(clientsock)
                elif r is sys.stdin:
                    if (sys.stdin.readline()).strip() == 'exit':
                        listenersock.close()
                        print "Server off"
                        sys.exit()
                # collect a message from each ready client
                else:
                    print 'client recv'
                    print (ready_rds, ready_wrs, exceptional)
                    # TODO
                    # results in err104 connection reset by peer if client
                    # has crashed
                    # When client closes socket, empty string is received
                    message = r.recv(8)
                    if not message:
                        self.delete_socket(r)
                    else:
                        print message
                    #else: # TEMP, TODO
                        # _, ready_to_write, _ = select.select([],[r],[],1)
                        # if ready_to_write:
                        #     ready_to_write[0].send('hello')
                        #     ready_to_write[0].send('hello')

            # for w in ready_wrs:
            #     print 'client send'
            #     print (ready_rds, ready_wrs, exceptional)
            # TODO when sending: recover from err 32 broken pipe in case
            # client quits
            #     w.send('hello')

            for e in exceptional:
                self.delete_socket(e)

    def delete_socket(self, sock):
        for iolist in [self.readables, self.writables, self.allsockets]:
            if sock in iolist:
                iolist.remove(sock)
        sock.close()
        print 'closed socket'
                #logger.info('Closed a socket.')

    # @property
    # def num_sessions(self):
    #     return len(set(self.sessiondict.values()))

    def assign_to_session(client):
        if len(self.sessions) != 0 and not self.sessions[-1].player2:
            # add client to last-created session; it is unpaired
            self.sessiondict[clientsock] = self.sessions[-1]
            self.sessions[-1].player2 = clientsock
            # send message to client that it is p1 and it's p2's turn
        else:
            gamesession = Session(clientsock)
            # TODO send message to client that it is p1 and it's p1's turn
            self.sessiondict[clientsock] = gamesession
            self.sessions.append(gamesession)

    # def broadcast_msg(self, source, destinations):
    #     ''' Send message from source to all other clients in destinations list.
    #     `destinations` is the result of a select call: if a client wasn't ready
    #     at that moment, it will never receive this message!
    #     '''
    #     msg = source.recv(SIZE)
    #     if msg:
    #         for recipient in destinations:
    #             if recipient is not source:
    #                 try:
    #                     recipient.send(msg)
    #                 except socket.error as e:
    #                     # remote peer disconnected
    #                     if isinstance(e.args, tuple) and e[0] == errno.EPIPE:
    #                         self.disconnect_client(recipient)
    #                     else:
    #                         # some other unanticipated issue. :(
    #                         raise e
    #     else:
    #         # no data received means that source has disconnected
    #         self.disconnect_client(source)

    def disconnect_client(self, s):
        # TODO discard session that the client belonged to
        print "Disconnecting: " + str(s.getpeername())
        if s in self.writables:
            self.writables.remove(s)
        if s in self.readables:
            self.readables.remove(s)
        if s in self.allsockets:
            self.allsockets.remove(s)
        s.close()

if __name__ == "__main__":
    Server().run()
<commit_msg>Mostly refactoring: separating the processing of requests and responses<commit_after>#!/usr/bin/env python2
import socket
import ttt
import logging
import sys
import select
import random

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
handler = logging.FileHandler('server.log')
handler.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - ' +
                              '%(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)

class Session(object):
    ''' Keeps track of game and its two participating client sockets.'''
    def __init__(self, firstclient, sessiontype):
        self.players = {ttt.BSTATES['P1']:firstclient}
        self.game = ttt.TicTacToeGame(start=random.choice([ttt.BSTATES['P1'],
                                                           ttt.BSTATES['P2']]))
        # 'a' for ai, 'p' for person
        self.sessiontype = sessiontype
        if sessiontype == 'a':
            self.players[ttt.BSTATES['P2']] = 'computer'

    def add_player(self, player2):
        ''' returns True if there was room for a second player (adding
            a second player succeeded)
        '''
        if self.players.get(ttt.BSTATES['P2']):
            return False
        else:
            self.players[ttt.BSTATES['P2']] = player2

        return True

    # For now, if a player leaves a gamesession, that gamesession is
    # concluded
    # and his partner is automatically put in a new empty gamesession.

class Server(object):
    HOST = 'localhost'
    PORT = 50000

    def __init__(self):
        self.readables = []
        self.writables = []
        self.allsockets = []
        # which clients belong to which games (sock:session)
        self.sessiondict = {}
        # sock:queue (queue of requests to respond to)
        self.requests = {}

        self.listenersock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        #prevent the "already in use error"
        self.listenersock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.listenersock.setblocking(False)
        # TODO: bind raises an exception if port is already in use
        self.listenersock.bind((self.HOST, self.PORT))
        self.listenersock.listen(5)
        print "Server on"

    def run(self):
        self.readables = [self.listenersock, sys.stdin]
        self.allsockets = [self.listenersock]
        while True:
            # wait for at least one of the sockets to be ready
            (ready_rds, ready_wrs,
                exceptional) = select.select(self.readables,
                                             self.writables,
                                             self.allsockets, 60)
            self.process_requests(ready_rds)
            self.process_responses(ready_wrs)
            for e in exceptional:
                self.delete_socket(e)

    def process_requests(self, ready_rds):
        for r in ready_rds:
                if r is self.listenersock:
                    clientsock, clientaddr = self.listenersock.accept()
                    clientsock.setblocking(False)
                    for iolist in [self.readables, self.writables,
                                   self.allsockets]:
                        iolist.append(clientsock)
                    # initialize request queue
                    self.requests[clientsock] = []
                    print 'Connected to ' + str(clientaddr)
                    #logger.info('Connected to ' + str(clientaddr))
                elif r is sys.stdin:
                    if (sys.stdin.readline()).strip() == 'exit':
                        self.listenersock.close()
                        print "Server off"
                        sys.exit()
                # collect a message from each ready client
                else:
                    # TODO
                    # results in err104 connection reset by peer if client
                    # has crashed
                    # When client closes socket, empty string is received
                    message = r.recv(1024)
                    if not message:
                        self.disconnect_client(r)
                    else:
                        self.requests[r].append(message)
                        print message

    def process_responses(self, ready_wrs):
        # for w in ready_wrs:
            #     print 'client send'
            #     print (ready_rds, ready_wrs, exceptional)
            # TODO when sending: recover from err 32 broken pipe in case
            # client quits
            #     w.send('hello')

        pass

    def disconnect_client(self, sock):
        for iolist in [self.readables, self.writables, self.allsockets]:
            if sock in iolist:
                iolist.remove(sock)
        sock.close()
        print 'closed socket'
        #logger.info('Closed a socket.')

    # @property
    # def num_sessions(self):
    #     return len(set(self.sessiondict.values()))

    def assign_to_session(client):
        if len(self.sessions) != 0 and not self.sessions[-1].player2:
            # add client to last-created session; it is unpaired
            self.sessiondict[clientsock] = self.sessions[-1]
            self.sessions[-1].player2 = clientsock
            # send message to client that it is p1 and it's p2's turn
        else:
            gamesession = Session(clientsock)
            # TODO send message to client that it is p1 and it's p1's turn
            self.sessiondict[clientsock] = gamesession
            self.sessions.append(gamesession)

    # def broadcast_msg(self, source, destinations):
    #     ''' Send message from source to all other clients in destinations list.
    #     `destinations` is the result of a select call: if a client wasn't ready
    #     at that moment, it will never receive this message!
    #     '''
    #     msg = source.recv(SIZE)
    #     if msg:
    #         for recipient in destinations:
    #             if recipient is not source:
    #                 try:
    #                     recipient.send(msg)
    #                 except socket.error as e:
    #                     # remote peer disconnected
    #                     if isinstance(e.args, tuple) and e[0] == errno.EPIPE:
    #                         self.disconnect_client(recipient)
    #                     else:
    #                         # some other unanticipated issue. :(
    #                         raise e
    #     else:
    #         # no data received means that source has disconnected
    #         self.disconnect_client(source)

if __name__ == "__main__":
    Server().run()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Utility functions for the neukrill-net classifier
"""
import glob
import io
import json
import os
import gzip
import numpy as np

import neukrill_net.image_processing as image_processing
import neukrill_net.constants as constants

class Settings:
    """
    A class to handle all the parsing of the settings.json as well as
    checking the values and providing defaults
    """
    def __init__(self, settings_file):
        """
        Initialise settings object
        """
        self.required_settings = ['data_dir']

        # read the provided settings json (or stringIO)
        parsed_settings = self.parse_settings(settings_file)

        # check all required options have been set
        self.user_input = self.check_mandatory_fields(parsed_settings)

        # check the data dir options and find the correct one
        self.data_dir = self.check_data_dir(self.user_input['data_dir'])

        # check user input for random seed and if absent set it
        self.random_seed = self.user_input.get('r_seed', 42)

        # check user defined classes if absent set it
        self.classes = self.user_input.get('classes',
                                            constants.classes)

        # a way to encode the superclasses if we need to but don't want to complete
        # just now as it is very monotonous to copy everything
        # self.super_classes = {'FISH': ('fish classes'),
        #                       'DIATOMS': ('diatom classes'),
        #                       'TRICHODESMIUM': ('you get the idea')
        #                       'PROTISTS': ('protists_classes')
        #                       'NO_SUPER_CLASS': ('unclassified')}

        self._image_fnames = {}

        self._class_priors = []

    def parse_settings(self, settings_file):
        """
        Parse json file or stringIO formatted settings file
        This function will likely be extended later to enable defaults and to
        check inputs
        input : settings_file - filename or stringIO obj
                required_settings - list of required fields
        output: settings - settings dict
        """
        if settings_file.__class__ is io.StringIO:
            settings = json.load(settings_file)

        else:
            if not os.path.exists(settings_file):
                raise ValueError('Settings file does not exist: {0}'.format(\
                                                                settings_file))
            with open(settings_file, 'r') as settings_fh:
                settings = json.load(settings_fh)

        return settings

    def check_mandatory_fields(self, parsed_user_input):
        """
        Ensure all the mandatory settings are present
        """
        for entry in self.required_settings:
            if entry not in parsed_user_input:
                raise ValueError('data_dir must be defined')

        return parsed_user_input

    def check_data_dir(self, data_dir_possibilities):
        """
        Make sure the data dirs exist and resolve the test and train dirs
        abspaths
        """
        for possible_dir in data_dir_possibilities:
            train_data_dir = os.path.join(possible_dir, 'train')
            test_data_dir = os.path.join(possible_dir, 'test')
            dirs_exist = os.path.exists(train_data_dir) and \
                                                os.path.exists(test_data_dir)

            if dirs_exist:
                return os.path.abspath(possible_dir)

        raise ValueError("Can't find data dir in options: {0}".format(\
                                                            data_dir_possibilities))

    @property
    def image_fnames(self):
        """
        Take in data dir and return dict of filenames
        input: data_directory - path as str
        output: image_fnames - dict {'test': (tuple of fnames abspaths),
                                     'train': {'class_1' : (tuple of fnames),
                                               'class_2  : (tuple of fnames),
                                               ...
                                               }
                                    }
        """

        if not self._image_fnames:
            test_fnames = tuple(sorted(glob.glob(os.path.join(self.data_dir,
                                                       'test',
                                                       '*.jpg'))))

            # check there are the correct number of images
            num_test_images = len(test_fnames)
            if num_test_images != 130400:
                raise ValueError('Wrong number of test images found: {0}'
                                 ' instead of 130400'.format(num_test_images))

            train_fnames = {}

            for name in sorted(glob.glob(os.path.join(self.data_dir,
                                               'train',
                                               '*',
                                               ''))):
                split_name = name.split(os.path.sep)
                class_name = split_name[-2]
                image_names = sorted(glob.glob(os.path.join(name, '*.jpg')))
                train_fnames.update({class_name: image_names})

            num_train_classes = len(train_fnames.keys())
            num_train_images = sum(map(len, train_fnames.values()))
            if num_train_classes != 121:
                raise ValueError('Incorrect num of training class directories '\
                        '121 expected: {0} found'.format(num_train_classes))

            if num_train_images != 30336:
                raise ValueError('Incorrect num of training images '\
                        ' 30336 expected: {0} found'.format(num_train_images))

            self._image_fnames = {'test': test_fnames,
                                  'train': train_fnames}

        return self._image_fnames

    @property
    def class_priors(self):
        """
        Get the proportion of training data in each class
        """
        if self._class_priors == []:
            class_probs = np.zeros(len(self.classes))
            for class_index, class_name in enumerate(self.classes):
                # Tally up how many filenames are in this class
                # Set to zero if this class is not a field
                class_probs[class_index] = len(self.image_fnames['train'].get(class_name, 0))
            # Normalise the classes
            class_probs /= sum(class_probs)
            self._class_priors = class_probs

        return self._class_priors

def load_data(image_fname_dict, classes=None,
              processing=None, verbose=False):
    """
    Loads training or test data using image_processing.load_images func
    which applies the supplied processing function as required

    If the classes kwarg is not none assumed to be loading labelled train
    data and returns two np objs:
        * data - image matrix
        * labels - vector of labels

    if classes kwarg is none, data will be loaded as test data and just return
        * data - list of image vectors
    """

    if not processing and verbose:
        print("Warning: no processing applied, it will \
        not be possible to stack these images due to \
        varying sizes.")

    # initialise lists
    data = []

    # e.g. labelled training data
    if classes:
        labels = []

        for class_index, class_name in enumerate(classes):
            if verbose:
                print("class: {0} of 120: {1}".format(class_index, class_name))

            image_fpaths = image_fname_dict['train'][class_name]
            data_subset = image_processing.load_images(image_fpaths,
                                                          processing,
                                                          verbose)
            data.append(data_subset)
            num_images = len(data_subset)
            # generate the class labels and add them to the list
            array_labels = num_images * [class_name]
            labels = labels + array_labels
        return np.vstack(data), np.array(labels)

    # e.g. test data
    else:
        data = image_processing.load_images(image_fname_dict['test'],
                                            processing,
                                            verbose)
        names = [os.path.basename(fpath) for fpath in image_fname_dict['test']]
        return np.vstack(data), names

<commit_msg>Function for writing submission CSV and compressing it<commit_after>#!/usr/bin/env python
"""
Utility functions for the neukrill-net classifier
"""
import glob
import io
import json
import os
import csv
import gzip
import numpy as np

import neukrill_net.image_processing as image_processing
import neukrill_net.constants as constants

class Settings:
    """
    A class to handle all the parsing of the settings.json as well as
    checking the values and providing defaults
    """
    def __init__(self, settings_file):
        """
        Initialise settings object
        """
        self.required_settings = ['data_dir']

        # read the provided settings json (or stringIO)
        parsed_settings = self.parse_settings(settings_file)

        # check all required options have been set
        self.user_input = self.check_mandatory_fields(parsed_settings)

        # check the data dir options and find the correct one
        self.data_dir = self.check_data_dir(self.user_input['data_dir'])

        # check user input for random seed and if absent set it
        self.random_seed = self.user_input.get('r_seed', 42)

        # check user defined classes if absent set it
        self.classes = self.user_input.get('classes',
                                            constants.classes)

        # a way to encode the superclasses if we need to but don't want to complete
        # just now as it is very monotonous to copy everything
        # self.super_classes = {'FISH': ('fish classes'),
        #                       'DIATOMS': ('diatom classes'),
        #                       'TRICHODESMIUM': ('you get the idea')
        #                       'PROTISTS': ('protists_classes')
        #                       'NO_SUPER_CLASS': ('unclassified')}

        self._image_fnames = {}

        self._class_priors = []

    def parse_settings(self, settings_file):
        """
        Parse json file or stringIO formatted settings file
        This function will likely be extended later to enable defaults and to
        check inputs
        input : settings_file - filename or stringIO obj
                required_settings - list of required fields
        output: settings - settings dict
        """
        if settings_file.__class__ is io.StringIO:
            settings = json.load(settings_file)

        else:
            if not os.path.exists(settings_file):
                raise ValueError('Settings file does not exist: {0}'.format(\
                                                                settings_file))
            with open(settings_file, 'r') as settings_fh:
                settings = json.load(settings_fh)

        return settings

    def check_mandatory_fields(self, parsed_user_input):
        """
        Ensure all the mandatory settings are present
        """
        for entry in self.required_settings:
            if entry not in parsed_user_input:
                raise ValueError('data_dir must be defined')

        return parsed_user_input

    def check_data_dir(self, data_dir_possibilities):
        """
        Make sure the data dirs exist and resolve the test and train dirs
        abspaths
        """
        for possible_dir in data_dir_possibilities:
            train_data_dir = os.path.join(possible_dir, 'train')
            test_data_dir = os.path.join(possible_dir, 'test')
            dirs_exist = os.path.exists(train_data_dir) and \
                                                os.path.exists(test_data_dir)

            if dirs_exist:
                return os.path.abspath(possible_dir)

        raise ValueError("Can't find data dir in options: {0}".format(\
                                                            data_dir_possibilities))

    @property
    def image_fnames(self):
        """
        Take in data dir and return dict of filenames
        input: data_directory - path as str
        output: image_fnames - dict {'test': (tuple of fnames abspaths),
                                     'train': {'class_1' : (tuple of fnames),
                                               'class_2  : (tuple of fnames),
                                               ...
                                               }
                                    }
        """

        if not self._image_fnames:
            test_fnames = tuple(sorted(glob.glob(os.path.join(self.data_dir,
                                                       'test',
                                                       '*.jpg'))))

            # check there are the correct number of images
            num_test_images = len(test_fnames)
            if num_test_images != 130400:
                raise ValueError('Wrong number of test images found: {0}'
                                 ' instead of 130400'.format(num_test_images))

            train_fnames = {}

            for name in sorted(glob.glob(os.path.join(self.data_dir,
                                               'train',
                                               '*',
                                               ''))):
                split_name = name.split(os.path.sep)
                class_name = split_name[-2]
                image_names = sorted(glob.glob(os.path.join(name, '*.jpg')))
                train_fnames.update({class_name: image_names})

            num_train_classes = len(train_fnames.keys())
            num_train_images = sum(map(len, train_fnames.values()))
            if num_train_classes != 121:
                raise ValueError('Incorrect num of training class directories '\
                        '121 expected: {0} found'.format(num_train_classes))

            if num_train_images != 30336:
                raise ValueError('Incorrect num of training images '\
                        ' 30336 expected: {0} found'.format(num_train_images))

            self._image_fnames = {'test': test_fnames,
                                  'train': train_fnames}

        return self._image_fnames

    @property
    def class_priors(self):
        """
        Get the proportion of training data in each class
        """
        if self._class_priors == []:
            class_probs = np.zeros(len(self.classes))
            for class_index, class_name in enumerate(self.classes):
                # Tally up how many filenames are in this class
                # Set to zero if this class is not a field
                class_probs[class_index] = len(self.image_fnames['train'].get(class_name, 0))
            # Normalise the classes
            class_probs /= sum(class_probs)
            self._class_priors = class_probs

        return self._class_priors

def load_data(image_fname_dict, classes=None,
              processing=None, verbose=False):
    """
    Loads training or test data using image_processing.load_images func
    which applies the supplied processing function as required

    If the classes kwarg is not none assumed to be loading labelled train
    data and returns two np objs:
        * data - image matrix
        * labels - vector of labels

    if classes kwarg is none, data will be loaded as test data and just return
        * data - list of image vectors
    """

    if not processing and verbose:
        print("Warning: no processing applied, it will \
        not be possible to stack these images due to \
        varying sizes.")

    # initialise lists
    data = []

    # e.g. labelled training data
    if classes:
        labels = []

        for class_index, class_name in enumerate(classes):
            if verbose:
                print("class: {0} of 120: {1}".format(class_index, class_name))

            image_fpaths = image_fname_dict['train'][class_name]
            data_subset = image_processing.load_images(image_fpaths,
                                                          processing,
                                                          verbose)
            data.append(data_subset)
            num_images = len(data_subset)
            # generate the class labels and add them to the list
            array_labels = num_images * [class_name]
            labels = labels + array_labels
        return np.vstack(data), np.array(labels)

    # e.g. test data
    else:
        data = image_processing.load_images(image_fname_dict['test'],
                                            processing,
                                            verbose)
        names = [os.path.basename(fpath) for fpath in image_fname_dict['test']]
        return np.vstack(data), names

def write_predictions(out_fname, p, names, settings):
    """Write probabilities to an output csv and then compress with gzip"""

    with open(out_fname, 'w') as csv_out:
        out_writer = csv.writer(csv_out, delimiter=',')
        out_writer.writerow(['image'] + list(settings.classes))
        for index in range(len(names)):
            out_writer.writerow([names[index]] + list(p[index,]))

    with open(out_fname, 'rb') as f_in:
        f_out = gzip.open(out_fname + '.gz', 'wb')
        f_out.writelines(f_in)
        f_out.close()

<|endoftext|>


<commit_before>"""
This is a training script for sequence to sequence learning.
"""

import argparse
import sys
import random
import os
import shlex
from shutil import copyfile
import numpy as np
import tensorflow as tf

from neuralmonkey.checking import CheckingException, check_dataset_and_coders
from neuralmonkey.logging import Logging, log
from neuralmonkey.config.configuration import Configuration
from neuralmonkey.learning_utils import training_loop

def create_config() -> Configuration:
    config = Configuration()

    # training loop arguments
    config.add_argument('tf_manager')
    config.add_argument('epochs', cond=lambda x: x >= 0)
    config.add_argument('trainer')
    config.add_argument('batch_size', cond=lambda x: x > 0)
    config.add_argument('train_dataset')
    config.add_argument('val_dataset')
    config.add_argument('output')
    config.add_argument('evaluation')
    config.add_argument('runners')
    config.add_argument('test_datasets', required=False, default=[])
    config.add_argument('logging_period', required=False, default=20)
    config.add_argument('validation_period', required=False, default=500)
    config.add_argument('val_preview_input_series',
                        required=False, default=None)
    config.add_argument('val_preview_output_series',
                        required=False, default=None)
    config.add_argument('val_preview_num_examples',
                        required=False, default=15)
    config.add_argument('train_start_offset', required=False, default=0)
    config.add_argument('runners_batch_size', required=False, default=None)
    config.add_argument('postprocess')
    config.add_argument('name')
    config.add_argument('random_seed', required=False)
    config.add_argument('initial_variables', required=False, default=None)
    config.add_argument('overwrite_output_dir', required=False, default=False)

    return config

# pylint: disable=too-many-statements
def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('config', metavar='INI-FILE',
                        help='the configuration file for the experiment')
    parser.add_argument('-s', '--set', type=str, metavar='SETTING',
                        action='append', dest='config_changes',
                        help='override an option in the configuration; the '
                        'syntax is [section.]option=value')
    parser.add_argument('-i', '--init', dest='init_only', action='store_true',
                        help='initialize the experiment directory and exit '
                        'without building the model')
    parser.add_argument('-f', '--overwrite', action='store_true',
                        help='force overwriting the output directory; can be '
                        'used to start an experiment created with --init')
    args = parser.parse_args()

    # define valid parameters and defaults
    cfg = create_config()
    # load the params from the config file, getting also the simple arguments
    cfg.load_file(args.config, changes=args.config_changes)
    # various things like randseed or summarywriter should be set up here
    # so that graph building can be recorded
    # build all the objects specified in the config

    if cfg.args.random_seed is None:
        cfg.args.random_seed = 2574600
    random.seed(cfg.args.random_seed)
    np.random.seed(cfg.args.random_seed)
    tf.set_random_seed(cfg.args.random_seed)

    # pylint: disable=no-member
    if (os.path.isdir(cfg.args.output) and
            os.path.exists(os.path.join(cfg.args.output, "experiment.ini"))):
        if cfg.args.overwrite_output_dir or args.overwrite:
            # we do not want to delete the directory contents
            log("Directory with experiment.ini '{}' exists, "
                "overwriting enabled, proceeding."
                .format(cfg.args.output))
        else:
            log("Directory with experiment.ini '{}' exists, "
                "overwriting disabled."
                .format(cfg.args.output), color='red')
            exit(1)

    # pylint: disable=broad-except
    if not os.path.isdir(cfg.args.output):
        try:
            os.mkdir(cfg.args.output)
        except Exception as exc:
            log("Failed to create experiment directory: {}. Exception: {}"
                .format(cfg.args.output, exc), color='red')
            exit(1)

    args_file = "{}/args".format(cfg.args.output)
    log_file = "{}/experiment.log".format(cfg.args.output)
    ini_file = "{}/experiment.ini".format(cfg.args.output)
    orig_ini_file = "{}/original.ini".format(cfg.args.output)
    git_commit_file = "{}/git_commit".format(cfg.args.output)
    git_diff_file = "{}/git_diff".format(cfg.args.output)
    variables_file_prefix = "{}/variables.data".format(cfg.args.output)

    cont_index = 0

    while (os.path.exists(log_file)
           or os.path.exists(ini_file)
           or os.path.exists(git_commit_file)
           or os.path.exists(git_diff_file)
           or os.path.exists(variables_file_prefix)
           or os.path.exists("{}.0".format(variables_file_prefix))):
        cont_index += 1

        args_file = "{}/args.cont-{}".format(
            cfg.args.output, cont_index)
        log_file = "{}/experiment.log.cont-{}".format(
            cfg.args.output, cont_index)
        ini_file = "{}/experiment.ini.cont-{}".format(
            cfg.args.output, cont_index)
        orig_ini_file = "{}/original.ini.cont-{}".format(
            cfg.args.output, cont_index)
        git_commit_file = "{}/git_commit.cont-{}".format(
            cfg.args.output, cont_index)
        git_diff_file = "{}/git_diff.cont-{}".format(
            cfg.args.output, cont_index)
        variables_file_prefix = "{}/variables.data.cont-{}".format(
            cfg.args.output, cont_index)

    with open(args_file, 'w') as file:
        print(' '.join(shlex.quote(a) for a in sys.argv), file=file)

    cfg.save_file(ini_file)
    copyfile(args.config, orig_ini_file)

    if args.init_only:
        log("Experiment directory initialized. Exiting.")
        exit(0)

    Logging.set_log_file(log_file)

    # this points inside the neuralmonkey/ dir inside the repo, but
    # it does not matter for git.
    repodir = os.path.dirname(os.path.realpath(__file__))

    # we need to execute the git log command in subshell, because if
    # the log file is specified via relative path, we need to do the
    # redirection of the git-log output to the right file
    os.system("(cd {}; git log -1 --format=%H) > {}"
              .format(repodir, git_commit_file))

    os.system("(cd {}; git --no-pager diff --color=always) > {}"
              .format(repodir, git_diff_file))

    cfg.build_model(warn_unused=True)

    cfg.model.tf_manager.init_saving(variables_file_prefix)

    try:
        check_dataset_and_coders(cfg.model.train_dataset,
                                 cfg.model.runners)
        check_dataset_and_coders(cfg.model.val_dataset,
                                 cfg.model.runners)
    except CheckingException as exc:
        log(str(exc), color='red')
        exit(1)

    Logging.print_header(cfg.model.name)

    # runners_batch_size must be set to avoid problems on GPU
    if cfg.model.runners_batch_size is None:
        cfg.model.runners_batch_size = cfg.model.batch_size

    training_loop(
        tf_manager=cfg.model.tf_manager,
        epochs=cfg.model.epochs,
        trainer=cfg.model.trainer,
        batch_size=cfg.model.batch_size,
        train_dataset=cfg.model.train_dataset,
        val_dataset=cfg.model.val_dataset,
        log_directory=cfg.model.output,
        evaluators=cfg.model.evaluation,
        runners=cfg.model.runners,
        test_datasets=cfg.model.test_datasets,
        logging_period=cfg.model.logging_period,
        validation_period=cfg.model.validation_period,
        val_preview_input_series=cfg.model.val_preview_input_series,
        val_preview_output_series=cfg.model.val_preview_output_series,
        val_preview_num_examples=cfg.model.val_preview_num_examples,
        postprocess=cfg.model.postprocess,
        train_start_offset=cfg.model.train_start_offset,
        runners_batch_size=cfg.model.runners_batch_size,
        initial_variables=cfg.model.initial_variables)
<commit_msg>Show command to start experiment after --init<commit_after>"""
This is a training script for sequence to sequence learning.
"""

import argparse
import sys
import random
import os
import shlex
from shutil import copyfile
import numpy as np
import tensorflow as tf

from neuralmonkey.checking import CheckingException, check_dataset_and_coders
from neuralmonkey.logging import Logging, log
from neuralmonkey.config.configuration import Configuration
from neuralmonkey.learning_utils import training_loop

def create_config() -> Configuration:
    config = Configuration()

    # training loop arguments
    config.add_argument('tf_manager')
    config.add_argument('epochs', cond=lambda x: x >= 0)
    config.add_argument('trainer')
    config.add_argument('batch_size', cond=lambda x: x > 0)
    config.add_argument('train_dataset')
    config.add_argument('val_dataset')
    config.add_argument('output')
    config.add_argument('evaluation')
    config.add_argument('runners')
    config.add_argument('test_datasets', required=False, default=[])
    config.add_argument('logging_period', required=False, default=20)
    config.add_argument('validation_period', required=False, default=500)
    config.add_argument('val_preview_input_series',
                        required=False, default=None)
    config.add_argument('val_preview_output_series',
                        required=False, default=None)
    config.add_argument('val_preview_num_examples',
                        required=False, default=15)
    config.add_argument('train_start_offset', required=False, default=0)
    config.add_argument('runners_batch_size', required=False, default=None)
    config.add_argument('postprocess')
    config.add_argument('name')
    config.add_argument('random_seed', required=False)
    config.add_argument('initial_variables', required=False, default=None)
    config.add_argument('overwrite_output_dir', required=False, default=False)

    return config

# pylint: disable=too-many-statements
def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('config', metavar='INI-FILE',
                        help='the configuration file for the experiment')
    parser.add_argument('-s', '--set', type=str, metavar='SETTING',
                        action='append', dest='config_changes',
                        help='override an option in the configuration; the '
                        'syntax is [section.]option=value')
    parser.add_argument('-i', '--init', dest='init_only', action='store_true',
                        help='initialize the experiment directory and exit '
                        'without building the model')
    parser.add_argument('-f', '--overwrite', action='store_true',
                        help='force overwriting the output directory; can be '
                        'used to start an experiment created with --init')
    args = parser.parse_args()

    # define valid parameters and defaults
    cfg = create_config()
    # load the params from the config file, getting also the simple arguments
    cfg.load_file(args.config, changes=args.config_changes)
    # various things like randseed or summarywriter should be set up here
    # so that graph building can be recorded
    # build all the objects specified in the config

    if cfg.args.random_seed is None:
        cfg.args.random_seed = 2574600
    random.seed(cfg.args.random_seed)
    np.random.seed(cfg.args.random_seed)
    tf.set_random_seed(cfg.args.random_seed)

    # pylint: disable=no-member
    if (os.path.isdir(cfg.args.output) and
            os.path.exists(os.path.join(cfg.args.output, "experiment.ini"))):
        if cfg.args.overwrite_output_dir or args.overwrite:
            # we do not want to delete the directory contents
            log("Directory with experiment.ini '{}' exists, "
                "overwriting enabled, proceeding."
                .format(cfg.args.output))
        else:
            log("Directory with experiment.ini '{}' exists, "
                "overwriting disabled."
                .format(cfg.args.output), color='red')
            exit(1)

    # pylint: disable=broad-except
    if not os.path.isdir(cfg.args.output):
        try:
            os.mkdir(cfg.args.output)
        except Exception as exc:
            log("Failed to create experiment directory: {}. Exception: {}"
                .format(cfg.args.output, exc), color='red')
            exit(1)

    args_file = "{}/args".format(cfg.args.output)
    log_file = "{}/experiment.log".format(cfg.args.output)
    ini_file = "{}/experiment.ini".format(cfg.args.output)
    orig_ini_file = "{}/original.ini".format(cfg.args.output)
    git_commit_file = "{}/git_commit".format(cfg.args.output)
    git_diff_file = "{}/git_diff".format(cfg.args.output)
    variables_file_prefix = "{}/variables.data".format(cfg.args.output)

    cont_index = 0

    while (os.path.exists(log_file)
           or os.path.exists(ini_file)
           or os.path.exists(git_commit_file)
           or os.path.exists(git_diff_file)
           or os.path.exists(variables_file_prefix)
           or os.path.exists("{}.0".format(variables_file_prefix))):
        cont_index += 1

        args_file = "{}/args.cont-{}".format(
            cfg.args.output, cont_index)
        log_file = "{}/experiment.log.cont-{}".format(
            cfg.args.output, cont_index)
        ini_file = "{}/experiment.ini.cont-{}".format(
            cfg.args.output, cont_index)
        orig_ini_file = "{}/original.ini.cont-{}".format(
            cfg.args.output, cont_index)
        git_commit_file = "{}/git_commit.cont-{}".format(
            cfg.args.output, cont_index)
        git_diff_file = "{}/git_diff.cont-{}".format(
            cfg.args.output, cont_index)
        variables_file_prefix = "{}/variables.data.cont-{}".format(
            cfg.args.output, cont_index)

    with open(args_file, 'w') as file:
        print(' '.join(shlex.quote(a) for a in sys.argv), file=file)

    cfg.save_file(ini_file)
    copyfile(args.config, orig_ini_file)

    if args.init_only:
        log("Experiment directory initialized.")

        cmd = [os.path.basename(sys.argv[0]), '-f', ini_file]
        log("To start experiment, run: {}".format(' '.join(shlex.quote(a)
                                                           for a in cmd)))
        exit(0)

    Logging.set_log_file(log_file)

    # this points inside the neuralmonkey/ dir inside the repo, but
    # it does not matter for git.
    repodir = os.path.dirname(os.path.realpath(__file__))

    # we need to execute the git log command in subshell, because if
    # the log file is specified via relative path, we need to do the
    # redirection of the git-log output to the right file
    os.system("(cd {}; git log -1 --format=%H) > {}"
              .format(repodir, git_commit_file))

    os.system("(cd {}; git --no-pager diff --color=always) > {}"
              .format(repodir, git_diff_file))

    cfg.build_model(warn_unused=True)

    cfg.model.tf_manager.init_saving(variables_file_prefix)

    try:
        check_dataset_and_coders(cfg.model.train_dataset,
                                 cfg.model.runners)
        check_dataset_and_coders(cfg.model.val_dataset,
                                 cfg.model.runners)
    except CheckingException as exc:
        log(str(exc), color='red')
        exit(1)

    Logging.print_header(cfg.model.name)

    # runners_batch_size must be set to avoid problems on GPU
    if cfg.model.runners_batch_size is None:
        cfg.model.runners_batch_size = cfg.model.batch_size

    training_loop(
        tf_manager=cfg.model.tf_manager,
        epochs=cfg.model.epochs,
        trainer=cfg.model.trainer,
        batch_size=cfg.model.batch_size,
        train_dataset=cfg.model.train_dataset,
        val_dataset=cfg.model.val_dataset,
        log_directory=cfg.model.output,
        evaluators=cfg.model.evaluation,
        runners=cfg.model.runners,
        test_datasets=cfg.model.test_datasets,
        logging_period=cfg.model.logging_period,
        validation_period=cfg.model.validation_period,
        val_preview_input_series=cfg.model.val_preview_input_series,
        val_preview_output_series=cfg.model.val_preview_output_series,
        val_preview_num_examples=cfg.model.val_preview_num_examples,
        postprocess=cfg.model.postprocess,
        train_start_offset=cfg.model.train_start_offset,
        runners_batch_size=cfg.model.runners_batch_size,
        initial_variables=cfg.model.initial_variables)
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# nodestructor
# Node.js application static code analysis tool
#
# bl4de | bloorq@gmail.com | Twitter: @_bl4de
#
"""
nodestructor.py - static code analysis for Node.js applications
by bl4de
GitHub: bl4de | Twitter: @_bl4de | hackerone.com/bl4de  bloorq@gmail.com
"""
import sys
import os
import re

from imports.beautyConsole import beautyConsole

BANNER = """
#####  nodestructor.py - static code analysis for Node.js applications  #####
# GitHub: bl4de | Twitter: @_bl4de | hackerone.com/bl4de | bloorq@gmail.com #
"""

PATTERNS = [
    ".*url\.parse\(",
    ".*parse\(",
    ".*normalize\(",
    ".*fs.readFileSync\(",
    ".*bodyParser()",
    ".*handlebars.SafeString\(",
    ".*eval\(",
    ".*(eval\()(.{0,40000})(req\.|req\.query|req\.body|req\.param)",
    ".*(setTimeout\()(.{0,40000})(req\.|req\.query|req\.body|req\.param)",
    ".*(setInterval\()(.{0,40000})(req\.|req\.query|req\.body|req\.param)",
    ".*(new Function\()(.{0,40000})(req\.|req\.query|req\.body|req\.param)",
    ".*(deserialize\(|unserialize\()",
    ".*(require\('js-yaml'\)\.load\(|yaml\.load\()",
    ".*(\[)*('|\")*NODE_TLS_REJECT_UNAUTHORIZED('|\")*(\])*(\s)*=(\s)*('|\")*0('|\")*",
    ".*noEscape(\s)*:(\s)*true",
    ".*SSL_VERIFYPEER(\s)*:(\s)*0",
    ".*createHash\(('|\")md5('|\")",
    ".*createHash\(('|\")sha1('|\")",
    ".*password\s*=\s*['|\"].+['|\"]\s{0,5}[;|.]",
    ".*\s*['|\"]password['|\"]\s*:",
    ".*\s*['|\"]+secret['|\"]+\s*:|\s*secret\s*:\s*['|\"]+",
    ".*username\s*=\s*['|\"].+['|\"]\s{0,5}[;|.]",
    ".*lusca.xssProtection\(false\)|X-XSS-Protection('|\")*(\s)*(:|,)(\s)*('|\")*0",
    ".*(\.createReadStream\()(.{0,40000})(req\.|req\.query|req\.body|req\.param)",
    ".*(\.readFile\()(.{0,40000})(req\.|req\.query|req\.body|req\.param)",
    ".*(res\.redirect\()( *)(req\.|req\.query|req\.body|req\.param)",
    ".*(SELECT|INSERT|UPDATE|DELETE|CREATE|EXPLAIN)(.{0,40000})(req\.|req\.query|req\.body|req\.param)",
    ".*(\.)(find|drop|create|explain|delete|count|bulk|copy)(.{0,4000})({(.{0,4000})\$where:)(.{0,4000})(req\.|req\.query|req\.body|req\.param)",
    ".*(res\.(write|send)\()(.{0,40000})(req\.|req\.query|req\.body|req\.param)",
    ".*(res\.set\()(.{0,40000})(req\.|req\.query|req\.body|req\.param)",
    ".*{{{\s*[\w.\[\]\(\)]+\s*}}}",
    ".*{\s*[\w.\[\]\(\)]+\s*\|\s*s\s*}",
    ".*#{\s*[\w.\[\]\(\)\'\"]+\s*}",
    ".*<%-\s*[\w.\[\]\(\)]+\s*%>",
    ".*<%-\s*@+[\w.\[\]\(\)]+\s*%>",
    ".*require( )*(\()( *)('|\")request('|\")( *)(\))",
    ".*require( )*(\()( *)('|\")request('|\")( *)(\))",
    ".*require( )*(\()( *)('|\")needle('|\")( *)(\))",
    ".*require\(('|\")helmet-csp('|\")\)|helmet.csp|lusca.csp\(|Content-Security-Policy",
    ".*helmet.xframe|lusca.xframe\(|require\(('|\")frameguard('|\")\)|frameguard\(|X-Frame-Options",
    ".*helmet.hsts\({|lusca.hsts\({|hsts\({|require\(('|\")hsts('|\")\)|Strict-Transport-Security",
    ".*Public-Key-Pins(:|,)*",
    ".*helmet.xssFilter\(\)|lusca.xssProtection\(true\)|X-XSS-Protection('|\")*(\s)*(:|,)(\s)*('|\")*1",
    ".*helmet.noSniff|require\(('|\")dont-sniff-mimetype('|\")\)|nosniff\(\)|X-Content-Type-Options('|\")*(\s)*(:|,)(\s)*('\"\")*nosniff",
    ".*require\(('|\")ienoopen('|\")\)|ienoopen\(|helmet.ienoopen\(|X-Download-Options('|\")*(\s)*(:|,)(\s)*('\"\")*noopen",
    "httpOnly(\s)*:(\s)*true|httpOnly",
    ".disable\(('|\")x-powered-by('|\")\)|require\(('|\")hide-powered-by('|\")\)|hidePoweredBy\(|helmet.hidePoweredBy\(|removeHeader\(('|\")X-Powered-By('|\")\)"
]

def show_banner():
    """
    Prints welcome banner with contact info
    """
    print BANNER

def printcodeline(_line, i, _fn, _message):
    """
    Formats and prints line of output
    """
    print "::  line %d :: \33[33;1m%s\33[0m %s " % (i, _fn, _message)
    print beautyConsole.getColor("grey") + _line + beautyConsole.getSpecialChar("endline")

def main(src):
    """
    performs code analysis, line by line
    """
    _file = open(src, "r")
    i = 0
    total = 0
    filenamelength = len(src)
    linelength = 97

    print "-" * 14, " FILE: \33[33m%s\33[0m " % src, "-" * (linelength - filenamelength - 21), "\n"

    for _line in _file:
        i += 1
        __line = _line.strip()
        for __pattern in PATTERNS:
            __rex = re.compile(__pattern)
            if __rex.match(__line):
                total += 1
                printcodeline(_line, i, __pattern + ')',
                              ' dangerous pattern identified')

    if total < 1:
        print "\n\n" + beautyConsole.getColor("green") + \
            "No patterns identified\n" + \
            beautyConsole.getSpecialChar("endline")
    else:
        print "\n\n" + beautyConsole.getColor("red") + \
            "Identified %d code pattern(s)\n" % (total) + \
            beautyConsole.getSpecialChar("endline")

    print beautyConsole.getColor("white") + "-" * 100

# main program
if __name__ == "__main__":

    if len(sys.argv) >= 2:
        show_banner()

        # main program loop
        if len(sys.argv) == 3 and (sys.argv[1] == "-R" or sys.argv[2] == "-R"):
            if sys.argv[1] == "-R":
                BASE_PATH = sys.argv[2] + '/'
                FILE_LIST = os.listdir(sys.argv[2])
            if sys.argv[2] == "-R":
                FILE_LIST = os.listdir(sys.argv[1])
                BASE_PATH = sys.argv[1] + '/'

            for __file in FILE_LIST:
                if __file[-3:] == ".js":
                    full_path = BASE_PATH + __file
                    if os.path.isfile(full_path):
                        main(full_path)
        else:
            main(sys.argv[1])

        print
    else:
        print "Enter JavaScript file name or directory name with file(s) to analyse"
        print "single file: ./nodestructor.py filename.js"
        print "directory: ./nodestructor.py -R dirname"
<commit_msg>-R implemented in the proper way - scan through the whole directory tree; summary added<commit_after>#!/usr/bin/python
#
# nodestructor
# Node.js application static code analysis tool
#
# bl4de | bloorq@gmail.com | Twitter: @_bl4de
#
"""
nodestructor.py - static code analysis for Node.js applications
by bl4de
GitHub: bl4de | Twitter: @_bl4de | hackerone.com/bl4de  bloorq@gmail.com
"""
import sys
import os
import re

from imports.beautyConsole import beautyConsole

BANNER = """
#####  nodestructor.py - static code analysis for Node.js applications  #####
# GitHub: bl4de | Twitter: @_bl4de | hackerone.com/bl4de | bloorq@gmail.com #
"""

PATTERNS = [
    ".*url\.parse\(",
    ".*parse\(",
    ".*normalize\(",
    ".*fs.readFileSync\(",
    ".*bodyParser()",
    ".*handlebars.SafeString\(",
    ".*eval\(",
    ".*(eval\()(.{0,40000})(req\.|req\.query|req\.body|req\.param)",
    ".*(setTimeout\()(.{0,40000})(req\.|req\.query|req\.body|req\.param)",
    ".*(setInterval\()(.{0,40000})(req\.|req\.query|req\.body|req\.param)",
    ".*(new Function\()(.{0,40000})(req\.|req\.query|req\.body|req\.param)",
    ".*(deserialize\(|unserialize\()",
    ".*(require\('js-yaml'\)\.load\(|yaml\.load\()",
    ".*(\[)*('|\")*NODE_TLS_REJECT_UNAUTHORIZED('|\")*(\])*(\s)*=(\s)*('|\")*0('|\")*",
    ".*noEscape(\s)*:(\s)*true",
    ".*SSL_VERIFYPEER(\s)*:(\s)*0",
    ".*createHash\(('|\")md5('|\")",
    ".*createHash\(('|\")sha1('|\")",
    ".*password\s*=\s*['|\"].+['|\"]\s{0,5}[;|.]",
    ".*\s*['|\"]password['|\"]\s*:",
    ".*\s*['|\"]+secret['|\"]+\s*:|\s*secret\s*:\s*['|\"]+",
    ".*username\s*=\s*['|\"].+['|\"]\s{0,5}[;|.]",
    ".*lusca.xssProtection\(false\)|X-XSS-Protection('|\")*(\s)*(:|,)(\s)*('|\")*0",
    ".*(\.createReadStream\()(.{0,40000})(req\.|req\.query|req\.body|req\.param)",
    ".*(\.readFile\()(.{0,40000})(req\.|req\.query|req\.body|req\.param)",
    ".*(res\.redirect\()( *)(req\.|req\.query|req\.body|req\.param)",
    ".*(SELECT|INSERT|UPDATE|DELETE|CREATE|EXPLAIN)(.{0,40000})(req\.|req\.query|req\.body|req\.param)",
    ".*(\.)(find|drop|create|explain|delete|count|bulk|copy)(.{0,4000})({(.{0,4000})\$where:)(.{0,4000})(req\.|req\.query|req\.body|req\.param)",
    ".*(res\.(write|send)\()(.{0,40000})(req\.|req\.query|req\.body|req\.param)",
    ".*(res\.set\()(.{0,40000})(req\.|req\.query|req\.body|req\.param)",
    ".*{{{\s*[\w.\[\]\(\)]+\s*}}}",
    ".*{\s*[\w.\[\]\(\)]+\s*\|\s*s\s*}",
    ".*#{\s*[\w.\[\]\(\)\'\"]+\s*}",
    ".*<%-\s*[\w.\[\]\(\)]+\s*%>",
    ".*<%-\s*@+[\w.\[\]\(\)]+\s*%>",
    ".*require( )*(\()( *)('|\")request('|\")( *)(\))",
    ".*require( )*(\()( *)('|\")request('|\")( *)(\))",
    ".*require( )*(\()( *)('|\")needle('|\")( *)(\))",
    ".*require\(('|\")helmet-csp('|\")\)|helmet.csp|lusca.csp\(|Content-Security-Policy",
    ".*helmet.xframe|lusca.xframe\(|require\(('|\")frameguard('|\")\)|frameguard\(|X-Frame-Options",
    ".*helmet.hsts\({|lusca.hsts\({|hsts\({|require\(('|\")hsts('|\")\)|Strict-Transport-Security",
    ".*Public-Key-Pins(:|,)*",
    ".*helmet.xssFilter\(\)|lusca.xssProtection\(true\)|X-XSS-Protection('|\")*(\s)*(:|,)(\s)*('|\")*1",
    ".*helmet.noSniff|require\(('|\")dont-sniff-mimetype('|\")\)|nosniff\(\)|X-Content-Type-Options('|\")*(\s)*(:|,)(\s)*('\"\")*nosniff",
    ".*require\(('|\")ienoopen('|\")\)|ienoopen\(|helmet.ienoopen\(|X-Download-Options('|\")*(\s)*(:|,)(\s)*('\"\")*noopen",
    "httpOnly(\s)*:(\s)*true|httpOnly",
    ".disable\(('|\")x-powered-by('|\")\)|require\(('|\")hide-powered-by('|\")\)|hidePoweredBy\(|helmet.hidePoweredBy\(|removeHeader\(('|\")X-Powered-By('|\")\)"
]

def show_banner():
    """
    Prints welcome banner with contact info
    """
    print BANNER

def printcodeline(_line, i, _fn, _message):
    """
    Formats and prints line of output
    """
    print "::  line %d :: \33[33;1m%s\33[0m %s " % (i, _fn, _message)
    print beautyConsole.getColor("grey") + _line + \
        beautyConsole.getSpecialChar("endline")

def build_file_list(start_dir):
    for subdir, dirs, files in os.walk(start_dir):
        for file in files:
            print os.path.join(subdir, file)

def main(src):
    """
    performs code analysis, line by line
    """
    global patterns_identified

    _file = open(src, "r")
    i = 0
    patterns_found_in_file = 0
    filenamelength = len(src)
    linelength = 97

    print "-" * 14, " FILE: \33[33m%s\33[0m " % src, "-" * \
        (linelength - filenamelength - 21)

    for _line in _file:
        i += 1
        __line = _line.strip()
        for __pattern in PATTERNS:
            __rex = re.compile(__pattern)
            if __rex.match(__line):
                patterns_found_in_file += 1
                printcodeline(_line, i, __pattern + ')',
                              ' dangerous pattern identified')

    if patterns_found_in_file < 1:
        print beautyConsole.getColor("green") + \
            "No patterns identified\n" + \
            beautyConsole.getSpecialChar("endline")
    else:
        patterns_identified = patterns_identified + patterns_found_in_file
        print "\n\n" + beautyConsole.getColor("red") + \
            "Identified %d code pattern(s)\n" % (patterns_found_in_file) + \
            beautyConsole.getSpecialChar("endline")

    print beautyConsole.getColor("white") + "-" * 100

# main program
if __name__ == "__main__":
    total_files = 0
    patterns_identified = 0

    if len(sys.argv) >= 2:
        show_banner()

        # main program loop
        if len(sys.argv) == 3 and (sys.argv[1] == "-R" or sys.argv[2] == "-R"):
            if sys.argv[1] == "-R":
                BASE_PATH = sys.argv[2]
                FILE_LIST = os.listdir(sys.argv[2])
            if sys.argv[2] == "-R":
                FILE_LIST = os.listdir(sys.argv[1])
                BASE_PATH = sys.argv[1]

            # build_file_list(BASE_PATH)

            for subdir, dirs, files in os.walk(BASE_PATH):
                for __file in files:
                    if __file[-3:] == ".js":
                        main(os.path.join(subdir, __file))
                        total_files = total_files + 1
        else:
            main(sys.argv[1])

        print

        # TODO summary jakies ladniejsze

        print beautyConsole.getColor("cyan")
        print "{} files scanned in total".format(total_files)
        print beautyConsole.getColor(
            "red"), "{} patterns identified".format(patterns_identified)
        print beautyConsole.getColor("white")

    else:
        print "Enter JavaScript file name or directory name with file(s) to analyse"
        print "single file: ./nodestructor.py filename.js"
        print "directory: ./nodestructor.py -R dirname"
<|endoftext|>


<commit_before>import itertools
import logging
import string

import nltk
from nltk.corpus import stopwords
from nltk.stem.porter import *

from encodedEntity import EncodedEntity

nltk.download('punkt');
nltk.download('stopwords');
nltk.download('maxent_treebank_pos_tagger');
nltk.download('maxent_ne_chunker');
nltk.download('words')

def _removePuntuation(text):
    if isinstance(text, str):
        return text.translate(None, string.punctuation)
    elif isinstance(text, unicode):
        remove_punctuation_map = dict((ord(char), None) for char in string.punctuation)
        return text.translate(remove_punctuation_map)

def _removeNonAsciiChars(text):
    return "".join([ch for ch in text if ord(ch)<= 128]);

def getTokens(text):
    lowers = text.lower()
    no_punctuation = _removePuntuation(lowers)
    tokens = nltk.word_tokenize(no_punctuation)
    return tokens

def getStemmedTokens(text):
    safe_text = _removeNonAsciiChars(text);
    tokens = getTokens(safe_text);
    stemmer = PorterStemmer();
    return [stemmer.stem(token) for token in tokens];

def filterStopwords(tokens):
    return [t for t in tokens if not t in stopwords.words('english')];

def getStemmedShingles(text, minLength, maxLength):
    if text is None:
        return [];

    tokens = filterStopwords(getStemmedTokens(text));
    shingles = [];

    for length in range(minLength, maxLength + 1):
        shingles = shingles + \
            [" ".join(tokens[i:i+length]) for i in range(len(tokens) - length + 1)]

    return shingles;

def compareEnglishTexts(text1, text2):
    text1ShinglesSet = set(getStemmedShingles(text1, 3, 3))
    text2ShinglesSet = set(getStemmedShingles(text2, 3, 3))

    intersection = text1ShinglesSet.intersection(text2ShinglesSet)
    shorterLen = min(len(text1ShinglesSet), len(text2ShinglesSet))

    if shorterLen == 0:
        return 0
    else:
        return float(len(intersection))/shorterLen

def compareEnglishTitles(title1, title2):
    title1Tokens = set(filterStopwords(getStemmedTokens(title1)))
    title2Tokens = set(filterStopwords(getStemmedTokens(title2)))

    intersection = title1Tokens.intersection(title2Tokens)
    shorterLen = min(len(title1Tokens), len(title2Tokens))

    if  shorterLen == 0:
        return 0
    else:
        return float(len(intersection))/shorterLen

def getEntities(text):
    try:
        if not text:
            return []

        text = _removeNonAsciiChars(text)

        sentences = nltk.sent_tokenize(text)
        sentences = [nltk.word_tokenize(sent) for sent in sentences]
        sentences = [nltk.pos_tag(sent) for sent in sentences]

        entities = [];
        for sentence in sentences:
            extractedEntities = nltk.ne_chunk(sentence, binary=True).subtrees(
                filter = lambda x: x.label() == 'NE')
            for entity in extractedEntities:
                newEntity = ' '.join([leaf[0] for leaf in entity.leaves()])
                entities.append(newEntity)

        return list(set(entities))
    except Exception as e:
        logging.warning("Could not extract entities for text: '%s'", text)
        return []

def compareEntities(entity1, entity2):
    entity1 = EncodedEntity(entity1)
    entity2 = EncodedEntity(entity2)

    if entity1.encoded == entity2.encoded:
        return 1.0
    else:
        entity1Words = set(entity1.encoded.split())
        entity2Words = set(entity2.encoded.split())
        commonWords = entity1Words.intersection(entity2Words)

        if len(commonWords) > 0:
            return float(len(commonWords))/(len(entity1Words) + len(entity2Words))
        else:
            return 0.0

def compareTextEntities(text1, text2):
    text1Entities = set(getEntities(text1))
    text2Entities = set(getEntities(text2))

    entityPairSimilarities = [compareEntities(x[0], x[1])
                 for x in itertools.product(text1Entities, text2Entities)]

    if len(entityPairSimilarities) == 0:
        return 0;
    else:
        return float(sum(entityPairSimilarities))/len(entityPairSimilarities)
<commit_msg>Weighting enties while comparing<commit_after>import itertools
import logging
import string

import nltk
from nltk.corpus import stopwords
from nltk.stem.porter import *

from encodedEntity import EncodedEntity
from entityTableManager import EntityTableManager

nltk.download('punkt');
nltk.download('stopwords');
nltk.download('maxent_treebank_pos_tagger');
nltk.download('maxent_ne_chunker');
nltk.download('words')

def _removePuntuation(text):
    if isinstance(text, str):
        return text.translate(None, string.punctuation)
    elif isinstance(text, unicode):
        remove_punctuation_map = dict((ord(char), None) for char in string.punctuation)
        return text.translate(remove_punctuation_map)

def _removeNonAsciiChars(text):
    return "".join([ch for ch in text if ord(ch)<= 128]);

def __getEncodedEntityWeight(encodedEntity):
    entityTableManager = EntityTableManager()

    docCount = len(list(entityTableManager.queryByEntity(encodedEntity.encoded)))
    if docCount > 50:
        return 0.0;
    else:
        return (50.0 - docCount)/50

def getTokens(text):
    lowers = text.lower()
    no_punctuation = _removePuntuation(lowers)
    tokens = nltk.word_tokenize(no_punctuation)
    return tokens

def getStemmedTokens(text):
    safe_text = _removeNonAsciiChars(text);
    tokens = getTokens(safe_text);
    stemmer = PorterStemmer();
    return [stemmer.stem(token) for token in tokens];

def filterStopwords(tokens):
    return [t for t in tokens if not t in stopwords.words('english')];

def getStemmedShingles(text, minLength, maxLength):
    if text is None:
        return [];

    tokens = filterStopwords(getStemmedTokens(text));
    shingles = [];

    for length in range(minLength, maxLength + 1):
        shingles = shingles + \
            [" ".join(tokens[i:i+length]) for i in range(len(tokens) - length + 1)]

    return shingles;

def compareEnglishTexts(text1, text2):
    text1ShinglesSet = set(getStemmedShingles(text1, 3, 3))
    text2ShinglesSet = set(getStemmedShingles(text2, 3, 3))

    intersection = text1ShinglesSet.intersection(text2ShinglesSet)
    shorterLen = min(len(text1ShinglesSet), len(text2ShinglesSet))

    if shorterLen == 0:
        return 0
    else:
        return float(len(intersection))/shorterLen

def compareEnglishTitles(title1, title2):
    title1Tokens = set(filterStopwords(getStemmedTokens(title1)))
    title2Tokens = set(filterStopwords(getStemmedTokens(title2)))

    intersection = title1Tokens.intersection(title2Tokens)
    shorterLen = min(len(title1Tokens), len(title2Tokens))

    if  shorterLen == 0:
        return 0
    else:
        return float(len(intersection))/shorterLen

def getEntities(text):
    try:
        if not text:
            return []

        text = _removeNonAsciiChars(text)

        sentences = nltk.sent_tokenize(text)
        sentences = [nltk.word_tokenize(sent) for sent in sentences]
        sentences = [nltk.pos_tag(sent) for sent in sentences]

        entities = [];
        for sentence in sentences:
            extractedEntities = nltk.ne_chunk(sentence, binary=True).subtrees(
                filter = lambda x: x.label() == 'NE')
            for entity in extractedEntities:
                newEntity = ' '.join([leaf[0] for leaf in entity.leaves()])
                entities.append(newEntity)

        return list(set(entities))
    except Exception as e:
        logging.warning("Could not extract entities for text: '%s'", text)
        return []

def compareEntities(entity1, entity2):
    entity1 = EncodedEntity(entity1)
    entity1Weigth = __getEncodedEntityWeight(entity1)
    entity2 = EncodedEntity(entity2)
    entity2Weigth = __getEncodedEntityWeight(entity2)
    combinedWeight = entity1Weigth * entity2Weigth;

    if entity1.encoded == entity2.encoded:
        return 1.0 * combinedWeight
    else:
        entity1Words = set(entity1.encoded.split())
        entity2Words = set(entity2.encoded.split())
        commonWords = entity1Words.intersection(entity2Words)

        if len(commonWords) > 0:
            return combinedWeight * float(len(commonWords))/(len(entity1Words) + len(entity2Words))
        else:
            return 0.0

def compareTextEntities(text1, text2):
    text1Entities = set(getEntities(text1))
    text2Entities = set(getEntities(text2))

    entityPairSimilarities = [compareEntities(x[0], x[1])
                 for x in itertools.product(text1Entities, text2Entities)]

    if len(entityPairSimilarities) == 0:
        return 0;
    else:
        return float(sum(entityPairSimilarities))/len(entityPairSimilarities)
<|endoftext|>