<commit_before>from __future__ import absolute_import

import os
import re
import numpy as np
import torch
from torch.autograd import Variable

stop_words=set(["a","an","the"])

def load_candidates(data_dir, task_id):
    assert task_id > 0 and task_id < 7
    candidates=[]
    candidates_f=None
    candid_dic={}
    if task_id==6:
        candidates_f='dialog-babi-task6-dstc2-candidates.txt'
    else:
        candidates_f='dialog-babi-candidates.txt'
    with open(os.path.join(data_dir,candidates_f)) as f:
        for i,line in enumerate(f):
            candid_dic[line.strip().split(' ',1)[1]] = i
            line=tokenize(line.strip())[1:]
            candidates.append(line)
    # return candidates,dict((' '.join(cand),i) for i,cand in enumerate(candidates))
    return candidates,candid_dic

def load_dialog_task(data_dir, task_id, candid_dic, isOOV):
    '''Load the nth task. There are 20 tasks in total.

    Returns a tuple containing the training and testing data for the task.
    '''
    assert task_id > 0 and task_id < 7

    files = os.listdir(data_dir)
    files = [os.path.join(data_dir, f) for f in files]
    s = 'dialog-babi-task{}-'.format(task_id)
    train_file = [f for f in files if s in f and 'trn' in f][0]
    if isOOV:
        test_file = [f for f in files if s in f and 'tst-OOV' in f][0]
    else:
        test_file = [f for f in files if s in f and 'tst.' in f][0]
    val_file = [f for f in files if s in f and 'dev' in f][0]
    train_data = get_dialogs(train_file,candid_dic)
    test_data = get_dialogs(test_file,candid_dic)
    val_data = get_dialogs(val_file,candid_dic)
    return train_data, test_data, val_data

def tokenize(sent):
    '''Return the tokens of a sentence including punctuation.
    >>> tokenize('Bob dropped the apple. Where is the apple?')
    ['Bob', 'dropped', 'the', 'apple', '.', 'Where', 'is', 'the', 'apple']
    '''
    sent=sent.lower()
    if sent=='<silence>':
        return [sent]
    result=[x.strip() for x in re.split('(\W+)?', sent) if x.strip() and x.strip() not in stop_words]
    if not result:
        result=['<silence>']
    if result[-1]=='.' or result[-1]=='?' or result[-1]=='!':
        result=result[:-1]
    return result

# def parse_dialogs(lines,candid_dic):
#     '''
#         Parse dialogs provided in the babi tasks format
#     '''
#     data=[]
#     context=[]
#     u=None
#     r=None
#     for line in lines:
#         line=str.lower(line.strip())
#         if line:
#             nid, line = line.split(' ', 1)
#             nid = int(nid)
#             if '\t' in line:
#                 u, r = line.split('\t')
#                 u = tokenize(u)
#                 r = tokenize(r)
#                 # temporal encoding, and utterance/response encoding
#                 u.append('$u')
#                 u.append('#'+str(nid))
#                 r.append('$r')
#                 r.append('#'+str(nid))
#                 context.append(u)
#                 context.append(r)
#             else:
#                 r=tokenize(line)
#                 r.append('$r')
#                 r.append('#'+str(nid))
#                 context.append(r)
#         else:
#             context=[x for x in context[:-2] if x]
#             u=u[:-2]
#             r=r[:-2]
#             key=' '.join(r)
#             if key in candid_dic:
#                 r=candid_dic[key]
#                 data.append((context, u,  r))
#             context=[]
#     return data

def parse_dialogs_per_response(lines,candid_dic):
    '''
        Parse dialogs provided in the babi tasks format
    '''
    data=[]
    context=[]
    u=None
    r=None
    for line in lines:
        line=line.strip()
        if line:
            nid, line = line.split(' ', 1)
            nid = int(nid)
            if '\t' in line:
                u, r = line.split('\t')
                a = candid_dic[r]
                u = tokenize(u)
                r = tokenize(r)
                # temporal encoding, and utterance/response encoding
                # data.append((context[:],u[:],candid_dic[' '.join(r)]))
                data.append((context[:],u[:],a))
                u.append('$u')
                u.append('#'+str(nid))
                r.append('$r')
                r.append('#'+str(nid))
                context.append(u)
                context.append(r)
            else:
                r=tokenize(line)
                r.append('$r')
                r.append('#'+str(nid))
                context.append(r)
        else:
            # clear context
            context=[]
    return data

def get_dialogs(f,candid_dic):
    '''Given a file name, read the file, retrieve the dialogs, and then convert the sentences into a single dialog.
    If max_length is supplied, any stories longer than max_length tokens will be discarded.
    '''
    with open(f) as f:
        return parse_dialogs_per_response(f.readlines(),candid_dic)

# def vectorize_candidates_sparse(candidates,word_idx):
#     shape=(len(candidates),len(word_idx)+1)
#     indices=[]
#     values=[]
#     for i,candidate in enumerate(candidates):
#         for w in candidate:
#             indices.append([i,word_idx[w]])
#             values.append(1.0)
#     return tf.SparseTensor(indices,values,shape)

def vectorize_candidates(candidates,word_idx,sentence_size):
    # shape=(len(candidates),sentence_size)
    C=[]
    for i,candidate in enumerate(candidates):
        lc=max(0,sentence_size-len(candidate))
        C.append([word_idx[w] if w in word_idx else 0 for w in candidate] + [0] * lc)
    return Variable(torch.from_numpy(np.array(C))).view(len(candidates), sentence_size)

def vectorize_data(data, word_idx, sentence_size, batch_size, candidates_size, max_memory_size):
    """
    Vectorize stories and queries.

    If a sentence length < sentence_size, the sentence will be padded with 0's.

    If a story length < memory_size, the story will be padded with empty memories.
    Empty memories are 1-D arrays of length sentence_size filled with 0's.

    The answer array is returned as a one-hot encoding.
    """
    S = []
    Q = []
    A = []
    data.sort(key=lambda x:len(x[0]),reverse=True)
    for i, (story, query, answer) in enumerate(data):
        if i%batch_size==0:
            memory_size=max(1,min(max_memory_size,len(story)))
        ss = []
        for i, sentence in enumerate(story, 1):
            ls = max(0, sentence_size - len(sentence))
            ss.append([word_idx[w] if w in word_idx else 0 for w in sentence] + [0] * ls)

        # take only the most recent sentences that fit in memory
        ss = ss[::-1][:memory_size][::-1]

        # pad to memory_size
        lm = max(0, memory_size - len(ss))
        for _ in range(lm):
            ss.append([0] * sentence_size)

        lq = max(0, sentence_size - len(query))
        q = [word_idx[w] if w in word_idx else 0 for w in query] + [0] * lq

        S.append(np.array(ss))
        Q.append(np.array(q))
        A.append(np.array(answer))
    return S, Q, A
<commit_msg>dialog index<commit_after>from __future__ import absolute_import

import os
import re
import numpy as np
import torch
from torch.autograd import Variable

stop_words=set(["a","an","the"])

def load_candidates(data_dir, task_id):
    assert task_id > 0 and task_id < 7
    candidates=[]
    candidates_f=None
    candid_dic={}
    if task_id==6:
        candidates_f='dialog-babi-task6-dstc2-candidates.txt'
    else:
        candidates_f='dialog-babi-candidates.txt'
    with open(os.path.join(data_dir,candidates_f)) as f:
        for i,line in enumerate(f):
            candid_dic[line.strip().split(' ',1)[1]] = i
            line=tokenize(line.strip())[1:]
            candidates.append(line)
    # return candidates,dict((' '.join(cand),i) for i,cand in enumerate(candidates))
    return candidates,candid_dic

def load_dialog_task(data_dir, task_id, candid_dic, isOOV):
    '''Load the nth task. There are 20 tasks in total.

    Returns a tuple containing the training and testing data for the task.
    '''
    assert task_id > 0 and task_id < 7

    files = os.listdir(data_dir)
    files = [os.path.join(data_dir, f) for f in files]
    s = 'dialog-babi-task{}-'.format(task_id)
    train_file = [f for f in files if s in f and 'trn' in f][0]
    if isOOV:
        test_file = [f for f in files if s in f and 'tst-OOV' in f][0]
    else:
        test_file = [f for f in files if s in f and 'tst.' in f][0]
    val_file = [f for f in files if s in f and 'dev' in f][0]
    train_data = get_dialogs(train_file,candid_dic)
    test_data = get_dialogs(test_file,candid_dic)
    val_data = get_dialogs(val_file,candid_dic)
    return train_data, test_data, val_data

def tokenize(sent):
    '''Return the tokens of a sentence including punctuation.
    >>> tokenize('Bob dropped the apple. Where is the apple?')
    ['Bob', 'dropped', 'the', 'apple', '.', 'Where', 'is', 'the', 'apple']
    '''
    sent=sent.lower()
    if sent=='<silence>':
        return [sent]
    result=[x.strip() for x in re.split('(\W+)?', sent) if x.strip() and x.strip() not in stop_words]
    if not result:
        result=['<silence>']
    if result[-1]=='.' or result[-1]=='?' or result[-1]=='!':
        result=result[:-1]
    return result

# def parse_dialogs(lines,candid_dic):
#     '''
#         Parse dialogs provided in the babi tasks format
#     '''
#     data=[]
#     context=[]
#     u=None
#     r=None
#     for line in lines:
#         line=str.lower(line.strip())
#         if line:
#             nid, line = line.split(' ', 1)
#             nid = int(nid)
#             if '\t' in line:
#                 u, r = line.split('\t')
#                 u = tokenize(u)
#                 r = tokenize(r)
#                 # temporal encoding, and utterance/response encoding
#                 u.append('$u')
#                 u.append('#'+str(nid))
#                 r.append('$r')
#                 r.append('#'+str(nid))
#                 context.append(u)
#                 context.append(r)
#             else:
#                 r=tokenize(line)
#                 r.append('$r')
#                 r.append('#'+str(nid))
#                 context.append(r)
#         else:
#             context=[x for x in context[:-2] if x]
#             u=u[:-2]
#             r=r[:-2]
#             key=' '.join(r)
#             if key in candid_dic:
#                 r=candid_dic[key]
#                 data.append((context, u,  r))
#             context=[]
#     return data

def parse_dialogs_per_response(lines,candid_dic):
    '''
        Parse dialogs provided in the babi tasks format
    '''
    data=[]
    context=[]
    u=None
    r=None
    for line in lines:
        line=line.strip()
        if line:
            nid, line = line.split(' ', 1)
            nid = int(nid)
            if '\t' in line:
                u, r = line.split('\t')
                a = candid_dic[r]
                u = tokenize(u)
                r = tokenize(r)
                # temporal encoding, and utterance/response encoding
                # data.append((context[:],u[:],candid_dic[' '.join(r)]))
                data.append((context[:],u[:],a))
                u.append('$u')
                u.append('#'+str(nid))
                r.append('$r')
                r.append('#'+str(nid))
                context.append(u)
                context.append(r)
            else:
                r=tokenize(line)
                r.append('$r')
                r.append('#'+str(nid))
                context.append(r)
        else:
            # clear context
            context=[]
    return data

def get_dialogs(f,candid_dic):
    '''Given a file name, read the file, retrieve the dialogs, and then convert the sentences into a single dialog.
    If max_length is supplied, any stories longer than max_length tokens will be discarded.
    '''
    with open(f) as f:
        return parse_dialogs_per_response(f.readlines(),candid_dic)

# def vectorize_candidates_sparse(candidates,word_idx):
#     shape=(len(candidates),len(word_idx)+1)
#     indices=[]
#     values=[]
#     for i,candidate in enumerate(candidates):
#         for w in candidate:
#             indices.append([i,word_idx[w]])
#             values.append(1.0)
#     return tf.SparseTensor(indices,values,shape)

def vectorize_candidates(candidates,word_idx,sentence_size):
    # shape=(len(candidates),sentence_size)
    C=[]
    for i,candidate in enumerate(candidates):
        lc=max(0,sentence_size-len(candidate))
        C.append([word_idx[w] if w in word_idx else 0 for w in candidate] + [0] * lc)
    return Variable(torch.from_numpy(np.array(C))).view(len(candidates), sentence_size)

def vectorize_data(data, word_idx, sentence_size, batch_size, candidates_size, max_memory_size):
    """
    Vectorize stories and queries.

    If a sentence length < sentence_size, the sentence will be padded with 0's.

    If a story length < memory_size, the story will be padded with empty memories.
    Empty memories are 1-D arrays of length sentence_size filled with 0's.

    The answer array is returned as a one-hot encoding.
    """
    S = []
    Q = []
    A = []
    # data.sort(key=lambda x:len(x[0]),reverse=True)
    dialog_idx = []
    start = -1
    for i, (story, query, answer) in enumerate(data):
        if len(story) == 0:
            if start == -1:
                start = i
            else:
                dialog_idx.append([start, i-1])
                start = i

        if i%batch_size==0:
            memory_size=max(1,min(max_memory_size,len(story)))
        ss = []
        for i, sentence in enumerate(story, 1):
            ls = max(0, sentence_size - len(sentence))
            ss.append([word_idx[w] if w in word_idx else 0 for w in sentence] + [0] * ls)

        # take only the most recent sentences that fit in memory
        ss = ss[::-1][:memory_size][::-1]

        # pad to memory_size
        lm = max(0, memory_size - len(ss))
        for _ in range(lm):
            ss.append([0] * sentence_size)

        lq = max(0, sentence_size - len(query))
        q = [word_idx[w] if w in word_idx else 0 for w in query] + [0] * lq

        S.append(np.array(ss))
        Q.append(np.array(q))
        A.append(np.array(answer))
    return S, Q, A
<|endoftext|>


<commit_before>import codecs
import glob
import json
import random

import numpy as np

class Vocabulary(object):

    def __init__(self):
        self._token_to_id = {}
        self._token_to_count = {}
        self._id_to_token = []
        self._num_tokens = 0
        self._s_id = None
        self._unk_id = None

    @property
    def num_tokens(self):
        return self._num_tokens

    @property
    def unk(self):
        return "<UNK>"

    @property
    def unk_id(self):
        return self._unk_id

    @property
    def s(self):
        return "<S>"

    @property
    def s_id(self):
        return self._s_id

    def add(self, token, count):
        self._token_to_id[token] = self._num_tokens
        self._token_to_count[token] = count
        self._id_to_token.append(token)
        self._num_tokens += 1

    def finalize(self):
        self._s_id = self.get_id(self.s)
        self._unk_id = self.get_id(self.unk)

    def get_id(self, token):
        return self._token_to_id.get(token, self.unk_id)

    def get_token(self, id_):
        return self._id_to_token[id_]

    @staticmethod
    def from_file(filename):
        vocab = Vocabulary()
        with codecs.open(filename, "r", "utf-8") as f:
            for line in f:
                word, count = line.strip().split()
                vocab.add(word, int(count))
        vocab.finalize()
        return vocab

class Dataset(object):

    def __init__(self, vocab, file_pattern, deterministic=False):
        self._vocab = vocab
        self._file_pattern = file_pattern
        self._deterministic = deterministic

    def _parse_sentence(self, line):
        s_id = self._vocab.s_id
        return [s_id] + [self._vocab.get_id(word) for word in line.strip().split()] + [s_id]

    def _parse_file(self, file_name):
        print("Processing file: %s" % file_name)
        with codecs.open(file_name, "r", "utf-8") as f:
            lines = [line.strip() for line in f]
            if not self._deterministic:
                random.shuffle(lines)
            print("Finished processing!")
            for line in lines:
                yield self._parse_sentence(line)

    def _sentence_stream(self, file_stream):
        for file_name in file_stream:
            for sentence in self._parse_file(file_name):
                yield sentence

    def _iterate(self, sentences, batch_size, num_steps):
        print('\x1b[6;30;42m' + '~~~~~>>Almog&Dor debug: ',sentences)
        streams = [None] * batch_size
        x = np.zeros([batch_size, num_steps], np.int32)
        y = np.zeros([batch_size, num_steps], np.int32)
        #w = np.zeros([batch_size, num_steps], np.uint8)
        do_iters = True
        while do_iters:
            x[:] = 0
            y[:] = 0
            #w[:] = 0
            for i in range(batch_size):
                tokens_filled = 0
                try:
                    while tokens_filled < num_steps:
                        if streams[i] is None or len(streams[i]) <= 1:
                            streams[i] = next(sentences)
                        num_tokens = min(len(streams[i]) - 1, num_steps - tokens_filled)
                        x[i, tokens_filled:tokens_filled+num_tokens] = streams[i][:num_tokens]
                        y[i, tokens_filled:tokens_filled + num_tokens] = streams[i][1:num_tokens+1]
                        #w[i, tokens_filled:tokens_filled + num_tokens] = 1
                        streams[i] = streams[i][num_tokens:]
                        tokens_filled += num_tokens
                except StopIteration:
                    do_iters = False
            #if not np.any(w):
            #    return

            #yield x, y, w
            yield x, y

    def iterate_once(self, batch_size, num_steps):
        def file_stream():
            for file_name in glob.glob(self._file_pattern):
                yield file_name
        for value in self._iterate(self._sentence_stream(file_stream()), batch_size, num_steps):
            yield value

    def iterate_forever(self, batch_size, num_steps):
        def file_stream():
            epoch_num = 0
            while True:
                file_patterns = glob.glob(self._file_pattern)
                if not self._deterministic:
                    random.shuffle(file_patterns)
                for file_name in file_patterns:
                    yield file_name
                print('Done epoch: %s' % epoch_num)
                epoch_num += 1
        for value in self._iterate(self._sentence_stream(file_stream()), batch_size, num_steps):
            yield value
<commit_msg>added some debug print<commit_after>import codecs
import glob
import json
import random

import numpy as np

class Vocabulary(object):

    def __init__(self):
        self._token_to_id = {}
        self._token_to_count = {}
        self._id_to_token = []
        self._num_tokens = 0
        self._s_id = None
        self._unk_id = None

    @property
    def num_tokens(self):
        return self._num_tokens

    @property
    def unk(self):
        return "<UNK>"

    @property
    def unk_id(self):
        return self._unk_id

    @property
    def s(self):
        return "<S>"

    @property
    def s_id(self):
        return self._s_id

    def add(self, token, count):
        self._token_to_id[token] = self._num_tokens
        self._token_to_count[token] = count
        self._id_to_token.append(token)
        self._num_tokens += 1

    def finalize(self):
        self._s_id = self.get_id(self.s)
        self._unk_id = self.get_id(self.unk)

    def get_id(self, token):
        return self._token_to_id.get(token, self.unk_id)

    def get_token(self, id_):
        return self._id_to_token[id_]

    @staticmethod
    def from_file(filename):
        vocab = Vocabulary()
        with codecs.open(filename, "r", "utf-8") as f:
            for line in f:
                word, count = line.strip().split()
                vocab.add(word, int(count))
        vocab.finalize()
        return vocab

class Dataset(object):

    def __init__(self, vocab, file_pattern, deterministic=False):
        self._vocab = vocab
        self._file_pattern = file_pattern
        self._deterministic = deterministic

    def _parse_sentence(self, line):
        s_id = self._vocab.s_id
        return [s_id] + [self._vocab.get_id(word) for word in line.strip().split()] + [s_id]

    def _parse_file(self, file_name):
        print("Processing file: %s" % file_name)
        with codecs.open(file_name, "r", "utf-8") as f:
            lines = [line.strip() for line in f]
            if not self._deterministic:
                random.shuffle(lines)
            print("Finished processing!")
            for line in lines:
                yield self._parse_sentence(line)

    def _sentence_stream(self, file_stream):
        for file_name in file_stream:
            for sentence in self._parse_file(file_name):
                yield sentence

    def _iterate(self, sentences, batch_size, num_steps):
        print('\x1b[6;30;42m' + '~~~~~>>Almog&Dor debug: ',sentences,'\x1b[0m')

        streams = [None] * batch_size
        x = np.zeros([batch_size, num_steps], np.int32)
        y = np.zeros([batch_size, num_steps], np.int32)
        #w = np.zeros([batch_size, num_steps], np.uint8)
        do_iters = True
        while do_iters:
            x[:] = 0
            y[:] = 0
            #w[:] = 0
            for i in range(batch_size):
                tokens_filled = 0
                try:
                    while tokens_filled < num_steps:
                        if streams[i] is None or len(streams[i]) <= 1:
                            streams[i] = next(sentences)
                        num_tokens = min(len(streams[i]) - 1, num_steps - tokens_filled)
                        x[i, tokens_filled:tokens_filled+num_tokens] = streams[i][:num_tokens]
                        y[i, tokens_filled:tokens_filled + num_tokens] = streams[i][1:num_tokens+1]
                        #w[i, tokens_filled:tokens_filled + num_tokens] = 1
                        streams[i] = streams[i][num_tokens:]
                        tokens_filled += num_tokens
                except StopIteration:
                    do_iters = False
            #if not np.any(w):
            #    return

            #yield x, y, w
            yield x, y

    def iterate_once(self, batch_size, num_steps):
        def file_stream():
            for file_name in glob.glob(self._file_pattern):
                yield file_name
        for value in self._iterate(self._sentence_stream(file_stream()), batch_size, num_steps):
            yield value

    def iterate_forever(self, batch_size, num_steps):
        def file_stream():
            epoch_num = 0
            while True:
                file_patterns = glob.glob(self._file_pattern)
                if not self._deterministic:
                    random.shuffle(file_patterns)
                for file_name in file_patterns:
                    yield file_name
                print('Done epoch: %s' % epoch_num)
                epoch_num += 1
        for value in self._iterate(self._sentence_stream(file_stream()), batch_size, num_steps):
            yield value
<|endoftext|>


<commit_before>"""
Spike train generation
======================

Functions to generate random spike trains.

Most of these functions were adapted from the NeuroTools stgen module, which was mostly written by Eilif Muller.

:copyright: Copyright 2015 by the Elephant team, see AUTHORS.txt.
:license: Modified BSD, see LICENSE.txt for details.
"""

from __future__ import division
import numpy as np
from quantities import ms, Hz, Quantity
from neo import SpikeTrain

def _homogeneous_process(interval_generator, args, mean_rate, t_start, t_stop, as_array):
    """
    Returns a spike train whose spikes are a realization of a random process generated by the function `interval_generator`
    with the given rate, starting at time `t_start` and stopping `time t_stop`.
    """
    def rescale(x):
        return (x / mean_rate.units).rescale(t_stop.units)

    n = int(((t_stop - t_start) * mean_rate).simplified)
    number = np.ceil(n + 3*np.sqrt(n))
    if number < 100:
        number = min(5 + np.ceil(2*n), 100)
    assert number > 4  # if positive, number cannot be less than 5
    isi = rescale(interval_generator(*args, size=number))
    spikes = np.cumsum(isi)
    spikes += t_start

    i = spikes.searchsorted(t_stop)
    if i == len(spikes):
        # ISI buffer overrun
        extra_spikes = []
        t_last = spikes[-1] + rescale(interval_generator(*args, size=1))[0]
        while t_last < t_stop:
            extra_spikes.append(t_last)
            t_last = t_last + rescale(interval_generator(*args, size=1))[0]
        # np.concatenate does not conserve units
        spikes = Quantity(np.concatenate((spikes, extra_spikes)).magnitude, units=spikes.units)
    else:
        spikes = spikes[:i]

    if as_array:
        spikes = spikes.magnitude
    else:
        spikes = SpikeTrain(spikes, t_start=t_start, t_stop=t_stop, units=spikes.units)

    return spikes

def homogeneous_poisson_process(rate, t_start=0.0*ms, t_stop=1000.0*ms, as_array=False):
    """
    Returns a spike train whose spikes are a realization of a Poisson process
    with the given rate, starting at time `t_start` and stopping time `t_stop`.

    All numerical values should be given as Quantities, e.g. 100*Hz.

    Parameters
    ----------

    rate : Quantity scalar with dimension 1/time
           The rate of the discharge.
    t_start : Quantity scalar with dimension time
              The beginning of the spike train.
    t_stop : Quantity scalar with dimension time
             The end of the spike train.
    as_array : bool
               If True, a NumPy array of sorted spikes is returned,
               rather than a SpikeTrain object.

    Examples
    --------
        >>> from quantities import Hz, ms
        >>> homogeneous_poisson_process(50*Hz, 0*ms, 1000*ms)
        >>> homogeneous_poisson_process(20*Hz, 5000*ms, 10000*ms, as_array=True)

    """
    mean_interval = 1/rate
    return _homogeneous_process(np.random.exponential, (mean_interval,), rate, t_start, t_stop, as_array)

def homogeneous_gamma_process(a, b, t_start=0.0*ms, t_stop=1000.0*ms, as_array=False):
    """
    Returns a spike train whose spikes are a realization of a gamma process
    with the given parameters, starting at time `t_start` and stopping `time t_stop`.
    (average rate will be b/a).

    All numerical values should be given as Quantities, e.g. 100*Hz.

    Parameters
    ----------

    a : int or float
        The shape parameter of the gamma distribution.
    b : Quantity scalar with dimension 1/time
        The rate parameter of the gamma distribution.
    t_start : Quantity scalar with dimension time
              The beginning of the spike train.
    t_stop : Quantity scalar with dimension time
             The end of the spike train.
    as_array : bool
               If True, a NumPy array of sorted spikes is returned,
               rather than a SpikeTrain object.

    Examples
    --------
        >>> from quantities import Hz, ms
        >>> homogeneous_gamma_process(2.0, 50*Hz, 0*ms, 1000*ms)
        >>> homogeneous_gamma_process(5.0, 20*Hz, 5000*ms, 10000*ms, as_array=True)

    """
    rate = b / a
    k, theta = a, (1/b)
    return _homogeneous_process(np.random.gamma, (k, theta), rate, t_start, t_stop, as_array)
<commit_msg>Update spike_train_generation.py<commit_after>"""
Spike train generation
======================

Functions to generate random spike trains.

Most of these functions were adapted from the NeuroTools stgen module, which was mostly written by Eilif Muller.

:copyright: Copyright 2015 by the Elephant team, see AUTHORS.txt.
:license: Modified BSD, see LICENSE.txt for details.
"""

from __future__ import division
import numpy as np
from quantities import ms, Hz, Quantity
from neo import SpikeTrain

def _homogeneous_process(interval_generator, args, mean_rate, t_start, t_stop, as_array):
    """
    Returns a spike train whose spikes are a realization of a random process generated by the function `interval_generator`
    with the given rate, starting at time `t_start` and stopping `time t_stop`.
    """
    def rescale(x):
        return (x / mean_rate.units).rescale(t_stop.units)

    n = int(((t_stop - t_start) * mean_rate).simplified)
    number = np.ceil(n + 3*np.sqrt(n))
    if number < 100:
        number = min(5 + np.ceil(2*n), 100)
    assert number > 4  # if positive, number cannot be less than 5
    isi = rescale(interval_generator(*args, size=number))
    spikes = np.cumsum(isi)
    spikes += t_start

    i = spikes.searchsorted(t_stop)
    if i == len(spikes):
        # ISI buffer overrun
        extra_spikes = []
        t_last = spikes[-1] + rescale(interval_generator(*args, size=1))[0]
        while t_last < t_stop:
            extra_spikes.append(t_last)
            t_last = t_last + rescale(interval_generator(*args, size=1))[0]
        # np.concatenate does not conserve units
        spikes = Quantity(np.concatenate((spikes, extra_spikes)).magnitude, units=spikes.units)
    else:
        spikes = spikes[:i]

    if as_array:
        spikes = spikes.magnitude
    else:
        spikes = SpikeTrain(spikes, t_start=t_start, t_stop=t_stop, units=spikes.units)

    return spikes

def homogeneous_poisson_process(rate, t_start=0.0*ms, t_stop=1000.0*ms, as_array=False):
    """
    Returns a spike train whose spikes are a realization of a Poisson process
    with the given rate, starting at time `t_start` and stopping time `t_stop`.

    All numerical values should be given as Quantities, e.g. 100*Hz.

    Parameters
    ----------

    rate : Quantity scalar with dimension 1/time
           The rate of the discharge.
    t_start : Quantity scalar with dimension time
              The beginning of the spike train.
    t_stop : Quantity scalar with dimension time
             The end of the spike train.
    as_array : bool
               If True, a NumPy array of sorted spikes is returned,
               rather than a SpikeTrain object.

    Examples
    --------
        >>> from quantities import Hz, ms
        >>> homogeneous_poisson_process(50*Hz, 0*ms, 1000*ms)
        >>> homogeneous_poisson_process(20*Hz, 5000*ms, 10000*ms, as_array=True)

    """
    mean_interval = 1/rate
    return _homogeneous_process(np.random.exponential, (mean_interval,), rate, t_start, t_stop, as_array)

def homogeneous_gamma_process(a, b, t_start=0.0*ms, t_stop=1000.0*ms, as_array=False):
    """
    Returns a spike train whose spikes are a realization of a gamma process
    with the given parameters, starting at time `t_start` and stopping time `t_stop`.
    (average rate will be b/a).

    All numerical values should be given as Quantities, e.g. 100*Hz.

    Parameters
    ----------

    a : int or float
        The shape parameter of the gamma distribution.
    b : Quantity scalar with dimension 1/time
        The rate parameter of the gamma distribution.
    t_start : Quantity scalar with dimension time
              The beginning of the spike train.
    t_stop : Quantity scalar with dimension time
             The end of the spike train.
    as_array : bool
               If True, a NumPy array of sorted spikes is returned,
               rather than a SpikeTrain object.

    Examples
    --------
        >>> from quantities import Hz, ms
        >>> homogeneous_gamma_process(2.0, 50*Hz, 0*ms, 1000*ms)
        >>> homogeneous_gamma_process(5.0, 20*Hz, 5000*ms, 10000*ms, as_array=True)

    """
    rate = b / a
    k, theta = a, (1/b)
    return _homogeneous_process(np.random.gamma, (k, theta), rate, t_start, t_stop, as_array)
<|endoftext|>


<commit_before>from django.contrib.auth.models import AbstractUser
from django.contrib.auth.models import UserManager
from django.db import models
from datetime import datetime

class CustomUserManager(UserManager):

    def agree_email_notification(self):
        return self.filter(email_notification=True)

class User(AbstractUser):

    objects = CustomUserManager()
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    email_notification = models.BooleanField(
        default=True,
    )

    @property
    def today_diary(self):
        today = datetime.now().strftime("%Y/%m/%d")
        return self.diary_set.get_or_none(datetime=today)

    def monthly_words(self, **kwargs):
        year = kwargs.get("year")
        month = kwargs.get("month")
        datetime = year+"/"+month+"/"
        diaries = self.diary_set.filter(datetime__contains=datetime)

        monthly_words = []
        for diary in diaries:
            monthly_words += diary.used_words
        return monthly_words

    def monthly_words_count(self, **kwargs):
        monthly_words = self.monthly_words(
            **kwargs,
        )
        monthly_words_count = len(set(monthly_words))
        return monthly_words_count
<commit_msg>fix User model: remove comma in monthly_words_count function<commit_after>from django.contrib.auth.models import AbstractUser
from django.contrib.auth.models import UserManager
from django.db import models
from datetime import datetime

class CustomUserManager(UserManager):

    def agree_email_notification(self):
        return self.filter(email_notification=True)

class User(AbstractUser):

    objects = CustomUserManager()
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    email_notification = models.BooleanField(
        default=True,
    )

    @property
    def today_diary(self):
        today = datetime.now().strftime("%Y/%m/%d")
        return self.diary_set.get_or_none(datetime=today)

    def monthly_words(self, **kwargs):
        year = kwargs.get("year")
        month = kwargs.get("month")
        datetime = year+"/"+month+"/"
        diaries = self.diary_set.filter(datetime__contains=datetime)

        monthly_words = []
        for diary in diaries:
            monthly_words += diary.used_words
        return monthly_words

    def monthly_words_count(self, **kwargs):
        monthly_words = self.monthly_words(**kwargs)
        monthly_words_count = len(set(monthly_words))
        return monthly_words_count
<|endoftext|>


<commit_before>from datetime import datetime

from django.contrib.admin.sites import AdminSite
from django.contrib.contenttypes.models import ContentType
from django.test import TestCase
from django_dynamic_fixture import G
from entity import Entity, EntityRelationship
from entity_subscription.models import Source

from entity_emailer import admin
from entity_emailer.models import Email, EmailTemplate

class SubentityContentTypeQsTest(TestCase):
    def setUp(self):
        self.sub_entity_type_1 = G(ContentType)
        self.sub_entity_type_2 = G(ContentType)
        self.super_entity_type = G(ContentType)
        sub_entity_1 = G(Entity, entity_type=self.sub_entity_type_1)
        sub_entity_2 = G(Entity, entity_type=self.sub_entity_type_2)
        super_entity = G(Entity, entity_type=self.super_entity_type)
        G(EntityRelationship, sub_entity=sub_entity_1, super_entity=super_entity)
        G(EntityRelationship, sub_entity=sub_entity_2, super_entity=super_entity)

    def test_filters_non_subentity_types(self):
        qs = admin.get_subentity_content_type_qs()
        self.assertNotIn(self.super_entity_type, list(qs))

    def test_returns_subentity_types(self):
        qs = admin.get_subentity_content_type_qs()
        self.assertIn(self.sub_entity_type_1, list(qs))
        self.assertIn(self.sub_entity_type_2, list(qs))

class EmailAdminTest(TestCase):
    def setUp(self):
        self.site = AdminSite()
        self.entity = G(Entity, entity_meta={'name':'entity_name'})
        self.email = Email(
            sent=datetime(2014, 1, 1, 12, 34),
            send_to=self.entity,
        )

    def test_has_been_sent(self):
        email_admin = admin.EmailAdmin(Email, self.site)
        sent = email_admin.has_been_sent(self.email)
        self.assertTrue(sent)

    def test_has_not_been_sent(self):
        email_admin = admin.EmailAdmin(Email, self.site)
        not_sent = email_admin.has_been_sent(Email())
        self.assertFalse(not_sent)

    def test_to(self):
        email_admin = admin.EmailAdmin(Email, self.site)
        to = email_admin.to(self.email)
        self.assertEqual(to, 'entity_name')

class CreateEmailFormTest(TestCase):
    def setUp(self):
        test_entity = G(Entity)
        G(Source, name='admin')
        G(EmailTemplate, template_name='html_safe', html_template='{{ html|safe }}')
        self.email_form_data = {
            'subject': 'A Test Email Subject',
            'from_email': 'test@example.com',
            'to_entity': test_entity.id,
            'body': '
This is the email body
',
        }

    def test_save_creates_email(self):
        form = admin.CreateEmailForm(self.email_form_data)
        form.is_valid()
        form.save()
        self.assertTrue(Email.objects.exists())

    def test_save_m2m_exists(self):
        pass
<commit_msg>test_save_m2m_exists and flake8<commit_after>from datetime import datetime

from django.contrib.admin.sites import AdminSite
from django.contrib.contenttypes.models import ContentType
from django.test import TestCase
from django_dynamic_fixture import G
from entity import Entity, EntityRelationship
from entity_subscription.models import Source

from entity_emailer import admin
from entity_emailer.models import Email, EmailTemplate

class SubentityContentTypeQsTest(TestCase):
    def setUp(self):
        self.sub_entity_type_1 = G(ContentType)
        self.sub_entity_type_2 = G(ContentType)
        self.super_entity_type = G(ContentType)
        sub_entity_1 = G(Entity, entity_type=self.sub_entity_type_1)
        sub_entity_2 = G(Entity, entity_type=self.sub_entity_type_2)
        super_entity = G(Entity, entity_type=self.super_entity_type)
        G(EntityRelationship, sub_entity=sub_entity_1, super_entity=super_entity)
        G(EntityRelationship, sub_entity=sub_entity_2, super_entity=super_entity)

    def test_filters_non_subentity_types(self):
        qs = admin.get_subentity_content_type_qs()
        self.assertNotIn(self.super_entity_type, list(qs))

    def test_returns_subentity_types(self):
        qs = admin.get_subentity_content_type_qs()
        self.assertIn(self.sub_entity_type_1, list(qs))
        self.assertIn(self.sub_entity_type_2, list(qs))

class EmailAdminTest(TestCase):
    def setUp(self):
        self.site = AdminSite()
        self.entity = G(Entity, entity_meta={'name': 'entity_name'})
        self.email = Email(
            sent=datetime(2014, 1, 1, 12, 34),
            send_to=self.entity,
        )

    def test_has_been_sent(self):
        email_admin = admin.EmailAdmin(Email, self.site)
        sent = email_admin.has_been_sent(self.email)
        self.assertTrue(sent)

    def test_has_not_been_sent(self):
        email_admin = admin.EmailAdmin(Email, self.site)
        not_sent = email_admin.has_been_sent(Email())
        self.assertFalse(not_sent)

    def test_to(self):
        email_admin = admin.EmailAdmin(Email, self.site)
        to = email_admin.to(self.email)
        self.assertEqual(to, 'entity_name')

class CreateEmailFormTest(TestCase):
    def setUp(self):
        test_entity = G(Entity)
        G(Source, name='admin')
        G(EmailTemplate, template_name='html_safe', html_template='{{ html|safe }}')
        self.email_form_data = {
            'subject': 'A Test Email Subject',
            'from_email': 'test@example.com',
            'to_entity': test_entity.id,
            'body': '
This is the email body
',
        }

    def test_save_creates_email(self):
        form = admin.CreateEmailForm(self.email_form_data)
        form.is_valid()
        form.save()
        self.assertTrue(Email.objects.exists())

    def test_save_m2m_exists(self):
        form = admin.CreateEmailForm(self.email_form_data)
        form.save_m2m()
<|endoftext|>


<commit_before># coding: utf-8

from __future__ import print_function
from __future__ import unicode_literals  # unicode by default

import collections
import logging
import os
import string
import time

import arrow
import bs4
import dateutil.parser
import flask
import internetarchive
from sqlalchemy.orm import joinedload
from sqlalchemy.orm.exc import NoResultFound

from esiclivre import models, extensions

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
VALID_ATTACHMENTS_NAME_CHARS = string.lowercase + string.digits + '.-_'

def parse_date(text):
    return arrow.get(dateutil.parser.parse(text, dayfirst=True))

class ParsedPedido(object):

    def __init__(self, raw_data, browser):

        self._browser = browser
        self._raw_data = raw_data
        self._main_data = self._get_main_data()

        if self._main_data:
            self._details = self._get_details()
            self.attachments = self._get_attachments()
            self.situation = self._get_situation()
            self.request_date = self._get_request_date()
            self.history = self._get_history()

    def _get_main_data(self):
        return self._raw_data.form

    def _get_details(self):
        return self._main_data.select('#ctl00_MainContent_dtv_pedido')[0]

    @property
    def protocol(self):
        data = self._details.tbody.select('tr')[0]
        _, protocol = data.select('td')
        return int(protocol.text.strip())

    @property
    def interessado(self):
        data = self._details.tbody.select('tr')[1]
        _, interessado = data.select('td')
        return interessado.text.strip()

    def _get_request_date(self):
        data = self._details.tbody.select('tr')[2]
        _, opened_at = data.select('td')
        # return dateutil.parser.parse(opened_at.text.strip(), dayfirst=True)
        return parse_date(opened_at.text.strip())

    @property
    def orgao(self):
        data = self._details.tbody.select('tr')[3]
        _, orgao = data.select('td')
        return orgao.text.strip()

    @property
    def contact_option(self):
        data = self._details.tbody.select('tr')[4]
        _, option = data.select('td')
        return option.text.strip()

    @property
    def description(self):
        data = self._details.tbody.select('tr')[5]
        _, desc = data.select('td')
        return desc.text.strip()

    def _get_attachments(self):
        grid = self._main_data.select(
            '#ctl00_MainContent_grid_anexos_resposta')

        if not grid:
            return ()  # 'Sem anexos.'
        else:
            grid = grid[0]

        data = grid.tbody.select('tr')[1:]
        if not data or not any([i.text.split() for i in data]):
            return ()  # 'Sem anexos.'

        result = ()
        for item in data:

            filename, created_at, fileid = item.select('td')

            attachment = collections.namedtuple(
                'PedidoAttachment', ['filename', 'created_at'])
            attachment.filename = clear_attachment_name(filename.text)
            # attachment.created_at = dateutil.parser.parse(
            #     created_at.text.strip(), dayfirst=True)
            attachment.created_at = parse_date(created_at.text.strip())

            upload_attachment_to_internet_archive(
                self.protocol, attachment.filename
            )

            logger.info("anexo")
            logger.info(attachment.filename)
            result += (attachment,)
        return result

    def _get_situation(self):
        fieldset = self._main_data.select('#fildSetSituacao')[0]
        data = fieldset.tbody.select('tr')[0]
        _, situation = data.select('td')[:2]
        return situation.text.strip()

    def _get_history(self):

        grid = self._main_data.select('#ctl00_MainContent_grid_historico')[0]

        # get the 2th to skip header...
        data = grid.tbody.select('tr')[1:]

        result = ()
        for item in data:
            situation, justification, responsible = item.select('td')[1:]
            date = item.span

            history = collections.namedtuple(
                'PedidoHistory',
                ['situation', 'justification', 'responsible', 'date']
            )
            history.situation = situation.text.strip()
            history.justification = justification.text.strip()
            history.responsible = responsible.text.strip()
            history.date = parse_date(date.text.strip())

            result += (history,)

        try:
            result = sorted(result, key=lambda h: h.date)
        except:
            pass

        return result

class Pedidos(object):

    _parsedpedidos = []
    _pedido_pagesource = []

    def set_full_data(self, browser):
        self._full_data = browser.navegador.find_element_by_id(
            'ctl00_MainContent_grid_pedido')

    def get_all_pages_source(self, browser):

        total_of_pedidos = len(self._full_data.find_elements_by_tag_name('a'))
        for pos in range(total_of_pedidos):

            self.set_full_data(browser)
            self._full_data.find_elements_by_tag_name('a')[pos].click()

            pagesource = bs4.BeautifulSoup(browser.navegador.page_source,
                                           "html5lib")

            self._pedido_pagesource.append(pagesource)

            if not pagesource.select('#ctl00_MainContent_grid_anexos_resposta'):  # noqa
                browser.navegador.back()
                continue
            else:
                # a ideia era baixar os anexo apenas durante o processo de
                # parsear o codigo fonte, mas ainda estou com dificuldades
                # para fazer uma requisição valida para o servidor sem usar o
                # selenium. Em um proximo refactoring esse processo pode ser
                # feito em durante o parsing do page source, background ou não.
                attachments = browser.navegador.find_element_by_id(
                    'ctl00_MainContent_grid_anexos_resposta'
                )
                if attachments:
                    self.get_pedido_attachments(attachments)

            browser.navegador.back()
        fix_attachment_name_and_extension()

    def get_pedido_attachments(self, attachments):

        for attachment in attachments.find_elements_by_tag_name('input'):

            # baixar o arquivo
            # TODO: Ignorar arquivos que já existem? Como lidar com a
            # atualização de um anexo?
            attachment.click()

            # a ideia aqui é que se houver algum arquivo .part, ou seja, algum
            # download ainda não terminou, o processo aguarde até esses
            # arquivos serem baixados ou completar N tentativas

            max_retries = 0
            if str('.part') in os.listdir(flask.current_app.config['DOWNLOADS_PATH']):  # noqa
                logger.info("Existe algum download inacabado...")
                while max_retries != 10:

                    download_dir = os.listdir(
                        flask.current_app.config['DOWNLOADS_PATH']
                    )

                    uncomplete_download = next(
                        (arq for arq in download_dir if arq.endswith('.part')),
                        None
                    )

                    if not uncomplete_download:
                        logger.info("Sem downloads inacabados...")
                        break
                    else:
                        logger.info("Aguardar 1 segundo...")
                        time.sleep(1)
                        max_retries += 1

    def process_pedidos(self, browser, page_source=None):

        # Existe a possibilidade da pagina não retornar um codigo fonte válido
        # a classe que estrutura o pedido retornará None se o código
        # fonte não for válido...
        if page_source:
            pedido = ParsedPedido(page_source, browser)
            self._parsedpedidos.append(pedido) if pedido else None
        else:
            self._parsedpedidos = list(p for p in (
                ParsedPedido(pp, browser) for pp in self._pedido_pagesource
                ) if p._main_data
            )

        return self._parsedpedidos

    def get_all_parsed_pedidos(self):
        return self._parsedpedidos

def clear_attachment_name(name):

    name = name.strip().lower()

    return ''.join([l for l in name if l in VALID_ATTACHMENTS_NAME_CHARS])

def fix_attachment_name_and_extension():
    # remover caracter invalidos
    # mudar extensão para lowercase
    # apagar arquivos .part (nessa etapa, se um arquivo ainda é .part é porque
    # o download falhou).
    download_dir = flask.current_app.config['DOWNLOADS_PATH']
    for _file in os.listdir(download_dir):

        _file = _file.decode('utf8')
        logger.info("file: {}".format(_file))
        _file_fullpath = '{}/{}'.format(download_dir, _file)

        if _file.endswith('.part'):
            os.remove(_file_fullpath)
        else:
            os.rename(
                _file_fullpath,
                '{}/{}'.format(download_dir, clear_attachment_name(_file))
            )

def update_pedido_messages(pre_pedido, pedido):
    '''Update messages for a pedido.'''
    new_insetion = False
    for item in pre_pedido.history:
        # Check if is a new msg
        already_inserted = False
        for itemDB in pedido.history:
            # justification can be empty, so using all fields
            if (item.date == itemDB.date and
               item.justification == itemDB.justification and
               item.situation == itemDB.situation and
               item.responsible == itemDB.responsible):
                already_inserted = True

        # Insert if is a new msg
        if not already_inserted:
            message = models.Message()
            message.date = item.date
            message.justification = item.justification
            message.responsible = item.responsible
            message.situation = item.situation
            message.pedido_id = pedido.id
            extensions.db.session.add(message)
            new_insetion = True

    extensions.db.session.commit() if new_insetion else None

def create_pedido_attachments(pre_pedido):

    attachments = []
    for item in pre_pedido.attachments:

        attachment = models.Attachment()
        attachment.created_at = item.created_at
        attachment.name = item.filename
        attachment.ia_url = u'https://archive.org/download/pedido_{}/{}'.format(
            pre_pedido.protocol, item.filename
        )

        extensions.db.session.add(attachment)
        attachments.append(attachment)
    extensions.db.session.commit() if attachments else None

    return attachments

def save_pedido_into_db(pre_pedido):
    # check if there is a object with the same protocol
    pedido = models.Pedido.query.filter(
        models.Pedido.protocol == pre_pedido.protocol)
    pedido = pedido.options(joinedload('history')).first()
    if not pedido:
        # if not, create one
        default_author = flask.current_app.config['DEFAULT_AUTHOR']
        try:
            author = extensions.db.session.query(models.Author).filter(
                models.Author.name == default_author
            ).one()
        except NoResultFound:
            author = models.Author(name=default_author)
            extensions.db.session.add(author)
            extensions.db.session.commit()

        pedido = models.Pedido(protocol=pre_pedido.protocol, author=author)
        pedido.add_keyword('recuperado')
        extensions.db.session.add(pedido)

    # TODO: O que fazer se o orgão não existir no DB?
    if not pre_pedido.orgao:
        orgao_name = 'desconhecido'
    else:
        orgao_name = pre_pedido.orgao

    orgao = models.Orgao.query.filter_by(name=orgao_name).first()
    if not orgao:
        orgao = models.Orgao(name=orgao_name)
        extensions.db.session.add(orgao)
        extensions.db.session.commit()
    pedido.orgao_name = orgao.name

    pedido.interessado = pre_pedido.interessado
    pedido.situation = pre_pedido.situation
    pedido.request_date = pre_pedido.request_date
    pedido.contact_option = pre_pedido.contact_option
    pedido.description = pre_pedido.description

    extensions.db.session.commit()

    update_pedido_messages(pre_pedido, pedido)

    if pre_pedido.attachments:
        pedido.attachments = create_pedido_attachments(pre_pedido)

def upload_attachment_to_internet_archive(pedido_protocol, filename):

    download_dir = flask.current_app.config['DOWNLOADS_PATH']
    downloaded_attachments = os.listdir(download_dir)

    if filename not in [a for a in downloaded_attachments]:
        logger.info("Arquivo {!r} não existe!.".format(filename))
        # TODO: O que fazer se o arquivo não estiver disponivel?
        # Já temos um caso onde o download não completa, mas por falha no
        # servidor do esic.
        return None
    else:

        item = internetarchive.Item('{prefix}_pedido_{protocol}'.format(
            prefix=flask.current_app.config['ATTACHMENT_URL_PREFIX'],
            protocol=pedido_protocol))
        metadata = dict(
            mediatype='pdf',
            creator='OKF',
            created_at=arrow.now().isoformat()
        )
        result = item.upload(
            '{}/{}'.format(download_dir, filename), metadata=metadata
        )

        if not result or result[0].status_code != 200:
            # TODO: O que fazer nessa situação?
            logger.info("Erro ao executar upload.")
        else:
            os.remove('{}/{}'.format(download_dir, filename))

def update_pedidos_list(browser):
    global logger
    logger = browser.logger

    # garantir que a tela inicial seja a de consulta de pedidos.
    browser.ir_para_consultar_pedido()

    pedidos = Pedidos()
    pedidos.set_full_data(browser)
    pedidos.get_all_pages_source(browser)
    pedidos.process_pedidos(browser)

    for pedido in pedidos.get_all_parsed_pedidos():
        save_pedido_into_db(pedido)

    # registrar atualização do dia
    extensions.db.session.add(
        models.PedidosUpdate(date=arrow.now())
    )
    extensions.db.session.commit()

    logger.info("Pedidos atualizados. Atualização registrada.")
<commit_msg>attempt to download/upload only attachments that changed<commit_after># coding: utf-8

from __future__ import print_function
from __future__ import unicode_literals  # unicode by default

import collections
import logging
import os
import string
import time

import arrow
import bs4
import dateutil.parser
import flask
import internetarchive
from sqlalchemy.orm import joinedload
from sqlalchemy.orm.exc import NoResultFound

from esiclivre import models, extensions

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
VALID_ATTACHMENTS_NAME_CHARS = string.lowercase + string.digits + '.-_'

def parse_date(text):
    return arrow.get(dateutil.parser.parse(text, dayfirst=True))

class ParsedPedido(object):

    def __init__(self, raw_data, browser):

        self._browser = browser
        self._raw_data = raw_data
        self._main_data = self._get_main_data()

        if self._main_data:
            self._details = self._get_details()
            self.attachments = self._get_attachments()
            self.situation = self._get_situation()
            self.request_date = self._get_request_date()
            self.history = self._get_history()

    def _get_main_data(self):
        return self._raw_data.form

    def _get_details(self):
        return self._main_data.select('#ctl00_MainContent_dtv_pedido')[0]

    @property
    def protocol(self):
        data = self._details.tbody.select('tr')[0]
        _, protocol = data.select('td')
        return int(protocol.text.strip())

    @property
    def interessado(self):
        data = self._details.tbody.select('tr')[1]
        _, interessado = data.select('td')
        return interessado.text.strip()

    def _get_request_date(self):
        data = self._details.tbody.select('tr')[2]
        _, opened_at = data.select('td')
        # return dateutil.parser.parse(opened_at.text.strip(), dayfirst=True)
        return parse_date(opened_at.text.strip())

    @property
    def orgao(self):
        data = self._details.tbody.select('tr')[3]
        _, orgao = data.select('td')
        return orgao.text.strip()

    @property
    def contact_option(self):
        data = self._details.tbody.select('tr')[4]
        _, option = data.select('td')
        return option.text.strip()

    @property
    def description(self):
        data = self._details.tbody.select('tr')[5]
        _, desc = data.select('td')
        return desc.text.strip()

    def _get_attachments(self):
        grid = self._main_data.select(
            '#ctl00_MainContent_grid_anexos_resposta')

        if not grid:
            return ()  # 'Sem anexos.'
        else:
            grid = grid[0]

        data = grid.tbody.select('tr')[1:]
        if not data or not any([i.text.split() for i in data]):
            return ()  # 'Sem anexos.'

        result = ()
        for item in data:

            filename, created_at, fileid = item.select('td')

            attachment = collections.namedtuple(
                'PedidoAttachment', ['filename', 'created_at'])
            attachment.filename = clear_attachment_name(filename.text)
            # attachment.created_at = dateutil.parser.parse(
            #     created_at.text.strip(), dayfirst=True)
            attachment.created_at = parse_date(created_at.text.strip())

            # upload_attachment_to_internet_archive(
            #     self.protocol, attachment.filename
            # )

            logger.info("anexo")
            logger.info(attachment.filename)
            result += (attachment,)
        return result

    def _get_situation(self):
        fieldset = self._main_data.select('#fildSetSituacao')[0]
        data = fieldset.tbody.select('tr')[0]
        _, situation = data.select('td')[:2]
        return situation.text.strip()

    def _get_history(self):

        grid = self._main_data.select('#ctl00_MainContent_grid_historico')[0]

        # get the 2th to skip header...
        data = grid.tbody.select('tr')[1:]

        result = ()
        for item in data:
            situation, justification, responsible = item.select('td')[1:]
            date = item.span

            history = collections.namedtuple(
                'PedidoHistory',
                ['situation', 'justification', 'responsible', 'date']
            )
            history.situation = situation.text.strip()
            history.justification = justification.text.strip()
            history.responsible = responsible.text.strip()
            history.date = parse_date(date.text.strip())

            result += (history,)

        try:
            result = sorted(result, key=lambda h: h.date)
        except:
            pass

        return result

    def upload_modified_attachments(self):

        attachments_el_id = 'ctl00_MainContent_grid_anexos_resposta'
        # If pedido has no attachments
        if not self._raw_data.select('#' + attachments_el_id):  # noqa
            return None

        # Get the current pedido and its attachments from the DB
        db_pedido = extensions.db.session.query(models.Pedido).filter_by(
            protocol=self.protocol).options(joinedload('attachments')).first()
        db_attachments = db_pedido.attachments if db_pedido else []

        for attachment in self.attachments:
            # Get last created_at time saved in DB
            old_created_at = [a.created_at for a in db_attachments
                              if a.name == attachment.filename]
            old_created_at = old_created_at[0] if old_created_at else None

            # Download and upload attachments that created_at changed
            if not old_created_at or (attachment.created_at != old_created_at):
                logger.info(
                    'Anexo modificado ou novo. Baixando e enviando para IA.')

                attachments_el = self._browser.navegador.find_element_by_id(
                    attachments_el_id)
                if attachments_el:
                    # TODO: atualmente está baixando TODOS os anexos do pedido,
                    # o certo seria baixar apenas o anexo cuja data realmente
                    # mudou
                    self.download_pedido_attachments(attachments_el)

                upload_attachment_to_internet_archive(
                    self.protocol, attachment.filename
                )

    def download_pedido_attachments(self, attachments):

        for attachment in attachments.find_elements_by_tag_name('input'):

            # baixar o arquivo
            attachment.click()

            # a ideia aqui é que se houver algum arquivo .part, ou seja, algum
            # download ainda não terminou, o processo aguarde até esses
            # arquivos serem baixados ou completar N tentativas

            max_retries = 0
            if str('.part') in os.listdir(flask.current_app.config['DOWNLOADS_PATH']):  # noqa
                logger.info("Existe algum download inacabado...")
                while max_retries != 10:

                    download_dir = os.listdir(
                        flask.current_app.config['DOWNLOADS_PATH']
                    )

                    uncomplete_download = next(
                        (arq for arq in download_dir if arq.endswith('.part')),
                        None
                    )

                    if not uncomplete_download:
                        logger.info("Sem downloads inacabados...")
                        break
                    else:
                        logger.info("Aguardar 1 segundo...")
                        time.sleep(1)
                        max_retries += 1

class Pedidos(object):

    _parsedpedidos = []
    _pedido_pagesource = []

    def set_full_data(self, browser):
        self._full_data = browser.navegador.find_element_by_id(
            'ctl00_MainContent_grid_pedido')

    def get_all_pages_source(self, browser):

        total_of_pedidos = len(self._full_data.find_elements_by_tag_name('a'))
        for pos in range(total_of_pedidos):

            self.set_full_data(browser)
            self._full_data.find_elements_by_tag_name('a')[pos].click()

            pagesource = bs4.BeautifulSoup(browser.navegador.page_source,
                                           "html5lib")

            self._pedido_pagesource.append(pagesource)
            pedido = self.process_pedidos(browser, pagesource)
            pedido.upload_modified_attachments()
            browser.navegador.back()
        fix_attachment_name_and_extension()

    def process_pedidos(self, browser, page_source=None):

        # Existe a possibilidade da pagina não retornar um codigo fonte válido
        # a classe que estrutura o pedido retornará None se o código
        # fonte não for válido...
        if page_source:
            pedido = ParsedPedido(page_source, browser)
            self._parsedpedidos.append(pedido) if pedido else None
            return pedido
        else:
            self._parsedpedidos = list(p for p in (
                ParsedPedido(pp, browser) for pp in self._pedido_pagesource
                ) if p._main_data
            )
            return self._parsedpedidos

    def get_all_parsed_pedidos(self):
        return self._parsedpedidos

def clear_attachment_name(name):

    name = name.strip().lower()

    return ''.join([l for l in name if l in VALID_ATTACHMENTS_NAME_CHARS])

def fix_attachment_name_and_extension():
    # remover caracter invalidos
    # mudar extensão para lowercase
    # apagar arquivos .part (nessa etapa, se um arquivo ainda é .part é porque
    # o download falhou).
    download_dir = flask.current_app.config['DOWNLOADS_PATH']
    for _file in os.listdir(download_dir):

        _file = _file.decode('utf8')
        logger.info("file: {}".format(_file))
        _file_fullpath = '{}/{}'.format(download_dir, _file)

        if _file.endswith('.part'):
            os.remove(_file_fullpath)
        else:
            os.rename(
                _file_fullpath,
                '{}/{}'.format(download_dir, clear_attachment_name(_file))
            )

def update_pedido_messages(pre_pedido, pedido):
    '''Update messages for a pedido.'''
    new_insetion = False
    for item in pre_pedido.history:
        # Check if is a new msg
        already_inserted = False
        for itemDB in pedido.history:
            # justification can be empty, so using all fields
            if (item.date == itemDB.date and
               item.justification == itemDB.justification and
               item.situation == itemDB.situation and
               item.responsible == itemDB.responsible):
                already_inserted = True

        # Insert if is a new msg
        if not already_inserted:
            message = models.Message()
            message.date = item.date
            message.justification = item.justification
            message.responsible = item.responsible
            message.situation = item.situation
            message.pedido_id = pedido.id
            extensions.db.session.add(message)
            new_insetion = True

    extensions.db.session.commit() if new_insetion else None

def create_pedido_attachments(pre_pedido):

    attachments = []
    for item in pre_pedido.attachments:

        attachment = models.Attachment()
        attachment.created_at = item.created_at
        attachment.name = item.filename
        attachment.ia_url = u'https://archive.org/download/pedido_{}/{}'.format(
            pre_pedido.protocol, item.filename
        )

        extensions.db.session.add(attachment)
        attachments.append(attachment)
    extensions.db.session.commit() if attachments else None

    return attachments

def save_pedido_into_db(pre_pedido):
    # check if there is a object with the same protocol
    pedido = models.Pedido.query.filter(
        models.Pedido.protocol == pre_pedido.protocol)
    pedido = pedido.options(joinedload('history')).first()
    if not pedido:
        # if not, create one
        default_author = flask.current_app.config['DEFAULT_AUTHOR']
        try:
            author = extensions.db.session.query(models.Author).filter(
                models.Author.name == default_author
            ).one()
        except NoResultFound:
            author = models.Author(name=default_author)
            extensions.db.session.add(author)
            extensions.db.session.commit()

        pedido = models.Pedido(protocol=pre_pedido.protocol, author=author)
        pedido.add_keyword('recuperado')
        extensions.db.session.add(pedido)

    # TODO: O que fazer se o orgão não existir no DB?
    if not pre_pedido.orgao:
        orgao_name = 'desconhecido'
    else:
        orgao_name = pre_pedido.orgao

    orgao = models.Orgao.query.filter_by(name=orgao_name).first()
    if not orgao:
        orgao = models.Orgao(name=orgao_name)
        extensions.db.session.add(orgao)
        extensions.db.session.commit()
    pedido.orgao_name = orgao.name

    pedido.interessado = pre_pedido.interessado
    pedido.situation = pre_pedido.situation
    pedido.request_date = pre_pedido.request_date
    pedido.contact_option = pre_pedido.contact_option
    pedido.description = pre_pedido.description

    extensions.db.session.commit()

    update_pedido_messages(pre_pedido, pedido)

    if pre_pedido.attachments:
        pedido.attachments = create_pedido_attachments(pre_pedido)

def upload_attachment_to_internet_archive(pedido_protocol, filename):

    download_dir = flask.current_app.config['DOWNLOADS_PATH']
    downloaded_attachments = os.listdir(download_dir)

    if filename not in [a.decode('utf8') for a in downloaded_attachments]:
        logger.info("Arquivo {!r} não existe!.".format(filename))
        # TODO: O que fazer se o arquivo não estiver disponivel?
        # Já temos um caso onde o download não completa, mas por falha no
        # servidor do esic.
        return None
    else:

        item = internetarchive.Item('{prefix}_pedido_{protocol}'.format(
            prefix=flask.current_app.config['ATTACHMENT_URL_PREFIX'],
            protocol=pedido_protocol))
        metadata = dict(
            mediatype='pdf',
            creator='OKF',
            created_at=arrow.now().isoformat()
        )
        result = item.upload(
            '{}/{}'.format(download_dir, filename), metadata=metadata
        )

        if not result or result[0].status_code != 200:
            # TODO: O que fazer nessa situação?
            logger.info("Erro ao executar upload.")
        else:
            os.remove('{}/{}'.format(download_dir, filename))

def update_pedidos_list(browser):
    global logger
    logger = browser.logger

    # garantir que a tela inicial seja a de consulta de pedidos.
    browser.ir_para_consultar_pedido()

    pedidos = Pedidos()
    pedidos.set_full_data(browser)
    pedidos.get_all_pages_source(browser)
    # pedidos.process_pedidos(browser)

    for pedido in pedidos.get_all_parsed_pedidos():
        save_pedido_into_db(pedido)

    # registrar atualização do dia
    extensions.db.session.add(
        models.PedidosUpdate(date=arrow.now())
    )
    extensions.db.session.commit()

    logger.info("Pedidos atualizados. Atualização registrada.")
<|endoftext|>


<commit_before>"""
    This file contains 2 examples on how to build raw transactions that can be send via the 'sendrawtransaction' RPC endpoint.

    Note that the perspective taken in these examples is that you want a light wallet like approach. This means that you will
    not have a fully synced Blockchain via neo-python and that you get certain pieces of data from external sources like the API at NEOSCAN.io

    The first example will build a Transaction for sending either NEO or GAS

    The second example will interact with a smart contract on the chain (this can be sending NEP5 tokens or simply quering data).
"""

import binascii
import hashlib
from neo.Implementations.Wallets.peewee.UserWallet import UserWallet
from neo.Core.TX.Transaction import ContractTransaction, TransactionOutput
from neo.Core.TX.InvocationTransaction import InvocationTransaction
from neo.Core.TX.TransactionAttribute import TransactionAttribute, TransactionAttributeUsage
from neo.Core.CoinReference import CoinReference
from neo.SmartContract.ContractParameterContext import ContractParametersContext
from neocore.Fixed8 import Fixed8
from neocore.UInt256 import UInt256
from neocore.UInt160 import UInt160
from neocore.KeyPair import KeyPair
from neo import Blockchain
from base58 import b58decode
from neo.VM.ScriptBuilder import ScriptBuilder

"""
    Example 1

    Sending NEO or GAS happens via a ContractTransaction
    For more information regarding different transaction types read point 3 of:
    http://docs.neo.org/en-us/network/network-protocol.html#data-type

    The standard ``Transaction`` looks as follows (taken from the above source)

    +------+------------+-----------+--------------------------------------------------+
    | Size |   Field    | DataType  |                   Description                    |
    +------+------------+-----------+--------------------------------------------------+
    | 1    | Type       | uint8     | Type of transaction                              |
    | 1    | Version    | uint8     | Trading version, currently 0                     |
    | ?    | -          | -         | Data specific to transaction types               |
    | ?*?  | Attributes | tx_attr[] | Additional features that the transaction has     |
    | 34*? | Inputs     | tx_in[]   | Input                                            |
    | 60*? | Outputs    | tx_out[]  | Output                                           |
    | ?*?  | Scripts    | script[]  | List of scripts used to validate the transaction |
    +------+------------+-----------+--------------------------------------------------+

"""

def example1():
    neo_asset_id = Blockchain.GetSystemShare().Hash
    gas_asset_id = Blockchain.GetSystemCoin().Hash

    source_address = "D3u1UuDJkzUCixp9UUtSSza6Rgt4F9KJqv3mPSzWopgb"
    source_script_hash = address_to_scripthash(source_address)

    destination_address = "Ad9A1xPbuA5YBFr1XPznDwBwQzdckAjCev"
    destination_script_hash = address_to_scripthash(destination_address)

    # Let's start with building a ContractTransaction
    # The constructor already sets the correct `Type` and `Version` fields, so we do not have to worry about that
    contract_tx = ContractTransaction()

    # the ContractTransaction type has no special data, so we do not have to do anything there

    # Next we can add Attributes if we want. Again the various types are described in point 4. of the main link above
    # We will add a simple "description"
    contract_tx.Attributes.append(TransactionAttribute(usage=TransactionAttributeUsage.Description, data="My raw contract transaction description"))

    # The next field we will set are the inputs. The inputs neo-python expects are of the type ``CoinReference``
    # To create these inputs we will need the usual `PrevHash` and `PrevIndex` values.
    # You can get the required data by using e.g. the neoscan.io API: https://api.neoscan.io/docs/index.html#api-v1-get-3
    # The `PrevHash` field equals to neoscan's `balance.unspent[index].txid` key, and `PrevIndex` comes from `balance.unspent[index].n`
    # It is up to the transaction creator to make sure that the sum of all input ``value`` fields is equal to or bigger than the amount that's intended to be send
    # The below values are fictuous and taken from the neoscan example
    input1 = CoinReference(prev_hash=UInt256(data=binascii.unhexlify('19edc4159d2bcf4c538256b17336555b71a3a6a81ecb07493fc7fa218cbafdbd')), prev_index=787)
    contract_tx.inputs = [input1]

    # Next we will create the outputs.
    # The above input has a value of 5. We will create 2 outputs.

    # 1 output for sending 3 NEO to a specific destination address
    send_to_destination_output = TransactionOutput(AssetId=neo_asset_id, Value=Fixed8.FromDecimal(3), script_hash=destination_script_hash)

    # and a second output for sending the change back to ourselves
    return_change_output = TransactionOutput(AssetId=neo_asset_id, Value=Fixed8.FromDecimal(2), script_hash=source_script_hash)

    contract_tx.outputs = [send_to_destination_output, return_change_output]

    # at this point we've build our unsigned transaction and it's time to sign it before we get the raw output that we can send to the network via RPC
    # we need to create a Wallet instance for helping us with signing
    wallet = UserWallet.Create('path', generate_default_key=False)

    # if you have a WIF use
    private_key = KeyPair.PrivateKeyFromWIF("WIF_string")

    # if you have a NEP2 encrypted key use the following instead
    private_key = KeyPair.PrivateKeyFromNEP2("NEP2 key string", "password string")

    # we add the key to our wallet
    wallet.CreateKey(private_key)

    # and now we're ready to sign
    context = ContractParametersContext(contract_tx)
    wallet.Sign(context)

    raw = contract_tx.ToArray()
    raw_signed_transaction = raw.hex()

"""
    Example 2

    Interacting with a Smart contract happens via an InvocationTransaction
    For more information regarding different transaction types read point 3 of:
    http://docs.neo.org/en-us/network/network-protocol.html#data-type

    The standard ``Transaction`` looks as follows (taken from the above source)

    +------+------------+-----------+--------------------------------------------------+
    | Size |   Field    | DataType  |                   Description                    |
    +------+------------+-----------+--------------------------------------------------+
    | 1    | Type       | uint8     | Type of transaction                              |
    | 1    | Version    | uint8     | Trading version, currently 0                     |
    | ?    | -          | -         | Data specific to transaction types               |
    | ?*?  | Attributes | tx_attr[] | Additional features that the transaction has     |
    | 34*? | Inputs     | tx_in[]   | Input                                            |
    | 60*? | Outputs    | tx_out[]  | Output                                           |
    | ?*?  | Scripts    | script[]  | List of scripts used to validate the transaction |
    +------+------------+-----------+--------------------------------------------------+

"""

def example2():
    source_address = "D3u1UuDJkzUCixp9UUtSSza6Rgt4F9KJqv3mPSzWopgb"
    source_script_hash = address_to_scripthash(source_address)

    # start by creating a base InvocationTransaction
    # the inputs, outputs and Type do not have to be set anymore.
    invocation_tx = InvocationTransaction()

    # often times smart contract developers use the function ``CheckWitness`` to determine if the transaction is signed by somebody eligible of calling a certain method
    # in order to pass that check you want to add the corresponding script_hash as a transaction attribute (this is generally the script_hash of the public key you use for signing)
    # Note that for public functions like the NEP-5 'getBalance' and alike this would not be needed, but it doesn't hurt either
    invocation_tx.Attributes.append(TransactionAttribute(usage=TransactionAttributeUsage.Script, data=source_script_hash))

    # next we need to build a 'script' that gets executed against the smart contract.
    # this is basically the script that calls the entry point of the contract with the necessary parameters
    smartcontract_scripthash = UInt160.ParseString("1578103c13e39df15d0d29826d957e85d770d8c9")
    sb = ScriptBuilder()
    # call the NEP-5 `name` method on the contract (assumes contract address is a NEP-5 token)
    sb.EmitAppCallWithOperation(smartcontract_scripthash, 'name')
    invocation_tx.Script = binascii.unhexlify(sb.ToArray())

    # at this point we've build our unsigned transaction and it's time to sign it before we get the raw output that we can send to the network via RPC
    # we need to create a Wallet instance for helping us with signing
    wallet = UserWallet.Create('path', generate_default_key=False)

    # if you have a WIF use
    private_key = KeyPair.PrivateKeyFromWIF("WIF_string")

    # if you have a NEP2 encrypted key use the following instead
    private_key = KeyPair.PrivateKeyFromNEP2("NEP2 key string", "password string")

    # we add the key to our wallet
    wallet.CreateKey(private_key)

    # and now we're ready to sign
    context = ContractParametersContext(invocation_tx)
    wallet.Sign(context)

    raw = invocation_tx.ToArray()
    raw_signed_transaction = raw.hex()

def address_to_scripthash(address: str) -> UInt160:
    """Just a helper method"""
    AddressVersion = 23  # fixed at this point
    data = b58decode(address)
    if len(data) != 25:
        raise ValueError('Not correct Address, wrong length.')
    if data[0] != AddressVersion:
        raise ValueError('Not correct Coin Version')

    checksum_data = data[:21]
    checksum = hashlib.sha256(hashlib.sha256(checksum_data).digest()).digest()[:4]
    if checksum != data[21:]:
        raise Exception('Address format error')
    return UInt160(data=data[1:21])
<commit_msg>Update raw TX examples to build a verifiable TX based on assets on the neo-python-privnet-unittest network (#674)<commit_after>"""
    This file contains 2 examples on how to build raw transactions that can be send via the 'sendrawtransaction' RPC endpoint.

    Note that the perspective taken in these examples is that you want a light wallet like approach. This means that you will
    not have a fully synced Blockchain via neo-python and that you get certain pieces of data from external sources like the API at NEOSCAN.io

    The first example will build a Transaction for sending either NEO or GAS

    The second example will interact with a smart contract on the chain (this can be sending NEP5 tokens or simply quering data).
"""

import binascii
import hashlib
from neo.Implementations.Wallets.peewee.UserWallet import UserWallet
from neo.Core.TX.Transaction import ContractTransaction, TransactionOutput
from neo.Core.TX.InvocationTransaction import InvocationTransaction
from neo.Core.TX.TransactionAttribute import TransactionAttribute, TransactionAttributeUsage
from neo.Core.CoinReference import CoinReference
from neo.SmartContract.ContractParameterContext import ContractParametersContext
from neocore.Fixed8 import Fixed8
from neocore.UInt256 import UInt256
from neocore.UInt160 import UInt160
from neocore.KeyPair import KeyPair
from neo import Blockchain
from base58 import b58decode
from neo.VM.ScriptBuilder import ScriptBuilder
from neo.Wallets.utils import to_aes_key

"""
    Example 1

    Sending NEO or GAS happens via a ContractTransaction
    For more information regarding different transaction types read point 3 of:
    http://docs.neo.org/en-us/network/network-protocol.html#data-type

    The standard ``Transaction`` looks as follows (taken from the above source)

    +------+------------+-----------+--------------------------------------------------+
    | Size |   Field    | DataType  |                   Description                    |
    +------+------------+-----------+--------------------------------------------------+
    | 1    | Type       | uint8     | Type of transaction                              |
    | 1    | Version    | uint8     | Trading version, currently 0                     |
    | ?    | -          | -         | Data specific to transaction types               |
    | ?*?  | Attributes | tx_attr[] | Additional features that the transaction has     |
    | 34*? | Inputs     | tx_in[]   | Input                                            |
    | 60*? | Outputs    | tx_out[]  | Output                                           |
    | ?*?  | Scripts    | script[]  | List of scripts used to validate the transaction |
    +------+------------+-----------+--------------------------------------------------+

"""

def example1():
    neo_asset_id = Blockchain.GetSystemShare().Hash
    gas_asset_id = Blockchain.GetSystemCoin().Hash

    source_address = "AJQ6FoaSXDFzA6wLnyZ1nFN7SGSN2oNTc3"
    source_script_hash = address_to_scripthash(source_address)

    destination_address = "Ad9A1xPbuA5YBFr1XPznDwBwQzdckAjCev"
    destination_script_hash = address_to_scripthash(destination_address)

    # Let's start with building a ContractTransaction
    # The constructor already sets the correct `Type` and `Version` fields, so we do not have to worry about that
    contract_tx = ContractTransaction()

    # the ContractTransaction type has no special data, so we do not have to do anything there

    # Next we can add Attributes if we want. Again the various types are described in point 4. of the main link above
    # We will add a simple "description"
    contract_tx.Attributes.append(TransactionAttribute(usage=TransactionAttributeUsage.Description, data="My raw contract transaction description"))

    # The next field we will set are the inputs. The inputs neo-python expects are of the type ``CoinReference``
    # To create these inputs we will need the usual `PrevHash` and `PrevIndex` values.
    # You can get the required data by using e.g. the neoscan.io API: https://api.neoscan.io/docs/index.html#api-v1-get-3
    # The `PrevHash` field equals to neoscan's `balance.unspent[index].txid` key, and `PrevIndex` comes from `balance.unspent[index].n`
    # It is up to the transaction creator to make sure that the sum of all input ``value`` fields is equal to or bigger than the amount that's intended to be send
    # The below values are taken from data out of the `neo-test1-w.wallet` fixture wallet (a wallet neo-python uses for internal testing)
    input1 = CoinReference(prev_hash=UInt256(data=binascii.unhexlify('949354ea0a8b57dfee1e257a1aedd1e0eea2e5837de145e8da9c0f101bfccc8e')), prev_index=1)
    contract_tx.inputs = [input1]

    # Next we will create the outputs.
    # The above input has a value of 50. We will create 2 outputs.

    # 1 output for sending 3 NEO to a specific destination address
    send_to_destination_output = TransactionOutput(AssetId=neo_asset_id, Value=Fixed8.FromDecimal(3), script_hash=destination_script_hash)

    # and a second output for sending the change back to ourselves
    return_change_output = TransactionOutput(AssetId=neo_asset_id, Value=Fixed8.FromDecimal(47), script_hash=source_script_hash)

    contract_tx.outputs = [send_to_destination_output, return_change_output]

    # at this point we've build our unsigned transaction and it's time to sign it before we get the raw output that we can send to the network via RPC
    # we need to create a Wallet instance for helping us with signing
    wallet = UserWallet.Create('path', to_aes_key('mypassword'), generate_default_key=False)

    # if you have a WIF use the following
    # this WIF comes from the `neo-test1-w.wallet` fixture wallet
    private_key = KeyPair.PrivateKeyFromWIF("Ky94Rq8rb1z8UzTthYmy1ApbZa9xsKTvQCiuGUZJZbaDJZdkvLRV")

    # if you have a NEP2 encrypted key use the following instead
    # private_key = KeyPair.PrivateKeyFromNEP2("NEP2 key string", "password string")

    # we add the key to our wallet
    wallet.CreateKey(private_key)

    # and now we're ready to sign
    context = ContractParametersContext(contract_tx)
    wallet.Sign(context)

    contract_tx.scripts = context.GetScripts()

    print(contract_tx.Hash.ToString())

    raw_tx = contract_tx.ToArray()

    # you can confirm that this transaction is correct by running it against our docker testnet image using the following instructions
    # docker pull cityofzion/neo-python-privnet-unittest:v0.0.1
    # docker run --rm -d --name neo-python-privnet-unittest -p 20333-20336:20333-20336/tcp -p 30333-30336:30333-30336/tcp cityofzion/neo-python-privnet-unittest:v0.0.1
    # curl -X POST http://localhost:30333 -H 'Content-Type: application/json' -d '{ "jsonrpc": "2.0", "id": 1, "method": "sendrawtransaction", "params": ["80000190274d792072617720636f6e7472616374207472616e73616374696f6e206465736372697074696f6e01949354ea0a8b57dfee1e257a1aedd1e0eea2e5837de145e8da9c0f101bfccc8e0100029b7cffdaa674beae0f930ebe6085af9093e5fe56b34a5c220ccdcf6efc336fc500a3e11100000000ea610aa6db39bd8c8556c9569d94b5e5a5d0ad199b7cffdaa674beae0f930ebe6085af9093e5fe56b34a5c220ccdcf6efc336fc5004f2418010000001cc9c05cefffe6cdd7b182816a9152ec218d2ec0014140dbd3cddac5cb2bd9bf6d93701f1a6f1c9dbe2d1b480c54628bbb2a4d536158c747a6af82698edf9f8af1cac3850bcb772bd9c8e4ac38f80704751cc4e0bd0e67232103cbb45da6072c14761c9da545749d9cfd863f860c351066d16df480602a2024c6ac"] }'
    #
    # you should then get the following result
    # {"jsonrpc":"2.0","id":1,"result":true}
    #
    # note that the `params` value comes from `raw_tx`
    # if you want to validate that the transaction is actually persisted on the chain then you can connect to the private net with neo-python and use the cli-command:
    # `tx 41b7b47aecf8573620ae28a844107f02ec14b69a6043b27138f38ae70e70f6b7` (were the TX id comes from calling: contract_tx.Hash.ToString())

"""
    Example 2

    Interacting with a Smart contract happens via an InvocationTransaction
    For more information regarding different transaction types read point 3 of:
    http://docs.neo.org/en-us/network/network-protocol.html#data-type

    The standard ``Transaction`` looks as follows (taken from the above source)

    +------+------------+-----------+--------------------------------------------------+
    | Size |   Field    | DataType  |                   Description                    |
    +------+------------+-----------+--------------------------------------------------+
    | 1    | Type       | uint8     | Type of transaction                              |
    | 1    | Version    | uint8     | Trading version, currently 0                     |
    | ?    | -          | -         | Data specific to transaction types               |
    | ?*?  | Attributes | tx_attr[] | Additional features that the transaction has     |
    | 34*? | Inputs     | tx_in[]   | Input                                            |
    | 60*? | Outputs    | tx_out[]  | Output                                           |
    | ?*?  | Scripts    | script[]  | List of scripts used to validate the transaction |
    +------+------------+-----------+--------------------------------------------------+

"""

def example2():
    source_address = "AJQ6FoaSXDFzA6wLnyZ1nFN7SGSN2oNTc3"
    source_script_hash = address_to_scripthash(source_address)

    # start by creating a base InvocationTransaction
    # the inputs, outputs and Type do not have to be set anymore.
    invocation_tx = InvocationTransaction()

    # often times smart contract developers use the function ``CheckWitness`` to determine if the transaction is signed by somebody eligible of calling a certain method
    # in order to pass that check you want to add the corresponding script_hash as a transaction attribute (this is generally the script_hash of the public key you use for signing)
    # Note that for public functions like the NEP-5 'getBalance' and alike this would not be needed, but it doesn't hurt either
    invocation_tx.Attributes.append(TransactionAttribute(usage=TransactionAttributeUsage.Script, data=source_script_hash))

    # next we need to build a 'script' that gets executed against the smart contract.
    # this is basically the script that calls the entry point of the contract with the necessary parameters
    smartcontract_scripthash = UInt160.ParseString("31730cc9a1844891a3bafd1aa929a4142860d8d3")
    sb = ScriptBuilder()
    # call the NEP-5 `name` method on the contract (assumes contract address is a NEP-5 token)
    sb.EmitAppCallWithOperation(smartcontract_scripthash, 'name')
    invocation_tx.Script = binascii.unhexlify(sb.ToArray())

    # at this point we've build our unsigned transaction and it's time to sign it before we get the raw output that we can send to the network via RPC
    # we need to create a Wallet instance for helping us with signing
    wallet = UserWallet.Create('path', to_aes_key('mypassword'), generate_default_key=False)

    # if you have a WIF use the following
    # this WIF comes from the `neo-test1-w.wallet` fixture wallet
    private_key = KeyPair.PrivateKeyFromWIF("Ky94Rq8rb1z8UzTthYmy1ApbZa9xsKTvQCiuGUZJZbaDJZdkvLRV")

    # if you have a NEP2 encrypted key use the following instead
    # private_key = KeyPair.PrivateKeyFromNEP2("NEP2 key string", "password string")

    # we add the key to our wallet
    wallet.CreateKey(private_key)

    # and now we're ready to sign
    context = ContractParametersContext(invocation_tx)
    wallet.Sign(context)

    invocation_tx.scripts = context.GetScripts()
    raw_tx = invocation_tx.ToArray()

    # you can confirm that this transaction is correct by running it against our docker testnet image using the following instructions
    # docker pull cityofzion/neo-python-privnet-unittest:v0.0.1
    # docker run --rm -d --name neo-python-privnet-unittest -p 20333-20336:20333-20336/tcp -p 30333-30336:30333-30336/tcp cityofzion/neo-python-privnet-unittest:v0.0.1
    # curl -X POST http://localhost:30333 -H 'Content-Type: application/json' -d '{ "jsonrpc": "2.0", "id": 1, "method": "sendrawtransaction", "params": ["d1001b00046e616d6567d3d8602814a429a91afdbaa3914884a1c90c733101201cc9c05cefffe6cdd7b182816a9152ec218d2ec000000141403387ef7940a5764259621e655b3c621a6aafd869a611ad64adcc364d8dd1edf84e00a7f8b11b630a377eaef02791d1c289d711c08b7ad04ff0d6c9caca22cfe6232103cbb45da6072c14761c9da545749d9cfd863f860c351066d16df480602a2024c6ac"] }'
    #
    # you should then get the following result
    # {"jsonrpc":"2.0","id":1,"result":true}
    #
    # note that the `params` value comes from `raw_tx`
    # if you want to validate that the transaction is actually persisted on the chain then you can connect to the private net with neo-python and use the cli-command:
    # `tx 1672df78b7dd21f3516fb0759518dfab29cbe106715504a59a3e12a359850397` (were the TX id comes from calling: invocation_tx.Hash.ToString())

def address_to_scripthash(address: str) -> UInt160:
    """Just a helper method"""
    AddressVersion = 23  # fixed at this point
    data = b58decode(address)
    if len(data) != 25:
        raise ValueError('Not correct Address, wrong length.')
    if data[0] != AddressVersion:
        raise ValueError('Not correct Coin Version')

    checksum_data = data[:21]
    checksum = hashlib.sha256(hashlib.sha256(checksum_data).digest()).digest()[:4]
    if checksum != data[21:]:
        raise Exception('Address format error')
    return UInt160(data=data[1:21])
<|endoftext|>