<commit_before>from Settings import DefineManager
from Utils import LoggingManager
from . import FirebaseDatabaseManager
import pandas as pd
import numpy as np
from fbprophet import Prophet
import tensorflow as tf
import matplotlib
import os
import matplotlib.pyplot as plt
from datetime import datetime
tf.set_random_seed(77)

mockForecastDictionary = {}
realForecastDictionary = {}

def LearningModuleRunner(rawArrayDatas, processId, forecastDay):
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "start of learning #" + str(processId), DefineManager.LOG_LEVEL_INFO)

    global mockForecastDictionary
    global realForecastDictionary
    trainSize=int(len(rawArrayDatas[0])*0.7)
    testSize=len(rawArrayDatas[0])-trainSize

    testY= rawArrayDatas[1][trainSize:]

####################################################################################LSTM
    salesForLstm = rawArrayDatas[1]
    # dataMean=np.mean(salesForLstm)
    # dataSd=np.std(salesForLstm)
    # for i in range(len(salesForLstm)):
    #     if salesForLstm[i]>dataMean+3*dataSd:
    #         salesForLstm[i]=dataMean+3*dataSd
    #     if salesForLstm[i]<dataMean-3*dataSd:
    #         salesForLstm[i] = dataMean + 3 * dataSd
    testY= salesForLstm[trainSize:]
    mockDs = rawArrayDatas[0][:trainSize]
    mockY = list((salesForLstm[:trainSize]))
    mockSales = list(zip(mockDs, mockY))
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "traindata create success",
                                   DefineManager.LOG_LEVEL_INFO)

    ds = rawArrayDatas[0]
    y = list(salesForLstm)
    sales = list(zip(ds, y))
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "testdata create success",
                                   DefineManager.LOG_LEVEL_INFO)
    mockForecastDictionary['LSTM']= LSTM(mockDs, mockY, testSize)
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "mockForecast success",
                                   DefineManager.LOG_LEVEL_INFO)
    tf.reset_default_graph()
    realForecastDictionary['LSTM'] = LSTM(ds,y, forecastDay)
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "realForecast success",
                                   DefineManager.LOG_LEVEL_INFO)

    # mockMinData = np.min(rawArrayDatas[1][:trainSize])
    # mockMaxData = np.max(rawArrayDatas[1][:trainSize])
    # mockForecastDictionary['LSTM'] = [i+mockMinData for i in np.random.beta(mockMinData, mockMaxData, testSize)*(mockMaxData-mockMinData)]
    #
    # minData = np.min(rawArrayDatas[1])
    # maxData = np.max(rawArrayDatas[1])
    # realForecastDictionary['LSTM'] = [i+minData for i in np.random.beta(minData, maxData, forecastDay)*(maxData-minData)]

####################################################################################BAYSEIAN
    trainSize = int(len(rawArrayDatas[0]) * 0.7)
    testSize = len(rawArrayDatas[0]) - trainSize

    testY = rawArrayDatas[1][trainSize:]
    mockDs = rawArrayDatas[0][:trainSize]
    mockY = list(np.log(rawArrayDatas[1][:trainSize]))
    mockSales = list(zip(mockDs, mockY))
    mockPreprocessedData = pd.DataFrame(data=mockSales, columns=['ds', 'y'])
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "traindata create success",
                                   DefineManager.LOG_LEVEL_INFO)

    ds = rawArrayDatas[0]
    y = list(np.log(rawArrayDatas[1]))
    sales = list(zip(ds, y))
    preprocessedData = pd.DataFrame(data=sales, columns=['ds', 'y'])
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "realdata create success",
                                   DefineManager.LOG_LEVEL_INFO)

    mockModel = Prophet(yearly_seasonality=True)
    mockModel.fit(mockPreprocessedData)
    mockFuture = mockModel.make_future_dataframe(periods=testSize, freq='W')
    mockForecastProphetTable = mockModel.predict(mockFuture)
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "mockforecast success",
                                   DefineManager.LOG_LEVEL_INFO)
    mockForecastDictionary['Bayseian'] = [np.exp(y) for y in mockForecastProphetTable['yhat'][-testSize:]]

    model = Prophet(yearly_seasonality=True)
    model.fit(preprocessedData)
    future = model.make_future_dataframe(periods=forecastDay,freq='W')
    forecastProphetTable = model.predict(future)
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "realforecast success",
                                   DefineManager.LOG_LEVEL_INFO)
    realForecastDictionary['Bayseian'] = [np.exp(y) for y in forecastProphetTable['yhat'][-forecastDay:]]

    date = [d.strftime('%Y-%m-%d') for d in forecastProphetTable['ds']]
##################################################################################################ALGORITHM COMPARE
    nameOfBestAlgorithm = 'LSTM'
    minData = rmse(testY, mockForecastDictionary[nameOfBestAlgorithm])
    rms = 0
    for algorithm in realForecastDictionary.keys():
        rms = rmse(testY, mockForecastDictionary[algorithm])
        if rms < minData:
            nameOfBestAlgorithm = algorithm

    data = rawArrayDatas[1] + realForecastDictionary[nameOfBestAlgorithm]

    FirebaseDatabaseManager.StoreOutputData(processId, resultArrayData=data, resultArrayDate= date, status=DefineManager.ALGORITHM_STATUS_DONE)
    return

def ProcessResultGetter(processId):

    status=FirebaseDatabaseManager.GetOutputDataStatus(processId)

    if(status==DefineManager.ALGORITHM_STATUS_DONE):
        date= FirebaseDatabaseManager.GetOutputDateArray(processId)
        data= FirebaseDatabaseManager.GetOutputDataArray(processId)
        return [date, data], DefineManager.ALGORITHM_STATUS_DONE
    elif(status==DefineManager.ALGORITHM_STATUS_WORKING):
        return [[], DefineManager.ALGORITHM_STATUS_WORKING]
    else:
        LoggingManager.PrintLogMessage("LearningManager", "ProcessResultGetter",
                                       "process not available #" + str(processId), DefineManager.LOG_LEVEL_ERROR)
        return [[], DefineManager.ALGORITHM_STATUS_WORKING]

def rmse(a,b):
    sum=0
    for i in range(len(a)):
        sum=sum+(a[i]-b[i])**2
    return np.sqrt(sum/len(a))

def minMaxNormalizer(data):
    numerator=data-np.min(data)
    denominator=np.max(data)-np.min(data)
    return numerator/(denominator+1e-7)

def minMaxDeNormalizer(data, originalData):
    shift=np.min(originalData)
    multiplier=np.max(originalData)-np.min(originalData)
    return (data+shift)*multiplier

def LSTM(ds, y, forecastDay):
    sales = list(zip(ds, y))
    txs = pd.DataFrame(data=sales, columns=['date', 'sales'])
    year = lambda x: datetime.strptime(x, "%Y-%m-%d").year
    day_of_week = lambda x: datetime.strptime(x, "%Y-%m-%d").weekday()
    month = lambda x: datetime.strptime(x, "%Y-%m-%d").month
    # please read docs on how week numbers are calculate
    week_number = lambda x: datetime.strptime(x, "%Y-%m-%d").strftime('%V')

    txs['year'] = txs['date'].map(year)
    txs['month'] = txs['date'].map(month)
    txs['week_number'] = txs['date'].map(week_number)
    txs['day_of_week'] = txs['date'].map(day_of_week)

    seasons = [0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 0]  # dec - feb is winter, then spring, summer, fall etc
    season = lambda x: seasons[(datetime.strptime(x, "%Y-%m-%d").month - 1)]
    day_of_week01s = [0, 0, 0, 0, 0, 1, 1]
    day_of_week01 = lambda x: day_of_week01s[(datetime.strptime(x, "%Y-%m-%d").weekday())]
    txs['season'] = txs['date'].map(season)
    txs['day_of_week01'] = txs['date'].map(day_of_week01)
    originalSales = list(txs['sales'])
    sales = list(txs['sales'])

    tempxy = [list(txs['season']), list(txs['day_of_week']), list(txs['week_number']), sales]

    xy = np.array(tempxy).transpose().astype(np.float)
    originalXY = np.array(tempxy).transpose().astype(np.float)
    xy = minMaxNormalizer(xy)
    # data_dim은 y값 도출을 위한 feature 가지수+1(독립변수 가지수 +1(y포함))
    data_dim = 4

    # data_dim크기의 data 한 묶음이 seq_length만큼 input으로 들어가
    seq_length = 5

    # output_dim(=forecastDays)만큼의 다음날 y_data를 예측
    output_dim = forecastDay

    # hidden_dim은 정말 임의로 설정
    hidden_dim = 10

    # learning rate은 배우는 속도(너무 크지도, 작지도 않게 설정)
    learning_rate = 0.01

    # iterations는 반복 횟수
    iterations = 1000

    x = xy
    y = xy[:, [-1]]

    # build a series dataset(seq_length에 해당하는 전날 X와 다음 forecastDays에 해당하는 Y)
    dataX = []
    dataY = []
    for i in range(0, len(y) - seq_length - forecastDay):
        _x = x[i:i + seq_length]
        _y = y[i + seq_length:i + seq_length + forecastDay]
        _y = np.reshape(_y, (forecastDay))

        dataX.append(_x)
        dataY.append(_y)

    train_size = int(len(dataY) * 0.7)
    test_size = len(dataY) - train_size
    trainX, testX = np.array(dataX[0:train_size]), np.array(dataX[train_size:])
    trainY, testY = np.array(dataY[0:train_size]), np.array(dataY[train_size:])

    X = tf.placeholder(tf.float32, [None, seq_length, data_dim])
    Y = tf.placeholder(tf.float32, [None, forecastDay])

    cell = tf.contrib.rnn.BasicLSTMCell(num_units=hidden_dim, state_is_tuple=True, activation=tf.nn.relu)

    outputs, _states = tf.nn.dynamic_rnn(cell, X, dtype=tf.float32)

    Y_pred = tf.contrib.layers.fully_connected(outputs[:, -1], output_dim, activation_fn=None)

    loss = tf.reduce_sum(tf.square(Y_pred - Y))  # sum of the squares

    optimizer = tf.train.AdamOptimizer(learning_rate)

    train = optimizer.minimize(loss)

    targets = tf.placeholder(tf.float32, [None, 1])
    predictions = tf.placeholder(tf.float32, [None, 1])
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "train prepare success",
                                   DefineManager.LOG_LEVEL_INFO)
    with tf.Session() as sess:
        # 초기화
        init = tf.global_variables_initializer()
        sess.run(init)

        # Training step
        for i in range(iterations):
            _, step_loss = sess.run([train, loss], feed_dict={X: trainX, Y: trainY})
            # print("[step: {}] loss: {}".format(i, step_loss))

        # Test step
        test_predict = minMaxDeNormalizer(sess.run(Y_pred, feed_dict={X: testX}), originalXY)
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "train  success",
                                   DefineManager.LOG_LEVEL_INFO)
    return list(test_predict[-1])

def Bayseian(rawArrayDatas, processId, forecastDay):
    global mockForecastDictionary
    global realForecastDictionary

    trainSize = int(len(rawArrayDatas[0]) * 0.7)
    testSize = len(rawArrayDatas[0]) - trainSize

    testY = rawArrayDatas[1][trainSize:]
    mockDs = rawArrayDatas[0][:trainSize]
    mockY = list(np.log(rawArrayDatas[1][:trainSize]))
    mockSales = list(zip(mockDs, mockY))
    mockPreprocessedData = pd.DataFrame(data=mockSales, columns=['ds', 'y'])
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "traindata create success",
                                   DefineManager.LOG_LEVEL_INFO)

    ds = rawArrayDatas[0]
    y = list(np.log(rawArrayDatas[1]))
    sales = list(zip(ds, y))
    preprocessedData = pd.DataFrame(data=sales, columns=['ds', 'y'])
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "realdata create success",
                                   DefineManager.LOG_LEVEL_INFO)

    mockModel = Prophet()
    mockModel.fit(mockPreprocessedData)
    mockFuture = mockModel.make_future_dataframe(periods=testSize)
    mockForecastProphetTable = mockModel.predict(mockFuture)
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "mockforecast success",
                                   DefineManager.LOG_LEVEL_INFO)
    mockForecastDictionary['Bayseian'] = [np.exp(y) for y in mockForecastProphetTable['yhat'][-testSize:]]

    model = Prophet()
    model.fit(preprocessedData)
    future = model.make_future_dataframe(periods=forecastDay)
    forecastProphetTable = model.predict(future)
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "realforecast success",
                                   DefineManager.LOG_LEVEL_INFO)
    realForecastDictionary['Bayseian'] = [np.exp(y) for y in forecastProphetTable['yhat'][-forecastDay:]]

    date = [d.strftime('%Y-%m-%d') for d in forecastProphetTable['ds']]

    return date<commit_msg>bayseian interval change(day, week)<commit_after>from Settings import DefineManager
from Utils import LoggingManager
from . import FirebaseDatabaseManager
import pandas as pd
import numpy as np
from fbprophet import Prophet
import tensorflow as tf
import matplotlib
import os
import matplotlib.pyplot as plt
from datetime import datetime
tf.set_random_seed(77)

mockForecastDictionary = {}
realForecastDictionary = {}

def LearningModuleRunner(rawArrayDatas, processId, forecastDay):
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "start of learning #" + str(processId), DefineManager.LOG_LEVEL_INFO)

    global mockForecastDictionary
    global realForecastDictionary
    trainSize=int(len(rawArrayDatas[0])*0.7)
    testSize=len(rawArrayDatas[0])-trainSize

    testY= rawArrayDatas[1][trainSize:]

####################################################################################LSTM
    salesForLstm = rawArrayDatas[1]
    # dataMean=np.mean(salesForLstm)
    # dataSd=np.std(salesForLstm)
    # for i in range(len(salesForLstm)):
    #     if salesForLstm[i]>dataMean+3*dataSd:
    #         salesForLstm[i]=dataMean+3*dataSd
    #     if salesForLstm[i]<dataMean-3*dataSd:
    #         salesForLstm[i] = dataMean + 3 * dataSd
    testY= salesForLstm[trainSize:]
    mockDs = rawArrayDatas[0][:trainSize]
    mockY = list((salesForLstm[:trainSize]))
    mockSales = list(zip(mockDs, mockY))
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "traindata create success",
                                   DefineManager.LOG_LEVEL_INFO)

    ds = rawArrayDatas[0]
    y = list(salesForLstm)
    sales = list(zip(ds, y))
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "testdata create success",
                                   DefineManager.LOG_LEVEL_INFO)
    mockForecastDictionary['LSTM']= LSTM(mockDs, mockY, testSize)
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "mockForecast success",
                                   DefineManager.LOG_LEVEL_INFO)
    tf.reset_default_graph()
    realForecastDictionary['LSTM'] = LSTM(ds,y, forecastDay)
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "realForecast success",
                                   DefineManager.LOG_LEVEL_INFO)

    # mockMinData = np.min(rawArrayDatas[1][:trainSize])
    # mockMaxData = np.max(rawArrayDatas[1][:trainSize])
    # mockForecastDictionary['LSTM'] = [i+mockMinData for i in np.random.beta(mockMinData, mockMaxData, testSize)*(mockMaxData-mockMinData)]
    #
    # minData = np.min(rawArrayDatas[1])
    # maxData = np.max(rawArrayDatas[1])
    # realForecastDictionary['LSTM'] = [i+minData for i in np.random.beta(minData, maxData, forecastDay)*(maxData-minData)]

####################################################################################BAYSEIAN
    trainSize = int(len(rawArrayDatas[0]) * 0.7)
    testSize = len(rawArrayDatas[0]) - trainSize

    testY = rawArrayDatas[1][trainSize:]
    mockDs = rawArrayDatas[0][:trainSize]
    mockY = list(np.log(rawArrayDatas[1][:trainSize]))
    mockSales = list(zip(mockDs, mockY))
    mockPreprocessedData = pd.DataFrame(data=mockSales, columns=['ds', 'y'])
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "traindata create success",
                                   DefineManager.LOG_LEVEL_INFO)

    ds = rawArrayDatas[0]
    y = list(np.log(rawArrayDatas[1]))
    sales = list(zip(ds, y))
    preprocessedData = pd.DataFrame(data=sales, columns=['ds', 'y'])
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "realdata create success",
                                   DefineManager.LOG_LEVEL_INFO)

    mockModel = Prophet(yearly_seasonality=True)
    mockModel.fit(mockPreprocessedData)
    mockFuture = mockModel.make_future_dataframe(periods=testSize)
    mockForecastProphetTable = mockModel.predict(mockFuture)
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "mockforecast success",
                                   DefineManager.LOG_LEVEL_INFO)
    mockForecastDictionary['Bayseian'] = [np.exp(y) for y in mockForecastProphetTable['yhat'][-testSize:]]

    model = Prophet(yearly_seasonality=True)
    model.fit(preprocessedData)
    future = model.make_future_dataframe(periods=forecastDay)
    forecastProphetTable = model.predict(future)
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "realforecast success",
                                   DefineManager.LOG_LEVEL_INFO)
    realForecastDictionary['Bayseian'] = [np.exp(y) for y in forecastProphetTable['yhat'][-forecastDay:]]

    date = [d.strftime('%Y-%m-%d') for d in forecastProphetTable['ds']]
##################################################################################################ALGORITHM COMPARE
    nameOfBestAlgorithm = 'LSTM'
    minData = rmse(testY, mockForecastDictionary[nameOfBestAlgorithm])
    rms = 0
    for algorithm in realForecastDictionary.keys():
        rms = rmse(testY, mockForecastDictionary[algorithm])
        if rms < minData:
            nameOfBestAlgorithm = algorithm

    data = rawArrayDatas[1] + realForecastDictionary[nameOfBestAlgorithm]

    FirebaseDatabaseManager.StoreOutputData(processId, resultArrayData=data, resultArrayDate= date, status=DefineManager.ALGORITHM_STATUS_DONE)
    return

def ProcessResultGetter(processId):

    status=FirebaseDatabaseManager.GetOutputDataStatus(processId)

    if(status==DefineManager.ALGORITHM_STATUS_DONE):
        date= FirebaseDatabaseManager.GetOutputDateArray(processId)
        data= FirebaseDatabaseManager.GetOutputDataArray(processId)
        return [date, data], DefineManager.ALGORITHM_STATUS_DONE
    elif(status==DefineManager.ALGORITHM_STATUS_WORKING):
        return [[], DefineManager.ALGORITHM_STATUS_WORKING]
    else:
        LoggingManager.PrintLogMessage("LearningManager", "ProcessResultGetter",
                                       "process not available #" + str(processId), DefineManager.LOG_LEVEL_ERROR)
        return [[], DefineManager.ALGORITHM_STATUS_WORKING]

def rmse(a,b):
    sum=0
    for i in range(len(a)):
        sum=sum+(a[i]-b[i])**2
    return np.sqrt(sum/len(a))

def minMaxNormalizer(data):
    numerator=data-np.min(data)
    denominator=np.max(data)-np.min(data)
    return numerator/(denominator+1e-7)

def minMaxDeNormalizer(data, originalData):
    shift=np.min(originalData)
    multiplier=np.max(originalData)-np.min(originalData)
    return (data+shift)*multiplier

def LSTM(ds, y, forecastDay):
    sales = list(zip(ds, y))
    txs = pd.DataFrame(data=sales, columns=['date', 'sales'])
    year = lambda x: datetime.strptime(x, "%Y-%m-%d").year
    day_of_week = lambda x: datetime.strptime(x, "%Y-%m-%d").weekday()
    month = lambda x: datetime.strptime(x, "%Y-%m-%d").month
    # please read docs on how week numbers are calculate
    week_number = lambda x: datetime.strptime(x, "%Y-%m-%d").strftime('%V')

    txs['year'] = txs['date'].map(year)
    txs['month'] = txs['date'].map(month)
    txs['week_number'] = txs['date'].map(week_number)
    txs['day_of_week'] = txs['date'].map(day_of_week)

    seasons = [0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 0]  # dec - feb is winter, then spring, summer, fall etc
    season = lambda x: seasons[(datetime.strptime(x, "%Y-%m-%d").month - 1)]
    day_of_week01s = [0, 0, 0, 0, 0, 1, 1]
    day_of_week01 = lambda x: day_of_week01s[(datetime.strptime(x, "%Y-%m-%d").weekday())]
    txs['season'] = txs['date'].map(season)
    txs['day_of_week01'] = txs['date'].map(day_of_week01)
    originalSales = list(txs['sales'])
    sales = list(txs['sales'])

    tempxy = [list(txs['season']), list(txs['day_of_week']), list(txs['week_number']), sales]

    xy = np.array(tempxy).transpose().astype(np.float)
    originalXY = np.array(tempxy).transpose().astype(np.float)
    xy = minMaxNormalizer(xy)
    # data_dim은 y값 도출을 위한 feature 가지수+1(독립변수 가지수 +1(y포함))
    data_dim = 4

    # data_dim크기의 data 한 묶음이 seq_length만큼 input으로 들어가
    seq_length = 5

    # output_dim(=forecastDays)만큼의 다음날 y_data를 예측
    output_dim = forecastDay

    # hidden_dim은 정말 임의로 설정
    hidden_dim = 10

    # learning rate은 배우는 속도(너무 크지도, 작지도 않게 설정)
    learning_rate = 0.01

    # iterations는 반복 횟수
    iterations = 1000

    x = xy
    y = xy[:, [-1]]

    # build a series dataset(seq_length에 해당하는 전날 X와 다음 forecastDays에 해당하는 Y)
    dataX = []
    dataY = []
    for i in range(0, len(y) - seq_length - forecastDay):
        _x = x[i:i + seq_length]
        _y = y[i + seq_length:i + seq_length + forecastDay]
        _y = np.reshape(_y, (forecastDay))

        dataX.append(_x)
        dataY.append(_y)

    train_size = int(len(dataY) * 0.7)
    test_size = len(dataY) - train_size
    trainX, testX = np.array(dataX[0:train_size]), np.array(dataX[train_size:])
    trainY, testY = np.array(dataY[0:train_size]), np.array(dataY[train_size:])

    X = tf.placeholder(tf.float32, [None, seq_length, data_dim])
    Y = tf.placeholder(tf.float32, [None, forecastDay])

    cell = tf.contrib.rnn.BasicLSTMCell(num_units=hidden_dim, state_is_tuple=True, activation=tf.nn.relu)

    outputs, _states = tf.nn.dynamic_rnn(cell, X, dtype=tf.float32)

    Y_pred = tf.contrib.layers.fully_connected(outputs[:, -1], output_dim, activation_fn=None)

    loss = tf.reduce_sum(tf.square(Y_pred - Y))  # sum of the squares

    optimizer = tf.train.AdamOptimizer(learning_rate)

    train = optimizer.minimize(loss)

    targets = tf.placeholder(tf.float32, [None, 1])
    predictions = tf.placeholder(tf.float32, [None, 1])
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "train prepare success",
                                   DefineManager.LOG_LEVEL_INFO)
    with tf.Session() as sess:
        # 초기화
        init = tf.global_variables_initializer()
        sess.run(init)

        # Training step
        for i in range(iterations):
            _, step_loss = sess.run([train, loss], feed_dict={X: trainX, Y: trainY})
            # print("[step: {}] loss: {}".format(i, step_loss))
        LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "test success",
                                       DefineManager.LOG_LEVEL_INFO)
        # Test step
        test_predict = minMaxDeNormalizer(sess.run(Y_pred, feed_dict={X: testX}), originalXY)
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "test success",
                                   DefineManager.LOG_LEVEL_INFO)
    return list(test_predict[-1])

def Bayseian(rawArrayDatas, processId, forecastDay):
    global mockForecastDictionary
    global realForecastDictionary

    trainSize = int(len(rawArrayDatas[0]) * 0.7)
    testSize = len(rawArrayDatas[0]) - trainSize

    testY = rawArrayDatas[1][trainSize:]
    mockDs = rawArrayDatas[0][:trainSize]
    mockY = list(np.log(rawArrayDatas[1][:trainSize]))
    mockSales = list(zip(mockDs, mockY))
    mockPreprocessedData = pd.DataFrame(data=mockSales, columns=['ds', 'y'])
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "traindata create success",
                                   DefineManager.LOG_LEVEL_INFO)

    ds = rawArrayDatas[0]
    y = list(np.log(rawArrayDatas[1]))
    sales = list(zip(ds, y))
    preprocessedData = pd.DataFrame(data=sales, columns=['ds', 'y'])
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "realdata create success",
                                   DefineManager.LOG_LEVEL_INFO)

    mockModel = Prophet()
    mockModel.fit(mockPreprocessedData)
    mockFuture = mockModel.make_future_dataframe(periods=testSize)
    mockForecastProphetTable = mockModel.predict(mockFuture)
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "mockforecast success",
                                   DefineManager.LOG_LEVEL_INFO)
    mockForecastDictionary['Bayseian'] = [np.exp(y) for y in mockForecastProphetTable['yhat'][-testSize:]]

    model = Prophet()
    model.fit(preprocessedData)
    future = model.make_future_dataframe(periods=forecastDay)
    forecastProphetTable = model.predict(future)
    LoggingManager.PrintLogMessage("LearningManager", "LearningModuleRunner", "realforecast success",
                                   DefineManager.LOG_LEVEL_INFO)
    realForecastDictionary['Bayseian'] = [np.exp(y) for y in forecastProphetTable['yhat'][-forecastDay:]]

    date = [d.strftime('%Y-%m-%d') for d in forecastProphetTable['ds']]

    return date<|endoftext|>


<commit_before>import redis
import json
import datetime
import click
import backoff
from opsgenie.swagger_client import AlertApi
from opsgenie.swagger_client import configuration
from opsgenie.swagger_client.models import CreateAlertRequest
from opsgenie.swagger_client.rest import ApiException
from textwrap import dedent

MAX_TRIES = 5
QUEUE_AGE_HASH_NAME = "queue_age_monitoring"
DATE_FORMAT = '%Y-%m-%d %H:%M:%S.%f'

class RedisWrapper(object):
    def __init__(self, *args, **kwargs):
        self.redis = redis.StrictRedis(*args, **kwargs)

    @backoff.on_exception(backoff.expo,
                          (redis.exceptions.TimeoutError,
                           redis.exceptions.ConnectionError),
                          max_tries=MAX_TRIES)
    def keys(self):
        return self.redis.keys()

    @backoff.on_exception(backoff.expo,
                          (redis.exceptions.TimeoutError,
                           redis.exceptions.ConnectionError),
                          max_tries=MAX_TRIES)
    def type(self, key):
        return self.redis.type(key)

    @backoff.on_exception(backoff.expo,
                          (redis.exceptions.TimeoutError,
                           redis.exceptions.ConnectionError),
                          max_tries=MAX_TRIES)
    def llen(self, key):
        return self.redis.llen(key)

    @backoff.on_exception(backoff.expo,
                          (redis.exceptions.TimeoutError,
                           redis.exceptions.ConnectionError),
                          max_tries=MAX_TRIES)
    def lindex(self, key, index):
        return self.redis.lindex(key, index)

    @backoff.on_exception(backoff.expo,
                          (redis.exceptions.TimeoutError,
                           redis.exceptions.ConnectionError),
                          max_tries=MAX_TRIES)
    def hgetall(self, key):
        return self.redis.hgetall(key)

    @backoff.on_exception(backoff.expo,
                          (redis.exceptions.TimeoutError,
                           redis.exceptions.ConnectionError),
                          max_tries=MAX_TRIES)
    def delete(self, key):
        return self.redis.delete(key)

    @backoff.on_exception(backoff.expo,
                          (redis.exceptions.TimeoutError,
                           redis.exceptions.ConnectionError),
                          max_tries=MAX_TRIES)
    def hset(self, *args):
        return self.redis.hset(*args)

    @backoff.on_exception(backoff.expo,
                          (redis.exceptions.TimeoutError,
                           redis.exceptions.ConnectionError),
                          max_tries=MAX_TRIES)
    def hmset(self, *args):
        return self.redis.hmset(*args)

def datetime_from_str(string):
    return datetime.datetime.strptime(string, DATE_FORMAT)

def str_from_datetime(dt):
    return dt.strftime(DATE_FORMAT)

def unpack_state(packed_state):
    decoded_state = {k.decode("utf-8"): v.decode("utf-8") for k, v in packed_state.items()}
    unpacked_state = {}

    for key, value in decoded_state.items():
        decoded_value = json.loads(value)
        unpacked_state[key] = {
            'correlation_id': decoded_value['correlation_id'],
            'first_occurance_time': datetime_from_str(decoded_value['first_occurance_time']),
        }

    return unpacked_state

def pack_state(unpacked_state):
    packed_state = {}
    for queue_name, queue_state in unpacked_state.items():
        dt_str = str_from_datetime(queue_state['first_occurance_time'])
        packed_state[queue_name] = json.dumps({
            'correlation_id': queue_state['correlation_id'],
            'first_occurance_time': dt_str,
        })
    return packed_state

def build_new_state(old_state, queue_first_items, current_time):
    new_state = {}
    for queue_name, first_item_encoded in queue_first_items.items():
        # TODO: Catch/Handle exception if not json
        first_item = json.loads(first_item_encoded.decode("utf-8"))
        # TODO: Handle keys missing in data
        correlation_id = first_item['properties']['correlation_id']
        first_occurance_time = current_time

        if queue_name in old_state:
            old_correlation_id = old_state[queue_name]['correlation_id']
            if old_correlation_id == correlation_id:
                first_occurance_time = old_state[queue_name]['first_occurance_time']

        new_state[queue_name] = {
            'correlation_id': correlation_id,
            'first_occurance_time': first_occurance_time,
        }

    return new_state

def should_send_alert(first_occurance_time, current_time, threshold):
    time_delta = current_time - first_occurance_time
    return time_delta.total_seconds() > threshold

@backoff.on_exception(backoff.expo,
                          (ApiException),
                          max_tries=MAX_TRIES)
def send_alert(opsgenie_api_key, environment, deploy, queue_name, threshold):

    configuration.api_key['Authorization'] = opsgenie_api_key
    configuration.api_key_prefix['Authorization'] = 'GenieKey'

    alert_message = str.format("{}-{} {} queue is stale. Stationary for {}s", environment, deploy, queue_name, threshold)
    print(alert_message)
    response = AlertApi().create_alert(body=CreateAlertRequest(message=alert_message))
    print('request id: {}'.format(response.request_id))
    print('took: {}'.format(response.took))
    print('result: {}'.format(response.result))

def print_info(queue_name, do_alert, first_occurance_time, current_time, threshold, default_threshold):
    output = str.format(
        """
            ---------------------------------------------
            queue_name = {}
            do_alert = {}
            first_occurance_time = {}
            current_time = {}
            threshold = {}
            default_threshold = {}
        """, queue_name, do_alert, first_occurance_time, current_time, threshold, default_threshold)
    print(dedent(output))

@click.command()
@click.option('--host', '-h', default='localhost',
              help='Hostname of redis server', required=True)
@click.option('--port', '-p', default=6379, help='Port of redis server')
@click.option('--environment', '-e', required=True)
@click.option('--deploy', '-d', required=True,
              help="Deployment (i.e. edx or edge)")
@click.option('--default-threshold', default=60,
              help='Default queue maximum item age in seconds')
@click.option('--queue-threshold', type=(str, int), multiple=True,
              help='Per queue maximum item age (seconds) in format --queue-threshold'
              + ' {queue_name} {threshold}. May be used multiple times.')
@click.option('--opsgenie-api-key', '-k', envvar='OPSGENIE_API_KEY', required=True)
def check_queues(host, port, environment, deploy, default_threshold, queue_threshold, opsgenie_api_key):

    thresholds = dict(queue_threshold)

    timeout = 1
    redis_client = RedisWrapper(host=host, port=port, socket_timeout=timeout,
                                socket_connect_timeout=timeout)
    queue_names = set([k.decode() for k in redis_client.keys()
                       if (redis_client.type(k) == b'list' and
                           not k.decode().endswith(".pidbox") and
                           not k.decode().startswith("_kombu"))])

    queue_age_hash = redis_client.hgetall(QUEUE_AGE_HASH_NAME)
    old_state = unpack_state(queue_age_hash)

    queue_first_items = {}
    current_time = datetime.datetime.now()
    for queue_name in queue_names:
        queue_first_items[queue_name] = redis_client.lindex(queue_name, 0)

    new_state = build_new_state(old_state, queue_first_items, current_time)

    redis_client.delete(QUEUE_AGE_HASH_NAME)
    if new_state:
        redis_client.hmset(QUEUE_AGE_HASH_NAME, pack_state(new_state))

    for queue_name in queue_names:
        threshold = default_threshold
        if queue_name in thresholds:
            threshold = thresholds[queue_name]

        first_occurance_time = new_state[queue_name]['first_occurance_time']
        do_alert = should_send_alert(first_occurance_time, current_time, threshold)

        if do_alert:
            send_alert(opsgenie_api_key, environment, deploy, queue_name, threshold)
        print_info(queue_name, do_alert, first_occurance_time, current_time, threshold, default_threshold)

if __name__ == '__main__':
    check_queues()
<commit_msg>Pep 8 fixes<commit_after>import json
import datetime
import redis
import click
import backoff
from opsgenie.swagger_client import AlertApi
from opsgenie.swagger_client import configuration
from opsgenie.swagger_client.models import CreateAlertRequest
from opsgenie.swagger_client.rest import ApiException
from textwrap import dedent

MAX_TRIES = 5
QUEUE_AGE_HASH_NAME = "queue_age_monitoring"
DATE_FORMAT = '%Y-%m-%d %H:%M:%S.%f'

class RedisWrapper(object):
    def __init__(self, *args, **kwargs):
        self.redis = redis.StrictRedis(*args, **kwargs)

    @backoff.on_exception(backoff.expo,
                          (redis.exceptions.TimeoutError,
                           redis.exceptions.ConnectionError),
                          max_tries=MAX_TRIES)
    def keys(self):
        return self.redis.keys()

    @backoff.on_exception(backoff.expo,
                          (redis.exceptions.TimeoutError,
                           redis.exceptions.ConnectionError),
                          max_tries=MAX_TRIES)
    def type(self, key):
        return self.redis.type(key)

    @backoff.on_exception(backoff.expo,
                          (redis.exceptions.TimeoutError,
                           redis.exceptions.ConnectionError),
                          max_tries=MAX_TRIES)
    def llen(self, key):
        return self.redis.llen(key)

    @backoff.on_exception(backoff.expo,
                          (redis.exceptions.TimeoutError,
                           redis.exceptions.ConnectionError),
                          max_tries=MAX_TRIES)
    def lindex(self, key, index):
        return self.redis.lindex(key, index)

    @backoff.on_exception(backoff.expo,
                          (redis.exceptions.TimeoutError,
                           redis.exceptions.ConnectionError),
                          max_tries=MAX_TRIES)
    def hgetall(self, key):
        return self.redis.hgetall(key)

    @backoff.on_exception(backoff.expo,
                          (redis.exceptions.TimeoutError,
                           redis.exceptions.ConnectionError),
                          max_tries=MAX_TRIES)
    def delete(self, key):
        return self.redis.delete(key)

    @backoff.on_exception(backoff.expo,
                          (redis.exceptions.TimeoutError,
                           redis.exceptions.ConnectionError),
                          max_tries=MAX_TRIES)
    def hset(self, *args):
        return self.redis.hset(*args)

    @backoff.on_exception(backoff.expo,
                          (redis.exceptions.TimeoutError,
                           redis.exceptions.ConnectionError),
                          max_tries=MAX_TRIES)
    def hmset(self, *args):
        return self.redis.hmset(*args)

def datetime_from_str(string):
    return datetime.datetime.strptime(string, DATE_FORMAT)

def str_from_datetime(dt):
    return dt.strftime(DATE_FORMAT)

def unpack_state(packed_state):
    decoded_state = {k.decode("utf-8"): v.decode("utf-8") for k, v in packed_state.items()}
    unpacked_state = {}

    for key, value in decoded_state.items():
        decoded_value = json.loads(value)
        unpacked_state[key] = {
            'correlation_id': decoded_value['correlation_id'],
            'first_occurance_time': datetime_from_str(decoded_value['first_occurance_time']),
        }

    return unpacked_state

def pack_state(unpacked_state):
    packed_state = {}
    for queue_name, queue_state in unpacked_state.items():
        dt_str = str_from_datetime(queue_state['first_occurance_time'])
        packed_state[queue_name] = json.dumps({
            'correlation_id': queue_state['correlation_id'],
            'first_occurance_time': dt_str,
        })
    return packed_state

def build_new_state(old_state, queue_first_items, current_time):
    new_state = {}
    for queue_name, first_item_encoded in queue_first_items.items():
        # TODO: Catch/Handle exception if not json
        first_item = json.loads(first_item_encoded.decode("utf-8"))
        # TODO: Handle keys missing in data
        correlation_id = first_item['properties']['correlation_id']
        first_occurance_time = current_time

        if queue_name in old_state:
            old_correlation_id = old_state[queue_name]['correlation_id']
            if old_correlation_id == correlation_id:
                first_occurance_time = old_state[queue_name]['first_occurance_time']

        new_state[queue_name] = {
            'correlation_id': correlation_id,
            'first_occurance_time': first_occurance_time,
        }

    return new_state

def should_send_alert(first_occurance_time, current_time, threshold):
    time_delta = current_time - first_occurance_time
    return time_delta.total_seconds() > threshold

@backoff.on_exception(backoff.expo,
                      (ApiException),
                      max_tries=MAX_TRIES)
def send_alert(opsgenie_api_key, environment, deploy, queue_name, threshold):

    configuration.api_key['Authorization'] = opsgenie_api_key
    configuration.api_key_prefix['Authorization'] = 'GenieKey'

    alert_message = str.format(
        "{}-{} {} queue is stale. Stationary for {}s", environment, deploy, queue_name, threshold
    )
    print(alert_message)
    response = AlertApi().create_alert(body=CreateAlertRequest(message=alert_message))
    print('request id: {}'.format(response.request_id))
    print('took: {}'.format(response.took))
    print('result: {}'.format(response.result))

def print_info(queue_name, do_alert, first_occurance_time, current_time, threshold, default_threshold):
    output = str.format(
        """
            ---------------------------------------------
            queue_name = {}
            do_alert = {}
            first_occurance_time = {}
            current_time = {}
            threshold = {}
            default_threshold = {}
        """, queue_name, do_alert, first_occurance_time, current_time, threshold, default_threshold)
    print(dedent(output))

@click.command()
@click.option('--host', '-h', default='localhost',
              help='Hostname of redis server', required=True)
@click.option('--port', '-p', default=6379, help='Port of redis server')
@click.option('--environment', '-e', required=True)
@click.option('--deploy', '-d', required=True,
              help="Deployment (i.e. edx or edge)")
@click.option('--default-threshold', default=60,
              help='Default queue maximum item age in seconds')
@click.option('--queue-threshold', type=(str, int), multiple=True,
              help='Per queue maximum item age (seconds) in format --queue-threshold'
              + ' {queue_name} {threshold}. May be used multiple times.')
@click.option('--opsgenie-api-key', '-k', envvar='OPSGENIE_API_KEY', required=True)
def check_queues(host, port, environment, deploy, default_threshold, queue_threshold, opsgenie_api_key):

    thresholds = dict(queue_threshold)

    timeout = 1
    redis_client = RedisWrapper(host=host, port=port, socket_timeout=timeout,
                                socket_connect_timeout=timeout)
    queue_names = set([k.decode() for k in redis_client.keys()
                       if (redis_client.type(k) == b'list' and
                           not k.decode().endswith(".pidbox") and
                           not k.decode().startswith("_kombu"))])

    queue_age_hash = redis_client.hgetall(QUEUE_AGE_HASH_NAME)
    old_state = unpack_state(queue_age_hash)

    queue_first_items = {}
    current_time = datetime.datetime.now()
    for queue_name in queue_names:
        queue_first_items[queue_name] = redis_client.lindex(queue_name, 0)

    new_state = build_new_state(old_state, queue_first_items, current_time)

    redis_client.delete(QUEUE_AGE_HASH_NAME)
    if new_state:
        redis_client.hmset(QUEUE_AGE_HASH_NAME, pack_state(new_state))

    for queue_name in queue_names:
        threshold = default_threshold
        if queue_name in thresholds:
            threshold = thresholds[queue_name]

        first_occurance_time = new_state[queue_name]['first_occurance_time']
        do_alert = should_send_alert(first_occurance_time, current_time, threshold)

        if do_alert:
            send_alert(opsgenie_api_key, environment, deploy, queue_name, threshold)
        print_info(queue_name, do_alert, first_occurance_time, current_time, threshold, default_threshold)

if __name__ == '__main__':
    check_queues()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
The Control, Escape and Graphics are taken from pyte (https://github.com/selectel/pyte)
"""
import os
import sys
import platform
import functools
import threading
import ctypes
from itertools import chain

IN_PYTHONISTA = sys.executable.find('Pythonista') >= 0

if IN_PYTHONISTA:
    import plistlib

    _properties = plistlib.readPlist(os.path.join(os.path.dirname(sys.executable), 'Info.plist'))
    PYTHONISTA_VERSION = _properties['CFBundleShortVersionString']
    PYTHONISTA_VERSION_LONG = _properties['CFBundleVersion']

    if PYTHONISTA_VERSION < '3.0':
        python_capi = ctypes.pythonapi
    else:
        # The default pythonapi always points to Python 3 in Pythonista 3
        # So we need to load the Python 2 API manually
        python_capi = ctypes.PyDLL(
            os.path.join(os.path.dirname(sys.executable),
                         'Frameworks/PythonistaKit.framework/PythonistaKit')
        )
else:
    PYTHONISTA_VERSION = '0.0'
    PYTHONISTA_VERSION_LONG = '000000'
    python_capi = ctypes.pythonapi

platform_string = platform.platform()

ON_IPAD = platform_string.find('iPad') >= 0
ON_IOS_8 = platform_string.split('-')[1].startswith('14')
M_64 = platform_string.find('64bit') != -1

CTRL_KEY_FLAG = (1 << 18)  # Control key for keyCommands
CMD_KEY_FLAG = (1 << 20)  # Command key

_STASH_ROOT = os.path.realpath(os.path.abspath(
    os.path.dirname(os.path.dirname(__file__))))
_STASH_CONFIG_FILES = ('.stash_config', 'stash.cfg')
_STASH_HISTORY_FILE = '.stash_history'

# Save the true IOs
if IN_PYTHONISTA:
    # The stdio catchers recreation is copied from code written by @dgelessus
    # https://forum.omz-software.com/topic/1946/pythonista-1-6-beta/167
    import _outputcapture

    if sys.stdin.__class__.__name__ == 'StdinCatcher':
        _SYS_STDIN = sys.__stdin__ = sys.stdin
    elif sys.__stdin__.__class__.__name__ == 'StdinCatcher':
        _SYS_STDIN = sys.__stdin__
    else:
        class StdinCatcher(object):
            def __init__(self):
                self.encoding = 'utf8'

            def read(self, limit=-1):
                return _outputcapture.ReadStdin(limit)

            def readline(self):
                return _outputcapture.ReadStdin()

        _SYS_STDIN = StdinCatcher()

    if sys.stdout.__class__.__name__ == 'StdoutCatcher':
        _SYS_STDOUT = sys.__stdout__ = sys.stdout
    elif sys.__stdout__.__class__.__name__ == 'StdoutCatcher':
        _SYS_STDOUT = sys.__stdout__
    else:
        class StdoutCatcher(object):
            def __init__(self):
                self.encoding = 'utf8'

            def flush(self):
                pass

            def write(self, s):
                if isinstance(s, str):
                    _outputcapture.CaptureStdout(s)
                elif isinstance(s, unicode):
                    _outputcapture.CaptureStdout(s.encode('utf8'))

            def writelines(self, lines):
                self.write(''.join(lines))

        _SYS_STDOUT = StdoutCatcher()

    if sys.stderr.__class__.__name__ == 'StderrCatcher':
        _SYS_STDERR = sys.__stderr__ = sys.stderr
    elif sys.stderr.__class__.__name__ == 'StderrCatcher':
        _SYS_STDERR = sys.__stderr__
    else:
        class StderrCatcher(object):
            def __init__(self):
                self.encoding = 'utf8'

            def flush(self):
                pass

            def write(self, s):
                if isinstance(s, str):
                    _outputcapture.CaptureStderr(s)
                elif isinstance(s, unicode):
                    _outputcapture.CaptureStderr(s.encode('utf8'))

            def writelines(self, lines):
                self.write(''.join(lines))

        _SYS_STDERR = StderrCatcher()

else:
    _SYS_STDOUT = sys.stdout
    _SYS_STDERR = sys.stderr
    _SYS_STDIN = sys.stdin

_SYS_PATH = sys.path
_OS_ENVIRON = os.environ

def is_binary_file(filename, nbytes=1024):
    """
    An approximate way to tell whether a file is binary.
    :param str filename: The name of the file to be tested.
    :param int nbytes: number of bytes to read for test
    :return:
    """
    with open(filename, 'rb') as ins:
        for c in ins.read(nbytes):
            oc = ord(c)
            if 127 < oc < 256 or (oc < 32 and oc not in (9, 10, 13)):
                return True
        else:
            return False

def sh_delay(func, nseconds):
    t = threading.Timer(nseconds, func)
    t.start()
    return t

def sh_background(name=None):
    def wrap(func):
        @functools.wraps(func)
        def wrapped_func(*args, **kwargs):
            t = threading.Thread(name=name, target=func, args=args, kwargs=kwargs)
            t.start()
            return t

        return wrapped_func

    return wrap

class ShFileNotFound(Exception):
    pass

class ShIsDirectory(Exception):
    pass

class ShNotExecutable(Exception):
    def __init__(self, filename):
        super(Exception, self).__init__('{}: not executable\n'.format(filename))

class ShSingleExpansionRequired(Exception):
    pass

class ShEventNotFound(Exception):
    pass

class ShBadSubstitution(Exception):
    pass

class ShSyntaxError(Exception):
    pass

class ShInternalError(Exception):
    pass

class Control(object):
    """
        pyte.control
        ~~~~~~~~~~~~

        This module defines simple control sequences, recognized by
        :class:`~pyte.streams.Stream`, the set of codes here is for
        ``TERM=linux`` which is a superset of VT102.

        :copyright: (c) 2011-2013 by Selectel, see AUTHORS for details.
        :license: LGPL, see LICENSE for more details.
    """

    #: *Space*: Not suprisingly -- ``" "``.
    SP = u" "

    #: *Null*: Does nothing.
    NUL = u"\u0000"

    #: *Bell*: Beeps.
    BEL = u"\u0007"

    #: *Backspace*: Backspace one column, but not past the beginning of the
    #: line.
    BS = u"\u0008"

    #: *Horizontal tab*: Move cursor to the next tab stop, or to the end
    #: of the line if there is no earlier tab stop.
    HT = u"\u0009"

    #: *Linefeed*: Give a line feed, and, if :data:`pyte.modes.LNM` (new
    #: line mode) is set also a carriage return.
    LF = u"\n"
    #: *Vertical tab*: Same as :data:`LF`.
    VT = u"\u000b"
    #: *Form feed*: Same as :data:`LF`.
    FF = u"\u000c"

    #: *Carriage return*: Move cursor to left margin on current line.
    CR = u"\r"

    #: *Shift out*: Activate G1 character set.
    SO = u"\u000e"

    #: *Shift in*: Activate G0 character set.
    SI = u"\u000f"

    #: *Cancel*: Interrupt escape sequence. If received during an escape or
    #: control sequence, cancels the sequence and displays substitution
    #: character.
    CAN = u"\u0018"
    #: *Substitute*: Same as :data:`CAN`.
    SUB = u"\u001a"

    #: *Escape*: Starts an escape sequence.
    ESC = u"\u001b"

    #: *Delete*: Is ignored.
    DEL = u"\u007f"

    #: *Control sequence introducer*: An equivalent for ``ESC [``.
    CSI = u"\u009b"

class Escape(object):
    """
        pyte.escape
        ~~~~~~~~~~~

        This module defines both CSI and non-CSI escape sequences, recognized
        by :class:`~pyte.streams.Stream` and subclasses.

        :copyright: (c) 2011-2013 by Selectel, see AUTHORS for details.
        :license: LGPL, see LICENSE for more details.
    """

    #: *Reset*.
    RIS = u"c"

    #: *Index*: Move cursor down one line in same column. If the cursor is
    #: at the bottom margin, the screen performs a scroll-up.
    IND = u"D"

    #: *Next line*: Same as :data:`pyte.control.LF`.
    NEL = u"E"

    #: Tabulation set: Set a horizontal tab stop at cursor position.
    HTS = u"H"

    #: *Reverse index*: Move cursor up one line in same column. If the
    #: cursor is at the top margin, the screen performs a scroll-down.
    RI = u"M"

    #: Save cursor: Save cursor position, character attribute (graphic
    #: rendition), character set, and origin mode selection (see
    #: :data:`DECRC`).
    DECSC = u"7"

    #: *Restore cursor*: Restore previously saved cursor position, character
    #: attribute (graphic rendition), character set, and origin mode
    #: selection. If none were saved, move cursor to home position.
    DECRC = u"8"

    # "Percent" escape sequences.
    # ---------------------------

    #: *Select default (ISO 646 / ISO 8859-1)*.
    DEFAULT = u"@"

    #: *Select UTF-8*.
    UTF8 = u"G"

    #: *Select UTF-8 (obsolete)*.
    UTF8_OBSOLETE = u"8"

    # "Sharp" escape sequences.
    # -------------------------

    #: *Alignment display*: Fill screen with uppercase E's for testing
    #: screen focus and alignment.
    DECALN = u"8"

    # ECMA-48 CSI sequences.
    # ---------------------

    #: *Insert character*: Insert the indicated # of blank characters.
    ICH = u"@"

    #: *Cursor up*: Move cursor up the indicated # of lines in same column.
    #: Cursor stops at top margin.
    CUU = u"A"

    #: *Cursor down*: Move cursor down the indicated # of lines in same
    #: column. Cursor stops at bottom margin.
    CUD = u"B"

    #: *Cursor forward*: Move cursor right the indicated # of columns.
    #: Cursor stops at right margin.
    CUF = u"C"

    #: *Cursor back*: Move cursor left the indicated # of columns. Cursor
    #: stops at left margin.
    CUB = u"D"

    #: *Cursor next line*: Move cursor down the indicated # of lines to
    #: column 1.
    CNL = u"E"

    #: *Cursor previous line*: Move cursor up the indicated # of lines to
    #: column 1.
    CPL = u"F"

    #: *Cursor horizontal align*: Move cursor to the indicated column in
    #: current line.
    CHA = u"G"

    #: *Cursor position*: Move cursor to the indicated line, column (origin
    #: at ``1, 1``).
    CUP = u"H"

    #: *Erase data* (default: from cursor to end of line).
    ED = u"J"

    #: *Erase in line* (default: from cursor to end of line).
    EL = u"K"

    #: *Insert line*: Insert the indicated # of blank lines, starting from
    #: the current line. Lines displayed below cursor move down. Lines moved
    #: past the bottom margin are lost.
    IL = u"L"

    #: *Delete line*: Delete the indicated # of lines, starting from the
    #: current line. As lines are deleted, lines displayed below cursor
    #: move up. Lines added to bottom of screen have spaces with same
    #: character attributes as last line move up.
    DL = u"M"

    #: *Delete character*: Delete the indicated # of characters on the
    #: current line. When character is deleted, all characters to the right
    #: of cursor move left.
    DCH = u"P"

    #: *Erase character*: Erase the indicated # of characters on the
    #: current line.
    ECH = u"X"

    #: *Horizontal position relative*: Same as :data:`CUF`.
    HPR = u"a"

    #: *Vertical position adjust*: Move cursor to the indicated line,
    #: current column.
    VPA = u"d"

    #: *Vertical position relative*: Same as :data:`CUD`.
    VPR = u"e"

    #: *Horizontal / Vertical position*: Same as :data:`CUP`.
    HVP = u"f"

    #: *Tabulation clear*: Clears a horizontal tab stop at cursor position.
    TBC = u"g"

    #: *Set mode*.
    SM = u"h"

    #: *Reset mode*.
    RM = u"l"

    #: *Select graphics rendition*: The terminal can display the following
    #: character attributes that change the character display without
    #: changing the character (see :mod:`pyte.graphics`).
    SGR = u"m"

    #: *Select top and bottom margins*: Selects margins, defining the
    #: scrolling region; parameters are top and bottom line. If called
    #: without any arguments, whole screen is used.
    DECSTBM = u"r"

    #: *Horizontal position adjust*: Same as :data:`CHA`.
    HPA = u"'"

class Graphics(object):
    # -*- coding: utf-8 -*-
    """
        pyte.graphics
        ~~~~~~~~~~~~~

        This module defines graphic-related constants, mostly taken from
        :manpage:`console_codes(4)` and
        http://pueblo.sourceforge.net/doc/manual/ansi_color_codes.html.

        :copyright: (c) 2011-2013 by Selectel, see AUTHORS for details.
        :license: LGPL, see LICENSE for more details.
    """

    #: A mapping of ANSI text style codes to style names, "+" means the:
    #: attribute is set, "-" -- reset; example:
    #:
    #: >>> text[1]
    #: '+bold'
    #: >>> text[9]
    #: '+strikethrough'
    TEXT = {
        1: "+bold",
        3: "+italics",
        4: "+underscore",
        7: "+reverse",
        9: "+strikethrough",
        22: "-bold",
        23: "-italics",
        24: "-underscore",
        27: "-reverse",
        29: "-strikethrough"
    }

    #: A mapping of ANSI foreground color codes to color names, example:
    #:
    #: >>> FG[30]
    #: 'black'
    #: >>> FG[38]
    #: 'default'
    FG = {
        30: "black",
        31: "red",
        32: "green",
        33: "brown",
        34: "blue",
        35: "magenta",
        36: "cyan",
        37: "white",
        39: "default",  # white.
        50: "gray",
        51: "yellow",
        52: "smoke",
    }

    #: A mapping of ANSI background color codes to color names, example:
    #:
    #: >>> BG[40]
    #: 'black'
    #: >>> BG[48]
    #: 'default'
    BG = {
        40: "black",
        41: "red",
        42: "green",
        43: "brown",
        44: "blue",
        45: "magenta",
        46: "cyan",
        47: "white",
        49: "default",  # black.
        60: "gray",
        61: "yellow",
        62: "smoke",
    }

    # Reverse mapping of all available attributes -- keep this private!
    _SGR = {v: k for k, v in chain(FG.items(),
                                   TEXT.items())}
    _SGR.update({'bg-' + v: k for k, v in BG.items()})
<commit_msg>fix for beta<commit_after># -*- coding: utf-8 -*-
"""
The Control, Escape and Graphics are taken from pyte (https://github.com/selectel/pyte)
"""
import os
import sys
import platform
import functools
import threading
import ctypes
from itertools import chain

IN_PYTHONISTA = sys.executable.find('Pythonista') >= 0

if IN_PYTHONISTA:
    import plistlib

    _properties = plistlib.readPlist(os.path.join(os.path.dirname(sys.executable), 'Info.plist'))
    PYTHONISTA_VERSION = _properties['CFBundleShortVersionString']
    PYTHONISTA_VERSION_LONG = _properties['CFBundleVersion']

    if PYTHONISTA_VERSION < '3.0':
        python_capi = ctypes.pythonapi
    else:
        # The default pythonapi always points to Python 3 in Pythonista 3
        # So we need to load the Python 2 API manually
        python_capi = ctypes.PyDLL(
            os.path.join(os.path.dirname(sys.executable),
                         'Frameworks/Py2Kit.framework/Py2Kit')
        )
else:
    PYTHONISTA_VERSION = '0.0'
    PYTHONISTA_VERSION_LONG = '000000'
    python_capi = ctypes.pythonapi

platform_string = platform.platform()

ON_IPAD = platform_string.find('iPad') >= 0
ON_IOS_8 = platform_string.split('-')[1].startswith('14')
M_64 = platform_string.find('64bit') != -1

CTRL_KEY_FLAG = (1 << 18)  # Control key for keyCommands
CMD_KEY_FLAG = (1 << 20)  # Command key

_STASH_ROOT = os.path.realpath(os.path.abspath(
    os.path.dirname(os.path.dirname(__file__))))
_STASH_CONFIG_FILES = ('.stash_config', 'stash.cfg')
_STASH_HISTORY_FILE = '.stash_history'

# Save the true IOs
if IN_PYTHONISTA:
    # The stdio catchers recreation is copied from code written by @dgelessus
    # https://forum.omz-software.com/topic/1946/pythonista-1-6-beta/167
    # In pythonista beta 301006, _outputcapture was replaced with pykit_io
    try:
        import _outputcapture
    except ImportError:
        import pykit_io
        class _outputcapture(object):
            ReadStdin=pykit_io.read_stdin
            CaptureStdout=pykit_io.write_stdout
            CaptureStderr=pykit_io.write_stderr

    if sys.stdin.__class__.__name__ == 'StdinCatcher':
        _SYS_STDIN = sys.__stdin__ = sys.stdin
    elif sys.__stdin__.__class__.__name__ == 'StdinCatcher':
        _SYS_STDIN = sys.__stdin__
    else:
        class StdinCatcher(object):
            def __init__(self):
                self.encoding = 'utf8'

            def read(self, limit=-1):
                return _outputcapture.ReadStdin(limit)

            def readline(self):
                return _outputcapture.ReadStdin()

        _SYS_STDIN = StdinCatcher()

    if sys.stdout.__class__.__name__ == 'StdoutCatcher':
        _SYS_STDOUT = sys.__stdout__ = sys.stdout
    elif sys.__stdout__.__class__.__name__ == 'StdoutCatcher':
        _SYS_STDOUT = sys.__stdout__
    else:
        class StdoutCatcher(object):
            def __init__(self):
                self.encoding = 'utf8'

            def flush(self):
                pass

            def write(self, s):
                if isinstance(s, str):
                    _outputcapture.CaptureStdout(s)
                elif isinstance(s, unicode):
                    _outputcapture.CaptureStdout(s.encode('utf8'))

            def writelines(self, lines):
                self.write(''.join(lines))

        _SYS_STDOUT = StdoutCatcher()

    if sys.stderr.__class__.__name__ == 'StderrCatcher':
        _SYS_STDERR = sys.__stderr__ = sys.stderr
    elif sys.stderr.__class__.__name__ == 'StderrCatcher':
        _SYS_STDERR = sys.__stderr__
    else:
        class StderrCatcher(object):
            def __init__(self):
                self.encoding = 'utf8'

            def flush(self):
                pass

            def write(self, s):
                if isinstance(s, str):
                    _outputcapture.CaptureStderr(s)
                elif isinstance(s, unicode):
                    _outputcapture.CaptureStderr(s.encode('utf8'))

            def writelines(self, lines):
                self.write(''.join(lines))

        _SYS_STDERR = StderrCatcher()

else:
    _SYS_STDOUT = sys.stdout
    _SYS_STDERR = sys.stderr
    _SYS_STDIN = sys.stdin

_SYS_PATH = sys.path
_OS_ENVIRON = os.environ

def is_binary_file(filename, nbytes=1024):
    """
    An approximate way to tell whether a file is binary.
    :param str filename: The name of the file to be tested.
    :param int nbytes: number of bytes to read for test
    :return:
    """
    with open(filename, 'rb') as ins:
        for c in ins.read(nbytes):
            oc = ord(c)
            if 127 < oc < 256 or (oc < 32 and oc not in (9, 10, 13)):
                return True
        else:
            return False

def sh_delay(func, nseconds):
    t = threading.Timer(nseconds, func)
    t.start()
    return t

def sh_background(name=None):
    def wrap(func):
        @functools.wraps(func)
        def wrapped_func(*args, **kwargs):
            t = threading.Thread(name=name, target=func, args=args, kwargs=kwargs)
            t.start()
            return t

        return wrapped_func

    return wrap

class ShFileNotFound(Exception):
    pass

class ShIsDirectory(Exception):
    pass

class ShNotExecutable(Exception):
    def __init__(self, filename):
        super(Exception, self).__init__('{}: not executable\n'.format(filename))

class ShSingleExpansionRequired(Exception):
    pass

class ShEventNotFound(Exception):
    pass

class ShBadSubstitution(Exception):
    pass

class ShSyntaxError(Exception):
    pass

class ShInternalError(Exception):
    pass

class Control(object):
    """
        pyte.control
        ~~~~~~~~~~~~

        This module defines simple control sequences, recognized by
        :class:`~pyte.streams.Stream`, the set of codes here is for
        ``TERM=linux`` which is a superset of VT102.

        :copyright: (c) 2011-2013 by Selectel, see AUTHORS for details.
        :license: LGPL, see LICENSE for more details.
    """

    #: *Space*: Not suprisingly -- ``" "``.
    SP = u" "

    #: *Null*: Does nothing.
    NUL = u"\u0000"

    #: *Bell*: Beeps.
    BEL = u"\u0007"

    #: *Backspace*: Backspace one column, but not past the beginning of the
    #: line.
    BS = u"\u0008"

    #: *Horizontal tab*: Move cursor to the next tab stop, or to the end
    #: of the line if there is no earlier tab stop.
    HT = u"\u0009"

    #: *Linefeed*: Give a line feed, and, if :data:`pyte.modes.LNM` (new
    #: line mode) is set also a carriage return.
    LF = u"\n"
    #: *Vertical tab*: Same as :data:`LF`.
    VT = u"\u000b"
    #: *Form feed*: Same as :data:`LF`.
    FF = u"\u000c"

    #: *Carriage return*: Move cursor to left margin on current line.
    CR = u"\r"

    #: *Shift out*: Activate G1 character set.
    SO = u"\u000e"

    #: *Shift in*: Activate G0 character set.
    SI = u"\u000f"

    #: *Cancel*: Interrupt escape sequence. If received during an escape or
    #: control sequence, cancels the sequence and displays substitution
    #: character.
    CAN = u"\u0018"
    #: *Substitute*: Same as :data:`CAN`.
    SUB = u"\u001a"

    #: *Escape*: Starts an escape sequence.
    ESC = u"\u001b"

    #: *Delete*: Is ignored.
    DEL = u"\u007f"

    #: *Control sequence introducer*: An equivalent for ``ESC [``.
    CSI = u"\u009b"

class Escape(object):
    """
        pyte.escape
        ~~~~~~~~~~~

        This module defines both CSI and non-CSI escape sequences, recognized
        by :class:`~pyte.streams.Stream` and subclasses.

        :copyright: (c) 2011-2013 by Selectel, see AUTHORS for details.
        :license: LGPL, see LICENSE for more details.
    """

    #: *Reset*.
    RIS = u"c"

    #: *Index*: Move cursor down one line in same column. If the cursor is
    #: at the bottom margin, the screen performs a scroll-up.
    IND = u"D"

    #: *Next line*: Same as :data:`pyte.control.LF`.
    NEL = u"E"

    #: Tabulation set: Set a horizontal tab stop at cursor position.
    HTS = u"H"

    #: *Reverse index*: Move cursor up one line in same column. If the
    #: cursor is at the top margin, the screen performs a scroll-down.
    RI = u"M"

    #: Save cursor: Save cursor position, character attribute (graphic
    #: rendition), character set, and origin mode selection (see
    #: :data:`DECRC`).
    DECSC = u"7"

    #: *Restore cursor*: Restore previously saved cursor position, character
    #: attribute (graphic rendition), character set, and origin mode
    #: selection. If none were saved, move cursor to home position.
    DECRC = u"8"

    # "Percent" escape sequences.
    # ---------------------------

    #: *Select default (ISO 646 / ISO 8859-1)*.
    DEFAULT = u"@"

    #: *Select UTF-8*.
    UTF8 = u"G"

    #: *Select UTF-8 (obsolete)*.
    UTF8_OBSOLETE = u"8"

    # "Sharp" escape sequences.
    # -------------------------

    #: *Alignment display*: Fill screen with uppercase E's for testing
    #: screen focus and alignment.
    DECALN = u"8"

    # ECMA-48 CSI sequences.
    # ---------------------

    #: *Insert character*: Insert the indicated # of blank characters.
    ICH = u"@"

    #: *Cursor up*: Move cursor up the indicated # of lines in same column.
    #: Cursor stops at top margin.
    CUU = u"A"

    #: *Cursor down*: Move cursor down the indicated # of lines in same
    #: column. Cursor stops at bottom margin.
    CUD = u"B"

    #: *Cursor forward*: Move cursor right the indicated # of columns.
    #: Cursor stops at right margin.
    CUF = u"C"

    #: *Cursor back*: Move cursor left the indicated # of columns. Cursor
    #: stops at left margin.
    CUB = u"D"

    #: *Cursor next line*: Move cursor down the indicated # of lines to
    #: column 1.
    CNL = u"E"

    #: *Cursor previous line*: Move cursor up the indicated # of lines to
    #: column 1.
    CPL = u"F"

    #: *Cursor horizontal align*: Move cursor to the indicated column in
    #: current line.
    CHA = u"G"

    #: *Cursor position*: Move cursor to the indicated line, column (origin
    #: at ``1, 1``).
    CUP = u"H"

    #: *Erase data* (default: from cursor to end of line).
    ED = u"J"

    #: *Erase in line* (default: from cursor to end of line).
    EL = u"K"

    #: *Insert line*: Insert the indicated # of blank lines, starting from
    #: the current line. Lines displayed below cursor move down. Lines moved
    #: past the bottom margin are lost.
    IL = u"L"

    #: *Delete line*: Delete the indicated # of lines, starting from the
    #: current line. As lines are deleted, lines displayed below cursor
    #: move up. Lines added to bottom of screen have spaces with same
    #: character attributes as last line move up.
    DL = u"M"

    #: *Delete character*: Delete the indicated # of characters on the
    #: current line. When character is deleted, all characters to the right
    #: of cursor move left.
    DCH = u"P"

    #: *Erase character*: Erase the indicated # of characters on the
    #: current line.
    ECH = u"X"

    #: *Horizontal position relative*: Same as :data:`CUF`.
    HPR = u"a"

    #: *Vertical position adjust*: Move cursor to the indicated line,
    #: current column.
    VPA = u"d"

    #: *Vertical position relative*: Same as :data:`CUD`.
    VPR = u"e"

    #: *Horizontal / Vertical position*: Same as :data:`CUP`.
    HVP = u"f"

    #: *Tabulation clear*: Clears a horizontal tab stop at cursor position.
    TBC = u"g"

    #: *Set mode*.
    SM = u"h"

    #: *Reset mode*.
    RM = u"l"

    #: *Select graphics rendition*: The terminal can display the following
    #: character attributes that change the character display without
    #: changing the character (see :mod:`pyte.graphics`).
    SGR = u"m"

    #: *Select top and bottom margins*: Selects margins, defining the
    #: scrolling region; parameters are top and bottom line. If called
    #: without any arguments, whole screen is used.
    DECSTBM = u"r"

    #: *Horizontal position adjust*: Same as :data:`CHA`.
    HPA = u"'"

class Graphics(object):
    # -*- coding: utf-8 -*-
    """
        pyte.graphics
        ~~~~~~~~~~~~~

        This module defines graphic-related constants, mostly taken from
        :manpage:`console_codes(4)` and
        http://pueblo.sourceforge.net/doc/manual/ansi_color_codes.html.

        :copyright: (c) 2011-2013 by Selectel, see AUTHORS for details.
        :license: LGPL, see LICENSE for more details.
    """

    #: A mapping of ANSI text style codes to style names, "+" means the:
    #: attribute is set, "-" -- reset; example:
    #:
    #: >>> text[1]
    #: '+bold'
    #: >>> text[9]
    #: '+strikethrough'
    TEXT = {
        1: "+bold",
        3: "+italics",
        4: "+underscore",
        7: "+reverse",
        9: "+strikethrough",
        22: "-bold",
        23: "-italics",
        24: "-underscore",
        27: "-reverse",
        29: "-strikethrough"
    }

    #: A mapping of ANSI foreground color codes to color names, example:
    #:
    #: >>> FG[30]
    #: 'black'
    #: >>> FG[38]
    #: 'default'
    FG = {
        30: "black",
        31: "red",
        32: "green",
        33: "brown",
        34: "blue",
        35: "magenta",
        36: "cyan",
        37: "white",
        39: "default",  # white.
        50: "gray",
        51: "yellow",
        52: "smoke",
    }

    #: A mapping of ANSI background color codes to color names, example:
    #:
    #: >>> BG[40]
    #: 'black'
    #: >>> BG[48]
    #: 'default'
    BG = {
        40: "black",
        41: "red",
        42: "green",
        43: "brown",
        44: "blue",
        45: "magenta",
        46: "cyan",
        47: "white",
        49: "default",  # black.
        60: "gray",
        61: "yellow",
        62: "smoke",
    }

    # Reverse mapping of all available attributes -- keep this private!
    _SGR = {v: k for k, v in chain(FG.items(),
                                   TEXT.items())}
    _SGR.update({'bg-' + v: k for k, v in BG.items()})
<|endoftext|>


<commit_before>
from .rule_primitive import RulePrimitive
from .messages import TransformationException
from core.himesis import Himesis
from core.himesis_utils import update_equations

from solver.simple_attribute_equation_evaluator import is_consistent

import traceback

import re

class Rewriter(RulePrimitive):
    '''
        Transforms the matched source model elements according to the specified post-condition pattern.
    '''
    def __init__(self, condition):
        '''
            Transforms the bound graph of the source graph into what the specification of the post-condition pattern.
            @param condition: The the post-condition pattern.
        '''
        super(Rewriter, self).__init__()
        self.condition = condition

    def __str__(self):
        s = super(Rewriter, self).__str__()
        s = s.split(' ')
        s.insert(1, '[%s]' % self.condition.name)
        return reduce(lambda x, y: x + ' ' + y, s)

    def packet_in(self, packet, verbosity = 0):
        self.exception = None
        self.is_success = False
        if self.condition.pre[Himesis.Constants.GUID] not in packet.match_sets:
            self.is_success = False
            # TODO: This should be a TransformationLanguageSpecificException
            self.exception = TransformationException()
            self.exception.packet = packet
            return packet
        else:
            match = packet.match_sets[self.condition.pre[Himesis.Constants.GUID]].match2rewrite
            try:
                mapping = match.to_label_mapping(packet.graph)
                # Apply the transformation on the match

                if verbosity > 0:
                    print("Rewriter mapping: " + str(mapping))

                graph_eqs = packet.graph["equations"]

                try:
                    cond_eqs = self.condition["equations"]
                except KeyError:
                    cond_eqs = []

                if cond_eqs and graph_eqs != cond_eqs:

                    #get dict from label to node num
                    RHS_labels = {}
                    for n in range(self.condition.vcount()):
                        node = self.condition.vs[n]
                        if node["MT_label__"] == '':
                            continue
                        RHS_labels[int(node["MT_label__"])] = n

                    new_mapping = {}
                    j=0
                    vcount = packet.graph.vcount()

                    #make sure to iterate in natural order
                    for label in sorted(RHS_labels.keys()):

                        #use mapping if possible
                        if str(label) in mapping:
                            new_mapping[label] = mapping[str(label)]

                        #assume nodes will be added in this order
                        #TODO: Handle deleted nodes
                        else:
                            new_mapping[label] = vcount + j
                            j += 1

                    new_cond_eqs = update_equations(cond_eqs, new_mapping)
                    packet.graph["equations"] += new_cond_eqs

                    if not is_consistent(packet.graph):
                        if verbosity >= 2:
                            print("Graph: " + packet.graph.name + " has inconsistent equations inside")
                        self.is_success = False
                        return packet

                self.condition.execute(packet, mapping)     # Sets dirty nodes as well

            except Exception as e:

                tb = traceback.format_exc()
                print("Rewriter Error: " + str(e))
                print(tb)
                print("packet.graph: " + packet.graph.name)
                print("self.condition: " + self.condition.name)
                raise
                self.is_success = False
                self.exception = TransformationException(e)
                self.exception.packet = packet
                self.exception.transformation_unit = self
                return packet

            # Remove the match
            packet.match_sets[self.condition.pre[Himesis.Constants.GUID]].match2rewrite = None
            if  len(packet.match_sets[self.condition.pre[Himesis.Constants.GUID]].matches) == 0:
                del packet.match_sets[self.condition.pre[Himesis.Constants.GUID]]

            #print self.condition

            self.is_success = True
            return packet

class Rewriter_Equation(Rewriter):

    def __init__(self, condition):
        super(Rewriter_Equation, self).__init__(condition)

    def packet_in(self, packet, verbosity = 0):

        packet = super(Rewriter_Equation, self).packet_in(packet, verbosity)

        if self.is_success:
            solver.combine_equations(packet.graph, self.condition)

        return packet
<commit_msg>Remove unneeded code in rewriter.<commit_after>
from .rule_primitive import RulePrimitive
from .messages import TransformationException
from core.himesis import Himesis
from core.himesis_utils import update_equations

from solver.simple_attribute_equation_evaluator import is_consistent

import traceback

import re

class Rewriter(RulePrimitive):
    '''
        Transforms the matched source model elements according to the specified post-condition pattern.
    '''
    def __init__(self, condition):
        '''
            Transforms the bound graph of the source graph into what the specification of the post-condition pattern.
            @param condition: The the post-condition pattern.
        '''
        super(Rewriter, self).__init__()
        self.condition = condition

    def __str__(self):
        s = super(Rewriter, self).__str__()
        s = s.split(' ')
        s.insert(1, '[%s]' % self.condition.name)
        return reduce(lambda x, y: x + ' ' + y, s)

    def packet_in(self, packet, verbosity = 0):
        self.exception = None
        self.is_success = False
        if self.condition.pre[Himesis.Constants.GUID] not in packet.match_sets:
            self.is_success = False
            # TODO: This should be a TransformationLanguageSpecificException
            self.exception = TransformationException()
            self.exception.packet = packet
            return packet
        else:
            match = packet.match_sets[self.condition.pre[Himesis.Constants.GUID]].match2rewrite
            try:
                mapping = match.to_label_mapping(packet.graph)
                # Apply the transformation on the match

                if verbosity > 0:
                    print("Rewriter mapping: " + str(mapping))

                graph_eqs = packet.graph["equations"]

                try:
                    cond_eqs = self.condition["equations"]
                except KeyError:
                    cond_eqs = []

                if cond_eqs and graph_eqs != cond_eqs:

                    #get dict from label to node num
                    RHS_labels = {}
                    for n, node in enumerate(self.condition.vs):
                        if node["MT_label__"] == '':
                            continue
                        RHS_labels[int(node["MT_label__"])] = n

                    new_mapping = {}
                    j=0
                    vcount = packet.graph.vcount()

                    #make sure to iterate in natural order
                    for label in sorted(RHS_labels.keys()):

                        #use mapping if possible
                        if str(label) in mapping:
                            new_mapping[label] = mapping[str(label)]

                        #assume nodes will be added in this order
                        #TODO: Handle deleted nodes
                        else:
                            new_mapping[label] = vcount + j
                            j += 1

                    new_cond_eqs = update_equations(cond_eqs, new_mapping)
                    packet.graph["equations"] += new_cond_eqs

                    if not is_consistent(packet.graph):
                        if verbosity >= 2:
                            print("Graph: " + packet.graph.name + " has inconsistent equations inside")
                        self.is_success = False
                        return packet

                self.condition.execute(packet, mapping)     # Sets dirty nodes as well

            except Exception as e:

                tb = traceback.format_exc()
                print("Rewriter Error: " + str(e))
                print(tb)
                print("packet.graph: " + packet.graph.name)
                print("self.condition: " + self.condition.name)
                raise
                self.is_success = False
                self.exception = TransformationException(e)
                self.exception.packet = packet
                self.exception.transformation_unit = self
                return packet

            # Remove the match
            packet.match_sets[self.condition.pre[Himesis.Constants.GUID]].match2rewrite = None
            if  len(packet.match_sets[self.condition.pre[Himesis.Constants.GUID]].matches) == 0:
                del packet.match_sets[self.condition.pre[Himesis.Constants.GUID]]

            #print self.condition

            self.is_success = True
            return packet

<|endoftext|>


<commit_before>from django.shortcuts import render
from django.http import HttpResponse
from django.template import RequestContext, loader

def index(request):
  template = loader.get_template('main/index.html')
  return HttpResponse(template.render())
<commit_msg>switch view to use shortcut<commit_after>from django.shortcuts import render
from django.http import HttpResponse
from django.shortcuts import render

def index(request):
  return render(request,"main/index.html")
<|endoftext|>


<commit_before>from functools import reduce
from operator import and_, or_
from common.funcfun import lmap
from retrieval.boolean_parser import Node
import shelve
from other.constants import INDEX_FOLDER_NAME, DOCUMENT_INFO_NAME, INFO_FOLDER_NAME, STEMSDICT_NAME,\
	KEYWORDSINDOCUMENTS_NAME
from retrieval.stem import Stem

class Index:

	def __init__(self, directory, documentsInfo = None):
		self.info = shelve.open(directory + INFO_FOLDER_NAME + 'info')
		self.keylen = 1
		self.directory = directory
		self.documents_info = documentsInfo if documentsInfo else self._get_translation()
		self.total_records = len(self.documents_info)
		self.stemsDict = None
		self.docKeywords = None
		self.allKeywords = None
		self.searchedStem = {}

	def get_documents(self, parsedQuery):
		documents = self._by_node(parsedQuery)
		return lmap(self._translate, documents)

	def get_stem_info(self, stem):
		stemObject = self.searchedStem.get(stem, False)
		if stemObject:
			return stemObject
		else:
			prefix = stem[:self.keylen]
			sh = shelve.open(self.directory + INDEX_FOLDER_NAME + '/' + prefix)
			try:
				record = sh[stem]
				stemObject = Stem(record, stem)
				self.searchedStem[stem] = stemObject
			except KeyError:
				stemObject = Stem()
			return stemObject

	def term_frequency(self, term, documentID):
		stem = self.get_stem_info(term)
		documents = stem.documents
		freq = documents.get(documentID, 0)
		return freq

	def contains_term(self, term, documentID):
		if not self.docKeywords:
			temp = self.info[KEYWORDSINDOCUMENTS_NAME]
			self.docKeywords = temp['inDocuments']
			self.allKeywords = temp['keywords']

		if term in self.allKeywords:
			return term in self.docKeywords[documentID]
		else:
			return self.term_frequency(term, documentID) > 0

	def document_frequency(self, term):
		return len(self.get_stem_info(term).documents)

	def getKeywords(self, docID):
		return self.documents_info[docID]['keywords']

	def stem2word(self, stem):
		if not self.stemsDict:
			self.stemsDict = self.info[STEMSDICT_NAME]

		return self.stemsDict.get(stem, stem)

	def _by_node(self, node):
		if isinstance(node, Node):
			if node.type == 'AND':
				return self._by_and_node(node.children)
			if node.type == 'OR':
				return self._by_or_node(node.children)
			if node.type == 'NOT':
				return self._by_not_node(node.children[0])
		else:
			return self._by_word(node)

	def _by_and_node(self, children):
		return reduce(and_, map(self._by_node, children))

	def _by_or_node(self, children):
		return reduce(or_, map(self._by_node, children))

	def _by_not_node(self, argument):
		allDocuments = set(range(len(self.documents_info)))
		matchedDoc = self._by_node(argument)
		return allDocuments - matchedDoc

	def _by_words(self, words):
		return lmap(self._translate, reduce(and_, map(self._by_word, words)))

	def _by_word(self, stem):
		documents = self.get_stem_info(stem)
		return documents.docids

	def _get_translation(self):
		return self.info[DOCUMENT_INFO_NAME]

	def _translate(self, docid):
		return self.documents_info[docid]<commit_msg>refactoring<commit_after>from functools import reduce
from operator import and_, or_
from common.funcfun import lmap
from retrieval.boolean_parser import Node
import shelve
from other.constants import INDEX_FOLDER_NAME, DOCUMENT_INFO_NAME, INFO_FOLDER_NAME, STEMSDICT_NAME,\
	KEYWORDSINDOCUMENTS_NAME
from retrieval.stem import Stem

class Index:

	def __init__(self, directory, documentsInfo = None):
		self.info = shelve.open(directory + INFO_FOLDER_NAME + 'info')
		self.keylen = 1
		self.directory = directory
		self.documents_info = documentsInfo if documentsInfo else self._get_translation()
		self.total_records = len(self.documents_info)
		self.stemsDict = None
		self.docKeywords = None
		self.allKeywords = None
		self.searchedStem = {}

	def get_documents(self, parsedQuery):
		documents = self._by_node(parsedQuery)
		return lmap(self._translate, documents)

	def get_stem_info(self, stem):
		stemObject = self.searchedStem.get(stem, False)
		if stemObject:
			return stemObject
		else:
			prefix = stem[:self.keylen]
			sh = shelve.open(self.directory + INDEX_FOLDER_NAME + '/' + prefix)
			try:
				record = sh[stem]
				stemObject = Stem(record, stem)
				self.searchedStem[stem] = stemObject
			except KeyError:
				stemObject = Stem()
			return stemObject

	def term_frequency(self, term, documentID):
		stem = self.get_stem_info(term)
		documents = stem.documents
		freq = documents.get(documentID, 0)
		return freq

	def contains_term(self, term, documentID):
		if not self.docKeywords:
			temp = self.info[KEYWORDSINDOCUMENTS_NAME]
			self.docKeywords = temp['inDocuments']
			self.allKeywords = temp['keywords']

		if term in self.allKeywords:
			return term in self.docKeywords[documentID]
		else:
			return self.term_frequency(term, documentID) > 0

	def document_frequency(self, term):
		return len(self.get_stem_info(term).documents)

	def getKeywords(self, docID):
		return self.documents_info[docID]['keywords']

	def stem2word(self, stem):
		if not self.stemsDict:
			self.stemsDict = self.info[STEMSDICT_NAME]

		return self.stemsDict.get(stem, stem)

	def _by_node(self, node):
		if isinstance(node, Node):
			if node.type == 'AND':
				return self._by_and_node(node.children)
			if node.type == 'OR':
				return self._by_or_node(node.children)
			if node.type == 'NOT':
				return self._by_not_node(node.children[0])
		else:
			return self._by_word(node)

	def _by_and_node(self, children):
		return reduce(and_, map(self._by_node, children))

	def _by_or_node(self, children):
		temp = list(map(self._by_node, children))
		return reduce(or_, temp)

	def _by_not_node(self, argument):
		allDocuments = set(range(len(self.documents_info)))
		matchedDoc = self._by_node(argument)
		return allDocuments - matchedDoc

	def _by_words(self, words):
		return lmap(self._translate, reduce(and_, map(self._by_word, words)))

	def _by_word(self, stem):
		documents = self.get_stem_info(stem)
		return documents.docids

	def _get_translation(self):
		return self.info[DOCUMENT_INFO_NAME]

	def _translate(self, docid):
		return self.documents_info[docid]<|endoftext|>


<commit_before>import ply.lex
import ply.yacc

tokens = (
    "VAR",
    "PLUS",
    "MINUS",
    "TIMES",
    "DIV",
    "NUM",
    "LPAREN",
    "RPAREN"
)

t_VAR   = (r'([a-zA-Z]|\_)([a-zA-Z]|\_|\d)*')
t_PLUS  = (r'\+')
t_MINUS = (r'-')
t_TIMES = (r'\*')
t_DIV   = (r'/')

rint    = r'([1-9]\d*|0)'
rexp    = r'((e|E)(\+|-)?\d+)'

rfloats = [rint + '\.\d+', '\.\d+', rint]

rfloats = list(map(lambda s: '(' + s + rexp + '?)', rfloats))

t_NUM = ('|'.join(rfloats))

t_LPAREN = (r'\(')

t_RPAREN = (r'\)')

t_ignore = (' \n\t\r')

def t_error(t):
  print(t);

lexer = ply.lex.lex()

def p_expression(t):
    '''expression : expression PLUS term
                  | expression MINUS term
                  | term'''
    if len(t) == 4:
        t[0] = [t[2], t[1], t[3]]
    else:
        t[0] = t[1]

def p_term(t):
    '''term : term TIMES uop
            | term DIV uop
            | uop'''
    if len(t) == 4:
        t[0] = [t[2], t[1], t[3]]
    else:
        t[0] = t[1]

def p_uop(t):
    '''uop : MINUS base
           | base'''
    if len(t) == 3:
        t[0] = ['Neg', t[2]]
    else:
        t[0] = t[1]

def p_base(t):
    '''base : name
            | number
            | group
            | func'''
    t[0] = t[1]

def p_name(t):
    '''name : VAR'''
    t[0] = ['Name', t[1]]

def p_number(t):
    '''number : NUM'''
    t[0] = ['Value', t[1]]

def p_group(t):
    '''group : LPAREN expression RPAREN'''
    t[0] = t[2]

def p_func(t):
    '''func : VAR LPAREN expression RPAREN'''
    t[0] = ['Call', t[1], t[3]]

def p_error(t):
    print("Syntax error at '{}'".format(t.value))

parser = ply.yacc.yacc()

tmp_count = 0;

def tmp_name():
    global tmp_count
    tmp_count += 1
    return ('Name', '_tmp_{}'.format(str(tmp_count)))

def collect_nums(exp):
    nums = set()
    if not isinstance(exp, list):
        return nums
    elif exp[0] == 'Value':
        new_name = tmp_name();
        nums.add((new_name, exp[1]))
        exp[0] = new_name[0]
        exp[1] = new_name[1]

    for sl in exp:
        nums = nums | collect_nums(sl)
    return nums

def process_ops(exp, func_transform = lambda x: x):
    result = ''
    new_exp = exp
    if len(exp) == 1: #
        pass
    elif len(exp) == 3:
        if exp[0] != 'Call':
            left, new_left = process_ops(exp[1])
            exp[1] = new_left
            right, new_right = process_ops(exp[2])
            exp[2] = new_right
            new_exp = tmp_name()
            result = left + right + 'const mpz_class ' + new_exp[1] + '(' + str(exp[1][1]) + str(exp[0]) + str(exp[2][1]) + ');\n'
        else:
            inner, new_inner = process_ops(exp[2])
            exp[2] = new_inner
            new_exp = tmp_name()
            result = inner + 'type ' + new_exp[1] + ' = ' + func_transform(exp[1]) + '(' + str(exp[2][1]) + ');\n'
    return result, new_exp;

#def process(exp):
#    nums = collect_nums(exp)
#    result = ''
#    for x in nums:
#        result += 'const mpz_class ' + x[0][1] + '("'+x[1] + '");\n'

#    op_list, exp_result = process_ops(exp)
#    result += op_list
#    result += 'return ' + exp_result[1] + ';\n';
#    return result

def collect_vars(exp):
    result = set()
    if len(exp) == 2:
        if exp[0] == 'Name':
          exp[1] = '_' + exp[1]
          result.add(exp[1])
        if exp[0] == 'Neg':
            result = result | collect_vars(exp[1])
    if len(exp) == 3:
        if exp[0] == 'Call':
            result = result | collect_vars(exp[2])
        else:
            result = result | collect_vars(exp[1]) | collect_vars(exp[2])
    return result

def decl_vars(variables):
    i = 0
    result = ''
    for v in variables.keys():
        result += 'const interval_t & _' + v + ' =  X[' + str(variables[v]) + '];\n'
        i += 1
    return result

def expressify(exp, val_trans = lambda x: 'interval("' + x + '", "' + x + '").get_value()',
               func_trans = lambda x: x):
    if len(exp) == 2:
        if exp[0] == 'Name':
            return exp[1]
        if exp[0] == 'Value':
            return '(' + val_trans(exp[1]) + ')'
        if exp[0] == 'Neg':
            return '(-' + expressify(exp[1]) + ')'
    if len(exp) == 3:
        if exp[0] == 'Call':
            return '(' + func_trans(exp[1]) + '(' + expressify(exp[2]) + '))'
        else:
            return '(' + expressify(exp[1]) + exp[0] + expressify(exp[2]) + ')'

def process(exp):
    return 'return interval( ' + expressify(exp) + ');\n'

def get_body(s, variables):
    exp = parser.parse(s)
    v = collect_vars(exp)
    return (decl_vars(variables) + process(exp))
<commit_msg>Added interval creation function to the parser.<commit_after>import ply.lex
import ply.yacc

tokens = (
    "VAR",
    "PLUS",
    "MINUS",
    "TIMES",
    "DIV",
    "NUM",
    "LPAREN",
    "RPAREN",
    "COMMA",
)

t_VAR   = (r'([a-zA-Z]|\_)([a-zA-Z]|\_|\d)*')
t_PLUS  = (r'\+')
t_MINUS = (r'-')
t_TIMES = (r'\*')
t_DIV   = (r'/')
t_COMMA = (r",")

rint    = r'([1-9]\d*|0)'
rexp    = r'((e|E)(\+|-)?\d+)'

rfloats = [rint + '\.\d+', '\.\d+', rint]

rfloats = list(map(lambda s: '(' + s + rexp + '?)', rfloats))

t_NUM = ('|'.join(rfloats))

t_LPAREN = (r'\(')

t_RPAREN = (r'\)')

t_ignore = (' \n\t\r')

def t_error(t):
  print(t);

lexer = ply.lex.lex()

def p_expression(t):
    '''expression : expression PLUS term
                  | expression MINUS term
                  | term'''
    if len(t) == 4:
        t[0] = [t[2], t[1], t[3]]
    else:
        t[0] = t[1]

def p_term(t):
    '''term : term TIMES uop
            | term DIV uop
            | uop'''
    if len(t) == 4:
        t[0] = [t[2], t[1], t[3]]
    else:
        t[0] = t[1]

def p_uop(t):
    '''uop : MINUS base
           | base'''
    if len(t) == 3:
        t[0] = ['Neg', t[2]]
    else:
        t[0] = t[1]

def p_base(t):
    '''base : name
            | number
            | group
            | func'''
    t[0] = t[1]

def p_name(t):
    '''name : VAR'''
    t[0] = ['Name', t[1]]

def p_number(t):
    '''number : NUM'''
    t[0] = ['Value', t[1]]

def p_group(t):
    '''group : LPAREN expression RPAREN'''
    t[0] = t[2]

def p_func(t):
    '''func : VAR LPAREN args RPAREN'''
    t[0] = ['Call', t[1]] + [t[3]]

def p_args(t):
  '''args : args COMMA expression
          | expression'''
  if t[1][0] == 'Args':
    t[0] = t[1] + [t[3]]
  else:
    t[0] = ['Args', t[1]]

def p_error(t):
    print("Syntax error at '{}'".format(t.value))

# Generate the parser
parser = ply.yacc.yacc()

tmp_count = 0;

def tmp_name():
    global tmp_count
    tmp_count += 1
    return ('Name', '_tmp_{}'.format(str(tmp_count)))

def collect_nums(exp):
    nums = set()
    if not isinstance(exp, list):
        return nums
    elif exp[0] == 'Value':
        new_name = tmp_name();
        nums.add((new_name, exp[1]))
        exp[0] = new_name[0]
        exp[1] = new_name[1]

    for sl in exp:
        nums = nums | collect_nums(sl)
    return nums

def process_ops(exp, func_transform = lambda x: x):
    result = ''
    new_exp = exp
    if len(exp) == 1: #
        pass
    elif len(exp) == 3:
        if exp[0] != 'Call':
            left, new_left = process_ops(exp[1])
            exp[1] = new_left
            right, new_right = process_ops(exp[2])
            exp[2] = new_right
            new_exp = tmp_name()
            result = left + right + 'const mpz_class ' + new_exp[1] + '(' + str(exp[1][1]) + str(exp[0]) + str(exp[2][1]) + ');\n'
        else:
            inner, new_inner = process_ops(exp[2])
            exp[2] = new_inner
            new_exp = tmp_name()
            result = inner + 'type ' + new_exp[1] + ' = ' + func_transform(exp[1]) + '(' + str(exp[2][1]) + ');\n'
    return result, new_exp;

#def process(exp):
#    nums = collect_nums(exp)
#    result = ''
#    for x in nums:
#        result += 'const mpz_class ' + x[0][1] + '("'+x[1] + '");\n'

#    op_list, exp_result = process_ops(exp)
#    result += op_list
#    result += 'return ' + exp_result[1] + ';\n';
#    return result

def collect_vars(exp):
    result = set()
    if len(exp) == 2:
        if exp[0] == 'Name':
          exp[1] = '_' + exp[1]
          result.add(exp[1])
        if exp[0] == 'Neg':
            result = result | collect_vars(exp[1])
    if len(exp) == 3:
        if exp[0] == 'Call':
            result = result | collect_vars(exp[2])
        else:
            result = result | collect_vars(exp[1]) | collect_vars(exp[2])
    return result

def decl_vars(variables):
    i = 0
    result = ''
    for v in variables.keys():
        result += 'const interval_t & _' + v + ' =  X[' + str(variables[v]) + '];\n'
        i += 1
    return result

def expressify(exp, val_trans = lambda x: 'interval("' + x + '", "' + x + '").get_value()',
               func_trans = lambda x: x):
    print(exp)
    if len(exp) == 2:
        if exp[0] == 'Name':
            return exp[1]
        if exp[0] == 'Value':
            return '(' + val_trans(exp[1]) + ')'
        if exp[0] == 'Neg':
            return '(-' + expressify(exp[1], val_trans, func_trans) + ')'
    if len(exp) == 3:
        if exp[0] == 'Call':
             return func_trans(exp);
#            return '(' + func_trans(exp[1]) + '(' + expressify(exp[2:]) + '))'
        else: # Binary operations
            return '(' + expressify(exp[1], val_trans, func_trans) + exp[0] + expressify(exp[2], val_trans, func_trans) + ')'

def process(exp):
    return 'return interval(' + expressify(exp, func_trans = gelpia_functions) + ');\n'

def get_body(s, variables):
    exp = parser.parse(s)
    v = collect_vars(exp)
    return (decl_vars(variables) + process(exp))

def gelpia_functions(expression):
  if expression[1] == 'interval':
    args = expression[2]
    assert(len(args) == 3) # ['Call', 'interval', ['Args', NUM, NUM]]
    assert(args[1][0] == 'Value')
    assert(args[2][0] == 'Value')
    return '(interval("' + args[1][1] + '", "' + args[2][1] + '").get_value())'
  else:
    return '(' + expression[1] + "(" + expressify(expression[2][1]) + ')' + ')'

<|endoftext|>