-
Notifications
You must be signed in to change notification settings - Fork 2
/
utils.py
314 lines (262 loc) · 16.1 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
'''
Editor: @Hyunhomo
python --vesion:3.6.10
Reference: https://github.com/mohyunho
Reference: https://www.sciencedirect.com/science/article/abs/pii/S0925231219309877
'''
## Import libraries in python
'''
Please use below pip to install all package requirements
pip install -r py_pkg_requirements.txt
'''
import logging
import sys, os
import pandas as pd
import numpy as np
from datetime import date
from ts_preprocessing import ts_prep
from ts_window import ts_win
from network_training import network_train
class network_launcher(object):
# Extract single input instance as the form of 4D tensor
def __init__(self):
'''
Constructor
@param none
'''
self.__logger = logging.getLogger("main of network part")
def Extract(self, lst, sample_idx):
extracted_input = []
for sensor_idx in range(len(lst)):
temp = lst[sensor_idx][sample_idx]
temp = np.reshape(temp, (1, temp.shape[0], temp.shape[1], temp.shape[2]))
extracted_input.append(temp)
return extracted_input
def preprocessing_main (self, csv_filename, cols, cols_non_sensor, train=True, piecewise_lin_ref=125):
'''
selecting columns in csv file, calculating RUL and all other kinds of data preparation
:param csv_filename: user defined csv filename
:return train_FD: save to the same csv file
:return cols_sensors: selected sensor columns in dataframe that are used as the network input
'''
tmp_path = 'tmp/'
csv_path = tmp_path + csv_filename
print (csv_path)
## Load data from csv and assign to dataframe
data_FD = ts_prep().load_train_csv(csv_path, cols)
## All processing within dataframe (RUL calculation, column drop and so on)
data_FD = ts_prep().df_preparation(data_FD, train, piecewise_lin_ref)
## Normalize sensor measurement data
data_FD, cols_sensors = ts_prep().df_preprocessing(data_FD, cols_non_sensor, train)
print (data_FD)
print ("pre-processing is successfully completed") # remark the end of the processing
## Return dataframe after preprocessing
return data_FD, cols_sensors
def network_input_generator (self, dataframe_norm, cols_non_sensor, test_engine_idx=None, train=True, application=False,
rul_file_path=None, sequence_length = 30, stride = 1, window_length = 3,
piecewise_lin_ref=125):
'''
generating the training input for multi-head CNN-LSTM network & the label of each input samples
:param dataframe_norm: dataframe after preprocessing
:param cols_non_sensor: columns in dataframe that are not the result of sensor measurement
:param train: If it is true, then the func generate training input. Otherwise, the func generates test input
:param application: If it is true, then the func generate tes input without ground truth (actual RUL).
:param sequence_length: the number of time stamps for sliced time series
:param stride: stride of ts window
:param window_length: segmented time series from sequence
:return network_input: ready to use input for CNN-LSTM network after applying time series window (python list)
:return network_input_label: label(target value) of network input
'''
n_window = int((sequence_length - window_length) / (stride) + 1)
n_engines = len(dataframe_norm['unit_nr'].unique())
if train == True:
## Generate numpy array of sequence
seq = ts_win().seq_generetion(dataframe_norm, cols_non_sensor,sequence_length)
# print("the number of input samples: ", seq.shape[0])
# print("the length of each sequence: ", seq.shape[1])
## Reshape the time series as the network input (python list of parallel input)
network_input = ts_win().networkinput_generetion(seq, stride, n_window, window_length)
## Generate numpy array of label(target value) of each sequence
network_input_label = ts_win().label_generetion(dataframe_norm, sequence_length)
else:
ts_array_test_engine = dataframe_norm[dataframe_norm['unit_nr'] == test_engine_idx+1]
test_seq = ts_win().test_seq_generetion(ts_array_test_engine, cols_non_sensor, sequence_length)
network_input = ts_win().networkinput_generetion(test_seq, stride, n_window, window_length)
# print ("network_input", network_input[1].shape)
if application == False:
network_input_label = []
else:
##Ground truth generator
y_mask = [len(dataframe_norm[dataframe_norm['unit_nr'] == id]) >= sequence_length for id in
dataframe_norm['unit_nr'].unique()]
# print ("y_mask", y_mask)
col_rul = ['RUL_truth']
RUL_FD = pd.read_csv(rul_file_path, sep=' ', header=None, names=col_rul, index_col=False)
# Cut max RUL ground truth
RUL_FD['RUL_truth'].loc[(RUL_FD['RUL_truth'] > piecewise_lin_ref)] = piecewise_lin_ref
label_array_test_last = RUL_FD['RUL_truth'][y_mask].values
label_array_test_last = label_array_test_last.reshape(label_array_test_last.shape[0], 1).astype(np.float32)
engine_RUL = label_array_test_last[test_engine_idx]
test_seq_ground_truth = np.arange(test_seq.shape[0] + engine_RUL - 1, engine_RUL - 1, -1)
test_seq_ground_truth[test_seq_ground_truth > piecewise_lin_ref] = piecewise_lin_ref
network_input_label = test_seq_ground_truth
return network_input, network_input_label
def opt_network_input_generator (self, dataframe_norm, cols_non_sensor, sequence_length = 30,
stride = 1, window_length = 3,piecewise_lin_ref=125):
'''
generating the training input for multi-head CNN-LSTM network & the label of each input samples
:param dataframe_norm: dataframe after preprocessing
:param cols_non_sensor: columns in dataframe that are not the result of sensor measurement
:param train: If it is true, then the func generate training input. Otherwise, the func generates test input
:param application: If it is true, then the func generate tes input without ground truth (actual RUL).
:param sequence_length: the number of time stamps for sliced time series
:param stride: stride of ts window
:param window_length: segmented time series from sequence
:return network_input: ready to use input for CNN-LSTM network after applying time series window (python list)
:return network_input_label: label(target value) of network input
'''
n_window = int((sequence_length - window_length) / (stride) + 1)
n_engines = len(dataframe_norm['unit_nr'].unique())
## Generate numpy array of sequence
seq = ts_win().seq_generetion(dataframe_norm, cols_non_sensor,sequence_length)
# print("the number of input samples: ", seq.shape[0])
# print("the length of each sequence: ", seq.shape[1])
## Reshape the time series as the network input (python list of parallel input)
network_input = ts_win().networkinput_generetion(seq, stride, n_window, window_length)
## Generate numpy array of label(target value) of each sequence
network_input_label = ts_win().label_generetion(dataframe_norm, sequence_length)
return network_input, network_input_label
# def rmse_test_input_generator (self, test_FD, cols_non_sensor, test_engine_idx=None,
# rul_file_path=None, sequence_length = 30, stride = 1, window_length = 3,
# piecewise_lin_ref=125):
# '''
# generating the training input for multi-head CNN-LSTM network & the label of each input samples
# :param dataframe_norm: dataframe after preprocessing
# :param cols_non_sensor: columns in dataframe that are not the result of sensor measurement
#
# '''
# n_window = int((sequence_length - window_length) / (stride) + 1)
#
#
# sequence_cols_test = test_FD.columns.difference(cols_non_sensor)
# # We pick the last sequence for each id in the test data
# seq_array_test_last = [test_FD[test_FD['unit_nr'] == id][sequence_cols_test].values[-sequence_length:]
# for id in test_FD['unit_nr'].unique() if
# len(test_FD[test_FD['unit_nr'] == id]) >= sequence_length]
# # print(seq_array_test_last[0].shape)
#
# seq_array_test_last = np.asarray(seq_array_test_last).astype(np.float32)
# # print("seq_array_test_last")
# # print(seq_array_test_last.shape)
#
# # Reshape test array to 4D tensor (as the input for CNN-LSTM)
# network_input = ts_win().networkinput_generetion(seq_array_test_last, stride, n_window, window_length)
# # print ("network_input", network_input[1].shape)
#
# ##Ground truth generator
# y_mask = [len(test_FD[test_FD['unit_nr'] == id]) >= sequence_length for id in
# test_FD['unit_nr'].unique()]
# # print("y_mask", y_mask)
#
# col_rul = ['RUL_truth']
# RUL_FD = pd.read_csv(rul_file_path, sep=' ', header=None, names=col_rul, index_col=False)
# # Cut max RUL ground truth
# RUL_FD['RUL_truth'].loc[(RUL_FD['RUL_truth'] > piecewise_lin_ref)] = piecewise_lin_ref
# label_array_test_last = RUL_FD['RUL_truth'][y_mask].values
# network_input_label = label_array_test_last.reshape(label_array_test_last.shape[0], 1).astype(np.float32)
# # print(network_input_label)
# # print(network_input_label.shape)
#
# return network_input, network_input_label
def rmse_test_input_generator (self, dataframe_norm, cols_non_sensor, test_engine_idx=None,
rul_file_path=None, sequence_length = 30, stride = 1, window_length = 3,
piecewise_lin_ref=125):
'''
generating the training input for multi-head CNN-LSTM network & the label of each input samples
:param dataframe_norm: dataframe after preprocessing
:param cols_non_sensor: columns in dataframe that are not the result of sensor measurement
'''
n_window = int((sequence_length - window_length) / (stride) + 1)
n_engines = len(dataframe_norm['unit_nr'].unique())
sequence_cols_test = dataframe_norm.columns.difference(cols_non_sensor)
# We pick the last sequence for each id in the test data
seq_array_test_last = [dataframe_norm[dataframe_norm['unit_nr'] == id][sequence_cols_test].values[-sequence_length:]
for id in dataframe_norm['unit_nr'].unique() if
len(dataframe_norm[dataframe_norm['unit_nr'] == id]) >= sequence_length]
# print(seq_array_test_last[0].shape)
seq_array_test_last = np.asarray(seq_array_test_last).astype(np.float32)
# print("seq_array_test_last")
# print(seq_array_test_last.shape)
# Reshape test array to 4D tensor (as the input for CNN-LSTM)
network_input = ts_win().networkinput_generetion(seq_array_test_last, stride, n_window, window_length)
# print ("network_input", network_input[1].shape)
##Ground truth generator
y_mask = [len(dataframe_norm[dataframe_norm['unit_nr'] == id]) >= sequence_length for id in
dataframe_norm['unit_nr'].unique()]
# print("y_mask", y_mask)
col_rul = ['RUL_truth']
RUL_FD = pd.read_csv(rul_file_path, sep=' ', header=None, names=col_rul, index_col=False)
# Cut max RUL ground truth
RUL_FD['RUL_truth'].loc[(RUL_FD['RUL_truth'] > piecewise_lin_ref)] = piecewise_lin_ref
label_array_test_last = RUL_FD['RUL_truth'][y_mask].values
network_input_label = label_array_test_last.reshape(label_array_test_last.shape[0], 1).astype(np.float32)
# print(network_input_label)
# print(network_input_label.shape)
return network_input, network_input_label
def network_training (self, training_input, training_input_label, cols_sensors, model_path, fitness_mode, log_file_path,
dropout, test=False, test_engine_idx=None,
n_channel=1, n_filters=2 , strides_len=1, kernel_size=3, n_conv_layer=2, LSTM1_ref=10, LSTM2_ref=5,
n_outputs=1, cross_val=True, k_value_fold=10, val_split=0.2, batch_size=400, max_epoch=40,
patience=10, bidirec = False, experiment = False, geno_list =None):
'''
generating the multi-head CNN-LSTM network & training the network
:param training_input: ready to use input for CNN-LSTM network after applying time series window (python list)
:param training_input_label: label(target value) of network input
:param cols_sensors: selected sensor columns in dataframe that are used as the network input
:param cols_sensors: selected sensor columns in dataframe that are used as the network input
:param model_path: h5 file path of the network model
:param n_channel: the number of network channel (in the case of multi-head, it should be 1)
:param n_filters: the number of convolutional filters, increase the number
if the network cannot recognize the pattern
:param strides_len: stride length of convolution
:param kernel_size: kernel size of convolution
:param n_conv_layer: the number of (stacked) convolutional layers, increase the number
if the network cannot recognize the pattern. Max number is 4
:param LSTM1_ref: reference number for the number of units in LSTM layer 1
:param LSTM2_ref: reference number for the number of units in LSTM layer 2
:param n_outputs: the number of output of the network (in the case of RUL prediction, it should be 1)
:param cross_val: whether to use k-fold cross validation
(please consider cross validation only the case of using GPU)
:param k_value_fold: k-value for fold cross validation (10 or 5 is recommended)
:param val_split: the ratio of validation (from training) for the decision of early stopping
:param batch_size: the batch size for optimization (gradient)
:param max_epoch: the maximum number of training epoch (if the training loss does not converge
without early stopping, increase the number)
:param patience: epoch patience for early stopping decision
:param bidirec: whether to use bidirectional for LSTM layers
:return trained network is saved to model path
'''
## Defining relative parameters
n_training_samples = training_input[0].shape[0]
n_window = training_input[0].shape[1]
window_length = training_input[0].shape[2]
# LSTM_u1 = n_window*LSTM1_ref
# LSTM_u2 = n_window*LSTM2_ref
# LSTM_u1 = LSTM1_ref
# LSTM_u2 = LSTM2_ref
LSTM_u1 = 20*LSTM1_ref
LSTM_u2 = 20*LSTM2_ref
## Training the network
if test == False:
fitness_net = network_train().network_training(training_input, training_input_label,
cols_sensors, model_path, fitness_mode, log_file_path,
LSTM_u1, LSTM_u2, cross_val, k_value_fold, val_split, n_channel,
n_filters, strides_len, kernel_size, window_length, n_window,
n_conv_layer, n_outputs, batch_size, max_epoch, patience, bidirec, dropout, experiment, geno_list)
# print ("log_file_path", log_file_path)
# temp_df.to_csv(log_file_path, mode='a', header=None)
return fitness_net
elif test == True:
y_predicted, y_actual = network_train().network_test(training_input, training_input_label, test_engine_idx,
model_path)
return y_predicted, y_actual