1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
use diem_metrics::{
register_histogram, register_histogram_vec, register_int_counter, register_int_counter_vec,
register_int_gauge, register_int_gauge_vec, DurationHistogram, Histogram, HistogramVec,
IntCounter, IntCounterVec, IntGauge, IntGaugeVec,
};
use once_cell::sync::Lazy;
pub const SEND_SUCCESS_LABEL: &str = "success";
pub const SEND_FAIL_LABEL: &str = "fail";
pub const SYNC_MSG_LABEL: &str = "sync";
pub const COMMIT_MSG_LABEL: &str = "commit";
pub const CHUNK_REQUEST_MSG_LABEL: &str = "chunk_request";
pub const CHUNK_RESPONSE_MSG_LABEL: &str = "chunk_response";
pub fn set_timestamp(timestamp_type: TimestampType, time_as_usecs: u64) {
TIMESTAMP
.with_label_values(&[timestamp_type.as_str()])
.set((time_as_usecs / 1000) as i64)
}
pub enum TimestampType {
Committed,
Real,
Synced,
}
impl TimestampType {
pub fn as_str(&self) -> &'static str {
match self {
TimestampType::Committed => "committed",
TimestampType::Real => "real",
TimestampType::Synced => "synced",
}
}
}
pub fn set_version(version_type: VersionType, version: u64) {
VERSION
.with_label_values(&[version_type.as_str()])
.set(version as i64)
}
pub fn get_version(version_type: VersionType) -> u64 {
VERSION.with_label_values(&[version_type.as_str()]).get() as u64
}
pub enum VersionType {
Committed,
Highest,
Synced,
Target,
}
impl VersionType {
pub fn as_str(&self) -> &'static str {
match self {
VersionType::Committed => "committed",
VersionType::Highest => "highest",
VersionType::Synced => "synced",
VersionType::Target => "target",
}
}
}
pub const CONSENSUS_SYNC_REQ_CALLBACK: &str = "consensus_sync_req_callback";
pub const WAYPOINT_INIT_CALLBACK: &str = "waypoint_init_callback";
pub const SUCCESS_LABEL: &str = "success";
pub const FAIL_LABEL: &str = "fail";
pub const MEMPOOL_LABEL: &str = "mempool";
pub const CONSENSUS_LABEL: &str = "consensus";
pub const COMPLETE_LABEL: &str = "complete";
pub const TIMEOUT_LABEL: &str = "timeout";
pub static PENDING_STATE_SYNC_NETWORK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
register_int_counter_vec!(
"diem_state_sync_pending_network_events",
"Counters(queued,dequeued,dropped) related to pending network notifications for State Sync",
&["state"]
)
.unwrap()
});
pub static REQUESTS_SENT: Lazy<IntCounterVec> = Lazy::new(|| {
register_int_counter_vec!(
"diem_state_sync_requests_sent_total",
"Number of chunk requests sent",
&["network", "peer", "result"]
)
.unwrap()
});
pub static RESPONSES_SENT: Lazy<IntCounterVec> = Lazy::new(|| {
register_int_counter_vec!(
"diem_state_sync_responses_sent_total",
"Number of chunk responses sent (including FN subscriptions)",
&["network", "peer", "result"]
)
.unwrap()
});
pub static RESPONSE_FROM_DOWNSTREAM_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
register_int_counter_vec!(
"diem_state_sync_responses_from_downstream_total",
"Number of chunk responses received from a downstream peer",
&["network", "peer"]
)
.unwrap()
});
pub static APPLY_CHUNK_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
register_int_counter_vec!(
"diem_state_sync_apply_chunk_total",
"Number of Success results of applying a chunk",
&["network", "sender", "result"]
)
.unwrap()
});
pub static PROCESS_CHUNK_REQUEST_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
register_int_counter_vec!(
"diem_state_sync_process_chunk_request_total",
"Number of times chunk request was processed",
&["network", "sender", "result"]
)
.unwrap()
});
pub static STATE_SYNC_CHUNK_SIZE: Lazy<HistogramVec> = Lazy::new(|| {
register_histogram_vec!(
"diem_state_sync_chunk_size",
"Number of transactions in a state sync chunk response",
&["network", "sender"]
)
.unwrap()
});
pub static ACTIVE_UPSTREAM_PEERS: Lazy<IntGaugeVec> = Lazy::new(|| {
register_int_gauge_vec!(
"diem_state_sync_active_upstream_peers",
"Number of upstream peers that are currently active",
&["network"]
)
.unwrap()
});
pub static MULTICAST_LEVEL: Lazy<IntGauge> = Lazy::new(|| {
register_int_gauge!(
"diem_state_sync_multicast_level",
"Max network preference of the networks state sync is sending chunk requests to"
)
.unwrap()
});
pub static TIMESTAMP: Lazy<IntGaugeVec> = Lazy::new(|| {
register_int_gauge_vec!(
"diem_state_sync_timestamp",
"Timestamp involved in state sync progress",
&["type"] )
.unwrap()
});
pub static VERSION: Lazy<IntGaugeVec> = Lazy::new(|| {
register_int_gauge_vec!(
"diem_state_sync_version",
"Version involved in state sync progress",
&["type"] )
.unwrap()
});
pub static EPOCH: Lazy<IntGauge> = Lazy::new(|| {
register_int_gauge!("diem_state_sync_epoch", "Current epoch in local state").unwrap()
});
pub static SYNC_PROGRESS_DURATION: Lazy<DurationHistogram> = Lazy::new(|| {
DurationHistogram::new(
register_histogram!(
"diem_state_sync_sync_progress_duration_s",
"Histogram of time it takes to sync a chunk, from requesting a chunk to processing the response and committing the chunk"
)
.unwrap()
)
});
pub static TIMEOUT: Lazy<IntCounter> = Lazy::new(|| {
register_int_counter!(
"diem_state_sync_timeout_total",
"Number of timeouts that occur during sync"
)
.unwrap()
});
pub static SYNC_REQUEST_RESULT: Lazy<IntCounterVec> = Lazy::new(|| {
register_int_counter_vec!(
"diem_state_sync_sync_request_total",
"Number of sync requests (from consensus) processed",
&["result"]
)
.unwrap()
});
pub static COMMIT_FLOW_FAIL: Lazy<IntCounterVec> = Lazy::new(|| {
register_int_counter_vec!(
"diem_state_sync_commit_flow_fail_total",
"Number of timeouts that occur during the commit flow across consensus, state sync, and mempool",
&["component"] )
.unwrap()
});
pub static FAILED_CHANNEL_SEND: Lazy<IntCounterVec> = Lazy::new(|| {
register_int_counter_vec!(
"diem_state_sync_failed_channel_sends_total",
"Number of times a channel send failed in state sync",
&["type"]
)
.unwrap()
});
pub static EXECUTE_CHUNK_DURATION: Lazy<Histogram> = Lazy::new(|| {
register_histogram!(
"diem_state_sync_execute_chunk_duration_s",
"Histogram of time it takes for state sync's executor proxy to fully execute a chunk"
)
.unwrap()
});
pub static SUBSCRIPTION_DELIVERY_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
register_int_counter_vec!(
"diem_state_sync_subscription_delivery_count",
"Number of times a node delivers a subscription for FN long-poll",
&["network", "recipient", "result"]
)
.unwrap()
});
pub static PROCESS_COORDINATOR_MSG_LATENCY: Lazy<HistogramVec> = Lazy::new(|| {
register_histogram_vec!(
"diem_state_sync_coordinator_msg_latency",
"Time it takes to process a message from consensus",
&["type"]
)
.unwrap()
});
pub static PROCESS_MSG_LATENCY: Lazy<HistogramVec> = Lazy::new(|| {
register_histogram_vec!(
"diem_state_sync_process_msg_latency",
"Time it takes to process a message in state sync",
&["network", "sender", "type"]
)
.unwrap()
});
pub static CONSENSUS_COMMIT_FAIL_COUNT: Lazy<IntCounter> = Lazy::new(|| {
register_int_counter!(
"diem_state_sync_consensus_commit_fail",
"Number of times a commit msg from consensus failed to be processed"
)
.unwrap()
});
pub static RECONFIG_PUBLISH_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
register_int_counter_vec!(
"diem_state_sync_reconfig_count",
"Number of times on-chain reconfig notification is published in state sync",
&["result"]
)
.unwrap()
});
pub static STORAGE_READ_FAIL_COUNT: Lazy<IntCounter> = Lazy::new(|| {
register_int_counter!(
"diem_state_sync_storage_read_fail_count",
"Number of times storage read failed in state sync"
)
.unwrap()
});
pub static NETWORK_ERROR_COUNT: Lazy<IntCounter> = Lazy::new(|| {
register_int_counter!(
"diem_state_sync_network_error_count",
"Number of network errors encountered in state sync"
)
.unwrap()
});
pub static MAIN_LOOP: Lazy<DurationHistogram> = Lazy::new(|| {
DurationHistogram::new(
register_histogram!(
"diem_state_sync_main_loop",
"Duration of the each run of the event loop"
)
.unwrap(),
)
});