175 lines
5.0 KiB
Python
175 lines
5.0 KiB
Python
import matplotlib.pyplot as plt
|
|
import pandas as pd
|
|
import re
|
|
import sys
|
|
|
|
from math import log
|
|
|
|
# Check if the filename is provided
|
|
if len(sys.argv) < 2:
|
|
print("Usage: python script_name.py your_log_file.log")
|
|
sys.exit(1)
|
|
|
|
filename = sys.argv[1]
|
|
|
|
# Initialize a list to store event data<
|
|
events_data = []
|
|
|
|
# Updated regular expression pattern
|
|
pattern = re.compile(
|
|
r".* time (\d+) pos (\d+) block (\d+) (\w+)(?: (.+))?"
|
|
)
|
|
|
|
# Open and read the log file
|
|
with open(filename, 'r') as file:
|
|
for line in file:
|
|
match = pattern.search(line)
|
|
if match:
|
|
time_ns = int(match.group(1))
|
|
pos = int(match.group(2))
|
|
block = int(match.group(3))
|
|
event = match.group(4)
|
|
action = match.group(5) if match.group(5) else ''
|
|
|
|
# Consider only 'push' and 'write' events
|
|
if event not in ['push', 'write']:
|
|
continue # Ignore other events
|
|
|
|
# Determine event type
|
|
if action == 'done':
|
|
event_type = 'end'
|
|
elif 'size' in action or (event == 'write' and action == ''):
|
|
event_type = 'start'
|
|
else:
|
|
continue # Ignore other events
|
|
|
|
events_data.append({
|
|
'time_ns': time_ns,
|
|
'pos': pos,
|
|
'block': block,
|
|
'event': event,
|
|
'event_type': event_type
|
|
})
|
|
|
|
# After the parsing loop, print the number of events parsed
|
|
print(f"Total events parsed: {len(events_data)}")
|
|
|
|
# Check if events_data is not empty
|
|
if not events_data:
|
|
print("No events were parsed from the log file.")
|
|
sys.exit(1)
|
|
|
|
# Create a DataFrame
|
|
data = pd.DataFrame(events_data)
|
|
|
|
# Convert time from nanoseconds to seconds
|
|
data['time_s'] = data['time_ns'] / 1e9
|
|
|
|
# Separate data by event type
|
|
push_events = data[data['event'] == 'push']
|
|
write_events = data[data['event'] == 'write']
|
|
|
|
|
|
# Function to compute time deltas
|
|
def compute_time_deltas(events_df, event_name):
|
|
# Separate start and end events
|
|
start_events = events_df[events_df['event_type'] == 'start']
|
|
end_events = events_df[events_df['event_type'] == 'end']
|
|
|
|
# Merge start and end events on 'block'
|
|
merged = pd.merge(start_events, end_events, on='block', suffixes=('_start', '_end'))
|
|
|
|
# Compute time deltas
|
|
merged['time_delta_s'] = merged['time_s_end'] - merged['time_s_start']
|
|
|
|
# Remove negative time deltas (if any)
|
|
merged = merged[merged['time_delta_s'] >= 0]
|
|
|
|
# Keep relevant columns
|
|
result = merged[['block', 'pos_start', 'time_delta_s']]
|
|
result['event'] = event_name
|
|
|
|
return result
|
|
|
|
|
|
# Compute time deltas for 'push' and 'write'
|
|
push_deltas = compute_time_deltas(push_events, 'push')
|
|
write_deltas = compute_time_deltas(write_events, 'write')
|
|
|
|
# Combine the deltas for analysis
|
|
deltas = pd.concat([push_deltas, write_deltas], ignore_index=True)
|
|
|
|
|
|
# Identify outliers (e.g., using 1.5 * IQR rule)
|
|
def identify_outliers(df):
|
|
outlier_indices = []
|
|
for event_name, group in df.groupby('event'):
|
|
# Q1 = group['time_delta_s'].quantile(0.25)
|
|
# Q3 = group['time_delta_s'].quantile(0.75)
|
|
# IQR = Q3 - Q1
|
|
# threshold = 1.9 * IQR
|
|
outliers = group[group['time_delta_s'] > 0.3]
|
|
outlier_indices.extend(outliers.index.tolist())
|
|
return outlier_indices
|
|
|
|
|
|
outlier_indices = identify_outliers(deltas)
|
|
outliers = deltas.loc[outlier_indices]
|
|
|
|
# Plot time deltas with outliers annotated
|
|
plt.figure(figsize=(12, 6))
|
|
for event_name, group in deltas.groupby('event'):
|
|
plt.plot(group['block'], group['time_delta_s'], label=f'{event_name.capitalize()} Duration',
|
|
marker='o')
|
|
|
|
# Annotate outliers
|
|
for idx, row in outliers.iterrows():
|
|
plt.annotate(f"Block {int(row['block'])}", (row['block'], row['time_delta_s']),
|
|
textcoords="offset points", xytext=(0, 10), ha='center', color='red')
|
|
|
|
# Optional: Print statistics
|
|
stats = deltas.groupby('event')['time_delta_s'].agg(['mean', 'max', 'min', 'std'])
|
|
print("Event Duration Statistics:")
|
|
print(stats)
|
|
|
|
|
|
def format_size(size):
|
|
units = [
|
|
("bytes", 0),
|
|
("kB", 0),
|
|
("MB", 1),
|
|
("GB", 2),
|
|
("TB", 2),
|
|
("PB", 2)
|
|
]
|
|
|
|
if size < 0:
|
|
raise ValueError("Negative size not allowed")
|
|
if size == 0:
|
|
return "0 bytes"
|
|
if size == 1:
|
|
return "1 byte"
|
|
|
|
exponent = min(int(log(size, 1024)), len(units) - 1)
|
|
quotient = float(size) / 1024 ** exponent
|
|
unit, precision = units[exponent]
|
|
return f"{quotient:.{precision}f} {unit}"
|
|
|
|
|
|
e0 = events_data[0]
|
|
e1 = events_data[-1]
|
|
|
|
t_delta_ns = e1["time_ns"] - e0["time_ns"]
|
|
t_delta_s = t_delta_ns / 1e9
|
|
blocks_delta = e1["block"] - e0["block"]
|
|
bytes_delta = blocks_delta * 512
|
|
|
|
print("Speed:", format_size(bytes_delta / t_delta_s), "/s")
|
|
print("Total time:", round(t_delta_s, 2), "seconds")
|
|
|
|
plt.xlabel('Block Number')
|
|
plt.ylabel('Duration (seconds)')
|
|
plt.title('Time Delta between Start and End of Events (Annotated Outliers)')
|
|
plt.legend()
|
|
plt.grid(True)
|
|
plt.show() |