Files
etchdroid/misc/ostream-trace-log-analyzer.py
T

175 lines
5.0 KiB
Python

import matplotlib.pyplot as plt
import pandas as pd
import re
import sys
from math import log
# Check if the filename is provided
if len(sys.argv) < 2:
print("Usage: python script_name.py your_log_file.log")
sys.exit(1)
filename = sys.argv[1]
# Initialize a list to store event data<
events_data = []
# Updated regular expression pattern
pattern = re.compile(
r".* time (\d+) pos (\d+) block (\d+) (\w+)(?: (.+))?"
)
# Open and read the log file
with open(filename, 'r') as file:
for line in file:
match = pattern.search(line)
if match:
time_ns = int(match.group(1))
pos = int(match.group(2))
block = int(match.group(3))
event = match.group(4)
action = match.group(5) if match.group(5) else ''
# Consider only 'push' and 'write' events
if event not in ['push', 'write']:
continue # Ignore other events
# Determine event type
if action == 'done':
event_type = 'end'
elif 'size' in action or (event == 'write' and action == ''):
event_type = 'start'
else:
continue # Ignore other events
events_data.append({
'time_ns': time_ns,
'pos': pos,
'block': block,
'event': event,
'event_type': event_type
})
# After the parsing loop, print the number of events parsed
print(f"Total events parsed: {len(events_data)}")
# Check if events_data is not empty
if not events_data:
print("No events were parsed from the log file.")
sys.exit(1)
# Create a DataFrame
data = pd.DataFrame(events_data)
# Convert time from nanoseconds to seconds
data['time_s'] = data['time_ns'] / 1e9
# Separate data by event type
push_events = data[data['event'] == 'push']
write_events = data[data['event'] == 'write']
# Function to compute time deltas
def compute_time_deltas(events_df, event_name):
# Separate start and end events
start_events = events_df[events_df['event_type'] == 'start']
end_events = events_df[events_df['event_type'] == 'end']
# Merge start and end events on 'block'
merged = pd.merge(start_events, end_events, on='block', suffixes=('_start', '_end'))
# Compute time deltas
merged['time_delta_s'] = merged['time_s_end'] - merged['time_s_start']
# Remove negative time deltas (if any)
merged = merged[merged['time_delta_s'] >= 0]
# Keep relevant columns
result = merged[['block', 'pos_start', 'time_delta_s']]
result['event'] = event_name
return result
# Compute time deltas for 'push' and 'write'
push_deltas = compute_time_deltas(push_events, 'push')
write_deltas = compute_time_deltas(write_events, 'write')
# Combine the deltas for analysis
deltas = pd.concat([push_deltas, write_deltas], ignore_index=True)
# Identify outliers (e.g., using 1.5 * IQR rule)
def identify_outliers(df):
outlier_indices = []
for event_name, group in df.groupby('event'):
# Q1 = group['time_delta_s'].quantile(0.25)
# Q3 = group['time_delta_s'].quantile(0.75)
# IQR = Q3 - Q1
# threshold = 1.9 * IQR
outliers = group[group['time_delta_s'] > 0.3]
outlier_indices.extend(outliers.index.tolist())
return outlier_indices
outlier_indices = identify_outliers(deltas)
outliers = deltas.loc[outlier_indices]
# Plot time deltas with outliers annotated
plt.figure(figsize=(12, 6))
for event_name, group in deltas.groupby('event'):
plt.plot(group['block'], group['time_delta_s'], label=f'{event_name.capitalize()} Duration',
marker='o')
# Annotate outliers
for idx, row in outliers.iterrows():
plt.annotate(f"Block {int(row['block'])}", (row['block'], row['time_delta_s']),
textcoords="offset points", xytext=(0, 10), ha='center', color='red')
# Optional: Print statistics
stats = deltas.groupby('event')['time_delta_s'].agg(['mean', 'max', 'min', 'std'])
print("Event Duration Statistics:")
print(stats)
def format_size(size):
units = [
("bytes", 0),
("kB", 0),
("MB", 1),
("GB", 2),
("TB", 2),
("PB", 2)
]
if size < 0:
raise ValueError("Negative size not allowed")
if size == 0:
return "0 bytes"
if size == 1:
return "1 byte"
exponent = min(int(log(size, 1024)), len(units) - 1)
quotient = float(size) / 1024 ** exponent
unit, precision = units[exponent]
return f"{quotient:.{precision}f} {unit}"
e0 = events_data[0]
e1 = events_data[-1]
t_delta_ns = e1["time_ns"] - e0["time_ns"]
t_delta_s = t_delta_ns / 1e9
blocks_delta = e1["block"] - e0["block"]
bytes_delta = blocks_delta * 512
print("Speed:", format_size(bytes_delta / t_delta_s), "/s")
print("Total time:", round(t_delta_s, 2), "seconds")
plt.xlabel('Block Number')
plt.ylabel('Duration (seconds)')
plt.title('Time Delta between Start and End of Events (Annotated Outliers)')
plt.legend()
plt.grid(True)
plt.show()