|
|
|
@ -18,7 +18,8 @@ def create_tables():
|
|
|
|
|
cur.execute("""
|
|
|
|
|
CREATE TABLE device (
|
|
|
|
|
id serial primary key,
|
|
|
|
|
name text
|
|
|
|
|
name text,
|
|
|
|
|
tell bigint default 0 -- Total number of attrlog lines imported since last time
|
|
|
|
|
)""")
|
|
|
|
|
cur.execute("""
|
|
|
|
|
CREATE TABLE attrlog (
|
|
|
|
@ -33,17 +34,22 @@ def create_tables():
|
|
|
|
|
cur.execute("COMMIT")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def create_or_find_device_id(name):
|
|
|
|
|
def create_or_find_device(name):
|
|
|
|
|
with conn.cursor() as cur:
|
|
|
|
|
cur.execute("SELECT id FROM device WHERE name = %s", (name,))
|
|
|
|
|
cur.execute("SELECT id, tell FROM device WHERE name = %s", (name,))
|
|
|
|
|
row = cur.fetchone()
|
|
|
|
|
if row is None:
|
|
|
|
|
cur.execute("INSERT INTO device (name) VALUES (%s) RETURNING id", (name,))
|
|
|
|
|
cur.execute("INSERT INTO device (name) VALUES (%s) RETURNING id, tell", (name,))
|
|
|
|
|
row = cur.fetchone()
|
|
|
|
|
return row[0]
|
|
|
|
|
return row
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def parse_attrlog_file(filename, device_id=None):
|
|
|
|
|
def set_tell(device_id, tell):
|
|
|
|
|
with conn.cursor() as cur:
|
|
|
|
|
cur.execute("UPDATE device SET tell = %s WHERE id = %s", (tell, device_id))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def parse_attrlog_file(filename, device_id=None, start_seek=0):
|
|
|
|
|
tz_dst = timezone("Europe/Tallinn")
|
|
|
|
|
utc = timezone("UTC")
|
|
|
|
|
# This is a date when smartmontools switched from UTC time to local time
|
|
|
|
@ -54,7 +60,7 @@ def parse_attrlog_file(filename, device_id=None):
|
|
|
|
|
|
|
|
|
|
fd = open(filename)
|
|
|
|
|
file_size = fd.seek(0, os.SEEK_END)
|
|
|
|
|
fd.seek(0)
|
|
|
|
|
fd.seek(start_seek)
|
|
|
|
|
|
|
|
|
|
while fd.tell() != file_size:
|
|
|
|
|
line = fd.readline()
|
|
|
|
@ -94,18 +100,17 @@ def parse_attrlog_file(filename, device_id=None):
|
|
|
|
|
id = int(line_parts.pop(0))
|
|
|
|
|
norm = int(line_parts.pop(0))
|
|
|
|
|
raw = int(line_parts.pop(0))
|
|
|
|
|
yield str(dtu), id, norm, raw, device_id
|
|
|
|
|
|
|
|
|
|
yield str(dtu), id, norm, raw, device_id, fd.tell()
|
|
|
|
|
# Pretty progress indicator
|
|
|
|
|
if fd.tell() % 100 == 0:
|
|
|
|
|
print(f"{int((fd.tell() / file_size)*100):>5}%", end='\r')
|
|
|
|
|
print(f"{int(((fd.tell() - start_seek) / (file_size - start_seek))*100):>5}%", end='\r')
|
|
|
|
|
print()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def import_attrlog_file(filename):
|
|
|
|
|
drive_name = drive_name_re.search(filename).group(1)
|
|
|
|
|
print(drive_name)
|
|
|
|
|
device_id = create_or_find_device_id(drive_name)
|
|
|
|
|
device_id, tell = create_or_find_device(drive_name)
|
|
|
|
|
with conn.cursor() as cur:
|
|
|
|
|
cur.execute("START TRANSACTION")
|
|
|
|
|
# for row in parse_attrlog_file(filename, device_id):
|
|
|
|
@ -114,9 +119,11 @@ def import_attrlog_file(filename):
|
|
|
|
|
# VALUES (%s, %s, %s, %s, %s)
|
|
|
|
|
# """, row)
|
|
|
|
|
with cur.copy("copy attrlog (time, id, norm, raw, device_id) FROM STDIN") as copy:
|
|
|
|
|
for row in parse_attrlog_file(filename, device_id):
|
|
|
|
|
for row in parse_attrlog_file(filename, device_id, tell):
|
|
|
|
|
tell = row[-1]
|
|
|
|
|
#print(row)
|
|
|
|
|
copy.write_row(row)
|
|
|
|
|
copy.write_row(row[:-1])
|
|
|
|
|
set_tell(device_id, tell)
|
|
|
|
|
#cur.execute("ROLLBACK")
|
|
|
|
|
cur.execute("COMMIT")
|
|
|
|
|
|
|
|
|
|