diff --git a/attrlogimport.py b/attrlogimport.py index 95b2455..0eec03b 100755 --- a/attrlogimport.py +++ b/attrlogimport.py @@ -18,7 +18,8 @@ def create_tables(): cur.execute(""" CREATE TABLE device ( id serial primary key, - name text + name text, + tell bigint default 0 -- Total number of attrlog lines imported since last time )""") cur.execute(""" CREATE TABLE attrlog ( @@ -33,17 +34,22 @@ def create_tables(): cur.execute("COMMIT") -def create_or_find_device_id(name): +def create_or_find_device(name): with conn.cursor() as cur: - cur.execute("SELECT id FROM device WHERE name = %s", (name,)) + cur.execute("SELECT id, tell FROM device WHERE name = %s", (name,)) row = cur.fetchone() if row is None: - cur.execute("INSERT INTO device (name) VALUES (%s) RETURNING id", (name,)) + cur.execute("INSERT INTO device (name) VALUES (%s) RETURNING id, tell", (name,)) row = cur.fetchone() - return row[0] + return row -def parse_attrlog_file(filename, device_id=None): +def set_tell(device_id, tell): + with conn.cursor() as cur: + cur.execute("UPDATE device SET tell = %s WHERE id = %s", (tell, device_id)) + + +def parse_attrlog_file(filename, device_id=None, start_seek=0): tz_dst = timezone("Europe/Tallinn") utc = timezone("UTC") # This is a date when smartmontools switched from UTC time to local time @@ -54,7 +60,7 @@ def parse_attrlog_file(filename, device_id=None): fd = open(filename) file_size = fd.seek(0, os.SEEK_END) - fd.seek(0) + fd.seek(start_seek) while fd.tell() != file_size: line = fd.readline() @@ -94,18 +100,17 @@ def parse_attrlog_file(filename, device_id=None): id = int(line_parts.pop(0)) norm = int(line_parts.pop(0)) raw = int(line_parts.pop(0)) - yield str(dtu), id, norm, raw, device_id - + yield str(dtu), id, norm, raw, device_id, fd.tell() # Pretty progress indicator if fd.tell() % 100 == 0: - print(f"{int((fd.tell() / file_size)*100):>5}%", end='\r') + print(f"{int(((fd.tell() - start_seek) / (file_size - start_seek))*100):>5}%", end='\r') print() def import_attrlog_file(filename): drive_name = drive_name_re.search(filename).group(1) print(drive_name) - device_id = create_or_find_device_id(drive_name) + device_id, tell = create_or_find_device(drive_name) with conn.cursor() as cur: cur.execute("START TRANSACTION") # for row in parse_attrlog_file(filename, device_id): @@ -114,9 +119,11 @@ def import_attrlog_file(filename): # VALUES (%s, %s, %s, %s, %s) # """, row) with cur.copy("copy attrlog (time, id, norm, raw, device_id) FROM STDIN") as copy: - for row in parse_attrlog_file(filename, device_id): + for row in parse_attrlog_file(filename, device_id, tell): + tell = row[-1] #print(row) - copy.write_row(row) + copy.write_row(row[:-1]) + set_tell(device_id, tell) #cur.execute("ROLLBACK") cur.execute("COMMIT")