Support importing only new data from end

attrlogimport.py

@@ -18,7 +18,8 @@ def create_tables():
         cur.execute("""
             CREATE TABLE device (
                 id serial primary key,
-                name text
+                name text,
+                tell bigint default 0  -- Total number of attrlog lines imported since last time
             )""")
         cur.execute("""
             CREATE TABLE attrlog (
@@ -33,17 +34,22 @@ def create_tables():
         cur.execute("COMMIT")
 
 
-def create_or_find_device_id(name):
+def create_or_find_device(name):
     with conn.cursor() as cur:
-        cur.execute("SELECT id FROM device WHERE name = %s", (name,))
+        cur.execute("SELECT id, tell FROM device WHERE name = %s", (name,))
         row = cur.fetchone()
         if row is None:
-            cur.execute("INSERT INTO device (name) VALUES (%s) RETURNING id", (name,))
+            cur.execute("INSERT INTO device (name) VALUES (%s) RETURNING id, tell", (name,))
             row = cur.fetchone()
-        return row[0]
+        return row
 
 
-def parse_attrlog_file(filename, device_id=None):
+def set_tell(device_id, tell):
+    with conn.cursor() as cur:
+        cur.execute("UPDATE device SET tell = %s WHERE id = %s", (tell, device_id))
+
+
+def parse_attrlog_file(filename, device_id=None, start_seek=0):
     tz_dst = timezone("Europe/Tallinn")
     utc = timezone("UTC")
     # This is a date when smartmontools switched from UTC time to local time
@@ -54,7 +60,7 @@ def parse_attrlog_file(filename, device_id=None):
 
     fd = open(filename)
     file_size = fd.seek(0, os.SEEK_END)
-    fd.seek(0)
+    fd.seek(start_seek)
 
     while fd.tell() != file_size:
         line = fd.readline()
@@ -94,18 +100,17 @@ def parse_attrlog_file(filename, device_id=None):
             id = int(line_parts.pop(0))
             norm = int(line_parts.pop(0))
             raw = int(line_parts.pop(0))
-            yield str(dtu), id, norm, raw, device_id
-
+            yield str(dtu), id, norm, raw, device_id, fd.tell()
             # Pretty progress indicator
             if fd.tell() % 100 == 0:
-                print(f"{int((fd.tell() / file_size)*100):>5}%", end='\r')
+                print(f"{int(((fd.tell() - start_seek) / (file_size - start_seek))*100):>5}%", end='\r')
     print()
 
 
 def import_attrlog_file(filename):
     drive_name = drive_name_re.search(filename).group(1)
     print(drive_name)
-    device_id = create_or_find_device_id(drive_name)
+    device_id, tell = create_or_find_device(drive_name)
     with conn.cursor() as cur:
         cur.execute("START TRANSACTION")
         # for row in parse_attrlog_file(filename, device_id):
@@ -114,9 +119,11 @@ def import_attrlog_file(filename):
         #     VALUES (%s, %s, %s, %s, %s)
         #     """, row)
         with cur.copy("copy attrlog (time, id, norm, raw, device_id) FROM STDIN") as copy:
-            for row in parse_attrlog_file(filename, device_id):
+            for row in parse_attrlog_file(filename, device_id, tell):
+                tell = row[-1]
                 #print(row)
-                copy.write_row(row)
+                copy.write_row(row[:-1])
+        set_tell(device_id, tell)
         #cur.execute("ROLLBACK")
         cur.execute("COMMIT")