Fix the global startid.

Previously, we would process the whole database every time (rejecting all the previously archived conversations row-by-row. Now we can skip most of the database during our select query if we've run through it previously. This still has to manually reject rows if there was a conversation ongoing in the last 30 minutes on the prior run, but we're talking about rejecting a handful of rows instead of 100,000 of them.
icculus · Jun 20, 2016 · d5015d8 · d5015d8
1 parent 964188d
commit d5015d8
Showing 1 changed file with 26 additions and 11 deletions.
diff --git a/archive_imessage.pl b/archive_imessage.pl
@@ -114,15 +114,19 @@ sub archive_fname {
 
 sub flush_startid {
     my ($with, $msgid) = @_;
-    $startids{$with} = $msgid;
-    my $startval = 0;
-    $startval = $startid if (defined $startid);
-    dbgprint("flushing startids (global: $startval, '$with': $msgid)\n");
+
+    if (defined $with and defined $msgid) {
+        $startids{$with} = $msgid;
+        dbgprint("flushing startids (global: $startid, '$with': $msgid)\n");
+    } else {
+        dbgprint("flushing startids (global: $startid)\n");
+    }
+
     if (not open(LASTID,'>',$lastarchivetmpfname)) {
         dbgprint("Open '$lastarchivetmpfname' for write failed: $!\n");
         return 0;
     }
-    print LASTID "$startval\n";
+    print LASTID "$startid\n";
     foreach(keys %startids) {
         my $key = $_;
         my $val = $startids{$key};
@@ -652,6 +656,14 @@ sub talk_gap {
 my $donerows = 0;
 my $totalrows = undef;
 my $percentdone = -1;
+
+$stmt = $db->prepare('select ROWID from message order by ROWID desc limit 1;') or fail("Couldn't prepare row count SELECT statement: " . $DBI::errstr);
+$stmt->execute() or fail("Couldn't execute row count SELECT statement: " . $DBI::errstr);
+my $ending_startid = $startid;
+if (my @row = $stmt->fetchrow_array()) {
+    $ending_startid = $row[0];
+}
+
 if ($report_progress) {
     $stmt = $db->prepare('select count(*) from message where (ROWID > ?)') or fail("Couldn't prepare message count SELECT statement: " . $DBI::errstr);
     $stmt->execute($startid) or fail("Couldn't execute message count SELECT statement: " . $DBI::errstr);
@@ -667,9 +679,6 @@ sub talk_gap {
 
 $stmt->execute($startid) or fail("Couldn't execute message SELECT statement: " . $DBI::errstr);
 
-
-$startid = undef;
-
 while (my @row = $stmt->fetchrow_array()) {
     if ($debug) {
         dbgprint("New row:\n");
@@ -705,9 +714,9 @@ sub talk_gap {
 
     if (($now - $date) < $gaptime) {
         dbgprint("timestamp '$date' is less than $gaptime seconds old.\n");
-        if ((not defined $startid) or ($msgid < $startid)) {
-            $startid = ($startmsgid-1);
-            dbgprint("forcing global startid to $startid\n");
+        if ($msgid < $ending_startid) {
+            $ending_startid = ($startmsgid-1);
+            dbgprint("forcing global startid to $ending_startid\n");
         }
         # trash this conversation, it might still be ongoing.
         flush_conversation(1) if ($handle_id == $lasthandle_id);
@@ -887,6 +896,12 @@ sub talk_gap {
     flush_conversation(0);
 }
 
+# Update the global startid.
+if ($ending_startid != $startid) {
+    $startid = $ending_startid;
+    flush_startid(undef, undef);
+}
+
 exit(0);
 
 # end of archive_imessage.pl ...