From b49138d0ad9909e267c860729e1eddfa37c2b65c Mon Sep 17 00:00:00 2001
From: nate smith <nateksmith@pm.me>
Date: Sun, 21 Apr 2024 20:02:22 -0700
Subject: [PATCH] speed up sqlite3 writes

---
 cmd/ingest/main.go               | 14 +++++++++++---
 ingest.sh => ingest_gutenberg.sh |  0
 2 files changed, 11 insertions(+), 3 deletions(-)
 rename ingest.sh => ingest_gutenberg.sh (100%)

diff --git a/cmd/ingest/main.go b/cmd/ingest/main.go
index 9fde174..e1a40fe 100644
--- a/cmd/ingest/main.go
+++ b/cmd/ingest/main.go
@@ -11,7 +11,7 @@ import (
 	_ "github.com/mattn/go-sqlite3"
 )
 
-const dsn = "phrase.db"
+const dsn = "phrase.db?_journal=OFF"
 
 func createSource(db *sql.DB, sourceName string) (int64, error) {
 	stmt, err := db.Prepare("INSERT INTO sources (name) VALUES (?) ON CONFLICT DO NOTHING RETURNING id")
@@ -24,6 +24,8 @@ func createSource(db *sql.DB, sourceName string) (int64, error) {
 		return -1, err
 	}
 
+	defer stmt.Close()
+
 	return result.LastInsertId()
 }
 
@@ -46,9 +48,15 @@ func _main(args []string) error {
 		return fmt.Errorf("could not make source: %w", err)
 	}
 
+	tx, err := db.Begin()
+	if err != nil {
+		return fmt.Errorf("failed to create transaction: %w", err)
+	}
+
+	stmt, err := tx.Prepare("INSERT INTO phrases (sourceid, phrase) VALUES (?, ?) ON CONFLICT DO NOTHING")
+	defer stmt.Close()
 	for s.Scan() {
 		phrase := s.Text()
-		stmt, err := db.Prepare("INSERT INTO phrases (sourceid, phrase) VALUES (?, ?) ON CONFLICT DO NOTHING")
 		if err != nil {
 			return err
 		}
@@ -58,7 +66,7 @@ func _main(args []string) error {
 		}
 	}
 
-	return nil
+	return tx.Commit()
 }
 
 func main() {
diff --git a/ingest.sh b/ingest_gutenberg.sh
similarity index 100%
rename from ingest.sh
rename to ingest_gutenberg.sh