diff --git a/download/mirror.go b/download/mirror.go index afa713d..69a059b 100644 --- a/download/mirror.go +++ b/download/mirror.go @@ -9,7 +9,6 @@ import ( const mirrorURL = "https://mirror.minhareceita.org" -// TODO: do we need to export these structs? type MirrorFile struct { URL string `json:"url"` Size uint `json:"size"` diff --git a/transform/kv.go b/transform/kv.go index c8d26aa..221477a 100644 --- a/transform/kv.go +++ b/transform/kv.go @@ -183,7 +183,9 @@ func newBadgerStorage(m bool) (*badgerStorage, error) { var dir string var err error var opt badger.Options - if !m { + if m { + opt = badger.DefaultOptions("").WithInMemory(m) + } else { dir, err = os.MkdirTemp("", badgerFilePrefix) if err != nil { return nil, fmt.Errorf("error creating temporary key-value storage: %w", err) @@ -192,8 +194,6 @@ func newBadgerStorage(m bool) (*badgerStorage, error) { log.Output(1, fmt.Sprintf("Creating temporary key-value storage at %s", dir)) } opt = badger.DefaultOptions(dir) - } else { - opt = badger.DefaultOptions("").WithInMemory(m) } db, err := badger.Open(opt.WithLogger(&badgerLogger{})) if err != nil { diff --git a/transform/transform.go b/transform/transform.go index c6314fe..18cfce6 100644 --- a/transform/transform.go +++ b/transform/transform.go @@ -43,9 +43,6 @@ func saveUpdatedAt(db database, dir string) error { // Transform the downloaded files for company venues creating a database record // per CNPJ func Transform(dir string, db database, maxParallelDBQueries, batchSize int, privacy, mem bool) error { - if err := saveUpdatedAt(db, dir); err != nil { - return fmt.Errorf("error saving the update at date: %w", err) - } l, err := newLookups(dir) if err != nil { return fmt.Errorf("error creating look up tables from %s: %w", dir, err) @@ -62,5 +59,8 @@ func Transform(dir string, db database, maxParallelDBQueries, batchSize int, pri if err != nil { return fmt.Errorf("error creating new task for venues in %s: %w", dir, err) } - return j.run(maxParallelDBQueries) + if err := j.run(maxParallelDBQueries); err != nil { + return fmt.Errorf("error writing venues to database: %w", err) + } + return saveUpdatedAt(db, dir) }