From a971b0dce95e3077a835c6c1c3093e4563177d69 Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Wed, 1 Feb 2023 18:54:49 +1100 Subject: [PATCH 01/23] Progress with creating better metadata This has some great early signs - it reduces the size of the assets/metadata dir from 40MB -> 3.5MB!! --- src/bin/pokedex/pokedex.go | 48 +++++++++++++++++++++++--------------- src/pokedex/metadata.go | 20 +++++++++++++--- src/pokedex/pokedex.go | 42 ++++++++++++++------------------- 3 files changed, 63 insertions(+), 47 deletions(-) diff --git a/src/bin/pokedex/pokedex.go b/src/bin/pokedex/pokedex.go index 8840dbe4..d986f683 100644 --- a/src/bin/pokedex/pokedex.go +++ b/src/bin/pokedex/pokedex.go @@ -8,7 +8,6 @@ import ( "path" "strings" - "github.com/tmck-code/pokesay/src/bin" "github.com/tmck-code/pokesay/src/pokedex" ) @@ -100,25 +99,36 @@ func main() { pokemonNames := pokedex.ReadNames(args.FromMetadataFname) fmt.Println("- Read", len(pokemonNames), "pokemon names from", args.FromMetadataFname) - // 1. Create the category struct using the cowfile paths, pokemon names and indexes\ - fmt.Println("- Writing categories to file") - pokedex.WriteStructToFile( - pokedex.CreateCategoryStruct(args.FromDir, cowfileFpaths, args.Debug), - categoryFpath, - ) - - // 2. Create the metadata files, containing name/category/japanese name info for each pokemon - metadata := pokedex.CreateMetadata(args.FromDir, cowfileFpaths, pokemonNames, args.Debug) - - fmt.Println("- Writing metadata and entries to file") - pbar := bin.NewProgressBar(len(metadata)) - for _, m := range metadata { - pokedex.WriteBytesToFile(m.Data, pokedex.EntryFpath(entryDirPath, m.Index), true) - pokedex.WriteStructToFile(m.Metadata, pokedex.MetadataFpath(metadataDirPath, m.Index)) - pbar.Add(1) + // 1. For each pokemon name, write a metadata file, containing the name information, and + // links to all of the matching cowfile indexes + i := 0 + for key, name := range pokemonNames { + metadata := pokedex.CreateNameMetadata(i, key, name, args.FromDir, cowfileFpaths) + fmt.Printf("%+v\n", metadata) + pokedex.WriteStructToFile(metadata, pokedex.MetadataFpath(metadataDirPath, i)) + i++ } - fmt.Println("- Writing total metadata to file") - pokedex.WriteIntToFile(len(metadata), totalFpath) + fmt.Println("wrote", i, "name metadata files to", metadataDirPath) + + // 1. Create the metadata files, containing name/category/japanese name info for each pokemon + // metadata := pokedex.CreateMetadata(args.FromDir, cowfileFpaths, pokemonNames, args.Debug) + + // 2. Create the category struct using the cowfile paths, pokemon names and indexes\ + // fmt.Println("- Writing categories to file") + // pokedex.WriteStructToFile( + // pokedex.CreateCategoryStruct(args.FromDir, cowfileFpaths, args.Debug), + // categoryFpath, + // ) + + // fmt.Println("- Writing metadata and entries to file") + // pbar := bin.NewProgressBar(len(metadata)) + // for _, m := range metadata { + // pokedex.WriteBytesToFile(m.Data, pokedex.EntryFpath(entryDirPath, m.Index), true) + // pokedex.WriteStructToFile(m.Metadata, pokedex.MetadataFpath(metadataDirPath, m.Index)) + // pbar.Add(1) + // } + // fmt.Println("- Writing total metadata to file") + // pokedex.WriteIntToFile(len(metadata), totalFpath) fmt.Println("✓ Complete! Indexed", len(cowfileFpaths), "total cowfiles") fmt.Println("✓ Wrote gzipped metadata to", metadataDirPath) diff --git a/src/pokedex/metadata.go b/src/pokedex/metadata.go index 63c03b8c..d0f5b2ac 100644 --- a/src/pokedex/metadata.go +++ b/src/pokedex/metadata.go @@ -2,19 +2,33 @@ package pokedex import "embed" +type PokemonEntryMapping struct { + EntryIndex int + Categories []string +} + type PokemonMetadata struct { - Categories string Name string JapaneseName string JapanesePhonetic string + Entries []PokemonEntryMapping } -func NewMetadata(categories string, name string, japaneseName string, japanesePhonetic string) *PokemonMetadata { +func NewMetadata(name string, japaneseName string, japanesePhonetic string, entryMap map[int][][]string) *PokemonMetadata { + + entries := make([]PokemonEntryMapping, 0) + + for idx, categories := range entryMap { + for _, category := range categories { + entries = append(entries, PokemonEntryMapping{idx, category}) + } + } + return &PokemonMetadata{ - Categories: categories, Name: name, JapaneseName: japaneseName, JapanesePhonetic: japanesePhonetic, + Entries: entries, } } diff --git a/src/pokedex/pokedex.go b/src/pokedex/pokedex.go index 542f2e98..1ddb7400 100644 --- a/src/pokedex/pokedex.go +++ b/src/pokedex/pokedex.go @@ -11,8 +11,8 @@ import ( "log" "os" "path" - "strings" "strconv" + "strings" ) func Check(e error) { @@ -122,32 +122,24 @@ func (m Metadata) WriteToFile(fpath string) { WriteStructToFile(m, fpath) } -func CreateMetadata(rootDir string, fpaths []string, pokemonNames map[string]PokemonName, debug bool) []Metadata { - metadata := []Metadata{} - for i, fpath := range fpaths { - data, err := os.ReadFile(fpath) - Check(err) +func CreateNameMetadata(idx int, key string, name PokemonName, rootDir string, fpaths []string) *PokemonMetadata { + entryCategories := make(map[int][][]string, 0) - cats := createCategories(strings.TrimPrefix(fpath, rootDir), data) - name := createName(fpath) - - v := pokemonNames[strings.Split(name, "-")[0]] - - metadata = append( - metadata, - Metadata{ - data, - i, - PokemonMetadata{ - Name: name, - JapaneseName: v.Japanese, - JapanesePhonetic: v.JapanesePhonetic, - Categories: strings.Join(cats, "/"), - }, - }, - ) + for _, fpath := range fpaths { + basename := strings.TrimPrefix(fpath, rootDir) + if strings.Contains(basename, strings.ToLower(name.English)) { + data, err := os.ReadFile(fpath) + Check(err) + cats := createCategories(strings.TrimPrefix(fpath, rootDir), data) + entryCategories[idx] = append(entryCategories[idx], cats) + } } - return metadata + return NewMetadata( + name.English, + name.Japanese, + name.JapanesePhonetic, + entryCategories, + ) } func CreateCategoryStruct(rootDir string, fpaths []string, debug bool) Trie { From a70cd43a08ececea4bcf3bf10307a69dde9cc896 Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Wed, 1 Feb 2023 19:03:34 +1100 Subject: [PATCH 02/23] Write each pokemon name to the category trie Instead of every single entry, 5000 nodes -> 900, 84 KB -> ~60KB --- src/bin/pokedex/pokedex.go | 13 ++++++++----- src/pokedex/pokedex.go | 20 ++++++++------------ 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/bin/pokedex/pokedex.go b/src/bin/pokedex/pokedex.go index d986f683..02d1314f 100644 --- a/src/bin/pokedex/pokedex.go +++ b/src/bin/pokedex/pokedex.go @@ -101,11 +101,14 @@ func main() { // 1. For each pokemon name, write a metadata file, containing the name information, and // links to all of the matching cowfile indexes + + pokemonMetadata := make([]pokedex.PokemonMetadata, 0) i := 0 for key, name := range pokemonNames { metadata := pokedex.CreateNameMetadata(i, key, name, args.FromDir, cowfileFpaths) fmt.Printf("%+v\n", metadata) pokedex.WriteStructToFile(metadata, pokedex.MetadataFpath(metadataDirPath, i)) + pokemonMetadata = append(pokemonMetadata, *metadata) i++ } fmt.Println("wrote", i, "name metadata files to", metadataDirPath) @@ -114,11 +117,11 @@ func main() { // metadata := pokedex.CreateMetadata(args.FromDir, cowfileFpaths, pokemonNames, args.Debug) // 2. Create the category struct using the cowfile paths, pokemon names and indexes\ - // fmt.Println("- Writing categories to file") - // pokedex.WriteStructToFile( - // pokedex.CreateCategoryStruct(args.FromDir, cowfileFpaths, args.Debug), - // categoryFpath, - // ) + fmt.Println("- Writing categories to file") + pokedex.WriteStructToFile( + pokedex.CreateCategoryStruct(args.FromDir, pokemonMetadata, args.Debug), + categoryFpath, + ) // fmt.Println("- Writing metadata and entries to file") // pbar := bin.NewProgressBar(len(metadata)) diff --git a/src/pokedex/pokedex.go b/src/pokedex/pokedex.go index 1ddb7400..0d83fe7f 100644 --- a/src/pokedex/pokedex.go +++ b/src/pokedex/pokedex.go @@ -142,19 +142,15 @@ func CreateNameMetadata(idx int, key string, name PokemonName, rootDir string, f ) } -func CreateCategoryStruct(rootDir string, fpaths []string, debug bool) Trie { +func CreateCategoryStruct(rootDir string, metadata []PokemonMetadata, debug bool) Trie { categories := NewTrie() - for i, fpath := range fpaths { - data, err := os.ReadFile(fpath) - Check(err) - - cats := createCategories(strings.TrimPrefix(fpath, rootDir), data) - name := createName(fpath) - - categories.Insert( - cats, - NewEntry(i, name), - ) + for _, m := range metadata { + for i, category := range m.Entries { + categories.Insert( + category.Categories, + NewEntry(i, m.Name), + ) + } } return *categories } From 231d7de8feaf97ff722667873a15240e21efee4b Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Wed, 1 Feb 2023 19:07:38 +1100 Subject: [PATCH 03/23] Write each cowfile to embedded FS --- src/bin/pokedex/pokedex.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/bin/pokedex/pokedex.go b/src/bin/pokedex/pokedex.go index 02d1314f..1224310a 100644 --- a/src/bin/pokedex/pokedex.go +++ b/src/bin/pokedex/pokedex.go @@ -8,6 +8,7 @@ import ( "path" "strings" + "github.com/tmck-code/pokesay/src/bin" "github.com/tmck-code/pokesay/src/pokedex" ) @@ -123,15 +124,17 @@ func main() { categoryFpath, ) - // fmt.Println("- Writing metadata and entries to file") - // pbar := bin.NewProgressBar(len(metadata)) - // for _, m := range metadata { - // pokedex.WriteBytesToFile(m.Data, pokedex.EntryFpath(entryDirPath, m.Index), true) - // pokedex.WriteStructToFile(m.Metadata, pokedex.MetadataFpath(metadataDirPath, m.Index)) - // pbar.Add(1) - // } - // fmt.Println("- Writing total metadata to file") - // pokedex.WriteIntToFile(len(metadata), totalFpath) + fmt.Println("- Writing metadata and entries to file") + pbar := bin.NewProgressBar(len(cowfileFpaths)) + for i, fpath := range cowfileFpaths { + data, err := os.ReadFile(fpath) + pokedex.Check(err) + + pokedex.WriteBytesToFile(data, pokedex.EntryFpath(entryDirPath, i), true) + pbar.Add(1) + } + fmt.Println("- Writing total metadata to file") + pokedex.WriteIntToFile(len(pokemonMetadata), totalFpath) fmt.Println("✓ Complete! Indexed", len(cowfileFpaths), "total cowfiles") fmt.Println("✓ Wrote gzipped metadata to", metadataDirPath) From 6dbc8ba5be6acf39cf8d81b026090d63bb881c33 Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Wed, 1 Feb 2023 22:05:21 +1100 Subject: [PATCH 04/23] Minor test updates --- src/bin/pokedex/pokedex.go | 9 ++++++++- src/pokedex/pokedex.go | 5 +++-- test/data/cows/4.metadata | Bin 155 -> 364 bytes test/pokedex_test.go | 13 +++++++++---- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/bin/pokedex/pokedex.go b/src/bin/pokedex/pokedex.go index 1224310a..761d2228 100644 --- a/src/bin/pokedex/pokedex.go +++ b/src/bin/pokedex/pokedex.go @@ -107,7 +107,14 @@ func main() { i := 0 for key, name := range pokemonNames { metadata := pokedex.CreateNameMetadata(i, key, name, args.FromDir, cowfileFpaths) - fmt.Printf("%+v\n", metadata) + fmt.Printf("-- %+v\n", metadata) + for _, entry := range metadata.Entries { + fpath := pokedex.EntryFpath(entryDirPath, entry.EntryIndex) + data, err := os.ReadFile(fpath) + pokedex.Check(err) + + fmt.Printf("%s\n", pokedex.Decompress(data)) + } pokedex.WriteStructToFile(metadata, pokedex.MetadataFpath(metadataDirPath, i)) pokemonMetadata = append(pokemonMetadata, *metadata) i++ diff --git a/src/pokedex/pokedex.go b/src/pokedex/pokedex.go index 0d83fe7f..559dd0b6 100644 --- a/src/pokedex/pokedex.go +++ b/src/pokedex/pokedex.go @@ -125,16 +125,17 @@ func (m Metadata) WriteToFile(fpath string) { func CreateNameMetadata(idx int, key string, name PokemonName, rootDir string, fpaths []string) *PokemonMetadata { entryCategories := make(map[int][][]string, 0) - for _, fpath := range fpaths { + for i, fpath := range fpaths { basename := strings.TrimPrefix(fpath, rootDir) if strings.Contains(basename, strings.ToLower(name.English)) { data, err := os.ReadFile(fpath) Check(err) cats := createCategories(strings.TrimPrefix(fpath, rootDir), data) - entryCategories[idx] = append(entryCategories[idx], cats) + entryCategories[i] = append(entryCategories[i], cats) } } return NewMetadata( + name.English, name.Japanese, name.JapanesePhonetic, diff --git a/test/data/cows/4.metadata b/test/data/cows/4.metadata index 96d776daf6dfe492e5fe06298ba2f4fd04d39c2d..6ea2d790ea51e07929c076e00e22119ce716a6a6 100644 GIT binary patch literal 364 zcmZus%}T>S5T0ogTS`Dd(37Bd^{Pif1QjYhDy4_P3`sEA4cQf1Z#K0Rd<0*`x9NLy zofuOPgxx)S|NG!xJ;d#ye&SrK0b7d561o|nkNzFyj3HonO9d%5+yXl-Fif;!n?Ba6 z?v=7-$_Cv#035qz2k~GuE+jkR$?0!LEt(FfC{h*IN_2C83s=dE-G3!>zz+6*Y&S}R z&8o+(H8TPKVwuxK}SJmgL`f6U>O7w{w`RF=~ v&b8c^U0T1JYrsIOG X Date: Wed, 1 Feb 2023 22:18:10 +1100 Subject: [PATCH 05/23] minor updates --- pokesay.go | 4 +++- src/bin/pokedex/pokedex.go | 20 ++++++++++---------- src/pokesay/lookup.go | 4 +--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pokesay.go b/pokesay.go index 0ca8413b..6d0bc551 100644 --- a/pokesay.go +++ b/pokesay.go @@ -118,7 +118,9 @@ func runPrintRandom(args pokesay.Args) { _, choice := pokesay.ChooseByRandomIndex(GOBTotal) metadata := pokedex.ReadMetadataFromEmbedded(GOBCowNames, MetadataFpath(choice)) - pokesay.Print(args, choice, GenerateNames(metadata, args), strings.Split(metadata.Categories, "/"), GOBCowData) + final := metadata.Entries[pokesay.RandomInt(len(metadata.Entries))] + + pokesay.Print(args, choice, GenerateNames(metadata, args), final.Categories, GOBCowData) } func main() { diff --git a/src/bin/pokedex/pokedex.go b/src/bin/pokedex/pokedex.go index 761d2228..56ad56d5 100644 --- a/src/bin/pokedex/pokedex.go +++ b/src/bin/pokedex/pokedex.go @@ -100,9 +100,18 @@ func main() { pokemonNames := pokedex.ReadNames(args.FromMetadataFname) fmt.Println("- Read", len(pokemonNames), "pokemon names from", args.FromMetadataFname) + fmt.Println("- Writing entries to file") + pbar := bin.NewProgressBar(len(cowfileFpaths)) + for i, fpath := range cowfileFpaths { + data, err := os.ReadFile(fpath) + pokedex.Check(err) + + pokedex.WriteBytesToFile(data, pokedex.EntryFpath(entryDirPath, i), true) + pbar.Add(1) + } + // 1. For each pokemon name, write a metadata file, containing the name information, and // links to all of the matching cowfile indexes - pokemonMetadata := make([]pokedex.PokemonMetadata, 0) i := 0 for key, name := range pokemonNames { @@ -131,15 +140,6 @@ func main() { categoryFpath, ) - fmt.Println("- Writing metadata and entries to file") - pbar := bin.NewProgressBar(len(cowfileFpaths)) - for i, fpath := range cowfileFpaths { - data, err := os.ReadFile(fpath) - pokedex.Check(err) - - pokedex.WriteBytesToFile(data, pokedex.EntryFpath(entryDirPath, i), true) - pbar.Add(1) - } fmt.Println("- Writing total metadata to file") pokedex.WriteIntToFile(len(pokemonMetadata), totalFpath) diff --git a/src/pokesay/lookup.go b/src/pokesay/lookup.go index 58e468db..71f8e15f 100644 --- a/src/pokesay/lookup.go +++ b/src/pokesay/lookup.go @@ -1,8 +1,6 @@ package pokesay import ( - "errors" - "log" "math/rand" "sort" "time" @@ -16,7 +14,7 @@ var ( func RandomInt(n int) int { if n <= 0 { - log.Fatal(errors.New("RandomInt arg must be >0")) + return 0 } return rand.New(Rand).Intn(n) } From b0157889ecf3a9fb8ca8fd9b900b017cef364520 Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Thu, 2 Feb 2023 18:51:12 +1100 Subject: [PATCH 06/23] update fix the indexes that link between the various structs --- pokesay.go | 3 ++- src/bin/pokedex/pokedex.go | 11 ++--------- src/pokedex/pokedex.go | 20 +++++--------------- src/pokedex/trie.go | 12 ++++-------- src/pokesay/lookup.go | 6 +++++- 5 files changed, 18 insertions(+), 34 deletions(-) diff --git a/pokesay.go b/pokesay.go index 6d0bc551..ca1721d5 100644 --- a/pokesay.go +++ b/pokesay.go @@ -103,8 +103,9 @@ func GenerateNames(metadata pokedex.PokemonMetadata, args pokesay.Args) []string func runPrintByName(args pokesay.Args, categories pokedex.Trie) { match := pokesay.ChooseByName(args.NameToken, categories) metadata := pokedex.ReadMetadataFromEmbedded(GOBCowNames, MetadataFpath(match.Entry.Index)) + final := metadata.Entries[pokesay.RandomInt(len(metadata.Entries))] - pokesay.Print(args, match.Entry.Index, GenerateNames(metadata, args), match.Keys, GOBCowData) + pokesay.Print(args, final.EntryIndex, GenerateNames(metadata, args), final.Categories, GOBCowData) } func runPrintByCategory(args pokesay.Args, categories pokedex.Trie) { diff --git a/src/bin/pokedex/pokedex.go b/src/bin/pokedex/pokedex.go index 56ad56d5..93cf795f 100644 --- a/src/bin/pokedex/pokedex.go +++ b/src/bin/pokedex/pokedex.go @@ -116,14 +116,7 @@ func main() { i := 0 for key, name := range pokemonNames { metadata := pokedex.CreateNameMetadata(i, key, name, args.FromDir, cowfileFpaths) - fmt.Printf("-- %+v\n", metadata) - for _, entry := range metadata.Entries { - fpath := pokedex.EntryFpath(entryDirPath, entry.EntryIndex) - data, err := os.ReadFile(fpath) - pokedex.Check(err) - - fmt.Printf("%s\n", pokedex.Decompress(data)) - } + fmt.Printf("-- %d %+v\n", i, metadata) pokedex.WriteStructToFile(metadata, pokedex.MetadataFpath(metadataDirPath, i)) pokemonMetadata = append(pokemonMetadata, *metadata) i++ @@ -146,6 +139,6 @@ func main() { fmt.Println("✓ Complete! Indexed", len(cowfileFpaths), "total cowfiles") fmt.Println("✓ Wrote gzipped metadata to", metadataDirPath) fmt.Println("✓ Wrote gzipped cowfiles to", entryDirPath) - fmt.Println("✓ Wrote 'total' metadata to", totalFpath) + fmt.Println("✓ Wrote 'total' metadata to", totalFpath, len(pokemonMetadata)) fmt.Println("✓ Wrote gzipped category trie to", categoryFpath) } diff --git a/src/pokedex/pokedex.go b/src/pokedex/pokedex.go index 559dd0b6..3b82ec57 100644 --- a/src/pokedex/pokedex.go +++ b/src/pokedex/pokedex.go @@ -112,16 +112,6 @@ func Decompress(data []byte) []byte { return resB.Bytes() } -type Metadata struct { - Data []byte - Index int - Metadata PokemonMetadata -} - -func (m Metadata) WriteToFile(fpath string) { - WriteStructToFile(m, fpath) -} - func CreateNameMetadata(idx int, key string, name PokemonName, rootDir string, fpaths []string) *PokemonMetadata { entryCategories := make(map[int][][]string, 0) @@ -135,7 +125,6 @@ func CreateNameMetadata(idx int, key string, name PokemonName, rootDir string, f } } return NewMetadata( - name.English, name.Japanese, name.JapanesePhonetic, @@ -145,14 +134,15 @@ func CreateNameMetadata(idx int, key string, name PokemonName, rootDir string, f func CreateCategoryStruct(rootDir string, metadata []PokemonMetadata, debug bool) Trie { categories := NewTrie() - for _, m := range metadata { - for i, category := range m.Entries { + for i, m := range metadata { + for _, entry := range m.Entries { categories.Insert( - category.Categories, - NewEntry(i, m.Name), + entry.Categories, + NewEntry(i, strings.ToLower(m.Name)), ) } } + // fmt.Println(categories.ToString(2)) return *categories } diff --git a/src/pokedex/trie.go b/src/pokedex/trie.go index 9f37675f..ae55e447 100644 --- a/src/pokedex/trie.go +++ b/src/pokedex/trie.go @@ -59,6 +59,7 @@ func (t *Trie) ToString(indentation ...int) string { } func (t *Trie) Insert(keyPath []string, data *Entry) { + // fmt.Println("adding", keyPath, data) current := t.Root found := false for _, k := range t.KeyPaths { @@ -119,10 +120,9 @@ func (current Node) Find(value string, keys []string) []*PokemonMatch { matches := []*PokemonMatch{} for _, entry := range current.Data { - for _, tk := range tokenizeValue(entry.Value) { - if tk == value { - matches = append(matches, &PokemonMatch{Entry: entry, Keys: keys}) - } + if strings.ToLower(entry.Value) == strings.ToLower(value) { + // fmt.Printf("%s matches %s: %v - %d\n", value, entry.Value, keys, len(current.Data)) + matches = append(matches, &PokemonMatch{Entry: entry, Keys: keys}) } } for k, node := range current.Children { @@ -134,10 +134,6 @@ func (current Node) Find(value string, keys []string) []*PokemonMatch { return matches } -func tokenizeValue(value string) []string { - return strings.Split(value, "-") -} - func (t Trie) FindKeyPaths(key string) ([][]string, error) { matches := [][]string{} for _, k := range t.KeyPaths { diff --git a/src/pokesay/lookup.go b/src/pokesay/lookup.go index 71f8e15f..debfb3c8 100644 --- a/src/pokesay/lookup.go +++ b/src/pokesay/lookup.go @@ -52,7 +52,11 @@ func ListCategories(categories pokedex.Trie) []string { func ChooseByName(name string, categories pokedex.Trie) *pokedex.PokemonMatch { matches, err := categories.Find(name) Check(err) - return matches[RandomInt(len(matches))] + // fmt.Printf("name matches for %s: %v\n", name, matches) + choice := matches[RandomInt(len(matches))] + + // fmt.Printf("name choice for %s\n", choice) + return choice } func ChooseByRandomIndex(totalInBytes []byte) (int, int) { From dbe309cbbfcb24b81cc4dabe190f8a061da57557 Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Thu, 2 Feb 2023 19:27:43 +1100 Subject: [PATCH 07/23] update --- pokesay.go | 3 ++- src/bin/pokedex/pokedex.go | 3 --- src/pokedex/pokedex.go | 4 ++-- src/pokedex/trie.go | 14 ++++++++++---- src/pokesay/lookup.go | 9 ++++++--- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/pokesay.go b/pokesay.go index ca1721d5..61b6f2cd 100644 --- a/pokesay.go +++ b/pokesay.go @@ -118,10 +118,11 @@ func runPrintByCategory(args pokesay.Args, categories pokedex.Trie) { func runPrintRandom(args pokesay.Args) { _, choice := pokesay.ChooseByRandomIndex(GOBTotal) metadata := pokedex.ReadMetadataFromEmbedded(GOBCowNames, MetadataFpath(choice)) + fmt.Println(choice, metadata) final := metadata.Entries[pokesay.RandomInt(len(metadata.Entries))] - pokesay.Print(args, choice, GenerateNames(metadata, args), final.Categories, GOBCowData) + pokesay.Print(args, final.EntryIndex, GenerateNames(metadata, args), final.Categories, GOBCowData) } func main() { diff --git a/src/bin/pokedex/pokedex.go b/src/bin/pokedex/pokedex.go index 93cf795f..27deb54b 100644 --- a/src/bin/pokedex/pokedex.go +++ b/src/bin/pokedex/pokedex.go @@ -123,9 +123,6 @@ func main() { } fmt.Println("wrote", i, "name metadata files to", metadataDirPath) - // 1. Create the metadata files, containing name/category/japanese name info for each pokemon - // metadata := pokedex.CreateMetadata(args.FromDir, cowfileFpaths, pokemonNames, args.Debug) - // 2. Create the category struct using the cowfile paths, pokemon names and indexes\ fmt.Println("- Writing categories to file") pokedex.WriteStructToFile( diff --git a/src/pokedex/pokedex.go b/src/pokedex/pokedex.go index 3b82ec57..32c81c19 100644 --- a/src/pokedex/pokedex.go +++ b/src/pokedex/pokedex.go @@ -134,11 +134,11 @@ func CreateNameMetadata(idx int, key string, name PokemonName, rootDir string, f func CreateCategoryStruct(rootDir string, metadata []PokemonMetadata, debug bool) Trie { categories := NewTrie() - for i, m := range metadata { + for _, m := range metadata { for _, entry := range m.Entries { categories.Insert( entry.Categories, - NewEntry(i, strings.ToLower(m.Name)), + NewEntry(entry.EntryIndex, strings.ToLower(m.Name)), ) } } diff --git a/src/pokedex/trie.go b/src/pokedex/trie.go index ae55e447..15d6ef75 100644 --- a/src/pokedex/trie.go +++ b/src/pokedex/trie.go @@ -107,8 +107,8 @@ type PokemonMatch struct { // Search every node in a trie for a given value. // Returns a slice of all nodes that have a matching value. // Returns an error if the value wasn't found -func (t Trie) Find(value string) ([]*PokemonMatch, error) { - matches := t.Root.Find(value, []string{}) +func (t Trie) Find(value string, findFirst bool) ([]*PokemonMatch, error) { + matches := t.Root.Find(value, []string{}, findFirst) if len(matches) > 0 { return matches, nil } else { @@ -116,18 +116,24 @@ func (t Trie) Find(value string) ([]*PokemonMatch, error) { } } -func (current Node) Find(value string, keys []string) []*PokemonMatch { +func (current Node) Find(value string, keys []string, findFirst bool) []*PokemonMatch { matches := []*PokemonMatch{} for _, entry := range current.Data { if strings.ToLower(entry.Value) == strings.ToLower(value) { + if findFirst { + return []*PokemonMatch{{Entry: entry, Keys: keys}} + } // fmt.Printf("%s matches %s: %v - %d\n", value, entry.Value, keys, len(current.Data)) matches = append(matches, &PokemonMatch{Entry: entry, Keys: keys}) } } for k, node := range current.Children { - more := node.Find(value, append(keys, k)) + more := node.Find(value, append(keys, k), findFirst) if len(more) > 0 { + if findFirst { + return more + } matches = append(matches, more...) } } diff --git a/src/pokesay/lookup.go b/src/pokesay/lookup.go index debfb3c8..8a9702a9 100644 --- a/src/pokesay/lookup.go +++ b/src/pokesay/lookup.go @@ -1,6 +1,7 @@ package pokesay import ( + "fmt" "math/rand" "sort" "time" @@ -50,12 +51,14 @@ func ListCategories(categories pokedex.Trie) []string { } func ChooseByName(name string, categories pokedex.Trie) *pokedex.PokemonMatch { - matches, err := categories.Find(name) + matches, err := categories.Find(name, true) Check(err) - // fmt.Printf("name matches for %s: %v\n", name, matches) + for i, m := range matches { + fmt.Printf("name matches for %s: %v (%d)\n", name, m, i) + } choice := matches[RandomInt(len(matches))] - // fmt.Printf("name choice for %s\n", choice) + fmt.Printf("name choice for %s\n", choice) return choice } From fd4630fb35eeb9b4cb136e925c2d96608a423eed Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Thu, 2 Feb 2023 20:35:04 +1100 Subject: [PATCH 08/23] get slug from names file --- src/pokedex/data.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pokedex/data.go b/src/pokedex/data.go index 8b2431c1..a69863a6 100644 --- a/src/pokedex/data.go +++ b/src/pokedex/data.go @@ -32,6 +32,7 @@ type PokemonName struct { English string Japanese string JapanesePhonetic string + Slug string } func NewPokemonName(entry DataEntry) *PokemonName { @@ -39,6 +40,7 @@ func NewPokemonName(entry DataEntry) *PokemonName { English: entry.Name.Eng, Japanese: entry.Name.Jpn, JapanesePhonetic: entry.Slug.Jpn, + Slug: entry.Slug.Eng, } } From 6cb6f40a2ff6208cb937b70ba4c44af67b7ade04 Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Tue, 7 Feb 2023 07:58:48 +1100 Subject: [PATCH 09/23] update test --- test/trie_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/trie_test.go b/test/trie_test.go index ab99e37b..1b6e12fc 100644 --- a/test/trie_test.go +++ b/test/trie_test.go @@ -52,7 +52,7 @@ func TestTrieFind(test *testing.T) { }, } - results, err := t.Find("pikachu") + results, err := t.Find("pikachu", false) pokesay.Check(err) sort.Slice(results, func(i, j int) bool { From 06a02fe0940d83332554e06bf0c727ce3dc3f73e Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Tue, 7 Feb 2023 08:23:45 +1100 Subject: [PATCH 10/23] Make tool to easily read assets while debugging --- build/read_assets.go | 52 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 build/read_assets.go diff --git a/build/read_assets.go b/build/read_assets.go new file mode 100644 index 00000000..d23c57b2 --- /dev/null +++ b/build/read_assets.go @@ -0,0 +1,52 @@ +package main + +import ( + "embed" + "flag" + "fmt" + + "github.com/tmck-code/pokesay/src/pokedex" +) + +var ( + //go:embed assets/metadata/*metadata + GOBCowNames embed.FS + //go:embed assets/cows/*cow + GOBCowFiles embed.FS + MetadataRoot string = "assets/metadata" + CowfileRoot string = "assets/cows" +) + +type Args struct { + Index int +} + +func parseFlags() Args { + index := flag.Int("index", 80, "the metadata file index") + flag.Parse() + + return Args{ + Index: *index, + } +} +func MetadataFpath(idx int) string { + return pokedex.MetadataFpath(MetadataRoot, idx) +} + +func EntryFpath(idx int) string { + return pokedex.EntryFpath(CowfileRoot, idx) +} + +func main() { + args := parseFlags() + metadata := pokedex.ReadMetadataFromEmbedded(GOBCowNames, MetadataFpath(args.Index)) + fmt.Println(pokedex.StructToJSON(metadata, 2)) + + for _, entry := range metadata.Entries { + fmt.Println( + entry.Categories, + "\n", + string(pokedex.ReadPokemonCow(GOBCowFiles, EntryFpath(entry.EntryIndex))), + ) + } +} From c7d173b72902577e5f4d088440852dcd30b62ed4 Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Tue, 7 Feb 2023 08:34:56 +1100 Subject: [PATCH 11/23] Fix index/path bugs --- pokesay.go | 2 +- src/pokedex/pokedex.go | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/pokesay.go b/pokesay.go index 61b6f2cd..46f6804b 100644 --- a/pokesay.go +++ b/pokesay.go @@ -65,7 +65,7 @@ func parseFlags() pokesay.Args { } func EntryFpath(idx int) string { - return pokedex.EntryFpath(MetadataRoot, idx) + return pokedex.EntryFpath(CowDataRoot, idx) } func MetadataFpath(idx int) string { diff --git a/src/pokedex/pokedex.go b/src/pokedex/pokedex.go index 32c81c19..510bb59a 100644 --- a/src/pokedex/pokedex.go +++ b/src/pokedex/pokedex.go @@ -134,12 +134,11 @@ func CreateNameMetadata(idx int, key string, name PokemonName, rootDir string, f func CreateCategoryStruct(rootDir string, metadata []PokemonMetadata, debug bool) Trie { categories := NewTrie() - for _, m := range metadata { + for i, m := range metadata { for _, entry := range m.Entries { - categories.Insert( - entry.Categories, - NewEntry(entry.EntryIndex, strings.ToLower(m.Name)), - ) + trieEntry := NewEntry(i, strings.ToLower(m.Name)) + // fmt.Printf("inserting %s, %+v\n", entry.Categories, trieEntry) + categories.Insert(entry.Categories, trieEntry) } } // fmt.Println(categories.ToString(2)) From 43d07d65ef59c9534e47458a445dd0c74339229e Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Tue, 7 Feb 2023 08:43:39 +1100 Subject: [PATCH 12/23] Match metadata to path via slug - This slug is the only link between the pokemon.json data and the file paths that hold the cowfile data. - Just use this to match up the metadata entries, don't need it after that --- src/pokedex/pokedex.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pokedex/pokedex.go b/src/pokedex/pokedex.go index 510bb59a..59df73d8 100644 --- a/src/pokedex/pokedex.go +++ b/src/pokedex/pokedex.go @@ -117,7 +117,7 @@ func CreateNameMetadata(idx int, key string, name PokemonName, rootDir string, f for i, fpath := range fpaths { basename := strings.TrimPrefix(fpath, rootDir) - if strings.Contains(basename, strings.ToLower(name.English)) { + if strings.Contains(basename, strings.ToLower(name.Slug)) { data, err := os.ReadFile(fpath) Check(err) cats := createCategories(strings.TrimPrefix(fpath, rootDir), data) @@ -137,7 +137,7 @@ func CreateCategoryStruct(rootDir string, metadata []PokemonMetadata, debug bool for i, m := range metadata { for _, entry := range m.Entries { trieEntry := NewEntry(i, strings.ToLower(m.Name)) - // fmt.Printf("inserting %s, %+v\n", entry.Categories, trieEntry) + fmt.Printf("inserting %s, %+v\n", entry.Categories, trieEntry) categories.Insert(entry.Categories, trieEntry) } } From 1f15de12d733032d6c7bfbc5e6018c55e20be394 Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Tue, 7 Feb 2023 08:54:24 +1100 Subject: [PATCH 13/23] add some timers --- pokesay.go | 12 +++++++++++- src/pokedex/metadata.go | 14 ++++++++++++-- src/pokedex/pokedex.go | 4 ++-- src/pokesay/lookup.go | 9 ++++----- src/timer/timer.go | 7 ++++--- 5 files changed, 33 insertions(+), 13 deletions(-) diff --git a/pokesay.go b/pokesay.go index 46f6804b..e2d05143 100644 --- a/pokesay.go +++ b/pokesay.go @@ -8,6 +8,7 @@ import ( "github.com/tmck-code/pokesay/src/pokedex" "github.com/tmck-code/pokesay/src/pokesay" + "github.com/tmck-code/pokesay/src/timer" ) var ( @@ -101,11 +102,20 @@ func GenerateNames(metadata pokedex.PokemonMetadata, args pokesay.Args) []string } func runPrintByName(args pokesay.Args, categories pokedex.Trie) { + t := timer.NewTimer() match := pokesay.ChooseByName(args.NameToken, categories) + t.Mark("match") metadata := pokedex.ReadMetadataFromEmbedded(GOBCowNames, MetadataFpath(match.Entry.Index)) - final := metadata.Entries[pokesay.RandomInt(len(metadata.Entries))] + t.Mark("read metadata") + choice := pokesay.RandomInt(len(metadata.Entries)) + t.Mark("choice") + final := metadata.Entries[choice] + // fmt.Println(pokedex.StructToJSON(metadata), "\n", choice, "\n", final) pokesay.Print(args, final.EntryIndex, GenerateNames(metadata, args), final.Categories, GOBCowData) + t.Mark("print") + t.Stop() + t.PrintJson() } func runPrintByCategory(args pokesay.Args, categories pokedex.Trie) { diff --git a/src/pokedex/metadata.go b/src/pokedex/metadata.go index d0f5b2ac..7bb1c670 100644 --- a/src/pokedex/metadata.go +++ b/src/pokedex/metadata.go @@ -1,6 +1,10 @@ package pokedex -import "embed" +import ( + "embed" + + "github.com/tmck-code/pokesay/src/timer" +) type PokemonEntryMapping struct { EntryIndex int @@ -37,7 +41,13 @@ func ReadMetadataFromBytes(data []byte) PokemonMetadata { } func ReadMetadataFromEmbedded(embeddedData embed.FS, fpath string) PokemonMetadata { + t := timer.NewTimer() metadata, err := embeddedData.ReadFile(fpath) + t.Mark("read file") Check(err) - return ReadMetadataFromBytes(metadata) + data := ReadMetadataFromBytes(metadata) + t.Mark("read metadata") + t.Stop() + t.PrintJson() + return data } diff --git a/src/pokedex/pokedex.go b/src/pokedex/pokedex.go index 59df73d8..20277c85 100644 --- a/src/pokedex/pokedex.go +++ b/src/pokedex/pokedex.go @@ -31,8 +31,8 @@ func MetadataFpath(subdir string, idx int) string { func ReadStructFromBytes[T any](data []byte) T { var d T - err := gob.NewDecoder(bytes.NewBuffer(data)).Decode(&d) - Check(err) + gob.NewDecoder(bytes.NewBuffer(data)).Decode(&d) + // Check(err) return d } diff --git a/src/pokesay/lookup.go b/src/pokesay/lookup.go index 8a9702a9..461a9028 100644 --- a/src/pokesay/lookup.go +++ b/src/pokesay/lookup.go @@ -1,7 +1,6 @@ package pokesay import ( - "fmt" "math/rand" "sort" "time" @@ -53,12 +52,12 @@ func ListCategories(categories pokedex.Trie) []string { func ChooseByName(name string, categories pokedex.Trie) *pokedex.PokemonMatch { matches, err := categories.Find(name, true) Check(err) - for i, m := range matches { - fmt.Printf("name matches for %s: %v (%d)\n", name, m, i) - } + // for i, m := range matches { + // fmt.Printf("name matches for %s: %v (%d)\n", name, m, i) + // } choice := matches[RandomInt(len(matches))] - fmt.Printf("name choice for %s\n", choice) + // fmt.Printf("name choice for %s\n", choice) return choice } diff --git a/src/timer/timer.go b/src/timer/timer.go index 2c2d0082..40d396cf 100644 --- a/src/timer/timer.go +++ b/src/timer/timer.go @@ -1,9 +1,9 @@ package timer import ( - "fmt" "encoding/json" - "os" + "fmt" + "strings" "time" ) @@ -46,5 +46,6 @@ func (t *Timer) Stop() { } func (t *Timer) PrintJson() { - json.NewEncoder(os.Stderr).Encode(t) + json, _ := json.MarshalIndent(t, "", strings.Repeat(" ", 2)) + fmt.Println(string(json)) } From b3b4047fcc89078e153e711ce7a3be97abddb439 Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Tue, 7 Feb 2023 18:11:48 +1100 Subject: [PATCH 14/23] update timer --- src/timer/timer.go | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/src/timer/timer.go b/src/timer/timer.go index 40d396cf..7322a0e4 100644 --- a/src/timer/timer.go +++ b/src/timer/timer.go @@ -8,18 +8,20 @@ import ( ) type Timer struct { - stageNames []string - StageTimes map[string]time.Time - StageDurations map[string]time.Duration - Total int64 + stageNames []string + StageTimes map[string]time.Time + StageDurations map[string]time.Duration + StagePercentages map[string]string + Total int64 } func NewTimer() *Timer { t := &Timer{ - stageNames: make([]string, 0), - StageTimes: make(map[string]time.Time), - StageDurations: make(map[string]time.Duration), - Total: 0, + stageNames: make([]string, 0), + StageTimes: make(map[string]time.Time), + StageDurations: make(map[string]time.Duration), + StagePercentages: make(map[string]string), + Total: 0, } t.Mark("Start") return t @@ -27,7 +29,7 @@ func NewTimer() *Timer { func (t *Timer) Mark(stage string) { now := time.Now() - stage = fmt.Sprintf("%d.%s", len(t.stageNames), stage) + stage = fmt.Sprintf("%02d.%s", len(t.stageNames), stage) t.StageTimes[stage] = now t.stageNames = append(t.stageNames, stage) @@ -43,6 +45,17 @@ func (t *Timer) Stop() { } // From the last stage, subtract the first stage, to get total duration t.Total = t.StageTimes[t.stageNames[len(t.stageNames)-1]].Sub(t.StageTimes[t.stageNames[0]]).Nanoseconds() + + for i, stage := range t.stageNames { + if i == 0 { + t.StagePercentages[stage] = "0.0%" + } else { + t.StagePercentages[stage] = fmt.Sprintf( + "%.2f%%", + float64(t.StageDurations[stage].Nanoseconds())*100.0/float64(t.Total), + ) + } + } } func (t *Timer) PrintJson() { From 91d08f3a472efc241324aea8697e37179c086f96 Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Tue, 7 Feb 2023 18:13:01 +1100 Subject: [PATCH 15/23] add timer to debug utility --- build/read_assets.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/build/read_assets.go b/build/read_assets.go index d23c57b2..4816a1e4 100644 --- a/build/read_assets.go +++ b/build/read_assets.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/tmck-code/pokesay/src/pokedex" + "github.com/tmck-code/pokesay/src/timer" ) var ( @@ -39,14 +40,24 @@ func EntryFpath(idx int) string { func main() { args := parseFlags() + t := timer.NewTimer() metadata := pokedex.ReadMetadataFromEmbedded(GOBCowNames, MetadataFpath(args.Index)) + t.Mark("metadata") fmt.Println(pokedex.StructToJSON(metadata, 2)) - for _, entry := range metadata.Entries { + t.Mark("toJSON") + + for i, entry := range metadata.Entries { + data := string(pokedex.ReadPokemonCow(GOBCowFiles, EntryFpath(entry.EntryIndex))) + t.Mark(fmt.Sprintf("read-cow-%d", i)) fmt.Println( entry.Categories, "\n", - string(pokedex.ReadPokemonCow(GOBCowFiles, EntryFpath(entry.EntryIndex))), + data, ) + + t.Mark(fmt.Sprintf("print-cow-%d", i)) } + t.Stop() + t.PrintJson() } From 7ece0c81719a51d34d3126a3849aef49765d0e2e Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Tue, 7 Feb 2023 19:36:26 +1100 Subject: [PATCH 16/23] time the category trie turns out its 80% of all execution time --- build/read_assets.go | 8 +++++++- pokesay.go | 20 +++++++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/build/read_assets.go b/build/read_assets.go index 4816a1e4..705fb5e0 100644 --- a/build/read_assets.go +++ b/build/read_assets.go @@ -13,7 +13,10 @@ var ( //go:embed assets/metadata/*metadata GOBCowNames embed.FS //go:embed assets/cows/*cow - GOBCowFiles embed.FS + GOBCowFiles embed.FS + //go:embed assets/pokedex.gob + GOBCategory []byte + MetadataRoot string = "assets/metadata" CowfileRoot string = "assets/cows" ) @@ -41,6 +44,9 @@ func EntryFpath(idx int) string { func main() { args := parseFlags() t := timer.NewTimer() + pokedex.NewTrieFromBytes(GOBCategory) + t.Mark("trie") + metadata := pokedex.ReadMetadataFromEmbedded(GOBCowNames, MetadataFpath(args.Index)) t.Mark("metadata") fmt.Println(pokedex.StructToJSON(metadata, 2)) diff --git a/pokesay.go b/pokesay.go index e2d05143..024e9b04 100644 --- a/pokesay.go +++ b/pokesay.go @@ -137,16 +137,30 @@ func runPrintRandom(args pokesay.Args) { func main() { args := parseFlags() + t := timer.NewTimer() if args.ListCategories { - runListCategories(pokedex.NewTrieFromBytes(GOBCategory)) + c := pokedex.NewTrieFromBytes(GOBCategory) + t.Mark("trie") + runListCategories(c) + t.Mark("op") } else if args.ListNames { runListNames() } else if args.NameToken != "" { - runPrintByName(args, pokedex.NewTrieFromBytes(GOBCategory)) + c := pokedex.NewTrieFromBytes(GOBCategory) + t.Mark("trie") + runPrintByName(args, c) + t.Mark("op") } else if args.Category != "" { - runPrintByCategory(args, pokedex.NewTrieFromBytes(GOBCategory)) + c := pokedex.NewTrieFromBytes(GOBCategory) + t.Mark("trie") + runPrintByCategory(args, c) + t.Mark("op") } else { runPrintRandom(args) + t.Mark("op") } + + t.Stop() + t.PrintJson() } From e904fc43803f37b9fc09f6f27185e04899db6a71 Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Tue, 7 Feb 2023 18:28:44 +1100 Subject: [PATCH 17/23] Create optional "alignKeys" argument - string pad the keys so that they vertically line up if true --- src/timer/timer.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/timer/timer.go b/src/timer/timer.go index 7322a0e4..0c5c7dab 100644 --- a/src/timer/timer.go +++ b/src/timer/timer.go @@ -13,15 +13,21 @@ type Timer struct { StageDurations map[string]time.Duration StagePercentages map[string]string Total int64 + AlignKeys bool } -func NewTimer() *Timer { +func NewTimer(alignKeys ...bool) *Timer { + align := false + if len(alignKeys) == 1 { + align = alignKeys[0] + } t := &Timer{ stageNames: make([]string, 0), StageTimes: make(map[string]time.Time), StageDurations: make(map[string]time.Duration), StagePercentages: make(map[string]string), Total: 0, + AlignKeys: align, } t.Mark("Start") return t @@ -29,7 +35,11 @@ func NewTimer() *Timer { func (t *Timer) Mark(stage string) { now := time.Now() - stage = fmt.Sprintf("%02d.%s", len(t.stageNames), stage) + if t.AlignKeys { + stage = fmt.Sprintf("%02d.%-15s", len(t.stageNames), stage) + } else { + stage = fmt.Sprintf("%02d.%s", len(t.stageNames), stage) + } t.StageTimes[stage] = now t.stageNames = append(t.stageNames, stage) From 588a2f868618aeef4d1cbe07c78adbed6857141a Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Tue, 7 Feb 2023 18:29:13 +1100 Subject: [PATCH 18/23] style tweaks --- build/read_assets.go | 10 +++------- src/timer/timer.go | 4 +++- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/build/read_assets.go b/build/read_assets.go index 705fb5e0..07d62de3 100644 --- a/build/read_assets.go +++ b/build/read_assets.go @@ -43,25 +43,21 @@ func EntryFpath(idx int) string { func main() { args := parseFlags() - t := timer.NewTimer() + t := timer.NewTimer("main", true) pokedex.NewTrieFromBytes(GOBCategory) t.Mark("trie") metadata := pokedex.ReadMetadataFromEmbedded(GOBCowNames, MetadataFpath(args.Index)) t.Mark("metadata") - fmt.Println(pokedex.StructToJSON(metadata, 2)) + fmt.Println(pokedex.StructToJSON(metadata, 2)) t.Mark("toJSON") for i, entry := range metadata.Entries { data := string(pokedex.ReadPokemonCow(GOBCowFiles, EntryFpath(entry.EntryIndex))) t.Mark(fmt.Sprintf("read-cow-%d", i)) - fmt.Println( - entry.Categories, - "\n", - data, - ) + fmt.Printf("%s\n%s\n", entry.Categories, data) t.Mark(fmt.Sprintf("print-cow-%d", i)) } t.Stop() diff --git a/src/timer/timer.go b/src/timer/timer.go index 0c5c7dab..3cb20bad 100644 --- a/src/timer/timer.go +++ b/src/timer/timer.go @@ -8,6 +8,7 @@ import ( ) type Timer struct { + Name string stageNames []string StageTimes map[string]time.Time StageDurations map[string]time.Duration @@ -16,12 +17,13 @@ type Timer struct { AlignKeys bool } -func NewTimer(alignKeys ...bool) *Timer { +func NewTimer(name string, alignKeys ...bool) *Timer { align := false if len(alignKeys) == 1 { align = alignKeys[0] } t := &Timer{ + Name: name, stageNames: make([]string, 0), StageTimes: make(map[string]time.Time), StageDurations: make(map[string]time.Duration), From ec43aa954d8c50fede47d38289f2347a21b65641 Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Tue, 7 Feb 2023 18:38:19 +1100 Subject: [PATCH 19/23] Add moar timers --- build/read_assets.go | 2 +- pokesay.go | 20 ++++++++++++++++++-- src/pokedex/metadata.go | 6 ++++-- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/build/read_assets.go b/build/read_assets.go index 07d62de3..a5d274d4 100644 --- a/build/read_assets.go +++ b/build/read_assets.go @@ -43,7 +43,7 @@ func EntryFpath(idx int) string { func main() { args := parseFlags() - t := timer.NewTimer("main", true) + t := timer.NewTimer("read_assets", true) pokedex.NewTrieFromBytes(GOBCategory) t.Mark("trie") diff --git a/pokesay.go b/pokesay.go index 024e9b04..cbf115aa 100644 --- a/pokesay.go +++ b/pokesay.go @@ -102,7 +102,7 @@ func GenerateNames(metadata pokedex.PokemonMetadata, args pokesay.Args) []string } func runPrintByName(args pokesay.Args, categories pokedex.Trie) { - t := timer.NewTimer() + t := timer.NewTimer("runPrintByName", true) match := pokesay.ChooseByName(args.NameToken, categories) t.Mark("match") metadata := pokedex.ReadMetadataFromEmbedded(GOBCowNames, MetadataFpath(match.Entry.Index)) @@ -114,25 +114,41 @@ func runPrintByName(args pokesay.Args, categories pokedex.Trie) { pokesay.Print(args, final.EntryIndex, GenerateNames(metadata, args), final.Categories, GOBCowData) t.Mark("print") + t.Stop() t.PrintJson() } func runPrintByCategory(args pokesay.Args, categories pokedex.Trie) { + t := timer.NewTimer("runPrintByCategory", true) choice, keys := pokesay.ChooseByCategory(args.Category, categories) + t.Mark("choose by category") + metadata := pokedex.ReadMetadataFromEmbedded(GOBCowNames, MetadataFpath(choice.Index)) + t.Mark("read metadata") pokesay.Print(args, choice.Index, GenerateNames(metadata, args), keys, GOBCowData) + t.Mark("print") + + t.Stop() + t.PrintJson() } func runPrintRandom(args pokesay.Args) { + t := timer.NewTimer("runPrintRandom", true) _, choice := pokesay.ChooseByRandomIndex(GOBTotal) + t.Mark("choose index") metadata := pokedex.ReadMetadataFromEmbedded(GOBCowNames, MetadataFpath(choice)) - fmt.Println(choice, metadata) + t.Mark("read metadata") final := metadata.Entries[pokesay.RandomInt(len(metadata.Entries))] + t.Mark("choose entry") pokesay.Print(args, final.EntryIndex, GenerateNames(metadata, args), final.Categories, GOBCowData) + t.Mark("print") + + t.Stop() + t.PrintJson() } func main() { diff --git a/src/pokedex/metadata.go b/src/pokedex/metadata.go index 7bb1c670..59e43e4c 100644 --- a/src/pokedex/metadata.go +++ b/src/pokedex/metadata.go @@ -41,12 +41,14 @@ func ReadMetadataFromBytes(data []byte) PokemonMetadata { } func ReadMetadataFromEmbedded(embeddedData embed.FS, fpath string) PokemonMetadata { - t := timer.NewTimer() + t := timer.NewTimer("ReadMetadataFromEmbedded") metadata, err := embeddedData.ReadFile(fpath) - t.Mark("read file") Check(err) + t.Mark("read file") + data := ReadMetadataFromBytes(metadata) t.Mark("read metadata") + t.Stop() t.PrintJson() return data From 8e02adb8a66ea5b4c9faeb9b00ea36d31c8743cc Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Wed, 8 Feb 2023 08:45:47 +1100 Subject: [PATCH 20/23] update timer usage --- pokesay.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pokesay.go b/pokesay.go index cbf115aa..b3fc6f08 100644 --- a/pokesay.go +++ b/pokesay.go @@ -153,7 +153,7 @@ func runPrintRandom(args pokesay.Args) { func main() { args := parseFlags() - t := timer.NewTimer() + t := timer.NewTimer("main", true) if args.ListCategories { c := pokedex.NewTrieFromBytes(GOBCategory) From 9cb72c960bd28b7d1f0a091d65e1278daa90eda3 Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Wed, 8 Feb 2023 08:45:59 +1100 Subject: [PATCH 21/23] Add a notes file to hold benchmark results --- notes.md | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 notes.md diff --git a/notes.md b/notes.md new file mode 100644 index 00000000..ffedcb8a --- /dev/null +++ b/notes.md @@ -0,0 +1,84 @@ +{ + "Name": "ReadMetadataFromEmbedded", + "StageTimes": { + "00.Start": "2023-02-08T08:45:23.517632+11:00", + "01.read file": "2023-02-08T08:45:23.517661+11:00", + "02.read metadata": "2023-02-08T08:45:23.517771+11:00" + }, + "StageDurations": { + "00.Start": 0, + "01.read file": 29208, + "02.read metadata": 109792 + }, + "StagePercentages": { + "00.Start": "0.0%", + "01.read file": "21.01%", + "02.read metadata": "78.99%" + }, + "Total": 139000, + "AlignKeys": false +} +/----------------------------------------------------------------------------------\ +| f | +\----------------------------------------------------------------------------------/ + \ + \ + \ + \ + ▄ ▄▄▄  +   ▄ ▄▄▄▄▄ + ▀▄▄▄▄ ▄▄▀ + ▄▄▄ ▄▄ +   ▄▄   + ▄▄▄▄ ▄ ▄  ▄▄ +  ▄▄ ▄▄▄▄ ▄▄ ▄▄▄ + ▀▄▀▀▀▀▄▄ ▄▄ ▄▄▄▄▄▄ + ▀▄▄▄▄  ▄ ▄▄▄▄ ▄▄  +   ▄▄  ▄▄▄▄▄▄▀  ▄▀ + ▄▄▄▄▄▀▀▄ ▄▀ ▀ + ▀▀  ▄ ▄▄ + ▀▀▀ +> Combusken | medium/gen8/shiny +{ + "Name": "runPrintRandom", + "StageTimes": { + "00.Start ": "2023-02-08T08:45:23.517622+11:00", + "01.choose index ": "2023-02-08T08:45:23.517626+11:00", + "02.read metadata ": "2023-02-08T08:45:23.51818+11:00", + "03.choose entry ": "2023-02-08T08:45:23.518182+11:00", + "04.print ": "2023-02-08T08:45:23.518408+11:00" + }, + "StageDurations": { + "00.Start ": 0, + "01.choose index ": 4209, + "02.read metadata ": 554291, + "03.choose entry ": 2000, + "04.print ": 225959 + }, + "StagePercentages": { + "00.Start ": "0.0%", + "01.choose index ": "0.54%", + "02.read metadata ": "70.48%", + "03.choose entry ": "0.25%", + "04.print ": "28.73%" + }, + "Total": 786459, + "AlignKeys": true +} +{ + "Name": "main", + "StageTimes": { + "00.Start ": "2023-02-08T08:45:23.517617+11:00", + "01.op ": "2023-02-08T08:45:23.518469+11:00" + }, + "StageDurations": { + "00.Start ": 0, + "01.op ": 852667 + }, + "StagePercentages": { + "00.Start ": "0.0%", + "01.op ": "100.00%" + }, + "Total": 852667, + "AlignKeys": true +} From 8aaac4e4a7d6d0b5f21f0f4f0f033989986a3558 Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Wed, 8 Feb 2023 09:13:39 +1100 Subject: [PATCH 22/23] move notes --- notes.md => test/notes.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename notes.md => test/notes.md (100%) diff --git a/notes.md b/test/notes.md similarity index 100% rename from notes.md rename to test/notes.md From fc97ea3d0a372e7c9d508d6f2153fb45f02f17cd Mon Sep 17 00:00:00 2001 From: Tom McKeesick Date: Wed, 8 Feb 2023 10:13:52 +1100 Subject: [PATCH 23/23] toggle timer via DEBUG env var to run in debug mode: ```shell echo f | DEBUG=true pokesay ``` --- src/timer/timer.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/timer/timer.go b/src/timer/timer.go index 3cb20bad..91deb504 100644 --- a/src/timer/timer.go +++ b/src/timer/timer.go @@ -3,10 +3,15 @@ package timer import ( "encoding/json" "fmt" + "os" "strings" "time" ) +var ( + DEBUG bool = os.Getenv("DEBUG") != "" +) + type Timer struct { Name string stageNames []string @@ -15,6 +20,7 @@ type Timer struct { StagePercentages map[string]string Total int64 AlignKeys bool + Enabled bool } func NewTimer(name string, alignKeys ...bool) *Timer { @@ -30,12 +36,16 @@ func NewTimer(name string, alignKeys ...bool) *Timer { StagePercentages: make(map[string]string), Total: 0, AlignKeys: align, + Enabled: DEBUG, } t.Mark("Start") return t } func (t *Timer) Mark(stage string) { + if !t.Enabled { + return + } now := time.Now() if t.AlignKeys { stage = fmt.Sprintf("%02d.%-15s", len(t.stageNames), stage) @@ -48,6 +58,9 @@ func (t *Timer) Mark(stage string) { } func (t *Timer) Stop() { + if !t.Enabled { + return + } for i, stage := range t.stageNames { if i == 0 { t.StageDurations[stage] = 0 @@ -71,6 +84,9 @@ func (t *Timer) Stop() { } func (t *Timer) PrintJson() { + if !t.Enabled { + return + } json, _ := json.MarshalIndent(t, "", strings.Repeat(" ", 2)) fmt.Println(string(json)) }