Remove Pgn column, get compact_games fast enough again, avoid bug in …

…Path

go/src/server/README.md

@@ -86,6 +86,20 @@ Restarting nginx:
 sudo service nginx restart
 ```
 
+Postgres online repack
+```
+sudo apt-get install postgresql-server-dev-9.5 mawk
+sudo easy_install pgxnclient
+sudo pgxn install pg_repack
+sudo -u postgres psql -c "CREATE EXTENSION pg_repack" -d gorm
+/usr/lib/postgresql/9.5/bin/pg_repack
+```
+
+Postgres performance tuning
+```
+https://github.com/jfcoz/postgresqltuner
+```
+
 ### Setting up backup
 
 ```

go/src/server/cmd/compact_games/main.go

@@ -42,20 +42,13 @@ func addFile(tw *tar.Writer, path string) error {
 }
 
 func tarGame(game *db.TrainingGame, dir string, tw *tar.Writer) error {
-	if len(game.Path) == 0 {
-		log.Printf("Skipping empty path\n")
-		return nil
-	}
-
-	if !strings.HasSuffix(game.Path, ".gz") {
-		log.Fatal("Not reading gz file?")
-	}
+	name := fmt.Sprintf("training.%d.gz", game.ID)
+	source := "../../games/run1/" + name
 
-	path := filepath.Base(game.Path)
-	path = filepath.Join(dir, path[0:len(path)-3])
-	// log.Printf("Compressing %s to %s\n", game.Path, path)
+	path := filepath.Join(dir, name[0:len(name)-3])
+	// log.Printf("Compressing %s to %s\n", source, path)
 
-	gzFile, err := os.Open("../../" + game.Path)
+	gzFile, err := os.Open(source)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -167,7 +160,7 @@ func compactGames() bool {
 	// Query for all the active games we haven't yet compacted.
 	games := []db.TrainingGame{}
 	var numGames int64 = 10000
-	err := db.GetDB().Order("id asc").Limit(numGames).Where("compacted = false AND id >= 40000").Find(&games).Error
+	err := db.GetDB().Order("id asc nulls first").Limit(numGames).Where("compacted = false").Find(&games).Error
 	if err != nil {
 		log.Fatal(err)
 	}

go/src/server/cmd/tweaks/main.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"fmt"
+	"io/ioutil"
 	"log"
 	"server/db"
 )
@@ -93,6 +95,28 @@ func updateMatchPassed() {
 	}
 }
 
+func dumpPgns() {
+	start := 9168243
+	end := 10014931
+	for id := start; id < end; id++ {
+		fmt.Printf("\r%d",id)
+		game := db.TrainingGame{
+			ID: uint64(id),
+		}
+		err := db.GetDB().Where(&game).First(&game).Error
+		if err != nil {
+			fmt.Printf(" - skipping\n")
+			continue
+		}
+		pgn_path := fmt.Sprintf("/home/web/leela-chess/go/src/server/pgns/run1/%d.pgn", game.ID)
+		err = ioutil.WriteFile(pgn_path, []byte(game.Pgn), 0644)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+	fmt.Printf("\n")
+}
+
 func main() {
 	db.Init(true)
 	db.SetupDB()
@@ -103,6 +127,7 @@ func main() {
 	// setTestOnly()
 	// updateNetworkCounts()
 	// updateMatchPassed()
+	dumpPgns()
 
 	defer db.Close()
 }

go/src/server/db/models.go

@@ -97,7 +97,6 @@ type TrainingGame struct {
 
 	Version   uint
 	Path      string
-	Pgn       string
 	Compacted bool
 
 	EngineVersion string

go/src/server/main.go

@@ -765,7 +765,6 @@ func game(c *gin.Context) {
 	}
 
 	c.HTML(http.StatusOK, "game", gin.H{
-		"pgn": game.Pgn,
 	})
 }
 
@@ -985,8 +984,11 @@ func viewTrainingData(c *gin.Context) {
 	}
 
 	files := []gin.H{}
-	game_id := uint(30000)
-	for game_id < id {
+	game_id := int(id + 1 - 500000)
+	if game_id < 0 {
+		game_id = 0
+	}
+	for game_id < int(id) {
 		files = append([]gin.H{
 			gin.H{"url": fmt.Sprintf("https://s3.amazonaws.com/lczero/training/games%d.tar.gz", game_id)},
 		}, files...)