From 79afabaeb3ec70a2f3b24adabfc0bd4c67fc9b72 Mon Sep 17 00:00:00 2001 From: DeepMind Lab Team Date: Mon, 12 Feb 2018 10:49:13 +0000 Subject: [PATCH] [tests] Testing the diversty of generated mazes. Testing the distribution of random mazes for two different mixer seeds, in the process validating the amount of overlaps remains below a 1% threshold. --- deepmind/engine/BUILD | 3 +- deepmind/engine/lua_maze_generation_test.cc | 70 ++++++++++++++++++- .../levels/tests/maze_generation_test.lua | 57 +++++++++++++++ python/tests/BUILD | 9 +++ python/tests/maze_generation_test.py | 68 ++++++++++++++++++ 5 files changed, 204 insertions(+), 3 deletions(-) create mode 100644 game_scripts/levels/tests/maze_generation_test.lua create mode 100644 python/tests/maze_generation_test.py diff --git a/deepmind/engine/BUILD b/deepmind/engine/BUILD index 43bbd84d0..8aafe9837 100644 --- a/deepmind/engine/BUILD +++ b/deepmind/engine/BUILD @@ -200,7 +200,7 @@ cc_library( cc_test( name = "lua_maze_generation_test", - size = "small", + size = "large", srcs = ["lua_maze_generation_test.cc"], tags = ["manual"], # Platform specific tests. deps = [ @@ -210,6 +210,7 @@ cc_test( "//deepmind/lua:call", "//deepmind/lua:n_results_or_test_util", "//deepmind/lua:push_script", + "//deepmind/lua:table_ref", "//deepmind/lua:vm_test_util", "@com_google_googletest//:gtest_main", ], diff --git a/deepmind/engine/lua_maze_generation_test.cc b/deepmind/engine/lua_maze_generation_test.cc index e15a0d704..4cff3a9e2 100644 --- a/deepmind/engine/lua_maze_generation_test.cc +++ b/deepmind/engine/lua_maze_generation_test.cc @@ -29,6 +29,7 @@ #include "deepmind/lua/call.h" #include "deepmind/lua/n_results_or_test_util.h" #include "deepmind/lua/push_script.h" +#include "deepmind/lua/table_ref.h" #include "deepmind/lua/vm_test_util.h" namespace deepmind { @@ -45,13 +46,19 @@ class LuaMazeGenerationTest : public lua::testing::TestWithVm { vm()->AddCModuleToSearchers( "dmlab.system.maze_generation", &lua::Bind, {reinterpret_cast(static_cast(0))}); + vm()->AddCModuleToSearchers( + "dmlab.system.maze_generation1", &lua::Bind, + {reinterpret_cast(static_cast(1))}); LuaRandom::Register(L); vm()->AddCModuleToSearchers( "dmlab.system.sys_random", &lua::Bind, - {&prbg_, reinterpret_cast(static_cast(0))}); + {&prbg_[0], reinterpret_cast(static_cast(0))}); + vm()->AddCModuleToSearchers( + "dmlab.system.sys_random1", &lua::Bind, + {&prbg_[1], reinterpret_cast(static_cast(1))}); } - std::mt19937_64 prbg_; + std::mt19937_64 prbg_[2]; }; constexpr char kCreateMaze[] = R"( @@ -468,6 +475,65 @@ TEST_F(LuaMazeGenerationTest, CreateRandomMazeNoRandom) { ASSERT_FALSE(lua::Call(L, 0).ok()); } +constexpr int kRandomMazeSeedCount = 10000; +constexpr char kCreateRandomMazes[] = R"( +local seed, maps, vers = ... +local sys_random = require('dmlab.system.sys_random' .. vers) +local maze_generation = require('dmlab.system.maze_generation' .. vers) +sys_random:seed(seed) +local maze = maze_generation.randomMazeGeneration{ + random = sys_random, + width = 17, + height = 17, + extraConnectionProbability = 0.0, + hasDoors = false, + roomCount = 4, + maxRooms = 4, + roomMaxSize = 5, + roomMinSize = 3 +} +local key = maze:entityLayer() +local count = maps[key] or 0 +maps[key] = count + 1 +)"; + +TEST_F(LuaMazeGenerationTest, CreateRandomMazes) { + auto maps = lua::TableRef::Create(L); + lua::PushScript(L, kCreateRandomMazes, sizeof(kCreateRandomMazes) - 1, + "kCreateRandomMazes"); + for (int i = 0; i < kRandomMazeSeedCount; ++i) { + LOG(INFO) << "Phase 1: step " << i + 1 << " out of " + << kRandomMazeSeedCount; + lua_pushvalue(L, -1); + lua::Push(L, i); + lua::Push(L, maps); + lua::Push(L, ""); + ASSERT_THAT(lua::Call(L, 3), IsOkAndHolds(0)); + } + const auto mixer_seed_0_maps = maps.KeyCount(); + const auto mixer_seed_0_repeat_ratio = + (kRandomMazeSeedCount - mixer_seed_0_maps) / + static_cast(kRandomMazeSeedCount); + ASSERT_GE(mixer_seed_0_repeat_ratio, 0.0); + ASSERT_LE(mixer_seed_0_repeat_ratio, 0.01); + for (int i = 0; i < kRandomMazeSeedCount; ++i) { + LOG(INFO) << "Phase 2: step " << i + 1 << " out of " + << kRandomMazeSeedCount; + lua_pushvalue(L, -1); + lua::Push(L, i); + lua::Push(L, maps); + lua::Push(L, "1"); + ASSERT_THAT(lua::Call(L, 3), IsOkAndHolds(0)); + } + const auto mixer_seed_1_maps = maps.KeyCount() - mixer_seed_0_maps; + const auto mixer_seed_1_repeat_ratio = + (kRandomMazeSeedCount - mixer_seed_1_maps) / + static_cast(kRandomMazeSeedCount); + ASSERT_GE(mixer_seed_1_repeat_ratio, 0.0); + ASSERT_LE(mixer_seed_1_repeat_ratio, 0.01); + lua_pop(L, 1); +} + constexpr char kVisitRandomPath[] = R"( local sys_random = require 'dmlab.system.sys_random' local maze_generation = require 'dmlab.system.maze_generation' diff --git a/game_scripts/levels/tests/maze_generation_test.lua b/game_scripts/levels/tests/maze_generation_test.lua new file mode 100644 index 000000000..66389d552 --- /dev/null +++ b/game_scripts/levels/tests/maze_generation_test.lua @@ -0,0 +1,57 @@ +--[[ Copyright (C) 2018 Google Inc. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +]] + +local custom_observations = require 'decorators.custom_observations' +local debug_observations = require 'decorators.debug_observations' +local make_map = require 'common.make_map' +local map_maker = require 'dmlab.system.map_maker' +local maze_generation = require 'dmlab.system.maze_generation' +local random = require 'common.random' + +local randomMap = random(map_maker:randomGen()) + +local api = {} + +function api:nextMap() + return api._map +end + +function api:start(episode, seed) + local mapName = 'maze' + random:seed(seed) + randomMap:seed(seed) + api._maze = maze_generation.randomMazeGeneration{ + seed = seed, + width = 17, + height = 17, + maxRooms = 4, + roomMaxSize = 5, + roomMinSize = 3, + extraConnectionProbability = 0.0, + } + api._map = make_map.makeMap{ + mapName = mapName, + mapEntityLayer = api._maze:entityLayer(), + mapVariationsLayer = api._maze:variationsLayer(), + useSkybox = true, + } + debug_observations.setMaze(api._maze) +end + +custom_observations.decorate(api) + +return api diff --git a/python/tests/BUILD b/python/tests/BUILD index 003bffe32..6867f7247 100644 --- a/python/tests/BUILD +++ b/python/tests/BUILD @@ -139,3 +139,12 @@ py_test( data = ["//:deepmind_lab.so"], imports = ["python"], ) + +py_test( + name = "maze_generation_test", + size = "enormous", + srcs = ["maze_generation_test.py"], + tags = ["manual"], + data = ["//:deepmind_lab.so"], + imports = ["python"], +) diff --git a/python/tests/maze_generation_test.py b/python/tests/maze_generation_test.py new file mode 100644 index 000000000..39603e1e6 --- /dev/null +++ b/python/tests/maze_generation_test.py @@ -0,0 +1,68 @@ +# Copyright 2018 Google Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import unittest +import numpy as np + +import deepmind_lab + +MAZE_LAYOUT_OBSERVATION = 'DEBUG.MAZE.LAYOUT' +MAZE_LAYOUT_TRIALS = 50 + + +class MazeGenerationTest(unittest.TestCase): + + def test_maze_layout_spread(self): + layouts = set() + for i in xrange(MAZE_LAYOUT_TRIALS): + print('phase 1: trial {} out of {}'.format(i+1, MAZE_LAYOUT_TRIALS)) + env = deepmind_lab.Lab( + 'tests/maze_generation_test', [MAZE_LAYOUT_OBSERVATION], config={}) + env.reset(seed=i+1) + layouts.add(env.observations()[MAZE_LAYOUT_OBSERVATION]) + num_layouts = len(layouts) + self.assertTrue(np.isclose(num_layouts, MAZE_LAYOUT_TRIALS)) + for i in xrange(MAZE_LAYOUT_TRIALS): + print('phase 2: trial {} out of {}'.format(i+1, MAZE_LAYOUT_TRIALS)) + env = deepmind_lab.Lab( + 'tests/maze_generation_test', [MAZE_LAYOUT_OBSERVATION], + config={ + 'mixerSeed': '0', + }) + env.reset(seed=i+1) + layouts.add(env.observations()[MAZE_LAYOUT_OBSERVATION]) + self.assertEqual(len(layouts), num_layouts) + for i in xrange(MAZE_LAYOUT_TRIALS): + print('phase 3: trial {} out of {}'.format(i+1, MAZE_LAYOUT_TRIALS)) + env = deepmind_lab.Lab( + 'tests/maze_generation_test', [MAZE_LAYOUT_OBSERVATION], + config={ + 'mixerSeed': '1', + }) + env.reset(seed=i+1) + layouts.add(env.observations()[MAZE_LAYOUT_OBSERVATION]) + self.assertTrue(np.isclose(len(layouts) - num_layouts, MAZE_LAYOUT_TRIALS)) + +if __name__ == '__main__': + if 'TEST_SRCDIR' in os.environ: + deepmind_lab.set_runfiles_path( + os.path.join(os.environ['TEST_SRCDIR'], + 'org_deepmind_lab')) + unittest.main()