Be more careful about the maxzoom in the mbtiles metadata table (#102)

* Stricter maxzoom standards for mbtiles uploads

* Update eslint and all the things it now complains about

* Revert "Update eslint and all the things it now complains about"

This reverts commit 17bccfa.

* Use outdated style in the interest of a small diff

* Incorporate review feedback:

* Use exceptions to detect invalid number formats
* Use switch/case to distinguish queries
* Try to clarify distinction between tiles maxzoom and metadata maxzoom

* Turn an unused, supposedly correct, mbtiles fixture into a failing test

* Fix test I just broke

* Pare this down to the bare minimum and fix broken tests

* Fix broken unit test

* 4.7.0-dev1

* Update changelog and version number

Co-authored-by: mapsam <[email protected]>

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 4.7.0
+
+* Require mbtiles metadata maxzoom to match the tiles table maxzoom
+
 ## 4.6.0
 
 * Added validation for a maximum possible zoom level in mbtiles

lib/validators/mbtiles.js

@@ -77,33 +77,88 @@ function deprecatedCarmen(db, callback) {
 
 function zoomCheck(db, limits, callback) {
   function getMaxZoom(type, cb) {
-    var query = type === 'tiles' ?
-      'SELECT MAX(zoom_level) as z FROM tiles' :
-      'SELECT MAX(zoom_level) as z FROM grids' ;
+    var query = null;
+    switch (type) {
+      case 'tiles_max':
+        query = 'SELECT MAX(zoom_level) as z FROM tiles';
+        break;
+
+      case 'grids_max':
+        query = 'SELECT MAX(zoom_level) as z FROM grids';
+        break;
+
+      case 'tiles_min':
+        query = 'SELECT MIN(zoom_level) as z FROM tiles';
+        break;
+
+      case 'maxzoom':
+        query = 'SELECT (value) as z FROM metadata WHERE name=\'maxzoom\'';
+        break;
+
+      case 'fillzoom':
+        query = 'SELECT (value) as z FROM metadata WHERE name=\'fillzoom\'';
+        break;
+    }
+
     db.get(query, function(err, result) {
       result = result || {};
       err = sqliteError(err);
       if (err) return cb(err);
 
       log.debug('zoomCheck::getMaxZoom(): ' + type + ' finished: ' + JSON.stringify(result));
-      cb(null, {
-        z: result.z || 0
-      });
+      cb(null, result);
     });
   }
 
   log.debug('zoomCheck(): checking...');
   queue()
-    .defer(getMaxZoom, 'tiles')
-    .defer(getMaxZoom, 'grids')
+    .defer(getMaxZoom, 'tiles_max')
+    .defer(getMaxZoom, 'grids_max')
+    .defer(getMaxZoom, 'tiles_min')
+    .defer(getMaxZoom, 'maxzoom')
+    .defer(getMaxZoom, 'fillzoom')
     .awaitAll(function(err, zoom) {
       err = sqliteError(err);
       if (err) return callback(err);
 
-      if (zoom[0].z > limits.max_zoomlevel)
-        return callback(invalid('Maxzoom exceeded for mbtiles file. There is a max zoom limit of ' + limits.max_zoomlevel + ' but tiles up to zoom level ' + zoom[0].z + ' were found. If you need support for 1cm tilesets please reach out to support.'));
-      if (zoom[1].z > limits.max_zoomlevel)
-        return callback(invalid('Maxzoom exceeded for mbtiles file. There is a max zoom limit of ' + limits.max_zoomlevel + ' but tiles up to zoom level ' + zoom[1].z + ' were found. If you need support for 1cm tilesets please reach out to support.'));
+      var tiles_max = zoom[0];
+      var grids_max = zoom[1];
+      var tiles_min = zoom[2];
+      var maxzoom = zoom[3];
+      var fillzoom = zoom[4];
+
+      if (tiles_max.z > limits.max_zoomlevel)
+        return callback(invalid('Maxzoom exceeded for mbtiles file. There is a max zoom limit of ' + limits.max_zoomlevel + ' but tiles up to zoom level ' + tiles_max.z + ' were found. If you need support for 1cm tilesets please reach out to support.'));
+      if (grids_max.z > limits.max_zoomlevel)
+        return callback(invalid('Maxzoom exceeded for mbtiles file. There is a max zoom limit of ' + limits.max_zoomlevel + ' but tiles up to zoom level ' + grids_max.z + ' were found. If you need support for 1cm tilesets please reach out to support.'));
+
+      if ('z' in fillzoom) {
+        var fz = Number.parseInt(fillzoom.z, 10);
+        if (fz < tiles_min.z) {
+          return callback(invalid('In mbtiles file, metadata.fillzoom (' + fz + ') is lower than actual tiles minzoom (' + tiles_min.z + ')'));
+        }
+        if (fz > tiles_max.z) {
+          return callback(invalid('In mbtiles file, metadata.fillzoom (' + fz + ') is higher than actual tiles maxzoom (' + tiles_max.z + ')'));
+        }
+      }
+
+      if ('z' in maxzoom) {
+        var mz = Number.parseInt(maxzoom.z, 10);
+        if (mz < tiles_min.z) {
+          return callback(invalid('In mbtiles file, metadata.maxzoom (' + mz + ') is lower than actual tiles minzoom (' + tiles_min.z + ')'));
+        }
+        if (mz < tiles_max.z) {
+          return callback(invalid('In mbtiles file, metadata.maxzoom (' + mz + ') is lower than actual tiles maxzoom (' + tiles_max.z + ')'));
+        }
+        if (mz > tiles_max.z) {
+          if ('z' in fillzoom) {
+            // Out-of range maxzoom is OK and expected in some raster uploads,
+            // if fillzoom also exists and was therefore validated to be in range above.
+          } else {
+            return callback(invalid('In mbtiles file, metadata.maxzoom (' + mz + ') is higher than actual tiles maxzoom (' + tiles_max.z + ')'));
+          }
+        }
+      }
 
       log.debug('zoomCheck(): OK.');
       callback();

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mapbox/mapbox-upload-validate",
-  "version": "4.6.0",
+  "version": "4.7.0",
   "description": "Validate that a file can be uploaded to Mapbox",
   "main": "index.js",
   "scripts": {

test/expected/index.js

@@ -12,12 +12,12 @@ module.exports = {
     'serialtiles_png': require('./valid.serialtiles_png.info.json'),
     'serialtiles_pbf': require('./valid.serialtiles_pbf.info.json'),
     'mbtiles-carmen2': require('./valid.mbtiles.carmen2.info.json'),
+    'mbtiles-metadata_maxzoom_beyond_tiles_but_fillzoom': require('./valid.mbtiles-metadata_maxzoom_beyond_tiles_but_fillzoom.info.json'),
     'mbtiles-onlygrids': require('./valid.mbtiles.onlygrids.info.json'),
     'mbtiles-onlytiles': require('./valid.mbtiles.onlytiles.info.json'),
     'mbtiles-tilesgrids': require('./valid.mbtiles.tilesgrid.info.json'),
     'mbtiles-update1': require('./valid.mbtiles.update1.info.json'),
     'mbtiles-update2': require('./valid.mbtiles.update2.info.json'),
-    'mbtiles-vector': require('./valid.mbtiles.vector.info.json'),
     'mbtiles-vectorgzip': require('./valid.mbtiles.vectorgzip.info.json'),
     'mbtiles-webp': require('./valid.mbtiles.webp.info.json'),
     'mbtiles-tilestats': require('./valid.mbtiles.tilestats.info.json')
@@ -64,6 +64,10 @@ module.exports = {
     'kmllayers': '16 layers found. Maximum of 15 layers allowed.'
   },
   mbtilesErrors: {
+    'metadata_maxzoom_beyond_tiles': 'In mbtiles file, metadata.maxzoom (4) is higher than actual tiles maxzoom (2)',
+    'metadata_maxzoom_below_tiles': 'In mbtiles file, metadata.maxzoom (0) is lower than actual tiles minzoom (1)',
+    'metadata_fillzoom_beyond_tiles': 'In mbtiles file, metadata.fillzoom (3) is higher than actual tiles maxzoom (2)',
+    'metadata_fillzoom_below_tiles': 'In mbtiles file, metadata.fillzoom (0) is lower than actual tiles minzoom (1)',
     'oldtemplate': 'Use TileMill 0.7 or later to export MBTiles with a valid template.',
     'notadb': 'Unknown filetype',
     'notiledata': 'SQLITE_ERROR: no such column: tile_data',

test/expected/valid.mbtiles.vector.info.json → test/expected/valid.mbtiles-metadata_maxzoom_beyond_tiles_but_fillzoom.info.json

@@ -1,31 +1,23 @@
 {
   "scheme": "tms",
-  "basename": "valid-vector.mbtiles",
-  "id": "valid-vector",
+  "basename": "valid.vector-metadata-maxzoom-beyond-tiles-but-fillzoom.mbtiles",
+  "id": "valid.vector-metadata-maxzoom-beyond-tiles-but-fillzoom",
   "filesize": 325632,
-  "center": [
-    -80.947266,
-    43.58039,
-    4
-  ],
+  "center": [ -80.947266, 43.58039, 4 ],
   "description": "Unpacker test fixture",
   "maxzoom": 4,
   "minzoom": 0,
   "name": "test-fixture",
   "vector_layers": [
-    {
+     {
       "id": "data",
       "description": "",
       "minzoom": 0,
       "maxzoom": 22
     }
   ],
-  "bounds": [
-    -180,
-    -85.051129,
-    180,
-    85.051128
-  ],
+  "fillzoom": "2",
+  "bounds": [ -180, -85.051129, 180, 85.051128 ],
   "version": "1.0.0",
   "legend": null
 }

test/fixtures/index.js

@@ -15,12 +15,12 @@ module.exports = {
     'serialtiles_pbf': path.join(__dirname, 'valid.serialtiles.pbf.gz'),
     'serialtiles_png': path.join(__dirname, 'valid.serialtiles.png.gz'),
     'mbtiles-carmen2': path.join(__dirname, 'valid-carmen2.mbtiles'),
+    'mbtiles-metadata_maxzoom_beyond_tiles_but_fillzoom': path.join(__dirname, 'valid.vector-metadata-maxzoom-beyond-tiles-but-fillzoom.mbtiles'),
     'mbtiles-onlygrids': path.join(__dirname, 'valid-onlygrids.mbtiles'),
     'mbtiles-onlytiles': path.join(__dirname, 'valid-onlytiles.mbtiles'),
     'mbtiles-tilesgrids': path.join(__dirname, 'valid-tilesgrid.mbtiles'),
     'mbtiles-update1': path.join(__dirname, 'valid-update1.mbtiles'),
     'mbtiles-update2': path.join(__dirname, 'valid-update2.mbtiles'),
-    'mbtiles-vector': path.join(__dirname, 'valid-vector.mbtiles'),
     'mbtiles-vectorgzip': path.join(__dirname, 'valid-vectorgzip.mbtiles'),
     'mbtiles-webp': path.join(__dirname, 'valid-webp.mbtiles'),
     'mbtiles-tilestats': path.join(__dirname, 'valid-tilestats-metadata.mbtiles')
@@ -57,6 +57,10 @@ module.exports = {
       'kmllayers': path.join(__dirname, 'invalid.omnivore.toomanylayers.kml')
     },
     'mbtiles': {
+      'metadata_maxzoom_beyond_tiles': path.join(__dirname, 'invalid.vector-metadata-maxzoom-beyond-tiles.mbtiles'),
+      'metadata_maxzoom_below_tiles': path.join(__dirname, 'invalid.vector-metadata-maxzoom-below-tiles.mbtiles'),
+      'metadata_fillzoom_beyond_tiles': path.join(__dirname, 'invalid.vector-metadata-fillzoom-beyond-tiles.mbtiles'),
+      'metadata_fillzoom_below_tiles': path.join(__dirname, 'invalid.vector-metadata-fillzoom-below-tiles.mbtiles'),
       'oldtemplate': path.join(__dirname, 'invalid.mbtiles-template.mbtiles'),
       'notadb': path.join(__dirname, 'invalid.mbtiles-notadb.mbtiles'),
       'notiledata': path.join(__dirname, 'invalid.mbtiles-notiledata.mbtiles'),

test/validators.mbtiles.test.js

@@ -142,3 +142,38 @@ test('lib.validators.mbtiles: grid maxzoom too big', function(t) {
   });
 });
 
+test('lib.validators.mbtiles: metadata maxzoom beyond tiles', function(t) {
+  validate(fixtures.invalid.mbtiles.metadata_maxzoom_beyond_tiles, function(err) {
+    t.ok(err, 'expected error');
+    t.equal(err.code, 'EINVALID', 'expected error code');
+    t.equal(err.message, expected.mbtilesErrors.metadata_maxzoom_beyond_tiles, 'expected error message');
+    t.end();
+  });
+});
+
+test('lib.validators.mbtiles: metadata maxzoom below tiles', function(t) {
+  validate(fixtures.invalid.mbtiles.metadata_maxzoom_below_tiles, function(err) {
+    t.ok(err, 'expected error');
+    t.equal(err.code, 'EINVALID', 'expected error code');
+    t.equal(err.message, expected.mbtilesErrors.metadata_maxzoom_below_tiles, 'expected error message');
+    t.end();
+  });
+});
+
+test('lib.validators.mbtiles: metadata fillzoom beyond tiles', function(t) {
+  validate(fixtures.invalid.mbtiles.metadata_fillzoom_beyond_tiles, function(err) {
+    t.ok(err, 'expected error');
+    t.equal(err.code, 'EINVALID', 'expected error code');
+    t.equal(err.message, expected.mbtilesErrors.metadata_fillzoom_beyond_tiles, 'expected error message');
+    t.end();
+  });
+});
+
+test('lib.validators.mbtiles: metadata fillzoom below tiles', function(t) {
+  validate(fixtures.invalid.mbtiles.metadata_fillzoom_below_tiles, function(err) {
+    t.ok(err, 'expected error');
+    t.equal(err.code, 'EINVALID', 'expected error code');
+    t.equal(err.message, expected.mbtilesErrors.metadata_fillzoom_below_tiles, 'expected error message');
+    t.end();
+  });
+});