Merge pull request #1070 from matsduf/merge-develop-into-master

Merge develop into master (Backend)

.travis.yml

@@ -46,7 +46,7 @@ addons:
       - liblocale-msgfmt-perl
       - libmail-rfc822-address-perl
       - libmodule-find-perl
-      - libnet-ip-perl
+      - libnet-ip-xs-perl
       - libpod-coverage-perl
       - libreadonly-perl
       - libreadonly-xs-perl

Changes

@@ -1,6 +1,43 @@
 Release history for Zonemaster component Zonemaster-Backend
 
 
+v10.0.0 2022-12-19 (public release version)
+
+ [Deprecation]
+ - The use of language tags that includes country code is
+   deprecated. See "docs/API.md#language-tag". Such
+   language codes will be made illegal in the v2023.1 release.
+   Use language codes without country code instead.
+ - The use of an empty string in the "LANGUAGE.locale"
+   setting is deprecated and will be made illegal in the
+   v2023.1 release. See "docs/Configuration.md#language-section".
+ - The use of two or more locale tags with the same language code
+   in the configuration is deprecated and will be made illegal in the
+   v2023.1 release. See "docs/Configuration.md#language-section".
+
+ [Breaking changes]
+ - Removes database primary key from API method "get_test_results"
+   (#946, also see #949).
+
+ [Features]
+ - Provides human readable messages when the test agent dies (#1058)
+ - Adds testcase descriptions in test results (#1055)
+ - Adds Zonemaster-LDNS version to API method "version_info" (#1050)
+
+ [Fixes]
+ - Updates installation instructions (#1069)
+ - Updates deprecation information in "docs/API.pm" and
+   "docs/Configuration.md" (#1067)
+ - Corrects the license statement in Backend.pm (#1059)
+ - Fixes get_data_from_parent_zone method (#1057)
+ - Updates to use Net::IP::XS directly (#1053)
+ - Updates documents to use lowercase fragments when referring
+   to internal headings (#1052)
+ - Sets IPV6_DISABLED message to INFO in testing profile (#1051)
+ - Removes enabling IPv4 and IPv6 by default in zmtest
+   (#1041, #1040, #1066)
+
+
 v9.0.1 2022-07-08 (public fix release)
 
  [Fixes]
@@ -9,7 +46,7 @@ v9.0.1 2022-07-08 (public fix release)
  - Updates the Danish translation (#1034)
 
 
-v9.0.0 2022-06-09 (public release release)
+v9.0.0 2022-06-09 (public release version)
 
  [Deprecation]
  - The use of language tags that includes country code is

Makefile.PL

@@ -26,15 +26,16 @@ requires
   'LWP::UserAgent'              => 0,
   'Mojolicious'                 => 7.28,
   'Moose'                       => 2.04,
+  'Net::IP::XS'                 => 0,
   'Parallel::ForkManager'       => 1.12,
   'Plack::Builder'              => 0,
   'Role::Tiny'                  => 1.001003,
   'Router::Simple::Declare'     => 0,
   'Starman'                     => 0,
   'String::ShellQuote'          => 0,
   'Try::Tiny'                   => 0.12,
-  'Zonemaster::Engine'          => 4.005001,
-  'Zonemaster::LDNS'            => 2.002002,
+  'Zonemaster::Engine'          => 4.006000,
+  'Zonemaster::LDNS'            => 3.000000,
   'Plack::Middleware::ReverseProxy' => 0,
   'Locale::TextDomain'          => 1.20,
   ;

README.md

@@ -59,8 +59,8 @@ The Zonemaster Backend documentation is split up into several documents:
 
 ## License
 
-The software is released under the 2-clause BSD license. See separate
-[LICENSE](LICENSE) file.
+This is free software under a 2-clause BSD license. The full text of the license can
+be found in the [LICENSE](LICENSE) file included in this respository.
 
 
 [Configuration documentation]: docs/Configuration.md

docs/API.md

@@ -212,7 +212,8 @@ Basic data type: string
 
 A string matching one of the following regular expression:
 * `/^[a-z]{2}$/`, preferred format.
-* `/^[a-z]{2}_[A-Z]{2}$/`, **deprecated** format, use the preferred format instead.
+* `/^[a-z]{2}_[A-Z]{2}$/`, **deprecated** format, use the preferred format instead
+  (planned removal: v2023.1).
 
 The set of valid *language tags* is further constrained by the
 [`LANGUAGE.locale`][LANGUAGE.locale] property.
@@ -225,7 +226,8 @@ The set of valid *language tags* is further constrained by the
 E.g. if [`LANGUAGE.locale`][LANGUAGE.locale] is "en_US en_UK sv_SE", all the valid *language tags*
 are "en_US", "en_UK", "sv_SE" and "sv".
 
-The use of `language tags` that include the country code is *deprecated*.
+The use of `language tags` that include the country code is *deprecated* (planned
+removal: v2023.1).
 
 #### Design
 
@@ -381,11 +383,12 @@ Each *test* has a unique *test id*.
 
 Basic data type: object
 
-The object has three keys, `"module"`, `"message"` and `"level"`.
+The object has four keys, `"module"`, `"message"`, `"level"` and `"testcase"`.
 
 * `"module"`: a string. The *test module* that produced the result.
 * `"message"`: a string. A human-readable *message* describing that particular result.
 * `"level"`: a [*severity level*][Severity level]. The severity of the message.
+* `"testcase"`: a string. The *[Test Case Identifier][Test Case Identifiers]* of the *[Test Case][Test Cases]* that produced the result.
 
 Sometimes additional keys are present.
 
@@ -434,7 +437,7 @@ The items of the array are objects with two keys, `"path"` and `"message"`:
 
 ## API method: `version_info`
 
-Returns the version of the Zonemaster Backend and Zonemaster Engine software combination
+Returns the version of the Zonemaster-LDNS, Zonemaster-Engine and Zonemaster-Backend software combination.
 
 Example request:
 ```json
@@ -451,6 +454,7 @@ Example response:
   "jsonrpc": "2.0",
   "id": 1,
   "result": {
+    "zonemaster_ldns": "1.0.1",
     "zonemaster_backend": "1.0.7",
     "zonemaster_engine": "v1.0.14"
   }
@@ -513,7 +517,8 @@ Returns the set of valid [*language tags*][Language tag].
 > that would give the same [short language tag][Language tag] then the short tag
 > is excluded from the set of valid [*language tags*][Language tag].
 >
-> Note: Language tags that include country code are *deprecated*.
+> Note: Language tags that include country code are *deprecated* (planned
+> removal: v2023.1).
 
 Example request:
 ```json
@@ -932,7 +937,6 @@ Example response:
   "result": {
     "creation_time": "2016-11-15 11:53:13.965982",
     "created_at": "2016-11-15T11:53:13Z",
-    "id": 25,
     "hash_id": "c45a3f8256c4a155",
     "params": {
       "ds_info": [],
@@ -953,6 +957,11 @@ Example response:
       "ipv4": true,
       "client_id": "Zonemaster Dancer Frontend"
     },
+    "testcase_descriptions": {
+      "ZONE08": "MX is not an alias",
+      "SYNTAX05": "Misuse of '@' character in the SOA RNAME field",
+      ...
+    },
     "results": [
       {
         "module": "SYSTEM",
@@ -991,11 +1000,12 @@ An object with the following properties:
   The time in UTC at which the *test* was created.
 * `"created_at"`: A [*timestamp*][Timestamp]. The time in UTC at which the *test*
   was created.
-* `"id"`: **Deprecated** (planned removal: v2022.2). An integer.
 * `"hash_id"`: A [*test id*][Test id]. The *test* in question.
 * `"params"`: See below.
   `start_domain_test` when the *test* was started.
 * `"results"`: A list of [*test result*][Test result] objects.
+* `"testcase_descriptions"`: A map with the *[Test Case Identifiers]* as keys and the
+  translated *Test Case Description* of the corresponding *[Test Cases]* as values.
 
 If the test was created by [`start_domain_test`][start_domain_test] then `"params"`
 is a normalized version `"params"` object sent to [`start_domain_test`][start_domain_test]
@@ -1016,9 +1026,9 @@ is a normalized version of an object created from the following parts:
 
 #### `"error"`
 
->
-> TODO: List all possible error codes and describe what they mean enough for clients to know how react to them.
->
+  >
+  > TODO: List all possible error codes and describe what they mean enough for clients to know how react to them.
+  >
 
 
 ## API method: `get_test_history`
@@ -1559,6 +1569,8 @@ The `"params"` object sent to [`start_domain_test`][start_domain_test] or
 [Severity Level Definitions]:         https://github.com/zonemaster/zonemaster/blob/master/docs/specifications/tests/SeverityLevelDefinitions.md
 [Severity level]:                     #severity-level
 [start_domain_test]:                  #api-method-start_domain_test
+[Test Cases]:                         https://github.com/zonemaster/zonemaster/tree/master/docs/specifications/tests#list-of-defined-test-cases
+[Test Case Identifiers]:              https://github.com/zonemaster/zonemaster/blob/master/docs/internal-documentation/templates/specifications/tests/TestCaseIdentifierSpecification.md
 [Test id]:                            #test-id
 [Test result]:                        #test-result
 [Timestamp]:                          #timestamp

docs/Configuration.md

@@ -2,38 +2,38 @@
 
 ## Table of contents
 
-* [Introduction](#Introduction)
+* [Introduction](#introduction)
 * [RPCAPI section](#rpcapi-section)
   * [enable_add_batch_job](#enable_add_batch_job)
   * [enable_add_api_user](#enable_add_api_user)
-* [DB section](#DB-section)
+* [DB section](#db-section)
   * [engine](#engine)
   * [user](#user)
   * [password](#password)
   * [database_host](#database_host)
   * [database_name](#database_name)
   * [polling_interval](#polling_interval)
-* [MYSQL section](#MYSQL-section)
+* [MYSQL section](#mysql-section)
   * [host](#host)
   * [port](#port)
   * [user](#user)
   * [password](#password)
   * [database](#database)
-* [POSTGRESQL section](#POSTGRESQL-section)
+* [POSTGRESQL section](#postgresql-section)
   * [host](#host-1)
   * [port](#port-1)
   * [user](#user-1)
   * [password](#password-1)
   * [database](#database-1)
-* [SQLITE section](#SQLITE-section)
+* [SQLITE section](#sqlite-section)
   * [database_file](#database_file)
-* [LANGUAGE section](#LANGUAGE-section)
+* [LANGUAGE section](#language-section)
   * [locale](#locale)
-* [METRICS section](#METRICS-section)
+* [METRICS section](#metrics-section)
   * [statsd_host](#statsd_host)
   * [statsd_port](#statsd_port)
-* [PUBLIC PROFILES and PRIVATE PROFILES sections](#PUBLIC-PROFILES-and-PRIVATE-PROFILES-sections)
-* [ZONEMASTER section](#ZONEMASTER-section)
+* [PUBLIC PROFILES and PRIVATE PROFILES sections](#public-profiles-and-private-profiles-sections)
+* [ZONEMASTER section](#zonemaster-section)
   * [max_zonemaster_execution_time](#max_zonemaster_execution_time)
   * [number_of_processes_for_frontend_testing](#number_of_processes_for_frontend_testing)
   * [number_of_processes_for_batch_testing](#number_of_processes_for_batch_testing)
@@ -207,13 +207,16 @@ A string matching one of the following descriptions:
 * A space separated list of one or more `locale tags` where each tag matches the
   regular expression `/^[a-z]{2}_[A-Z]{2}$/`.
 * The empty string. **Deprecated**, remove the LANGUAGE.locale entry or specify
-  LANGUAGE.locale = en_US instead.
+  LANGUAGE.locale = en_US instead. (Planned removal in version v2023.1.)
 
 It is an error to repeat the same `locale tag`.
 
 If the `locale` key is empty or absent, the `locale tag` value
 "en_US" is set by default.
 
+The use of two or more `locale tags` with the same language code is
+**deprecated** and is planned to be illegal from version v2023.1.
+
 #### Design
 
 The two first characters of a `locale tag` are intended to be an

docs/Installation.md

@@ -80,17 +80,13 @@ for Zonemaster::Backend, see the [declaration of prerequisites].
 Install dependencies available from binary packages:
 
 ```sh
-sudo dnf -y install jq perl-Class-Method-Modifiers perl-Config-IniFiles perl-DBD-SQLite perl-DBI perl-HTML-Parser perl-JSON-RPC perl-libwww-perl perl-Log-Dispatch perl-Mojolicious perl-Net-Server perl-Parallel-ForkManager perl-Plack perl-Plack-Test perl-Role-Tiny perl-Router-Simple perl-String-ShellQuote perl-Test-Differences perl-Test-Exception perl-Test-Mojo perl-Test-NoWarnings perl-Test-Warn perl-Try-Tiny perl-libintl redhat-lsb-core
+sudo dnf -y install jq perl-Class-Method-Modifiers perl-Config-IniFiles perl-DBD-SQLite perl-DBI perl-HTML-Parser perl-JSON-RPC perl-libwww-perl perl-Log-Dispatch perl-Mojolicious perl-Parallel-ForkManager perl-Plack perl-Plack-Middleware-ReverseProxy perl-Plack-Test perl-Role-Tiny perl-Router-Simple perl-String-ShellQuote perl-Test-Differences perl-Test-Exception perl-Test-Mojo perl-Test-NoWarnings perl-Try-Tiny perl-libintl redhat-lsb-core
 ```
 
-> **Note:** perl-Net-Server and perl-Test-Warn are listed here even though they
-> are not direct dependencies. They are transitive dependencies with build
-> problems when installed using cpanm.
-
 Install dependencies not available from binary packages:
 
 ```sh
-sudo cpanm Daemon::Control JSON::Validator Log::Any Log::Any::Adapter::Dispatch Starman Plack::Middleware::ReverseProxy
+sudo cpanm Daemon::Control JSON::Validator Log::Any Log::Any::Adapter::Dispatch Starman
 ```
 
 Install Zonemaster::Backend:

lib/Zonemaster/Backend.pm

@@ -1,6 +1,6 @@
 package Zonemaster::Backend;
 
-our $VERSION = '9.0.1';
+our $VERSION = '10.0.0';
 
 use strict;
 use warnings;
@@ -16,12 +16,8 @@ Michal Toma <[email protected]>
 
 =head1 LICENSE
 
-This is free software, licensed under:
-
-The (three-clause) BSD License
-
-The full text of the license can be found in the
-F<LICENSE> file included with this distribution.
+This is free software under a 2-clause BSD license. The full text of the license can
+be found in the F<LICENSE> file included with this distribution.
 
 =cut
 

lib/Zonemaster/Backend/DB.pm

@@ -269,7 +269,6 @@ sub select_test_results {
     my ( $hrefs ) = $self->dbh->selectall_hashref(
         q[
             SELECT
-                id,
                 hash_id,
                 created_at AS creation_time,
                 params,

lib/Zonemaster/Backend/RPCAPI.pm

@@ -107,9 +107,9 @@ sub version_info {
 
     my %ver;
     eval {
+        $ver{zonemaster_ldns} = Zonemaster::LDNS->VERSION;
         $ver{zonemaster_engine} = Zonemaster::Engine->VERSION;
         $ver{zonemaster_backend} = Zonemaster::Backend->VERSION;
-
     };
     if ($@) {
         handle_exception( $@ );
@@ -345,6 +345,7 @@ sub get_test_results {
         eval { $translator->data } if $translator; # Provoke lazy loading of translation data
 
         my @zm_results;
+        my %testcases;
 
         my $test_info = $self->{db}->test_results( $params->{id} );
         foreach my $test_res ( @{ $test_info->{results} } ) {
@@ -364,6 +365,8 @@ sub get_test_results {
             $res->{message} =~ s/,/, /isg;
             $res->{message} =~ s/;/; /isg;
             $res->{level} = $test_res->{level};
+            $res->{testcase} = $test_res->{testcase} // 'UNSPECIFIED';
+            $testcases{$res->{testcase}} = $translator->test_case_description($test_res->{testcase});
 
             if ( $test_res->{module} eq 'SYSTEM' ) {
                 if ( $res->{message} =~ /policy\.json/ ) {
@@ -394,6 +397,7 @@ sub get_test_results {
         }
 
         $result = $test_info;
+        $result->{testcase_descriptions} = \%testcases;
         $result->{results} = \@zm_results;
 
         $translator->locale( $previous_locale );

lib/Zonemaster/Backend/Translator.pm

@@ -6,7 +6,9 @@ use 5.14.2;
 
 use Moose;
 use Encode;
+use Readonly;
 use POSIX qw[setlocale LC_MESSAGES LC_CTYPE];
+use Locale::TextDomain qw[Zonemaster-Backend];
 use Zonemaster::Backend::Config;
 
 # Zonemaster Modules
@@ -15,6 +17,19 @@ require Zonemaster::Engine::Logger::Entry;
 
 extends 'Zonemaster::Engine::Translator';
 
+Readonly my %TAG_DESCRIPTIONS => (
+    UNABLE_TO_FINISH_TEST => sub {
+        __x    # BACKEND_TEST_AGENT:UNABLE_TO_FINISH_TEST
+          "Zonemaster was unable to start or finish the test.", @_;
+    },
+);
+
+sub _build_all_tag_descriptions {
+    my $all_tag_descriptions = Zonemaster::Engine::Translator::_build_all_tag_descriptions();
+    $all_tag_descriptions->{BACKEND_TEST_AGENT} = \%TAG_DESCRIPTIONS;
+    return $all_tag_descriptions;
+}
+
 sub translate_tag {
     my ( $self, $hashref ) = @_;
 
@@ -24,4 +39,7 @@ sub translate_tag {
     return decode_utf8( $octets );
 }
 
+sub test_case_description {
+    return decode_utf8(Zonemaster::Engine::Translator::test_case_description(@_));
+}
 1;