Merge branch 'xfr-middleware' into xfr

Cargo.toml

@@ -127,6 +127,10 @@ required-features = ["zonefile", "unstable-zonetree"]
 name = "serve-zone"
 required-features = ["zonefile", "net", "unstable-server-transport", "unstable-zonetree"]
 
+[[example]]
+name = "ixfr-client"
+required-features = ["zonefile", "net", "unstable-client-transport", "unstable-zonetree"]
+
 # This example is commented out because it is difficult, if not impossible,
 # when including the sqlx dependency, to make the dependency tree compatible
 # with both `cargo +nightly update -Z minimal versions` and the crate minimum

Changelog.md

@@ -12,6 +12,8 @@ New
   ([#358])
 * Added `Rtype::is_glue` to check if the Rtype may be used as glue. ([#363])
 * Added `MessageBuilder::start_error`, like `start_answer` but infallible. ([#369])
+* Added `AnswerBuilder::push_ref`, like `push` but takes the record by
+  reference. ([#383])
 
 Bug fixes
 
@@ -27,9 +29,9 @@ Unstable features
 
 * New unstable feature `unstable-validator` that adds a DNSSEC validator.
   ([#328])
-* New unstable feature `unstable-xfr` that adds an XFR response
-  interpreter in `net::xfr` for iterating over XFR responses
-  as a sequence of high level `ZoneUpdate`s. ([#375])
+* New unstable feature `unstable-xfr` that adds `XfrResponseInterpreter` in
+  `net::xfr` for iterating over XFR responses as a sequence of high level
+  `ZoneUpdate`s. ([#375])
 * `unstable-client-transport`:
   * Fixed an issue with slow responses in the
     `multi_stream` transport by not waiting in the first iteration if an
@@ -46,16 +48,32 @@ Unstable features
     transport in `net::client:tsig`. ([#373])
 * `unstable-server-transport`
   * Breaking changes to the `Service` and middleware traits. ([#369])
-  * Added a TSIG request validating and response signing middleware in
-    `net::server::middleware::tsig`. ([#380])
-  * The cookies middleware now allows requests with invalid cookies to
-    proceed if they are authenticated or not required to authenticate. ([#336])
+  * Added `TsigMiddlewareSvc` request validating and response signing
+    middleware in `net::server::middleware::tsig`. ([#380])
+  * Added `NotifyMiddlewareSvc` in `net::server::middleware::notify` to parse
+    and acknowledge SOA NOTIFY requests, for use by secondary nameservers to
+    detect outdated zones compared to the primary. ([#382])
+  * `CookiesMiddlewareSvc` now allows requests with invalid cookies to proceed
+    if they are authenticated or not required to authenticate. ([#336])
   * Added an `enabled` flag to `CookiesMiddlewareSvc`. ([#369])
+  * Added trait `ResourceRecordBatcher` and impl `CallbackBatcher` in
+    `net::server::batcher` for pushing as many records into a response as will
+    fit according to defined limits. ([#383])
+* `unstable-zonetree`:
+  * Added `ZoneUpdate`. ([#375])
+  * Added `ZoneUpdater`, `ZoneDiffBuilder` and `ZoneDiff` and improved
+    `ZoneUpdate`. ([#376])
   * Improved zonefile parsing error messages. ([#362]). 
   * `TryFrom<inplace::Zonefile> for Zonefile` now returns the set of
     errors instead of logging and ignoring them. ([#362])
   * Allow both glue (A/AAAA) and zone cuts at the same owner when zone
     parsing. ([#363])
+  * Altered the logic in `Versioned::remove_all()` (formerly
+    `Versioned::clean()`) as it made destructive changes to the zone that
+    would have impacted readers of the current zone version while the new zone
+    version was being created. ([#376])
+  * Removed / renamed references to `clean` in `zonetree::in_memory` to
+    `remove`. ([#376])
 
 Other changes
 
@@ -77,8 +95,11 @@ Other changes
 [#373]: https://github.com/NLnetLabs/domain/pull/373
 [#374]: https://github.com/NLnetLabs/domain/pull/374
 [#375]: https://github.com/NLnetLabs/domain/pull/375
+[#376]: https://github.com/NLnetLabs/domain/pull/376
 [#377]: https://github.com/NLnetLabs/domain/pull/377
 [#380]: https://github.com/NLnetLabs/domain/pull/380
+[#382]: https://github.com/NLnetLabs/domain/pull/382
+[#383]: https://github.com/NLnetLabs/domain/pull/383
 [@dklbreitling]: https://github.com/dklbreitling
 
 ## 0.10.1

examples/common/serve-utils.rs

@@ -2,7 +2,9 @@ use bytes::Bytes;
 use domain::base::{Message, MessageBuilder, Name, ParsedName, Rtype};
 use domain::rdata::ZoneRecordData;
 use domain::zonetree::Answer;
+use octseq::Octets;
 
+#[allow(dead_code)]
 pub fn generate_wire_query(
     qname: &Name<Bytes>,
     qtype: Rtype,
@@ -13,6 +15,7 @@ pub fn generate_wire_query(
     query.into()
 }
 
+#[allow(dead_code)]
 pub fn generate_wire_response(
     wire_query: &Message<Vec<u8>>,
     zone_answer: Answer,
@@ -22,9 +25,10 @@ pub fn generate_wire_response(
     response.into()
 }
 
-pub fn print_dig_style_response(
-    query: &Message<Vec<u8>>,
-    response: &Message<Vec<u8>>,
+#[allow(dead_code)]
+pub fn print_dig_style_response<Octs1: Octets, Octs2: Octets>(
+    query: &Message<Octs1>,
+    response: &Message<Octs2>,
     short: bool,
 ) {
     if !short {

examples/ixfr-client.rs

@@ -0,0 +1,69 @@
+/// Using the `domain::net::client` module for sending a query.
+use core::str::FromStr;
+
+use std::vec::Vec;
+
+use tokio::net::TcpStream;
+
+use domain::base::Name;
+use domain::base::Rtype;
+use domain::base::{MessageBuilder, Serial, Ttl};
+use domain::net::client::request::SendRequestMulti;
+use domain::net::client::request::{RequestMessage, RequestMessageMulti};
+use domain::net::client::stream;
+use domain::rdata::Soa;
+
+#[path = "common/serve-utils.rs"]
+mod common;
+
+#[tokio::main]
+async fn main() {
+    let args: Vec<String> = std::env::args().collect();
+
+    if args.len() != 4 {
+        eprintln!(
+            "Usage: {} <ip addr:port> <zone name> <SOA serial>",
+            args[0]
+        );
+        eprintln!("E.g.:  {} 127.0.0.1:8053 example.com 2020080302", args[0]);
+        std::process::exit(1);
+    }
+
+    let server_addr = &args[1];
+    let qname = Name::<Vec<u8>>::from_str(&args[2]).unwrap();
+    let soa_serial: u32 = args[3].parse().unwrap();
+
+    eprintln!("Requesting IXFR from {server_addr} for zone {qname} from serial {soa_serial}");
+
+    let tcp_conn = TcpStream::connect(server_addr).await.unwrap();
+    let (tcp, transport) = stream::Connection::<
+        RequestMessage<Vec<u8>>,
+        RequestMessageMulti<Vec<u8>>,
+    >::new(tcp_conn);
+    tokio::spawn(async move {
+        transport.run().await;
+        println!("single TSIG TCP run terminated");
+    });
+
+    let mname = Name::<Vec<u8>>::from_str("mname").unwrap();
+    let rname = Name::<Vec<u8>>::from_str("rname").unwrap();
+    let ttl = Ttl::from_secs(3600);
+    let soa = Soa::new(mname, rname, Serial(soa_serial), ttl, ttl, ttl, ttl);
+
+    let mut msg = MessageBuilder::new_vec();
+    msg.header_mut().set_rd(true);
+    msg.header_mut().set_ad(true);
+    let mut msg = msg.question();
+    msg.push((&qname, Rtype::IXFR)).unwrap();
+    let mut msg = msg.authority();
+    msg.push((&qname, 3600, soa)).unwrap();
+    let req = RequestMessageMulti::new(msg.clone()).unwrap();
+
+    let mut request = SendRequestMulti::send_request(&tcp, req);
+
+    // Get the reply
+    let mock_req = msg.into_message();
+    while let Some(reply) = request.get_response().await.unwrap() {
+        common::print_dig_style_response(&mock_req, &reply, false);
+    }
+}

examples/serve-zone.rs

@@ -29,6 +29,7 @@
 //!   ;; WHEN: Thu May 02 00:14:04 CEST 2024
 //!   ;; XFR size: 43347447 records (messages 16393621, bytes 2557835040)
 
+use core::future::Future;
 use core::str::FromStr;
 
 use std::any::Any;
@@ -44,20 +45,12 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use bytes::Bytes;
-use domain::zonemaintenance::maintainer::{
-    self, DefaultConnFactory, TypedZone, ZoneLookup, ZoneMaintainer,
-};
-use domain::zonemaintenance::types::{
-    CompatibilityMode, NotifyConfig, TransportStrategy, XfrConfig,
-    XfrStrategy, ZoneConfig, ZoneMaintainerKeyStore,
-};
 use octseq::Parser;
 use tokio::net::{TcpListener, UdpSocket};
 use tokio::sync::mpsc;
 use tracing::{debug, info};
 use tracing_subscriber::EnvFilter;
 
-use core::future::Future;
 use domain::base::iana::{Class, Rcode};
 use domain::base::record::ComposeRecord;
 use domain::base::{Name, ParsedName, ParsedRecord, Record, ToName};
@@ -78,9 +71,16 @@ use domain::rdata::ZoneRecordData;
 use domain::tsig::{Algorithm, Key, KeyName};
 use domain::utils::base64;
 use domain::zonefile::inplace;
+use domain::zonemaintenance::maintainer::{
+    self, DefaultConnFactory, TypedZone, ZoneLookup, ZoneMaintainer,
+};
+use domain::zonemaintenance::types::{
+    CompatibilityMode, NotifyConfig, TransportStrategy, XfrConfig,
+    XfrStrategy, ZoneConfig, ZoneMaintainerKeyStore,
+};
 use domain::zonetree::{
-    Answer, ReadableZone, Rrset, StoredName, WritableZone, WritableZoneNode,
-    Zone, ZoneBuilder, ZoneDiff, ZoneStore,
+    Answer, InMemoryZoneDiff, ReadableZone, Rrset, StoredName, WritableZone,
+    WritableZoneNode, Zone, ZoneBuilder, ZoneStore,
 };
 
 #[tokio::main()]
@@ -115,7 +115,7 @@ async fn main() {
 
     // Create a service to answer queries for the zone.
     let svc = service_fn(my_service, zones.clone());
-    let svc = XfrMiddlewareSvc::<_, _, _, Option<Key>>::new(
+    let svc = XfrMiddlewareSvc::<_, _, Option<Key>, _>::new(
         svc,
         zones.clone(),
         max_concurrency,
@@ -345,8 +345,9 @@ impl WritableZone for WritableArchiveZone {
         bump_soa_serial: bool,
     ) -> Pin<
         Box<
-            dyn Future<Output = Result<Option<ZoneDiff>, std::io::Error>>
-                + Send
+            dyn Future<
+                    Output = Result<Option<InMemoryZoneDiff>, std::io::Error>,
+                > + Send
                 + Sync,
         >,
     > {

src/net/server/batcher.rs

@@ -142,9 +142,9 @@ where
     /// Do something with the completed message.
     #[allow(clippy::result_unit_err)]
     fn batch_ready(
-        _state: &T,
-        _answer: AnswerBuilder<StreamTarget<Target>>,
-        _finished: bool,
+        state: &T,
+        answer: AnswerBuilder<StreamTarget<Target>>,
+        finished: bool,
     ) -> Result<(), Self::Error>;
 }
 

src/net/server/middleware/mod.rs

@@ -12,11 +12,30 @@
 //! post-processing the resulting responses and propagating them back down
 //! through the layers to the server.
 //!
+//! # Middleware layering strategies
+//!
+//! The simplest strategy for using middleware is to use a single layered
+//! stack of middleware for all incoming requests.
+//!
+//! If however some middleware layers impose a disproportionately high cost on
+//! request processing for request types that occur rarely, an alternate
+//! strategy could be to add a middleware layer that routes requests to the
+//! appropriate middleware "chain" based on some property or properties of the
+//! request. Rather than a liner processing "chain" one would then have a tree
+//! like processing path.
+//!
+//! Another option that may be suitable in some cases could be to use separate
+//! server instances listening on separate ports or interfaces, each with
+//! their own differing middleware "chains".
+//!
+//! # Middleware-to-middleware communication
+//!
 //! If needed middleware services can pass service specific data to upstream
 //! services for consumption, via the  `RequestMeta` custom data support of
 //! the [`Service`] trait. An example of this can be seen in the
 //! [`TsigMiddlewareSvc`][tsig::TsigMiddlewareSvc].
 //!
+//!
 //! Currently the following middleware are available:
 //!
 //! [`Service`]: crate::net::server::service::Service

src/net/server/middleware/notify.rs

@@ -248,6 +248,10 @@ where
     fn get_relevant_question(
         msg: &Message<RequestOctets>,
     ) -> Option<Question<ParsedName<RequestOctets::Range<'_>>>> {
+        // NOTE: If this middleware is used with a server that primarily
+        // receives Opcode::QUERY it would be more efficient to place a
+        // "router" middleware in front of this middleware that routes
+        // requests by Opcode to separate dedicated middleware "chains".
         if Opcode::NOTIFY == msg.header().opcode() {
             if let Some(q) = msg.first_question() {
                 if q.qtype() == Rtype::SOA {

src/net/server/middleware/tsig.rs

@@ -161,13 +161,15 @@ where
             }
 
             Err(err) => {
+                // Message is incorrectly signed or signed with an unknown key.
                 warn!(
                     "{} from {} refused: {err}",
                     req.message().header().opcode(),
                     req.client_addr(),
                 );
 
                 let builder = mk_builder_for_target();
+
                 let res = match err.build_message(req.message(), builder) {
                     Ok(additional) => Ok(ControlFlow::Break(additional)),
                     Err(err) => {