Release 2.25.0

- [API, FEATURE] Add es_delay_onclose option to delay on_close until all
  data is ACKed.  Use new function lsquic_stream_has_unacked_data() to
  learn whether peer acknowledged all data written to stream.
- [API] Add optional on_reset() stream callback to get notifications
  when RESET or STOP_SENDING frames are received.
- [BUGFIX] On STOP_SENDING, make conn tickable is _writeable_, not
  readable.

CHANGELOG

@@ -1,3 +1,13 @@
+2020-12-04
+    - 2.25.0
+    - [API, FEATURE] Add es_delay_onclose option to delay on_close until all
+      data is ACKed.  Use new function lsquic_stream_has_unacked_data() to
+      learn whether peer acknowledged all data written to stream.
+    - [API] Add optional on_reset() stream callback to get notifications
+      when RESET or STOP_SENDING frames are received.
+    - [BUGFIX] On STOP_SENDING, make conn tickable is _writeable_, not
+      readable.
+
 2020-11-24
     - 2.24.5
     - [FEATURE] Improve Delayed ACKs extension and turn it on by default.

bin/http_server.c

@@ -1045,7 +1045,8 @@ http_server_on_close (lsquic_stream_t *stream, lsquic_stream_ctx_t *st_h)
     if (st_h->req)
         interop_server_hset_destroy(st_h->req);
     free(st_h);
-    LSQ_INFO("%s called", __func__);
+    LSQ_INFO("%s called, has unacked data: %d", __func__,
+                                lsquic_stream_has_unacked_data(stream));
 }
 
 

bin/test_common.c

@@ -1931,6 +1931,11 @@ set_engine_option (struct lsquic_engine_settings *settings,
             settings->es_ptpc_int_gain = atof(val);
             return 0;
         }
+        if (0 == strncmp(name, "delay_onclose", 13))
+        {
+            settings->es_delay_onclose = atoi(val);
+            return 0;
+        }
         break;
     case 14:
         if (0 == strncmp(name, "max_streams_in", 14))

docs/apiref.rst

@@ -856,6 +856,16 @@ settings structure:
 
        Default value is :macro:`LSQUIC_DF_QPACK_EXPERIMENT`
 
+    .. member:: int             es_delay_onclose
+
+       When set to true, :member:`lsquic_stream_if.on_close` will be delayed until the
+       peer acknowledges all data sent on the stream.  (Or until the connection
+       is destroyed in some manner -- either explicitly closed by the user or
+       as a result of an engine shutdown.)  To find out whether all data written
+       to peer has been acknowledged, use `lsquic_stream_has_unacked_data()`.
+
+       Default value is :macro:`LSQUIC_DF_DELAY_ONCLOSE`
+
 To initialize the settings structure to library defaults, use the following
 convenience function:
 
@@ -1088,6 +1098,10 @@ out of date.  Please check your :file:`lsquic.h` for actual values.*
 
     By default, QPACK experiments are turned off.
 
+.. macro:: LSQUIC_DF_DELAY_ONCLOSE
+
+    By default, calling :member:`lsquic_stream_if.on_close()` is not delayed.
+
 Receiving Packets
 -----------------
 
@@ -1272,6 +1286,20 @@ the engine to communicate with the user code:
 
         This callback is mandatory.
 
+    .. member:: void (*on_reset)    (lsquic_stream_t *s, lsquic_stream_ctx_t *h, int how)
+
+        This callback is called as soon as the peer resets a stream.
+        The argument `how` is either 0, 1, or 2, meaning "read", "write", and
+        "read and write", respectively (just like in ``shutdown(2)``).  This
+        signals the user to stop reading, writing, or both.
+
+        Note that resets differ in gQUIC and IETF QUIC.  In gQUIC, `how` is
+        always 2; in IETF QUIC, `how` is either 0 or 1 because on can reset
+        just one direction in IETF QUIC.
+
+        This callback is optional.  The reset error can still be collected
+        during next "on read" or "on write" event.
+
     .. member:: void (*on_hsk_done)(lsquic_conn_t *c, enum lsquic_hsk_status s)
 
         When handshake is completed, this callback is called.
@@ -1945,6 +1973,11 @@ Miscellaneous Stream Functions
     Returns true if this stream was rejected, false otherwise.  Use this as
     an aid to distinguish between errors.
 
+.. function:: int lsquic_stream_has_unacked_data (const lsquic_stream_t *stream)
+
+    Return true if peer has not ACKed all data written to the stream.  This
+    includes both packetized and buffered data.
+
 Other Functions
 ---------------
 

docs/conf.py

@@ -24,9 +24,9 @@
 author = u'LiteSpeed Technologies'
 
 # The short X.Y version
-version = u'2.24'
+version = u'2.25'
 # The full version, including alpha/beta/rc tags
-release = u'2.24.5'
+release = u'2.25.0'
 
 
 # -- General configuration ---------------------------------------------------

include/lsquic.h

@@ -24,8 +24,8 @@ extern "C" {
 #endif
 
 #define LSQUIC_MAJOR_VERSION 2
-#define LSQUIC_MINOR_VERSION 24
-#define LSQUIC_PATCH_VERSION 5
+#define LSQUIC_MINOR_VERSION 25
+#define LSQUIC_PATCH_VERSION 0
 
 /**
  * Engine flags:
@@ -210,6 +210,17 @@ struct lsquic_stream_if {
      * perform a session resumption next time around.
      */
     void (*on_sess_resume_info)(lsquic_conn_t *c, const unsigned char *, size_t);
+    /**
+     * Optional callback is called as soon as the peer resets a stream.
+     * The argument `how' is either 0, 1, or 2, meaning "read", "write", and
+     * "read and write", respectively (just like in shutdown(2)).  This
+     * signals the user to stop reading, writing, or both.
+     *
+     * Note that resets differ in gQUIC and IETF QUIC.  In gQUIC, `how' is
+     * always 2; in IETF QUIC, `how' is either 0 or 1 because on can reset
+     * just one direction in IETF QUIC.
+     */
+    void (*on_reset)    (lsquic_stream_t *s, lsquic_stream_ctx_t *h, int how);
 };
 
 struct ssl_ctx_st;
@@ -411,6 +422,9 @@ typedef struct ssl_ctx_st * (*lsquic_lookup_cert_f)(
 /** By default, we use the minimum timer of 1000 milliseconds */
 #define LSQUIC_DF_MTU_PROBE_TIMER 1000
 
+/** By default, calling on_close() is not delayed */
+#define LSQUIC_DF_DELAY_ONCLOSE 0
+
 struct lsquic_engine_settings {
     /**
      * This is a bit mask wherein each bit corresponds to a value in
@@ -1005,6 +1019,16 @@ struct lsquic_engine_settings {
              es_ptpc_int_gain,      /* LSQUIC_DF_PTPC_INT_GAIN */
              es_ptpc_err_thresh,    /* LSQUIC_DF_PTPC_ERR_THRESH */
              es_ptpc_err_divisor;   /* LSQUIC_DF_PTPC_ERR_DIVISOR */
+
+    /**
+     * When set to true, the on_close() callback will be delayed until the
+     * peer acknowledges all data sent on the stream.  (Or until the connection
+     * is destroyed in some manner -- either explicitly closed by the user or
+     * as a result of an engine shutdown.)
+     *
+     * Default value is @ref LSQUIC_DF_DELAY_ONCLOSE
+     */
+    int             es_delay_onclose;
 };
 
 /* Initialize `settings' to default values */
@@ -1636,6 +1660,13 @@ int lsquic_stream_shutdown(lsquic_stream_t *s, int how);
 
 int lsquic_stream_close(lsquic_stream_t *s);
 
+/**
+ * Return true if peer has not ACKed all data written to the stream.  This
+ * includes both packetized and buffered data.
+ */
+int
+lsquic_stream_has_unacked_data (lsquic_stream_t *s);
+
 /**
  * Get certificate chain returned by the server.  This can be used for
  * server certificate verification.

src/liblsquic/lsquic_engine.c

@@ -390,6 +390,7 @@ lsquic_engine_init_settings (struct lsquic_engine_settings *settings,
     settings->es_ptpc_int_gain   = LSQUIC_DF_PTPC_INT_GAIN;
     settings->es_ptpc_err_thresh = LSQUIC_DF_PTPC_ERR_THRESH;
     settings->es_ptpc_err_divisor= LSQUIC_DF_PTPC_ERR_DIVISOR;
+    settings->es_delay_onclose   = LSQUIC_DF_DELAY_ONCLOSE;
 }
 
 

src/liblsquic/lsquic_full_conn.c

@@ -1330,6 +1330,8 @@ new_stream (struct full_conn *conn, lsquic_stream_id_t stream_id,
         flags |= SCF_HTTP;
     if (conn->fc_enpub->enp_settings.es_rw_once)
         flags |= SCF_DISP_RW_ONCE;
+    if (conn->fc_enpub->enp_settings.es_delay_onclose)
+        flags |= SCF_DELAY_ONCLOSE;
 
     return new_stream_ext(conn, stream_id, STREAM_IF_STD, flags);
 }
@@ -4025,6 +4027,7 @@ headers_stream_on_push_promise (void *ctx, struct uncompressed_headers *uh)
     }
 
     stream = new_stream_ext(conn, uh->uh_oth_stream_id, STREAM_IF_STD,
+                (conn->fc_enpub->enp_settings.es_delay_onclose?SCF_DELAY_ONCLOSE:0)|
                 SCF_DI_AUTOSWITCH|(conn->fc_enpub->enp_settings.es_rw_once ?
                                                         SCF_DISP_RW_ONCE : 0));
     if (!stream)

src/liblsquic/lsquic_full_conn_ietf.c

@@ -1079,6 +1079,8 @@ create_bidi_stream_out (struct ietf_full_conn *conn)
     flags = SCF_IETF|SCF_DI_AUTOSWITCH;
     if (conn->ifc_enpub->enp_settings.es_rw_once)
         flags |= SCF_DISP_RW_ONCE;
+    if (conn->ifc_enpub->enp_settings.es_delay_onclose)
+        flags |= SCF_DELAY_ONCLOSE;
     if (conn->ifc_flags & IFC_HTTP)
     {
         flags |= SCF_HTTP;
@@ -1117,6 +1119,8 @@ create_push_stream (struct ietf_full_conn *conn)
     flags = SCF_IETF|SCF_HTTP;
     if (conn->ifc_enpub->enp_settings.es_rw_once)
         flags |= SCF_DISP_RW_ONCE;
+    if (conn->ifc_enpub->enp_settings.es_delay_onclose)
+        flags |= SCF_DELAY_ONCLOSE;
 
     stream_id = generate_stream_id(conn, SD_UNI);
     stream = lsquic_stream_new(stream_id, &conn->ifc_pub,
@@ -5267,6 +5271,8 @@ new_stream (struct ietf_full_conn *conn, lsquic_stream_id_t stream_id,
         stream_ctx = conn->ifc_enpub->enp_stream_if_ctx;
         if (conn->ifc_enpub->enp_settings.es_rw_once)
             flags |= SCF_DISP_RW_ONCE;
+        if (conn->ifc_enpub->enp_settings.es_delay_onclose)
+            flags |= SCF_DELAY_ONCLOSE;
         if (conn->ifc_flags & IFC_HTTP)
         {
             flags |= SCF_HTTP;

src/liblsquic/lsquic_stream.c

@@ -1,20 +1,6 @@
 /* Copyright (c) 2017 - 2020 LiteSpeed Technologies Inc.  See LICENSE. */
 /*
  * lsquic_stream.c -- stream processing
- *
- * To clear up terminology, here are some of our stream states (in order).
- * They are not codified, but they are referred to in both code and comments.
- *
- *  CLOSED      STREAM_U_READ_DONE and STREAM_U_WRITE_DONE are set.  At this
- *                point, on_close() gets called.
- *  FINISHED    FIN or RST has been sent to peer.  Stream is scheduled to be
- *                finished (freed): it gets put onto the `service_streams'
- *                list for connection to clean it up.
- *  DESTROYED   All remaining memory associated with the stream is released.
- *                If on_close() has not been called yet, it is called now.
- *                The stream pointer is now invalid.
- *
- * When connection is aborted, a stream may go directly to DESTROYED state.
  */
 
 #include <assert.h>
@@ -476,7 +462,7 @@ lsquic_stream_new (lsquic_stream_id_t id,
             stream->sm_readable = stream_readable_non_http;
         if ((ctor_flags & (SCF_HTTP|SCF_HTTP_PRIO))
                                                 == (SCF_HTTP|SCF_HTTP_PRIO))
-        lsquic_stream_set_priority_internal(stream, LSQUIC_DEF_HTTP_URGENCY);
+            lsquic_stream_set_priority_internal(stream, LSQUIC_DEF_HTTP_URGENCY);
         else
             lsquic_stream_set_priority_internal(stream,
                                             LSQUIC_STREAM_DEFAULT_PRIO);
@@ -708,13 +694,16 @@ lsquic_stream_destroy (lsquic_stream_t *stream)
 
 
 static int
-stream_is_finished (const lsquic_stream_t *stream)
+stream_is_finished (struct lsquic_stream *stream)
 {
     return lsquic_stream_is_closed(stream)
+        && (stream->sm_bflags & SMBF_DELAY_ONCLOSE ?
+           /* Need a stricter check when on_close() is delayed: */
+            !lsquic_stream_has_unacked_data(stream) :
            /* n_unacked checks that no outgoing packets that reference this
             * stream are outstanding:
             */
-        && 0 == stream->n_unacked
+            0 == stream->n_unacked)
         && 0 == (stream->sm_qflags & (
            /* This checks that no packets that reference this stream will
             * become outstanding:
@@ -756,6 +745,8 @@ maybe_schedule_call_on_close (lsquic_stream_t *stream)
     if ((stream->stream_flags & (STREAM_U_READ_DONE|STREAM_U_WRITE_DONE|
                      STREAM_ONNEW_DONE|STREAM_ONCLOSE_DONE))
             == (STREAM_U_READ_DONE|STREAM_U_WRITE_DONE|STREAM_ONNEW_DONE)
+            && (!(stream->sm_bflags & SMBF_DELAY_ONCLOSE)
+                                || !lsquic_stream_has_unacked_data(stream))
             && !(stream->sm_qflags & SMQF_CALL_ONCLOSE))
     {
         if (0 == (stream->sm_qflags & SMQF_SERVICE_FLAGS))
@@ -1224,6 +1215,28 @@ lsquic_stream_rst_in (lsquic_stream_t *stream, uint64_t offset,
         return -1;
     }
 
+    if (stream->stream_if->on_reset
+                            && !(stream->stream_flags & STREAM_ONCLOSE_DONE))
+    {
+        if (stream->sm_bflags & SMBF_IETF)
+        {
+            if (!(stream->sm_dflags & SMDF_ONRESET0))
+            {
+                stream->stream_if->on_reset(stream, stream->st_ctx, 0);
+                stream->sm_dflags |= SMDF_ONRESET0;
+            }
+        }
+        else
+        {
+            if ((stream->sm_dflags & (SMDF_ONRESET0|SMDF_ONRESET1))
+                                    != (SMDF_ONRESET0|SMDF_ONRESET1))
+            {
+                stream->stream_if->on_reset(stream, stream->st_ctx, 2);
+                stream->sm_dflags |= SMDF_ONRESET0|SMDF_ONRESET1;
+            }
+        }
+    }
+
     /* Let user collect error: */
     maybe_conn_to_tickable_if_readable(stream);
 
@@ -1270,8 +1283,15 @@ lsquic_stream_stop_sending_in (struct lsquic_stream *stream,
     SM_HISTORY_APPEND(stream, SHE_STOP_SENDIG_IN);
     stream->stream_flags |= STREAM_SS_RECVD;
 
+    if (stream->stream_if->on_reset && !(stream->sm_dflags & SMDF_ONRESET1)
+                            && !(stream->stream_flags & STREAM_ONCLOSE_DONE))
+    {
+        stream->stream_if->on_reset(stream, stream->st_ctx, 1);
+        stream->sm_dflags |= SMDF_ONRESET1;
+    }
+
     /* Let user collect error: */
-    maybe_conn_to_tickable_if_readable(stream);
+    maybe_conn_to_tickable_if_writeable(stream, 0);
 
     lsquic_sfcw_consume_rem(&stream->fc);
     drop_frames_in(stream);
@@ -4283,7 +4303,10 @@ lsquic_stream_acked (struct lsquic_stream *stream,
         stream->stream_flags |= STREAM_RST_ACKED;
     }
     if (0 == stream->n_unacked)
+    {
+        maybe_schedule_call_on_close(stream);
         maybe_finish_stream(stream);
+    }
 }
 
 
@@ -5411,3 +5434,10 @@ lsquic_stream_set_http_prio (struct lsquic_stream *stream,
     else
         return -1;
 }
+
+
+int
+lsquic_stream_has_unacked_data (struct lsquic_stream *stream)
+{
+    return stream->n_unacked > 0 || stream->sm_n_buffered > 0;
+}

src/liblsquic/lsquic_stream.h

@@ -191,7 +191,17 @@ enum stream_b_flags
     SMBF_HTTP_PRIO    = 1 <<10,  /* Extensible HTTP Priorities are used */
     SMBF_INCREMENTAL  = 1 <<11,  /* Value of the "incremental" HTTP Priority parameter */
     SMBF_HPRIO_SET    = 1 <<12,  /* Extensible HTTP Priorities have been set once */
-#define N_SMBF_FLAGS 13
+    SMBF_DELAY_ONCLOSE= 1 <<13,  /* Delay calling on_close() until peer ACKs everything */
+#define N_SMBF_FLAGS 14
+};
+
+
+/* Stream "callback done" flags */
+/* TODO: move STREAM.*DONE flags from stream_flags here */
+enum stream_d_flags
+{
+    SMDF_ONRESET0       =   1 << 0, /* Called on_reset(0) */
+    SMDF_ONRESET1       =   1 << 1, /* Called on_reset(1) */
 };
 
 
@@ -364,6 +374,7 @@ struct lsquic_stream
         SSHS_ENC_SENDING,   /* Sending encoder stream data */
         SSHS_HBLOCK_SENDING,/* Sending header block data */
     }                               sm_send_headers_state:8;
+    enum stream_d_flags             sm_dflags:8;
     signed char                     sm_saved_want_write;
     signed char                     sm_has_frame;
 
@@ -396,6 +407,7 @@ enum stream_ctor_flags
     SCF_CRYPTO        = SMBF_CRYPTO,
     SCF_HEADERS       = SMBF_HEADERS,
     SCF_HTTP_PRIO     = SMBF_HTTP_PRIO,
+    SCF_DELAY_ONCLOSE = SMBF_DELAY_ONCLOSE,
 };