vsnprintf(): If size is zero, then vsnprintf() should return the size of the required buffer without writing anything. This is same fix that was done for snprintf in 2014 by commit 59846a8fe9.

libc/stdio/lib_vsnprintf.c

@@ -55,17 +55,38 @@
 int vsnprintf(FAR char *buf, size_t size, FAR const IPTR char *format,
               va_list ap)
 {
-  struct lib_memoutstream_s memoutstream;
-  int     n;
+  union
+  {
+    struct lib_outstream_s    nulloutstream;
+    struct lib_memoutstream_s memoutstream;
+  } u;
 
-  /* Initialize a memory stream to write to the buffer */
+  FAR struct lib_outstream_s *stream;
+  int n;
 
-  lib_memoutstream((FAR struct lib_memoutstream_s *)&memoutstream,
-                    buf, size);
+  /* "If the value of [size] is zero on a call to vsnprintf(), nothing shall
+   *  be written, the number of bytes that would have been written had [size]
+   *  been sufficiently large excluding the terminating null shall be returned,
+   *  and [buf] may be a null pointer." -- opengroup.org
+   */
+
+  if (size > 0)
+    {
+      /* Initialize a memory stream to write to the buffer */
+
+      lib_memoutstream(&u.memoutstream, buf, size);
+      stream = &u.memoutstream.public;
+    }
+  else
+    {
+      /* Use a null stream to get the size of the buffer */
+
+      lib_nulloutstream(&u.nulloutstream);
+      stream = &u.nulloutstream;
+    }
 
   /* Then let lib_vsprintf do the real work */
 
-  n = lib_vsprintf((FAR struct lib_outstream_s *)&memoutstream.public,
-                   format, ap);
+  n = lib_vsprintf(stream, format, ap);
   return n;
 }