buffered output and fine grained debug macros

contrib/CMakeLists.txt

@@ -80,12 +80,6 @@ add_executable(gendir gendir.c)
 target_link_libraries(gendir ${COMMON_LIBRARIES})
 add_dependencies(gendir GUFI)
 
-
-add_executable(fill_queues fill_queues.c)
-target_link_libraries(fill_queues ${COMMON_LIBRARIES})
-add_dependencies(fill_queues GUFI)
-
-
 # potentially useful C++ executables
 if (CMAKE_CXX_COMPILER)
   # a more complex index generator

include/OutputBuffers.h

@@ -0,0 +1,95 @@
+/*
+This file is part of GUFI, which is part of MarFS, which is released
+under the BSD license.
+
+
+Copyright (c) 2017, Los Alamos National Security (LANS), LLC
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+may be used to endorse or promote products derived from this software without
+specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+-----
+NOTE:
+-----
+
+GUFI uses the C-Thread-Pool library.  The original version, written by
+Johan Hanssen Seferidis, is found at
+https://github.com/Pithikos/C-Thread-Pool/blob/master/LICENSE, and is
+released under the MIT License.  LANS, LLC added functionality to the
+original work.  The original work, plus LANS, LLC added functionality is
+found at https://github.com/jti-lanl/C-Thread-Pool, also under the MIT
+License.  The MIT License can be found at
+https://opensource.org/licenses/MIT.
+
+
+From Los Alamos National Security, LLC:
+LA-CC-15-039
+
+Copyright (c) 2017, Los Alamos National Security, LLC All rights reserved.
+Copyright 2017. Los Alamos National Security, LLC. This software was produced
+under U.S. Government contract DE-AC52-06NA25396 for Los Alamos National
+Laboratory (LANL), which is operated by Los Alamos National Security, LLC for
+the U.S. Department of Energy. The U.S. Government has rights to use,
+reproduce, and distribute this software.  NEITHER THE GOVERNMENT NOR LOS
+ALAMOS NATIONAL SECURITY, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR
+ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE.  If software is
+modified to produce derivative works, such modified software should be
+clearly marked, so as not to confuse it with the version available from
+LANL.
+
+THIS SOFTWARE IS PROVIDED BY LOS ALAMOS NATIONAL SECURITY, LLC AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL LOS ALAMOS NATIONAL SECURITY, LLC OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+OF SUCH DAMAGE.
+*/
+
+
+
+#include <pthread.h>
+#include <stddef.h>
+
+/* Single Buffer */
+struct OutputBuffer {
+    char * buf;
+    size_t capacity;
+    size_t filled;
+};
+
+/* Buffers for all threads */
+struct OutputBuffers {
+    struct OutputBuffer * buffers;
+    pthread_mutex_t mutex;
+};
+
+struct OutputBuffers * OutputBuffers_init(struct OutputBuffers * obufs, const size_t count, const size_t capacity);
+void OutputBuffers_destroy(struct OutputBuffers * obufs, const size_t count);

scripts/gufi_find

@@ -403,10 +403,9 @@ if __name__=='__main__':
     parser.add_argument('--smallest',                                dest='smallest',           action='store_true',                          help='top n smallest files')
     parser.add_argument('--largest',                                 dest='largest',            action='store_true',                          help='top n largest files')
     parser.add_argument('--threads', '-t',     metavar='n',          dest='threads',            type=qb.get_positive, default=qb.cpus(),      help='number of threads')
-    parser.add_argument('--intermediate_dbs',  metavar='n',          dest='intermediates',      type=qb.get_non_negative,                     help='number of intermediate databases')
-    parser.add_argument('--intermediate_skip', metavar='n',          dest='skip',               type=qb.get_non_negative,                     help='number of intermediate databases to skip when selecting the next database')
     parser.add_argument('--aggregate',                               dest='aggregate',          action='store_true',                          help='aggregate results before printing')
     parser.add_argument('--exec',                                    dest='query_exec',         default=EXEC,                                 help='Location of gufi_query executable')
+    parser.add_argument('--output',                                  dest='output',             type=str,                                     help='Output file prefix (Creates file <output>.tid)')
 
     args = parser.parse_args();
 
@@ -489,6 +488,9 @@ if __name__=='__main__':
     if args.mindepth:
         query_cmd += ['-y', str(args.mindepth)]
 
+    if args.output:
+        query_cmd += ['-o', args.output]
+
     query = subprocess.Popen(query_cmd + find_out.split()) # positional arguments must appear after flags
     query.communicate()                                    # block until query finishes
 

src/CMakeLists.txt

@@ -106,7 +106,20 @@ endif()
 # ##########################################
 
 # create the GUFI library, which contains all of the common source files
-set(GUFI_SOURCES bf.c dbutils.c debug.c opendb.c outfiles.c outdbs.c QueuePerThreadPool.c SinglyLinkedList.c structq.c template_db.c trace.c utils.c)
+set(GUFI_SOURCES
+  bf.c
+  dbutils.c
+  debug.c
+  opendb.c
+  outfiles.c
+  outdbs.c
+  OutputBuffers.c
+  QueuePerThreadPool.c
+  SinglyLinkedList.c
+  structq.c
+  template_db.c
+  trace.c
+  utils.c)
 add_library(GUFI STATIC ${GUFI_SOURCES})
 add_dependencies(GUFI install_dependencies)
 install(TARGETS GUFI DESTINATION lib COMPONENT Server)

src/OutputBuffers.c

@@ -0,0 +1,126 @@
+/*
+This file is part of GUFI, which is part of MarFS, which is released
+under the BSD license.
+
+
+Copyright (c) 2017, Los Alamos National Security (LANS), LLC
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+may be used to endorse or promote products derived from this software without
+specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+-----
+NOTE:
+-----
+
+GUFI uses the C-Thread-Pool library.  The original version, written by
+Johan Hanssen Seferidis, is found at
+https://github.com/Pithikos/C-Thread-Pool/blob/master/LICENSE, and is
+released under the MIT License.  LANS, LLC added functionality to the
+original work.  The original work, plus LANS, LLC added functionality is
+found at https://github.com/jti-lanl/C-Thread-Pool, also under the MIT
+License.  The MIT License can be found at
+https://opensource.org/licenses/MIT.
+
+
+From Los Alamos National Security, LLC:
+LA-CC-15-039
+
+Copyright (c) 2017, Los Alamos National Security, LLC All rights reserved.
+Copyright 2017. Los Alamos National Security, LLC. This software was produced
+under U.S. Government contract DE-AC52-06NA25396 for Los Alamos National
+Laboratory (LANL), which is operated by Los Alamos National Security, LLC for
+the U.S. Department of Energy. The U.S. Government has rights to use,
+reproduce, and distribute this software.  NEITHER THE GOVERNMENT NOR LOS
+ALAMOS NATIONAL SECURITY, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR
+ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE.  If software is
+modified to produce derivative works, such modified software should be
+clearly marked, so as not to confuse it with the version available from
+LANL.
+
+THIS SOFTWARE IS PROVIDED BY LOS ALAMOS NATIONAL SECURITY, LLC AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL LOS ALAMOS NATIONAL SECURITY, LLC OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+OF SUCH DAMAGE.
+*/
+
+#include "OutputBuffers.h"
+
+#include <stdlib.h>
+
+static struct OutputBuffer * OutputBuffer_init(struct OutputBuffer * obuf, const size_t capacity) {
+    if (obuf) {
+        if (!(obuf->buf = malloc(capacity))) {
+            return NULL;
+        }
+        obuf->capacity = capacity;
+        obuf->filled = 0;
+    }
+
+    return obuf;
+}
+
+static void OutputBuffer_destroy(struct OutputBuffer * obuf) {
+    if (obuf) {
+        free(obuf->buf);
+    }
+}
+
+struct OutputBuffers * OutputBuffers_init(struct OutputBuffers * obufs, const size_t count, const size_t capacity) {
+    if (!obufs) {
+        return NULL;
+    }
+
+    if (pthread_mutex_init(&obufs->mutex, NULL)) {
+        return NULL;
+    }
+
+    obufs->buffers = malloc(count * sizeof(struct OutputBuffer));
+    for(size_t i = 0; i < count; i++) {
+        if (!OutputBuffer_init(&obufs->buffers[i], capacity)) {
+            OutputBuffers_destroy(obufs, i);
+            return NULL;
+        }
+    }
+
+    return obufs;
+}
+
+void OutputBuffers_destroy(struct OutputBuffers * obufs, const size_t count) {
+    for(size_t i = 0; i < count; i++) {
+        OutputBuffer_destroy(&obufs->buffers[i]);
+    }
+    free(obufs->buffers);
+    obufs->buffers = NULL;
+
+    pthread_mutex_destroy(&obufs->mutex);
+}