Some bitcode and symbol debugging aids (#831)

* Add a new shadingsys option "llvm_optput_bitcode" that will dump the
bitcode for each group, even if the other debug options aren't turned all
the way on.

* bitcode now dumps in text as well as binary. (It saves a step of running
"llvm-dis" if your goal is just ot see the text in the first place, and also
it helps the llvm bug where sometimes it refuses to work because of mismatched
version strings.)

* Make sure that both in the bitcode output and other debug symbols,
we are more careful about being sure the shader group name is included,
not just the layer name, and also if a layer name is not specified to
ss->Shader(), make one up based on the shader master name and the
layer number.

Author: lgritz

src/include/OSL/llvm_util.h

@@ -474,7 +474,10 @@ class OSLEXECPUBLIC LLVM_Util {
     /// file.  If err is not NULL, errors will be deposited there.
     void write_bitcode_file (const char *filename, std::string *err=NULL);
 
-    /// Convert a function's bitcode to a string.
+    /// Convert a whole module's bitcode to a string.
+    std::string bitcode_string (llvm::Module *module);
+
+    /// Convert one function's bitcode to a string.
     std::string bitcode_string (llvm::Function *func);
 
     /// Delete the IR for the body of the given function to reclaim its

src/include/OSL/oslexec.h

@@ -157,6 +157,8 @@ class OSLEXECPUBLIC ShadingSystem
     ///                              layer functions.
     ///    int llvm_debug_ops     Extra printfs for each OSL op (helpful
     ///                              for devs to find crashes)
+    ///    int llvm_output_bitcode  Output the full bitcode for each group,
+    ///                              for debugging. (0)
     ///    int max_local_mem_KB   Error if shader group needs more than this
     ///                              much local storage to execute (1024K)
     ///    string debug_groupname Name of shader group -- debug only this one

src/liboslexec/llvm_gen.cpp

@@ -157,10 +157,8 @@ BackendLLVM::llvm_call_layer (int layer, bool unconditional)
         // insert point is now then_block
     }
 
-    std::string name = Strutil::format ("%s_%d", parent->layername().c_str(),
-                                        parent->id());
     // Mark the call as a fast call
-    llvm::Value *funccall = ll.call_function (name.c_str(), args, 2);
+    llvm::Value *funccall = ll.call_function (layer_function_name(group(), *parent).c_str(), args, 2);
     if (!parent->entry_layer())
         ll.mark_fast_func_call (funccall);
 

src/liboslexec/llvm_instance.cpp

@@ -27,6 +27,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
 #include <cmath>
+#include <iostream>
 #include <unordered_map>
 
 #include <OpenImageIO/timer.h>
@@ -761,8 +762,7 @@ BackendLLVM::build_llvm_instance (bool groupentry)
 {
     // Make a layer function: void layer_func(ShaderGlobals*, GroupData*)
     // Note that the GroupData* is passed as a void*.
-    std::string unique_layer_name = Strutil::format ("%s_%d", inst()->layername(), inst()->id());
-
+    std::string unique_layer_name = layer_function_name();
     bool is_entry_layer = group().is_entry_layer(layer());
     ll.current_function (
            ll.make_function (unique_layer_name,
@@ -1103,6 +1103,20 @@ BackendLLVM::run ()
     }
     ll.internalize_module_functions ("osl_", external_function_names, entry_function_names);
 
+    // Debug code to dump the pre-optimized bitcode to a file
+    if (llvm_debug() >= 2 || shadingsys().llvm_output_bitcode()) {
+        std::string name = Strutil::format ("%s_%s_%d.bc", group().name(),
+                                            inst()->layername(), inst()->id());
+        ll.write_bitcode_file (name.c_str());
+        name = Strutil::format ("%s_%s_%d.ll", group().name(),
+                                inst()->layername(), inst()->id());
+        std::ofstream out (name, std::ios_base::out | std::ios_base::trunc);
+        if (out.good()) {
+            out << ll.bitcode_string (ll.module());
+            out.close ();
+        }
+    }
+
     // Optimize the LLVM IR unless it's a do-nothing group.
     if (! group().does_nothing())
         ll.do_optimize();
@@ -1116,11 +1130,18 @@ BackendLLVM::run ()
         std::cout.flush();
     }
 
-    // Debug code to dump the resulting bitcode to a file
-    if (llvm_debug() >= 2) {
-        std::string name = Strutil::format ("%s_%d.bc", inst()->layername(),
-                                            inst()->id());
+    // Debug code to dump the post-optimized bitcode to a file
+    if (llvm_debug() >= 2 || shadingsys().llvm_output_bitcode()) {
+        std::string name = Strutil::format ("%s_%s_%d_opt.bc", group().name(),
+                                            inst()->layername(), inst()->id());
         ll.write_bitcode_file (name.c_str());
+        name = Strutil::format ("%s_%s_%d_opt.ll", group().name(),
+                                inst()->layername(), inst()->id());
+        std::ofstream out (name, std::ios_base::out | std::ios_base::trunc);
+        if (out.good()) {
+            out << ll.bitcode_string (ll.module());
+            out.close ();
+        }
     }
 
     // Force the JIT to happen now and retrieve the JITed function pointers

src/liboslexec/llvm_util.cpp

@@ -1495,6 +1495,30 @@ LLVM_Util::bitcode_string (llvm::Function *func)
 
 
 
+std::string
+LLVM_Util::bitcode_string (llvm::Module *module)
+{
+    std::string s;
+    llvm::raw_string_ostream stream (s);
+
+#if OSL_LLVM_VERSION < 40
+    for (llvm::Module::iterator iter = module->begin(); iter != module->end(); iter++) {
+        llvm::Function *funcptr = static_cast<llvm::Function*>(iter);
+        stream << (*funcptr);
+        stream << "\n";
+    }
+#else
+    for (const llvm::Function& func : module->getFunctionList()) {
+        stream << func;
+        stream << "\n";
+    }
+#endif
+
+    return stream.str();
+}
+
+
+
 void
 LLVM_Util::delete_func_body (llvm::Function *func)
 {

src/liboslexec/oslexec_pvt.h

@@ -609,6 +609,7 @@ class ShadingSystemImpl
     int llvm_debug () const { return m_llvm_debug; }
     int llvm_debug_layers () const { return m_llvm_debug_layers; }
     int llvm_debug_ops () const { return m_llvm_debug_ops; }
+    int llvm_output_bitcode () const { return m_llvm_output_bitcode; }
     bool fold_getattribute () const { return m_opt_fold_getattribute; }
     bool opt_texture_handle () const { return m_opt_texture_handle; }
     int opt_passes() const { return m_opt_passes; }
@@ -802,6 +803,7 @@ class ShadingSystemImpl
     int m_llvm_debug;                     ///< More LLVM debugging output
     int m_llvm_debug_layers;              ///< Add layer enter/exit printfs
     int m_llvm_debug_ops;                 ///< Add printfs to every op
+    int m_llvm_output_bitcode;            ///< Output bitcode for each group
     ustring m_debug_groupname;            ///< Name of sole group to debug
     ustring m_debug_layername;            ///< Name of sole layer to debug
     ustring m_opt_layername;              ///< Name of sole layer to optimize
@@ -1997,6 +1999,16 @@ class OSOProcessorBase {
         return m_bblockids[opnum];
     }
 
+    // Mangle the group and layer into a unique function name
+    std::string layer_function_name (const ShaderGroup &group,
+                                     const ShaderInstance &inst) {
+        return Strutil::format ("%s_%s_%d", group.name(),
+                                inst.layername(), inst.id());
+    }
+    std::string layer_function_name () {
+        return layer_function_name (group(), *inst());
+    }
+
 protected:
     ShadingSystemImpl &m_shadingsys;  ///< Backpointer to shading system
     ShaderGroup &m_group;             ///< Group we're processing

src/liboslexec/shadingsys.cpp

@@ -669,6 +669,7 @@ ShadingSystemImpl::ShadingSystemImpl (RendererServices *renderer,
       m_llvm_optimize(0),
       m_debug(0), m_llvm_debug(0),
       m_llvm_debug_layers(0), m_llvm_debug_ops(0),
+      m_llvm_output_bitcode(0),
       m_commonspace_synonym("world"),
       m_colorspace("Rec709"),
       m_max_local_mem_KB(2048),
@@ -1092,6 +1093,7 @@ ShadingSystemImpl::attribute (string_view name, TypeDesc type,
     ATTR_SET ("llvm_debug", int, m_llvm_debug);
     ATTR_SET ("llvm_debug_layers", int, m_llvm_debug_layers);
     ATTR_SET ("llvm_debug_ops", int, m_llvm_debug_ops);
+    ATTR_SET ("llvm_output_bitcode", int, m_llvm_output_bitcode);
     ATTR_SET ("strict_messages", int, m_strict_messages);
     ATTR_SET ("range_checking", int, m_range_checking);
     ATTR_SET ("unknown_coordsys_error", int, m_unknown_coordsys_error);
@@ -1203,6 +1205,7 @@ ShadingSystemImpl::getattribute (string_view name, TypeDesc type,
     ATTR_DECODE ("llvm_debug", int, m_llvm_debug);
     ATTR_DECODE ("llvm_debug_layers", int, m_llvm_debug_layers);
     ATTR_DECODE ("llvm_debug_ops", int, m_llvm_debug_ops);
+    ATTR_DECODE ("llvm_output_bitcode", int, m_llvm_output_bitcode);
     ATTR_DECODE ("strict_messages", int, m_strict_messages);
     ATTR_DECODE ("range_checking", int, m_range_checking);
     ATTR_DECODE ("unknown_coordsys_error", int, m_unknown_coordsys_error);
@@ -1619,6 +1622,7 @@ ShadingSystemImpl::getstats (int level) const
     INTOPT (llvm_debug);
     BOOLOPT (llvm_debug_layers);
     BOOLOPT (llvm_debug_ops);
+    BOOLOPT (llvm_output_bitcode);
     BOOLOPT (lazylayers);
     BOOLOPT (lazyglobals);
     BOOLOPT (lazyunconnected);
@@ -1957,6 +1961,14 @@ ShadingSystemImpl::Shader (string_view shaderusage,
         return false;
     }
 
+    // If a layer name was not supplied, make one up.
+    std::string local_layername;
+    if (layername.empty()) {
+        local_layername = OIIO::Strutil::format ("%s_%d", master->shadername(),
+                                                 m_curgroup->nlayers());
+        layername = string_view (local_layername);
+    }
+
     ShaderInstanceRef instance (new ShaderInstance (master, layername));
     instance->parameters (m_pending_params);
     m_pending_params.clear ();

testsuite/array-range/ref/out-alt.txt

@@ -2,19 +2,19 @@ Compiled test.osl -> test.oso
 constant index:
  reading:
   array[1] = 1
-ERROR: Index [10] out of range $const1[0..4]: test.osl:8 (group <unnamed group>, layer 0 <unnamed layer>, shader test)
+ERROR: Index [10] out of range $const1[0..4]: test.osl:8 (group <unnamed group>, layer 0 test_0, shader test)
   array[10] = 4
  writing:
-ERROR: Index [10] out of range array[0..4]: test.osl:10 (group <unnamed group>, layer 0 <unnamed layer>, shader test)
+ERROR: Index [10] out of range array[0..4]: test.osl:10 (group <unnamed group>, layer 0 test_0, shader test)
 variable index:
  reading:
   array[0] = 0
   array[1] = 1
   array[2] = 2
   array[3] = 3
   array[4] = 42
-ERROR: Index [5] out of range array[0..4]: test.osl:15 (group <unnamed group>, layer 0 <unnamed layer>, shader test)
+ERROR: Index [5] out of range array[0..4]: test.osl:15 (group <unnamed group>, layer 0 test_0, shader test)
   array[5] = 42
  writing:
-ERROR: Index [5] out of range array[0..4]: test.osl:19 (group <unnamed group>, layer 0 <unnamed layer>, shader test)
+ERROR: Index [5] out of range array[0..4]: test.osl:19 (group <unnamed group>, layer 0 test_0, shader test)
 

testsuite/array-range/ref/out.txt

@@ -2,19 +2,19 @@ Compiled test.osl -> test.oso
 constant index:
  reading:
   array[1] = 1
-ERROR: Index [10] out of range array[0..4]: test.osl:8 (group <unnamed group>, layer 0 <unnamed layer>, shader test)
+ERROR: Index [10] out of range array[0..4]: test.osl:8 (group <unnamed group>, layer 0 test_0, shader test)
   array[10] = 4
  writing:
-ERROR: Index [10] out of range array[0..4]: test.osl:10 (group <unnamed group>, layer 0 <unnamed layer>, shader test)
+ERROR: Index [10] out of range array[0..4]: test.osl:10 (group <unnamed group>, layer 0 test_0, shader test)
 variable index:
  reading:
   array[0] = 0
   array[1] = 1
   array[2] = 2
   array[3] = 3
   array[4] = 42
-ERROR: Index [5] out of range array[0..4]: test.osl:15 (group <unnamed group>, layer 0 <unnamed layer>, shader test)
+ERROR: Index [5] out of range array[0..4]: test.osl:15 (group <unnamed group>, layer 0 test_0, shader test)
   array[5] = 42
  writing:
-ERROR: Index [5] out of range array[0..4]: test.osl:19 (group <unnamed group>, layer 0 <unnamed layer>, shader test)
+ERROR: Index [5] out of range array[0..4]: test.osl:19 (group <unnamed group>, layer 0 test_0, shader test)
 

testsuite/component-range/ref/out-alt.txt

@@ -2,17 +2,17 @@ Compiled test.osl -> test.oso
 constant index:
  reading:
   V[1] = 1
-ERROR: Index [10] out of range $const1[0..2]: test.osl:8 (group <unnamed group>, layer 0 <unnamed layer>, shader test)
+ERROR: Index [10] out of range $const1[0..2]: test.osl:8 (group <unnamed group>, layer 0 test_0, shader test)
   V[10] = 2
  writing:
-ERROR: Index [10] out of range V[0..2]: test.osl:10 (group <unnamed group>, layer 0 <unnamed layer>, shader test)
+ERROR: Index [10] out of range V[0..2]: test.osl:10 (group <unnamed group>, layer 0 test_0, shader test)
 variable index:
  reading:
   V[0] = 0
   V[1] = 1
   V[2] = 42
-ERROR: Index [3] out of range V[0..2]: test.osl:15 (group <unnamed group>, layer 0 <unnamed layer>, shader test)
+ERROR: Index [3] out of range V[0..2]: test.osl:15 (group <unnamed group>, layer 0 test_0, shader test)
   V[3] = 42
  writing:
-ERROR: Index [3] out of range V[0..2]: test.osl:19 (group <unnamed group>, layer 0 <unnamed layer>, shader test)
+ERROR: Index [3] out of range V[0..2]: test.osl:19 (group <unnamed group>, layer 0 test_0, shader test)