bench-stand: the jit lane stops rebuilding the JIT emitter once per benchmark

.github/workflows/build.yml

@@ -781,8 +781,6 @@ jobs:
     # follow-up; we lose master's previous bind_llvm.das CI coverage in
     # the meantime.
     - name: "Run self-binder (bind_clangbind.das)"
-      env:
-        DAS_CLANG_INCLUDE: ${{ env.MSYSTEM_PREFIX }}/include
       run: cmake --build ./build-mingw --config Release --target check_clangbind_fresh
 
     - name: "Run cbind const-gen regression test"

.github/workflows/pages.yml

@@ -172,9 +172,7 @@ jobs:
       # the log names the .so and the missing symbol directly.
       env:
         DAS_TRACE_MODULE_LOAD: 1
-      run: |
-        set -eux
-        ./bin/daslang -documentation doc/reflections/das2rst.das
+      run: cmake --build ./build --target check_docs_generated
 
     - name: "Run imgui2rst"
       # dasImgui's stdlib pages (doc/source/stdlib/generated/imgui_*.rst) come

CMakeCommon.txt

@@ -91,6 +91,15 @@ MACRO(SETUP_SANITIZERS)
             ADD_COMPILE_OPTIONS(/fsanitize=address)
         ENDIF()
     ENDIF()
+    IF(DAS_USE_SANITIZER AND NOT DAS_USE_SANITIZER STREQUAL "none")
+        SET(DAS_SANITIZER_ENV ${CMAKE_COMMAND} -E env
+            "LSAN_OPTIONS=suppressions=${CMAKE_CURRENT_SOURCE_DIR}/.github/lsan_suppressions.txt"
+            "TSAN_OPTIONS=suppressions=${CMAKE_CURRENT_SOURCE_DIR}/.github/tsan_suppressions.txt"
+            "UBSAN_OPTIONS=print_stacktrace=1"
+            CACHE INTERNAL "sanitizer runtime options for every lane that runs a built binary")
+    ELSE()
+        SET(DAS_SANITIZER_ENV "" CACHE INTERNAL "sanitizer runtime options for every lane that runs a built binary")
+    ENDIF()
 ENDMACRO()
 
 # Some options are defines and should be exported together with library.

CMakeLists.txt

@@ -512,6 +512,7 @@ ENDIF()
 # Every dynamic module should pass through this macro.
 MACRO(ADD_DAS_SHARED_MODULE_LIB module_name)
     MESSAGE("REGISTER DAS SHARED MODULE ${module_name}")
+    SET_PROPERTY(GLOBAL APPEND PROPERTY DAS_SHARED_MODULE_TARGETS ${module_name})
     SETUP_DAS_SHARED_LIBRARY(${module_name} ${ARGN})
     DAS_APPLY_STRICT_WARNINGS(${module_name})
 
@@ -640,7 +641,6 @@ MACRO(ADD_EXAMPLE_RUN das_file)
         IF("${ARGN}" STREQUAL "FALSE")
             SET(_exe FALSE)
         ENDIF()
-        MESSAGE("EXAMPLE_RUN ${das_file} (exe=${_exe})")
         LIST(APPEND DAS_EXAMPLES_TO_RUN "${das_file}|${_exe}")
     ENDIF()
 ENDMACRO()
@@ -1411,29 +1411,30 @@ target_sources(libDaScriptDyn PRIVATE src/misc/alloc_tracker_overrides.cpp)
 target_link_libraries(libDaScript libDaScript_runtime)
 target_link_libraries(libDaScriptDyn libDaScriptDyn_runtime)
 
+set(AotDaslibList
+    daslib/ast_boost.das
+    daslib/ast_cursor.das
+    daslib/functional.das
+    daslib/json_boost.das
+    daslib/json.das
+    daslib/math_boost.das
+    daslib/random.das
+    daslib/regex_boost.das
+    daslib/regex.das
+    daslib/shader_lingua_franca.das
+    daslib/strings_boost.das
+    daslib/strings_convert.das
+    daslib/templates_boost.das
+    daslib/utf8_utils.das
+)
+
 if(NOT ${DAS_AOT_EXAMPLES_DISABLED})
 
     if(${DAS_TOOLS_DISABLED})
         # error, we need tools to generate aot
         message(FATAL_ERROR "DAS_AOT_EXAMPLES_DISABLED requires DAS_TOOLS_DISABLED to be OFF")
     endif()
 
-    set(AotDaslibList
-        daslib/ast_boost.das
-        daslib/ast_cursor.das
-        daslib/functional.das
-        daslib/json_boost.das
-        daslib/json.das
-        daslib/math_boost.das
-        daslib/random.das
-        daslib/regex_boost.das
-        daslib/regex.das
-        daslib/strings_boost.das
-        daslib/strings_convert.das
-        daslib/templates_boost.das
-        daslib/utf8_utils.das
-    )
-
     add_custom_target(dasAotStub)
     DAS_AOT_LIB("${AotDaslibList}" AOT_GENERATED_SRC dasAotStub daslang)
 

ci/CMakeLists.txt

@@ -112,3 +112,27 @@ das_lane_group(run_nightly "Running everything CI runs only on the cron"
     run_build_nightly ${DAS_EXTENDED_NIGHTLY})
 das_lane_group(run_all "Running every lane this configure can build"
     run_extended_all run_doc run_build run_build_nightly)
+
+# A lane runs daslang against the tree's own shared modules and descriptors. Under a single
+# ninja invocation a relink lands while another lane is reading that module, and its descriptor
+# manifest (keyed on dep size and hash) goes stale mid-read. CI never sees this: it builds in
+# one step and tests in the next. The edge goes on every leaf, not on the groups - a dependency
+# of a group is a sibling of its members and ninja may still run them together.
+add_custom_target(das_build_products COMMENT "Building what the lanes load at runtime")
+add_dependencies(das_build_products daslang)
+get_property(_das_modules GLOBAL PROPERTY DAS_SHARED_MODULE_TARGETS)
+if(_das_modules)
+    add_dependencies(das_build_products ${_das_modules})
+endif()
+foreach(_lane ${DAS_EXTENDED_CORE} ${DAS_EXTENDED_MODULES} ${DAS_EXTENDED_NIGHTLY}
+              test-small test-big check_bundle_smoke
+              run_tests_interpreter run_tests_interpreter_isolated
+              run_tests_jit run_tests_jit_isolated
+              run_tests_aot run_tests_aot_subset run_tests_llvm_aot
+              run_tests_slow run_standalone_sweep_aot standalone_sweep_jit
+              check_ast_verify_tree run_examples dry_run_tutorials
+              check_docs_generated run_imgui2rst run_vulkan2rst stage_doc_assets)
+    if(TARGET ${_lane})
+        add_dependencies(${_lane} das_build_products)
+    endif()
+endforeach()

ci/smoke_test_bundle.sh

@@ -134,16 +134,6 @@ LOG="$(mktemp)"
 # run; Windows has no rpath and locks open DLL dirs, so only POSIX.
 BUILD_LIB="$(cd "$CI_DIR/.." && pwd -P)/lib"
 HIDDEN_LIB=""
-# A previous run killed outright (SIGKILL, a hard ninja abort) never reached its trap and
-# left lib/ hidden, which breaks every link and every daslang launch in the tree until
-# someone notices. Put it back before doing anything else.
-if [[ -d "$BUILD_LIB.smokehidden" ]]; then
-    if [[ -e "$BUILD_LIB" ]]; then
-        cp -a "$BUILD_LIB.smokehidden"/. "$BUILD_LIB"/ && rm -rf "$BUILD_LIB.smokehidden"
-    else
-        mv "$BUILD_LIB.smokehidden" "$BUILD_LIB"
-    fi
-fi
 if [[ -z "$CPP_SUFFIX" && -d "$BUILD_LIB" ]]; then
     HIDDEN_LIB="$BUILD_LIB.smokehidden"
     mv "$BUILD_LIB" "$HIDDEN_LIB"

daslib/ast.das

@@ -478,6 +478,7 @@ class AstVisitor {
     def abstract preVisitExprContinue(expr : ExprContinue?) : void
     def abstract visitExprContinue(expr : ExprContinue?) : ExpressionPtr
     def abstract canVisitMakeBlockBody(expr : ExprMakeBlock?) : bool
+    def abstract canVisitTryCatchBody(expr : ExprTryCatch?) : bool
     def abstract preVisitExprMakeBlock(expr : ExprMakeBlock?) : void
     def abstract visitExprMakeBlock(expr : ExprMakeBlock?) : ExpressionPtr
     def abstract preVisitExprMakeGenerator(expr : ExprMakeGenerator?) : void

daslib/macro_boost.das

@@ -82,10 +82,11 @@ def public capture_block(expr : ExpressionPtr) : array<CapturedVariable> {
 }
 
 [macro]
-class ColletFinally : AstVisitor {
+class CollectFinally : AstVisitor {
     //! AST visitor that collects finally blocks.
     @do_not_delete blocks : table<ExprBlock?>
     alwaysFor : bool
+    skipTryCatch : bool = false
     def override preVisitExprBlock(blk : ExprBlock?) : void {
         unsafe {
             if (!empty(blk.finalList)) {
@@ -98,14 +99,17 @@ class ColletFinally : AstVisitor {
     def override canVisitMakeBlockBody(expr : ExprMakeBlock?) : bool {
         return false
     }
+    def override canVisitTryCatchBody(expr : ExprTryCatch?) : bool {
+        return !skipTryCatch
+    }
 }
 
 [macro_function]
-def public collect_finally(expr : ExpressionPtr; alwaysFor : bool = false) {
-    //! Collect all finally blocks in the expression.
-    //! Returns array of ExprBlock? with all the blocks which have `finally` section
-    //! Does not go into 'make_block' expression, such as `lambda`, or 'block' expressions
-    var astVisitor = new ColletFinally(alwaysFor = alwaysFor)
+def public collect_finally(expr : ExpressionPtr; alwaysFor : bool = false; skipTryCatch : bool = false) {
+    //! Collect all finally blocks in the expression - array of ExprBlock? with a `finally` section.
+    //! Does not go into 'make_block' expression, such as `lambda`, or 'block' expressions;
+    //! skipTryCatch also stops at a try/recover arm, for a consumer emitting each arm as its own block
+    var astVisitor = new CollectFinally(alwaysFor = alwaysFor, skipTryCatch = skipTryCatch)
     make_visitor(*astVisitor) $(adapter) {
         visit(expr, adapter)
     }
@@ -117,22 +121,27 @@ def public collect_finally(expr : ExpressionPtr; alwaysFor : bool = false) {
 }
 
 [macro]
-class ColletLabels : AstVisitor {
+class CollectLabels : AstVisitor {
     //! AST visitor that collects label statements.
     labels : table<int>
+    skipTryCatch : bool = false
     def override preVisitExprLabel(expr : ExprLabel?) : void {
         labels |> insert(expr.labelName)
     }
     def override canVisitMakeBlockBody(expr : ExprMakeBlock?) : bool {
         return false
     }
+    def override canVisitTryCatchBody(expr : ExprTryCatch?) : bool {
+        return !skipTryCatch
+    }
 }
 
 [macro_function]
-def public collect_labels(expr : ExpressionPtr) {
-    //! Collect all labels in the expression. Returns array of integer with label indices
-    //! Does not go into 'make_block' expression, such as `lambda`, or 'block' expressions
-    var astVisitor = new ColletLabels()
+def public collect_labels(expr : ExpressionPtr; skipTryCatch : bool = false) {
+    //! Collect all labels in the expression. Returns array of integer with label indices.
+    //! Does not go into 'make_block' expression, such as `lambda`, or 'block' expressions;
+    //! skipTryCatch also stops at a try/recover arm, for a consumer emitting each arm as its own block
+    var astVisitor = new CollectLabels(skipTryCatch = skipTryCatch)
     make_visitor(*astVisitor) $(adapter) {
         visit(expr, adapter)
     }

daslib/quote.das

@@ -466,7 +466,6 @@ class QuotePass : AstPassMacro {
     //! Pass macro that processes quoted AST expressions.
     def override apply(prog : ProgramPtr; mod : Module?) : bool {
         let lower = (compiling_program().policies.aot_macros
-            || (compiling_program().policies.jit_enabled && !prog.flags.needMacroModule)
             || (prog._options |> find_arg("aot_macros") ?as tBool ?? false))
         if (!lower) return false
         var astVisitor = new QuoteConverter(mod = compiling_module())

daslib/rst.das

@@ -82,7 +82,15 @@ var private pageless_ref_modules : table<string>
 
 var private nosearch_symbols : table<string>
 
+struct private MdRow {
+    at : int
+    name : string
+    topic : string
+    entry_lines : int
+}
+
 var private md_section_lines : array<string>
+var private md_rows : array<MdRow>
 var private md_row_at : table<string; int>
 var private md_pending_group = ""
 var private md_index_lines : array<string>
@@ -158,23 +166,31 @@ def private md_row(name : string; topic : string; entry_lines : int = 0) {
     //! One row per name: the first overload claims it, and a later overload with a description
     //! fills a row the first one left bare.
     if (empty(digest_path)) return
-    let text = md_first_sentence(md_first_paragraph(md_read_topic(topic), entry_lines))
     var at = -1
     md_row_at |> get(name) $(idx) {
         at = idx
     }
     if (at >= 0) {
-        if (!empty(text) && md_section_lines[at] == "- `{name}`") {
-            md_section_lines[at] = "- `{name}` - {text}"
-        }
+        md_rows |> emplace(MdRow(at = at, name = name, topic = topic, entry_lines = entry_lines))
         return
     }
     if (!empty(md_pending_group)) {
         md_section_lines |> push("\n### {md_pending_group}\n")
         md_pending_group = ""
     }
-    md_section_lines |> push(empty(text) ? "- `{name}`" : "- `{name}` - {text}")
+    md_section_lines |> push("- `{name}`")
     md_row_at[name] = length(md_section_lines) - 1
+    md_rows |> emplace(MdRow(at = length(md_section_lines) - 1, name = name, topic = topic, entry_lines = entry_lines))
+}
+
+def private md_resolve_rows() {
+    for (row in md_rows) {
+        if (md_section_lines[row.at] != "- `{row.name}`") continue
+        let text = md_first_sentence(md_first_paragraph(md_read_topic(row.topic), row.entry_lines))
+        if (!empty(text)) {
+            md_section_lines[row.at] = "- `{row.name}` - {text}"
+        }
+    }
 }
 
 def private md_begin(mod_name : string; mod : Module?) {
@@ -196,6 +212,7 @@ def public write_digest() {
     //! documented so far - its intro paragraph and every public symbol with a one-line description.
     //! A no-op while `digest_path` is empty.
     if (empty(digest_path)) return
+    md_resolve_rows()
     mkdir_rec(dir_name(digest_path))
     sort(md_index_lines)
     fopen(digest_path, "wb") $(f) {

doc/CMakeLists.txt

@@ -8,12 +8,27 @@
 if(NOT DAS_TOOLS_DISABLED)
     add_das_target(check_docs_generated
         COMMAND $<TARGET_FILE:daslang> -documentation ${PROJECT_SOURCE_DIR}/doc/reflections/das2rst.das
+                -- --digest ${CMAKE_CURRENT_BINARY_DIR}/everything.md
         DEPENDS daslang
         WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
         COMMENT "Regenerating stdlib RST (das2rst)"
     )
+    # The shipped digest is generated but tracked: skills/daslang ships to agents with no
+    # daslang to generate it. So the lane checks, and updating the tracked copy is this target -
+    # a copy of what the generator already wrote, never a second das2rst run to disagree with it.
+    add_custom_target(regen_docs_digest
+        COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/everything.md
+                ${PROJECT_SOURCE_DIR}/skills/daslang/references/everything.md
+        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+        COMMENT "Updating the shipped module digest (skills/daslang/references/everything.md)"
+    )
     add_custom_target(check_docs_fresh
-        COMMAND ${CMAKE_COMMAND} -DDAS_ROOT=${PROJECT_SOURCE_DIR}
+        COMMAND ${CMAKE_COMMAND} -DDAS_ROOT=${PROJECT_SOURCE_DIR} -DDAS_DOC_BUILD=${CMAKE_CURRENT_BINARY_DIR}
+                # OFF: the digest carries the symbols of every module the build loaded, so a
+                # checkout with one module more than the doc lane's generates a different file and
+                # no local configuration reproduces what CI compares against. Back ON once the
+                # digest is generated from a fixed module list.
+                -DDAS_DOC_GATE=OFF
                 -P ${PROJECT_SOURCE_DIR}/doc/check_docs_fresh.cmake
         WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
         COMMENT "Checking generated documentation is fresh"
@@ -59,14 +74,14 @@ if(DAS_BUILD_DOCUMENTATION)
         set(DAS_DOC_BUILD_DIR   "${CMAKE_CURRENT_BINARY_DIR}/doc_html")
         set(DAS_DOC_DOCTREE_DIR "${CMAKE_CURRENT_BINARY_DIR}/doc_doctrees")
 
-        # Step 1: regenerate stdlib RST from daslib source comments (requires daslang)
+        # Step 1: the pages come from check_docs_generated - one das2rst per build, never two
+        # racing over the same pages and digest
         add_custom_command(
             OUTPUT  ${CMAKE_CURRENT_BINARY_DIR}/das2rst.stamp
-            COMMAND $<TARGET_FILE:daslang> -documentation "${PROJECT_SOURCE_DIR}/doc/reflections/das2rst.das"
             COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/das2rst.stamp
-            DEPENDS daslang
+            DEPENDS check_docs_generated
             WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
-            COMMENT "Generating stdlib RST documentation (das2rst)"
+            COMMENT "Staging generated stdlib RST for Sphinx"
         )
 
         # Step 2: build HTML with Sphinx
@@ -130,6 +145,10 @@ if(NOT DAS_TOOLS_DISABLED)
         endif()
     endforeach()
     add_dependencies(check_docs_fresh ${_doc_generators})
+    add_dependencies(regen_docs_digest check_docs_generated)
+    # check_doc_verify walks doc/source, which the generators rewrite; it is a nightly member of
+    # run_extended_all, so under run_all it would otherwise read a page mid-write
+    add_dependencies(check_doc_verify ${_doc_generators})
 
     set(_doc_all ${_doc_generators} check_docs_fresh)
     if(TARGET build_doc)

doc/check_docs_fresh.cmake

@@ -38,6 +38,21 @@ if(_untracked)
     set(_failed 1)
 endif()
 
+# The shipped digest is generated but tracked: skills/daslang ships to agents with no daslang to
+# generate it with. The generator wrote its copy to the build dir; the tree's copy must match.
+# A digest carries the symbols of the modules its build loaded, so only the configuration the doc
+# lane runs - DAS_BUILD_DOCUMENTATION, which it sets and an ordinary build does not - holds the
+# reading the tracked copy is compared against.
+if(DAS_DOC_BUILD AND DAS_DOC_GATE)
+    file(SHA256 "${DAS_DOC_BUILD}/everything.md" _digest_built)
+    file(SHA256 "${DAS_ROOT}/skills/daslang/references/everything.md" _digest_tracked)
+    if(NOT _digest_built STREQUAL _digest_tracked)
+        message("ERROR: skills/daslang/references/everything.md does not match what das2rst generates.")
+        message("Update it: cmake --build <build> --target regen_docs_digest")
+        set(_failed 1)
+    endif()
+endif()
+
 if(_failed)
     message(FATAL_ERROR "generated documentation is not fresh")
 endif()

doc/reflections/das2rst.das

@@ -272,7 +272,7 @@ def document_module_dashv(_root : string) {
     var mod = get_module("dashv")
     var groups <- array<DocGroup>(
         group_by_regex("WebSocket client", mod, %regex~(make_web_socket_client|destroy_web_socket_client|open|close|is_connected|tick|send)$%%),
-        group_by_regex("WebSocket server lifecycle", mod, %regex~(make_web_socket_server|destroy_web_socket_server|set_bind_host|start|stop)$%%),
+        group_by_regex("WebSocket server lifecycle", mod, %regex~(make_web_socket_server|destroy_web_socket_server|set_bind_host|start|stop|bound_port)$%%),
         group_by_regex("WebSocket channel send", mod, %regex~(send)$%%),
         group_by_regex("Handle operations", mod, %regex~(is_alive|==|!=)$%%),
         group_by_regex("HTTP route registration", mod, %regex~(GET|POST|PUT|DELETE|PATCH|HEAD|ANY|SSE|STREAM)$%%),

doc/source/reference/tutorials/dasLLAMA_16_text_to_speech.rst

@@ -310,6 +310,7 @@ speech from the restored prompt:
 
 .. code-block:: das
 
+   let clip <- load_audio_mono("me.wav", c.sample_rate)
    var inscope prompt <- tts_prepare_voice_prompt(m, clip, c.sample_rate)
    var inscope restored = TtsVoicePrompt()
    if (sscan_json(sprint_json(prompt, false), restored)

doc/source/stdlib/handmade/function-dashv-bound_port-0xe09303f8211650b6.rst

@@ -0,0 +1,2 @@
+The port the server listens on: the one passed to ``make_web_socket_server``, or, when that was 0,
+the free port the operating system picked. Valid once ``start`` has succeeded; -1 otherwise.

include/daScript/ast/ast_infer_type.h

@@ -542,7 +542,6 @@ namespace das {
         virtual ExpressionPtr visit(ExprClone *expr) override;
         // ExprTryCatch
         void preVisit(ExprTryCatch *expr) override;
-        ExpressionPtr visit(ExprTryCatch *expr) override;
         // ExprReturn
         bool inferReturnType(TypeDeclPtr &resType, ExprReturn *expr);
         bool isVoidReturnValueSettled(Expression *subexpr) const;