build and run on Linux

.github/actions/build-libloot/action.yml

@@ -0,0 +1,38 @@
+name: build libloot for Linux
+description: >
+  Builds the C++ wrapper of the libloot version loot_api's headers name, from its release tag, and
+  writes it stripped to loot_api/libloot.so.0. LOOT publishes no Linux binary. The image the calling
+  job runs on sets the library's glibc floor, so callers use the oldest one GitHub hosts.
+
+runs:
+  using: composite
+  steps:
+    - id: version
+      shell: bash
+      run: |
+        version=$(sed -nE 's/.*LIBLOOT_VERSION_(MAJOR|MINOR|PATCH) = ([0-9]+);/\2/p' loot_api/include/loot/loot_version.h | paste -sd.)
+        echo "version=$version" >> "$GITHUB_OUTPUT"
+
+    - uses: actions/checkout@v4
+      with:
+        repository: loot/libloot
+        ref: ${{ steps.version.outputs.version }}
+        path: libloot
+
+    - uses: actions/cache@v4
+      with:
+        path: |
+          ~/.cargo/registry/index
+          ~/.cargo/registry/cache
+          ~/.cargo/git/db
+          libloot/target
+        key: libloot-${{ steps.version.outputs.version }}-${{ env.ImageOS }}
+
+    # the cmake invocation libloot's own CI uses, minus its test binary; the SONAME is libloot.so.0
+    - shell: bash
+      working-directory: libloot/cpp
+      run: |
+        cmake -DCMAKE_BUILD_TYPE=Release -DLIBLOOT_BUILD_TESTS=OFF -B build
+        cmake --build build --parallel
+        cp --dereference build/libloot.so.0 "$GITHUB_WORKSPACE/loot_api/libloot.so.0"
+        strip --strip-unneeded "$GITHUB_WORKSPACE/loot_api/libloot.so.0"

.github/workflows/build.yml

@@ -5,9 +5,17 @@ on:
     branches: [master]
   pull_request:
 
+permissions:
+  contents: read
+
 jobs:
   build:
-    runs-on: windows-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # the oldest Linux image GitHub hosts, see .github/actions/build-libloot
+        os: [windows-latest, ubuntu-22.04]
+    runs-on: ${{ matrix.os }}
 
     steps:
       - uses: actions/checkout@v4
@@ -17,6 +25,11 @@ jobs:
           node-version: 22
           cache: yarn
 
+      # rebuilt rather than taken from loot_api, so the sources under test always link against a
+      # library of exactly the version the headers describe
+      - if: runner.os == 'Linux'
+        uses: ./.github/actions/build-libloot
+
       # the package install script prefers a published prebuild, which would leave the sources under
       # test unbuilt, so skip it and compile the addon explicitly
       - run: yarn install --frozen-lockfile --ignore-scripts

.github/workflows/vendor-libloot.yml

@@ -0,0 +1,47 @@
+name: vendor libloot
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: the libloot release to vendor, e.g. 0.29.6
+        required: true
+
+permissions:
+  contents: write
+
+jobs:
+  vendor:
+    # the oldest Linux image GitHub hosts, see .github/actions/build-libloot
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      # the Windows binaries and the headers are the official release, unmodified
+      - env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release download "${{ inputs.version }}" -R loot/libloot -p "libloot-${{ inputs.version }}-win64.7z" -D release
+          7z x -orelease release/*.7z
+          src="release/libloot-${{ inputs.version }}-win64"
+          cp "$src/bin/libloot.dll" "$src/lib/libloot.lib" "$src/lib/libloot.pdb" loot_api/
+          rm -rf loot_api/include
+          cp -r "$src/include" loot_api/include
+          rm -rf release
+
+      - uses: ./.github/actions/build-libloot
+
+      - run: yarn version --patch --no-git-tag-version
+
+      # the branch is pushed, not turned into a pull request: one the workflow token opened would
+      # start no CI run, so a person opens it from the link in the run summary
+      - run: |
+          branch="vendor/libloot-${{ inputs.version }}"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git switch -c "$branch"
+          git add loot_api package.json
+          git commit -m "update to libloot ${{ inputs.version }}"
+          git push --force origin "$branch"
+          echo "Open the pull request: https://github.com/${{ github.repository }}/compare/$branch?expand=1" >> "$GITHUB_STEP_SUMMARY"

README.md

@@ -6,7 +6,7 @@ See https://github.com/loot/loot for LOOT itself
 # Caveats
 
 This doesn't support the entire API and the API has been changed in a few places to fit better in a node.js environment.
-For easier distribution, this currently includes build artifacts (.dll and .lib as well as headers) for loot-api.
+For easier distribution, this includes libloot in binary form (see loot_api/readme.txt).
 
 # Async
 
@@ -16,34 +16,10 @@ to the second process where they will be queued and processed in sequence.
 
 # Keeping this module up to date
 
-The following procedure should be followed to ensure a smooth transition to a new version of the LOOT API.
+[1] Run the "vendor libloot" workflow from the Actions tab with the libloot release version. It replaces the Windows binaries and headers in loot_api with the official release, builds and strips the Linux library from the same tag (LOOT publishes no Linux binary), increments this module's version and pushes a `vendor/libloot-<version>` branch. Open the pull request from the link in the run's summary; one the workflow opened itself would start no CI run.
 
-[1] Create a new branch based on the current master/main and clone it locally.
+[2] On that pull request, go through https://loot-api.readthedocs.io/en/latest/api/changelog.html for each release since the previous version and check src/lootwrapper.cpp against any API change: renamed or removed functions, changed parameters. The wrapper function names match the API functions unless renamed. CI builds and tests the addon on Windows and Linux against the new library.
 
-[2] Download the latest 64bit API release from https://github.com/loot/libloot/releases and replace the contents of the loot_api folder with the contents of the new release.
+[3] Merge once CI is green.
 
-[3] Go to https://loot-api.readthedocs.io/en/latest/api/changelog.html#id1 and go through the changelog of each incremental API release (since the last version) to see what changed.
-
-[4] Go to src/lootwrapper.cpp and verify whether any of the changed API functionality could potentially affect the module's wrapper. The wrapper function names will match the API functions (unless renamed) Things like changed function parameters, newly added/renamed functions should be reflected in the lootwrapper.cpp file as required. It's usually safe to assume that the return values will not change as LOOT is quite robust at this point and is widely used by Vortex and other applications.
-
-[5] Increment the version of this module in the package.json file.
-
-[6] Commit your changes to your new branch.
-
-The below steps are only valid for Vortex. Please make sure you have a functioning development build of the source code.
-
-[7] Open the package.json file in the gamebryo-plugin-management extension and change the loot dependency from `"loot": "Nexus-Mods/node-loot"` to `"loot": "Nexus-Mods/node-loot#YOUR_BRANCH",` (obviously change the Nexus-Mods/node-loot part if your repo is located somewhere else)
-
-[8] Run `yarn install` to ensure the package manager pulls your updated loot module and `yarn build`. 
-
-[9] Run your Vortex development build and test all relevant functionality depending on the changes made to the LOOT API as noted in the changelogs.
-
-[10] Once all relevant functionality has been fully tested, submit a pull request.
-
-[10.1] Go back to the gamebryo-plugin-management extension and ensure the package file's loot dependency change is reverted. It's a good idea to `rm -rf node_modules` just in case.
-
-[11] Once the PR has been merged, go to the gamebryo-plugin-management extension directory and run `yarn upgrade-interactive` - use the interface to upgrade the loot dependency to the latest version.
-
-[12] Increment the version of the gamebryo-plugin-management extension and submit a PR. The PR should include an updated `package.json` and `yarn.lock` files.
-
-[13] Once merged - make sure to update the gamebryo-plugin-management submodule.
+For Vortex: pin `loot` in pnpm-workspace.yaml (the catalog entry and the built-dependencies entry) to the merge commit, run `pnpm install`, and test the plugin list, plugin details and sorting on a development build before opening the Vortex pull request.

async.js

@@ -18,7 +18,7 @@ process.on('uncaughtException', error => {
 
 let currentLogLevel = 2; // default: info (matches previous hardcoded filter)
 
-const client = net.connect(`\\\\?\\pipe\\loot-ipc-${process.argv[2]}`, (arg) => {
+const client = net.connect(process.argv[2], (arg) => {
   let instance;
   let dataBuffer = '';
   // a read can end mid-character; the decoder holds the incomplete sequence back until the

binding.gyp

@@ -10,8 +10,6 @@
                 "src/lootwrapper.h",
                 "src/exceptions.cpp",
                 "src/exceptions.h",
-                "src/string_cast.cpp",
-                "src/string_cast.h",
                 "src/napi_helpers.cpp",
                 "src/napi_helpers.h",
                 "src/util.cpp",
@@ -21,9 +19,6 @@
                 "./loot_api/include",
                 "<!(node -p \"require('node-addon-api').include_dir\")"
             ],
-            "libraries": [
-                "-l../loot_api/libloot"
-            ],
             'cflags!': ['-fno-exceptions', '-g', '-O0'],
             'cflags_cc!': ['-fno-exceptions'],
             'msvs_settings': {
@@ -49,6 +44,7 @@
                   "WINVER=0x600"
                 ],
                 "libraries": [
+                  "-l../loot_api/libloot",
                   "-DelayLoad:node.exe",
                 ],
                 'msvs_settings': {
@@ -59,6 +55,18 @@
                     'LinkTimeCodeGeneration': 1
                   }
                 }
+              }],
+              ["OS=='linux'", {
+                "cflags_cc": [
+                  "-std=c++20"
+                ],
+                "ldflags": [
+                  "-Wl,-rpath,\\$$ORIGIN:\\$$ORIGIN/../../loot_api"
+                ],
+                "libraries": [
+                  "-L../loot_api",
+                  "-l:libloot.so.0"
+                ]
               }]
             ]
         }

index.js

@@ -1,4 +1,7 @@
+const crypto = require('crypto');
+const fs = require('fs');
 const net = require('net');
+const os = require('os');
 const path = require('path');
 const { StringDecoder } = require('string_decoder');
 
@@ -12,6 +15,29 @@ const LogLevel = {
   error: 4,
 };
 
+// where the worker's socket goes: the user's runtime directory, which the desktop spec keeps
+// private, when there is a usable one, otherwise the temp directory
+function socketDir() {
+  const runtimeDir = process.env.XDG_RUNTIME_DIR;
+  if (runtimeDir !== undefined) {
+    try {
+      fs.accessSync(runtimeDir, fs.constants.W_OK);
+      return runtimeDir;
+    } catch (err) {
+      // inherited from another user, or from a session that no longer exists
+    }
+  }
+  return os.tmpdir();
+}
+
+// the endpoint the worker connects to: a named pipe on Windows, a unix socket elsewhere
+function ipcPath() {
+  const id = crypto.randomUUID();
+  return process.platform === 'win32'
+    ? `\\\\?\\pipe\\loot-ipc-${id}`
+    : path.join(socketDir(), `loot-ipc-${id}.sock`);
+}
+
 // interface IPluginsNotLoadedArgs {
 //   name: string;
 //   plugin: string;
@@ -134,12 +160,12 @@ class LootAsync {
     this.makeProxy('clearConditionCache');
     this.makeProxy('setLogLevel');
 
-    this.id = this.generateId();
+    this.ipcPath = ipcPath();
     this.ipc = new net.Server();
     try {
       // this seems to fail for some users with EINVAL. why?
       // May be a wine-only problem but that's not confirmed
-      this.ipc.listen(`\\\\?\\pipe\\loot-ipc-${this.id}`, () => {
+      this.ipc.listen(this.ipcPath, () => {
         this.ipc.on('connection', socket => {
           this.socket = socket;
           socket
@@ -208,12 +234,12 @@ class LootAsync {
         initCallback(err);
       });
     } catch (err) {
-      initCallback(new Error('failed to establish pipe'));
+      initCallback(new Error('failed to establish IPC endpoint'));
     }
   }
 
   restart(callback) {
-    this.worker = this.onFork(`${__dirname}${path.sep}async.js`, [this.id]);
+    this.worker = this.onFork(`${__dirname}${path.sep}async.js`, [this.ipcPath]);
     this.currentCallback = () => {
       this.enqueue({
         type: 'init',
@@ -222,18 +248,15 @@ class LootAsync {
     }
   }
 
-  generateId() {
-    const chars = 'abcdefghijklmnopqrstuvwxyz0123456789';
-    let res = [];
-    for (let i = 0; i < 8; ++i) {
-      res.push(chars[Math.floor(Math.random() * chars.length)]);
+  close() {
+    if (this.didClose) {
+      return;
     }
-    return res.join('');
-  }
 
-  close() {
     this.enqueue({ type: 'terminate' }, () => {
       this.worker = undefined;
+      // closing the server removes a unix socket's file
+      this.ipc.close();
     });
     this.didClose = true;
   }

loot_api/readme.txt

@@ -1,3 +1,9 @@
 LOOT is being developed by "Ortham" at https://github.com/loot/libloot under the GPLv3.
 
-This directory contains an unmodified copy of an official release, it is only included here in binary form to simplify the build process.
+This directory holds libloot in binary form to simplify the build process.
+
+libloot.dll, libloot.lib, libloot.pdb and include/ are an unmodified copy of the official Windows
+release of the version named in include/loot/loot_version.h.
+
+libloot.so.0 is the same version's C++ wrapper built from source by the "vendor libloot" workflow
+on ubuntu-22.04 (glibc 2.34, GLIBCXX 3.4.29) and stripped, since LOOT publishes no Linux binary.

src/exceptions.cpp

@@ -1,10 +1,9 @@
 #include "exceptions.h"
-#include "string_cast.h"
 #include "util.h"
 #include <napi.h>
 #include <optional>
 
-#ifdef WIN32
+#ifdef _WIN32
 #include <windows.h>
 
 std::wstring strerror(unsigned long errorno) {
@@ -31,13 +30,13 @@ std::wstring strerror(unsigned long errorno) {
   }
 }
 
-#endif // WIN32
+#endif // _WIN32
 
 Napi::Error ErrnoException(const Napi::Env &env, unsigned long lastError, const char * func, const char * path) {
 
-#ifdef WIN32
+#ifdef _WIN32
   std::wstring errStr = strerror(lastError);
-  std::string err = toMB(errStr.c_str(), CodePage::UTF8, errStr.size());
+  std::string err = Napi::String::New(env, reinterpret_cast<const char16_t*>(errStr.c_str())).Utf8Value();
 #else
   std::string err = strerror(lastError);
 #endif

src/exceptions.h

@@ -1,9 +1,9 @@
 #pragma once
 
 
-#ifdef WIN32
+#ifdef _WIN32
 #include <windows.h>
-#endif // WIN32
+#endif // _WIN32
 #include <exception>
 #include <napi.h>
 #include <loot/api.h>