Skip to content

fuse: experimental write-path FUSE passthrough (WIP) #7202

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
EngHabu wants to merge 14 commits into
base: main
Choose a base branch
from
Open

fuse: experimental write-path FUSE passthrough (WIP) #7202

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
5e0a1bd
fuse: experimental write-path FUSE passthrough (WIP)
EngHabu Jun 24, 2026
d037dd5
fuse: never passthrough internal/control files (fixes checkpoint hang)
EngHabu Jul 7, 2026
38c04c6
fuse: pool passthrough backing registrations across opens
EngHabu Jul 7, 2026
1938b2e
fuse: disable passthrough for the mount after EPERM registration
EngHabu Jul 7, 2026
9a50674
vfs+fuse: fence passthrough reconcile for consistency points
EngHabu Jul 7, 2026
c8ba37e
vfs+fuse: cover the unmount path for in-flight passthrough reconciles
EngHabu Jul 8, 2026
c4b2f86
fuse: fix passthrough data-corruption bugs found in review
EngHabu Jul 8, 2026
d378c47
fuse: one passthrough writer per inode (close reconcile-window reuse …
EngHabu Jul 8, 2026
b5f97b9
fuse: wait out in-flight reconcile on reopen (close-to-open consistency)
EngHabu Jul 8, 2026
2bdc902
fuse: close remaining passthrough durability holes (ENG26-874)
EngHabu Jul 8, 2026
26bf421
fuse: note the fsync exception in the passthrough durability contract
EngHabu Jul 8, 2026
d1c5b3d
build: point go-fuse replace at the unionai fork (repo transferred fr…
EngHabu Jul 10, 2026
aa93a4f
fuse: never reuse passthrough backings across inodes
EngHabu Aug 6, 2026
3796448
fuse: account passthrough reconcile's copy buffer against the memory …
EngHabu Aug 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,4 +47,5 @@ test*.bak
/juicefs.lite
dist/
juicefs.egg-info/
pkg/meta/jfs-load-dump/
pkg/meta/jfs-load-dump/juicefs-fork
juicefs-fork
23 changes: 23 additions & 0 deletions cmd/mount_unix.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -937,6 +937,16 @@ func installHandler(m meta.Meta, mp string, v *vfs.VFS, blob object.ObjectStorag
sig := <-signalChan
logger.Infof("Received signal %s, exiting...", sig.String())
if sig == syscall.SIGHUP {
// A passthrough open's data lives only in this process's
// staging files; the successor inherits the FUSE session but
// has no record of them, so their release-time reconciles
// would never run and every byte written through them would
// be lost. Drain first; if live opens remain, refuse the
// handover and keep serving.
if !fuse.DrainPassthrough(time.Second * 10) {
logger.Warnf("passthrough opens still in flight, don't restart")
continue
}
path := fmt.Sprintf("/tmp/state%d.json", os.Getppid())
if err := v.FlushAll(""); err == nil {
fuse.Shutdown()
Expand All @@ -955,6 +965,19 @@ func installHandler(m meta.Meta, mp string, v *vfs.VFS, blob object.ObjectStorag
}
go func() {
time.Sleep(time.Second * 30)
// The umount is stuck, but a passthrough reconcile may still
// be copying staging data whose close(2) already returned;
// FlushAll now would commit the mid-copy prefix as a durable
// short file. Extend the deadline while reconciles are in
// flight, with a hard cap so a wedged reconcile can't hold
// the exit hostage.
hardStop := time.Now().Add(time.Minute * 10)
for v.ExternalFlushes() > 0 && time.Now().Before(hardStop) {
time.Sleep(time.Millisecond * 100)
}
if n := v.ExternalFlushes(); n > 0 {
logger.Errorf("force exit with %d unfinished passthrough reconcile(s); their files may be committed incomplete", n)
}
if err := v.FlushAll(""); err != nil {
logger.Errorf("flush all: %s", err)
}
Expand Down
94 changes: 48 additions & 46 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@ require (
github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.4.1
github.com/aliyun/credentials-go v1.4.5
github.com/apple/foundationdb/bindings/go v0.0.0-20211207225159-47b9a81d1c10
github.com/aws/aws-sdk-go-v2 v1.41.5
github.com/aws/aws-sdk-go-v2 v1.42.0
github.com/aws/aws-sdk-go-v2/config v1.32.14
github.com/aws/aws-sdk-go-v2/credentials v1.19.14
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0
github.com/aws/smithy-go v1.24.2
github.com/aws/smithy-go v1.27.1
github.com/baidubce/bce-sdk-go v0.9.263
github.com/bytedance/mockey v1.2.14
github.com/ceph/go-ceph v0.18.0
Expand All @@ -34,7 +34,7 @@ require (
github.com/go-sql-driver/mysql v1.9.3
github.com/goccy/go-json v0.10.5
github.com/gofrs/flock v0.8.1
github.com/golang/snappy v0.0.4
github.com/golang/snappy v1.0.0
github.com/google/btree v1.1.2
github.com/google/uuid v1.6.0
github.com/grafana/pyroscope-go v1.2.1
Expand Down Expand Up @@ -71,7 +71,7 @@ require (
github.com/qingstor/qingstor-sdk-go/v4 v4.4.1
github.com/qiniu/go-sdk/v7 v7.26.8
github.com/redis/go-redis/v9 v9.18.0
github.com/sirupsen/logrus v1.9.3
github.com/sirupsen/logrus v1.9.4
github.com/smartystreets/goconvey v1.7.2
github.com/spf13/cast v1.7.1
github.com/stretchr/testify v1.11.1
Expand All @@ -88,16 +88,16 @@ require (
github.com/volcengine/ve-tos-golang-sdk/v2 v2.9.0
github.com/winfsp/cgofuse v1.6.0
go.etcd.io/etcd v3.3.27+incompatible
go.etcd.io/etcd/client/v3 v3.5.9
go.etcd.io/etcd/client/v3 v3.6.12
go.uber.org/automaxprocs v1.6.0
go.uber.org/zap v1.27.0
golang.org/x/crypto v0.49.0
golang.org/x/net v0.52.0
golang.org/x/oauth2 v0.34.0
golang.org/x/sync v0.20.0
golang.org/x/sys v0.42.0
golang.org/x/term v0.41.0
golang.org/x/text v0.35.0
go.uber.org/zap v1.28.0
golang.org/x/crypto v0.51.0
golang.org/x/net v0.54.0
golang.org/x/oauth2 v0.36.0
golang.org/x/sync v0.21.0
golang.org/x/sys v0.45.0
golang.org/x/term v0.43.0
golang.org/x/text v0.38.0
google.golang.org/api v0.210.0
google.golang.org/protobuf v1.36.11
gopkg.in/kothar/go-backblaze.v0 v0.0.0-20210124194846-35409b867216
Expand All @@ -113,13 +113,13 @@ require (
cloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect
cloud.google.com/go/iam v1.2.2 // indirect
cloud.google.com/go/monitoring v1.21.2 // indirect
filippo.io/edwards25519 v1.1.0 // indirect
filippo.io/edwards25519 v1.2.0 // indirect
git.apache.org/thrift.git v0.13.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect
github.com/IBM/go-sdk-core/v5 v5.21.2 // indirect
Expand All @@ -133,8 +133,8 @@ require (
github.com/armon/go-metrics v0.4.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
Expand All @@ -152,13 +152,14 @@ require (
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cheggaaa/pb v1.0.29 // indirect
github.com/clbanning/mxj v1.8.4 // indirect
github.com/clipperhouse/uax29/v2 v2.2.0 // indirect
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect
github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect
github.com/coredns/coredns v1.4.0 // indirect
github.com/coreos/etcd v3.3.27+incompatible // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-semver v0.3.1 // indirect
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/coreos/go-systemd/v22 v22.7.0 // indirect
github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
Expand All @@ -170,8 +171,8 @@ require (
github.com/djherbis/atime v1.0.0 // indirect
github.com/dswarbrick/smart v0.0.0-20190505152634-909a45200d6d // indirect
github.com/elastic/gosigar v0.14.2 // indirect
github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect
github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect
github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect
github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect
github.com/fatih/color v1.16.0 // indirect
github.com/fatih/structs v1.1.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
Expand All @@ -194,21 +195,22 @@ require (
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
github.com/golang-jwt/jwt/v5 v5.3.0 // indirect
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
github.com/golang/glog v1.2.5 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/flatbuffers v25.2.10+incompatible // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/readahead v0.0.0-20161222183148-eaceba169032 // indirect
github.com/google/s2a-go v0.1.8 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
github.com/googleapis/gax-go/v2 v2.14.0 // indirect
github.com/gopherjs/gopherjs v1.12.80 // indirect
github.com/gorilla/handlers v1.5.1 // indirect
github.com/gorilla/mux v1.8.1 // indirect
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
Expand All @@ -229,7 +231,7 @@ require (
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jtolds/gls v4.20.0+incompatible // indirect
github.com/jtolio/noiseconn v0.0.0-20230301220541-88105e6c8ac6 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/compress v1.18.6 // indirect
github.com/klauspost/cpuid v1.3.1 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
Expand All @@ -240,7 +242,7 @@ require (
github.com/leodido/go-urn v1.4.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/mattn/go-runewidth v0.0.24 // indirect
github.com/miekg/dns v1.1.61 // indirect
github.com/minio/highwayhash v1.0.2 // indirect
github.com/minio/md5-simd v1.1.1 // indirect
Expand Down Expand Up @@ -269,7 +271,7 @@ require (
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/prometheus/procfs v0.20.1 // indirect
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rivo/uniseg v0.4.7 // indirect
Expand All @@ -283,7 +285,7 @@ require (
github.com/smartystreets/assertions v1.2.0 // indirect
github.com/spacemonkeygo/monkit/v3 v3.0.25-0.20251022131615-eb24eb109368 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect
github.com/syndtr/goleveldb v1.0.0 // indirect
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
github.com/tidwall/gjson v1.6.7 // indirect
Expand All @@ -302,33 +304,33 @@ require (
github.com/yusufpapurcu/wmi v1.2.3 // indirect
github.com/zeebo/blake3 v0.2.3 // indirect
github.com/zeebo/errs v1.4.0 // indirect
go.etcd.io/etcd/api/v3 v3.5.9 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect
go.etcd.io/etcd/api/v3 v3.6.12 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.6.12 // indirect
go.mongodb.org/mongo-driver v1.17.6 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.42.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect
go.opentelemetry.io/otel v1.43.0 // indirect
go.opentelemetry.io/otel/metric v1.43.0 // indirect
go.opentelemetry.io/otel/sdk v1.43.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
go.opentelemetry.io/otel/trace v1.43.0 // indirect
go.opentelemetry.io/otel v1.44.0 // indirect
go.opentelemetry.io/otel/metric v1.44.0 // indirect
go.opentelemetry.io/otel/sdk v1.44.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect
go.opentelemetry.io/otel/trace v1.44.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
golang.org/x/arch v0.11.0 // indirect
golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect
golang.org/x/mod v0.33.0 // indirect
golang.org/x/mod v0.36.0 // indirect
golang.org/x/time v0.8.0 // indirect
golang.org/x/tools v0.42.0 // indirect
golang.org/x/tools v0.45.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d // indirect
google.golang.org/grpc v1.80.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect
google.golang.org/grpc v1.81.1 // indirect
gopkg.in/ini.v1 v1.67.3 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
Expand All @@ -344,7 +346,7 @@ require (

replace github.com/minio/minio v0.0.0-20210206053228-97fe57bba92c => github.com/juicedata/minio v0.0.0-20260515071949-69a6cfc9da65

replace github.com/hanwen/go-fuse/v2 v2.1.1-0.20210611132105-24a1dfe6b4f8 => github.com/juicedata/go-fuse/v2 v2.1.1-0.20260610024748-b44a81936922
replace github.com/hanwen/go-fuse/v2 => github.com/unionai/go-fuse/v2 v2.1.1-0.20260624053043-ef6e8dc3c14e

replace github.com/dgrijalva/jwt-go v3.2.0+incompatible => github.com/golang-jwt/jwt v3.2.1+incompatible

Expand Down
Loading