Skip to content

Commit 6457158

Browse files
yoursunnyyoursunny
committed
fetch: fallocate fallback to ftruncate
ext3 filesystem does not support fallocate.
1 parent 5fef2e8 commit 6457158

File tree

6 files changed

+90
-59
lines changed

6 files changed

+90
-59
lines changed

‎app/fetch/fetchtest/fetcher_test.go

+2
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,8 @@ func TestFetcher(t *testing.T) {
9696
}
9797
ticker.Stop()
9898
}
99+
taskA.Stop()
100+
taskB.Stop()
99101

100102
cntA, cntB := taskA.Counters(), taskB.Counters()
101103
assert.EqualValues(defA.SegmentEnd-defA.SegmentBegin, cntA.NRxData)

‎app/fetch/task.go

+28-13
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,11 @@ func (ts *taskSlot) Init(d TaskDef) error {
116116
}
117117
ts.tpl.prefixV[ts.tpl.prefixL] = an.TtSegmentNameComponent
118118

119+
logEntry := logger.With(
120+
zap.Int("slot-index", int(ts.index)),
121+
zap.Stringer("prefix", d.Prefix),
122+
)
123+
119124
if d.Filename != "" {
120125
if d.SegmentLen <= 0 || d.SegmentLen > math.MaxUint32 {
121126
return errors.New("bad SegmentLen")
@@ -129,24 +134,27 @@ func (ts *taskSlot) Init(d TaskDef) error {
129134
return fmt.Errorf("unix.Open(%s): %w", d.Filename, e)
130135
}
131136

137+
logEntry = logEntry.With(
138+
zap.String("filename", d.Filename),
139+
zap.Int("fd", fd),
140+
zap.Int("segment-len", d.SegmentLen),
141+
)
142+
132143
offsetBegin := int64(d.SegmentBegin) * int64(d.SegmentLen)
133144
offsetEnd := int64(d.SegmentEnd) * int64(d.SegmentLen)
134145
if e := unix.Fallocate(fd, 0, offsetBegin, offsetEnd-offsetBegin); e != nil {
135-
unix.Close(fd)
136-
unix.Unlink(d.Filename)
137-
return fmt.Errorf("unix.Fallocate(%s): %w", d.Filename, e)
146+
logEntry.Warn("unix.Fallocate error, this may affect write performance",
147+
zap.Int64("offset-begin", offsetBegin),
148+
zap.Int64("offset-end", offsetEnd),
149+
zap.Error(e),
150+
)
138151
}
139152

140153
ts.fd, ts.segmentLen = C.int(fd), C.uint32_t(d.SegmentLen)
141154
}
142155

143-
logger.Info("task init",
144-
zap.Int("slot-index", int(ts.index)),
145-
zap.Stringer("prefix", d.Prefix),
156+
logEntry.Info("task init",
146157
zap.Uint64s("segment-range", []uint64{d.SegmentBegin, d.SegmentEnd}),
147-
zap.String("filename", d.Filename),
148-
zap.Int("fd", int(ts.fd)),
149-
zap.Int("segment-len", d.SegmentLen),
150158
)
151159
return nil
152160
}
@@ -162,15 +170,22 @@ func (ts *taskSlot) Logic() *Logic {
162170
}
163171

164172
func (ts *taskSlot) closeFd() {
165-
if ts.fd < 0 {
173+
fd := int(ts.fd)
174+
if fd < 0 {
166175
return
167176
}
168-
if e := unix.Close(int(ts.fd)); e != nil {
169-
logger.Warn("unix.Close error",
170-
zap.Int("fd", int(ts.fd)),
177+
178+
logEntry := logger.With(
179+
zap.Int("slot-index", int(ts.index)),
180+
zap.Int("fd", fd),
181+
)
182+
if e := unix.Close(fd); e != nil {
183+
logEntry.Warn("unix.Close error",
171184
zap.Error(e),
172185
)
173186
}
187+
188+
logEntry.Info("task output file closed")
174189
ts.fd = -1
175190
}
176191

‎csrc/core/mmapfd.c

+6-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,12 @@ MmapFd_Open(MmapFd* m, const char* filename, size_t size) {
3030
}
3131
if (fallocate(m->fd, 0, 0, size) != 0) {
3232
MmapFd_Error(fallocate);
33-
goto FAIL;
33+
if (ftruncate(m->fd, size) == 0) {
34+
N_LOGW("ftruncate succeeded in place of fallocate, this may affect write performance");
35+
} else {
36+
MmapFd_Error(ftruncate);
37+
goto FAIL;
38+
}
3439
}
3540

3641
m->map = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, m->fd, 0);

‎dpdk/eal/malloc.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ func ZmallocAligned[T any, S constraints.Integer](dbgtype string, size S, align
2626

2727
func zmallocImpl(dbgtype string, size uintptr, align int, socket NumaSocket) unsafe.Pointer {
2828
var typ [32]byte
29-
copy(typ[:len(typ)-1], []byte(dbgtype))
29+
copy(typ[:len(typ)-1], dbgtype)
3030
typC := (*C.char)(unsafe.Pointer(unsafe.SliceData(typ[:])))
3131

3232
ptr := C.rte_zmalloc_socket(typC, C.size_t(size), C.uint(align*C.RTE_CACHE_LINE_SIZE), C.int(socket.ID()))

‎go.mod

+16-16
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ require (
1414
github.com/gopacket/gopacket v1.2.0
1515
github.com/gorilla/schema v1.4.0
1616
github.com/graphql-go/graphql v0.8.1
17-
github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240618164540-8f65cb80ce42
18-
github.com/jacobsa/fuse v0.0.0-20240607092844-7285af0d05b0
17+
github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240626161320-231a28f62b43
18+
github.com/jacobsa/fuse v0.0.0-20240626143436-8a36813dc074
1919
github.com/jfoster/binary-utilities v0.2.1
2020
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
2121
github.com/korylprince/go-graphql-ws v0.3.6
@@ -25,7 +25,7 @@ require (
2525
github.com/pascaldekloe/name v1.0.1
2626
github.com/powerman/rpc-codec v1.2.2
2727
github.com/rickb777/plural v1.4.2
28-
github.com/safchain/ethtool v0.4.0
28+
github.com/safchain/ethtool v0.4.1
2929
github.com/sethvargo/go-retry v0.2.4
3030
github.com/sirupsen/logrus v1.9.3
3131
github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8
@@ -49,28 +49,28 @@ require (
4949
github.com/alecthomas/participle/v2 v2.0.0-alpha6 // indirect
5050
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
5151
github.com/davecgh/go-spew v1.1.1 // indirect
52-
github.com/gofrs/uuid v3.2.0+incompatible // indirect
53-
github.com/google/uuid v1.3.0 // indirect
54-
github.com/gorilla/websocket v1.4.2 // indirect
52+
github.com/gofrs/uuid v4.4.0+incompatible // indirect
53+
github.com/google/uuid v1.6.0 // indirect
54+
github.com/gorilla/websocket v1.5.3 // indirect
5555
github.com/matryer/is v1.4.1 // indirect
56-
github.com/mattn/go-colorable v0.1.12 // indirect
57-
github.com/mattn/go-isatty v0.0.14 // indirect
56+
github.com/mattn/go-colorable v0.1.13 // indirect
57+
github.com/mattn/go-isatty v0.0.20 // indirect
5858
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
5959
github.com/onsi/ginkgo v1.16.5 // indirect
6060
github.com/onsi/gomega v1.33.1 // indirect
61-
github.com/pkg/errors v0.8.1 // indirect
61+
github.com/pkg/errors v0.9.1 // indirect
6262
github.com/pmezard/go-difflib v1.0.0 // indirect
6363
github.com/russross/blackfriday/v2 v2.1.0 // indirect
6464
github.com/segmentio/fasthash v1.0.3 // indirect
6565
github.com/smartystreets/goconvey v1.8.1 // indirect
66-
github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
66+
github.com/vishvananda/netns v0.0.4 // indirect
6767
github.com/x-cray/logrus-prefixed-formatter v0.5.2 // indirect
68-
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
68+
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
6969
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
70-
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
71-
go.uber.org/multierr v1.10.0 // indirect
72-
golang.org/x/crypto v0.22.0 // indirect
73-
golang.org/x/net v0.24.0 // indirect
74-
golang.org/x/term v0.19.0 // indirect
70+
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
71+
go.uber.org/multierr v1.11.0 // indirect
72+
golang.org/x/crypto v0.24.0 // indirect
73+
golang.org/x/net v0.26.0 // indirect
74+
golang.org/x/term v0.21.0 // indirect
7575
gopkg.in/yaml.v3 v3.0.1 // indirect
7676
)

0 commit comments

Comments
 (0)