Skip to content

Commit f5f480e

Browse files
tarutibradfitz
taruti
authored and
bradfitz
committed
os: reduce allocations in Readdir on unix
Include syscall.Stat_t on unix to the unexported fileStat structure rather than accessing it though an interface. Additionally add a benchmark for Readdir (and Readdirnames). Tested on linux, freebsd, netbsd, openbsd darwin, solaris, does not touch windows stuff. Does not change the API, as discussed on golang-dev. E.g. on linux/amd64 with a directory of 65 files: benchmark old ns/op new ns/op delta BenchmarkReaddir-4 67774 66225 -2.29% benchmark old allocs new allocs delta BenchmarkReaddir-4 334 269 -19.46% benchmark old bytes new bytes delta BenchmarkReaddir-4 25208 24168 -4.13% Change-Id: I44ef72a04ad7055523a980f29aa11122040ae8fe Reviewed-on: https://go-review.googlesource.com/16423 Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org> Run-TryBot: Brad Fitzpatrick <bradfitz@golang.org> TryBot-Result: Gobot Gobot <gobot@golang.org>
1 parent a21b4bc commit f5f480e

File tree

12 files changed

+158
-166
lines changed

12 files changed

+158
-166
lines changed

‎src/os/file_unix.go‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@ import (
1212
"syscall"
1313
)
1414

15+
func sameFile(fs1, fs2 *fileStat) bool {
16+
return fs1.sys.Dev == fs2.sys.Dev && fs1.sys.Ino == fs2.sys.Ino
17+
}
18+
1519
func rename(oldname, newname string) error {
1620
e := syscall.Rename(oldname, newname)
1721
if e != nil {
@@ -152,36 +156,39 @@ func (f *File) Stat() (FileInfo, error) {
152156
if f == nil {
153157
return nil, ErrInvalid
154158
}
155-
var stat syscall.Stat_t
156-
err := syscall.Fstat(f.fd, &stat)
159+
var fs fileStat
160+
err := syscall.Fstat(f.fd, &fs.sys)
157161
if err != nil {
158162
return nil, &PathError{"stat", f.name, err}
159163
}
160-
return fileInfoFromStat(&stat, f.name), nil
164+
fillFileStatFromSys(&fs, f.name)
165+
return &fs, nil
161166
}
162167

163168
// Stat returns a FileInfo describing the named file.
164169
// If there is an error, it will be of type *PathError.
165170
func Stat(name string) (FileInfo, error) {
166-
var stat syscall.Stat_t
167-
err := syscall.Stat(name, &stat)
171+
var fs fileStat
172+
err := syscall.Stat(name, &fs.sys)
168173
if err != nil {
169174
return nil, &PathError{"stat", name, err}
170175
}
171-
return fileInfoFromStat(&stat, name), nil
176+
fillFileStatFromSys(&fs, name)
177+
return &fs, nil
172178
}
173179

174180
// Lstat returns a FileInfo describing the named file.
175181
// If the file is a symbolic link, the returned FileInfo
176182
// describes the symbolic link. Lstat makes no attempt to follow the link.
177183
// If there is an error, it will be of type *PathError.
178184
func Lstat(name string) (FileInfo, error) {
179-
var stat syscall.Stat_t
180-
err := syscall.Lstat(name, &stat)
185+
var fs fileStat
186+
err := syscall.Lstat(name, &fs.sys)
181187
if err != nil {
182188
return nil, &PathError{"lstat", name, err}
183189
}
184-
return fileInfoFromStat(&stat, name), nil
190+
fillFileStatFromSys(&fs, name)
191+
return &fs, nil
185192
}
186193

187194
func (f *File) readdir(n int) (fi []FileInfo, err error) {

‎src/os/os_test.go‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,48 @@ func TestReaddir(t *testing.T) {
296296
testReaddir(sysdir.name, sysdir.files, t)
297297
}
298298

299+
func benchmarkReaddirname(path string, b *testing.B) {
300+
var nentries int
301+
for i := 0; i < b.N; i++ {
302+
f, err := Open(path)
303+
if err != nil {
304+
b.Fatalf("open %q failed: %v", path, err)
305+
}
306+
ns, err := f.Readdirnames(-1)
307+
f.Close()
308+
if err != nil {
309+
b.Fatalf("readdirnames %q failed: %v", path, err)
310+
}
311+
nentries = len(ns)
312+
}
313+
b.Logf("benchmarkReaddirname %q: %d entries", path, nentries)
314+
}
315+
316+
func benchmarkReaddir(path string, b *testing.B) {
317+
var nentries int
318+
for i := 0; i < b.N; i++ {
319+
f, err := Open(path)
320+
if err != nil {
321+
b.Fatalf("open %q failed: %v", path, err)
322+
}
323+
fs, err := f.Readdir(-1)
324+
f.Close()
325+
if err != nil {
326+
b.Fatalf("readdir %q failed: %v", path, err)
327+
}
328+
nentries = len(fs)
329+
}
330+
b.Logf("benchmarkReaddir %q: %d entries", path, nentries)
331+
}
332+
333+
func BenchmarkReaddirname(b *testing.B) {
334+
benchmarkReaddirname(".", b)
335+
}
336+
337+
func BenchmarkReaddir(b *testing.B) {
338+
benchmarkReaddir(".", b)
339+
}
340+
299341
// Read the directory one entry at a time.
300342
func smallReaddirnames(file *File, length int, t *testing.T) []string {
301343
names := make([]string, length)

‎src/os/stat_darwin.go‎

Lines changed: 9 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9,21 +9,12 @@ import (
99
"time"
1010
)
1111

12-
func sameFile(fs1, fs2 *fileStat) bool {
13-
stat1 := fs1.sys.(*syscall.Stat_t)
14-
stat2 := fs2.sys.(*syscall.Stat_t)
15-
return stat1.Dev == stat2.Dev && stat1.Ino == stat2.Ino
16-
}
17-
18-
func fileInfoFromStat(st *syscall.Stat_t, name string) FileInfo {
19-
fs := &fileStat{
20-
name: basename(name),
21-
size: int64(st.Size),
22-
modTime: timespecToTime(st.Mtimespec),
23-
sys: st,
24-
}
25-
fs.mode = FileMode(st.Mode & 0777)
26-
switch st.Mode & syscall.S_IFMT {
12+
func fillFileStatFromSys(fs *fileStat, name string) {
13+
fs.name = basename(name)
14+
fs.size = int64(fs.sys.Size)
15+
fs.modTime = timespecToTime(fs.sys.Mtimespec)
16+
fs.mode = FileMode(fs.sys.Mode & 0777)
17+
switch fs.sys.Mode & syscall.S_IFMT {
2718
case syscall.S_IFBLK, syscall.S_IFWHT:
2819
fs.mode |= ModeDevice
2920
case syscall.S_IFCHR:
@@ -39,16 +30,15 @@ func fileInfoFromStat(st *syscall.Stat_t, name string) FileInfo {
3930
case syscall.S_IFSOCK:
4031
fs.mode |= ModeSocket
4132
}
42-
if st.Mode&syscall.S_ISGID != 0 {
33+
if fs.sys.Mode&syscall.S_ISGID != 0 {
4334
fs.mode |= ModeSetgid
4435
}
45-
if st.Mode&syscall.S_ISUID != 0 {
36+
if fs.sys.Mode&syscall.S_ISUID != 0 {
4637
fs.mode |= ModeSetuid
4738
}
48-
if st.Mode&syscall.S_ISVTX != 0 {
39+
if fs.sys.Mode&syscall.S_ISVTX != 0 {
4940
fs.mode |= ModeSticky
5041
}
51-
return fs
5242
}
5343

5444
func timespecToTime(ts syscall.Timespec) time.Time {

‎src/os/stat_dragonfly.go‎

Lines changed: 9 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9,21 +9,12 @@ import (
99
"time"
1010
)
1111

12-
func sameFile(fs1, fs2 *fileStat) bool {
13-
stat1 := fs1.sys.(*syscall.Stat_t)
14-
stat2 := fs2.sys.(*syscall.Stat_t)
15-
return stat1.Dev == stat2.Dev && stat1.Ino == stat2.Ino
16-
}
17-
18-
func fileInfoFromStat(st *syscall.Stat_t, name string) FileInfo {
19-
fs := &fileStat{
20-
name: basename(name),
21-
size: int64(st.Size),
22-
modTime: timespecToTime(st.Mtim),
23-
sys: st,
24-
}
25-
fs.mode = FileMode(st.Mode & 0777)
26-
switch st.Mode & syscall.S_IFMT {
12+
func fillFileStatFromSys(fs *fileStat, name string) {
13+
fs.name = basename(name)
14+
fs.size = int64(fs.sys.Size)
15+
fs.modTime = timespecToTime(fs.sys.Mtim)
16+
fs.mode = FileMode(fs.sys.Mode & 0777)
17+
switch fs.sys.Mode & syscall.S_IFMT {
2718
case syscall.S_IFBLK:
2819
fs.mode |= ModeDevice
2920
case syscall.S_IFCHR:
@@ -39,16 +30,15 @@ func fileInfoFromStat(st *syscall.Stat_t, name string) FileInfo {
3930
case syscall.S_IFSOCK:
4031
fs.mode |= ModeSocket
4132
}
42-
if st.Mode&syscall.S_ISGID != 0 {
33+
if fs.sys.Mode&syscall.S_ISGID != 0 {
4334
fs.mode |= ModeSetgid
4435
}
45-
if st.Mode&syscall.S_ISUID != 0 {
36+
if fs.sys.Mode&syscall.S_ISUID != 0 {
4637
fs.mode |= ModeSetuid
4738
}
48-
if st.Mode&syscall.S_ISVTX != 0 {
39+
if fs.sys.Mode&syscall.S_ISVTX != 0 {
4940
fs.mode |= ModeSticky
5041
}
51-
return fs
5242
}
5343

5444
func timespecToTime(ts syscall.Timespec) time.Time {

‎src/os/stat_freebsd.go‎

Lines changed: 9 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9,21 +9,12 @@ import (
99
"time"
1010
)
1111

12-
func sameFile(fs1, fs2 *fileStat) bool {
13-
stat1 := fs1.sys.(*syscall.Stat_t)
14-
stat2 := fs2.sys.(*syscall.Stat_t)
15-
return stat1.Dev == stat2.Dev && stat1.Ino == stat2.Ino
16-
}
17-
18-
func fileInfoFromStat(st *syscall.Stat_t, name string) FileInfo {
19-
fs := &fileStat{
20-
name: basename(name),
21-
size: int64(st.Size),
22-
modTime: timespecToTime(st.Mtimespec),
23-
sys: st,
24-
}
25-
fs.mode = FileMode(st.Mode & 0777)
26-
switch st.Mode & syscall.S_IFMT {
12+
func fillFileStatFromSys(fs *fileStat, name string) {
13+
fs.name = basename(name)
14+
fs.size = int64(fs.sys.Size)
15+
fs.modTime = timespecToTime(fs.sys.Mtimespec)
16+
fs.mode = FileMode(fs.sys.Mode & 0777)
17+
switch fs.sys.Mode & syscall.S_IFMT {
2718
case syscall.S_IFBLK:
2819
fs.mode |= ModeDevice
2920
case syscall.S_IFCHR:
@@ -39,16 +30,15 @@ func fileInfoFromStat(st *syscall.Stat_t, name string) FileInfo {
3930
case syscall.S_IFSOCK:
4031
fs.mode |= ModeSocket
4132
}
42-
if st.Mode&syscall.S_ISGID != 0 {
33+
if fs.sys.Mode&syscall.S_ISGID != 0 {
4334
fs.mode |= ModeSetgid
4435
}
45-
if st.Mode&syscall.S_ISUID != 0 {
36+
if fs.sys.Mode&syscall.S_ISUID != 0 {
4637
fs.mode |= ModeSetuid
4738
}
48-
if st.Mode&syscall.S_ISVTX != 0 {
39+
if fs.sys.Mode&syscall.S_ISVTX != 0 {
4940
fs.mode |= ModeSticky
5041
}
51-
return fs
5242
}
5343

5444
func timespecToTime(ts syscall.Timespec) time.Time {

‎src/os/stat_linux.go‎

Lines changed: 9 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9,21 +9,12 @@ import (
99
"time"
1010
)
1111

12-
func sameFile(fs1, fs2 *fileStat) bool {
13-
stat1 := fs1.sys.(*syscall.Stat_t)
14-
stat2 := fs2.sys.(*syscall.Stat_t)
15-
return stat1.Dev == stat2.Dev && stat1.Ino == stat2.Ino
16-
}
17-
18-
func fileInfoFromStat(st *syscall.Stat_t, name string) FileInfo {
19-
fs := &fileStat{
20-
name: basename(name),
21-
size: int64(st.Size),
22-
modTime: timespecToTime(st.Mtim),
23-
sys: st,
24-
}
25-
fs.mode = FileMode(st.Mode & 0777)
26-
switch st.Mode & syscall.S_IFMT {
12+
func fillFileStatFromSys(fs *fileStat, name string) {
13+
fs.name = basename(name)
14+
fs.size = int64(fs.sys.Size)
15+
fs.modTime = timespecToTime(fs.sys.Mtim)
16+
fs.mode = FileMode(fs.sys.Mode & 0777)
17+
switch fs.sys.Mode & syscall.S_IFMT {
2718
case syscall.S_IFBLK:
2819
fs.mode |= ModeDevice
2920
case syscall.S_IFCHR:
@@ -39,16 +30,15 @@ func fileInfoFromStat(st *syscall.Stat_t, name string) FileInfo {
3930
case syscall.S_IFSOCK:
4031
fs.mode |= ModeSocket
4132
}
42-
if st.Mode&syscall.S_ISGID != 0 {
33+
if fs.sys.Mode&syscall.S_ISGID != 0 {
4334
fs.mode |= ModeSetgid
4435
}
45-
if st.Mode&syscall.S_ISUID != 0 {
36+
if fs.sys.Mode&syscall.S_ISUID != 0 {
4637
fs.mode |= ModeSetuid
4738
}
48-
if st.Mode&syscall.S_ISVTX != 0 {
39+
if fs.sys.Mode&syscall.S_ISVTX != 0 {
4940
fs.mode |= ModeSticky
5041
}
51-
return fs
5242
}
5343

5444
func timespecToTime(ts syscall.Timespec) time.Time {

‎src/os/stat_nacl.go‎

Lines changed: 9 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9,21 +9,12 @@ import (
99
"time"
1010
)
1111

12-
func sameFile(fs1, fs2 *fileStat) bool {
13-
stat1 := fs1.sys.(*syscall.Stat_t)
14-
stat2 := fs2.sys.(*syscall.Stat_t)
15-
return stat1.Dev == stat2.Dev && stat1.Ino == stat2.Ino
16-
}
17-
18-
func fileInfoFromStat(st *syscall.Stat_t, name string) FileInfo {
19-
fs := &fileStat{
20-
name: basename(name),
21-
size: int64(st.Size),
22-
modTime: timespecToTime(st.Mtime, st.MtimeNsec),
23-
sys: st,
24-
}
25-
fs.mode = FileMode(st.Mode & 0777)
26-
switch st.Mode & syscall.S_IFMT {
12+
func fillFileStatFromSys(fs *fileStat, name string) {
13+
fs.name = basename(name)
14+
fs.size = int64(fs.sys.Size)
15+
fs.modTime = timespecToTime(fs.sys.Mtime, fs.sys.MtimeNsec)
16+
fs.mode = FileMode(fs.sys.Mode & 0777)
17+
switch fs.sys.Mode & syscall.S_IFMT {
2718
case syscall.S_IFBLK:
2819
fs.mode |= ModeDevice
2920
case syscall.S_IFCHR:
@@ -39,16 +30,15 @@ func fileInfoFromStat(st *syscall.Stat_t, name string) FileInfo {
3930
case syscall.S_IFSOCK:
4031
fs.mode |= ModeSocket
4132
}
42-
if st.Mode&syscall.S_ISGID != 0 {
33+
if fs.sys.Mode&syscall.S_ISGID != 0 {
4334
fs.mode |= ModeSetgid
4435
}
45-
if st.Mode&syscall.S_ISUID != 0 {
36+
if fs.sys.Mode&syscall.S_ISUID != 0 {
4637
fs.mode |= ModeSetuid
4738
}
48-
if st.Mode&syscall.S_ISVTX != 0 {
39+
if fs.sys.Mode&syscall.S_ISVTX != 0 {
4940
fs.mode |= ModeSticky
5041
}
51-
return fs
5242
}
5343

5444
func timespecToTime(sec, nsec int64) time.Time {

‎src/os/stat_netbsd.go‎

Lines changed: 9 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9,21 +9,12 @@ import (
99
"time"
1010
)
1111

12-
func sameFile(fs1, fs2 *fileStat) bool {
13-
stat1 := fs1.sys.(*syscall.Stat_t)
14-
stat2 := fs2.sys.(*syscall.Stat_t)
15-
return stat1.Dev == stat2.Dev && stat1.Ino == stat2.Ino
16-
}
17-
18-
func fileInfoFromStat(st *syscall.Stat_t, name string) FileInfo {
19-
fs := &fileStat{
20-
name: basename(name),
21-
size: int64(st.Size),
22-
modTime: timespecToTime(st.Mtimespec),
23-
sys: st,
24-
}
25-
fs.mode = FileMode(st.Mode & 0777)
26-
switch st.Mode & syscall.S_IFMT {
12+
func fillFileStatFromSys(fs *fileStat, name string) {
13+
fs.name = basename(name)
14+
fs.size = int64(fs.sys.Size)
15+
fs.modTime = timespecToTime(fs.sys.Mtimespec)
16+
fs.mode = FileMode(fs.sys.Mode & 0777)
17+
switch fs.sys.Mode & syscall.S_IFMT {
2718
case syscall.S_IFBLK:
2819
fs.mode |= ModeDevice
2920
case syscall.S_IFCHR:
@@ -39,16 +30,15 @@ func fileInfoFromStat(st *syscall.Stat_t, name string) FileInfo {
3930
case syscall.S_IFSOCK:
4031
fs.mode |= ModeSocket
4132
}
42-
if st.Mode&syscall.S_ISGID != 0 {
33+
if fs.sys.Mode&syscall.S_ISGID != 0 {
4334
fs.mode |= ModeSetgid
4435
}
45-
if st.Mode&syscall.S_ISUID != 0 {
36+
if fs.sys.Mode&syscall.S_ISUID != 0 {
4637
fs.mode |= ModeSetuid
4738
}
48-
if st.Mode&syscall.S_ISVTX != 0 {
39+
if fs.sys.Mode&syscall.S_ISVTX != 0 {
4940
fs.mode |= ModeSticky
5041
}
51-
return fs
5242
}
5343

5444
func timespecToTime(ts syscall.Timespec) time.Time {

0 commit comments

Comments
 (0)