阿里企业网站建设评估商丘市网站建设推广
原生标准库在文件量过大时效率和内存均表现不好
1400万文件遍历Filepath.Walk
1400万文件重写直接调用windows api并处理细节
结论
1400万文件遍历时对比
| 对比条目 | filepath.walk | windows api并触发黑科技 |
|---|---|---|
| 运行时间 | 710秒 | 22秒 |
| 内存占用 | 480M | 38M |
关键代码
//超级快的文件遍历
func FindFileWin(dir string, callbackfunc MyFindFileCallBack) {dir = dir + `\`finstruct := win.WIN32_FIND_DATAW{}handle := win.FindFirstFileW(dir+`*`, &finstruct)if win.IsInvalidHandle(handle) {for {//文件夾if (finstruct.DwFileAttributes & win.FILE_ATTRIBUTE_DIRECTORY) != 0 {if (finstruct.CfileNameGo != "..") && (finstruct.CfileNameGo != ".") {FindFileWin(dir+finstruct.CfileNameGo, callbackfunc)}} else {callbackfunc(dir + finstruct.CfileNameGo)}if win.FindNextFileW(handle, &finstruct) == 0 {break}}}
}var (kernel32 = syscall.NewLazyDLL("Kernel32.dll")procCreateFileW = kernel32.NewProc("CreateFileW")procOpenEventW = kernel32.NewProc("OpenEventW")procSetEvent = kernel32.NewProc("SetEvent")procFindFirstFileW = kernel32.NewProc("FindFirstFileW")procFindNextFileW = kernel32.NewProc("FindNextFileW")//procCreateFileA = kernel32.NewProc("CreateFileA")
)func FindFirstFileW(fileName string, lpFindFileData *WIN32_FIND_DATAW) HANDLE {strname := unsafe.Pointer(syscall.StringToUTF16Ptr(fileName))handle, _, _ := procFindFirstFileW.Call(uintptr(strname),uintptr(unsafe.Pointer(lpFindFileData)),)if handle != 0 {lpFindFileData.CfileNameGo = syscall.UTF16ToString(lpFindFileData.cFileName[:])}return HANDLE(handle)
}
func IsInvalidHandle(handle HANDLE) bool {if handle != 0 && int(handle) != -1 {return true}return false
}
func FindNextFileW(hFindFile HANDLE, lpFindFileData *WIN32_FIND_DATAW) BOOL {ret, _, _ := procFindNextFileW.Call(uintptr(hFindFile),uintptr(unsafe.Pointer(lpFindFileData)),)if ret != 0 {lpFindFileData.CfileNameGo = syscall.UTF16ToString(lpFindFileData.cFileName[:])}return BOOL(ret)
}