1 /*
2 Copyright (c) 2015-2021 Timur Gafarov
3 
4 Boost Software License - Version 1.0 - August 17th, 2003
5 
6 Permission is hereby granted, free of charge, to any person or organization
7 obtaining a copy of the software and accompanying documentation covered by
8 this license (the "Software") to use, reproduce, display, distribute,
9 execute, and transmit the Software, and to prepare derivative works of the
10 Software, and to permit third-parties to whom the Software is furnished to
11 do so, all subject to the following:
12 
13 The copyright notices in the Software and this entire statement, including
14 the above license grant, this restriction and the following disclaimer,
15 must be included in all copies of the Software, in whole or in part, and
16 all derivative works of the Software, unless such copies or derivative
17 works are solely in the form of machine-executable object code generated by
18 a source language processor.
19 
20 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
23 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
24 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
25 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 DEALINGS IN THE SOFTWARE.
27 */
28 
29 /**
30  * GC-free filesystem
31  * Copyright: Timur Gafarov 2015-2021.
32  * License: $(LINK2 boost.org/LICENSE_1_0.txt, Boost License 1.0).
33  * Authors: Timur Gafarov
34  */
35 module dlib.filesystem.stdfs;
36 
37 import core.stdc.stdio;
38 import std.file;
39 import std..string;
40 import std.datetime;
41 import dlib.core.memory;
42 import dlib.core.stream;
43 import dlib.container.dict;
44 import dlib.container.array;
45 import dlib.text.str;
46 import dlib.filesystem.filesystem;
47 
48 version(Posix)
49 {
50     import dlib.filesystem.posix.common;
51     import dlib.filesystem.stdposixdir;
52 }
53 version(Windows)
54 {
55     import std.stdio;
56     import dlib.filesystem.windows.common;
57     import dlib.filesystem.stdwindowsdir;
58 }
59 
60 import dlib.text.utils;
61 import dlib.text.utf16;
62 
63 // TODO: where are these definitions in druntime?
64 version(Windows)
65 {
66    extern(C) int _wmkdir(const wchar*);
67    extern(C) int _wremove(const wchar*);
68 
69    extern(Windows) int RemoveDirectoryW(const wchar*);
70 }
71 
72 /// InputStream that wraps FILE
73 class StdInFileStream: InputStream
74 {
75     FILE* file;
76     StreamSize _size;
77     bool eof;
78 
79     this(FILE* file)
80     {
81         this.file = file;
82 
83         fseek(file, 0, SEEK_END);
84         _size = ftell(file);
85         fseek(file, 0, SEEK_SET);
86 
87         eof = false;
88     }
89 
90     ~this()
91     {
92         fclose(file);
93     }
94 
95     StreamPos getPosition() @property
96     {
97         return ftell(file);
98     }
99 
100     bool setPosition(StreamPos p)
101     {
102         import core.stdc.config : c_long;
103         return !fseek(file, cast(c_long)p, SEEK_SET);
104     }
105 
106     StreamSize size()
107     {
108         return _size;
109     }
110 
111     void close()
112     {
113         fclose(file);
114     }
115 
116     bool seekable()
117     {
118         return true;
119     }
120 
121     bool readable()
122     {
123         return !eof;
124     }
125 
126     size_t readBytes(void* buffer, size_t count)
127     {
128         auto bytesRead = fread(buffer, 1, count, file);
129         if (count > bytesRead)
130             eof = true;
131         return bytesRead;
132     }
133 }
134 
135 /// OutputStream that wraps FILE
136 class StdOutFileStream: OutputStream
137 {
138     FILE* file;
139     bool _writeable;
140 
141     this(FILE* file)
142     {
143         this.file = file;
144         this._writeable = true;
145     }
146 
147     ~this()
148     {
149         fclose(file);
150     }
151 
152     StreamPos getPosition() @property
153     {
154         return 0;
155     }
156 
157     bool setPosition(StreamPos pos)
158     {
159         return false;
160     }
161 
162     StreamSize size()
163     {
164         return 0;
165     }
166 
167     void close()
168     {
169         fclose(file);
170     }
171 
172     bool seekable()
173     {
174         return false;
175     }
176 
177     void flush()
178     {
179         fflush(file);
180     }
181 
182     bool writeable()
183     {
184         return _writeable;
185     }
186 
187     size_t writeBytes(const void* buffer, size_t count)
188     {
189         size_t res = fwrite(buffer, 1, count, file);
190         if (res != count)
191             _writeable = false;
192         return res;
193     }
194 }
195 
196 /// IOStream that wraps FILE
197 class StdIOStream: IOStream
198 {
199     FILE* file;
200     StreamSize _size;
201     bool _eof;
202     bool _writeable;
203 
204     this(FILE* file)
205     {
206         this.file = file;
207         this._writeable = true;
208 
209         fseek(file, 0, SEEK_END);
210         this._size = ftell(file);
211         fseek(file, 0, SEEK_SET);
212 
213         this._eof = false;
214     }
215 
216     ~this()
217     {
218         fclose(file);
219     }
220 
221     StreamPos getPosition() @property
222     {
223         return ftell(file);
224     }
225 
226     bool setPosition(StreamPos p)
227     {
228         import core.stdc.config : c_long;
229         return !fseek(file, cast(c_long)p, SEEK_SET);
230     }
231 
232     StreamSize size()
233     {
234         return _size;
235     }
236 
237     void close()
238     {
239         fclose(file);
240     }
241 
242     bool seekable()
243     {
244         return true;
245     }
246 
247     bool readable()
248     {
249         return !_eof;
250     }
251 
252     size_t readBytes(void* buffer, size_t count)
253     {
254         auto bytesRead = fread(buffer, 1, count, file);
255         if (count > bytesRead)
256             _eof = true;
257         return bytesRead;
258     }
259 
260     void flush()
261     {
262         fflush(file);
263     }
264 
265     bool writeable()
266     {
267         return _writeable;
268     }
269 
270     size_t writeBytes(const void* buffer, size_t count)
271     {
272         size_t res = fwrite(buffer, 1, count, file);
273         if (res != count)
274             _writeable = false;
275         return res;
276     }
277 }
278 
279 /// FileSystem that wraps libc filesystem functions + some Posix and WinAPI parts for additional functionality
280 class StdFileSystem: FileSystem
281 {
282     Dict!(Directory, string) openedDirs;
283     Array!string openedDirPaths;
284 
285     this()
286     {
287         openedDirs = New!(Dict!(Directory, string));
288     }
289 
290     ~this()
291     {
292         foreach(k, v; openedDirs)
293             Delete(v);
294         Delete(openedDirs);
295 
296         foreach(p; openedDirPaths)
297             Delete(p);
298         openedDirPaths.free();
299     }
300 
301     bool stat(string filename, out FileStat stat)
302     {
303         if (std.file.exists(filename))
304         {
305             with(stat)
306             {
307                 version (Posix)
308                 {
309                     stat_t st;
310                     String filenamez = String(filename);
311                     stat_(filenamez.ptr, &st);
312                     filenamez.free();
313 
314                     isFile = S_ISREG(st.st_mode);
315                     isDirectory = S_ISDIR(st.st_mode);
316                     sizeInBytes = st.st_size;
317                     creationTimestamp = SysTime(unixTimeToStdTime(st.st_ctime));
318                     auto modificationStdTime = unixTimeToStdTime(st.st_mtime);
319                     static if (is(typeof(st.st_mtimensec)))
320                     {
321                         modificationStdTime += st.st_mtimensec / 100;
322                     }
323                     modificationTimestamp = SysTime(modificationStdTime);
324 
325                     if ((st.st_mode & S_IRUSR) | (st.st_mode & S_IRGRP) | (st.st_mode & S_IROTH))
326                         permissions |= PRead;
327                     if ((st.st_mode & S_IWUSR) | (st.st_mode & S_IWGRP) | (st.st_mode & S_IWOTH))
328                         permissions |= PWrite;
329                     if ((st.st_mode & S_IXUSR) | (st.st_mode & S_IXGRP) | (st.st_mode & S_IXOTH))
330                         permissions |= PExecute;
331                 }
332                 else version(Windows)
333                 {
334                     wchar[] filename_utf16 = convertUTF8toUTF16(filename, true);
335 
336                     WIN32_FILE_ATTRIBUTE_DATA data;
337 
338                     if (!GetFileAttributesExW(filename_utf16.ptr, GET_FILEEX_INFO_LEVELS.GetFileExInfoStandard, &data))
339                         return false;
340 
341                     if (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
342                         isDirectory = true;
343                     else
344                         isFile = true;
345 
346                     sizeInBytes = (cast(FileSize) data.nFileSizeHigh << 32) | data.nFileSizeLow;
347                     creationTimestamp = SysTime(FILETIMEToStdTime(&data.ftCreationTime));
348                     modificationTimestamp = SysTime(FILETIMEToStdTime(&data.ftLastWriteTime));
349 
350                     permissions = 0;
351 
352                     PACL pacl;
353                     PSECURITY_DESCRIPTOR secDesc;
354                     TRUSTEE_W trustee;
355                     trustee.pMultipleTrustee = null;
356                     trustee.MultipleTrusteeOperation = MULTIPLE_TRUSTEE_OPERATION.NO_MULTIPLE_TRUSTEE;
357                     trustee.TrusteeForm = TRUSTEE_FORM.TRUSTEE_IS_NAME;
358                     trustee.TrusteeType = TRUSTEE_TYPE.TRUSTEE_IS_UNKNOWN;
359                     trustee.ptstrName = cast(wchar*)"CURRENT_USER"w.ptr;
360                     GetNamedSecurityInfoW(filename_utf16.ptr, SE_OBJECT_TYPE.SE_FILE_OBJECT, DACL_SECURITY_INFORMATION, null, null, &pacl, null, &secDesc);
361                     if (pacl)
362                     {
363                         uint access;
364                         GetEffectiveRightsFromAcl(pacl, &trustee, &access);
365 
366                         if (access & ACTRL_FILE_READ)
367                             permissions |= PRead;
368                         if ((access & ACTRL_FILE_WRITE) && !(data.dwFileAttributes & FILE_ATTRIBUTE_READONLY))
369                             permissions |= PWrite;
370                         if (access & ACTRL_FILE_EXECUTE)
371                             permissions |= PExecute;
372                     }
373 
374                     Delete(filename_utf16);
375                 }
376                 else
377                 {
378                     isFile = std.file.isFile(filename);
379                     isDirectory = std.file.isDir(filename);
380                     sizeInBytes = std.file.getSize(filename);
381                     getTimes(filename,
382                         modificationTimestamp,
383                         modificationTimestamp);
384                 }
385             }
386             return true;
387         }
388         else
389             return false;
390     }
391 
392     StdInFileStream openForInput(string filename)
393     {
394         version(Posix)
395         {
396             String filenamez = String(filename);
397             FILE* file = fopen(filenamez.ptr, "rb");
398             filenamez.free();
399         }
400         version(Windows)
401         {
402             wchar[] filename_utf16 = convertUTF8toUTF16(filename, true);
403             wchar[] mode_utf16 = convertUTF8toUTF16("rb", true);
404             FILE* file = _wfopen(filename_utf16.ptr, mode_utf16.ptr);
405             Delete(filename_utf16);
406             Delete(mode_utf16);
407         }
408         return New!StdInFileStream(file);
409     }
410 
411     StdOutFileStream openForOutput(string filename, uint creationFlags = FileSystem.create)
412     {
413         version(Posix)
414         {
415             String filenamez = String(filename);
416             FILE* file = fopen(filenamez.ptr, "wb");
417             filenamez.free();
418         }
419         version(Windows)
420         {
421             wchar[] filename_utf16 = convertUTF8toUTF16(filename, true);
422             wchar[] mode_utf16 = convertUTF8toUTF16("wb", true);
423             FILE* file = _wfopen(filename_utf16.ptr, mode_utf16.ptr);
424             Delete(filename_utf16);
425             Delete(mode_utf16);
426         }
427         return New!StdOutFileStream(file);
428     }
429 
430     StdIOStream openForIO(string filename, uint creationFlags = FileSystem.create)
431     {
432         version(Posix)
433         {
434             String filenamez = String(filename);
435             FILE* file = fopen(filename.ptr, "rb+");
436             filenamez.free();
437         }
438         version(Windows)
439         {
440             wchar[] filename_utf16 = convertUTF8toUTF16(filename, true);
441             wchar[] mode_utf16 = convertUTF8toUTF16("rb+", true);
442             FILE* file = _wfopen(filename_utf16.ptr, mode_utf16.ptr);
443             Delete(filename_utf16);
444             Delete(mode_utf16);
445         }
446         return New!StdIOStream(file);
447     }
448 
449     Directory openDir(string path)
450     {
451         if (path in openedDirs)
452         {
453             Directory d = openedDirs[path];
454             Delete(d);
455         }
456 
457         Directory dir;
458 
459         version(Posix)
460         {
461             dir = New!StdPosixDirectory(path);
462         }
463         version(Windows)
464         {
465             string s = catStr(path, "\\*.*");
466             wchar[] ws = convertUTF8toUTF16(s, true);
467             Delete(s);
468             dir = New!StdWindowsDirectory(ws.ptr);
469         }
470 
471         auto p = New!(char[])(path.length);
472         p[] = path[];
473         openedDirPaths.append(cast(string)p);
474         openedDirs[cast(string)p] = dir;
475         return dir;
476     }
477 
478     bool createDir(string path, bool recursive = true)
479     {
480         version(Posix)
481         {
482             String pathz = String(path);
483             int res = mkdir(pathz.ptr, 777);
484             pathz.free();
485             return (res == 0);
486         }
487         version(Windows)
488         {
489             wchar[] wp = convertUTF8toUTF16(path, true);
490             int res = _wmkdir(wp.ptr);
491             Delete(wp);
492             return (res == 0);
493         }
494     }
495 
496     bool remove(string path, bool recursive = true)
497     {
498         version(Posix)
499         {
500             String pathz = String(path);
501             int res = core.stdc.stdio.remove(pathz.ptr);
502             pathz.free();
503             return (res == 0);
504         }
505         version(Windows)
506         {
507             import std.stdio;
508             bool res;
509             if (std.file.isDir(path))
510             {
511                 if (recursive)
512                 foreach(e; openDir(path).contents)
513                 {
514                     string path2 = catStr(path, "\\");
515                     string path3 = catStr(path2, e.name);
516                     Delete(path2);
517                     writeln(path3);
518                     this.remove(path3, recursive);
519                     Delete(path3);
520                 }
521 
522                 wchar[] wp = convertUTF8toUTF16(path, true);
523                 res = RemoveDirectoryW(wp.ptr) != 0;
524                 Delete(wp);
525             }
526             else
527             {
528                 wchar[] wp = convertUTF8toUTF16(path, true);
529                 res = _wremove(wp.ptr) == 0;
530                 Delete(wp);
531             }
532             return res;
533         }
534     }
535 }
536 
537 /// Reads string from InputStream and stores it in unmanaged memory
538 string readText(InputStream istrm)
539 {
540     ubyte[] arr = New!(ubyte[])(cast(size_t)istrm.size);
541     istrm.fillArray(arr);
542     istrm.setPosition(0);
543     return cast(string)arr;
544 }
545 
546 /// Reads struct from InputStream
547 T readStruct(T)(InputStream istrm) if (is(T == struct))
548 {
549     T res;
550     istrm.readBytes(&res, T.sizeof);
551     return res;
552 }
553 
554 enum MAX_PATH_LEN = 4096;
555 
556 struct PathBuilder
557 {
558     // TODO: use dlib.text.unmanaged.String instead
559     char[MAX_PATH_LEN] str;
560     uint length = 0;
561 
562     void append(string s)
563     {
564         if (length && str[length-1] != '/')
565         {
566             str[length] = '/';
567             length++;
568         }
569 
570         str[length..length+s.length] = s[];
571         length += s.length;
572     }
573 
574     string toString() return
575     {
576         if (length)
577             return cast(string)(str[0..length]);
578         else
579             return "";
580     }
581 }
582 
583 struct RecursiveFileIterator
584 {
585     PathBuilder pb;
586     ReadOnlyFileSystem rofs;
587     string directory;
588     bool rec;
589 
590     this(ReadOnlyFileSystem fs, string dir, bool recursive)
591     {
592         rofs = fs;
593         directory = dir;
594         pb.append(dir);
595         rec = recursive;
596     }
597 
598     int opApply(scope int delegate(string path, ref dlib.filesystem.filesystem.DirEntry) dg)
599     {
600         int result = 0;
601 
602         if (!rofs)
603             return 0;
604 
605         foreach(e; rofs.openDir(directory).contents)
606         {
607             uint pathPos = pb.length;
608             pb.append(e.name);
609 
610             string oldPath = directory;
611             directory = pb.toString;
612 
613             result = dg(directory, e);
614             if (result)
615                 break;
616 
617             if (e.isDirectory && rec)
618                 result = opApply(dg);
619 
620             directory = oldPath;
621             pb.length = pathPos;
622 
623             if (result)
624                 break;
625         }
626 
627         return 0;
628     }
629 
630     int opApply(scope int delegate(ref dlib.filesystem.filesystem.DirEntry) dg)
631     {
632         int result = 0;
633 
634         auto dir = rofs.openDir(directory);
635 
636         foreach(e; dir.contents)
637         {
638             uint pathPos = pb.length;
639             pb.append(e.name);
640 
641             string oldPath = directory;
642             directory = pb.toString;
643 
644             result = dg(e);
645             if (result)
646                 break;
647 
648             if (e.isDirectory)
649                 result = opApply(dg);
650 
651             directory = oldPath;
652             pb.length = pathPos;
653 
654             if (result)
655                 break;
656         }
657 
658         return 0;
659     }
660 }
661 
662 /// Enumerate directory contents, optionally recursive
663 RecursiveFileIterator traverseDir(ReadOnlyFileSystem rofs, string baseDir, bool recursive)
664 {
665     FileStat s;
666     if (!rofs.stat(baseDir, s))
667         return RecursiveFileIterator(null, baseDir, recursive);
668     else
669         return RecursiveFileIterator(rofs, baseDir, recursive);
670 }