Skip to content

Commit 4199bad

Browse files
mareclmarecl
committed
updated dumps
Testing consistency of read/getdirentries Normal errors Testing random reads against single dump (reverse lookup) (PFS WIP)
1 parent a562893 commit 4199bad

File tree

7 files changed

+238
-84
lines changed

7 files changed

+238
-84
lines changed

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,5 @@ build/
88
!.github
99
!.vscode
1010
.DS_Store
11-
.venv
11+
.venv
12+
__pycache__

‎tests/code/filesystem_dirents/code/fs_test.cpp‎

Lines changed: 177 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,6 @@
1111
#include <string>
1212
#include <vector>
1313

14-
std::vector<u32> read_sizes {8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072};
15-
std::vector<u16> read_offsets {0};
16-
1714
const char* clone_source_app0 = "/app0/assets/misc";
1815
const char* clone_destination_read = "/data/enderman/clone_read";
1916
const char* clone_destination_getdirentries = "/data/enderman/clone_getdents";
@@ -33,7 +30,6 @@ bool PrepareTests() {
3330

3431
RegenerateDir("/data/enderman");
3532
sceKernelMkdir(clone_dir.c_str(), 0777);
36-
sceKernelWrite(1, "!@#!@#!WD", 8);
3733

3834
for (auto& dent: fs::directory_iterator("/app0/assets/misc")) {
3935
target = clone_dir / dent.path().filename();
@@ -49,29 +45,16 @@ bool PrepareTests() {
4945
return false;
5046
}
5147
return true;
48+
}
5249

53-
// Log("---------------------");
54-
// Log("Dump normal directory");
55-
// Log("---------------------");
56-
57-
// int fd = sceKernelOpen("/data/enderman", O_DIRECTORY | O_RDONLY, 0777);
58-
// for (auto read_size: read_sizes) {
59-
// for (auto read_offset: read_offsets) {
60-
// DumpDirectory(fd, read_size, read_offset);
61-
// }
62-
// }
63-
// sceKernelClose(fd);
64-
65-
// Log("------------------");
66-
// Log("Dump PFS directory");
67-
// Log("------------------");
68-
// fd = sceKernelOpen("/app0/assets/misc", O_DIRECTORY | O_RDONLY, 0777);
69-
// for (auto read_size: read_sizes) {
70-
// for (auto read_offset: read_offsets) {
71-
// DumpDirectory(fd, read_size, read_offset, true);
72-
// }
73-
// }
74-
// sceKernelClose(fd);
50+
s64 undump_file(const char* path, char* data, u64 length) {
51+
int fd = sceKernelOpen(path, O_RDONLY, 0777);
52+
if (fd < 0) return fd;
53+
memset(data, 'A', length);
54+
int tbr = sceKernelRead(fd, data, length);
55+
if (tbr < 0) return tbr;
56+
if (auto res = sceKernelClose(fd); res < 0) return res;
57+
return tbr;
7558
}
7659

7760
TEST_GROUP (DirentTests) {
@@ -88,7 +71,10 @@ TEST_GROUP (DirentTests) {
8871
int fd;
8972
s64 tbr;
9073

91-
void setup() {}
74+
void setup() {
75+
fd = -1;
76+
tbr = 0;
77+
}
9278
void teardown() {
9379
sceKernelClose(fd);
9480
fd = -1;
@@ -102,21 +88,18 @@ TEST(DirentTests, PFSGetdirentries) {
10288
char buffer[65536];
10389
int result_cast {};
10490

105-
auto pattern = [](s64 size, s64 offset) -> fs::path { return "pfs_getdirentries_fail_o" + std::to_string(offset) + "_s" + std::to_string(size) + ".bin"; };
106-
107-
RegenerateDir(output_root.c_str());
108-
10991
fd = sceKernelOpen(input_pfs, O_DIRECTORY, 0777);
11092
s64 basep {};
11193
for (auto& spec: pfs_dirent_variants) {
11294
basep = 0;
11395
memset(buffer, 0xAA, 65536);
11496
result_cast = int(spec.expected_result);
97+
11598
if (spec.read_offset >= 0) CHECK_EQUAL(spec.read_offset, sceKernelLseek(fd, spec.read_offset, 0));
11699
errno = 0;
117100
tbr = sceKernelGetdirentries(fd, buffer, spec.read_size, &basep);
118101
LogTest(spec.read_size, spec.read_offset, spec.expected_basep, result_cast, spec.expected_errno, "\t->\t", basep, tbr, errno, "\t",
119-
to_hex_string(buffer,16,""));
102+
to_hex_string(buffer, 16, ""));
120103
if (tbr < 0) {
121104
CHECK_EQUAL(result_cast, tbr);
122105
} else {
@@ -132,25 +115,21 @@ TEST(DirentTests, PFSGetdirentries) {
132115
TEST(DirentTests, NormalGetdirentries) {
133116
LogTest("<<<< Normal getdirentries tests >>>>");
134117

135-
fs::path output_root = "/data/enderman/normal_getdirentries";
136-
char buffer[65536];
137-
int result_cast {};
138-
139-
auto pattern = [](s64 size, s64 offset) -> fs::path { return "normal_getdirentries_fail_o" + std::to_string(offset) + "_s" + std::to_string(size) + ".bin"; };
140-
141-
RegenerateDir(output_root.c_str());
118+
char buffer[65536];
119+
int result_cast {};
142120

143121
fd = sceKernelOpen(input_normal, O_DIRECTORY, 0777);
144122
s64 basep {};
145123
for (auto& spec: normal_dirent_variants) {
146124
basep = 0;
147125
memset(buffer, 0xAA, 65536);
148126
result_cast = int(spec.expected_result);
127+
149128
if (spec.read_offset >= 0) CHECK_EQUAL(spec.read_offset, sceKernelLseek(fd, spec.read_offset, 0));
150129
errno = 0;
151130
tbr = sceKernelGetdirentries(fd, buffer, spec.read_size, &basep);
152131
LogTest(spec.read_size, spec.read_offset, spec.expected_basep, result_cast, spec.expected_errno, "\t->\t", basep, tbr, errno,
153-
to_hex_string(buffer,16,""));
132+
to_hex_string(buffer, 16, ""));
154133
if (tbr < 0) {
155134
CHECK_EQUAL(result_cast, tbr);
156135
} else {
@@ -163,56 +142,112 @@ TEST(DirentTests, NormalGetdirentries) {
163142
sceKernelClose(fd);
164143
}
165144

166-
s64 qqq(const void* master, const void* test, s64 buffer_size, s64 tbr, struct oi::DirentCombinationRead* spec) {
145+
s64 compare_data_dump(const void* master, const void* test, s64 buffer_size, s64 tbr, struct oi::DirentCombinationRead* spec) {
167146
const char* master_data = reinterpret_cast<const char*>(master) + spec->read_offset;
168147
const char* test_data = reinterpret_cast<const char*>(test);
169148

170149
if (auto qw = imemcmp(master_data, test_data, tbr); qw != -1) {
171-
LogError("Incorrect read at", spec->read_offset + qw);
172-
LogError("Global dump:", to_hex_string(master_data + qw, std::min((s64)32, tbr)));
173-
LogError("Recent dump:", to_hex_string(test_data + qw, std::min((s64)32, tbr)));
150+
LogError("Inconsistent read at", spec->read_offset + qw);
151+
LogError("Global dump:", to_hex_string(master_data + qw, std::min((s64)48, tbr)));
152+
LogError("Recent dump:", to_hex_string(test_data + qw, std::min((s64)48, tbr)));
174153

175154
return -1;
176155
}
177156
return 0;
178157
}
179158

159+
TEST(DirentTests, PFSRead) {
160+
LogTest("<<<< PFS read tests >>>>");
161+
162+
char buffer[65536];
163+
char master_buffer[65536];
164+
const s64 master_length = undump_file(output_pfs_read, master_buffer, 65536);
165+
s64 expected_length_adj {};
166+
167+
LogTest("Master PFS read length is", master_length);
168+
169+
fd = sceKernelOpen(input_pfs, O_DIRECTORY, 0777);
170+
for (auto& spec: normal_read_variants) {
171+
memset(buffer, 'A', 65536);
172+
173+
CHECK_EQUAL(spec.read_offset, sceKernelLseek(fd, spec.read_offset, 0));
174+
errno = 0;
175+
tbr = sceKernelRead(fd, buffer, spec.read_size);
176+
177+
expected_length_adj = (master_length - spec.read_offset) > 0 ? master_length - spec.read_offset : 0;
178+
expected_length_adj = std::min(expected_length_adj, spec.read_size);
179+
180+
LogTest(spec.read_size, spec.read_offset, int(spec.expected_result), spec.expected_errno, "\t->\t", tbr, errno, expected_length_adj, "\t",
181+
to_hex_string(buffer, 16, ""));
182+
CHECK_EQUAL(expected_length_adj, tbr);
183+
CHECK_EQUAL(spec.expected_errno, errno);
184+
compare_data_dump(master_buffer, buffer, 65536, tbr, &spec);
185+
// dump good ones to file
186+
}
187+
sceKernelClose(fd);
188+
}
189+
180190
TEST(DirentTests, NormalRead) {
181191
LogTest("<<<< Normal read tests >>>>");
182192

183-
fs::path output_root = "/data/enderman/normal_read";
184-
char buffer[65536];
185-
int result_cast {};
186-
187-
auto pattern = [](s64 size, s64 offset) -> fs::path { return "normal_read_fail_o" + std::to_string(offset) + "_s" + std::to_string(size) + ".bin"; };
193+
char buffer[65536];
194+
char master_buffer[65536];
195+
const s64 master_length = undump_file(output_normal_read, master_buffer, 65536);
196+
s64 expected_length_adj {};
188197

189-
RegenerateDir(output_root.c_str());
198+
LogTest("Master PFS read length is", master_length);
190199

191200
fd = sceKernelOpen(input_normal, O_DIRECTORY, 0777);
192201
for (auto& spec: normal_read_variants) {
193-
result_cast = int(spec.expected_result);
194202
memset(buffer, 'A', 65536);
195203

196204
CHECK_EQUAL(spec.read_offset, sceKernelLseek(fd, spec.read_offset, 0));
197205
errno = 0;
198206
tbr = sceKernelRead(fd, buffer, spec.read_size);
199207

200-
LogTest(spec.read_size, spec.read_offset, result_cast, spec.expected_errno, "\t->\t", tbr, errno, "\t", to_hex_string(buffer,16,""));
208+
expected_length_adj = (master_length - spec.read_offset) > 0 ? master_length - spec.read_offset : 0;
209+
expected_length_adj = std::min(expected_length_adj, spec.read_size);
201210

202-
{
203-
int f = sceKernelOpen(output_normal_read, O_RDONLY, 0777);
204-
char b[65536] {'A'};
205-
sceKernelRead(f, b, 65536);
206-
sceKernelClose(f);
207-
qqq(b, buffer, 65536, tbr, &spec);
208-
}
209-
CHECK_EQUAL(spec.expected_result, tbr);
211+
LogTest(spec.read_size, spec.read_offset, int(spec.expected_result), spec.expected_errno, "\t->\t", tbr, errno, expected_length_adj, "\t",
212+
to_hex_string(buffer, 16, ""));
213+
CHECK_EQUAL(expected_length_adj, tbr);
210214
CHECK_EQUAL(spec.expected_errno, errno);
215+
compare_data_dump(master_buffer, buffer, 65536, tbr, &spec);
211216
// dump good ones to file
212217
}
213218
sceKernelClose(fd);
214219
}
215220

221+
TEST(DirentTests, NormalGetdirentriesErrors) {
222+
LogTest("<<<< Normal getdirentries test illegal reads >>>>");
223+
LogTest("Test reads that do not break 512b alignment");
224+
225+
char buffer[1024];
226+
int result_cast {};
227+
s64 spec_read_size {};
228+
s64 spec_offset_base {};
229+
s64 spec_offset {};
230+
s64 basep {};
231+
memset(buffer, 0xAA, 1024);
232+
233+
fd = sceKernelOpen(input_normal, O_DIRECTORY, 0777);
234+
235+
for (spec_offset_base = 0; spec_offset_base < 65536; spec_offset_base += 512) {
236+
for (spec_read_size = 0; spec_read_size < 512; ++spec_read_size) {
237+
basep = 0;
238+
spec_offset = spec_offset_base + 511 - spec_read_size;
239+
CHECK_EQUAL(spec_offset, sceKernelLseek(fd, spec_offset, 0));
240+
errno = 0;
241+
tbr = sceKernelGetdirentries(fd, buffer, spec_read_size, &basep);
242+
CHECK_EQUAL(0xAAAAAAAAAAAAAAAA, *reinterpret_cast<u64*>(buffer));
243+
CHECK_EQUAL(ORBIS_KERNEL_ERROR_EINVAL, int(tbr));
244+
CHECK_EQUAL(EINVAL, errno);
245+
CHECK_EQUAL(0, basep);
246+
}
247+
}
248+
sceKernelClose(fd);
249+
}
250+
216251
TEST(DirentTests, PFSGetdirentriesErrors) {
217252
LogTest("<<<< PFS getdirentries test illegal reads >>>>");
218253
LogTest("Test reads that do not break 512b alignment");
@@ -243,6 +278,75 @@ TEST(DirentTests, PFSGetdirentriesErrors) {
243278
sceKernelClose(fd);
244279
}
245280

281+
s64 NormalComparator(const char* read, const char* getdirentries, u64 length) {
282+
s64 offset {0};
283+
while (offset < length) {
284+
const oi::FolderDirent* dirent_read = reinterpret_cast<const oi::FolderDirent*>(read + offset);
285+
const oi::FolderDirent* dirent_getdirentries = reinterpret_cast<const oi::FolderDirent*>(getdirentries + offset);
286+
287+
if (dirent_read->d_namlen != dirent_getdirentries->d_namlen) break;
288+
if (dirent_read->d_reclen != dirent_getdirentries->d_reclen) break;
289+
if (dirent_read->d_type != dirent_getdirentries->d_type) break;
290+
if (memcmp(dirent_read->d_name, dirent_getdirentries->d_name, dirent_read->d_namlen)) break; // namlen is the same
291+
offset += dirent_read->d_reclen;
292+
if (dirent_read->d_reclen == 0) break;
293+
}
294+
return offset;
295+
}
296+
297+
TEST(DirentTests, Normal_Consistency) {
298+
LogTest("<<<< Normal read and getdirentries consistency >>>>");
299+
300+
char master_read[65536] {'A'};
301+
char master_getdirentries[65536] {'A'};
302+
303+
auto master_read_size = undump_file(output_normal_read, master_read, 65536);
304+
auto master_getdirentries_size = undump_file(output_normal_getdirentries, master_getdirentries, 65536);
305+
306+
if (auto res = NormalComparator(master_read, master_getdirentries, master_read_size); master_getdirentries_size != res) {
307+
LogError("Normal read and getdirentries have a different dirent at", res, ":");
308+
LogError("Normal Read:\t ", to_hex_string(master_read + res, 24));
309+
LogError("Normal Getdirentries: ", to_hex_string(master_read + res, 24));
310+
FAIL("Normal read and getdirentries returned different amount of data");
311+
}
312+
}
313+
314+
s64 PFSComparator(const char* read, const char* getdirentries, u64 length) {
315+
316+
s64 offset {0};
317+
while (offset < length) {
318+
const oi::PfsDirent* dirent_read = reinterpret_cast<const oi::PfsDirent*>(read + offset);
319+
const oi::FolderDirent* dirent_getdirentries = reinterpret_cast<const oi::FolderDirent*>(getdirentries + offset);
320+
321+
if (dirent_read->d_namlen != dirent_getdirentries->d_namlen) break;
322+
if (dirent_read->d_reclen != dirent_getdirentries->d_reclen) break;
323+
// if (dirent_read->d_type != dirent_getdirentries->d_type) break;
324+
if (memcmp(dirent_read->d_name, dirent_getdirentries->d_name, dirent_read->d_namlen)) break; // namlen is the same
325+
if (dirent_read->d_reclen == 0) break; // reclen is the same
326+
offset += dirent_read->d_reclen;
327+
if (dirent_read->d_reclen == 0) break;
328+
}
329+
return offset;
330+
}
331+
332+
TEST(DirentTests, PFS_Consistency) {
333+
LogTest("<<<< PFS read and getdirentries consistency >>>>");
334+
LogWarning("This function does not translate d_type between PFS and user filesystems");
335+
336+
char master_read[65536] {'A'};
337+
char master_getdirentries[65536] {'A'};
338+
339+
auto master_read_size = undump_file(output_pfs_read, master_read, 65536);
340+
auto master_getdirentries_size = undump_file(output_pfs_getdirentries, master_getdirentries, 65536);
341+
342+
if (auto res = PFSComparator(master_read, master_getdirentries, 65536); master_getdirentries_size != res) {
343+
LogError("PFS read and getdirentries have a different dirent at", res, ":");
344+
LogError("PFS Read:\t ", to_hex_string(master_read + res, 24));
345+
LogError("PFS Getdirentries: ", to_hex_string(master_read + res, 24));
346+
FAIL("PFS read and getdirentries returned different amount of data");
347+
}
348+
}
349+
246350
TEST(DirentTests, DumpEverythingRaw) {
247351
LogTest("<<<< Dump everything >>>>");
248352

@@ -265,8 +369,10 @@ TEST(DirentTests, DumpEverythingRaw) {
265369
memset(buffer, 0xAA, 65536);
266370
tbr = sceKernelRead(fd_read, buffer, 65536);
267371
CHECK_COMPARE_TEXT(tbr, >=, 0, "PFS read failed");
372+
LogTest("PFS read got", tbr, "bytes");
268373
if (tbr == 0) break;
269-
CHECK_EQUAL(65536, sceKernelWrite(fd_dump, buffer, 65536));
374+
CHECK_EQUAL_TEXT(65536, tbr, "Incorrect read size"); // not ready for multiples of buffer size
375+
CHECK_EQUAL(tbr, sceKernelWrite(fd_dump, buffer, tbr));
270376
} while (tbr);
271377
CHECK_EQUAL_ZERO(sceKernelClose(fd_dump));
272378

@@ -279,8 +385,10 @@ TEST(DirentTests, DumpEverythingRaw) {
279385
memset(buffer, 0xAA, 65536);
280386
tbr = sceKernelGetdirentries(fd_read, buffer, 65536, nullptr);
281387
CHECK_COMPARE_TEXT(tbr, >=, 0, "PFS sceKernelGetdirentries failed");
388+
LogTest("PFS sceKernelGetdirentries got", tbr, "bytes");
282389
if (tbr == 0) break;
283-
CHECK_EQUAL(65536, sceKernelWrite(fd_dump, buffer, 65536));
390+
CHECK_EQUAL_TEXT(10616, tbr, "Incorrect read size"); // not ready for multiples of buffer size
391+
CHECK_EQUAL(tbr, sceKernelWrite(fd_dump, buffer, tbr));
284392
} while (tbr);
285393
CHECK_EQUAL_ZERO(sceKernelClose(fd_dump));
286394

@@ -297,9 +405,11 @@ TEST(DirentTests, DumpEverythingRaw) {
297405
do {
298406
memset(buffer, 0xAA, 65536);
299407
tbr = sceKernelRead(fd_read, buffer, 65536);
300-
CHECK_COMPARE_TEXT(tbr, >=, 0, "Normal sceKernelRead failed");
408+
CHECK_COMPARE_TEXT(tbr, >=, 0, "Normal read failed");
409+
LogTest("Normal read got", tbr, "bytes");
301410
if (tbr == 0) break;
302-
CHECK_EQUAL(65536, sceKernelWrite(fd_dump, buffer, 65536));
411+
CHECK_EQUAL_TEXT(8704, tbr, "Incorrect read size"); // not ready for multiples of buffer size
412+
CHECK_EQUAL(tbr, sceKernelWrite(fd_dump, buffer, tbr));
303413
} while (tbr);
304414
CHECK_EQUAL_ZERO(sceKernelClose(fd_dump));
305415

@@ -311,8 +421,10 @@ TEST(DirentTests, DumpEverythingRaw) {
311421
memset(buffer, 0xAA, 65536);
312422
tbr = sceKernelGetdirentries(fd_read, buffer, 65536, nullptr);
313423
CHECK_COMPARE_TEXT(tbr, >=, 0, "Normal sceKernelGetdirentries failed");
424+
LogTest("Normal sceKernelGetdirentries got", tbr, "bytes");
314425
if (tbr == 0) break;
315-
CHECK_EQUAL(65536, sceKernelWrite(fd_dump, buffer, 65536));
426+
CHECK_EQUAL_TEXT(8704, tbr, "Incorrect read size"); // not ready for multiples of buffer size
427+
CHECK_EQUAL(tbr, sceKernelWrite(fd_dump, buffer, tbr));
316428
} while (tbr);
317429
CHECK_EQUAL_ZERO(sceKernelClose(fd_dump));
318430

0 commit comments

Comments
 (0)