Skip to content

Commit bb9d32f

Browse files
tomasbrodtomasbrod
committed
save
save work
1 parent 4386db6 commit bb9d32f

8 files changed

Lines changed: 567 additions & 97 deletions

File tree

‎BtrDbase.pas‎

Lines changed: 344 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -67,15 +67,25 @@
6767
6868
* When block is freed:
6969
- added to unstable list
70-
-
70+
-
71+
* Whent to verify csums?
72+
- on every access
73+
- ideally only when it is loaded from disk
74+
- but os does not tell wheter it was loaded or already in pagecache
75+
- bitfield for already verified pages? (in primary area)
76+
- only when requested
77+
78+
* To read from multiple processes:
79+
- read superblock on every tx open
80+
-
7181
7282
RTrx gen=1
7383
WTrx gen=1 -> 2, A:U->F, B:x->F, C:F->x
7484
-
7585
}
7686
INTERFACE
7787
IMPLEMENTATION
78-
uses bn_avl_tree;
88+
uses Classes,SysUtils,bn_avl_tree,ObjectModel,BaseUnix,crypto;
7989

8090
Type tDatabase= Class
8191
type tGenNum= QWord;
@@ -89,39 +99,356 @@ type tLLPageGenNode=class
8999
type tLLGeneration=class
90100
private {ro by transactions}
91101
gen: tGenNum;
92-
root: tPageix;
102+
root: tCSPtr;
103+
super: tPageix;
93104
private {atomic by transaction}
94105
refc: tRefCount;
95106
private {under write mutex}
96107
next: tLLGeneration;
97108
sync: boolean;
98109
end;
110+
type tCSPtr= packed object
111+
page: Word4;
112+
csum: DWord;
113+
//procedure clear;
114+
//procedure set(ipage:tPageix; isum: dword);
115+
end;
99116

100117
public
101118

102-
WriteMutex: TRTLCriticalSection;
119+
FileHandle: THandle;
103120

104121
(* Memorry-mapped Window into the Primary Branch Area *)
105122
PrimaryWindow: Pointer;
106-
WindowSize: LongWord;
107-
108-
(* Current writeback root *)
109-
Generation: tLLGeneration
123+
PrimaryWindowEnd: Pointer;
124+
PrimarySize: LongWord;
125+
MixedStart: tPageIx;
126+
MixedSize: LongWord;
110127

111-
(* Current synced root *)
112-
GenerationSync: tLLGeneration;
113128

114129
(* Generation number on which the oldest open transaction was created *)
115-
OldestGeneration: tLLGeneration;
116-
LatestGeneration: tLLGeneration;
130+
{ Unused space: page list, each page one generation
131+
$UnusedLatest -> page[gen, next, older, arr[pageix]]
132+
$UnusedOldest -> page (not modified, only moved, freed pages go to unused)
133+
Open read transaction share-locks the latest node page.
134+
@In write trx the oldest used node found by following the older pointer until
135+
exclusive lock succeeds. It still costs syscalls, but so does writing.
136+
@Put list of pages that blocked reclaim in ascending gen to superblock.
137+
@Only store Reusable in superblock, not unused.
138+
$freelist for all areas
139+
$freetree stub for mix only
140+
}
141+
142+
{ these should be part of Write transaction (memory only) }
143+
DirtyPages: tAVLTree;
144+
145+
146+
pagesize: LongWord;
147+
148+
constructor OpenH(const ihandle: THandle);
149+
constructor Format(const ihandle: THandle);
150+
151+
procedure LoadDatabase;
152+
procedure ReadUnaligned(var dest; offset: PtrUInt; size: PtrUInt);
153+
function ChecksumBuf(buffer: pointer; size: PtrUInt): DWord;
154+
function ChecksumPage(buffer: pointer): DWord;
155+
156+
procedure FreePage( p: pointer );
157+
function GetPage( ix: tPageIx ): pointer;
158+
function GetPage( const ptr: tCSPtr ): pointer;
159+
(* Special pages: 0=super0 1=super1 2=super2 3=locks -1=last *)
160+
var SpecialPageIndex: array [0..4] of tPageIx;
161+
end;
162+
163+
Type TWriteTrx= Class
164+
db: tDatabase;
165+
constructor Create(idb: tDatabase);
166+
end;
167+
168+
var Log1: tEventLogBaseSink;
169+
var Log: TEventLog;
170+
171+
constructor tWriteTrx.Create(idb: tDatabase);
172+
begin
173+
Fail;
174+
end;
175+
176+
177+
type tSuperblockH = packed record
178+
ident: array [1..8] of char;
179+
checksum: DWord;
180+
gen: Word8;
181+
pagesize: Word4;
182+
version: word2;
183+
sync: bytebool;
184+
height: byte;
185+
prisize, //1C
186+
secsize,
187+
mixsize, //24
188+
secpos,
189+
mixpos: Word4; //2C
190+
root: tDatabase.tCSPtr;
191+
utilroot: tDatabase.tCSPtr;
192+
prifree: tDatabase.tCSPtr;
193+
secfree: tDatabase.tCSPtr;
194+
mixfree: tDatabase.tCSPtr;
195+
levelmix: byte;
196+
levelsec: byte;
197+
end;
198+
const c_sb_ident: array [1..8] of char = 'BnBtrDb'#0;
199+
200+
type EDbCheckSum= class(Exception)
201+
//constructor Create(ipageno:tPageNo);
202+
//property Message: string read What;
203+
end;
204+
type EDbInitialize= class(Exception)
205+
end;
206+
type EDbSuperInit= class(EDbInitialize)
207+
end;
208+
209+
procedure tDatabase.ReadUnaligned(var dest; offset: PtrUInt; size: PtrUInt);
210+
var rc:TsSize;
211+
begin
212+
rc:=baseunix.fppread(FileHandle,@dest,size,offset);
213+
if rc<>size then raise Exception.Create('PRead failed');
214+
end;
215+
216+
function WrapMMap( fd:THandle; ofs:PtrUInt; size:PtrUInt ): pointer;
217+
var rc: pointer;
218+
begin
219+
rc:= BaseUnix.Fpmmap( {addr} nil, {len} size,
220+
{prot} PROT_READ,
221+
{flag} MAP_SHARED,
222+
{fd} fd, {off} ofs );
223+
if rc=MAP_FAILED then raise Exception.Create('MMap failed');
224+
result:=rc;
225+
end;
226+
227+
procedure tDatabase.FreePage( p: pointer );
228+
begin
229+
if (p>=PrimaryWindow) and (p<PrimaryWindowEnd)
230+
then exit;
231+
FreeMem(p,pagesize);
232+
end;
233+
234+
function tDatabase.GetPage( ix: tPageIx ): pointer;
235+
begin
236+
if ix < PrimarySize then begin
237+
result:= PrimaryWindow + ( ix * pagesize );
238+
end else begin
239+
result:=GetMem(pagesize);
240+
try
241+
ReadUnaligned( result^, ix * pagesize, pagesize );
242+
except
243+
FreeMem(result,pagesize);
244+
raise;
245+
end
246+
end
247+
end;
117248

118-
(* Pages that can be used for allocation if node.gen<FirstTransaction Populated during BeginWriteTrx from
119-
* synced unstable list. Flushed on Sync (not Confirm). *)
120-
ReusableFirst tLLPageGenNode;
121-
ReusableLast: tLLPageGenNode;
249+
function tDatabase.GetPage( ptr: tCSPtr ): pointer;
250+
var sum: DWord;
251+
var ix: tPageIx;
252+
begin
253+
ix:= ptr.page;
254+
if ix < PrimarySize then begin
255+
result:= PrimaryWindow + ( ix * pagesize );
256+
sum:= ChecksumPage(result);
257+
if(sum<>ptr.csum)
258+
then raise EDbCheckSum.CreateFmt('Checksum error at page %Dp, correct %S',[DWord(ptr.page),ToHexStr(sum,4)]);
259+
end else begin
260+
result:=GetMem(pagesize);
261+
try
262+
ReadUnaligned( result^, ix * pagesize, pagesize );
263+
sum:= ChecksumPage(result);
264+
if(sum<>ptr.csum)
265+
then raise EDbCheckSum.CreateFmt('Checksum error at page %Dp, correct %S',[DWord(ptr.page),ToHexStr(sum,4)]);
266+
except
267+
FreeMem(result,pagesize);
268+
raise;
269+
end
270+
end
271+
end;
272+
273+
procedure tDatabase.LoadDatabase();
274+
var sbpagesize: LongWord;
275+
var super: ^tSuperblockH;
276+
procedure insertgen(gen: tLLGeneration);
277+
begin
278+
end;
279+
procedure CheckSbAt(sbpos:PtrUInt);
280+
(* Early Superblock Check *)
281+
var buf:^tSuperblockH;
282+
var tmp:DWORD;
283+
var gen: tLLGeneration;
284+
var p:^tLLGeneration;
285+
begin
286+
buf:=GetMem(512);
287+
try
288+
ReadUnaligned(buf^,sbpos,512);
289+
if buf^.ident <> c_sb_ident then exit;
290+
if (DWord(buf^.pagesize) = 0)
291+
or ((sbpos mod DWord(buf^.pagesize)) >0) then exit;
292+
if buf^.checksum<>ChecksumBuf(pointer(buf)+12,500) then begin
293+
tmp:=ChecksumBuf(pointer(buf)+12,500);
294+
log.Warn('.CheckSbAt: Invalid bufblock checksum at %DB',[sbpos]);
295+
log.info('.CheckSbAt: correct %S',[ToHexStr(tmp,4)]);
296+
exit end;
297+
gen:= tLLGeneration.Create;
298+
gen.gen:= buf^.gen;
299+
gen.super:= sbpos div DWord(buf^.pagesize);
300+
gen.sync:= buf^.sync;
301+
gen.root:= buf^.root;
302+
gen.refc:= 1; {the 1 reference is from written superblock}
303+
p:=@OldestGeneration;
304+
while assigned(p^) and (p^.gen<gen.gen)
305+
do p:=@p^.next;
306+
gen.next:=p^;
307+
p^:=gen;
308+
if gen.next=nil then begin
309+
LatestGeneration:=gen;
310+
if assigned(super) then FreeMem(super,512);
311+
super:=buf;
312+
sbpagesize:=DWord(buf^.pagesize);
313+
end;
314+
finally
315+
if buf<>super then FreeMem(buf);
316+
end
317+
end;
318+
319+
procedure LoadAreas;
320+
begin
321+
{Focus on base functionality. Custom page size may come later.}
322+
{Btw: pagesize means mul instead of shl. performace ??}
323+
if sbpagesize<>4096
324+
then raise EDbSuperInit.Create('unsupported page size');
325+
pagesize:= sbpagesize;
326+
327+
{find a way to calculate this with diffrent page size}
328+
SpecialPageIndex[0]:=0; {byte offset 0}
329+
SpecialPageIndex[1]:=65536 div pagesize;
330+
if SpecialPageIndex[1] <= 2
331+
then SpecialPageIndex[2]:=SpecialPageIndex[1]+1
332+
else SpecialPageIndex[2]:=1;
333+
SpecialPageIndex[3]:=SpecialPageIndex[2]+1;
334+
335+
PrimaryWindow:= nil;
336+
PrimarySize:= 0;
337+
PrimaryWindowEnd:=nil;
338+
339+
if Word(super^.version)<>1
340+
then raise EDbInitialize.Create('unknown superblock version');
122341

123-
AllocatedPages: tAVLTree;
342+
343+
if DWord(super^.prisize)>$800000
344+
then raise EDbInitialize.Create('primary area too large');
345+
PrimarySize:= DWord(super^.prisize);
346+
if PrimarySize <= SpecialPageIndex[3]
347+
then raise EDbInitialize.Create('primary area too small');
348+
349+
PrimaryWindow:= WrapMMap(FileHandle, 0, PrimarySize * pagesize);
350+
PrimaryWindowEnd:= PrimaryWindow + (PrimarySize * pagesize) - 1;
351+
log.debug('.LoadAreas: Primary window at $%P..$%P size %Dp',[PrimaryWindow, PrimaryWindowEnd, PrimarySize]);
352+
353+
if DWord(super^.secsize)<>0
354+
then raise EDbInitialize.Create('unsupported secondary area');
355+
356+
MixedStart:=super^.mixpos;
357+
MixedSize:= super^.mixsize;
358+
359+
if MixedStart<PrimarySize
360+
then raise EDbInitialize.Create('mixed area overlaps primary area');
361+
if MixedSize<24
362+
then raise EDbInitialize.Create('mixed area too small');
363+
SpecialPageIndex[4]:=MixedStart+(MixedSize-1);
364+
FreePage(GetPage(SpecialPageIndex[4]));
365+
log.debug('.LoadAreas: Mixed area at %Dp size %Dp',[MixedStart, MixedSize]);
366+
end;
367+
368+
begin
369+
sbpagesize:=0;
370+
super:=nil;
371+
{mark that the allocator is not initialized yet}
372+
DirtyPages:=nil;
373+
try
374+
CheckSbAt(0);
375+
CheckSbAt(65536);
376+
if LatestGeneration=nil then
377+
raise EDbSuperInit.Create('No valid superblocks found');
378+
log.Info('.LoadSuperblocks: found sb at %Dp with page size %DB generation %D',[LatestGeneration.super,sbpagesize,LatestGeneration.gen]);
379+
LoadAreas;
380+
finally
381+
if assigned(super) then FreeMem(super,512);
382+
end;
383+
384+
385+
{Find latest synced sb and use config from it.}
386+
{Check if dirty sb is pesent and attempt to recover from it.}
387+
388+
{find superblock}
389+
{get config from sb}
390+
end;
391+
392+
{procedure tDatabase.LoadAreas;
393+
begin
394+
log.Debug('.LoadPrimaryArea ...',[]);
395+
check size
396+
mmap
397+
end;
398+
}
399+
400+
//procedure tDatabase.DoFormat;
401+
// begin
402+
{better?
403+
- only set memory state
404+
- create write transaction
405+
- free all pages
406+
- sync
407+
}
408+
{what?
409+
- determine areas
410+
411+
- write superblock(s)
412+
}
413+
//end;
414+
415+
constructor tDatabase.OpenH(const ihandle: THandle);
416+
begin
417+
Assert(ihandle>=0);
418+
FileHandle:= ihandle;
419+
log.Debug('.OpenH handle: %D',[FileHandle]);
420+
LoadDatabase();
124421
end;
125422

423+
constructor tDatabase.Format(const ihandle: THandle);
424+
begin
425+
Assert(ihandle>=0);
426+
FileHandle:= ihandle;
427+
log.Debug('.Format handle: %D',[FileHandle]);
428+
//DoFormat;
429+
end;
430+
431+
(* Unaligned checksum calculation. *)
432+
function tDatabase.ChecksumBuf(buffer:pointer; size: PtrUInt): DWord;
433+
begin
434+
result:= NtoLE( Crypto.CRC32c(0, buffer^, size) );
435+
end;
436+
437+
(* Page-aligned checksum calculation. Open for optimization *)
438+
function tDatabase.ChecksumPage(buffer:pointer): DWord;
439+
begin
440+
assert((PtrUInt(@buffer) and (pagesize-1)) = 0);
441+
result:= NtoLE( Crypto.CRC32c(0, buffer^, self.pagesize) );
442+
end;
443+
444+
445+
var db:tDatabase;
446+
126447
BEGIN
448+
Log1:=tEventLogBaseSink.Create;
449+
Log:=TEventLog.Create(Log1,'db');
450+
Log.Info(' BtrDBase: %S',['0.0']);
451+
log.Info(' sb size: %D',[sizeof(tSuperblockH)]);
452+
db:=tDatabase.OpenH(FileOpen('btrdb.dat',fmOpenReadWrite));
453+
db.Free;
127454
END.

0 commit comments

Comments
 (0)