Skip to content

Commit b6d6baa

Browse files
pikatchupikatchu
committed
Added the collect option in lazyDirs
dont update readers when a lazy data is computed Added reusing for lazyDirs First version of concurrent runtime. - A global lock is take for commits - The lock is allocated in an mmapped shared area with all the bells and whistles. fixed node handled constants added sk_new_const to wasm renamed sync_context_test cleaned up concurrence test first stab at real command line argument parsing more work on command args added a auto-generated magic number revisited sql loop made the concurrent test work added isATTY fixed wasm make the obstack use malloc instead of palloc
1 parent 0daf351 commit b6d6baa

21 files changed

+917
-530
lines changed

‎Cmd.sk

+2-6
Original file line numberDiff line numberDiff line change
@@ -233,12 +233,8 @@ untracked fun toplevel(topContext: mutable Context): void {
233233
prompt = env.pwd + "$ ";
234234
print_raw(prompt);
235235
cmd = vtry(() -> read_line(), _exn -> "");
236-
if (cmd == "") return None();
236+
if (cmd == "") return CAbort();
237237
if (isSQL(cmd)) {
238-
if (!SKFS.isSqlMode) {
239-
print_error("You cannot run an SQL query when not in SQL MODE");
240-
return None();
241-
};
242238
SKSQL.setupContext(context, SKSQL.Buffer(cmd));
243239
shouldContinue = SKSQL.evalWithContext(context, false);
244240
if (shouldContinue) {
@@ -249,6 +245,6 @@ untracked fun toplevel(topContext: mutable Context): void {
249245
};
250246
context.!pwd = env.pwd;
251247
context.update();
252-
Some(context.clone())
248+
CContinue(context.clone())
253249
})
254250
}

‎Context.sk

+61-24
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,13 @@
1111

1212
module SKFS;
1313

14+
/*****************************************************************************/
15+
/* Unique number generator. */
16+
/*****************************************************************************/
17+
18+
@cpp_extern("SKIP_genSym")
19+
native fun genSym(Int): Int;
20+
1421
/*****************************************************************************/
1522
/* Constants */
1623
/*****************************************************************************/
@@ -190,7 +197,7 @@ mutable class Context{
190197
mutable pwd: String = "/",
191198
mutable dirs: Dirs = Dirs{},
192199
mutable reads: SortedSet<Path> = SortedSet[],
193-
mutable time: Int = 1,
200+
mutable time: Int = 0,
194201
mutable tick: Int = 1,
195202
mutable lazyCapacity: Int = 10,
196203
mutable toUpdate: SortedMap<(Int, Int), Arrow> = SortedMap[],
@@ -493,6 +500,7 @@ mutable class Context{
493500
};
494501

495502
this.!tick = this.tick + 1;
503+
_ = this.timeStamp();
496504

497505
this.updateLazyGets();
498506

@@ -584,7 +592,7 @@ mutable class Context{
584592
}
585593

586594
mutable fun timeStamp(): Int {
587-
this.!time = this.time + 1;
595+
this.!time = genSym(this.time + 1);
588596
this.time
589597
}
590598

@@ -646,7 +654,7 @@ mutable class Context{
646654
totalSize = fixedData.data.size();
647655
creator = this.currentArrow();
648656
dir = EagerDir{time, input => true, dirName, fixedData, totalSize, creator};
649-
this.!dirs = this.dirs.set(dir.time, dirName, dir);
657+
this.setDir(dirName, dir);
650658
this.!newDirs = this.newDirs.add(dirName);
651659
}
652660

@@ -665,26 +673,13 @@ mutable class Context{
665673
totalSize = fixedData.data.size();
666674
creator = this.currentArrow();
667675
dir = EagerDir{time, input => true, dirName, fixedData, totalSize, creator};
668-
this.!dirs = this.dirs.set(dir.time, dirName, dir);
676+
this.setDir(dirName, dir);
669677
this.!newDirs = this.newDirs.add(dirName);
670678
EHandle(conv, dirName)
671679
}
672680

673681
mutable fun setDir(dirName: DirName, dir: Dir): void {
674-
/*
675-
dirNamePwdStr = this.pwd.join("/");
676-
dirNameStr = dirName.toString();
677-
dirNameArr = dirNameStr.split("/");
678-
if (dirNameArr.size() < 2) {
679-
error(`Invalid directory name: ${dirNameStr}`);
680-
};
681-
lastDirName = dirNameArr[dirNameArr.size() - 2];
682-
if (dirNamePwdStr + lastDirName + "/" != dirNameStr) {
683-
error(`Cannot create directory ${dirNameStr} from path ${dirNamePwdStr}`);
684-
};
685-
*/
686-
// TODO: re-enable this when the dirNames are represented as lists
687-
this.!dirs = this.dirs.set(dir.getTime(), dirName, dir);
682+
this.!dirs = this.dirs.set(this.timeStamp(), dirName, dir);
688683
}
689684

690685
mutable fun getDir(dirName: DirName): Dir {
@@ -826,6 +821,35 @@ class Dirs{state: DMap<DirName, Dir> = DMap::empty()} {
826821
}
827822
}
828823

824+
@cpp_export("SKIP_syncContext")
825+
fun syncContext(txTime: Int, newRoot: Context, ctx: mutable Context): void {
826+
if (txTime == newRoot.time) return void;
827+
828+
context = Context::fromSaved(newRoot);
829+
830+
changedDirNames = ctx.dirs.state.getChangesAfter(txTime);
831+
for (dirName in changedDirNames) {
832+
ctx.unsafeMaybeGetDir(dirName) match {
833+
| Some(dir @ EagerDir{input => true}) ->
834+
changedKeys = dir.getChangesAfter(txTime);
835+
context.unsafeMaybeGetDir(dirName) match {
836+
| None() -> context.mkdirMulti(dirName, Array[])
837+
| Some(EagerDir{input => true}) -> void
838+
| Some(_) -> invariant_violation("Error: incompactible dir types")
839+
};
840+
for (key in changedKeys) {
841+
values = dir.getArrayRaw(key);
842+
inputDir = context.unsafeMaybeGetEagerDir(dirName).fromSome();
843+
inputDir.writeArray(context, key, values);
844+
}
845+
| _ -> void
846+
}
847+
};
848+
849+
context.update();
850+
ctx.replaceFromSaved(context.clone());
851+
}
852+
829853
@cpp_extern
830854
private native base class Obstack
831855

@@ -843,7 +867,7 @@ private native fun destroyObstackWithValue<T>(Obstack, List<T>): List<T>;
843867

844868
@debug
845869
@cpp_extern("SKIP_context_sync")
846-
private native fun gContextSync(Context): Context;
870+
private native fun gContextSync(Int, Context, Context): Context;
847871

848872
@debug
849873
@cpp_extern("SKIP_context_init")
@@ -869,25 +893,38 @@ native fun disableGC(): void;
869893
@cpp_extern("SKIP_enable_GC")
870894
native fun enableGC(): void;
871895

896+
base class ContextOp {
897+
children =
898+
| CContinue(Context)
899+
| CStop(Context)
900+
| CAbort()
901+
}
902+
872903
@cpp_noinline
873904
fun runWithGcReturnContext(
874905
origContext: Context,
875-
f: Context ~> ?Context,
906+
f: Context ~> ContextOp,
876907
): mutable Context {
877908
context = gContextInit(origContext);
909+
startTime = context.time;
878910
loop {
879911
pos = newObstack();
880912
newContextOpt = f(context);
881913
newContextOpt match {
882-
| None() -> return Context::fromSaved(context)
883-
| Some(newContext) ->
884-
!context = gContextSync(newContext);
914+
| CAbort() -> return Context::fromSaved(context)
915+
| CContinue(newContext) ->
916+
!context = gContextSync(startTime, context, newContext);
917+
!startTime = context.time;
885918
destroyObstack(pos)
919+
| CStop(newContext) ->
920+
!context = gContextSync(startTime, context, newContext);
921+
!startTime = context.time;
922+
return Context::fromSaved(context)
886923
}
887924
}
888925
}
889926

890-
fun runWithGc(origContext: Context, f: Context ~> ?Context): void {
927+
fun runWithGc(origContext: Context, f: Context ~> ContextOp): void {
891928
_ = runWithGcReturnContext(origContext, f);
892929
}
893930

‎EagerDir.sk

+2-1
Original file line numberDiff line numberDiff line change
@@ -851,7 +851,8 @@ class EagerDir{
851851
};
852852

853853
!map = map.set(source, rvalues);
854-
data = this.data.set(TimeTag::create(context.tick), k, map);
854+
tag = TimeTag::create(context.timeStamp());
855+
data = this.data.set(tag, k, map);
855856

856857
if (native_eq(rvalues, oldValues) == 0) {
857858
return this with {data};

‎Main.sk

+60-28
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,43 @@
1-
module SKFS;
1+
module alias AP = ArgumentParser;
22

3-
const args: Array<String> = arguments();
4-
const isSqlMode: Bool = args.size() > 1 && args[0] == "sql";
3+
const argsSchema: Array<AP.Param> = Array[
4+
AP.BoolParam{name => "all", help => "run all the tests", negatable => false},
5+
AP.BoolParam{name => "debug", help => "", negatable => false},
6+
AP.StringParam{name => "test", help => "The name of the test to run"},
7+
AP.StringParam{name => "init", help => "Initializes a new data file"},
8+
AP.StringParam{name => "data", help => "Loads an existing data file"},
9+
];
510

6-
module end;
11+
const parsedArgs: AP.ParseResult = AP.parse(argsSchema, arguments()) match {
12+
| Success(value) -> value
13+
| Failure(error) ->
14+
print_error(error.getMessage());
15+
exit(2)
16+
};
17+
18+
const argMap: Map<String, AP.Value> = parsedArgs.argumentValues;
719

820
untracked fun main(): void {
9-
_ = SKFS.gContextInit(SKFS.Context{});
10-
if (SKFS.args.size() == 0) {
11-
print_error("Command parameters missing");
21+
if (parsedArgs.unknown.size() != 0) {
22+
unknownOptions = parsedArgs.unknown.filter(x ->
23+
x != "" && x.getIter().next() == Some('-')
24+
);
25+
for (option in unknownOptions) {
26+
if (option != "--help") print_error("Unknown option: " + option);
27+
};
28+
if (unknownOptions.size() != 0) {
29+
print_error(AP.help(argsSchema));
30+
exit(2);
31+
}
32+
};
33+
argFiles = parsedArgs.unknown.toArray();
34+
if (argFiles.size() > 0) {
35+
for (file in argFiles) {
36+
print_error("Error: don't know what to do with " + file);
37+
};
1238
exit(2);
1339
};
14-
SKFS.args[0] match {
15-
| "sql" ->
16-
context = SKSQL.eval(SKFS.args);
17-
if (SKFS.args.size() > 1 && SKFS.args.contains("--toplevel")) {
18-
SKFS.toplevel(context)
19-
}
20-
| "all" ->
40+
if (argMap.maybeGet("all") is Some(AP.BoolValue _)) {
2141
testFuns = SKFSTest.getAllTests();
2242
for (testName => testFun in testFuns) {
2343
print_string(
@@ -29,26 +49,38 @@ untracked fun main(): void {
2949
);
3050
_ = testFun();
3151
print_string("");
52+
return void;
3253
}
33-
| "toplevel" | "test" ->
34-
if (SKFS.args.size() <= 1) {
35-
print_error("Error: missing test name");
36-
exit(3);
37-
};
54+
} else if (argMap.maybeGet("test") is Some(AP.StringValue _)) {
3855
testFuns = SKFSTest.getAllTests();
39-
testName = SKFS.args[1];
40-
if (!testFuns.containsKey(testName)) {
41-
print_error("Error: unknown test " + testName + "\n");
42-
for (key => _ in testFuns) {
43-
print_error(" " + key + "\n");
56+
testName = argMap["test"] match {
57+
| AP.StringValue{value => testName} ->
58+
if (!testFuns.containsKey(testName)) {
59+
print_error("Error: unknown test " + testName + "\n");
60+
for (key => _ in testFuns) {
61+
print_error(" " + key + "\n");
62+
};
63+
exit(4);
4464
};
45-
exit(4);
65+
testName
66+
| _ ->
67+
print_error("Invalid test name type");
68+
exit(2)
4669
};
4770
context = testFuns[testName]();
48-
_ = SKFS.gContextSync(context.clone());
49-
if (SKFS.args[0] == "toplevel") {
71+
if (argMap.maybeGet("debug") is Some(AP.BoolValue _)) {
5072
SKFS.toplevel(context)
5173
}
52-
| cmd -> print_error("Error: unknown command " + cmd + "\n")
74+
} else if (argMap.maybeGet("init") is Some(AP.StringValue _)) {
75+
SKFS.runWithGc(SKSQL.makeSqlContext().clone(), (ctx) ~> SKFS.CStop(ctx));
76+
return void;
77+
} else if (argMap.maybeGet("data") is Some(AP.StringValue _)) {
78+
// this was handled in the C++ side
79+
void
80+
};
81+
if (argMap.maybeGet("debug") is Some(AP.BoolValue _)) {
82+
SKFS.toplevel(mutable SKFS.Context{});
83+
} else {
84+
SKSQL.eval(SKSQL.makeSqlContext())
5385
}
5486
}

‎Makefile

+18-9
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ SKC=~/skip/build/bin/skip_to_llvm
44
BCLINK=llvm-link-10
55
MEMSIZE32=1073741824
66

7-
OLEVEL=-O3
7+
OLEVEL=-O0 -g
88
CC32FLAGS=-DSKIP32 --target=wasm32 -emit-llvm
99
CC64FLAGS=$(OLEVEL) -DSKIP64
1010
SKFLAGS=
@@ -24,10 +24,15 @@ CFILES=\
2424
runtime/string.c \
2525
runtime/native_eq.c
2626

27+
NATIVE_FILES=\
28+
runtime/palloc.c\
29+
runtime/consts.c
30+
2731
CFILES32=$(CFILES) runtime/runtime32_specific.c
28-
CFILES64=$(CFILES) runtime/runtime64_specific.cpp runtime/alloc.c
32+
CFILES64=$(CFILES) runtime/runtime64_specific.cpp $(NATIVE_FILES)
2933
BCFILES32=$(addprefix build/,$(CFILES32:.c=.bc))
3034
OFILES=$(addprefix build/,$(CFILES:.c=.o))
35+
ONATIVE_FILES=build/magic.h $(addprefix build/,$(NATIVE_FILES:.c=.o))
3136

3237
SKFUNS=\
3338
getCompositeName \
@@ -46,11 +51,15 @@ SKFUNS=\
4651

4752
EXPORTJS=$(addprefix -export=,$(SKFUNS))
4853

49-
default: build/out32.wasm build/a.out
54+
default: build/out32.wasm build/sqlive
55+
56+
build/magic.h:
57+
echo -n "#define MAGIC " > build/magic.h
58+
date | cksum | awk '{print $$1}' >> build/magic.h
5059

51-
test: build/out32.wasm build/a.out
60+
test: build/out32.wasm build/sqlive
5261
node run.js
53-
build/a.out all
62+
build/sqlive all
5463

5564
build/out32.wasm: build/out32.ll build/full_runtime32.bc
5665
cat preamble32.ll build/out32.ll > build/preamble_and_out32.ll
@@ -69,16 +78,16 @@ build/%.bc: %.c
6978
mkdir -p build/runtime
7079
$(CC) $(OLEVEL) $(CC32FLAGS) -o $@ -c $<
7180

72-
build/a.out: build/out64.ll build/libskip_runtime64.a
81+
build/sqlive: build/out64.ll build/libskip_runtime64.a
7382
cat preamble64.ll build/out64.ll > build/preamble_and_out64.ll
74-
$(CPP) $(OLEVEL) build/preamble_and_out64.ll build/libskip_runtime64.a -o build/a.out
83+
$(CPP) $(OLEVEL) build/preamble_and_out64.ll build/libskip_runtime64.a -lpthread -o build/sqlive
7584

7685
build/out64.ll: $(SKIP_FILES)
7786
mkdir -p build/
7887
$(SKC) --embedded64 . --export-function-as main=skip_main $(SKFLAGS) --output build/out64.ll
7988

80-
build/libskip_runtime64.a: $(OFILES) build/runtime/runtime64_specific.o build/runtime/alloc.o
81-
ar rcs build/libskip_runtime64.a $(OFILES) build/runtime/runtime64_specific.o build/runtime/alloc.o
89+
build/libskip_runtime64.a: $(OFILES) build/runtime/runtime64_specific.o $(ONATIVE_FILES)
90+
ar rcs build/libskip_runtime64.a $(OFILES) build/runtime/runtime64_specific.o $(ONATIVE_FILES)
8291

8392
build/runtime/runtime64_specific.o: runtime/runtime64_specific.cpp
8493
$(CPP) $(OLEVEL) -c runtime/runtime64_specific.cpp -o build/runtime/runtime64_specific.o

0 commit comments

Comments
 (0)