Merge pull request #1259 from undingen/rewriter_slots4

ICs: variable size IC slots

Author: undingen

src/asm_writing/assembler.h

@@ -206,6 +206,7 @@ class Assembler {
     uint8_t* startAddr() const { return start_addr; }
     int bytesLeft() const { return end_addr - addr; }
     int bytesWritten() const { return addr - start_addr; }
+    int size() const { return end_addr - start_addr; }
     uint8_t* curInstPointer() { return addr; }
     void setCurInstPointer(uint8_t* ptr) { addr = ptr; }
     bool isExactlyFull() const { return addr == end_addr; }

src/asm_writing/icinfo.cpp

@@ -76,37 +76,46 @@ void ICInvalidator::invalidateAll() {
 void ICSlotInfo::clear() {
     ic->clear(this);
     decref_infos.clear();
+    used = false;
 }
 
-ICSlotRewrite::ICSlotRewrite(ICInfo* ic, const char* debug_name)
-    : ic(ic), debug_name(debug_name), buf((uint8_t*)malloc(ic->getSlotSize())), assembler(buf, ic->getSlotSize()) {
-    assembler.nop();
+std::unique_ptr<ICSlotRewrite> ICSlotRewrite::create(ICInfo* ic, const char* debug_name) {
+    auto ic_entry = ic->pickEntryForRewrite(debug_name);
+    if (!ic_entry)
+        return NULL;
+    return std::unique_ptr<ICSlotRewrite>(new ICSlotRewrite(ic_entry, debug_name));
+}
 
+ICSlotRewrite::ICSlotRewrite(ICSlotInfo* ic_entry, const char* debug_name)
+    : ic_entry(ic_entry),
+      debug_name(debug_name),
+      buf((uint8_t*)malloc(ic_entry->size)),
+      assembler(buf, ic_entry->size) {
+    // set num_inside = 1 to make sure that we will not have multiple rewriters at the same time rewriting the same slot
+    assert(ic_entry->num_inside == 0);
+    ++ic_entry->num_inside;
+
+    assembler.nop();
     if (VERBOSITY() >= 4)
         printf("starting %s icentry\n", debug_name);
 }
 
 ICSlotRewrite::~ICSlotRewrite() {
     free(buf);
+    --ic_entry->num_inside;
 }
 
 void ICSlotRewrite::abort() {
+    auto ic = getICInfo();
     ic->retry_backoff = std::min(MAX_RETRY_BACKOFF, 2 * ic->retry_backoff);
     ic->retry_in = ic->retry_backoff;
 }
 
-ICSlotInfo* ICSlotRewrite::prepareEntry() {
-    this->ic_entry = ic->pickEntryForRewrite(debug_name);
-    return this->ic_entry;
-}
 
-uint8_t* ICSlotRewrite::getSlotStart() {
-    assert(ic_entry != NULL);
-    return (uint8_t*)ic->start_addr + ic_entry->idx * ic->getSlotSize();
-}
 
 void ICSlotRewrite::commit(CommitHook* hook, std::vector<void*> gc_references,
-                           std::vector<std::pair<uint64_t, std::vector<Location>>> decref_infos) {
+                           std::vector<std::pair<uint64_t, std::vector<Location>>> decref_infos,
+                           llvm::ArrayRef<NextSlotJumpInfo> next_slot_jumps) {
     bool still_valid = true;
     for (int i = 0; i < dependencies.size(); i++) {
         int orig_version = dependencies[i].second;
@@ -124,10 +133,13 @@ void ICSlotRewrite::commit(CommitHook* hook, std::vector<void*> gc_references,
         return;
     }
 
+    auto ic = getICInfo();
     uint8_t* slot_start = getSlotStart();
     uint8_t* continue_point = (uint8_t*)ic->continue_addr;
 
-    bool do_commit = hook->finishAssembly(continue_point - slot_start);
+    bool should_fill_with_nops = true;
+    bool variable_size_slots = true;
+    bool do_commit = hook->finishAssembly(continue_point - slot_start, should_fill_with_nops, variable_size_slots);
 
     if (!do_commit) {
         for (auto p : gc_references)
@@ -136,6 +148,15 @@ void ICSlotRewrite::commit(CommitHook* hook, std::vector<void*> gc_references,
     }
 
     assert(!assembler.hasFailed());
+    int original_size = ic_entry->size;
+    int actual_size = assembler.bytesWritten();
+    int empty_space = original_size - actual_size;
+    assert(actual_size <= original_size);
+    assert(assembler.size() == original_size);
+    if (should_fill_with_nops) {
+        assembler.fillWithNops();
+        assert(original_size == assembler.bytesWritten());
+    }
 
     for (int i = 0; i < dependencies.size(); i++) {
         ICInvalidator* invalidator = dependencies[i].first;
@@ -144,14 +165,43 @@ void ICSlotRewrite::commit(CommitHook* hook, std::vector<void*> gc_references,
 
     ic->next_slot_to_try++;
 
+    // we can create a new IC slot if this is the last slot in the IC in addition we are checking that the new slot is
+    // at least as big as the current one.
+    bool should_create_new_slot = variable_size_slots && &ic->slots.back() == ic_entry && empty_space >= actual_size;
+    if (should_create_new_slot) {
+        // reduce size of the current slot to the real size
+        ic_entry->size = actual_size;
+
+        // after resizing this slot we need to patch the jumps to the next slot
+        Assembler new_asm(assembler.getStartAddr(), original_size);
+        for (auto&& jump : next_slot_jumps) {
+            auto jmp_inst_offset = std::get<0>(jump);
+            auto jmp_inst_end = std::get<1>(jump);
+            auto jmp_condition = std::get<2>(jump);
+            new_asm.setCurInstPointer(assembler.getStartAddr() + jmp_inst_offset);
+            new_asm.jmp_cond(assembler::JumpDestination::fromStart(actual_size), jmp_condition);
+
+            // we often end up using a smaller encoding so we have to make sure we fill the space with nops
+            while (new_asm.bytesWritten() < jmp_inst_end)
+                new_asm.nop();
+        }
+
+        // put a jump to the slowpath at the beginning of the new slot
+        Assembler asm_next_slot(assembler.getStartAddr() + actual_size, empty_space);
+        asm_next_slot.jmp(JumpDestination::fromStart(empty_space));
+
+        // add the new slot
+        ic->slots.emplace_back(ic, ic_entry->start_addr + actual_size, empty_space);
+    }
+
     // if (VERBOSITY()) printf("Commiting to %p-%p\n", start, start + ic->slot_size);
-    memcpy(slot_start, buf, ic->getSlotSize());
+    memcpy(slot_start, buf, original_size);
 
     for (auto p : ic_entry->gc_references) {
         Py_DECREF(p);
     }
     ic_entry->gc_references = std::move(gc_references);
-
+    ic_entry->used = true;
     ic->times_rewritten++;
 
     if (ic->times_rewritten == IC_MEGAMORPHIC_THRESHOLD) {
@@ -174,49 +224,66 @@ void ICSlotRewrite::commit(CommitHook* hook, std::vector<void*> gc_references,
         ic_entry->decref_infos.emplace_back(decref_info.first, std::move(merged_locations));
     }
 
-    llvm::sys::Memory::InvalidateInstructionCache(slot_start, ic->getSlotSize());
+    llvm::sys::Memory::InvalidateInstructionCache(slot_start, original_size);
 }
 
 void ICSlotRewrite::addDependenceOn(ICInvalidator& invalidator) {
     dependencies.push_back(std::make_pair(&invalidator, invalidator.version()));
 }
 
-int ICSlotRewrite::getSlotSize() {
-    return ic->getSlotSize();
-}
-
-int ICSlotRewrite::getScratchRspOffset() {
-    assert(ic->stack_info.scratch_size);
-    return ic->stack_info.scratch_rsp_offset;
-}
+int ICInfo::calculateSuggestedSize() {
+    // if we never rewrote this IC just return the whole IC size for now
+    if (!times_rewritten)
+        return slots[0].size;
 
-int ICSlotRewrite::getScratchSize() {
-    return ic->stack_info.scratch_size;
-}
-
-TypeRecorder* ICSlotRewrite::getTypeRecorder() {
-    return ic->type_recorder;
-}
+    int additional_space_per_slot = 30;
+    // if there are less rewrites than slots we can give a very accurate estimate
+    if (times_rewritten < slots.size()) {
+        // add up the sizes of all used slots
+        int size = 0;
+        for (int i = 0; i < times_rewritten; ++i) {
+            size += slots[i].size + additional_space_per_slot;
+        }
+        return size;
+    }
 
-assembler::GenericRegister ICSlotRewrite::returnRegister() {
-    return ic->return_register;
+    // get total size of IC
+    int size = 0;
+    for (auto&& slot : slots) {
+        size += slot.size;
+    }
+    // make it bigger
+    if (isMegamorphic())
+        size *= 4;
+    else
+        size *= 2;
+    return std::min(size, 4096);
 }
 
-
-
 std::unique_ptr<ICSlotRewrite> ICInfo::startRewrite(const char* debug_name) {
-    return std::unique_ptr<ICSlotRewrite>(new ICSlotRewrite(this, debug_name));
+    return ICSlotRewrite::create(this, debug_name);
 }
 
 ICSlotInfo* ICInfo::pickEntryForRewrite(const char* debug_name) {
-    int num_slots = getNumSlots();
+    int num_slots = slots.size();
+    int fallback_to_in_use_slot = -1;
+
+    // we prefer to use a unused slot and if non is available we will fallback to a slot which is in use (but no one is
+    // inside)
     for (int _i = 0; _i < num_slots; _i++) {
         int i = (_i + next_slot_to_try) % num_slots;
 
         ICSlotInfo& sinfo = slots[i];
         assert(sinfo.num_inside >= 0);
-        if (sinfo.num_inside)
+
+        if (sinfo.num_inside || sinfo.size == 0)
+            continue;
+
+        if (sinfo.used) {
+            if (fallback_to_in_use_slot == -1)
+                fallback_to_in_use_slot = i;
             continue;
+        }
 
         if (VERBOSITY() >= 4) {
             printf("picking %s icentry to in-use slot %d at %p\n", debug_name, i, start_addr);
@@ -225,21 +292,28 @@ ICSlotInfo* ICInfo::pickEntryForRewrite(const char* debug_name) {
         next_slot_to_try = i;
         return &sinfo;
     }
+
+    if (fallback_to_in_use_slot != -1) {
+        if (VERBOSITY() >= 4) {
+            printf("picking %s icentry to in-use slot %d at %p\n", debug_name, fallback_to_in_use_slot, start_addr);
+        }
+
+        next_slot_to_try = fallback_to_in_use_slot;
+        return &slots[fallback_to_in_use_slot];
+    }
+
     if (VERBOSITY() >= 4)
         printf("not committing %s icentry since there are no available slots\n", debug_name);
     return NULL;
 }
 
 static llvm::DenseMap<void*, ICInfo*> ics_by_return_addr;
 
-ICInfo::ICInfo(void* start_addr, void* slowpath_rtn_addr, void* continue_addr, StackInfo stack_info, int num_slots,
-               int slot_size, llvm::CallingConv::ID calling_conv, LiveOutSet _live_outs,
-               assembler::GenericRegister return_register, TypeRecorder* type_recorder,
-               std::vector<Location> ic_global_decref_locations)
+ICInfo::ICInfo(void* start_addr, void* slowpath_rtn_addr, void* continue_addr, StackInfo stack_info, int size,
+               llvm::CallingConv::ID calling_conv, LiveOutSet _live_outs, assembler::GenericRegister return_register,
+               TypeRecorder* type_recorder, std::vector<Location> ic_global_decref_locations)
     : next_slot_to_try(0),
       stack_info(stack_info),
-      num_slots(num_slots),
-      slot_size(slot_size),
       calling_conv(calling_conv),
       live_outs(std::move(_live_outs)),
       return_register(return_register),
@@ -251,10 +325,7 @@ ICInfo::ICInfo(void* start_addr, void* slowpath_rtn_addr, void* continue_addr, S
       start_addr(start_addr),
       slowpath_rtn_addr(slowpath_rtn_addr),
       continue_addr(continue_addr) {
-    slots.reserve(num_slots);
-    for (int i = 0; i < num_slots; i++) {
-        slots.emplace_back(this, i);
-    }
+    slots.emplace_back(this, (uint8_t*)start_addr, size);
     if (slowpath_rtn_addr && !this->ic_global_decref_locations.empty())
         slowpath_decref_info = DecrefInfo((uint64_t)slowpath_rtn_addr, this->ic_global_decref_locations);
 }
@@ -283,7 +354,7 @@ std::unique_ptr<ICInfo> registerCompiledPatchpoint(uint8_t* start_addr, uint8_t*
                                                    uint8_t* continue_addr, uint8_t* slowpath_rtn_addr,
                                                    const ICSetupInfo* ic, StackInfo stack_info, LiveOutSet live_outs,
                                                    std::vector<Location> decref_info) {
-    assert(slowpath_start_addr - start_addr >= ic->num_slots * ic->slot_size);
+    assert(slowpath_start_addr - start_addr >= ic->size);
     assert(slowpath_rtn_addr > slowpath_start_addr);
     assert(slowpath_rtn_addr <= start_addr + ic->totalSize());
 
@@ -305,22 +376,13 @@ std::unique_ptr<ICInfo> registerCompiledPatchpoint(uint8_t* start_addr, uint8_t*
     // we can let the user just slide down the nop section, but instead
     // emit jumps to the end.
     // Not sure if this is worth it or not?
-    for (int i = 0; i < ic->num_slots; i++) {
-        uint8_t* start = start_addr + i * ic->slot_size;
-        // std::unique_ptr<MCWriter> writer(createMCWriter(start, ic->slot_size * (ic->num_slots - i), 0));
-        // writer->emitNop();
-        // writer->emitGuardFalse();
-
-        Assembler writer(start, ic->slot_size);
-        writer.nop();
-        // writer.trap();
-        // writer.jmp(JumpDestination::fromStart(ic->slot_size * (ic->num_slots - i)));
-        writer.jmp(JumpDestination::fromStart(slowpath_start_addr - start));
-    }
+    Assembler writer(start_addr, ic->size);
+    writer.nop();
+    writer.jmp(JumpDestination::fromStart(slowpath_start_addr - start_addr));
 
     ICInfo* icinfo
-        = new ICInfo(start_addr, slowpath_rtn_addr, continue_addr, stack_info, ic->num_slots, ic->slot_size,
-                     ic->getCallingConvention(), std::move(live_outs), return_register, ic->type_recorder, decref_info);
+        = new ICInfo(start_addr, slowpath_rtn_addr, continue_addr, stack_info, ic->size, ic->getCallingConvention(),
+                     std::move(live_outs), return_register, ic->type_recorder, decref_info);
 
     assert(!ics_by_return_addr.count(slowpath_rtn_addr));
     ics_by_return_addr[slowpath_rtn_addr] = icinfo;
@@ -350,14 +412,14 @@ ICInfo* getICInfo(void* rtn_addr) {
 void ICInfo::clear(ICSlotInfo* icentry) {
     assert(icentry);
 
-    uint8_t* start = (uint8_t*)start_addr + icentry->idx * getSlotSize();
+    uint8_t* start = (uint8_t*)icentry->start_addr;
 
     if (VERBOSITY() >= 4)
         printf("clearing patchpoint %p, slot at %p\n", start_addr, start);
 
-    Assembler writer(start, getSlotSize());
+    Assembler writer(start, icentry->size);
     writer.nop();
-    writer.jmp(JumpDestination::fromStart(getSlotSize()));
+    writer.jmp(JumpDestination::fromStart(icentry->size));
     assert(writer.bytesWritten() <= IC_INVALDITION_HEADER_SIZE);
 
     for (auto p : icentry->gc_references) {
@@ -370,7 +432,17 @@ void ICInfo::clear(ICSlotInfo* icentry) {
     // writer->emitGuardFalse();
 
     // writer->endWithSlowpath();
-    llvm::sys::Memory::InvalidateInstructionCache(start, getSlotSize());
+
+    llvm::sys::Memory::InvalidateInstructionCache(start, icentry->size);
+
+    for (int i = 0; i < slots.size(); ++i) {
+        if (&slots[i] == icentry) {
+            next_slot_to_try = i;
+            break;
+        }
+    }
+
+    icentry->used = false;
 }
 
 bool ICInfo::shouldAttempt() {