Backport trafo optimization

monticore-grammar/src/main/java/de/monticore/tf/odrulegeneration/_ast/ASTTransformationStructure.java

@@ -1,8 +1,8 @@
 /* (c) https://github.com/MontiCore/monticore */
 package de.monticore.tf.odrulegeneration._ast;
 
-import java.util.HashMap;
-import java.util.List;
+import java.util.*;
+import java.util.stream.Collectors;
 
 public class ASTTransformationStructure extends ASTTransformationStructureTOP {
 
@@ -70,5 +70,36 @@ public HashMap<String, List<String>> getFoldingHash ()  {
     public  void setFoldingHash (HashMap<String, List<String>> foldingHash)  {
        this.foldingHash = foldingHash;
     }
-
+
+    public List<String> getCompositionDependencyNames(ASTMatchingObject object) {
+        return this.getPattern().getLinkConditionsList().stream().filter(
+                l -> l.getLinktype().equals("composition") && l.getDependency().getContent()
+                    .equals(object.getObjectName())).map(ASTCondition::getObjectName)
+            .collect(Collectors.toList());
+    }
+
+    public Map<String, String> getReplacementChangeMapping() {
+        Map<String, String> replacementChangeMapping = new HashMap<>();
+        this.getReplacement().getChangesList().stream().filter(ASTChange::isPresentValue)
+            .filter(x -> x.getValue().charAt(0) == '_').forEach(change -> {
+                String truncValueName =
+                    change.getValue().substring(1, change.getValue().lastIndexOf("_"));
+                replacementChangeMapping.put(change.getObjectName(), truncValueName);
+            });
+        return replacementChangeMapping;
+    }
+
+    public Set<String> getAllInnerNonOptionalNames(List<ASTMatchingObject> allObjects,
+        ASTMatchingObject matchObject) {
+        Set<String> result = new HashSet<>();
+        for (String innerLinkObjectName : matchObject.getInnerLinkObjectNamesList()) {
+            ASTMatchingObject innerLinkObject = this.getPattern().getMatchingObjectsList().stream()
+                .filter(f -> f.getObjectName().equals(innerLinkObjectName)).findFirst().get();
+            if (!innerLinkObject.isOptObject()) {
+                result.add(innerLinkObjectName);
+            }
+            result.addAll(getAllInnerNonOptionalNames(allObjects, innerLinkObject));
+        }
+        return result;
+    }
 }

monticore-runtime/src/main/java/de/monticore/tf/runtime/FastLookupList.java

@@ -0,0 +1,67 @@
+package de.monticore.tf.runtime;
+
+import java.util.AbstractList;
+import java.util.ConcurrentModificationException;
+import java.util.List;
+
+public class FastLookupList<T> extends AbstractList<T> {
+
+  protected List<T> list;
+
+  public FastLookupList(List<T> list) {
+    this.list = list;
+    this.size = list.size();
+  }
+
+  public FastLookupList(List<T> list, int removalCounter) {
+    this(list);
+    this.removalCounter = removalCounter;
+  }
+
+  protected int removalCounter = 0;
+  protected int size;
+
+  public void reset() {
+    removalCounter = 0;
+  }
+
+  @Override
+  public T get(int index) {
+    if (list.size() != size) {
+      throw new ConcurrentModificationException("FastLookupList size changed");
+    }
+    return this.list.get(index + removalCounter);
+  }
+
+  @Override
+  public T remove(int index) {
+    if (index != 0) {
+      throw new IllegalArgumentException("You may only remove index 0");
+    }
+    removalCounter++;
+    return null;
+  }
+
+  @Override
+  public int size() {
+    return size - removalCounter;
+  }
+
+  @Override
+  public boolean isEmpty() {
+    if (list.size() != size) {
+      throw new ConcurrentModificationException("FastLookupList size changed");
+    }
+    return removalCounter == size;
+  }
+
+  public FastLookupList<T> matchCopy() {
+    return new FastLookupList<>(this.list, this.removalCounter);
+  }
+
+  @Override
+  public String toString() {
+    return "FastLookupList{" + removalCounter + "/ " + size + ": " + list.subList(removalCounter,
+        size) + "}";
+  }
+}

monticore-runtime/src/main/java/de/monticore/tf/runtime/matching/CommentBasedModelTraversal.java

@@ -0,0 +1,53 @@
+/* (c) https://github.com/MontiCore/monticore */
+package de.monticore.tf.runtime.matching;
+
+import de.monticore.ast.ASTNode;
+import de.monticore.ast.Comment;
+import de.monticore.visitor.ITraverser;
+
+import java.util.*;
+
+public class CommentBasedModelTraversal<E extends ITraverser> extends ModelTraversal<E> {
+
+  protected CommentBasedModelTraversal(E traverser) {
+    super(traverser);
+  }
+
+  @Override
+  public ASTNode getParent(ASTNode node) {
+    return ((WComment) node.get_PostCommentList().get(0)).getParent();
+  }
+
+  public void init() {
+    for (Map.Entry<ASTNode, ASTNode> node : this.getParents().entrySet()) {
+      if (node.getKey().get_PostCommentList().isEmpty()) {
+        node.getKey().get_PostCommentList().add(new WComment("", node.getValue()));
+      }
+      else {
+        node.getKey().get_PostCommentList()
+            .set(0, new WComment(node.getKey().get_PostCommentList().get(0), node.getValue()));
+      }
+    }
+  }
+
+  static class WComment extends Comment {
+
+    protected ASTNode parent;
+
+    public WComment(String text, ASTNode parent) {
+      super(text);
+      this.parent = parent;
+    }
+
+    public WComment(Comment c, ASTNode parent) {
+      this(c.getText(), parent);
+      this.set_SourcePositionStart(c.get_SourcePositionStart());
+      this.set_SourcePositionEnd(c.get_SourcePositionEnd());
+    }
+
+    public ASTNode getParent() {
+      return parent;
+    }
+  }
+
+}

monticore-runtime/src/main/java/de/monticore/tf/runtime/matching/CommentBasedModelTraversalFactory.java

@@ -0,0 +1,33 @@
+/* (c) https://github.com/MontiCore/monticore */
+package de.monticore.tf.runtime.matching;
+
+import de.monticore.visitor.ITraverser;
+
+public class CommentBasedModelTraversalFactory extends ModelTraversalFactory {
+
+  private static CommentBasedModelTraversalFactory instance;
+
+  public static CommentBasedModelTraversalFactory getInstance() {
+    if (instance == null) {
+      instance = new CommentBasedModelTraversalFactory();
+    }
+    return instance;
+  }
+
+  /**
+   * @return a {@link ModelTraversal} for the given model
+   */
+  public ModelTraversalVisitor createVisitor(ModelTraversal<?> modelTraversal) {
+    return new CommentBasedModelTraversalVisitor(modelTraversal);
+  }
+
+  public <E extends ITraverser> ModelTraversal<E> create(java.util.function.Supplier<E> traverserSupplier){
+    return this.create(traverserSupplier.get());
+  }
+
+  public <E extends ITraverser> ModelTraversal<E> create(E traverser){
+    ModelTraversal<E> modelTraversal = new CommentBasedModelTraversal<>(traverser);
+    traverser.add4IVisitor(createVisitor(modelTraversal));
+    return modelTraversal;
+  }
+}

monticore-runtime/src/main/java/de/monticore/tf/runtime/matching/CommentBasedModelTraversalVisitor.java

@@ -0,0 +1,11 @@
+/* (c) https://github.com/MontiCore/monticore */
+package de.monticore.tf.runtime.matching;
+
+import de.monticore.visitor.IVisitor;
+
+public class CommentBasedModelTraversalVisitor extends ModelTraversalVisitor implements IVisitor {
+
+  protected CommentBasedModelTraversalVisitor(ModelTraversal<?> modelTraversal) {
+    super(modelTraversal);
+  }
+}

monticore-runtime/src/main/resources/de/monticore/tf/odrules/ConstantCode.ftl

@@ -4,10 +4,14 @@
 <#assign matchObjects = hierarchyHelper.getMandatoryMatchObjects(ast.getPattern().getMatchingObjectsList())>
 <#assign optionalMatchObjects = hierarchyHelper.getOptionalMatchObjects(ast.getPattern().getLHSObjectsList())>
 
+  public static boolean optimizeSP = true;
+
   private List<ASTNode> hostGraph;
   private GlobalExtensionManagement glex = new GlobalExtensionManagement();
   private List<Match> allMatches;
   private boolean doReplacementExecuted = false;
+  private boolean isHostGraphDirty = true;
+  List<Runnable> resetOptionalCans = new LinkedList<>();
   <#-- for each object creates a _candidates, _candidates_temp nodelist and an _cand object-->
 
   // Matches
@@ -19,7 +23,7 @@
   protected boolean ${variable.getName()}_is_fix = false;
   private ${variable.getType()} ${variable.getName()};
   </#list>
-  private ModelTraversal <?> t = ModelTraversalFactory.getInstance().create((java.util.function.Supplier)${grammarName}Mill::inheritanceTraverser);
+  private ModelTraversal <?> t = CommentBasedModelTraversalFactory.getInstance().create((java.util.function.Supplier)${grammarName}Mill::inheritanceTraverser);
   <#list ast.getPattern().getAssocList() as association>
   private mc.ast.MCAssociation ${association.getName()};
   </#list>
@@ -67,3 +71,22 @@
     doPatternMatching();
     doReplacement();
   }
+
+  protected void loadIntoModelTraverser() {
+    for (ASTNode astNode : Log.errorIfNull(hostGraph,
+            "0xE1200: Hostgraph is null, check constructor arguments!")) {
+      astNode.accept(t.getTraverser());
+    }
+
+    if (t instanceof CommentBasedModelTraversal) {
+      ((CommentBasedModelTraversal<?>) t).init();
+    }
+  }
+
+  /**
+  * Marks the original model as dirty, same as if {@link #doReplacement} was called.
+  * @see ${ast.getClassname()}#doReplacement()
+  */
+  public void markDirty() {
+    this.isHostGraphDirty = true;
+  }

monticore-runtime/src/main/resources/de/monticore/tf/odrules/DoPatternMatching.ftl

@@ -13,11 +13,22 @@ public boolean doPatternMatching() {
   // (this will skip all attempts to match negative nodes)
   boolean isBacktracking = true;
   boolean isBacktrackingNegative = false;
-  for(ASTNode a: hostGraph){
-    a.accept(t.getTraverser());
+
+  resetOptionalCans.forEach(Runnable::run);
+  resetOptionalCans = new LinkedList<>();
+
+  if (isHostGraphDirty || searchPlan == null) {
+    this.loadIntoModelTraverser();
+    isHostGraphDirty= false;
   }
+
   if (searchPlan == null) {
     searchPlan = findSearchPlan();
+
+    if(optimizeSP) {
+      optimizeSearchplan();
+    }
+    initializeFastLookupList();
     splitSearchplan(); // for OptList structures
     isBacktracking = false;
   }

monticore-runtime/src/main/resources/de/monticore/tf/odrules/DoPatternMatchingList.ftl

@@ -149,7 +149,7 @@ public boolean doPatternMatching_${structure.getObjectName()}(boolean isParentBa
               // reset candidates list
               if(!isBacktracking){
                 if (!isBacktrackingNegative) {
-                  ${object.getObjectName()}_candidates_temp = new ArrayList<>(${object.getObjectName()}_candidates);
+                  ((FastLookupList<?>)${object.getObjectName()}_candidates_temp).reset();
                 }
                 //try to find a match
                 ${object.getObjectName()}_cand = match_${object.getObjectName()}();
@@ -172,7 +172,7 @@ public boolean doPatternMatching_${structure.getObjectName()}(boolean isParentBa
                     //put the first object of the backtracking stack
                     searchPlan.push(backtrackingNegative.pop());
                     //reset candidates list
-                    ${object.getObjectName()}_candidates_temp = new ArrayList<>(${object.getObjectName()}_candidates);
+                    ((FastLookupList<?>)${object.getObjectName()}_candidates_temp).reset();
                   }
                 }else{
 
@@ -195,7 +195,7 @@ public boolean doPatternMatching_${structure.getObjectName()}(boolean isParentBa
                       searchPlan.push(backtracking.pop());
                     }
                     //reset candidates list
-                    ${object.getObjectName()}_candidates_temp = new ArrayList<>(${object.getObjectName()}_candidates);
+                    ((FastLookupList<?>)${object.getObjectName()}_candidates_temp).reset();
 
                   }
                 }
@@ -211,7 +211,7 @@ public boolean doPatternMatching_${structure.getObjectName()}(boolean isParentBa
                 clear${structure.getObjectName()}NegativeObjects();
               }
               if (!isBacktracking) {
-                ${object.getObjectName()}_candidates_temp = new ArrayList<>(${object.getObjectName()}_candidates);
+                ((FastLookupList<?>)${object.getObjectName()}_candidates_temp).reset();
               }
               //try to find a match
               ${object.getObjectName()}_cand = match_${object.getObjectName()}();
@@ -230,7 +230,7 @@ public boolean doPatternMatching_${structure.getObjectName()}(boolean isParentBa
                   //put the first object of the backtracking stack
                   searchPlan.push(backtracking.pop());
                   //reset candidates list
-                  ${object.getObjectName()}_candidates_temp = new ArrayList<>(${object.getObjectName()}_candidates);
+                  ((FastLookupList<?>)${object.getObjectName()}_candidates_temp).reset();
                 }
               }else{
                 // stop backtracking
@@ -274,13 +274,23 @@ public boolean doPatternMatching_${structure.getObjectName()}(boolean isParentBa
         Match${structure.getObjectName()} match = new Match${structure.getObjectName()}(<@commaSeperatedNames/>);
         match.backtracking = (Stack<String>) backtracking.clone();
         <#list mandatoryObjects as o>// save context of every object and then clear it
-          match.${o.getObjectName()}_temp_candidates = ${o.getObjectName()}_candidates_temp;
-          ${o.getObjectName()}_cand = null;
+        match.${o.getObjectName()}_temp_candidates = ((FastLookupList<ASTNode>)${o.getObjectName()}_candidates_temp).matchCopy();
+        ${o.getObjectName()}_cand = null;
         </#list>
         ${structure.getObjectName()}_candidates.add(match);
         backtracking.clear();
       }
     }
+
+    // Reset list candidates are match
+    <#list structure.getInnerLinkObjectNamesList() as innerLinkObjectName>
+    <#if hierarchyHelper.isNoOptionalName(ast.getPattern().getLHSObjectsList(), innerLinkObjectName)>
+    ${innerLinkObjectName}_cand = null;
+    </#if>
+    </#list>
+
+    // TODO: Do something similar for optionals (but somehow do not loose them?)
+
     if(${structure.getObjectName()}_candidates.isEmpty()) {
       return false;
     }

monticore-runtime/src/main/resources/de/monticore/tf/odrules/DoPatternMatchingOptional.ftl

@@ -12,12 +12,6 @@ protected boolean doPatternMatching_${structure.getObjectName()}(boolean isParen
   Stack<String> backtrackingNegative = new Stack<String>();
   Stack<String> searchPlan = (Stack<String>) searchPlan_${structure.getObjectName()}.clone();
 
- <#list structure.getInnerLinkObjectNamesList() as elem>
-   <#if hierarchyHelper.isNoOptionalName(ast.getPattern().getLHSObjectsList(), elem)>
-     ${elem}_cand = null;
-   </#if>
- </#list>
-
   String nextNode = null;
   while(!searchPlan.isEmpty()){
     nextNode = searchPlan.pop();
@@ -38,6 +32,14 @@ protected boolean doPatternMatching_${structure.getObjectName()}(boolean isParen
     </#list>
   }
 
+  // Cleanup after optionals
+  // TODO: correct here? -> not for lists?
+  resetOptionalCans.add(() -> {
+  <#list ast.getAllInnerNonOptionalNames(ast.getPattern().getLHSObjectsList(), structure) as elem>
+    ${elem}_cand = null;
+  </#list>
+  });
+
   return true;
 }
 </#list>

monticore-runtime/src/main/resources/de/monticore/tf/odrules/DoReplacement.ftl

@@ -1,6 +1,7 @@
 <#-- (c) https://github.com/MontiCore/monticore -->
 
 public void doReplacement() {
+  isHostGraphDirty = true;
 
   for(Match m:allMatches){