Dynamically switching from all-file processing to single-file processing of multi file JavaSources.

ide/parsing.api/apichanges.xml

@@ -87,6 +87,21 @@ is the proper place.
 <!-- ACTUAL CHANGES BEGIN HERE: -->
 
   <changes>
+      <change id="Parser.Result.processingFinished">
+          <api name="ParsingAPI"/>
+          <summary>Adding Parser.Result.processingFinished</summary>
+          <version major="9" minor="20"/>
+          <date day="12" month="9" year="2021"/>
+          <author login="jlahoda"/>
+          <compatibility source="compatible" binary="compatible" addition="yes"/>
+          <description>
+              <p>
+                  Adding a new method, Parser.Result.processingFinished, used to
+                  mark a result as completely finished, or not completely finished.
+              </p>
+          </description>
+          <class package="org.netbeans.modules.parsing.spi" name="Parser"/>
+      </change>
       <change id="Source.Context.Lookup">
           <api name="ParsingAPI"/>
           <summary>Source provides access to Lookup</summary>

ide/parsing.api/src/org/netbeans/modules/parsing/api/ParserManager.java

@@ -20,10 +20,12 @@
 package org.netbeans.modules.parsing.api;
 
 import java.lang.ref.Reference;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Future;
 import org.netbeans.api.annotations.common.NonNull;
@@ -138,20 +140,21 @@ public Void run () throws Exception {
     private static class MultiUserTaskAction implements Mutex.ExceptionAction<Void> {
 
         private final UserTask userTask;
-        private final Collection<Source> sources;
+        private final List<Source> sources;
 
         public MultiUserTaskAction (final Collection<Source> sources, final UserTask userTask) {
             assert sources != null;
             assert userTask != null;
             this.userTask = userTask;
-            this.sources = sources;
+            this.sources = new ArrayList<>(sources);
         }
 
         public Void run () throws Exception {
             final LowMemoryWatcher lMListener = LowMemoryWatcher.getInstance();
             Parser parser = null;
             final Collection<Snapshot> snapShots = new LazySnapshots(sources);
-            for (Source source : sources) {
+            for (int i = 0; i < sources.size(); ) {
+                Source source = sources.get(i);
                 if (parser == null) {
                     Lookup lookup = MimeLookup.getLookup (source.getMimeType ());
                     ParserFactory parserFactory = lookup.lookup (ParserFactory.class);
@@ -165,6 +168,9 @@ public Void run () throws Exception {
                 try {
                     TaskProcessor.callUserTask(userTask, resultIterator);
                 } finally {
+                    if (ParserAccessor.getINSTANCE().processingFinished(resultIterator.getParserResult())) {
+                        i++;
+                    }
                     ResultIteratorAccessor.getINSTANCE().invalidate(resultIterator);
                     SourceAccessor.getINSTANCE().getAndSetCache(source, origCache);
                 }

ide/parsing.api/src/org/netbeans/modules/parsing/impl/ParserAccessor.java

@@ -48,5 +48,6 @@ public static void setINSTANCE (final ParserAccessor _instance) {
     }
 
     public abstract void invalidate (Parser.Result result);    
+    public abstract boolean processingFinished (Parser.Result result);
 
 }

ide/parsing.api/src/org/netbeans/modules/parsing/spi/Parser.java

@@ -142,7 +142,19 @@ public Snapshot getSnapshot () {
          * This method is called by Parsing API, when {@link Task} is finished.
          */
         protected abstract void invalidate ();
-
+
+        /**
+         * Return {@code true} if the parser was able to process the file fully.
+         *
+         * Return {@code false}, if the parser had to stop the processing (e.g. for
+         * memory limit reasons), and the file should be processed again.
+         *
+         * @return whether file processing finished or not
+         * @since 9.20
+         */
+        protected boolean processingFinished() {
+            return true;
+        }
     }
 
     /**
@@ -183,7 +195,13 @@ public void invalidate (
             assert result != null;
             result.invalidate();
         }        
-
+
+        @Override
+        public boolean processingFinished(Result result) {
+            assert result != null;
+            return result.processingFinished();
+        }
+
     }
 }
 

java/java.source.base/src/org/netbeans/modules/java/source/parsing/CompilationInfoImpl.java

@@ -44,6 +44,8 @@
 import java.util.Map.Entry;
 import java.util.SortedMap;
 import java.util.TreeMap;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import javax.swing.text.Document;
 import javax.tools.Diagnostic;
@@ -93,7 +95,10 @@ public final class CompilationInfoImpl {
     JavaSource.Phase parserCrashed = JavaSource.Phase.UP_TO_DATE;      //When javac throws an error, the moveToPhase sets this to the last safe phase
     private final Map<CacheClearPolicy, Map<Object, Object>> userCache = new EnumMap<CacheClearPolicy, Map<Object, Object>>(CacheClearPolicy.class);
     //cache of already parsed files
-    private Map<String, CompilationUnitTree> parsedTrees;
+    private Map<JavaFileObject, CompilationUnitTree> parsedTrees;
+    private Map<FileObject, AbstractSourceFileObject> ide2javacFileObject;
+    private Map<FileObject, Snapshot> fileObject2Snapshot;
+    private boolean incomplete;
 
     /**
      * Creates a new CompilationInfoImpl for given source file
@@ -111,7 +116,9 @@ public CompilationInfoImpl (final JavacParser parser,
                          final JavacTaskImpl javacTask,
                          final DiagnosticListener<JavaFileObject> diagnosticListener,
                          final Snapshot snapshot,
-                         final boolean detached) throws IOException {
+                         final boolean detached,
+                         final Map<FileObject, AbstractSourceFileObject> ide2javacFileObject,
+                         final Map<FileObject, Snapshot> fileObject2Snapshot) throws IOException {
         assert parser != null;
         this.parser = parser;
         this.cpInfo = parser.getClasspathInfo();
@@ -121,13 +128,33 @@ public CompilationInfoImpl (final JavacParser parser,
         this.snapshot = snapshot;
         this.partialReparseLastGoodSnapshot = new SoftReference<>(snapshot);
         assert file == null || snapshot != null;
-        this.jfo = file != null ?
-            FileObjects.sourceFileObject(file, root, JavaFileFilterQuery.getFilter(file), snapshot.getText()) :
-            null;
         this.javacTask = javacTask;
         this.diagnosticListener = diagnosticListener;
         this.isClassFile = false;
         this.isDetached = detached;
+        this.ide2javacFileObject = ide2javacFileObject;
+        this.fileObject2Snapshot = fileObject2Snapshot;
+        this.jfo = file != null ?
+            getSourceFileObject(file) :
+            null;
+    }
+
+    private AbstractSourceFileObject getSourceFileObject(FileObject file) throws IOException {
+        AbstractSourceFileObject result = ide2javacFileObject.get(file);
+
+        if (result == null) {
+            Snapshot snapshot = fileObject2Snapshot.get(file);
+
+            if (snapshot != null) {
+                result = FileObjects.sourceFileObject(file, root, JavaFileFilterQuery.getFilter(file), snapshot.getText());
+            } else {
+                result = FileObjects.sourceFileObject(file, root); //TODO: filter?
+            }
+
+            ide2javacFileObject.put(file, result);
+        }
+
+        return result;
     }
 
     /**
@@ -346,7 +373,7 @@ public Document getDocument() {
         return null;
     }
 
-    public Map<String, CompilationUnitTree> getParsedTrees() {
+    public Map<JavaFileObject, CompilationUnitTree> getParsedTrees() {
         return this.parsedTrees;
     }
 
@@ -407,21 +434,26 @@ public JavaSource.Phase toPhase(JavaSource.Phase phase, List<FileObject> forcedS
      * @return JavacTaskImpl
      */
     public synchronized JavacTaskImpl getJavacTask() {
-        return getJavacTask(Collections.emptyList());
+        try {
+            return getJavacTask(Collections.emptyList());
+        } catch (IOException ex) {
+            //should not happen
+            throw new IllegalStateException(ex);
+        }
     }
 
     /**
      * Returns {@link JavacTaskImpl}, when it doesn't exist
      * it's created.
      * @return JavacTaskImpl
      */
-    public synchronized JavacTaskImpl getJavacTask(List<FileObject> forcedSources) {
+    public synchronized JavacTaskImpl getJavacTask(List<FileObject> forcedSources) throws IOException {
         if (javacTask == null) {
             List<JavaFileObject> jfos = new ArrayList<>();
             if (jfo != null) {
                 jfos.add(jfo);
                 forcedSources.stream()
-                             .map(fo -> FileObjects.sourceFileObject(fo, root)) //TODO: filter?
+                             .map(fo -> runAndThrow(this::getSourceFileObject, fo))
                              .forEach(jfos::add);
             }
             diagnosticListener = new DiagnosticListenerImpl(this.root, this.jfo, this.cpInfo);
@@ -503,23 +535,52 @@ void setCompilationUnit(final CompilationUnitTree compilationUnit) {
         this.compilationUnit = compilationUnit;
     }
 
-    public void setParsedTrees(Map<String, CompilationUnitTree> parsedTrees) {
+    public void setParsedTrees(Map<JavaFileObject, CompilationUnitTree> parsedTrees) {
         this.parsedTrees = parsedTrees;
     }
 
     private boolean hasSource() {
         return this.jfo != null && !isClassFile;
     }
 
-    List<JavaFileObject> parsedFiles;
-    void setParsedFiles(List<JavaFileObject> parsedFiles) {
-        this.parsedFiles = parsedFiles;
+    List<AbstractSourceFileObject> getFiles(List<FileObject> sourceFiles) throws IOException {
+        return sourceFiles.stream()
+                          .map(fo -> runAndThrow(this::getSourceFileObject, fo))
+                          .collect(Collectors.toList());
     }
 
-    List<JavaFileObject> getParsedFiles() {
-        return parsedFiles;
+    private <P, R> R runAndThrow(Convert<P, R> run, P p) {
+        try {
+            return run.run(p);
+        } catch (Exception ex) {
+            throw this.<RuntimeException>thrw(ex);
+        }
     }
-
+
+    private <T extends Exception> RuntimeException thrw(Exception e) throws T {
+        throw (T) e;
+    }
+
+    interface Convert<P, R> {
+        public R run(P p) throws Exception;
+    }
+
+    public Map<FileObject, AbstractSourceFileObject> getIde2javacFileObject() {
+        return ide2javacFileObject;
+    }
+
+    public Map<FileObject, Snapshot> getFileObject2Snapshot() {
+        return fileObject2Snapshot;
+    }
+
+    public boolean isIncomplete() {
+        return incomplete;
+    }
+
+    public void markIncomplete() {
+        this.incomplete = true;
+    }
+
     // Innerclasses ------------------------------------------------------------
     @Trusted
     public static class DiagnosticListenerImpl implements DiagnosticListener<JavaFileObject> {