From 2558300a79ee9e8fd18c2a4a8fea0283ccf6c8a3 Mon Sep 17 00:00:00 2001 From: Jan Lahoda Date: Sat, 15 May 2021 13:56:59 +0200 Subject: [PATCH 1/5] Trying to dynamically switch from all-file processing to single-file processing of multi file JavaSources. --- ide/parsing.api/nbproject/project.properties | 2 +- .../modules/parsing/api/ParserManager.java | 12 +- .../modules/parsing/impl/ParserAccessor.java | 1 + .../netbeans/modules/parsing/spi/Parser.java | 22 ++- .../source/parsing/CompilationInfoImpl.java | 93 +++++++++--- .../java/source/parsing/JavacParser.java | 136 +++++++++++++----- .../source/parsing/JavacParserResult.java | 5 + .../java/source/parsing/JavacParserTest.java | 126 ++++++++++++---- .../netbeans/lib/nbjavac/services/NBAttr.java | 13 ++ 9 files changed, 320 insertions(+), 90 deletions(-) diff --git a/ide/parsing.api/nbproject/project.properties b/ide/parsing.api/nbproject/project.properties index 549ffe072766..8b1db54ac46d 100644 --- a/ide/parsing.api/nbproject/project.properties +++ b/ide/parsing.api/nbproject/project.properties @@ -19,7 +19,7 @@ javac.compilerargs=-Xlint -Xlint:-serial javac.source=1.8 javadoc.apichanges=${basedir}/apichanges.xml javadoc.arch=${basedir}/arch.xml -spec.version.base=9.19.0 +spec.version.base=9.20.0 test.config.stableBTD.includes=**/*Test.class test.config.stableBTD.excludes=\ diff --git a/ide/parsing.api/src/org/netbeans/modules/parsing/api/ParserManager.java b/ide/parsing.api/src/org/netbeans/modules/parsing/api/ParserManager.java index 9b710933a637..5698c28ce49c 100644 --- a/ide/parsing.api/src/org/netbeans/modules/parsing/api/ParserManager.java +++ b/ide/parsing.api/src/org/netbeans/modules/parsing/api/ParserManager.java @@ -20,10 +20,12 @@ package org.netbeans.modules.parsing.api; import java.lang.ref.Reference; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.concurrent.Future; import org.netbeans.api.annotations.common.NonNull; @@ -138,20 +140,21 @@ public Void run () throws Exception { private static class MultiUserTaskAction implements Mutex.ExceptionAction { private final UserTask userTask; - private final Collection sources; + private final List sources; public MultiUserTaskAction (final Collection sources, final UserTask userTask) { assert sources != null; assert userTask != null; this.userTask = userTask; - this.sources = sources; + this.sources = new ArrayList<>(sources); } public Void run () throws Exception { final LowMemoryWatcher lMListener = LowMemoryWatcher.getInstance(); Parser parser = null; final Collection snapShots = new LazySnapshots(sources); - for (Source source : sources) { + for (int i = 0; i < sources.size(); ) { + Source source = sources.get(i); if (parser == null) { Lookup lookup = MimeLookup.getLookup (source.getMimeType ()); ParserFactory parserFactory = lookup.lookup (ParserFactory.class); @@ -165,6 +168,9 @@ public Void run () throws Exception { try { TaskProcessor.callUserTask(userTask, resultIterator); } finally { + if (ParserAccessor.getINSTANCE().processingFinished(resultIterator.getParserResult())) { + i++; + } ResultIteratorAccessor.getINSTANCE().invalidate(resultIterator); SourceAccessor.getINSTANCE().getAndSetCache(source, origCache); } diff --git a/ide/parsing.api/src/org/netbeans/modules/parsing/impl/ParserAccessor.java b/ide/parsing.api/src/org/netbeans/modules/parsing/impl/ParserAccessor.java index 3e279f661e2b..066c0281a14a 100644 --- a/ide/parsing.api/src/org/netbeans/modules/parsing/impl/ParserAccessor.java +++ b/ide/parsing.api/src/org/netbeans/modules/parsing/impl/ParserAccessor.java @@ -48,5 +48,6 @@ public static void setINSTANCE (final ParserAccessor _instance) { } public abstract void invalidate (Parser.Result result); + public abstract boolean processingFinished (Parser.Result result); } diff --git a/ide/parsing.api/src/org/netbeans/modules/parsing/spi/Parser.java b/ide/parsing.api/src/org/netbeans/modules/parsing/spi/Parser.java index 33bcaa8325c3..8648f2326b4e 100644 --- a/ide/parsing.api/src/org/netbeans/modules/parsing/spi/Parser.java +++ b/ide/parsing.api/src/org/netbeans/modules/parsing/spi/Parser.java @@ -142,7 +142,19 @@ public Snapshot getSnapshot () { * This method is called by Parsing API, when {@link Task} is finished. */ protected abstract void invalidate (); - + + /** + * Return {@code true} if the parser was able to process the file fully. + * + * Return {@code false}, if the parser had to stop the processing (e.g. for + * memory limit reasons), and the file should be processed again. + * + * @return whether file processing finished or not + * @since 9.20 + */ + protected boolean processingFinished() { + return true; + } } /** @@ -183,7 +195,13 @@ public void invalidate ( assert result != null; result.invalidate(); } - + + @Override + public boolean processingFinished(Result result) { + assert result != null; + return result.processingFinished(); + } + } } diff --git a/java/java.source.base/src/org/netbeans/modules/java/source/parsing/CompilationInfoImpl.java b/java/java.source.base/src/org/netbeans/modules/java/source/parsing/CompilationInfoImpl.java index de33a422e192..ddbccfed49fb 100644 --- a/java/java.source.base/src/org/netbeans/modules/java/source/parsing/CompilationInfoImpl.java +++ b/java/java.source.base/src/org/netbeans/modules/java/source/parsing/CompilationInfoImpl.java @@ -44,6 +44,8 @@ import java.util.Map.Entry; import java.util.SortedMap; import java.util.TreeMap; +import java.util.function.Function; +import java.util.stream.Collectors; import javax.swing.text.Document; import javax.tools.Diagnostic; @@ -93,7 +95,10 @@ public final class CompilationInfoImpl { JavaSource.Phase parserCrashed = JavaSource.Phase.UP_TO_DATE; //When javac throws an error, the moveToPhase sets this to the last safe phase private final Map> userCache = new EnumMap>(CacheClearPolicy.class); //cache of already parsed files - private Map parsedTrees; + private Map parsedTrees; + private Map ide2javacFileObject; + private Map fileObject2Snapshot; + private boolean incomplete; /** * Creates a new CompilationInfoImpl for given source file @@ -111,7 +116,9 @@ public CompilationInfoImpl (final JavacParser parser, final JavacTaskImpl javacTask, final DiagnosticListener diagnosticListener, final Snapshot snapshot, - final boolean detached) throws IOException { + final boolean detached, + final Map ide2javacFileObject, + final Map fileObject2Snapshot) throws IOException { assert parser != null; this.parser = parser; this.cpInfo = parser.getClasspathInfo(); @@ -121,13 +128,33 @@ public CompilationInfoImpl (final JavacParser parser, this.snapshot = snapshot; this.partialReparseLastGoodSnapshot = new SoftReference<>(snapshot); assert file == null || snapshot != null; - this.jfo = file != null ? - FileObjects.sourceFileObject(file, root, JavaFileFilterQuery.getFilter(file), snapshot.getText()) : - null; this.javacTask = javacTask; this.diagnosticListener = diagnosticListener; this.isClassFile = false; this.isDetached = detached; + this.ide2javacFileObject = ide2javacFileObject; + this.fileObject2Snapshot = fileObject2Snapshot; + this.jfo = file != null ? + getSourceFileObject(file) : + null; + } + + private AbstractSourceFileObject getSourceFileObject(FileObject file) throws IOException { + AbstractSourceFileObject result = ide2javacFileObject.get(file); + + if (result == null) { + Snapshot snapshot = fileObject2Snapshot.get(file); + + if (snapshot != null) { + result = FileObjects.sourceFileObject(file, root, JavaFileFilterQuery.getFilter(file), snapshot.getText()); + } else { + result = FileObjects.sourceFileObject(file, root); //TODO: filter? + } + + ide2javacFileObject.put(file, result); + } + + return result; } /** @@ -346,7 +373,7 @@ public Document getDocument() { return null; } - public Map getParsedTrees() { + public Map getParsedTrees() { return this.parsedTrees; } @@ -407,7 +434,12 @@ public JavaSource.Phase toPhase(JavaSource.Phase phase, List forcedS * @return JavacTaskImpl */ public synchronized JavacTaskImpl getJavacTask() { - return getJavacTask(Collections.emptyList()); + try { + return getJavacTask(Collections.emptyList()); + } catch (IOException ex) { + //should not happen + throw new IllegalStateException(ex); + } } /** @@ -415,13 +447,13 @@ public synchronized JavacTaskImpl getJavacTask() { * it's created. * @return JavacTaskImpl */ - public synchronized JavacTaskImpl getJavacTask(List forcedSources) { + public synchronized JavacTaskImpl getJavacTask(List forcedSources) throws IOException { if (javacTask == null) { List jfos = new ArrayList<>(); if (jfo != null) { jfos.add(jfo); forcedSources.stream() - .map(fo -> FileObjects.sourceFileObject(fo, root)) //TODO: filter? + .map(fo -> runAndThrow(this::getSourceFileObject, fo)) .forEach(jfos::add); } diagnosticListener = new DiagnosticListenerImpl(this.root, this.jfo, this.cpInfo); @@ -503,7 +535,7 @@ void setCompilationUnit(final CompilationUnitTree compilationUnit) { this.compilationUnit = compilationUnit; } - public void setParsedTrees(Map parsedTrees) { + public void setParsedTrees(Map parsedTrees) { this.parsedTrees = parsedTrees; } @@ -511,15 +543,44 @@ private boolean hasSource() { return this.jfo != null && !isClassFile; } - List parsedFiles; - void setParsedFiles(List parsedFiles) { - this.parsedFiles = parsedFiles; + List getFiles(List sourceFiles) throws IOException { + return sourceFiles.stream() + .map(fo -> runAndThrow(this::getSourceFileObject, fo)) + .collect(Collectors.toList()); } - List getParsedFiles() { - return parsedFiles; + private R runAndThrow(Convert run, P p) { + try { + return run.run(p); + } catch (Exception ex) { + throw this.thrw(ex); + } } - + + private RuntimeException thrw(Exception e) throws T { + throw (T) e; + } + + interface Convert { + public R run(P p) throws Exception; + } + + public Map getIde2javacFileObject() { + return ide2javacFileObject; + } + + public Map getFileObject2Snapshot() { + return fileObject2Snapshot; + } + + public boolean isIncomplete() { + return incomplete; + } + + public void markIncomplete() { + this.incomplete = true; + } + // Innerclasses ------------------------------------------------------------ @Trusted public static class DiagnosticListenerImpl implements DiagnosticListener { diff --git a/java/java.source.base/src/org/netbeans/modules/java/source/parsing/JavacParser.java b/java/java.source.base/src/org/netbeans/modules/java/source/parsing/JavacParser.java index 6af9650d6880..0ede34b0933f 100644 --- a/java/java.source.base/src/org/netbeans/modules/java/source/parsing/JavacParser.java +++ b/java/java.source.base/src/org/netbeans/modules/java/source/parsing/JavacParser.java @@ -117,6 +117,7 @@ import org.netbeans.modules.parsing.api.Task; import org.netbeans.modules.parsing.api.UserTask; import org.netbeans.modules.parsing.impl.Utilities; +import org.netbeans.modules.parsing.spi.LowMemoryWatcher; import org.netbeans.modules.parsing.spi.ParseException; import org.netbeans.modules.parsing.spi.Parser; import org.netbeans.modules.parsing.spi.ParserResultTask; @@ -154,6 +155,8 @@ public class JavacParser extends Parser { //Command line switch disabling partial reparse private static final boolean DISABLE_PARTIAL_REPARSE = Boolean.getBoolean("org.netbeans.modules.java.source.parsing.JavacParser.no_reparse"); //NOI18N private static final boolean DISABLE_PARAMETER_NAMES_READING = Boolean.getBoolean("org.netbeans.modules.java.source.parsing.JavacParser.no_parameter_names"); //NOI18N + private static final Set> HUGE_SNAPSHOTS = new HashSet<>(); + private static final LowMemoryWatcher LOW_MEMORY_WATCHER = LowMemoryWatcher.getInstance(); public static final String LOMBOK_DETECTED = "lombokDetected"; /** @@ -171,6 +174,7 @@ public class JavacParser extends Parser { private final ChangeSupport listeners = new ChangeSupport(this); //Cancelling of parser private final AtomicBoolean parserCanceled = new AtomicBoolean(); + private final AtomicBoolean lowMemoryCancel = new AtomicBoolean(); //Cancelling of index private final AtomicBoolean indexCanceled = new AtomicBoolean(); @@ -184,8 +188,6 @@ public class JavacParser extends Parser { private ClasspathInfo cpInfo; //all the files for which parser was created for private final Collection snapshots; - //size of all the files to check the memory leack - private long snapshotSize; //Incremental parsing support private final boolean supportsReparse; //Incremental parsing support @@ -212,16 +214,13 @@ public class JavacParser extends Parser { private ChangeListener weakCpListener; //Current source for parse optmalization of task with no Source (identity) private Reference currentSource; + private final boolean perFileProcessing; JavacParser (final Collection snapshots, boolean privateParser) { this.privateParser = privateParser; this.snapshots = snapshots; final boolean singleJavaFile = this.snapshots.size() == 1 && MIME_TYPE.equals(snapshots.iterator().next().getSource().getMimeType()); this.supportsReparse = singleJavaFile && !DISABLE_PARTIAL_REPARSE; - this.snapshotSize=0; - for (Snapshot snapshot : snapshots) { - this.snapshotSize+=snapshot.getSource().getFileObject().getSize(); - } JavaFileFilterImplementation filter = null; if (singleJavaFile) { final Source source = snapshots.iterator().next().getSource(); @@ -243,6 +242,27 @@ public void run() { } }); this.sequentialParsing = Lookup.getDefault().lookup(SequentialParsing.class); + this.perFileProcessing = perFileProcessing(); + } + + private boolean perFileProcessing() { + if (snapshots.size() <= 1) { + return true; + } + + boolean result = false; + + for (Iterator> it = HUGE_SNAPSHOTS.iterator(); it.hasNext();) { + Reference ref = it.next(); + Object obj = ref.get(); + if (obj == null) { + it.remove(); + } else if (obj == snapshots) { + result = true; + } + } + + return result; } private void init (final Snapshot snapshot, final Task task, final boolean singleSource) { @@ -385,13 +405,13 @@ private void parseImpl( assert privateParser || Utilities.holdsParserLock(); parseId++; parserCanceled.set(false); + lowMemoryCancel.set(false); indexCanceled.set(false); cachedSnapShot = snapshot; LOGGER.log(Level.FINE, "parse: task: {0}\n{1}", new Object[]{ //NOI18N task.toString(), snapshot == null ? "null" : snapshot.getText()}); //NOI18N final CompilationInfoImpl oldInfo = ciImpl; - Map oldParsedTrees = new HashMap<>(); boolean success = false; try { switch (this.snapshots.size()) { @@ -417,7 +437,7 @@ private void parseImpl( } if (needsFullReparse) { positions.clear(); - ciImpl = createCurrentInfo(this, file, root, snapshot, null, null, oldParsedTrees); + ciImpl = createCurrentInfo(this, file, root, snapshot, null, null, new HashMap<>(), new HashMap<>(), Collections.singletonMap(file, snapshot)); LOGGER.fine("\t:created new javac"); //NOI18N } break; @@ -425,26 +445,30 @@ private void parseImpl( init(snapshot, task, false); DiagnosticListener diagnosticListener; JavacTaskImpl javacTask; - boolean oneInstanceJava=Boolean.getBoolean("java.enable.single.javac") && this.snapshotSize <= this.MAX_FILE_SIZE; - if (sequentialParsing == null && ciImpl == null && oneInstanceJava) { - List jfos = new ArrayList<>(); - for (Snapshot s : snapshots) { - jfos.add(FileObjects.sourceFileObject(s.getSource().getFileObject(), root, JavaFileFilterQuery.getFilter(s.getSource().getFileObject()), s.getText())); - } - diagnosticListener = new CompilationInfoImpl.DiagnosticListenerImpl(this.root, jfos.get(0), this.cpInfo); - javacTask = JavacParser.createJavacTask(this.file, jfos, this.root, this.cpInfo, - this, diagnosticListener, false); - } else if (ciImpl != null && (sequentialParsing != null || oneInstanceJava)) { + Map oldParsedTrees; + Map ide2javacFileObject; + Map file2Snapshot; + + if (ciImpl != null && !perFileProcessing) { diagnosticListener = ciImpl.getDiagnosticListener(); javacTask = ciImpl.getJavacTask(); oldParsedTrees = ciImpl.getParsedTrees(); + ide2javacFileObject = ciImpl.getIde2javacFileObject(); + file2Snapshot = ciImpl.getFileObject2Snapshot(); } else { diagnosticListener = null; javacTask = null; + oldParsedTrees = new HashMap<>(); + ide2javacFileObject = new HashMap<>(); + file2Snapshot = new HashMap<>(); + for (Snapshot s : snapshots) { + file2Snapshot.put(s.getSource().getFileObject(), s); + } } ciImpl = createCurrentInfo(this, file, root, snapshot, javacTask, - diagnosticListener,oldParsedTrees); + diagnosticListener, oldParsedTrees, ide2javacFileObject, + file2Snapshot); } success = true; } finally { @@ -538,7 +562,7 @@ else if (isUserTask) { final NewComilerTask nct = (NewComilerTask)task; if (nct.getCompilationController() == null || nct.getTimeStamp() != parseId) { try { - CompilationInfoImpl cii = new CompilationInfoImpl(this, file, root, null, null, cachedSnapShot, true); + CompilationInfoImpl cii = new CompilationInfoImpl(this, file, root, null, null, cachedSnapShot, true, new HashMap<>(), new HashMap<>()); cii.setParsedTrees(new HashMap<>()); nct.setCompilationController(JavaSourceAccessor.getINSTANCE().createCompilationController(cii), parseId); @@ -616,14 +640,27 @@ Phase moveToPhase (final Phase phase, final CompilationInfoImpl currentInfo, Lis Iterable trees = null; Iterator it = null; CompilationUnitTree unit = null; - if (snapshots.size() > 1 && currentInfo.getParsedTrees() != null && currentInfo.getParsedTrees().containsKey(currentInfo.jfo.getName())) { - unit = currentInfo.getParsedTrees().get(currentInfo.jfo.getName()); + if (currentInfo.getParsedTrees() != null && currentInfo.getParsedTrees().containsKey(currentInfo.jfo)) { + unit = currentInfo.getParsedTrees().get(currentInfo.jfo); } else { - if (sequentialParsing != null) { trees = sequentialParsing.parse(currentInfo.getJavacTask(), currentInfo.jfo); } else { - trees = currentInfo.getJavacTask(forcedSources).parse(); + List files = new ArrayList<>(); + + if (perFileProcessing) { + files.add(file); + } else { + snapshots.stream() + .map(s -> s.getSource().getFileObject()) + .forEach(files::add); + } + + files.addAll(forcedSources); + + JavacTaskImpl javacTask = currentInfo.getJavacTask(files); + + trees = javacTask.parse(); } if (unit == null) { if (trees == null) { @@ -640,9 +677,8 @@ Phase moveToPhase (final Phase phase, final CompilationInfoImpl currentInfo, Lis while (it.hasNext()) { CompilationUnitTree oneFileTree = it.next(); parsedFiles.add(oneFileTree.getSourceFile()); - CompilationUnitTree put = currentInfo.getParsedTrees().put(oneFileTree.getSourceFile().getName(), oneFileTree); + CompilationUnitTree put = currentInfo.getParsedTrees().put(oneFileTree.getSourceFile(), oneFileTree); } - currentInfo.setParsedFiles(parsedFiles); unit = trees.iterator().next(); } @@ -724,7 +760,11 @@ Phase moveToPhase (final Phase phase, final CompilationInfoImpl currentInfo, Lis JavaCompiler compiler = JavaCompiler.instance(jti.getContext()); List> savedTodo = new ArrayList<>(compiler.todo); try { - compiler.todo.retainFiles(currentInfo.getParsedFiles()); + List currentFileObjects = new ArrayList<>(); + currentFileObjects.addAll(forcedSources); + currentFileObjects.add(file); + List currentFiles = currentInfo.getFiles(currentFileObjects); + compiler.todo.retainFiles(currentFiles); savedTodo.removeAll(compiler.todo); PostFlowAnalysis.analyze(jti.analyze(), jti.getContext()); } finally { @@ -740,18 +780,29 @@ Phase moveToPhase (final Phase phase, final CompilationInfoImpl currentInfo, Lis currentPhase = Phase.UP_TO_DATE; } } catch (CancelAbort ca) { - currentPhase = Phase.MODIFIED; - invalidate(false); + if (lowMemoryCancel.get()) { + currentInfo.markIncomplete(); + HUGE_SNAPSHOTS.add(new WeakReference<>(snapshots)); + } else { + //real cancel + currentPhase = Phase.MODIFIED; + invalidate(false); + } } catch (Abort abort) { parserError = currentPhase; } catch (RuntimeException | Error ex) { - if (cancellable && parserCanceled.get()) { - currentPhase = Phase.MODIFIED; - invalidate(false); + if (lowMemoryCancel.get()) { + currentInfo.markIncomplete(); + HUGE_SNAPSHOTS.add(new WeakReference<>(snapshots)); } else { - parserError = currentPhase; - dumpSource(currentInfo, ex); - throw ex; + if (cancellable && parserCanceled.get()) { + currentPhase = Phase.MODIFIED; + invalidate(false); + } else { + parserError = currentPhase; + dumpSource(currentInfo, ex); + throw ex; + } } } finally { currentInfo.setPhase(currentPhase); @@ -766,8 +817,10 @@ private static CompilationInfoImpl createCurrentInfo (final JavacParser parser, final Snapshot snapshot, final JavacTaskImpl javac, final DiagnosticListener diagnosticListener, - final Map parsedTrees) throws IOException { - CompilationInfoImpl info = new CompilationInfoImpl(parser, file, root, javac, diagnosticListener, snapshot, false); + final Map parsedTrees, + final Map ide2javacFileObject, + final Map file2Snapshot) throws IOException { + CompilationInfoImpl info = new CompilationInfoImpl(parser, file, root, javac, diagnosticListener, snapshot, false, ide2javacFileObject, file2Snapshot); if (file != null) { Logger.getLogger("TIMER").log(Level.FINE, "CompilationInfo", //NOI18N new Object[] {file, info}); @@ -1326,7 +1379,14 @@ public static DefaultCancelService instance(final Context ctx) { @Override public boolean isCanceled() { - return mayCancel.get() && parser.parserCanceled.get(); + if (!mayCancel.get() && parser.parserCanceled.get()) { + return true; + } + if (!parser.perFileProcessing && LOW_MEMORY_WATCHER.isLowMemory()) { + parser.lowMemoryCancel.set(true); + return true; + } + return false; } } diff --git a/java/java.source.base/src/org/netbeans/modules/java/source/parsing/JavacParserResult.java b/java/java.source.base/src/org/netbeans/modules/java/source/parsing/JavacParserResult.java index 8446e430273a..a7139de9c00d 100644 --- a/java/java.source.base/src/org/netbeans/modules/java/source/parsing/JavacParserResult.java +++ b/java/java.source.base/src/org/netbeans/modules/java/source/parsing/JavacParserResult.java @@ -20,6 +20,7 @@ package org.netbeans.modules.java.source.parsing; import org.netbeans.api.java.source.CompilationInfo; +import org.netbeans.modules.java.source.CompilationInfoAccessor; import org.netbeans.modules.java.source.JavaSourceAccessor; import org.netbeans.modules.parsing.spi.Parser; import org.openide.util.Parameters; @@ -57,4 +58,8 @@ public void invalidate() { JavaSourceAccessor.getINSTANCE().invalidate (info); } + @Override + public boolean processingFinished() { + return !CompilationInfoAccessor.getInstance().getCompilationInfoImpl(info).isIncomplete(); + } } diff --git a/java/java.source.base/test/unit/src/org/netbeans/modules/java/source/parsing/JavacParserTest.java b/java/java.source.base/test/unit/src/org/netbeans/modules/java/source/parsing/JavacParserTest.java index e4dbec945121..d772e3b94774 100644 --- a/java/java.source.base/test/unit/src/org/netbeans/modules/java/source/parsing/JavacParserTest.java +++ b/java/java.source.base/test/unit/src/org/netbeans/modules/java/source/parsing/JavacParserTest.java @@ -32,14 +32,19 @@ import java.io.OutputStreamWriter; import java.net.URI; import java.net.URL; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiPredicate; import javax.lang.model.element.TypeElement; import javax.swing.event.ChangeListener; import javax.swing.text.Document; @@ -56,6 +61,7 @@ import org.netbeans.api.java.source.CompilationController; import org.netbeans.api.java.source.JavaSource; import org.netbeans.api.java.source.JavaSource.Phase; +import org.netbeans.api.java.source.SourceUtils; import org.netbeans.api.java.source.SourceUtilsTestUtil; import org.netbeans.api.java.source.Task; import org.netbeans.api.java.source.TestUtilities; @@ -78,6 +84,7 @@ import org.netbeans.modules.parsing.api.ResultIterator; import org.netbeans.modules.parsing.api.Snapshot; import org.netbeans.modules.parsing.api.UserTask; +import org.netbeans.modules.parsing.lucene.support.LowMemoryWatcherAccessor; import org.netbeans.modules.parsing.spi.EmbeddingProvider; import org.netbeans.modules.parsing.spi.ParseException; import org.netbeans.modules.parsing.spi.Parser; @@ -118,12 +125,13 @@ public void testMultiSource() throws Exception { js.runUserActionTask(new Task() { TypeElement storedJLObject; public void run(CompilationController parameter) throws Exception { + SourceUtils.forceSource(parameter, f1); + assertEquals(Phase.PARSED, parameter.toPhase(Phase.PARSED)); if ("Test3".equals(parameter.getFileObject().getName())) { TypeElement te = parameter.getElements().getTypeElement("test.Test1"); assertNotNull(te); assertNotNull(parameter.getTrees().getPath(te)); } - assertEquals(Phase.PARSED, parameter.toPhase(Phase.PARSED)); assertNotNull(parameter.getCompilationUnit()); TypeElement jlObject = parameter.getElements().getTypeElement("java.lang.Object"); @@ -136,37 +144,95 @@ public void run(CompilationController parameter) throws Exception { }, true); } - public void testMultiSourceVanilla() throws Exception { - Lookup noSP = Lookups.exclude(Lookup.getDefault(), JavacParser.SequentialParsing.class); - GlobalLookup.execute(noSP, () -> { - try { - FileObject f1 = createFile("test/Test1.java", "package test; class Test1"); - FileObject f2 = createFile("test/Test2.java", "package test; class Test2{}"); - FileObject f3 = createFile("test/Test3.java", "package test; class Test3{}"); - - ClasspathInfo cpInfo = ClasspathInfo.create(f2); - JavaSource js = JavaSource.create(cpInfo, f2, f3); - - SourceUtilsTestUtil.compileRecursively(sourceRoot); - - js.runUserActionTask(new Task() { - TypeElement storedJLObject; - public void run(CompilationController parameter) throws Exception { - assertEquals(Phase.PARSED, parameter.toPhase(Phase.PARSED)); - assertNotNull(parameter.getCompilationUnit()); - TypeElement jlObject = parameter.getElements().getTypeElement("java.lang.Object"); - - if (storedJLObject == null) { - storedJLObject = jlObject; - } else { - assertFalse(Objects.equals(storedJLObject, jlObject)); + public void testMultiSourceOutOfMemory() throws Exception { + FileObject f1 = createFile("test/Test1.java", "package test; class Test1 {}"); + FileObject f2 = createFile("test/Test2.java", "package test; class Test2 {}"); + FileObject f3 = createFile("test/Test3.java", "package test; class Test3 {}"); + + ClasspathInfo cpInfo = ClasspathInfo.create(f1); + JavaSource js = JavaSource.create(cpInfo, f1, f2, f3); + + class TestCase { + BiPredicate stopAt; + List expectedLog; + public TestCase(BiPredicate stopAt, String... expectedLog) { + this.stopAt = stopAt; + this.expectedLog = Arrays.asList(expectedLog); + } + } + + TestCase[] cases = new TestCase[] { + new TestCase((phase, file) -> phase == Phase.PARSED && file.equals(f1), + "failed: Test1", + "success: Test1, phase: PARSED", "0", + "success: Test1, phase: ELEMENTS_RESOLVED", "1", + "success: Test1, phase: RESOLVED", "1", + "success: Test2, phase: PARSED", "2", + "success: Test2, phase: ELEMENTS_RESOLVED", "3", + "success: Test2, phase: RESOLVED", "3", + "success: Test3, phase: PARSED", "4", + "success: Test3, phase: ELEMENTS_RESOLVED", "5", + "success: Test3, phase: RESOLVED", "5"), + new TestCase((phase, file) -> phase == Phase.ELEMENTS_RESOLVED && file.equals(f1), + "success: Test1, phase: PARSED", "0", + "failed: Test1", + "success: Test1, phase: PARSED", "1", + "success: Test1, phase: ELEMENTS_RESOLVED", "2", + "success: Test1, phase: RESOLVED", "2", + "success: Test2, phase: PARSED", "3", + "success: Test2, phase: ELEMENTS_RESOLVED", "4", + "success: Test2, phase: RESOLVED", "4", + "success: Test3, phase: PARSED", "5", + "success: Test3, phase: ELEMENTS_RESOLVED", "6", + "success: Test3, phase: RESOLVED", "6"), + new TestCase((phase, file) -> phase == Phase.RESOLVED && file.equals(f2), + "success: Test1, phase: PARSED", "0", + "success: Test1, phase: ELEMENTS_RESOLVED", "1", + "success: Test1, phase: RESOLVED", "1", + "success: Test2, phase: PARSED", "2", + "success: Test2, phase: ELEMENTS_RESOLVED", "1", + "failed: Test2", + "success: Test2, phase: PARSED", "3", + "success: Test2, phase: ELEMENTS_RESOLVED", "4", + "success: Test2, phase: RESOLVED", "4", + "success: Test3, phase: PARSED", "5", + "success: Test3, phase: ELEMENTS_RESOLVED", "6", + "success: Test3, phase: RESOLVED", "6") + }; + + for (TestCase testCase : cases) { + List log = new ArrayList<>(); + + js.runUserActionTask(new Task() { + Map keys = new IdentityHashMap<>(); + boolean wasCanceled; + public void run(CompilationController cc) throws Exception { + for (Phase p : new Phase[] {Phase.PARSED, Phase.ELEMENTS_RESOLVED, Phase.RESOLVED}) { + LowMemoryWatcherAccessor.setLowMemory(!wasCanceled && testCase.stopAt.test(p, cc.getFileObject())); + if (cc.toPhase(p).compareTo(p) < 0) { + log.add("failed: " + cc.getFileObject().getName()); + wasCanceled = true; + return ; + } + log.add("success: " + cc.getFileObject().getName() + ", phase: " + p); + Object key; + switch (p) { + case PARSED: key = cc.getCompilationUnit(); break; + case ELEMENTS_RESOLVED: + case RESOLVED: + key = cc.getElements().getTypeElement("java.lang.Object"); + break; + default: + throw new IllegalStateException(p.name()); } + log.add("" + keys.computeIfAbsent(key, k -> keys.size())); } - }, true); - } catch (Exception ex) { - throw new AssertionError(ex); - } - }); + } + }, true); + + assertEquals(testCase.expectedLog, log); + LowMemoryWatcherAccessor.setLowMemory(false); + } } public void test199332() throws Exception { diff --git a/java/lib.nbjavac/src/org/netbeans/lib/nbjavac/services/NBAttr.java b/java/lib.nbjavac/src/org/netbeans/lib/nbjavac/services/NBAttr.java index e3cde76702d1..a564f6079b54 100644 --- a/java/lib.nbjavac/src/org/netbeans/lib/nbjavac/services/NBAttr.java +++ b/java/lib.nbjavac/src/org/netbeans/lib/nbjavac/services/NBAttr.java @@ -18,6 +18,7 @@ */ package org.netbeans.lib.nbjavac.services; +import com.sun.tools.javac.code.Symbol.ClassSymbol; import com.sun.tools.javac.code.Type; import com.sun.tools.javac.comp.Attr; import com.sun.tools.javac.comp.AttrContext; @@ -31,6 +32,7 @@ import com.sun.tools.javac.tree.JCTree.JCVariableDecl; import com.sun.tools.javac.tree.TreeMaker; import com.sun.tools.javac.util.Context; +import com.sun.tools.javac.util.JCDiagnostic.DiagnosticPosition; import com.sun.tools.javac.util.List; import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodType; @@ -58,6 +60,11 @@ public NBAttr(Context context) { tm = TreeMaker.instance(context); } + @Override + public void attribClass(DiagnosticPosition pos, ClassSymbol c) { + super.attribClass(pos, c); //To change body of generated methods, choose Tools | Templates. + } + @Override public void visitClassDef(JCClassDecl tree) { cancelService.abortIfCanceled(); @@ -83,6 +90,12 @@ public void visitVarDef(JCVariableDecl tree) { super.visitVarDef(tree); } + @Override + public Type attribType(JCTree tree, Env env) { + cancelService.abortIfCanceled(); + return super.attribType(tree, env); + } + @Override public void visitCatch(JCCatch that) { super.visitBlock(tm.Block(0, List.of(that.param, that.body))); From 382494ee9e9737ca7d7068cd6ac5d6939c93a0c4 Mon Sep 17 00:00:00 2001 From: Jan Lahoda Date: Fri, 3 Sep 2021 06:34:09 +0200 Subject: [PATCH 2/5] Enabling tests --- .../src/org/netbeans/api/java/source/JavaSourceTest.java | 4 ---- .../modules/java/source/parsing/JavacParserTest.java | 5 ----- 2 files changed, 9 deletions(-) diff --git a/java/java.source.base/test/unit/src/org/netbeans/api/java/source/JavaSourceTest.java b/java/java.source.base/test/unit/src/org/netbeans/api/java/source/JavaSourceTest.java index b3c7137e0409..eaf8baeff634 100644 --- a/java/java.source.base/test/unit/src/org/netbeans/api/java/source/JavaSourceTest.java +++ b/java/java.source.base/test/unit/src/org/netbeans/api/java/source/JavaSourceTest.java @@ -295,10 +295,6 @@ public void run(CompilationController parameter) throws Exception { } public void testMultipleFilesSameJavac() throws Exception { - if (!NoJavacHelper.hasNbJavac()) { - //nb-javac (NB_JAVAC_): JDK's javac we currently don't correctly run JavaSource with multiple files: - return ; - } final FileObject testFile1 = createTestFile("Test1"); final FileObject testFile2 = createTestFile("Test2"); final ClassPath bootPath = createBootPath(); diff --git a/java/java.source.base/test/unit/src/org/netbeans/modules/java/source/parsing/JavacParserTest.java b/java/java.source.base/test/unit/src/org/netbeans/modules/java/source/parsing/JavacParserTest.java index 689e25c0050b..9d434f4df689 100644 --- a/java/java.source.base/test/unit/src/org/netbeans/modules/java/source/parsing/JavacParserTest.java +++ b/java/java.source.base/test/unit/src/org/netbeans/modules/java/source/parsing/JavacParserTest.java @@ -114,11 +114,6 @@ protected void setUp() throws Exception { } public void testMultiSource() throws Exception { - if (!NoJavacHelper.hasNbJavac()) { - //nb-javac (NB_JAVAC_): JDK's javac we currently don't correctly run JavaSource with multiple files: - return ; - } - FileObject f1 = createFile("test/Test1.java", "package test; class Test1"); FileObject f2 = createFile("test/Test2.java", "package test; class Test2{}"); FileObject f3 = createFile("test/Test3.java", "package test; class Test3{}"); From 11cef3726ef610b18b617b836e37521a0fc4fde0 Mon Sep 17 00:00:00 2001 From: Jan Lahoda Date: Sun, 5 Sep 2021 07:17:58 +0200 Subject: [PATCH 3/5] Fixing cancellation logic. --- .../org/netbeans/modules/java/source/parsing/JavacParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/java.source.base/src/org/netbeans/modules/java/source/parsing/JavacParser.java b/java/java.source.base/src/org/netbeans/modules/java/source/parsing/JavacParser.java index ef7b28120af1..5eadd691094f 100644 --- a/java/java.source.base/src/org/netbeans/modules/java/source/parsing/JavacParser.java +++ b/java/java.source.base/src/org/netbeans/modules/java/source/parsing/JavacParser.java @@ -1379,7 +1379,7 @@ public static DefaultCancelService instance(final Context ctx) { @Override public boolean isCanceled() { - if (!mayCancel.get() && parser.parserCanceled.get()) { + if (mayCancel.get() && parser.parserCanceled.get()) { return true; } if (!parser.perFileProcessing && LOW_MEMORY_WATCHER.isLowMemory()) { From 94f8442006aaf3779c6c3b4ed74a302bd2b08a00 Mon Sep 17 00:00:00 2001 From: Jan Lahoda Date: Sun, 5 Sep 2021 07:22:24 +0200 Subject: [PATCH 4/5] Cleanup. --- ide/parsing.api/apichanges.xml | 15 +++++++++++++++ .../org/netbeans/lib/nbjavac/services/NBAttr.java | 5 ----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/ide/parsing.api/apichanges.xml b/ide/parsing.api/apichanges.xml index 615a7f9307e6..1bb50cfe44e6 100644 --- a/ide/parsing.api/apichanges.xml +++ b/ide/parsing.api/apichanges.xml @@ -87,6 +87,21 @@ is the proper place. + + + Adding Parser.Result.processingFinished + + + + + +

+ Adding a new method, Parser.Result.processingFinished, used to + mark a result as completely finished, or not completely finished. +

+ + + Source provides access to Lookup diff --git a/java/lib.nbjavac/src/org/netbeans/lib/nbjavac/services/NBAttr.java b/java/lib.nbjavac/src/org/netbeans/lib/nbjavac/services/NBAttr.java index a564f6079b54..bca9eb351e6d 100644 --- a/java/lib.nbjavac/src/org/netbeans/lib/nbjavac/services/NBAttr.java +++ b/java/lib.nbjavac/src/org/netbeans/lib/nbjavac/services/NBAttr.java @@ -60,11 +60,6 @@ public NBAttr(Context context) { tm = TreeMaker.instance(context); } - @Override - public void attribClass(DiagnosticPosition pos, ClassSymbol c) { - super.attribClass(pos, c); //To change body of generated methods, choose Tools | Templates. - } - @Override public void visitClassDef(JCClassDecl tree) { cancelService.abortIfCanceled(); From 2a13ad2c70e04794cce105e981b53da21a7db1cd Mon Sep 17 00:00:00 2001 From: Jan Lahoda Date: Fri, 24 Sep 2021 21:14:06 +0200 Subject: [PATCH 5/5] Fixing test. --- .../netbeans/modules/java/source/parsing/JavacParserTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/java/java.source.base/test/unit/src/org/netbeans/modules/java/source/parsing/JavacParserTest.java b/java/java.source.base/test/unit/src/org/netbeans/modules/java/source/parsing/JavacParserTest.java index 9d434f4df689..3d96316a3460 100644 --- a/java/java.source.base/test/unit/src/org/netbeans/modules/java/source/parsing/JavacParserTest.java +++ b/java/java.source.base/test/unit/src/org/netbeans/modules/java/source/parsing/JavacParserTest.java @@ -146,6 +146,10 @@ public void run(CompilationController parameter) throws Exception { } public void testMultiSourceOutOfMemory() throws Exception { + if (NoJavacHelper.hasNbJavac()) { + return ; + } + FileObject f1 = createFile("test/Test1.java", "package test; class Test1 {}"); FileObject f2 = createFile("test/Test2.java", "package test; class Test2 {}"); FileObject f3 = createFile("test/Test3.java", "package test; class Test3 {}");