Add extended-analysis=local to bound data-flow memory on large source trees

CHANGELOG.md

@@ -1,5 +1,9 @@
 ## Git
 ### Added
+- `extended-analysis=local`: a new value for the `--extended-analysis` flag and
+  directive that performs dataflow analysis per file, treating sourced files as
+  opaque boundaries. This bounds memory when many checked files source a shared
+  tree of libraries, while keeping per-file dataflow checks (unlike `false`).
 
 ### Changed
 

shellcheck.1.md

@@ -56,12 +56,17 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts.
     options are cumulative, but all the codes can be specified at once,
     comma-separated as a single argument.
 
-**--extended-analysis=true/false**
-
-:   Enable/disable Dataflow Analysis to identify more issues (default true). If
-    ShellCheck uses too much CPU/RAM when checking scripts with several
-    thousand lines of code, extended analysis can be disabled with this flag
-    or a directive. This flag overrides directives and rc files.
+**--extended-analysis=true/local/false**
+
+:   Set the scope of Dataflow Analysis, which identifies more issues (default
+    true). `true` performs whole-program analysis, following data flow into
+    sourced files. `local` performs the analysis per file, treating sourced
+    files as opaque boundaries; this bounds memory when many checked files
+    source a shared tree of libraries (each includer would otherwise re-analyze
+    the whole tree), at the cost of data flow that crosses `source` boundaries.
+    `false` disables Dataflow Analysis entirely. Use `local` or `false` if
+    ShellCheck uses too much CPU/RAM. This flag overrides directives and rc
+    files.
 
 **-f** *FORMAT*, **--format=***FORMAT*
 
@@ -264,10 +269,13 @@ Valid keys are:
     Only file-wide `enable` directives are considered.
 
 **extended-analysis**
-:   Set to true/false to enable/disable dataflow analysis. Specifying
+:   Set to true/local/false to control dataflow analysis. `true` (default)
+    performs whole-program analysis; `local` performs it per file, treating
+    sourced files as opaque boundaries; `false` disables it. Specifying
     `# shellcheck extended-analysis=false` in particularly large (2000+ line)
     auto-generated scripts will reduce ShellCheck's resource usage at the
-    expense of certain checks. Extended analysis is enabled by default.
+    expense of certain checks. `local` bounds memory when many checked files
+    source a shared tree of libraries, while keeping per-file dataflow checks.
 
 **external-sources**
 :   Set to `true` in `.shellcheckrc` to always allow ShellCheck to open

shellcheck.hs

@@ -18,6 +18,7 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 -}
 import qualified ShellCheck.Analyzer
+import           ShellCheck.AST (ExtendedAnalysisMode(..))
 import           ShellCheck.Checker
 import           ShellCheck.Data
 import           ShellCheck.Interface
@@ -103,7 +104,7 @@ options = [
     Option "e" ["exclude"]
         (ReqArg (Flag "exclude") "CODE1,CODE2..") "Exclude types of warnings",
     Option "" ["extended-analysis"]
-        (ReqArg (Flag "extended-analysis") "bool") "Perform dataflow analysis (default true)",
+        (ReqArg (Flag "extended-analysis") "mode") "Perform dataflow analysis: true|local|false (default true; local treats sourced files as boundaries)",
     Option "f" ["format"]
         (ReqArg (Flag "format") "FORMAT") $
         "Output format (" ++ formatList ++ ")",
@@ -428,7 +429,7 @@ parseOption flag options =
             }
 
         Flag "extended-analysis" str -> do
-            value <- parseBool str
+            value <- parseExtendedAnalysis str
             return options {
                 checkSpec = (checkSpec options) {
                     csExtendedAnalysis = Just value
@@ -463,6 +464,15 @@ parseOption flag options =
                 printErr $ "Invalid boolean, expected true/false: " ++ str
                 throwError SyntaxFailure
 
+    parseExtendedAnalysis str =
+        case str of
+            "true" -> return EAFull
+            "local" -> return EALocal
+            "false" -> return EAOff
+            _ -> do
+                printErr $ "Invalid extended-analysis value, expected true/local/false: " ++ str
+                throwError SyntaxFailure
+
 ioInterface :: Options -> [FilePath] -> IO (SystemInterface IO)
 ioInterface options files = do
     inputs <- mapM normalize files

src/ShellCheck/AST.hs

@@ -153,8 +153,19 @@ data Annotation =
     | ShellOverride String
     | SourcePath String
     | ExternalSources Bool
-    | ExtendedAnalysis Bool
+    | ExtendedAnalysis ExtendedAnalysisMode
     deriving (Show, Eq)
+
+-- The scope of the data-flow ("extended") analysis, set via the
+-- `extended-analysis` directive/flag. Wire values: true=EAFull (default),
+-- false=EAOff, local=EALocal.
+data ExtendedAnalysisMode =
+    EAOff       -- No data-flow analysis.
+    | EALocal   -- Data-flow analysis within each file; sourced files are
+                -- treated as opaque boundaries (not inlined into the CFG).
+    | EAFull    -- Whole-program: data flow is followed into sourced files.
+    deriving (Show, Eq)
+
 data ConditionType = DoubleBracket | SingleBracket deriving (Show, Eq)
 
 pattern T_AND_IF id = OuterToken id Inner_T_AND_IF

src/ShellCheck/ASTLib.hs

@@ -923,7 +923,7 @@ getEnableDirectives root =
         T_Annotation _ list _ -> [s | EnableComment s <- list]
         _ -> []
 
-getExtendedAnalysisDirective :: Token -> Maybe Bool
+getExtendedAnalysisDirective :: Token -> Maybe ExtendedAnalysisMode
 getExtendedAnalysisDirective root =
     case root of
         T_Annotation _ list _ -> listToMaybe $ [s | ExtendedAnalysis s <- list]

src/ShellCheck/AnalyzerLib.hs

@@ -204,7 +204,7 @@ makeCommentWithFix severity id code str fix =
 -- makeParameters :: CheckSpec -> Parameters
 makeParameters spec = params
   where
-    extendedAnalysis = fromMaybe True $ msum [asExtendedAnalysis spec, getExtendedAnalysisDirective root]
+    extendedAnalysisMode = fromMaybe EAFull $ msum [asExtendedAnalysis spec, getExtendedAnalysisDirective root]
     params = Parameters {
         rootNode = root,
         shellType = fromMaybe (determineShell (asFallbackShell spec) root) $ asShellType spec,
@@ -241,12 +241,16 @@ makeParameters spec = params
         variableFlow = getVariableFlow params root,
         tokenPositions = asTokenPositions spec,
         cfgAnalysis = do
-            guard extendedAnalysis
+            guard $ extendedAnalysisMode /= EAOff
             return $ CF.analyzeControlFlow cfParams root
     }
     cfParams = CF.CFGParameters {
         CF.cfLastpipe = hasLastpipe params,
-        CF.cfPipefail = hasPipefail params
+        CF.cfPipefail = hasPipefail params,
+        -- In 'local' mode, sourced files are data-flow boundaries: their
+        -- bodies are not inlined into the CFG (bounds memory on large
+        -- mutually-sourcing trees). See CFG.cfSourceAsBoundary.
+        CF.cfSourceAsBoundary = extendedAnalysisMode == EALocal
     }
     root = asScript spec
 

src/ShellCheck/CFG.hs

@@ -168,7 +168,11 @@ data CFGParameters = CFGParameters {
     -- Whether the last element in a pipeline runs in the current shell
     cfLastpipe :: Bool,
     -- Whether all elements in a pipeline count towards the exit status
-    cfPipefail :: Bool
+    cfPipefail :: Bool,
+    -- Treat 'source'd files as data-flow boundaries: don't inline their
+    -- bodies into the CFG. Bounds memory when many inputs source a shared
+    -- tree (each includer would otherwise re-analyze the whole tree).
+    cfSourceAsBoundary :: Bool
 }
 
 data CFGResult = CFGResult {
@@ -871,11 +875,19 @@ build t = do
 
         T_SourceCommand _ originalCommand inlinedSource -> do
             cmd <- build originalCommand
-            end <- newStructuralNode
-            inline <- withReturn end $ build inlinedSource
-            linkRange cmd inline
-            linkRange inline end
-            return $ spanRange cmd inline
+            sourceAsBoundary <- reader $ cfSourceAsBoundary . cfParameters
+            if sourceAsBoundary
+              then
+                -- Data-flow boundary: do not inline the sourced body into the CFG.
+                -- Functions/vars it defines become opaque here; the file is
+                -- analyzed on its own when it is itself an input.
+                return cmd
+              else do
+                end <- newStructuralNode
+                inline <- withReturn end $ build inlinedSource
+                linkRange cmd inline
+                linkRange inline end
+                return $ spanRange cmd inline
 
         T_Subshell id body -> do
             main <- subshell id "explicit (..) subshell" $ sequentially body

src/ShellCheck/Checker.hs

@@ -21,6 +21,7 @@
 module ShellCheck.Checker (checkScript, ShellCheck.Checker.runTests) where
 
 import ShellCheck.Analyzer
+import ShellCheck.AST (ExtendedAnalysisMode(..))
 import ShellCheck.ASTLib
 import ShellCheck.Interface
 import ShellCheck.Parser
@@ -561,14 +562,40 @@ prop_flagWinsWhenSuppressingDfa1 = result == [2317]
   where
     result = checkWithRc "extended-analysis=false" emptyCheckSpec {
         csScript = "#!/bin/sh\n# shellcheck extended-analysis=false\nexit; foo;",
-        csExtendedAnalysis = Just True
+        csExtendedAnalysis = Just EAFull
     }
 
 prop_flagWinsWhenSuppressingDfa2 = null result
   where
     result = checkWithRc "extended-analysis=true" emptyCheckSpec {
         csScript = "#!/bin/sh\n# shellcheck extended-analysis=true\nexit; foo;",
-        csExtendedAnalysis = Just False
+        csExtendedAnalysis = Just EAOff
+    }
+
+-- 'local' still does data-flow analysis within a file...
+prop_localKeepsIntraFileDfa = 2317 `elem` result
+  where
+    result = getErrors (mockedSystemInterface []) emptyCheckSpec {
+        csScript = "#!/bin/sh\nexit; foo;",
+        csExtendedAnalysis = Just EALocal
+    }
+
+-- ...but treats sourced files as boundaries, so it does not use data flow
+-- from a sourced file (here: that f() always exits) the way 'full' does.
+prop_localTreatsSourcesAsBoundary = 2317 `notElem` result
+  where
+    result = getErrors (mockedSystemInterface [("lib", "f() { exit 1; }")]) emptyCheckSpec {
+        csScript = "#!/bin/sh\nsource lib\nf\nfoo",
+        csCheckSourced = True,
+        csExtendedAnalysis = Just EALocal
+    }
+
+prop_fullFollowsDataFlowIntoSources = 2317 `elem` result
+  where
+    result = getErrors (mockedSystemInterface [("lib", "f() { exit 1; }")]) emptyCheckSpec {
+        csScript = "#!/bin/sh\nsource lib\nf\nfoo",
+        csCheckSourced = True,
+        csExtendedAnalysis = Just EAFull
     }
 
 return []

src/ShellCheck/Debug.hs

@@ -117,7 +117,8 @@ dummySystemInterface = mockedSystemInterface [
 cfgParams :: CFGParameters
 cfgParams = CFGParameters {
     cfLastpipe = False,
-    cfPipefail = False
+    cfPipefail = False,
+    cfSourceAsBoundary = False
 }
 
 -- An example script to play with

src/ShellCheck/Interface.hs

@@ -100,7 +100,7 @@ data CheckSpec = CheckSpec {
     csIncludedWarnings :: Maybe [Integer],
     csShellTypeOverride :: Maybe Shell,
     csMinSeverity :: Severity,
-    csExtendedAnalysis :: Maybe Bool,
+    csExtendedAnalysis :: Maybe ExtendedAnalysisMode,
     csOptionalChecks :: [String]
 } deriving (Show, Eq)
 
@@ -176,7 +176,7 @@ data AnalysisSpec = AnalysisSpec {
     asExecutionMode :: ExecutionMode,
     asCheckSourced :: Bool,
     asOptionalChecks :: [String],
-    asExtendedAnalysis :: Maybe Bool,
+    asExtendedAnalysis :: Maybe ExtendedAnalysisMode,
     asTokenPositions :: Map.Map Id (Position, Position)
 }
 

src/ShellCheck/Parser.hs

@@ -1062,10 +1062,11 @@ readAnnotationWithoutPrefix sandboxed = do
                 pos <- getPosition
                 value <- plainOrQuoted $ many1 letter
                 case value of
-                    "true" -> return [ExtendedAnalysis True]
-                    "false" -> return [ExtendedAnalysis False]
+                    "true" -> return [ExtendedAnalysis EAFull]
+                    "local" -> return [ExtendedAnalysis EALocal]
+                    "false" -> return [ExtendedAnalysis EAOff]
                     _ -> do
-                        parseNoteAt pos ErrorC 1146 "Unknown extended-analysis value. Expected true/false."
+                        parseNoteAt pos ErrorC 1146 "Unknown extended-analysis value. Expected true/local/false."
                         return []
 
             "external-sources" -> do