diff --git a/CHANGELOG.md b/CHANGELOG.md index d4880001c..79f562eb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## Git ### Added +- `extended-analysis=local`: a new value for the `--extended-analysis` flag and + directive that performs dataflow analysis per file, treating sourced files as + opaque boundaries. This bounds memory when many checked files source a shared + tree of libraries, while keeping per-file dataflow checks (unlike `false`). ### Changed diff --git a/shellcheck.1.md b/shellcheck.1.md index 2fdc5f4d8..b9d6c2dae 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -56,12 +56,17 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. options are cumulative, but all the codes can be specified at once, comma-separated as a single argument. -**--extended-analysis=true/false** - -: Enable/disable Dataflow Analysis to identify more issues (default true). If - ShellCheck uses too much CPU/RAM when checking scripts with several - thousand lines of code, extended analysis can be disabled with this flag - or a directive. This flag overrides directives and rc files. +**--extended-analysis=true/local/false** + +: Set the scope of Dataflow Analysis, which identifies more issues (default + true). `true` performs whole-program analysis, following data flow into + sourced files. `local` performs the analysis per file, treating sourced + files as opaque boundaries; this bounds memory when many checked files + source a shared tree of libraries (each includer would otherwise re-analyze + the whole tree), at the cost of data flow that crosses `source` boundaries. + `false` disables Dataflow Analysis entirely. Use `local` or `false` if + ShellCheck uses too much CPU/RAM. This flag overrides directives and rc + files. **-f** *FORMAT*, **--format=***FORMAT* @@ -264,10 +269,13 @@ Valid keys are: Only file-wide `enable` directives are considered. **extended-analysis** -: Set to true/false to enable/disable dataflow analysis. Specifying +: Set to true/local/false to control dataflow analysis. `true` (default) + performs whole-program analysis; `local` performs it per file, treating + sourced files as opaque boundaries; `false` disables it. Specifying `# shellcheck extended-analysis=false` in particularly large (2000+ line) auto-generated scripts will reduce ShellCheck's resource usage at the - expense of certain checks. Extended analysis is enabled by default. + expense of certain checks. `local` bounds memory when many checked files + source a shared tree of libraries, while keeping per-file dataflow checks. **external-sources** : Set to `true` in `.shellcheckrc` to always allow ShellCheck to open diff --git a/shellcheck.hs b/shellcheck.hs index 9378b78f0..c0b6a9f12 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -18,6 +18,7 @@ along with this program. If not, see . -} import qualified ShellCheck.Analyzer +import ShellCheck.AST (ExtendedAnalysisMode(..)) import ShellCheck.Checker import ShellCheck.Data import ShellCheck.Interface @@ -103,7 +104,7 @@ options = [ Option "e" ["exclude"] (ReqArg (Flag "exclude") "CODE1,CODE2..") "Exclude types of warnings", Option "" ["extended-analysis"] - (ReqArg (Flag "extended-analysis") "bool") "Perform dataflow analysis (default true)", + (ReqArg (Flag "extended-analysis") "mode") "Perform dataflow analysis: true|local|false (default true; local treats sourced files as boundaries)", Option "f" ["format"] (ReqArg (Flag "format") "FORMAT") $ "Output format (" ++ formatList ++ ")", @@ -428,7 +429,7 @@ parseOption flag options = } Flag "extended-analysis" str -> do - value <- parseBool str + value <- parseExtendedAnalysis str return options { checkSpec = (checkSpec options) { csExtendedAnalysis = Just value @@ -463,6 +464,15 @@ parseOption flag options = printErr $ "Invalid boolean, expected true/false: " ++ str throwError SyntaxFailure + parseExtendedAnalysis str = + case str of + "true" -> return EAFull + "local" -> return EALocal + "false" -> return EAOff + _ -> do + printErr $ "Invalid extended-analysis value, expected true/local/false: " ++ str + throwError SyntaxFailure + ioInterface :: Options -> [FilePath] -> IO (SystemInterface IO) ioInterface options files = do inputs <- mapM normalize files diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index b04abee42..019cbaa36 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -153,8 +153,19 @@ data Annotation = | ShellOverride String | SourcePath String | ExternalSources Bool - | ExtendedAnalysis Bool + | ExtendedAnalysis ExtendedAnalysisMode deriving (Show, Eq) + +-- The scope of the data-flow ("extended") analysis, set via the +-- `extended-analysis` directive/flag. Wire values: true=EAFull (default), +-- false=EAOff, local=EALocal. +data ExtendedAnalysisMode = + EAOff -- No data-flow analysis. + | EALocal -- Data-flow analysis within each file; sourced files are + -- treated as opaque boundaries (not inlined into the CFG). + | EAFull -- Whole-program: data flow is followed into sourced files. + deriving (Show, Eq) + data ConditionType = DoubleBracket | SingleBracket deriving (Show, Eq) pattern T_AND_IF id = OuterToken id Inner_T_AND_IF diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index f02e9f341..3ee7bf024 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -923,7 +923,7 @@ getEnableDirectives root = T_Annotation _ list _ -> [s | EnableComment s <- list] _ -> [] -getExtendedAnalysisDirective :: Token -> Maybe Bool +getExtendedAnalysisDirective :: Token -> Maybe ExtendedAnalysisMode getExtendedAnalysisDirective root = case root of T_Annotation _ list _ -> listToMaybe $ [s | ExtendedAnalysis s <- list] diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index f6d7defd7..6caae5eb2 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -204,7 +204,7 @@ makeCommentWithFix severity id code str fix = -- makeParameters :: CheckSpec -> Parameters makeParameters spec = params where - extendedAnalysis = fromMaybe True $ msum [asExtendedAnalysis spec, getExtendedAnalysisDirective root] + extendedAnalysisMode = fromMaybe EAFull $ msum [asExtendedAnalysis spec, getExtendedAnalysisDirective root] params = Parameters { rootNode = root, shellType = fromMaybe (determineShell (asFallbackShell spec) root) $ asShellType spec, @@ -241,12 +241,16 @@ makeParameters spec = params variableFlow = getVariableFlow params root, tokenPositions = asTokenPositions spec, cfgAnalysis = do - guard extendedAnalysis + guard $ extendedAnalysisMode /= EAOff return $ CF.analyzeControlFlow cfParams root } cfParams = CF.CFGParameters { CF.cfLastpipe = hasLastpipe params, - CF.cfPipefail = hasPipefail params + CF.cfPipefail = hasPipefail params, + -- In 'local' mode, sourced files are data-flow boundaries: their + -- bodies are not inlined into the CFG (bounds memory on large + -- mutually-sourcing trees). See CFG.cfSourceAsBoundary. + CF.cfSourceAsBoundary = extendedAnalysisMode == EALocal } root = asScript spec diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index c235cb7d4..3de78b4b5 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -168,7 +168,11 @@ data CFGParameters = CFGParameters { -- Whether the last element in a pipeline runs in the current shell cfLastpipe :: Bool, -- Whether all elements in a pipeline count towards the exit status - cfPipefail :: Bool + cfPipefail :: Bool, + -- Treat 'source'd files as data-flow boundaries: don't inline their + -- bodies into the CFG. Bounds memory when many inputs source a shared + -- tree (each includer would otherwise re-analyze the whole tree). + cfSourceAsBoundary :: Bool } data CFGResult = CFGResult { @@ -871,11 +875,19 @@ build t = do T_SourceCommand _ originalCommand inlinedSource -> do cmd <- build originalCommand - end <- newStructuralNode - inline <- withReturn end $ build inlinedSource - linkRange cmd inline - linkRange inline end - return $ spanRange cmd inline + sourceAsBoundary <- reader $ cfSourceAsBoundary . cfParameters + if sourceAsBoundary + then + -- Data-flow boundary: do not inline the sourced body into the CFG. + -- Functions/vars it defines become opaque here; the file is + -- analyzed on its own when it is itself an input. + return cmd + else do + end <- newStructuralNode + inline <- withReturn end $ build inlinedSource + linkRange cmd inline + linkRange inline end + return $ spanRange cmd inline T_Subshell id body -> do main <- subshell id "explicit (..) subshell" $ sequentially body diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 8060d05ee..c8a075f8d 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -21,6 +21,7 @@ module ShellCheck.Checker (checkScript, ShellCheck.Checker.runTests) where import ShellCheck.Analyzer +import ShellCheck.AST (ExtendedAnalysisMode(..)) import ShellCheck.ASTLib import ShellCheck.Interface import ShellCheck.Parser @@ -561,14 +562,40 @@ prop_flagWinsWhenSuppressingDfa1 = result == [2317] where result = checkWithRc "extended-analysis=false" emptyCheckSpec { csScript = "#!/bin/sh\n# shellcheck extended-analysis=false\nexit; foo;", - csExtendedAnalysis = Just True + csExtendedAnalysis = Just EAFull } prop_flagWinsWhenSuppressingDfa2 = null result where result = checkWithRc "extended-analysis=true" emptyCheckSpec { csScript = "#!/bin/sh\n# shellcheck extended-analysis=true\nexit; foo;", - csExtendedAnalysis = Just False + csExtendedAnalysis = Just EAOff + } + +-- 'local' still does data-flow analysis within a file... +prop_localKeepsIntraFileDfa = 2317 `elem` result + where + result = getErrors (mockedSystemInterface []) emptyCheckSpec { + csScript = "#!/bin/sh\nexit; foo;", + csExtendedAnalysis = Just EALocal + } + +-- ...but treats sourced files as boundaries, so it does not use data flow +-- from a sourced file (here: that f() always exits) the way 'full' does. +prop_localTreatsSourcesAsBoundary = 2317 `notElem` result + where + result = getErrors (mockedSystemInterface [("lib", "f() { exit 1; }")]) emptyCheckSpec { + csScript = "#!/bin/sh\nsource lib\nf\nfoo", + csCheckSourced = True, + csExtendedAnalysis = Just EALocal + } + +prop_fullFollowsDataFlowIntoSources = 2317 `elem` result + where + result = getErrors (mockedSystemInterface [("lib", "f() { exit 1; }")]) emptyCheckSpec { + csScript = "#!/bin/sh\nsource lib\nf\nfoo", + csCheckSourced = True, + csExtendedAnalysis = Just EAFull } return [] diff --git a/src/ShellCheck/Debug.hs b/src/ShellCheck/Debug.hs index 23b87062b..be1903f67 100644 --- a/src/ShellCheck/Debug.hs +++ b/src/ShellCheck/Debug.hs @@ -117,7 +117,8 @@ dummySystemInterface = mockedSystemInterface [ cfgParams :: CFGParameters cfgParams = CFGParameters { cfLastpipe = False, - cfPipefail = False + cfPipefail = False, + cfSourceAsBoundary = False } -- An example script to play with diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index 16a7e3641..e112851ca 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -100,7 +100,7 @@ data CheckSpec = CheckSpec { csIncludedWarnings :: Maybe [Integer], csShellTypeOverride :: Maybe Shell, csMinSeverity :: Severity, - csExtendedAnalysis :: Maybe Bool, + csExtendedAnalysis :: Maybe ExtendedAnalysisMode, csOptionalChecks :: [String] } deriving (Show, Eq) @@ -176,7 +176,7 @@ data AnalysisSpec = AnalysisSpec { asExecutionMode :: ExecutionMode, asCheckSourced :: Bool, asOptionalChecks :: [String], - asExtendedAnalysis :: Maybe Bool, + asExtendedAnalysis :: Maybe ExtendedAnalysisMode, asTokenPositions :: Map.Map Id (Position, Position) } diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 2902f9b99..9d266d2b7 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1062,10 +1062,11 @@ readAnnotationWithoutPrefix sandboxed = do pos <- getPosition value <- plainOrQuoted $ many1 letter case value of - "true" -> return [ExtendedAnalysis True] - "false" -> return [ExtendedAnalysis False] + "true" -> return [ExtendedAnalysis EAFull] + "local" -> return [ExtendedAnalysis EALocal] + "false" -> return [ExtendedAnalysis EAOff] _ -> do - parseNoteAt pos ErrorC 1146 "Unknown extended-analysis value. Expected true/false." + parseNoteAt pos ErrorC 1146 "Unknown extended-analysis value. Expected true/local/false." return [] "external-sources" -> do