{-# LANGUAGE TemplateHaskell #-}

{-# LANGUAGE FlexibleContexts #-}

module ShellCheck.Analytics (runAnalytics, optionalChecks, ShellCheck.Analytics.runTests) where

import ShellCheck.AST

import ShellCheck.ASTLib

import ShellCheck.AnalyzerLib hiding (producesComments)

import ShellCheck.Data

import ShellCheck.Parser

import ShellCheck.Interface

import ShellCheck.Regex

import Control.Arrow (first)

import Control.Monad

import Control.Monad.Identity

import Control.Monad.State

import Control.Monad.Writer hiding ((<>))

import Control.Monad.Reader

import Data.Char

import Data.Functor

import Data.Function (on)

import Data.List

import Data.Maybe

import Data.Ord

import Data.Semigroup

import Debug.Trace

import qualified Data.Map.Strict as Map

import Test.QuickCheck.All (forAllProperties)

import Test.QuickCheck.Test (quickCheckWithResult, stdArgs, maxSuccess)

treeChecks :: [Parameters -> Token -> [TokenComment]]

treeChecks = [

nodeChecksToTreeCheck nodeChecks

,subshellAssignmentCheck

,checkSpacefulness

,checkQuotesInLiterals

,checkShebangParameters

,checkFunctionsUsedExternally

,checkUnusedAssignments

,checkUnpassedInFunctions

,checkArrayWithoutIndex

,checkShebang

,checkUnassignedReferences

,checkUncheckedCdPushdPopd

,checkArrayAssignmentIndices

,checkUseBeforeDefinition

,checkAliasUsedInSameParsingUnit

]

runAnalytics :: AnalysisSpec -> [TokenComment]

runAnalytics options =

runList options treeChecks ++ runList options optionalChecks

where

root = asScript options

optionals = getEnableDirectives root ++ asOptionalChecks options

optionalChecks =

if "all" `elem` optionals

then map snd optionalTreeChecks

else mapMaybe (\c -> Map.lookup c optionalCheckMap) optionals

runList :: AnalysisSpec -> [Parameters -> Token -> [TokenComment]]

-> [TokenComment]

runList spec list = notes

where

root = asScript spec

params = makeParameters spec

notes = concatMap (\f -> f params root) list

getEnableDirectives root =

case root of

T_Annotation _ list _ -> [s | EnableComment s <- list]

_ -> []

checkList l t = concatMap (\f -> f t) l

runNodeAnalysis f p t = execWriter (doAnalysis (f p) t)

nodeChecksToTreeCheck checkList =

runNodeAnalysis

(\p t -> (mapM_ ((\ f -> f t) . (\ f -> f p))

checkList))

nodeChecks :: [Parameters -> Token -> Writer [TokenComment] ()]

nodeChecks = [

checkUuoc

,checkPipePitfalls

,checkForInQuoted

,checkForInLs

,checkShorthandIf

,checkDollarStar

,checkUnquotedDollarAt

,checkStderrRedirect

,checkUnquotedN

,checkNumberComparisons

,checkSingleBracketOperators

,checkDoubleBracketOperators

,checkLiteralBreakingTest

,checkConstantNullary

,checkDivBeforeMult

,checkArithmeticDeref

,checkArithmeticBadOctal

,checkComparisonAgainstGlob

,checkCaseAgainstGlob

,checkCommarrays

,checkOrNeq

,checkEchoWc

,checkConstantIfs

,checkPipedAssignment

,checkAssignAteCommand

,checkUuoeVar

,checkQuotedCondRegex

,checkForInCat

,checkFindExec

,checkValidCondOps

,checkGlobbedRegex

,checkTestRedirects

,checkIndirectExpansion

,checkPS1Assignments

,checkBackticks

,checkInexplicablyUnquoted

,checkTildeInQuotes

,checkLonelyDotDash

,checkSpuriousExec

,checkSpuriousExpansion

,checkDollarBrackets

,checkSshHereDoc

,checkGlobsAsOptions

,checkWhileReadPitfalls

,checkArithmeticOpCommand

,checkCharRangeGlob

,checkUnquotedExpansions

,checkSingleQuotedVariables

,checkRedirectToSame

,checkPrefixAssignmentReference

,checkLoopKeywordScope

,checkCdAndBack

,checkWrongArithmeticAssignment

,checkConditionalAndOrs

,checkFunctionDeclarations

,checkStderrPipe

,checkOverridingPath

,checkArrayAsString

,checkUnsupported

,checkMultipleAppends

,checkSuspiciousIFS

,checkShouldUseGrepQ

,checkTestArgumentSplitting

,checkConcatenatedDollarAt

,checkTildeInPath

,checkMaskedReturns

,checkReadWithoutR

,checkLoopVariableReassignment

,checkTrailingBracket

,checkReturnAgainstZero

,checkRedirectedNowhere

,checkUnmatchableCases

,checkSubshellAsTest

,checkSplittingInArrays

,checkRedirectionToNumber

,checkGlobAsCommand

,checkFlagAsCommand

,checkEmptyCondition

,checkPipeToNowhere

,checkForLoopGlobVariables

,checkSubshelledTests

,checkInvertedStringTest

,checkRedirectionToCommand

,checkDollarQuoteParen

,checkUselessBang

,checkTranslatedStringVariable

,checkModifiedArithmeticInRedirection

,checkBlatantRecursion

,checkBadTestAndOr

]

optionalChecks = map fst optionalTreeChecks

prop_verifyOptionalExamples = all check optionalTreeChecks

where

check (desc, check) =

verifyTree check (cdPositive desc)

&& verifyNotTree check (cdNegative desc)

optionalTreeChecks :: [(CheckDescription, (Parameters -> Token -> [TokenComment]))]

optionalTreeChecks = [

(newCheckDescription {

cdName = "quote-safe-variables",

cdDescription = "Suggest quoting variables without metacharacters",

cdPositive = "var=hello; echo $var",

cdNegative = "var=hello; echo \"$var\""

}, checkVerboseSpacefulness)

,(newCheckDescription {

cdName = "avoid-nullary-conditions",

cdDescription = "Suggest explicitly using -n in `[ $var ]`",

cdPositive = "[ \"$var\" ]",

cdNegative = "[ -n \"$var\" ]"

}, nodeChecksToTreeCheck [checkNullaryExpansionTest])

,(newCheckDescription {

cdName = "add-default-case",

cdDescription = "Suggest adding a default case in `case` statements",

cdPositive = "case $? in 0) echo 'Success';; esac",

cdNegative = "case $? in 0) echo 'Success';; *) echo 'Fail' ;; esac"

}, nodeChecksToTreeCheck [checkDefaultCase])

,(newCheckDescription {

cdName = "require-variable-braces",

cdDescription = "Suggest putting braces around all variable references",

cdPositive = "var=hello; echo $var",

cdNegative = "var=hello; echo ${var}"

}, nodeChecksToTreeCheck [checkVariableBraces])

,(newCheckDescription {

cdName = "check-unassigned-uppercase",

cdDescription = "Warn when uppercase variables are unassigned",

cdPositive = "echo $VAR",

cdNegative = "VAR=hello; echo $VAR"

}, checkUnassignedReferences' True)

,(newCheckDescription {

cdName = "avoid-x-comparisons",

cdDescription = "Warn about 'x'-prefix in comparisons",

cdPositive = "[ \"x$var\" = xval ]",

cdNegative = "[ \"$var\" = val ]"

}, nodeChecksToTreeCheck [checkComparisonWithLeadingX])

]

optionalCheckMap :: Map.Map String (Parameters -> Token -> [TokenComment])

optionalCheckMap = Map.fromList $ map item optionalTreeChecks

where

item (desc, check) = (cdName desc, check)

wouldHaveBeenGlob s = '*' `elem` s

verify :: (Parameters -> Token -> Writer [TokenComment] ()) -> String -> Bool

verify f s = checkNode f s == Just True

verifyNot :: (Parameters -> Token -> Writer [TokenComment] ()) -> String -> Bool

verifyNot f s = checkNode f s == Just False

verifyTree :: (Parameters -> Token -> [TokenComment]) -> String -> Bool

verifyTree f s = producesComments f s == Just True

verifyNotTree :: (Parameters -> Token -> [TokenComment]) -> String -> Bool

verifyNotTree f s = producesComments f s == Just False

checkCommand str f t@(T_SimpleCommand id _ (cmd:rest))

| t `isCommand` str = f cmd rest

checkCommand _ _ _ = return ()

checkUnqualifiedCommand str f t@(T_SimpleCommand id _ (cmd:rest))

| t `isUnqualifiedCommand` str = f cmd rest

checkUnqualifiedCommand _ _ _ = return ()

checkNode f = producesComments (runNodeAnalysis f)

producesComments :: (Parameters -> Token -> [TokenComment]) -> String -> Maybe Bool

producesComments f s = do

let pr = pScript s

prRoot pr

let spec = defaultSpec pr

let params = makeParameters spec

return . not . null $

filterByAnnotation spec params $

runList spec [f]

dist :: Eq a => [a] -> [a] -> Int

dist a b

= last (if lab == 0 then mainDiag

else if lab > 0 then lowers !! (lab - 1)

else{- < 0 -} uppers !! (-1 - lab))

where mainDiag = oneDiag a b (head uppers) (-1 : head lowers)

uppers = eachDiag a b (mainDiag : uppers) -- upper diagonals

lowers = eachDiag b a (mainDiag : lowers) -- lower diagonals

eachDiag a [] diags = []

eachDiag a (bch:bs) (lastDiag:diags) = oneDiag a bs nextDiag lastDiag : eachDiag a bs diags

where nextDiag = head (tail diags)

oneDiag a b diagAbove diagBelow = thisdiag

where doDiag [] b nw n w = []

doDiag a [] nw n w = []

doDiag (ach:as) (bch:bs) nw n w = me : doDiag as bs me (tail n) (tail w)

where me = if ach == bch then nw else 1 + min3 (head w) nw (head n)

firstelt = 1 + head diagBelow

thisdiag = firstelt : doDiag a b firstelt diagAbove (tail diagBelow)

lab = length a - length b

min3 x y z = if x < y then x else min y z

hasFloatingPoint params = shellType params == Ksh

isCondition [] = False

isCondition [_] = False

isCondition (child:parent:rest) =

case child of

T_BatsTest {} -> True -- count anything in a @test as conditional

_ -> getId child `elem` map getId (getConditionChildren parent) || isCondition (parent:rest)

where

getConditionChildren t =

case t of

T_AndIf _ left right -> [left]

T_OrIf id left right -> [left]

T_IfExpression id conditions elses -> concatMap (take 1 . reverse . fst) conditions

T_WhileExpression id c l -> take 1 . reverse $ c

T_UntilExpression id c l -> take 1 . reverse $ c

_ -> []

replaceStart id params n r =

let tp = tokenPositions params

(start, _) = tp Map.! id

new_end = start {

posColumn = posColumn start + n

}

depth = length $ getPath (parentMap params) (T_EOF id)

in

newReplacement {

repStartPos = start,

repEndPos = new_end,

repString = r,

repPrecedence = depth,

repInsertionPoint = InsertAfter

}

replaceEnd id params n r =

let tp = tokenPositions params

(_, end) = tp Map.! id

new_start = end {

posColumn = posColumn end - n

}

new_end = end {

posColumn = posColumn end

}

depth = length $ getPath (parentMap params) (T_EOF id)

in

newReplacement {

repStartPos = new_start,

repEndPos = new_end,

repString = r,

repPrecedence = depth,

repInsertionPoint = InsertBefore

}

surroundWidth id params s = fixWith [replaceStart id params 0 s, replaceEnd id params 0 s]

fixWith fixes = newFix { fixReplacements = fixes }

prop_checkEchoWc3 = verify checkEchoWc "n=$(echo $foo | wc -c)"

checkEchoWc _ (T_Pipeline id _ [a, b]) =

when (acmd == ["echo", "${VAR}"]) $

case bcmd of

["wc", "-c"] -> countMsg

["wc", "-m"] -> countMsg

_ -> return ()

where

acmd = oversimplify a

bcmd = oversimplify b

countMsg = style id 2000 "See if you can use ${#variable} instead."

checkEchoWc _ _ = return ()

prop_checkPipedAssignment1 = verify checkPipedAssignment "A=ls | grep foo"

prop_checkPipedAssignment2 = verifyNot checkPipedAssignment "A=foo cmd | grep foo"

prop_checkPipedAssignment3 = verifyNot checkPipedAssignment "A=foo"

checkPipedAssignment _ (T_Pipeline _ _ (T_Redirecting _ _ (T_SimpleCommand id (_:_) []):_:_)) =

warn id 2036 "If you wanted to assign the output of the pipeline, use a=$(b | c) ."

checkPipedAssignment _ _ = return ()

prop_checkAssignAteCommand1 = verify checkAssignAteCommand "A=ls -l"

prop_checkAssignAteCommand2 = verify checkAssignAteCommand "A=ls --sort=$foo"

prop_checkAssignAteCommand3 = verify checkAssignAteCommand "A=cat foo | grep bar"

prop_checkAssignAteCommand4 = verifyNot checkAssignAteCommand "A=foo ls -l"

prop_checkAssignAteCommand5 = verify checkAssignAteCommand "PAGER=cat grep bar"

prop_checkAssignAteCommand6 = verifyNot checkAssignAteCommand "PAGER=\"cat\" grep bar"

prop_checkAssignAteCommand7 = verify checkAssignAteCommand "here=pwd"

checkAssignAteCommand _ (T_SimpleCommand id (T_Assignment _ _ _ _ assignmentTerm:[]) list) =

-- Check if first word is intended as an argument (flag or glob).

if firstWordIsArg list

then

err id 2037 "To assign the output of a command, use var=$(cmd) ."

else

-- Check if it's a known, unquoted command name.

when (isCommonCommand $ getUnquotedLiteral assignmentTerm) $

warn id 2209 "Use var=$(command) to assign output (or quote to assign string)."

where

isCommonCommand (Just s) = s `elem` commonCommands

isCommonCommand _ = False

firstWordIsArg list = fromMaybe False $ do

head <- list !!! 0

return $ isGlob head || isUnquotedFlag head

checkAssignAteCommand _ _ = return ()

prop_checkArithmeticOpCommand1 = verify checkArithmeticOpCommand "i=i + 1"

prop_checkArithmeticOpCommand2 = verify checkArithmeticOpCommand "foo=bar * 2"

prop_checkArithmeticOpCommand3 = verifyNot checkArithmeticOpCommand "foo + opts"

checkArithmeticOpCommand _ (T_SimpleCommand id [T_Assignment {}] (firstWord:_)) =

mapM_ check $ getGlobOrLiteralString firstWord

where

check op =

when (op `elem` ["+", "-", "*", "/"]) $

warn (getId firstWord) 2099 $

"Use $((..)) for arithmetics, e.g. i=$((i " ++ op ++ " 2))"

checkArithmeticOpCommand _ _ = return ()

prop_checkWrongArit = verify checkWrongArithmeticAssignment "i=i+1"

prop_checkWrongArit2 = verify checkWrongArithmeticAssignment "n=2; i=n*2"

checkWrongArithmeticAssignment params (T_SimpleCommand id (T_Assignment _ _ _ _ val:[]) []) =

sequence_ $ do

str <- getNormalString val

match <- matchRegex regex str

var <- match !!! 0

op <- match !!! 1

Map.lookup var references

return . warn (getId val) 2100 $

"Use $((..)) for arithmetics, e.g. i=$((i " ++ op ++ " 2))"

where

regex = mkRegex "^([_a-zA-Z][_a-zA-Z0-9]*)([+*-]).+$"

references = foldl (flip ($)) Map.empty (map insertRef $ variableFlow params)

insertRef (Assignment (_, _, name, _)) =

Map.insert name ()

insertRef _ = Prelude.id

getNormalString (T_NormalWord _ words) = do

parts <- mapM getLiterals words

return $ concat parts

getNormalString _ = Nothing

getLiterals (T_Literal _ s) = return s

getLiterals (T_Glob _ s) = return s

getLiterals _ = Nothing

checkWrongArithmeticAssignment _ _ = return ()

prop_checkUuoc1 = verify checkUuoc "cat foo | grep bar"

prop_checkUuoc2 = verifyNot checkUuoc "cat * | grep bar"

prop_checkUuoc3 = verify checkUuoc "cat $var | grep bar"

prop_checkUuoc4 = verifyNot checkUuoc "cat $var"

prop_checkUuoc5 = verifyNot checkUuoc "cat \"$@\""

prop_checkUuoc6 = verifyNot checkUuoc "cat -n | grep bar"

checkUuoc _ (T_Pipeline _ _ (T_Redirecting _ _ cmd:_:_)) =

checkCommand "cat" (const f) cmd

where

f [word] | not (mayBecomeMultipleArgs word || isOption word) =

style (getId word) 2002 "Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead."

f _ = return ()

isOption word = "-" `isPrefixOf` onlyLiteralString word

checkUuoc _ _ = return ()

prop_checkPipePitfalls3 = verify checkPipePitfalls "ls | grep -v mp3"

prop_checkPipePitfalls4 = verifyNot checkPipePitfalls "find . -print0 | xargs -0 foo"

prop_checkPipePitfalls5 = verifyNot checkPipePitfalls "ls -N | foo"

prop_checkPipePitfalls6 = verify checkPipePitfalls "find . | xargs foo"

prop_checkPipePitfalls7 = verifyNot checkPipePitfalls "find . -printf '%s\\n' | xargs foo"

prop_checkPipePitfalls8 = verify checkPipePitfalls "foo | grep bar | wc -l"

prop_checkPipePitfalls9 = verifyNot checkPipePitfalls "foo | grep -o bar | wc -l"

prop_checkPipePitfalls10 = verifyNot checkPipePitfalls "foo | grep -o bar | wc"

prop_checkPipePitfalls11 = verifyNot checkPipePitfalls "foo | grep bar | wc"

prop_checkPipePitfalls12 = verifyNot checkPipePitfalls "foo | grep -o bar | wc -c"

prop_checkPipePitfalls13 = verifyNot checkPipePitfalls "foo | grep bar | wc -c"

prop_checkPipePitfalls14 = verifyNot checkPipePitfalls "foo | grep -o bar | wc -cmwL"

prop_checkPipePitfalls15 = verifyNot checkPipePitfalls "foo | grep bar | wc -cmwL"

prop_checkPipePitfalls16 = verifyNot checkPipePitfalls "foo | grep -r bar | wc -l"

checkPipePitfalls _ (T_Pipeline id _ commands) = do

for ["find", "xargs"] $

\(find:xargs:_) ->

let args = oversimplify xargs ++ oversimplify find

in

unless (any ($ args) [

hasShortParameter '0',

hasParameter "null",

hasParameter "print0",

hasParameter "printf"

]) $ warn (getId find) 2038

"Use -print0/-0 or -exec + to allow for non-alphanumeric filenames."

for' ["ps", "grep"] $

\x -> info x 2009 "Consider using pgrep instead of grepping ps output."

for ["grep", "wc"] $

\(grep:wc:_) ->

let flagsGrep = maybe [] (map snd . getAllFlags) $ getCommand grep

flagsWc = maybe [] (map snd . getAllFlags) $ getCommand wc

in

unless (any (`elem` ["o", "only-matching", "r", "R", "recursive"]) flagsGrep || any (`elem` ["m", "chars", "w", "words", "c", "bytes", "L", "max-line-length"]) flagsWc || null flagsWc) $

style (getId grep) 2126 "Consider using grep -c instead of grep|wc -l."

didLs <- fmap or . sequence $ [

for' ["ls", "grep"] $

\x -> warn x 2010 "Don't use ls | grep. Use a glob or a for loop with a condition to allow non-alphanumeric filenames.",

for' ["ls", "xargs"] $

\x -> warn x 2011 "Use 'find .. -print0 | xargs -0 ..' or 'find .. -exec .. +' to allow non-alphanumeric filenames."

]

unless didLs $ void $

for ["ls", "?"] $

\(ls:_) -> unless (hasShortParameter 'N' (oversimplify ls)) $

info (getId ls) 2012 "Use find instead of ls to better handle non-alphanumeric filenames."

where

for l f =

let indices = indexOfSublists l (map (headOrDefault "" . oversimplify) commands)

in do

mapM_ (f . (\ n -> take (length l) $ drop n commands)) indices

return . not . null $ indices

for' l f = for l (first f)

first func (x:_) = func (getId $ getCommandTokenOrThis x)

first _ _ = return ()

hasShortParameter char = any (\x -> "-" `isPrefixOf` x && char `elem` x)

hasParameter string =

any (isPrefixOf string . dropWhile (== '-'))

checkPipePitfalls _ _ = return ()

indexOfSublists sub = f 0

where

f _ [] = []

f n a@(r:rest) =

let others = f (n+1) rest in

if match sub a

then n:others

else others

match ("?":r1) (_:r2) = match r1 r2

match (x1:r1) (x2:r2) | x1 == x2 = match r1 r2

match [] _ = True

match _ _ = False

prop_checkShebangParameters1 = verifyTree checkShebangParameters "#!/usr/bin/env bash -x\necho cow"

prop_checkShebangParameters2 = verifyNotTree checkShebangParameters "#! /bin/sh -l "

checkShebangParameters p (T_Annotation _ _ t) = checkShebangParameters p t

checkShebangParameters _ (T_Script _ (T_Literal id sb) _) =

[makeComment ErrorC id 2096 "On most OS, shebangs can only specify a single parameter." | length (words sb) > 2]

prop_checkShebang1 = verifyNotTree checkShebang "#!/usr/bin/env bash -x\necho cow"

prop_checkShebang2 = verifyNotTree checkShebang "#! /bin/sh -l "

prop_checkShebang3 = verifyTree checkShebang "ls -l"

prop_checkShebang4 = verifyNotTree checkShebang "#shellcheck shell=sh\nfoo"

prop_checkShebang5 = verifyTree checkShebang "#!/usr/bin/env ash"

prop_checkShebang6 = verifyNotTree checkShebang "#!/usr/bin/env ash\n# shellcheck shell=dash\n"

prop_checkShebang7 = verifyNotTree checkShebang "#!/usr/bin/env ash\n# shellcheck shell=sh\n"

prop_checkShebang8 = verifyTree checkShebang "#!bin/sh\ntrue"

prop_checkShebang9 = verifyNotTree checkShebang "# shellcheck shell=sh\ntrue"

prop_checkShebang10= verifyNotTree checkShebang "#!foo\n# shellcheck shell=sh ignore=SC2239\ntrue"

prop_checkShebang11= verifyTree checkShebang "#!/bin/sh/\ntrue"

prop_checkShebang12= verifyTree checkShebang "#!/bin/sh/ -xe\ntrue"

checkShebang params (T_Annotation _ list t) =

if any isOverride list then [] else checkShebang params t

where

isOverride (ShellOverride _) = True

isOverride _ = False

checkShebang params (T_Script _ (T_Literal id sb) _) = execWriter $ do

unless (shellTypeSpecified params) $ do

when (null sb) $

err id 2148 "Tips depend on target shell and yours is unknown. Add a shebang or a 'shell' directive."

when (executableFromShebang sb == "ash") $

warn id 2187 "Ash scripts will be checked as Dash. Add '# shellcheck shell=dash' to silence."

unless (null sb) $ do

unless ("/" `isPrefixOf` sb) $

err id 2239 "Ensure the shebang uses an absolute path to the interpreter."

when ("/" `isSuffixOf` head (words sb)) $

err id 2246 "This shebang specifies a directory. Ensure the interpreter is a file."

prop_checkForInQuoted = verify checkForInQuoted "for f in \"$(ls)\"; do echo foo; done"

prop_checkForInQuoted2 = verifyNot checkForInQuoted "for f in \"$@\"; do echo foo; done"

prop_checkForInQuoted2a = verifyNot checkForInQuoted "for f in *.mp3; do echo foo; done"

prop_checkForInQuoted2b = verify checkForInQuoted "for f in \"*.mp3\"; do echo foo; done"

prop_checkForInQuoted3 = verify checkForInQuoted "for f in 'find /'; do true; done"

prop_checkForInQuoted4 = verify checkForInQuoted "for f in 1,2,3; do true; done"

prop_checkForInQuoted4a = verifyNot checkForInQuoted "for f in foo{1,2,3}; do true; done"

prop_checkForInQuoted5 = verify checkForInQuoted "for f in ls; do true; done"

prop_checkForInQuoted6 = verifyNot checkForInQuoted "for f in \"${!arr}\"; do true; done"

prop_checkForInQuoted7 = verify checkForInQuoted "for f in ls, grep, mv; do true; done"

prop_checkForInQuoted8 = verify checkForInQuoted "for f in 'ls', 'grep', 'mv'; do true; done"

prop_checkForInQuoted9 = verifyNot checkForInQuoted "for f in 'ls,' 'grep,' 'mv'; do true; done"

checkForInQuoted _ (T_ForIn _ f [T_NormalWord _ [word@(T_DoubleQuoted id list)]] _)

| any (\x -> willSplit x && not (mayBecomeMultipleArgs x)) list

|| maybe False wouldHaveBeenGlob (getLiteralString word) =

err id 2066 "Since you double quoted this, it will not word split, and the loop will only run once."

checkForInQuoted _ (T_ForIn _ f [T_NormalWord _ [T_SingleQuoted id _]] _) =

warn id 2041 "This is a literal string. To run as a command, use $(..) instead of '..' . "

checkForInQuoted _ (T_ForIn _ _ [single] _)

| maybe False (',' `elem`) $ getUnquotedLiteral single =

warn (getId single) 2042 "Use spaces, not commas, to separate loop elements."

| not (willSplit single || mayBecomeMultipleArgs single) =

warn (getId single) 2043 "This loop will only ever run once. Bad quoting or missing glob/expansion?"

checkForInQuoted params (T_ForIn _ _ multiple _) =

forM_ multiple $ \arg -> sequence_ $ do

suffix <- getTrailingUnquotedLiteral arg

string <- getLiteralString suffix

guard $ "," `isSuffixOf` string

return $

warnWithFix (getId arg) 2258

"The trailing comma is part of the value, not a separator. Delete or quote it."

(fixWith [replaceEnd (getId suffix) params 1 ""])

checkForInQuoted _ _ = return ()

prop_checkForInCat1 = verify checkForInCat "for f in $(cat foo); do stuff; done"

prop_checkForInCat1a= verify checkForInCat "for f in `cat foo`; do stuff; done"

prop_checkForInCat2 = verify checkForInCat "for f in $(cat foo | grep lol); do stuff; done"

prop_checkForInCat2a= verify checkForInCat "for f in `cat foo | grep lol`; do stuff; done"

prop_checkForInCat3 = verifyNot checkForInCat "for f in $(cat foo | grep bar | wc -l); do stuff; done"

checkForInCat _ (T_ForIn _ f [T_NormalWord _ w] _) = mapM_ checkF w

where

checkF (T_DollarExpansion id [T_Pipeline _ _ r])

| all isLineBased r =

info id 2013 "To read lines rather than words, pipe/redirect to a 'while read' loop."

checkF (T_Backticked id cmds) = checkF (T_DollarExpansion id cmds)

checkF _ = return ()

isLineBased cmd = any (cmd `isCommand`)

["grep", "fgrep", "egrep", "sed", "cat", "awk", "cut", "sort"]

checkForInCat _ _ = return ()

prop_checkForInLs = verify checkForInLs "for f in $(ls *.mp3); do mplayer \"$f\"; done"

prop_checkForInLs2 = verify checkForInLs "for f in `ls *.mp3`; do mplayer \"$f\"; done"

prop_checkForInLs3 = verify checkForInLs "for f in `find / -name '*.mp3'`; do mplayer \"$f\"; done"

checkForInLs _ = try

where

try (T_ForIn _ f [T_NormalWord _ [T_DollarExpansion id [x]]] _) =

check id f x

try (T_ForIn _ f [T_NormalWord _ [T_Backticked id [x]]] _) =

check id f x

try _ = return ()

check id f x =

case oversimplify x of

("ls":n) ->

let warntype = if any ("-" `isPrefixOf`) n then warn else err in

warntype id 2045 "Iterating over ls output is fragile. Use globs."

("find":_) -> warn id 2044 "For loops over find output are fragile. Use find -exec or a while read loop."

_ -> return ()

prop_checkFindExec1 = verify checkFindExec "find / -name '*.php' -exec rm {};"

prop_checkFindExec2 = verify checkFindExec "find / -exec touch {} && ls {} \\;"

prop_checkFindExec3 = verify checkFindExec "find / -execdir cat {} | grep lol +"

prop_checkFindExec4 = verifyNot checkFindExec "find / -name '*.php' -exec foo {} +"

prop_checkFindExec5 = verifyNot checkFindExec "find / -execdir bash -c 'a && b' \\;"

prop_checkFindExec6 = verify checkFindExec "find / -type d -execdir rm *.jpg \\;"

checkFindExec _ cmd@(T_SimpleCommand _ _ t@(h:r)) | cmd `isCommand` "find" = do

c <- broken r False

when c $

let wordId = getId $ last t in

err wordId 2067 "Missing ';' or + terminating -exec. You can't use |/||/&&, and ';' has to be a separate, quoted argument."

where

broken [] v = return v

broken (w:r) v = do

when v (mapM_ warnFor $ fromWord w)

case getLiteralString w of

Just "-exec" -> broken r True

Just "-execdir" -> broken r True

Just "+" -> broken r False

Just ";" -> broken r False

_ -> broken r v

shouldWarn x =

case x of

T_DollarExpansion _ _ -> True

T_Backticked _ _ -> True

T_Glob _ _ -> True

T_Extglob {} -> True

_ -> False

warnFor x =

when(shouldWarn x) $

info (getId x) 2014 "This will expand once before find runs, not per file found."

fromWord (T_NormalWord _ l) = l

fromWord _ = []

checkFindExec _ _ = return ()

prop_checkUnquotedExpansions1 = verify checkUnquotedExpansions "rm $(ls)"

prop_checkUnquotedExpansions1a= verify checkUnquotedExpansions "rm `ls`"

prop_checkUnquotedExpansions2 = verify checkUnquotedExpansions "rm foo$(date)"

prop_checkUnquotedExpansions3 = verify checkUnquotedExpansions "[ $(foo) == cow ]"

prop_checkUnquotedExpansions3a= verify checkUnquotedExpansions "[ ! $(foo) ]"

prop_checkUnquotedExpansions4 = verifyNot checkUnquotedExpansions "[[ $(foo) == cow ]]"

prop_checkUnquotedExpansions5 = verifyNot checkUnquotedExpansions "for f in $(cmd); do echo $f; done"

prop_checkUnquotedExpansions6 = verifyNot checkUnquotedExpansions "$(cmd)"

prop_checkUnquotedExpansions7 = verifyNot checkUnquotedExpansions "cat << foo\n$(ls)\nfoo"

prop_checkUnquotedExpansions8 = verifyNot checkUnquotedExpansions "set -- $(seq 1 4)"

prop_checkUnquotedExpansions9 = verifyNot checkUnquotedExpansions "echo foo `# inline comment`"

checkUnquotedExpansions params =

check

where

check t@(T_DollarExpansion _ c) = examine t c

check t@(T_Backticked _ c) = examine t c

check t@(T_DollarBraceCommandExpansion _ c) = examine t c

check _ = return ()

tree = parentMap params

examine t contents =

unless (null contents || shouldBeSplit t || isQuoteFree tree t || usedAsCommandName tree t) $

warn (getId t) 2046 "Quote this to prevent word splitting."

shouldBeSplit t =

getCommandNameFromExpansion t == Just "seq"

prop_checkRedirectToSame = verify checkRedirectToSame "cat foo > foo"

prop_checkRedirectToSame2 = verify checkRedirectToSame "cat lol | sed -e 's/a/b/g' > lol"

prop_checkRedirectToSame3 = verifyNot checkRedirectToSame "cat lol | sed -e 's/a/b/g' > foo.bar && mv foo.bar lol"

prop_checkRedirectToSame4 = verifyNot checkRedirectToSame "foo /dev/null > /dev/null"

prop_checkRedirectToSame5 = verifyNot checkRedirectToSame "foo > bar 2> bar"

prop_checkRedirectToSame6 = verifyNot checkRedirectToSame "echo foo > foo"

prop_checkRedirectToSame7 = verifyNot checkRedirectToSame "sed 's/foo/bar/g' file | sponge file"

prop_checkRedirectToSame8 = verifyNot checkRedirectToSame "while read -r line; do _=\"$fname\"; done <\"$fname\""

checkRedirectToSame params s@(T_Pipeline _ _ list) =

mapM_ (\l -> (mapM_ (\x -> doAnalysis (checkOccurrences x) l) (getAllRedirs list))) list

where

note x = makeComment InfoC x 2094

"Make sure not to read and write the same file in the same pipeline."

checkOccurrences t@(T_NormalWord exceptId x) u@(T_NormalWord newId y) |

exceptId /= newId

&& x == y

&& not (isOutput t && isOutput u)

&& not (special t)

&& not (any isHarmlessCommand [t,u])

&& not (any containsAssignment [u]) = do

addComment $ note newId

addComment $ note exceptId

checkOccurrences _ _ = return ()

getAllRedirs = concatMap (\t ->

case t of

T_Redirecting _ ls _ -> concatMap getRedirs ls

_ -> [])

getRedirs (T_FdRedirect _ _ (T_IoFile _ op file)) =

case op of T_Greater _ -> [file]

T_Less _ -> [file]

T_DGREAT _ -> [file]

_ -> []

getRedirs _ = []

special x = "/dev/" `isPrefixOf` concat (oversimplify x)

isOutput t =

case drop 1 $ getPath (parentMap params) t of

T_IoFile _ op _:_ ->

case op of

T_Greater _ -> True

T_DGREAT _ -> True

_ -> False

_ -> False

isHarmlessCommand arg = fromMaybe False $ do

cmd <- getClosestCommand (parentMap params) arg

name <- getCommandBasename cmd

return $ name `elem` ["echo", "printf", "sponge"]

containsAssignment arg = fromMaybe False $ do

cmd <- getClosestCommand (parentMap params) arg

return $ isAssignment cmd

checkRedirectToSame _ _ = return ()

prop_checkShorthandIf = verify checkShorthandIf "[[ ! -z file ]] && scp file host || rm file"

prop_checkShorthandIf2 = verifyNot checkShorthandIf "[[ ! -z file ]] && { scp file host || echo 'Eek'; }"

prop_checkShorthandIf3 = verifyNot checkShorthandIf "foo && bar || echo baz"

prop_checkShorthandIf4 = verifyNot checkShorthandIf "foo && a=b || a=c"

prop_checkShorthandIf5 = verifyNot checkShorthandIf "foo && rm || printf b"

prop_checkShorthandIf6 = verifyNot checkShorthandIf "if foo && bar || baz; then true; fi"

prop_checkShorthandIf7 = verifyNot checkShorthandIf "while foo && bar || baz; do true; done"

prop_checkShorthandIf8 = verify checkShorthandIf "if true; then foo && bar || baz; fi"

checkShorthandIf params x@(T_AndIf id _ (T_OrIf _ _ (T_Pipeline _ _ t)))

| not (isOk t || inCondition) =

info id 2015 "Note that A && B || C is not if-then-else. C may run when A is true."

where

isOk [t] = isAssignment t || fromMaybe False (do

name <- getCommandBasename t

return $ name `elem` ["echo", "exit", "return", "printf"])

isOk _ = False

inCondition = isCondition $ getPath (parentMap params) x

checkShorthandIf _ _ = return ()

prop_checkDollarStar = verify checkDollarStar "for f in $*; do ..; done"

prop_checkDollarStar2 = verifyNot checkDollarStar "a=$*"

prop_checkDollarStar3 = verifyNot checkDollarStar "[[ $* = 'a b' ]]"

checkDollarStar p t@(T_NormalWord _ [T_DollarBraced id _ l])

| concat (oversimplify l) == "*" &&

not (isStrictlyQuoteFree (parentMap p) t) =

warn id 2048 "Use \"$@\" (with quotes) to prevent whitespace problems."

checkDollarStar _ _ = return ()

prop_checkUnquotedDollarAt = verify checkUnquotedDollarAt "ls $@"

prop_checkUnquotedDollarAt1= verifyNot checkUnquotedDollarAt "ls ${#@}"

prop_checkUnquotedDollarAt2 = verify checkUnquotedDollarAt "ls ${foo[@]}"

prop_checkUnquotedDollarAt3 = verifyNot checkUnquotedDollarAt "ls ${#foo[@]}"

prop_checkUnquotedDollarAt4 = verifyNot checkUnquotedDollarAt "ls \"$@\""

prop_checkUnquotedDollarAt5 = verifyNot checkUnquotedDollarAt "ls ${foo/@/ at }"

prop_checkUnquotedDollarAt6 = verifyNot checkUnquotedDollarAt "a=$@"

prop_checkUnquotedDollarAt7 = verify checkUnquotedDollarAt "for f in ${var[@]}; do true; done"

prop_checkUnquotedDollarAt8 = verifyNot checkUnquotedDollarAt "echo \"${args[@]:+${args[@]}}\""

prop_checkUnquotedDollarAt9 = verifyNot checkUnquotedDollarAt "echo ${args[@]:+\"${args[@]}\"}"

prop_checkUnquotedDollarAt10 = verifyNot checkUnquotedDollarAt "echo ${@+\"$@\"}"

checkUnquotedDollarAt p word@(T_NormalWord _ parts) | not $ isStrictlyQuoteFree (parentMap p) word =

forM_ (find isArrayExpansion parts) $ \x ->

unless (isQuotedAlternativeReference x) $

err (getId x) 2068

"Double quote array expansions to avoid re-splitting elements."

checkUnquotedDollarAt _ _ = return ()

prop_checkConcatenatedDollarAt1 = verify checkConcatenatedDollarAt "echo \"foo$@\""

prop_checkConcatenatedDollarAt2 = verify checkConcatenatedDollarAt "echo ${arr[@]}lol"

prop_checkConcatenatedDollarAt3 = verify checkConcatenatedDollarAt "echo $a$@"

prop_checkConcatenatedDollarAt4 = verifyNot checkConcatenatedDollarAt "echo $@"

prop_checkConcatenatedDollarAt5 = verifyNot checkConcatenatedDollarAt "echo \"${arr[@]}\""

checkConcatenatedDollarAt p word@T_NormalWord {}

| not $ isQuoteFree (parentMap p) word

|| null (drop 1 parts) =

mapM_ for array

where

parts = getWordParts word

array = find isArrayExpansion parts

for t = err (getId t) 2145 "Argument mixes string and array. Use * or separate argument."

checkConcatenatedDollarAt _ _ = return ()

prop_checkArrayAsString1 = verify checkArrayAsString "a=$@"

prop_checkArrayAsString2 = verify checkArrayAsString "a=\"${arr[@]}\""

prop_checkArrayAsString3 = verify checkArrayAsString "a=*.png"

prop_checkArrayAsString4 = verify checkArrayAsString "a={1..10}"

prop_checkArrayAsString5 = verifyNot checkArrayAsString "a='*.gif'"

prop_checkArrayAsString6 = verifyNot checkArrayAsString "a=$*"

prop_checkArrayAsString7 = verifyNot checkArrayAsString "a=( $@ )"

checkArrayAsString _ (T_Assignment id _ _ _ word) =

if willConcatInAssignment word

then

warn (getId word) 2124

"Assigning an array to a string! Assign as array, or use * instead of @ to concatenate."

else

when (willBecomeMultipleArgs word) $

warn (getId word) 2125

"Brace expansions and globs are literal in assignments. Quote it or use an array."

checkArrayAsString _ _ = return ()

prop_checkArrayWithoutIndex1 = verifyTree checkArrayWithoutIndex "foo=(a b); echo $foo"

prop_checkArrayWithoutIndex2 = verifyNotTree checkArrayWithoutIndex "foo='bar baz'; foo=($foo); echo ${foo[0]}"

prop_checkArrayWithoutIndex3 = verifyTree checkArrayWithoutIndex "coproc foo while true; do echo cow; done; echo $foo"

prop_checkArrayWithoutIndex4 = verifyTree checkArrayWithoutIndex "coproc tail -f log; echo $COPROC"

prop_checkArrayWithoutIndex5 = verifyTree checkArrayWithoutIndex "a[0]=foo; echo $a"

prop_checkArrayWithoutIndex6 = verifyTree checkArrayWithoutIndex "echo $PIPESTATUS"

prop_checkArrayWithoutIndex7 = verifyTree checkArrayWithoutIndex "a=(a b); a+=c"

prop_checkArrayWithoutIndex8 = verifyTree checkArrayWithoutIndex "declare -a foo; foo=bar;"

prop_checkArrayWithoutIndex9 = verifyTree checkArrayWithoutIndex "read -r -a arr <<< 'foo bar'; echo \"$arr\""

prop_checkArrayWithoutIndex10 = verifyTree checkArrayWithoutIndex "read -ra arr <<< 'foo bar'; echo \"$arr\""

prop_checkArrayWithoutIndex11 = verifyNotTree checkArrayWithoutIndex "read -rpfoobar r; r=42"

checkArrayWithoutIndex params _ =

doVariableFlowAnalysis readF writeF defaultMap (variableFlow params)

where

defaultMap = Map.fromList $ map (\x -> (x,())) arrayVariables

readF _ (T_DollarBraced id _ token) _ = do

map <- get

return . maybeToList $ do

name <- getLiteralString token

assigned <- Map.lookup name map

return $ makeComment WarningC id 2128

"Expanding an array without an index only gives the first element."

readF _ _ _ = return []

writeF _ (T_Assignment id mode name [] _) _ (DataString _) = do

isArray <- gets (Map.member name)

return $ if not isArray then [] else

case mode of

Assign -> [makeComment WarningC id 2178 "Variable was used as an array but is now assigned a string."]

Append -> [makeComment WarningC id 2179 "Use array+=(\"item\") to append items to an array."]

writeF _ t name (DataArray _) = do

modify (Map.insert name ())

return []

writeF _ expr name _ = do

if isIndexed expr

then modify (Map.insert name ())

else modify (Map.delete name)

return []

isIndexed expr =

case expr of

T_Assignment _ _ _ (_:_) _ -> True

_ -> False

prop_checkStderrRedirect = verify checkStderrRedirect "test 2>&1 > cow"

prop_checkStderrRedirect2 = verifyNot checkStderrRedirect "test > cow 2>&1"

prop_checkStderrRedirect3 = verifyNot checkStderrRedirect "test 2>&1 > file | grep stderr"

prop_checkStderrRedirect4 = verifyNot checkStderrRedirect "errors=$(test 2>&1 > file)"

prop_checkStderrRedirect5 = verifyNot checkStderrRedirect "read < <(test 2>&1 > file)"

prop_checkStderrRedirect6 = verify checkStderrRedirect "foo | bar 2>&1 > /dev/null"

prop_checkStderrRedirect7 = verifyNot checkStderrRedirect "{ cmd > file; } 2>&1"

checkStderrRedirect params redir@(T_Redirecting _ [

T_FdRedirect id "2" (T_IoDuplicate _ (T_GREATAND _) "1"),

T_FdRedirect _ _ (T_IoFile _ op _)

] _) = case op of

T_Greater _ -> error

T_DGREAT _ -> error

_ -> return ()

where

usesOutput t =

case t of

(T_Pipeline _ _ list) -> length list > 1 && not (isParentOf (parentMap params) (last list) redir)

T_ProcSub {} -> True

T_DollarExpansion {} -> True

T_Backticked {} -> True

_ -> False

isCaptured = any usesOutput $ getPath (parentMap params) redir

error = unless isCaptured $

warn id 2069 "To redirect stdout+stderr, 2>&1 must be last (or use '{ cmd > file; } 2>&1' to clarify)."

checkStderrRedirect _ _ = return ()

lt x = trace ("Tracing " ++ show x) x

ltt t = trace ("Tracing " ++ show t)

prop_checkSingleQuotedVariables = verify checkSingleQuotedVariables "echo '$foo'"

prop_checkSingleQuotedVariables2 = verify checkSingleQuotedVariables "echo 'lol$1.jpg'"

prop_checkSingleQuotedVariables3 = verifyNot checkSingleQuotedVariables "sed 's/foo$/bar/'"

prop_checkSingleQuotedVariables3a= verify checkSingleQuotedVariables "sed 's/${foo}/bar/'"

prop_checkSingleQuotedVariables3b= verify checkSingleQuotedVariables "sed 's/$(echo cow)/bar/'"

prop_checkSingleQuotedVariables3c= verify checkSingleQuotedVariables "sed 's/$((1+foo))/bar/'"

prop_checkSingleQuotedVariables4 = verifyNot checkSingleQuotedVariables "awk '{print $1}'"

prop_checkSingleQuotedVariables5 = verifyNot checkSingleQuotedVariables "trap 'echo $SECONDS' EXIT"

prop_checkSingleQuotedVariables6 = verifyNot checkSingleQuotedVariables "sed -n '$p'"

prop_checkSingleQuotedVariables6a= verify checkSingleQuotedVariables "sed -n '$pattern'"

prop_checkSingleQuotedVariables7 = verifyNot checkSingleQuotedVariables "PS1='$PWD \\$ '"

prop_checkSingleQuotedVariables8 = verify checkSingleQuotedVariables "find . -exec echo '$1' {} +"

prop_checkSingleQuotedVariables9 = verifyNot checkSingleQuotedVariables "find . -exec awk '{print $1}' {} \\;"

prop_checkSingleQuotedVariables10= verify checkSingleQuotedVariables "echo '`pwd`'"

prop_checkSingleQuotedVariables11= verifyNot checkSingleQuotedVariables "sed '${/lol/d}'"

prop_checkSingleQuotedVariables12= verifyNot checkSingleQuotedVariables "eval 'echo $1'"

prop_checkSingleQuotedVariables13= verifyNot checkSingleQuotedVariables "busybox awk '{print $1}'"

prop_checkSingleQuotedVariables14= verifyNot checkSingleQuotedVariables "[ -v 'bar[$foo]' ]"

prop_checkSingleQuotedVariables15= verifyNot checkSingleQuotedVariables "git filter-branch 'test $GIT_COMMIT'"

prop_checkSingleQuotedVariables16= verify checkSingleQuotedVariables "git '$a'"

prop_checkSingleQuotedVariables17= verifyNot checkSingleQuotedVariables "rename 's/(.)a/$1/g' *"

prop_checkSingleQuotedVariables18= verifyNot checkSingleQuotedVariables "echo '``'"

prop_checkSingleQuotedVariables19= verifyNot checkSingleQuotedVariables "echo '```'"

prop_checkSingleQuotedVariables20= verifyNot checkSingleQuotedVariables "mumps -run %XCMD 'W $O(^GLOBAL(5))'"

prop_checkSingleQuotedVariables21= verifyNot checkSingleQuotedVariables "mumps -run LOOP%XCMD --xec 'W $O(^GLOBAL(6))'"

prop_checkSingleQuotedVariables22= verifyNot checkSingleQuotedVariables "jq '$__loc__'"

prop_checkSingleQuotedVariables23= verifyNot checkSingleQuotedVariables "command jq '$__loc__'"

prop_checkSingleQuotedVariables24= verifyNot checkSingleQuotedVariables "exec jq '$__loc__'"

prop_checkSingleQuotedVariables25= verifyNot checkSingleQuotedVariables "exec -c -a foo jq '$__loc__'"

checkSingleQuotedVariables params t@(T_SingleQuoted id s) =

when (s `matches` re) $

if "sed" == commandName

then unless (s `matches` sedContra) showMessage

else unless isProbablyOk showMessage

where

parents = parentMap params

showMessage = info id 2016

"Expressions don't expand in single quotes, use double quotes for that."

commandName = fromMaybe "" $ do

cmd <- getClosestCommand parents t

name <- getCommandBasename cmd

return $ if name == "find" then getFindCommand cmd else if name == "git" then getGitCommand cmd else if name == "mumps" then getMumpsCommand cmd else name

isProbablyOk =

any isOkAssignment (take 3 $ getPath parents t)

|| commandName `elem` [

"trap"

,"sh"

,"bash"

,"ksh"

,"zsh"

,"ssh"

,"eval"

,"xprop"

,"alias"

,"sudo" -- covering "sudo sh" and such

,"docker" -- like above

,"dpkg-query"

,"jq" -- could also check that user provides --arg

,"rename"

,"unset"

,"git filter-branch"

,"mumps -run %XCMD"

,"mumps -run LOOP%XCMD"

]

|| "awk" `isSuffixOf` commandName

|| "perl" `isPrefixOf` commandName

commonlyQuoted = ["PS1", "PS2", "PS3", "PS4", "PROMPT_COMMAND"]