{-# LANGUAGE TemplateHaskell, FlexibleContexts #-}

module ShellCheck.Analytics (runAnalytics, ShellCheck.Analytics.runTests) where

import ShellCheck.AST

import ShellCheck.ASTLib

import ShellCheck.AnalyzerLib

import ShellCheck.Data

import ShellCheck.Parser

import ShellCheck.Interface

import ShellCheck.Regex

import Control.Arrow (first)

import Control.Monad

import Control.Monad.Identity

import Control.Monad.State

import Control.Monad.Writer

import Control.Monad.Reader

import Data.Char

import Data.Functor

import Data.Function (on)

import Data.List

import Data.Maybe

import Data.Ord

import Debug.Trace

import qualified Data.Map as Map

import Test.QuickCheck.All (forAllProperties)

import Test.QuickCheck.Test (quickCheckWithResult, stdArgs, maxSuccess)

treeChecks :: [Parameters -> Token -> [TokenComment]]

treeChecks = [

runNodeAnalysis

(\p t -> (mapM_ ((\ f -> f t) . (\ f -> f p))

(nodeChecks ++ checksFor (shellType p))))

,subshellAssignmentCheck

,checkSpacefulness

,checkQuotesInLiterals

,checkShebangParameters

,checkFunctionsUsedExternally

,checkUnusedAssignments

,checkUnpassedInFunctions

,checkArrayWithoutIndex

,checkShebang

,checkUnassignedReferences

,checkUncheckedCd

]

checksFor Sh = [

checkBashisms

,checkTimeParameters

,checkForDecimals

,checkTimedCommand

]

checksFor Dash = [

checkBashisms

,checkForDecimals

,checkLocalScope

,checkTimedCommand

]

checksFor Ksh = [

checkEchoSed

]

checksFor Bash = [

checkTimeParameters

,checkBraceExpansionVars

,checkEchoSed

,checkForDecimals

,checkLocalScope

]

runAnalytics :: AnalysisSpec -> [TokenComment]

runAnalytics options =

runList options treeChecks

runList :: AnalysisSpec -> [Parameters -> Token -> [TokenComment]]

-> [TokenComment]

runList spec list = notes

where

root = asScript spec

params = makeParameters spec

notes = concatMap (\f -> f params root) list

checkList l t = concatMap (\f -> f t) l

runNodeAnalysis f p t = execWriter (doAnalysis (f p) t)

nodeChecks :: [Parameters -> Token -> Writer [TokenComment] ()]

nodeChecks = [

checkUuoc

,checkPipePitfalls

,checkForInQuoted

,checkForInLs

,checkShorthandIf

,checkDollarStar

,checkUnquotedDollarAt

,checkStderrRedirect

,checkUnquotedN

,checkNumberComparisons

,checkSingleBracketOperators

,checkDoubleBracketOperators

,checkLiteralBreakingTest

,checkConstantNoary

,checkDivBeforeMult

,checkArithmeticDeref

,checkArithmeticBadOctal

,checkComparisonAgainstGlob

,checkCommarrays

,checkOrNeq

,checkEchoWc

,checkConstantIfs

,checkPipedAssignment

,checkAssignAteCommand

,checkUuoeVar

,checkQuotedCondRegex

,checkForInCat

,checkFindExec

,checkValidCondOps

,checkGlobbedRegex

,checkTestRedirects

,checkIndirectExpansion

,checkSudoRedirect

,checkPS1Assignments

,checkBackticks

,checkInexplicablyUnquoted

,checkTildeInQuotes

,checkLonelyDotDash

,checkSpuriousExec

,checkSpuriousExpansion

,checkDollarBrackets

,checkSshHereDoc

,checkGlobsAsOptions

,checkWhileReadPitfalls

,checkArithmeticOpCommand

,checkCharRangeGlob

,checkUnquotedExpansions

,checkSingleQuotedVariables

,checkRedirectToSame

,checkPrefixAssignmentReference

,checkLoopKeywordScope

,checkCdAndBack

,checkWrongArithmeticAssignment

,checkConditionalAndOrs

,checkFunctionDeclarations

,checkCatastrophicRm

,checkStderrPipe

,checkOverridingPath

,checkArrayAsString

,checkUnsupported

,checkMultipleAppends

,checkSuspiciousIFS

,checkShouldUseGrepQ

,checkTestGlobs

,checkConcatenatedDollarAt

,checkTildeInPath

,checkMaskedReturns

,checkReadWithoutR

,checkLoopVariableReassignment

,checkTrailingBracket

]

wouldHaveBeenGlob s = '*' `elem` s

verify :: (Parameters -> Token -> Writer [TokenComment] ()) -> String -> Bool

verify f s = checkNode f s == Just True

verifyNot :: (Parameters -> Token -> Writer [TokenComment] ()) -> String -> Bool

verifyNot f s = checkNode f s == Just False

verifyTree :: (Parameters -> Token -> [TokenComment]) -> String -> Bool

verifyTree f s = producesComments f s == Just True

verifyNotTree :: (Parameters -> Token -> [TokenComment]) -> String -> Bool

verifyNotTree f s = producesComments f s == Just False

checkCommand str f t@(T_SimpleCommand id _ (cmd:rest)) =

when (t `isCommand` str) $ f cmd rest

checkCommand _ _ _ = return ()

checkUnqualifiedCommand str f t@(T_SimpleCommand id _ (cmd:rest)) =

when (t `isUnqualifiedCommand` str) $ f cmd rest

checkUnqualifiedCommand _ _ _ = return ()

checkNode f = producesComments (runNodeAnalysis f)

producesComments :: (Parameters -> Token -> [TokenComment]) -> String -> Maybe Bool

producesComments f s = do

root <- pScript s

return . not . null $ runList (defaultSpec root) [f]

dist :: Eq a => [a] -> [a] -> Int

dist a b

= last (if lab == 0 then mainDiag

else if lab > 0 then lowers !! (lab - 1)

else{- < 0 -} uppers !! (-1 - lab))

where mainDiag = oneDiag a b (head uppers) (-1 : head lowers)

uppers = eachDiag a b (mainDiag : uppers) -- upper diagonals

lowers = eachDiag b a (mainDiag : lowers) -- lower diagonals

eachDiag a [] diags = []

eachDiag a (bch:bs) (lastDiag:diags) = oneDiag a bs nextDiag lastDiag : eachDiag a bs diags

where nextDiag = head (tail diags)

oneDiag a b diagAbove diagBelow = thisdiag

where doDiag [] b nw n w = []

doDiag a [] nw n w = []

doDiag (ach:as) (bch:bs) nw n w = me : doDiag as bs me (tail n) (tail w)

where me = if ach == bch then nw else 1 + min3 (head w) nw (head n)

firstelt = 1 + head diagBelow

thisdiag = firstelt : doDiag a b firstelt diagAbove (tail diagBelow)

lab = length a - length b

min3 x y z = if x < y then x else min y z

hasFloatingPoint params = shellType params == Ksh

isCondition [] = False

isCondition [_] = False

isCondition (child:parent:rest) =

case child of

T_BatsTest {} -> True -- count anything in a @test as conditional

_ -> getId child `elem` map getId (getConditionChildren parent) || isCondition (parent:rest)

where

getConditionChildren t =

case t of

T_AndIf _ left right -> [left]

T_OrIf id left right -> [left]

T_IfExpression id conditions elses -> concatMap (take 1 . reverse . fst) conditions

T_WhileExpression id c l -> take 1 . reverse $ c

T_UntilExpression id c l -> take 1 . reverse $ c

_ -> []

prop_checkEchoWc3 = verify checkEchoWc "n=$(echo $foo | wc -c)"

checkEchoWc _ (T_Pipeline id _ [a, b]) =

when (acmd == ["echo", "${VAR}"]) $

case bcmd of

["wc", "-c"] -> countMsg

["wc", "-m"] -> countMsg

_ -> return ()

where

acmd = oversimplify a

bcmd = oversimplify b

countMsg = style id 2000 "See if you can use ${#variable} instead."

checkEchoWc _ _ = return ()

prop_checkEchoSed1 = verify checkEchoSed "FOO=$(echo \"$cow\" | sed 's/foo/bar/g')"

prop_checkEchoSed2 = verify checkEchoSed "rm $(echo $cow | sed -e 's,foo,bar,')"

checkEchoSed _ (T_Pipeline id _ [a, b]) =

when (acmd == ["echo", "${VAR}"]) $

case bcmd of

["sed", v] -> checkIn v

["sed", "-e", v] -> checkIn v

_ -> return ()

where

-- This should have used backreferences, but TDFA doesn't support them

sedRe = mkRegex "^s(.)([^\n]*)g?$"

isSimpleSed s = fromMaybe False $ do

[first,rest] <- matchRegex sedRe s

let delimiters = filter (== (head first)) rest

guard $ length delimiters == 2

return True

acmd = oversimplify a

bcmd = oversimplify b

checkIn s =

when (isSimpleSed s) $

style id 2001 "See if you can use ${variable//search/replace} instead."

checkEchoSed _ _ = return ()

prop_checkPipedAssignment1 = verify checkPipedAssignment "A=ls | grep foo"

prop_checkPipedAssignment2 = verifyNot checkPipedAssignment "A=foo cmd | grep foo"

prop_checkPipedAssignment3 = verifyNot checkPipedAssignment "A=foo"

checkPipedAssignment _ (T_Pipeline _ _ (T_Redirecting _ _ (T_SimpleCommand id (_:_) []):_:_)) =

warn id 2036 "If you wanted to assign the output of the pipeline, use a=$(b | c) ."

checkPipedAssignment _ _ = return ()

prop_checkAssignAteCommand1 = verify checkAssignAteCommand "A=ls -l"

prop_checkAssignAteCommand2 = verify checkAssignAteCommand "A=ls --sort=$foo"

prop_checkAssignAteCommand3 = verify checkAssignAteCommand "A=cat foo | grep bar"

prop_checkAssignAteCommand4 = verifyNot checkAssignAteCommand "A=foo ls -l"

prop_checkAssignAteCommand5 = verifyNot checkAssignAteCommand "PAGER=cat grep bar"

checkAssignAteCommand _ (T_SimpleCommand id (T_Assignment _ _ _ _ assignmentTerm:[]) (firstWord:_)) =

when ("-" `isPrefixOf` concat (oversimplify firstWord) ||

isCommonCommand (getLiteralString assignmentTerm)

&& not (isCommonCommand (getLiteralString firstWord))) $

warn id 2037 "To assign the output of a command, use var=$(cmd) ."

where

isCommonCommand (Just s) = s `elem` commonCommands

isCommonCommand _ = False

checkAssignAteCommand _ _ = return ()

prop_checkArithmeticOpCommand1 = verify checkArithmeticOpCommand "i=i + 1"

prop_checkArithmeticOpCommand2 = verify checkArithmeticOpCommand "foo=bar * 2"

prop_checkArithmeticOpCommand3 = verifyNot checkArithmeticOpCommand "foo + opts"

checkArithmeticOpCommand _ (T_SimpleCommand id [T_Assignment {}] (firstWord:_)) =

fromMaybe (return ()) $ check <$> getGlobOrLiteralString firstWord

where

check op =

when (op `elem` ["+", "-", "*", "/"]) $

warn (getId firstWord) 2099 $

"Use $((..)) for arithmetics, e.g. i=$((i " ++ op ++ " 2))"

checkArithmeticOpCommand _ _ = return ()

prop_checkWrongArit = verify checkWrongArithmeticAssignment "i=i+1"

prop_checkWrongArit2 = verify checkWrongArithmeticAssignment "n=2; i=n*2"

checkWrongArithmeticAssignment params (T_SimpleCommand id (T_Assignment _ _ _ _ val:[]) []) =

fromMaybe (return ()) $ do

str <- getNormalString val

match <- matchRegex regex str

var <- match !!! 0

op <- match !!! 1

Map.lookup var references

return . warn (getId val) 2100 $

"Use $((..)) for arithmetics, e.g. i=$((i " ++ op ++ " 2))"

where

regex = mkRegex "^([_a-zA-Z][_a-zA-Z0-9]*)([+*-]).+$"

references = foldl (flip ($)) Map.empty (map insertRef $ variableFlow params)

insertRef (Assignment (_, _, name, _)) =

Map.insert name ()

insertRef _ = Prelude.id

getNormalString (T_NormalWord _ words) = do

parts <- foldl (liftM2 (\x y -> x ++ [y])) (Just []) $ map getLiterals words

return $ concat parts

getNormalString _ = Nothing

getLiterals (T_Literal _ s) = return s

getLiterals (T_Glob _ s) = return s

getLiterals _ = Nothing

checkWrongArithmeticAssignment _ _ = return ()

prop_checkUuoc1 = verify checkUuoc "cat foo | grep bar"

prop_checkUuoc2 = verifyNot checkUuoc "cat * | grep bar"

prop_checkUuoc3 = verify checkUuoc "cat $var | grep bar"

prop_checkUuoc4 = verifyNot checkUuoc "cat $var"

prop_checkUuoc5 = verifyNot checkUuoc "cat \"$@\""

prop_checkUuoc6 = verifyNot checkUuoc "cat -n | grep bar"

checkUuoc _ (T_Pipeline _ _ (T_Redirecting _ _ cmd:_:_)) =

checkCommand "cat" (const f) cmd

where

f [word] = unless (mayBecomeMultipleArgs word || isOption word) $

style (getId word) 2002 "Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead."

f _ = return ()

isOption word = "-" `isPrefixOf` onlyLiteralString word

checkUuoc _ _ = return ()

prop_checkPipePitfalls3 = verify checkPipePitfalls "ls | grep -v mp3"

prop_checkPipePitfalls4 = verifyNot checkPipePitfalls "find . -print0 | xargs -0 foo"

prop_checkPipePitfalls5 = verifyNot checkPipePitfalls "ls -N | foo"

prop_checkPipePitfalls6 = verify checkPipePitfalls "find . | xargs foo"

prop_checkPipePitfalls7 = verifyNot checkPipePitfalls "find . -printf '%s\\n' | xargs foo"

checkPipePitfalls _ (T_Pipeline id _ commands) = do

for ["find", "xargs"] $

\(find:xargs:_) ->

let args = oversimplify xargs ++ oversimplify find

in

unless (any ($ args) [

hasShortParameter '0',

hasParameter "null",

hasParameter "print0",

hasParameter "printf"

]) $ warn (getId find) 2038

"Use -print0/-0 or -exec + to allow for non-alphanumeric filenames."

for ["?", "echo"] $

\(_:echo:_) -> info (getId echo) 2008 "echo doesn't read from stdin, are you sure you should be piping to it?"

for' ["ps", "grep"] $

\x -> info x 2009 "Consider using pgrep instead of grepping ps output."

for' ["grep", "wc"] $

\x -> style x 2126 "Consider using grep -c instead of grep|wc."

didLs <- liftM or . sequence $ [

for' ["ls", "grep"] $

\x -> warn x 2010 "Don't use ls | grep. Use a glob or a for loop with a condition to allow non-alphanumeric filenames.",

for' ["ls", "xargs"] $

\x -> warn x 2011 "Use 'find .. -print0 | xargs -0 ..' or 'find .. -exec .. +' to allow non-alphanumeric filenames."

]

unless didLs $ do

for ["ls", "?"] $

\(ls:_) -> unless (hasShortParameter 'N' (oversimplify ls)) $

info (getId ls) 2012 "Use find instead of ls to better handle non-alphanumeric filenames."

return ()

where

for l f =

let indices = indexOfSublists l (map (headOrDefault "" . oversimplify) commands)

in do

mapM_ (f . (\ n -> take (length l) $ drop n commands)) indices

return . not . null $ indices

for' l f = for l (first f)

first func (x:_) = func (getId x)

first _ _ = return ()

hasShortParameter char = any (\x -> "-" `isPrefixOf` x && char `elem` x)

hasParameter string =

any (isPrefixOf string . dropWhile (== '-'))

checkPipePitfalls _ _ = return ()

indexOfSublists sub = f 0

where

f _ [] = []

f n a@(r:rest) =

let others = f (n+1) rest in

if match sub a

then n:others

else others

match ("?":r1) (_:r2) = match r1 r2

match (x1:r1) (x2:r2) | x1 == x2 = match r1 r2

match [] _ = True

match _ _ = False

prop_checkShebangParameters1 = verifyTree checkShebangParameters "#!/usr/bin/env bash -x\necho cow"

prop_checkShebangParameters2 = verifyNotTree checkShebangParameters "#! /bin/sh -l "

checkShebangParameters p (T_Annotation _ _ t) = checkShebangParameters p t

checkShebangParameters _ (T_Script id sb _) =

[makeComment ErrorC id 2096 "On most OS, shebangs can only specify a single parameter." | length (words sb) > 2]

prop_checkShebang1 = verifyNotTree checkShebang "#!/usr/bin/env bash -x\necho cow"

prop_checkShebang2 = verifyNotTree checkShebang "#! /bin/sh -l "

prop_checkShebang3 = verifyTree checkShebang "ls -l"

checkShebang params (T_Annotation _ _ t) = checkShebang params t

checkShebang params (T_Script id sb _) =

[makeComment ErrorC id 2148 "Tips depend on target shell and yours is unknown. Add a shebang."

| not (shellTypeSpecified params) && sb == "" ]

prop_checkBashisms = verify checkBashisms "while read a; do :; done < <(a)"

prop_checkBashisms2 = verify checkBashisms "[ foo -nt bar ]"

prop_checkBashisms3 = verify checkBashisms "echo $((i++))"

prop_checkBashisms4 = verify checkBashisms "rm !(*.hs)"

prop_checkBashisms5 = verify checkBashisms "source file"

prop_checkBashisms6 = verify checkBashisms "[ \"$a\" == 42 ]"

prop_checkBashisms7 = verify checkBashisms "echo ${var[1]}"

prop_checkBashisms8 = verify checkBashisms "echo ${!var[@]}"

prop_checkBashisms9 = verify checkBashisms "echo ${!var*}"

prop_checkBashisms10= verify checkBashisms "echo ${var:4:12}"

prop_checkBashisms11= verifyNot checkBashisms "echo ${var:-4}"

prop_checkBashisms12= verify checkBashisms "echo ${var//foo/bar}"

prop_checkBashisms13= verify checkBashisms "exec -c env"

prop_checkBashisms14= verify checkBashisms "echo -n \"Foo: \""

prop_checkBashisms15= verify checkBashisms "let n++"

prop_checkBashisms16= verify checkBashisms "echo $RANDOM"

prop_checkBashisms17= verify checkBashisms "echo $((RANDOM%6+1))"

prop_checkBashisms18= verify checkBashisms "foo &> /dev/null"

prop_checkBashisms19= verify checkBashisms "foo > file*.txt"

prop_checkBashisms20= verify checkBashisms "read -ra foo"

prop_checkBashisms21= verify checkBashisms "[ -a foo ]"

prop_checkBashisms22= verifyNot checkBashisms "[ foo -a bar ]"

prop_checkBashisms23= verify checkBashisms "trap mything ERR INT"

prop_checkBashisms24= verifyNot checkBashisms "trap mything INT TERM"

prop_checkBashisms25= verify checkBashisms "cat < /dev/tcp/host/123"

prop_checkBashisms26= verify checkBashisms "trap mything ERR SIGTERM"

prop_checkBashisms27= verify checkBashisms "echo *[^0-9]*"

prop_checkBashisms28= verify checkBashisms "exec {n}>&2"

prop_checkBashisms29= verify checkBashisms "echo ${!var}"

prop_checkBashisms30= verify checkBashisms "printf -v '%s' \"$1\""

prop_checkBashisms31= verify checkBashisms "printf '%q' \"$1\""

prop_checkBashisms32= verifyNot checkBashisms "#!/bin/dash\n[ foo -nt bar ]"

prop_checkBashisms33= verify checkBashisms "#!/bin/sh\necho -n foo"

prop_checkBashisms34= verifyNot checkBashisms "#!/bin/dash\necho -n foo"

prop_checkBashisms35= verifyNot checkBashisms "#!/bin/dash\nlocal foo"

prop_checkBashisms36= verifyNot checkBashisms "#!/bin/dash\nread -p foo -r bar"

prop_checkBashisms37= verifyNot checkBashisms "HOSTNAME=foo; echo $HOSTNAME"

prop_checkBashisms38= verify checkBashisms "RANDOM=9; echo $RANDOM"

prop_checkBashisms39= verify checkBashisms "foo-bar() { true; }"

prop_checkBashisms40= verify checkBashisms "echo $(<file)"

prop_checkBashisms41= verify checkBashisms "echo `<file`"

prop_checkBashisms42= verify checkBashisms "trap foo int"

prop_checkBashisms43= verify checkBashisms "trap foo sigint"

prop_checkBashisms44= verifyNot checkBashisms "#!/bin/dash\ntrap foo int"

prop_checkBashisms45= verifyNot checkBashisms "#!/bin/dash\ntrap foo INT"

prop_checkBashisms46= verify checkBashisms "#!/bin/dash\ntrap foo SIGINT"

prop_checkBashisms47= verify checkBashisms "#!/bin/dash\necho foo 42>/dev/null"

prop_checkBashisms48= verifyNot checkBashisms "#!/bin/dash\necho $LINENO"

prop_checkBashisms49= verify checkBashisms "#!/bin/dash\necho $MACHTYPE"

checkBashisms params = bashism

where

isDash = shellType params == Dash

warnMsg id s =

if isDash

then warn id 2169 $ "In dash, " ++ s ++ " not supported."

else warn id 2039 $ "In POSIX sh, " ++ s ++ " undefined."

bashism (T_ProcSub id _ _) = warnMsg id "process substitution is"

bashism (T_Extglob id _ _) = warnMsg id "extglob is"

bashism (T_DollarSingleQuoted id _) = warnMsg id "$'..' is"

bashism (T_DollarDoubleQuoted id _) = warnMsg id "$\"..\" is"

bashism (T_ForArithmetic id _ _ _ _) = warnMsg id "arithmetic for loops are"

bashism (T_Arithmetic id _) = warnMsg id "standalone ((..)) is"

bashism (T_DollarBracket id _) = warnMsg id "$[..] in place of $((..)) is"

bashism (T_SelectIn id _ _ _) = warnMsg id "select loops are"

bashism (T_BraceExpansion id _) = warnMsg id "brace expansion is"

bashism (T_Condition id DoubleBracket _) = warnMsg id "[[ ]] is"

bashism (T_HereString id _) = warnMsg id "here-strings are"

bashism (TC_Binary id SingleBracket op _ _)

| op `elem` [ "-nt", "-ef", "\\<", "\\>"] =

unless isDash $ warnMsg id $ op ++ " is"

bashism (TC_Binary id SingleBracket "==" _ _) =

warnMsg id "== in place of = is"

bashism (TC_Unary id _ "-a" _) =

warnMsg id "unary -a in place of -e is"

bashism (TA_Unary id op _)

| op `elem` [ "|++", "|--", "++|", "--|"] =

warnMsg id $ filter (/= '|') op ++ " is"

bashism (TA_Binary id "**" _ _) = warnMsg id "exponentials are"

bashism (T_FdRedirect id "&" (T_IoFile _ (T_Greater _) _)) = warnMsg id "&> is"

bashism (T_FdRedirect id ('{':_) _) = warnMsg id "named file descriptors are"

bashism (T_FdRedirect id num _)

| all isDigit num && length num > 1 = warnMsg id "FDs outside 0-9 are"

bashism (T_IoFile id _ word) | isNetworked =

warnMsg id "/dev/{tcp,udp} is"

where

file = onlyLiteralString word

isNetworked = any (`isPrefixOf` file) ["/dev/tcp", "/dev/udp"]

bashism (T_Glob id str) | "[^" `isInfixOf` str =

warnMsg id "^ in place of ! in glob bracket expressions is"

bashism t@(TA_Expansion id _) | isBashism =

warnMsg id $ fromJust str ++ " is"

where

str = getLiteralString t

isBashism = isJust str && isBashVariable (fromJust str)

bashism t@(T_DollarBraced id token) = do

mapM_ check expansion

when (isBashVariable var) $

warnMsg id $ var ++ " is"

where

str = bracedString t

var = getBracedReference str

check (regex, feature) =

when (isJust $ matchRegex regex str) $ warnMsg id feature

bashism t@(T_Pipe id "|&") =

warnMsg id "|& in place of 2>&1 | is"

bashism (T_Array id _) =

warnMsg id "arrays are"

bashism (T_IoFile id _ t) | isGlob t =

warnMsg id "redirecting to/from globs is"

bashism (T_CoProc id _ _) =

warnMsg id "coproc is"

bashism (T_Function id _ _ str _) | not (isVariableName str) =

warnMsg id "naming functions outside [a-zA-Z_][a-zA-Z0-9_]* is"

bashism (T_DollarExpansion id [x]) | isOnlyRedirection x =

warnMsg id "$(<file) to read files is"

bashism (T_Backticked id [x]) | isOnlyRedirection x =

warnMsg id "`<file` to read files is"

bashism t@(T_SimpleCommand _ _ (cmd:arg:_))

| t `isCommand` "echo" && "-" `isPrefixOf` argString =

unless ("--" `isPrefixOf` argString) $ -- echo "-----"

if isDash

then

when (argString /= "-n") $

warnMsg (getId arg) "echo flags besides -n"

else

warnMsg (getId arg) "echo flags are"

where argString = concat $ oversimplify arg

bashism t@(T_SimpleCommand _ _ (cmd:arg:_))

| t `isCommand` "exec" && "-" `isPrefixOf` concat (oversimplify arg) =

warnMsg (getId arg) "exec flags are"

bashism t@(T_SimpleCommand id _ _)

| t `isCommand` "let" = warnMsg id "'let' is"

bashism t@(T_SimpleCommand id _ (cmd:rest)) =

let name = fromMaybe "" $ getCommandName t

flags = getLeadingFlags t

in do

when (name `elem` unsupportedCommands) $

warnMsg id $ "'" ++ name ++ "' is"

potentially $ do

allowed <- Map.lookup name allowedFlags

(word, flag) <- listToMaybe $

filter (\x -> (not . null . snd $ x) && snd x `notElem` allowed) flags

return . warnMsg (getId word) $ name ++ " -" ++ flag ++ " is"

when (name == "source") $ warnMsg id "'source' in place of '.' is"

when (name == "trap") $

let

check token = potentially $ do

str <- getLiteralString token

let upper = map toUpper str

return $ do

when (upper `elem` ["ERR", "DEBUG", "RETURN"]) $

warnMsg (getId token) $ "trapping " ++ str ++ " is"

when ("SIG" `isPrefixOf` upper) $

warnMsg (getId token)

"prefixing signal names with 'SIG' is"

when (not isDash && upper /= str) $

warnMsg (getId token)

"using lower/mixed case for signal names is"

in

mapM_ check (drop 1 rest)

when (name == "printf") $ potentially $ do

format <- rest !!! 0 -- flags are covered by allowedFlags

let literal = onlyLiteralString format

guard $ "%q" `isInfixOf` literal

return $ warnMsg (getId format) "printf %q is"

where

unsupportedCommands = [

"let", "caller", "builtin", "complete", "compgen", "declare", "dirs", "disown",

"enable", "mapfile", "readarray", "pushd", "popd", "shopt", "suspend",

"typeset"

] ++ if not isDash then ["local", "type"] else []

allowedFlags = Map.fromList [

("read", if isDash then ["r", "p"] else ["r"]),

("ulimit", ["f"]),

("printf", []),

("exec", [])

]

bashism _ = return ()

varChars="_0-9a-zA-Z"

expansion = let re = mkRegex in [

(re $ "^![" ++ varChars ++ "]", "indirect expansion is"),

(re $ "^[" ++ varChars ++ "]+\\[.*\\]$", "array references are"),

(re $ "^![" ++ varChars ++ "]+\\[[*@]]$", "array key expansion is"),

(re $ "^![" ++ varChars ++ "]+[*@]$", "name matching prefixes are"),

(re $ "^[" ++ varChars ++ "]+:[^-=?+]", "string indexing is"),

(re $ "^[" ++ varChars ++ "]+(\\[.*\\])?/", "string replacement is")

]

bashVars = [

"LINENO", "OSTYPE", "MACHTYPE", "HOSTTYPE", "HOSTNAME",

"DIRSTACK", "EUID", "UID", "SHLVL", "PIPESTATUS", "SHELLOPTS"

]

bashDynamicVars = [ "RANDOM", "SECONDS" ]

dashVars = [ "LINENO" ]

isBashVariable var =

(var `elem` bashDynamicVars

|| var `elem` bashVars && not (isAssigned var))

&& not (isDash && var `elem` dashVars)

isAssigned var = any f (variableFlow params)

where

f x = case x of

Assignment (_, _, name, _) -> name == var

_ -> False

prop_checkForInQuoted = verify checkForInQuoted "for f in \"$(ls)\"; do echo foo; done"

prop_checkForInQuoted2 = verifyNot checkForInQuoted "for f in \"$@\"; do echo foo; done"

prop_checkForInQuoted2a = verifyNot checkForInQuoted "for f in *.mp3; do echo foo; done"

prop_checkForInQuoted2b = verify checkForInQuoted "for f in \"*.mp3\"; do echo foo; done"

prop_checkForInQuoted3 = verify checkForInQuoted "for f in 'find /'; do true; done"

prop_checkForInQuoted4 = verify checkForInQuoted "for f in 1,2,3; do true; done"

prop_checkForInQuoted4a = verifyNot checkForInQuoted "for f in foo{1,2,3}; do true; done"

prop_checkForInQuoted5 = verify checkForInQuoted "for f in ls; do true; done"

prop_checkForInQuoted6 = verifyNot checkForInQuoted "for f in \"${!arr}\"; do true; done"

checkForInQuoted _ (T_ForIn _ f [T_NormalWord _ [word@(T_DoubleQuoted id list)]] _) =

when (any (\x -> willSplit x && not (mayBecomeMultipleArgs x)) list

|| (liftM wouldHaveBeenGlob (getLiteralString word) == Just True)) $

err id 2066 "Since you double quoted this, it will not word split, and the loop will only run once."

checkForInQuoted _ (T_ForIn _ f [T_NormalWord _ [T_SingleQuoted id s]] _) =

warn id 2041 $ "This is a literal string. To run as a command, use $(" ++ s ++ ")."

checkForInQuoted _ (T_ForIn _ f [T_NormalWord _ [T_Literal id s]] _) =

if ',' `elem` s

then unless ('{' `elem` s) $

warn id 2042 "Use spaces, not commas, to separate loop elements."

else warn id 2043 $ "This loop will only run once, with " ++ f ++ "='" ++ s ++ "'."

checkForInQuoted _ _ = return ()

prop_checkForInCat1 = verify checkForInCat "for f in $(cat foo); do stuff; done"

prop_checkForInCat1a= verify checkForInCat "for f in `cat foo`; do stuff; done"

prop_checkForInCat2 = verify checkForInCat "for f in $(cat foo | grep lol); do stuff; done"

prop_checkForInCat2a= verify checkForInCat "for f in `cat foo | grep lol`; do stuff; done"

prop_checkForInCat3 = verifyNot checkForInCat "for f in $(cat foo | grep bar | wc -l); do stuff; done"

checkForInCat _ (T_ForIn _ f [T_NormalWord _ w] _) = mapM_ checkF w

where

checkF (T_DollarExpansion id [T_Pipeline _ _ r])

| all isLineBased r =

info id 2013 "To read lines rather than words, pipe/redirect to a 'while read' loop."

checkF (T_Backticked id cmds) = checkF (T_DollarExpansion id cmds)

checkF _ = return ()

isLineBased cmd = any (cmd `isCommand`)

["grep", "fgrep", "egrep", "sed", "cat", "awk", "cut", "sort"]

checkForInCat _ _ = return ()

prop_checkForInLs = verify checkForInLs "for f in $(ls *.mp3); do mplayer \"$f\"; done"

prop_checkForInLs2 = verify checkForInLs "for f in `ls *.mp3`; do mplayer \"$f\"; done"

prop_checkForInLs3 = verify checkForInLs "for f in `find / -name '*.mp3'`; do mplayer \"$f\"; done"

checkForInLs _ = try

where

try (T_ForIn _ f [T_NormalWord _ [T_DollarExpansion id [x]]] _) =

check id f x

try (T_ForIn _ f [T_NormalWord _ [T_Backticked id [x]]] _) =

check id f x

try _ = return ()

check id f x =

case oversimplify x of

("ls":n) ->

let warntype = if any ("-" `isPrefixOf`) n then warn else err in

warntype id 2045 "Iterating over ls output is fragile. Use globs."

("find":_) -> warn id 2044 "For loops over find output are fragile. Use find -exec or a while read loop."

_ -> return ()

prop_checkFindExec1 = verify checkFindExec "find / -name '*.php' -exec rm {};"

prop_checkFindExec2 = verify checkFindExec "find / -exec touch {} && ls {} \\;"

prop_checkFindExec3 = verify checkFindExec "find / -execdir cat {} | grep lol +"

prop_checkFindExec4 = verifyNot checkFindExec "find / -name '*.php' -exec foo {} +"

prop_checkFindExec5 = verifyNot checkFindExec "find / -execdir bash -c 'a && b' \\;"

prop_checkFindExec6 = verify checkFindExec "find / -type d -execdir rm *.jpg \\;"

checkFindExec _ cmd@(T_SimpleCommand _ _ t@(h:r)) | cmd `isCommand` "find" = do

c <- broken r False

when c $

let wordId = getId $ last t in

err wordId 2067 "Missing ';' or + terminating -exec. You can't use |/||/&&, and ';' has to be a separate, quoted argument."

where

broken [] v = return v

broken (w:r) v = do

when v (mapM_ warnFor $ fromWord w)

case getLiteralString w of

Just "-exec" -> broken r True

Just "-execdir" -> broken r True

Just "+" -> broken r False

Just ";" -> broken r False

_ -> broken r v

shouldWarn x =

case x of

T_DollarExpansion _ _ -> True

T_Backticked _ _ -> True

T_Glob _ _ -> True

T_Extglob {} -> True

_ -> False

warnFor x =

when(shouldWarn x) $

info (getId x) 2014 "This will expand once before find runs, not per file found."

fromWord (T_NormalWord _ l) = l

fromWord _ = []

checkFindExec _ _ = return ()

prop_checkUnquotedExpansions1 = verify checkUnquotedExpansions "rm $(ls)"

prop_checkUnquotedExpansions1a= verify checkUnquotedExpansions "rm `ls`"

prop_checkUnquotedExpansions2 = verify checkUnquotedExpansions "rm foo$(date)"

prop_checkUnquotedExpansions3 = verify checkUnquotedExpansions "[ $(foo) == cow ]"

prop_checkUnquotedExpansions3a= verify checkUnquotedExpansions "[ ! $(foo) ]"

prop_checkUnquotedExpansions4 = verifyNot checkUnquotedExpansions "[[ $(foo) == cow ]]"

prop_checkUnquotedExpansions5 = verifyNot checkUnquotedExpansions "for f in $(cmd); do echo $f; done"

prop_checkUnquotedExpansions6 = verifyNot checkUnquotedExpansions "$(cmd)"

prop_checkUnquotedExpansions7 = verifyNot checkUnquotedExpansions "cat << foo\n$(ls)\nfoo"

checkUnquotedExpansions params =

check

where

check t@(T_DollarExpansion _ _) = examine t

check t@(T_Backticked _ _) = examine t

check t@(T_DollarBraceCommandExpansion _ _) = examine t

check _ = return ()

tree = parentMap params

examine t =

unless (isQuoteFree tree t || usedAsCommandName tree t) $

warn (getId t) 2046 "Quote this to prevent word splitting."

prop_checkRedirectToSame = verify checkRedirectToSame "cat foo > foo"

prop_checkRedirectToSame2 = verify checkRedirectToSame "cat lol | sed -e 's/a/b/g' > lol"

prop_checkRedirectToSame3 = verifyNot checkRedirectToSame "cat lol | sed -e 's/a/b/g' > foo.bar && mv foo.bar lol"

prop_checkRedirectToSame4 = verifyNot checkRedirectToSame "foo /dev/null > /dev/null"

prop_checkRedirectToSame5 = verifyNot checkRedirectToSame "foo > bar 2> bar"

prop_checkRedirectToSame6 = verifyNot checkRedirectToSame "echo foo > foo"

prop_checkRedirectToSame7 = verifyNot checkRedirectToSame "sed 's/foo/bar/g' file | sponge file"

checkRedirectToSame params s@(T_Pipeline _ _ list) =

mapM_ (\l -> (mapM_ (\x -> doAnalysis (checkOccurrences x) l) (getAllRedirs list))) list

where

note x = makeComment InfoC x 2094

"Make sure not to read and write the same file in the same pipeline."

checkOccurrences t@(T_NormalWord exceptId x) u@(T_NormalWord newId y) =

when (exceptId /= newId

&& x == y

&& not (isOutput t && isOutput u)

&& not (special t)

&& not (any isHarmlessCommand [t,u])) $ do

addComment $ note newId

addComment $ note exceptId

checkOccurrences _ _ = return ()

getAllRedirs = concatMap (\t ->

case t of

T_Redirecting _ ls _ -> concatMap getRedirs ls

_ -> [])

getRedirs (T_FdRedirect _ _ (T_IoFile _ op file)) =

case op of T_Greater _ -> [file]

T_Less _ -> [file]

T_DGREAT _ -> [file]

_ -> []

getRedirs _ = []

special x = "/dev/" `isPrefixOf` concat (oversimplify x)

isOutput t =

case drop 1 $ getPath (parentMap params) t of

T_IoFile _ op _:_ ->

case op of

T_Greater _ -> True

T_DGREAT _ -> True

_ -> False

_ -> False

isHarmlessCommand arg = fromMaybe False $ do

cmd <- getClosestCommand (parentMap params) arg

name <- getCommandBasename cmd

return $ name `elem` ["echo", "printf", "sponge"]

checkRedirectToSame _ _ = return ()

prop_checkShorthandIf = verify checkShorthandIf "[[ ! -z file ]] && scp file host || rm file"

prop_checkShorthandIf2 = verifyNot checkShorthandIf "[[ ! -z file ]] && { scp file host || echo 'Eek'; }"

prop_checkShorthandIf3 = verifyNot checkShorthandIf "foo && bar || echo baz"

prop_checkShorthandIf4 = verifyNot checkShorthandIf "foo && a=b || a=c"

prop_checkShorthandIf5 = verifyNot checkShorthandIf "foo && rm || printf b"

prop_checkShorthandIf6 = verifyNot checkShorthandIf "if foo && bar || baz; then true; fi"

prop_checkShorthandIf7 = verifyNot checkShorthandIf "while foo && bar || baz; do true; done"

prop_checkShorthandIf8 = verify checkShorthandIf "if true; then foo && bar || baz; fi"

checkShorthandIf params x@(T_AndIf id _ (T_OrIf _ _ (T_Pipeline _ _ t)))

| not (isOk t || inCondition) =

info id 2015 "Note that A && B || C is not if-then-else. C may run when A is true."

where

isOk [t] = isAssignment t || fromMaybe False (do

name <- getCommandBasename t

return $ name `elem` ["echo", "exit", "return", "printf"])

isOk _ = False

inCondition = isCondition $ getPath (parentMap params) x

checkShorthandIf _ _ = return ()

prop_checkDollarStar = verify checkDollarStar "for f in $*; do ..; done"

prop_checkDollarStar2 = verifyNot checkDollarStar "a=$*"

checkDollarStar p t@(T_NormalWord _ [b@(T_DollarBraced id _)])

| bracedString b == "*" =

unless isAssigned $

warn id 2048 "Use \"$@\" (with quotes) to prevent whitespace problems."

where

path = getPath (parentMap p) t

isAssigned = any isAssignment . take 2 $ path

checkDollarStar _ _ = return ()

prop_checkUnquotedDollarAt = verify checkUnquotedDollarAt "ls $@"

prop_checkUnquotedDollarAt1= verifyNot checkUnquotedDollarAt "ls ${#@}"

prop_checkUnquotedDollarAt2 = verify checkUnquotedDollarAt "ls ${foo[@]}"

prop_checkUnquotedDollarAt3 = verifyNot checkUnquotedDollarAt "ls ${#foo[@]}"

prop_checkUnquotedDollarAt4 = verifyNot checkUnquotedDollarAt "ls \"$@\""

prop_checkUnquotedDollarAt5 = verifyNot checkUnquotedDollarAt "ls ${foo/@/ at }"

prop_checkUnquotedDollarAt6 = verifyNot checkUnquotedDollarAt "a=$@"

prop_checkUnquotedDollarAt7 = verify checkUnquotedDollarAt "for f in ${var[@]}; do true; done"

prop_checkUnquotedDollarAt8 = verifyNot checkUnquotedDollarAt "echo \"${args[@]:+${args[@]}}\""

prop_checkUnquotedDollarAt9 = verifyNot checkUnquotedDollarAt "echo ${args[@]:+\"${args[@]}\"}"

checkUnquotedDollarAt p word@(T_NormalWord _ parts) | not $ isStrictlyQuoteFree (parentMap p) word =

forM_ (take 1 $ filter isArrayExpansion parts) $ \x ->

unless (isAlternative x) $

err (getId x) 2068

"Double quote array expansions to avoid re-splitting elements."

where

-- Fixme: should detect whether the alternative is quoted

isAlternative b@(T_DollarBraced _ t) = ":+" `isInfixOf` bracedString b

isAlternative _ = False

checkUnquotedDollarAt _ _ = return ()

prop_checkConcatenatedDollarAt1 = verify checkConcatenatedDollarAt "echo \"foo$@\""

prop_checkConcatenatedDollarAt2 = verify checkConcatenatedDollarAt "echo ${arr[@]}lol"

prop_checkConcatenatedDollarAt3 = verify checkConcatenatedDollarAt "echo $a$@"

prop_checkConcatenatedDollarAt4 = verifyNot checkConcatenatedDollarAt "echo $@"

prop_checkConcatenatedDollarAt5 = verifyNot checkConcatenatedDollarAt "echo \"${arr[@]}\""

checkConcatenatedDollarAt p word@(T_NormalWord {})

| not $ isQuoteFree (parentMap p) word =

unless (null $ drop 1 parts) $

mapM_ for array

where

parts = getWordParts word

array = take 1 $ filter isArrayExpansion parts

for t = err (getId t) 2145 "Argument mixes string and array. Use * or separate argument."

checkConcatenatedDollarAt _ _ = return ()

prop_checkArrayAsString1 = verify checkArrayAsString "a=$@"

prop_checkArrayAsString2 = verify checkArrayAsString "a=\"${arr[@]}\""

prop_checkArrayAsString3 = verify checkArrayAsString "a=*.png"

prop_checkArrayAsString4 = verify checkArrayAsString "a={1..10}"

prop_checkArrayAsString5 = verifyNot checkArrayAsString "a='*.gif'"

prop_checkArrayAsString6 = verifyNot checkArrayAsString "a=$*"

prop_checkArrayAsString7 = verifyNot checkArrayAsString "a=( $@ )"

checkArrayAsString _ (T_Assignment id _ _ _ word) =

if willConcatInAssignment word

then

warn (getId word) 2124

"Assigning an array to a string! Assign as array, or use * instead of @ to concatenate."

else

when (willBecomeMultipleArgs word) $

warn (getId word) 2125

"Brace expansions and globs are literal in assignments. Quote it or use an array."

checkArrayAsString _ _ = return ()

prop_checkArrayWithoutIndex1 = verifyTree checkArrayWithoutIndex "foo=(a b); echo $foo"

prop_checkArrayWithoutIndex2 = verifyNotTree checkArrayWithoutIndex "foo='bar baz'; foo=($foo); echo ${foo[0]}"

prop_checkArrayWithoutIndex3 = verifyTree checkArrayWithoutIndex "coproc foo while true; do echo cow; done; echo $foo"

prop_checkArrayWithoutIndex4 = verifyTree checkArrayWithoutIndex "coproc tail -f log; echo $COPROC"

prop_checkArrayWithoutIndex5 = verifyTree checkArrayWithoutIndex "a[0]=foo; echo $a"

prop_checkArrayWithoutIndex6 = verifyTree checkArrayWithoutIndex "echo $PIPESTATUS"

prop_checkArrayWithoutIndex7 = verifyTree checkArrayWithoutIndex "a=(a b); a+=c"

prop_checkArrayWithoutIndex8 = verifyTree checkArrayWithoutIndex "declare -a foo; foo=bar;"

checkArrayWithoutIndex params _ =

doVariableFlowAnalysis readF writeF defaultMap (variableFlow params)

where

defaultMap = Map.fromList $ map (\x -> (x,())) arrayVariables

readF _ (T_DollarBraced id token) _ = do

map <- get

return . maybeToList $ do

name <- getLiteralString token

assigned <- Map.lookup name map

return $ makeComment WarningC id 2128

"Expanding an array without an index only gives the first element."

readF _ _ _ = return []

writeF _ (T_Assignment id mode name Nothing _) _ (DataString _) = do

isArray <- gets (isJust . Map.lookup name)

return $ if not isArray then [] else

case mode of

Assign -> [makeComment WarningC id 2178 "Variable was used as an array but is now assigned a string."]

Append -> [makeComment WarningC id 2179 "Use array+=(\"item\") to append items to an array."]

writeF _ t name (DataArray _) = do

modify (Map.insert name ())

return []

writeF _ expr name _ = do

if isIndexed expr

then modify (Map.insert name ())

else modify (Map.delete name)

return []

isIndexed expr =

case expr of

T_Assignment _ _ _ (Just _) _ -> True

_ -> False

prop_checkStderrRedirect = verify checkStderrRedirect "test 2>&1 > cow"

prop_checkStderrRedirect2 = verifyNot checkStderrRedirect "test > cow 2>&1"

prop_checkStderrRedirect3 = verifyNot checkStderrRedirect "test 2>&1 > file | grep stderr"

prop_checkStderrRedirect4 = verifyNot checkStderrRedirect "errors=$(test 2>&1 > file)"

prop_checkStderrRedirect5 = verifyNot checkStderrRedirect "read < <(test 2>&1 > file)"

prop_checkStderrRedirect6 = verify checkStderrRedirect "foo | bar 2>&1 > /dev/null"

checkStderrRedirect params redir@(T_Redirecting _ [

T_FdRedirect id "2" (T_IoFile _ (T_GREATAND _) (T_NormalWord _ [T_Literal _ "1"])),

T_FdRedirect _ _ (T_IoFile _ op _)

] _) = case op of

T_Greater _ -> error

T_DGREAT _ -> error

_ -> return ()

where

usesOutput t =

case t of

(T_Pipeline _ _ list) -> length list > 1 && not (isParentOf (parentMap params) (last list) redir)

(T_ProcSub {}) -> True

(T_DollarExpansion {}) -> True

(T_Backticked {}) -> True

_ -> False

isCaptured = any usesOutput $ getPath (parentMap params) redir

error = unless isCaptured $

err id 2069 "The order of the 2>&1 and the redirect matters. The 2>&1 has to be last."

checkStderrRedirect _ _ = return ()

lt x = trace ("Tracing " ++ show x) x

ltt t = trace ("Tracing " ++ show t)

prop_checkSingleQuotedVariables = verify checkSingleQuotedVariables "echo '$foo'"

prop_checkSingleQuotedVariables2 = verify checkSingleQuotedVariables "echo 'lol$1.jpg'"

prop_checkSingleQuotedVariables3 = verifyNot checkSingleQuotedVariables "sed 's/foo$/bar/'"

prop_checkSingleQuotedVariables3a= verify checkSingleQuotedVariables "sed 's/${foo}/bar/'"

prop_checkSingleQuotedVariables3b= verify checkSingleQuotedVariables "sed 's/$(echo cow)/bar/'"

prop_checkSingleQuotedVariables3c= verify checkSingleQuotedVariables "sed 's/$((1+foo))/bar/'"

prop_checkSingleQuotedVariables4 = verifyNot checkSingleQuotedVariables "awk '{print $1}'"

prop_checkSingleQuotedVariables5 = verifyNot checkSingleQuotedVariables "trap 'echo $SECONDS' EXIT"

prop_checkSingleQuotedVariables6 = verifyNot checkSingleQuotedVariables "sed -n '$p'"

prop_checkSingleQuotedVariables6a= verify checkSingleQuotedVariables "sed -n '$pattern'"