Skip to content

daca: Add compare-normal-exhaustive.py script to compare normal and exhaustive check level #5860

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
danmar merged 4 commits into from
Jan 9, 2024
Merged

daca: Add compare-normal-exhaustive.py script to compare normal and exhaustive check level #5860

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5635b6c
daca: Add compare-normal-exhaustive.py script to compare normal and e…
danmar Jan 5, 2024
6bbcf7d
update version
danmar Jan 8, 2024
0933865
fix format string
danmar Jan 8, 2024
8eea839
format string
danmar Jan 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
213 changes: 213 additions & 0 deletions tools/compare-normal-exhaustive.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
#!/usr/bin/env python3

# Compare "normal" check level and "exhaustive" check level

import donate_cpu_lib as lib
import argparse
import glob
import os
import sys
import random
import subprocess


def format_float(a, b=1):
if a > 0 and b > 0:
return '{:.2f}'.format(a / b)
return 'N/A'


if __name__ == "__main__":
__my_script_name = os.path.splitext(os.path.basename(sys.argv[0]))[0]
__work_path = os.path.expanduser(os.path.join('~', 'cppcheck-' + __my_script_name + '-workfolder'))

parser = argparse.ArgumentParser(description='Compare --check-level=normal and --check-level=exhaustive output')
parser.add_argument('-j', default=1, type=int, help='Concurency execution threads')
parser.add_argument('--cppcheck-path', default=None, type=str, help='Path to Cppcheck binary, if not given then clone and compile')
package_group = parser.add_mutually_exclusive_group()
package_group.add_argument('-p', default=256, type=int, help='Count of packages to check')
package_group.add_argument('--packages', nargs='+', help='Check specific packages and then stop.')
package_group.add_argument('--packages-path', default=None, type=str, help='Check packages in path.')
parser.add_argument('-o', default='my_check_diff.log', help='Filename of result inside a working path dir')

language_group = parser.add_mutually_exclusive_group()
language_group.add_argument('--c-only', dest='c_only', help='Only process c packages', action='store_true')
language_group.add_argument('--cpp-only', dest='cpp_only', help='Only process c++ packages', action='store_true')
parser.add_argument('--work-path', '--work-path=', default=__work_path, type=str, help='Working directory for reference repo')
args = parser.parse_args()

print(args)

if not lib.check_requirements():
print("Error: Check requirements")
sys.exit(1)

work_path = os.path.abspath(args.work_path)
if not os.path.exists(work_path):
os.makedirs(work_path)

lib.set_jobs('-j' + str(args.j))
result_file = os.path.join(work_path, args.o)
(f, ext) = os.path.splitext(result_file)
timing_file = f + '_timing' + ext
normal_results = f + '_normal' + ext
exhaustive_results = f + '_exhaustive' + ext

if os.path.exists(result_file):
os.remove(result_file)
if os.path.exists(timing_file):
os.remove(timing_file)
if os.path.exists(normal_results):
os.remove(normal_results)
if os.path.exists(exhaustive_results):
os.remove(exhaustive_results)

cppcheck_path = args.cppcheck_path

if cppcheck_path is None:
cppcheck_path = os.path.join(work_path, 'cppcheck')
try:
lib.clone_cppcheck(cppcheck_path, '')
pass
except Exception as e:
print('Failed to clone Cppcheck repository ({}), retry later'.format(e))
sys.exit(1)

if not lib.compile_cppcheck(cppcheck_path):
print('Failed to compile Cppcheck')
sys.exit(1)

if args.packages_path:
# You can download packages using daca2-download.py
args.packages = glob.glob(os.path.join(args.packages_path, '*.tar.xz'))
args.p = len(args.packages)
packages_idxs = list(range(args.p))
random.shuffle(packages_idxs)
elif args.packages:
args.p = len(args.packages)
packages_idxs = []
else:
packages_count = lib.get_packages_count()
if not packages_count:
print("network or server might be temporarily down..")
sys.exit(1)

packages_idxs = list(range(packages_count))
random.shuffle(packages_idxs)

packages_processed = 0
crashes = []
timeouts = []

while (packages_processed < args.p and len(packages_idxs) > 0) or args.packages:
if args.packages:
package = args.packages.pop()
else:
package = lib.get_package(packages_idxs.pop())

if package.startswith('ftp://') or package.startswith('http://'):
tgz = lib.download_package(work_path, package, None)
if tgz is None:
print("No package downloaded")
continue
else:
print('Package: ' + package)
tgz = package

source_path, source_found = lib.unpack_package(work_path, tgz, c_only=args.c_only, cpp_only=args.cpp_only)
if not source_found:
print("No files to process")
continue

results_to_diff = list()

normal_crashed = False
exhaustive_crashed = False

normal_timeout = False
exhaustive_timeout = False

enable = 'style'
debug_warnings = False

libraries = lib.library_includes.get_libraries(source_path)
c, errout, info, time_normal, cppcheck_options, timing_info = lib.scan_package(cppcheck_path, source_path, libraries, enable=enable, debug_warnings=debug_warnings, check_level='normal')
if c < 0:
if c == -101 and 'error: could not find or open any of the paths given.' in errout:
# No sourcefile found (for example only headers present)
print('Error: 101')
elif c == lib.RETURN_CODE_TIMEOUT:
print('Normal check level timed out!')
normal_timeout = True
continue # we don't want to compare timeouts
else:
print('Normal check level crashed!')
normal_crashed = True
results_to_diff.append(errout)

c, errout, info, time_exhaustive, cppcheck_options, timing_info = lib.scan_package(cppcheck_path, source_path, libraries, enable=enable, debug_warnings=debug_warnings, check_level='exhaustive')
if c < 0:
if c == -101 and 'error: could not find or open any of the paths given.' in errout:
# No sourcefile found (for example only headers present)
print('Error: 101')
elif c == lib.RETURN_CODE_TIMEOUT:
print('Exhaustive check level timed out!')
exhaustive_timeout = True
continue # we don't want to compare timeouts
else:
print('Exhaustive check level crashed!')
exhaustive_crashed = True
results_to_diff.append(errout)

if normal_crashed or exhaustive_crashed:
who = None
if normal_crashed and exhaustive_crashed:
who = 'Both'
elif normal_crashed:
who = 'Normal'
else:
who = 'Exhaustive'
crashes.append(package + ' ' + who)

if normal_timeout or exhaustive_timeout:
who = None
if normal_timeout and exhaustive_timeout:
who = 'Both'
elif normal_timeout:
who = 'Normal'
else:
who = 'Exhaustive'
timeouts.append(package + ' ' + who)

with open(result_file, 'a') as myfile:
myfile.write(package + '\n')
diff = lib.diff_results('normal', results_to_diff[0], 'exhaustive', results_to_diff[1])
if not normal_crashed and not exhaustive_crashed and diff != '':
myfile.write('libraries:' + ','.join(libraries) +'\n')
myfile.write('diff:\n' + diff + '\n')

if not normal_crashed and not exhaustive_crashed:
with open(timing_file, 'a') as myfile:
package_width = '140'
timing_width = '>7'
myfile.write('{:{package_width}} {:{timing_width}} {:{timing_width}} {:{timing_width}}\n'.format(
package, format_float(time_normal),
format_float(time_exhaustive), format_float(time_normal, time_exhaustive),
package_width=package_width, timing_width=timing_width))
with open(normal_results, 'a') as myfile:
myfile.write(results_to_diff[0])
with open(exhaustive_results, 'a') as myfile:
myfile.write(results_to_diff[1])

packages_processed += 1
print(str(packages_processed) + ' of ' + str(args.p) + ' packages processed\n')

with open(result_file, 'a') as myfile:
myfile.write('\n\ncrashes\n')
myfile.write('\n'.join(crashes))

with open(result_file, 'a') as myfile:
myfile.write('\n\ntimeouts\n')
myfile.write('\n'.join(timeouts) + '\n')

print('Result saved to: ' + result_file)
16 changes: 11 additions & 5 deletions tools/donate_cpu_lib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
# Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/
# Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic
# changes)
CLIENT_VERSION = "1.3.53"
CLIENT_VERSION = "1.3.54"

# Timeout for analysis with Cppcheck in seconds
CPPCHECK_TIMEOUT = 30 * 60
Expand Down Expand Up @@ -431,7 +431,7 @@ def __run_command(cmd, print_cmd=True):
return return_code, stdout, stderr, elapsed_time


def scan_package(cppcheck_path, source_path, libraries, capture_callstack=True):
def scan_package(cppcheck_path, source_path, libraries, capture_callstack=True, enable='style,information', debug_warnings=True, check_level=None):
print('Analyze..')
libs = ''
for library in libraries:
Expand All @@ -441,10 +441,16 @@ def scan_package(cppcheck_path, source_path, libraries, capture_callstack=True):
dir_to_scan = source_path

# TODO: temporarily disabled timing information - use --showtime=top5_summary when next version is released
# TODO: remove missingInclude disabling when it no longer is implied by --enable=information
# Reference for GNU C: https://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html
options = libs + ' --check-library --inconclusive --enable=style,information --inline-suppr --disable=missingInclude --suppress=unmatchedSuppression --template=daca2'
options += ' --debug-warnings --suppress=autoNoType --suppress=valueFlowBailout --suppress=bailoutUninitVar --suppress=symbolDatabaseWarning'
options = '{} --inconclusive --enable={} --inline-suppr --template=daca2'.format(libs, enable)
if 'information' in enable:
# TODO: remove missingInclude disabling when it no longer is implied by --enable=information
options += ' --disable=missingInclude --suppress=unmatchedSuppression'
if check_level:
options += ' --check-level=' + check_level
if debug_warnings:
options += ' --check-library --debug-warnings --suppress=autoNoType --suppress=valueFlowBailout' \
' --suppress=bailoutUninitVar --suppress=symbolDatabaseWarning'
options += ' -D__GNUC__ --platform=unix64'
options_rp = options + ' -rp={}'.format(dir_to_scan)
if __make_cmd == 'msbuild.exe':
Expand Down