QA: PCB file input parse test program (fuzzable)
This adds a test program which can be used to test the parsing of a given KiCad PCB file. This interface is useful for both manual or automated debugging of given files, as well as providing an interface suitable for fuzz-testing tools. Also adds to the testing docs to detail how fuzzing can be used. Also moves some useful re-usable code from io-benchmark to a new library qa_utils, which can contain code that isn't need in the actual KiCad libs.
This commit is contained in:
John Beard
Wayne Stambaugh
parent
2a170c9847
commit
acd103631b
|
|
@ -103,6 +103,51 @@ You can run the tests in GDB to trace this:
|
|||
If the test segfaults, you will get a familiar backtrace, just like
|
||||
if you were running pcbnew under GDB.
|
||||
|
||||
## Fuzz testing ##
|
||||
|
||||
It is possible to run fuzz testing on some parts of KiCad. To do this for a
|
||||
generic function, you need to be able to pass some kind of input from the fuzz
|
||||
testing tool to the function under test.
|
||||
|
||||
For example, to use the [AFL fuzzing tool][], you will need:
|
||||
|
||||
* A test executable that can:
|
||||
** Receive input from `stdin` to be run by `afl-fuzz`.
|
||||
** Optional: process input from a filename to allow `afl-tmin` to minimise the
|
||||
input files.
|
||||
* To compile this executable with an AFL compiler, to enable the instrumentation
|
||||
that allows the fuzzer to detect the fuzzing state.
|
||||
|
||||
For example, the `qa_pcb_parse_input` executable can be compiled like this:
|
||||
|
||||
mkdir build
|
||||
cd build
|
||||
cmake -DCMAKE_CXX_COMPILER=/usr/bin/afl-clang-fast++ -DCMAKE_C_COMPILER=/usr/bin/afl-clang-fast ../kicad_src
|
||||
make qa_pcb_parse_input
|
||||
|
||||
You may need to disable core dumps and CPU frequency scaling on your system (AFL
|
||||
will warn you if you should do this). For example, as root:
|
||||
|
||||
# echo core >/proc/sys/kernel/core_pattern
|
||||
# echo performance | tee cpu*/cpufreq/scaling_governor
|
||||
|
||||
To fuzz:
|
||||
|
||||
afl-fuzz -i fuzzin -o fuzzout -m500 qa/pcb_parse_input/qa_pcb_parse_input
|
||||
|
||||
where:
|
||||
|
||||
* `-i` is a directory of files to use as fuzz input "seeds"
|
||||
* `-o` is a directory to write the results (including inputs that provoke crashes
|
||||
or hangs)
|
||||
* `-t` is the maximum time that a run is allowed to take before being declared a "hang"
|
||||
* `-m` is the memory allowed to use (this often needs to be bumped, as KiCad code
|
||||
tends to use a lot of memory to initialise)
|
||||
|
||||
The AFL TUI will then display the fuzzing progress, and you can use the hang- or
|
||||
crash-provoking inputs to debug code as needed.
|
||||
|
||||
[CTest]: https://cmake.org/cmake/help/latest/module/CTest.html
|
||||
[Boost Unit Test framework]: https://www.boost.org/doc/libs/1_68_0/libs/test/doc/html/index.html
|
||||
[boost-test-functions]: https://www.boost.org/doc/libs/1_68_0/libs/test/doc/html/boost_test/utf_reference/testing_tool_ref.html
|
||||
[boost-test-functions]: https://www.boost.org/doc/libs/1_68_0/libs/test/doc/html/boost_test/utf_reference/testing_tool_ref.html
|
||||
[AFL fuzzing tool]: http://lcamtuf.coredump.cx/afl/
|
||||
|
|
@ -12,8 +12,12 @@ if( KICAD_SCRIPTING_MODULES )
|
|||
|
||||
endif()
|
||||
|
||||
# common QA helpers
|
||||
add_subdirectory( qa_utils )
|
||||
|
||||
add_subdirectory( common )
|
||||
add_subdirectory( shape_poly_set_refactor )
|
||||
add_subdirectory( pcb_parse_input )
|
||||
# add_subdirectory( pcb_test_window )
|
||||
# add_subdirectory( polygon_triangulation )
|
||||
# add_subdirectory( polygon_generator )
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
# This program source code file is part of KiCad, a free EDA CAD application.
|
||||
#
|
||||
# Copyright (C) 2018 KiCad Developers, see CHANGELOG.TXT for contributors.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, you may find one here:
|
||||
# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
|
||||
# or you may search the http://www.gnu.org website for the version 2 license,
|
||||
# or you may write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
|
||||
|
||||
|
||||
if( BUILD_GITHUB_PLUGIN )
|
||||
set( GITHUB_PLUGIN_LIBRARIES github_plugin )
|
||||
endif()
|
||||
|
||||
add_dependencies( pnsrouter pcbcommon pcad2kicadpcb ${GITHUB_PLUGIN_LIBRARIES} )
|
||||
|
||||
add_executable( qa_pcb_parse_input
|
||||
# This is needed for the global mock objects
|
||||
../qa_utils/mocks.cpp
|
||||
|
||||
main.cpp
|
||||
|
||||
../../common/base_units.cpp
|
||||
../../common/xnode.cpp
|
||||
../../common/base_screen.cpp
|
||||
)
|
||||
|
||||
include_directories( BEFORE ${INC_BEFORE} )
|
||||
include_directories(
|
||||
${CMAKE_SOURCE_DIR}
|
||||
${CMAKE_SOURCE_DIR}/include
|
||||
${CMAKE_SOURCE_DIR}/polygon
|
||||
${CMAKE_SOURCE_DIR}/pcbnew
|
||||
${CMAKE_SOURCE_DIR}/common
|
||||
${CMAKE_SOURCE_DIR}/pcbnew/router
|
||||
${CMAKE_SOURCE_DIR}/pcbnew/tools
|
||||
${CMAKE_SOURCE_DIR}/pcbnew/dialogs
|
||||
${Boost_INCLUDE_DIR}
|
||||
${INC_AFTER}
|
||||
)
|
||||
|
||||
target_link_libraries( qa_pcb_parse_input
|
||||
pcbcommon
|
||||
legacy_wx
|
||||
polygon
|
||||
pnsrouter
|
||||
common
|
||||
pcbcommon
|
||||
bitmaps
|
||||
polygon
|
||||
pnsrouter
|
||||
common
|
||||
pcbcommon
|
||||
bitmaps
|
||||
polygon
|
||||
pnsrouter
|
||||
common
|
||||
pcbcommon
|
||||
bitmaps
|
||||
polygon
|
||||
pnsrouter
|
||||
common
|
||||
pcbcommon
|
||||
3d-viewer
|
||||
bitmaps
|
||||
gal
|
||||
pcad2kicadpcb
|
||||
common
|
||||
pcbcommon
|
||||
${GITHUB_PLUGIN_LIBRARIES}
|
||||
qa_utils
|
||||
${wxWidgets_LIBRARIES}
|
||||
)
|
||||
|
||||
# we need to pretend to be something to appease the units code
|
||||
target_compile_definitions( qa_pcb_parse_input
|
||||
PRIVATE PCBNEW
|
||||
)
|
||||
|
|
@ -0,0 +1,158 @@
|
|||
/*
|
||||
* This program source code file is part of KiCad, a free EDA CAD application.
|
||||
*
|
||||
* Copyright (C) 2018 KiCad Developers, see CHANGELOG.TXT for contributors.
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, you may find one here:
|
||||
* http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
|
||||
* or you may search the http://www.gnu.org website for the version 2 license,
|
||||
* or you may write to the Free Software Foundation, Inc.,
|
||||
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
|
||||
*/
|
||||
|
||||
|
||||
#include <kicad_plugin.h>
|
||||
#include <pcb_parser.h>
|
||||
#include <richio.h>
|
||||
#include <class_board_item.h>
|
||||
|
||||
#include <wx/cmdline.h>
|
||||
|
||||
#include <stdstream_line_reader.h>
|
||||
#include <scoped_timer.h>
|
||||
|
||||
using PARSE_DURATION = std::chrono::microseconds;
|
||||
|
||||
/**
|
||||
* Parse a PCB or footprint file from the given input stream
|
||||
*
|
||||
* @param aStream the input stream to read from
|
||||
* @return success, duration (in us)
|
||||
*/
|
||||
bool parse(std::istream& aStream, bool aVerbose )
|
||||
{
|
||||
// Take input from stdin
|
||||
STDISTREAM_LINE_READER reader;
|
||||
reader.SetStream( aStream );
|
||||
|
||||
PCB_PARSER parser;
|
||||
|
||||
parser.SetLineReader( &reader );
|
||||
|
||||
BOARD_ITEM* board = nullptr;
|
||||
|
||||
PARSE_DURATION duration {};
|
||||
|
||||
try
|
||||
{
|
||||
SCOPED_TIMER<PARSE_DURATION> timer( duration );
|
||||
board = parser.Parse();
|
||||
}
|
||||
catch( const IO_ERROR& parse_error )
|
||||
{
|
||||
std::cerr << parse_error.Problem() << std::endl;
|
||||
std::cerr << parse_error.Where() << std::endl;
|
||||
}
|
||||
|
||||
if( aVerbose )
|
||||
{
|
||||
std::cout << "Took: " << duration.count() << "us" << std::endl;
|
||||
}
|
||||
|
||||
return board != nullptr;
|
||||
}
|
||||
|
||||
|
||||
static const wxCmdLineEntryDesc g_cmdLineDesc [] =
|
||||
{
|
||||
{ wxCMD_LINE_SWITCH, "h", "help",
|
||||
_( "displays help on the command line parameters" ),
|
||||
wxCMD_LINE_VAL_NONE, wxCMD_LINE_OPTION_HELP },
|
||||
{ wxCMD_LINE_SWITCH, "v", "verbose",
|
||||
_( "print parsing information") },
|
||||
{ wxCMD_LINE_PARAM, nullptr, nullptr,
|
||||
_( "input file" ),
|
||||
wxCMD_LINE_VAL_STRING, wxCMD_LINE_PARAM_OPTIONAL | wxCMD_LINE_PARAM_MULTIPLE },
|
||||
{ wxCMD_LINE_NONE }
|
||||
};
|
||||
|
||||
|
||||
enum RET_CODES
|
||||
{
|
||||
OK = 0,
|
||||
BAD_CMDLINE = 1,
|
||||
PARSE_FAILED = 2,
|
||||
};
|
||||
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
#ifdef __AFL_COMPILER
|
||||
__AFL_INIT();
|
||||
#endif
|
||||
|
||||
wxMessageOutput::Set(new wxMessageOutputStderr);
|
||||
wxCmdLineParser cl_parser( argc, argv );
|
||||
cl_parser.SetDesc( g_cmdLineDesc );
|
||||
cl_parser.AddUsageText( _("This program parses PCB files, either from the "
|
||||
"stdin stream or from the given filenames. This can be used either for "
|
||||
"standalone testing of the parser or for fuzz testing." ) );
|
||||
|
||||
int cmd_parsed_ok = cl_parser.Parse();
|
||||
if( cmd_parsed_ok != 0 )
|
||||
{
|
||||
// Help and invalid input both stop here
|
||||
return ( cmd_parsed_ok == -1 ) ? RET_CODES::OK : RET_CODES::BAD_CMDLINE;
|
||||
}
|
||||
|
||||
const bool verbose = cl_parser.Found( "verbose" );
|
||||
|
||||
bool ok = true;
|
||||
PARSE_DURATION duration;
|
||||
|
||||
const auto file_count = cl_parser.GetParamCount();
|
||||
|
||||
if ( file_count == 0 )
|
||||
{
|
||||
// Parse the file provided on stdin - used by AFL to drive the
|
||||
// program
|
||||
// while (__AFL_LOOP(2))
|
||||
{
|
||||
ok = parse( std::cin, verbose );
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Parse 'n' files given on the command line
|
||||
// (this is useful for input minimisation (e.g. afl-tmin) as
|
||||
// well as manual testing
|
||||
for( unsigned i = 0; i < file_count; i++ )
|
||||
{
|
||||
const auto filename = cl_parser.GetParam( i ).ToStdString();
|
||||
|
||||
if( verbose )
|
||||
std::cout << "Parsing: " << filename << std::endl;
|
||||
|
||||
std::ifstream fin;
|
||||
fin.open( filename );
|
||||
|
||||
ok = ok && parse( fin, verbose );
|
||||
}
|
||||
}
|
||||
|
||||
if( !ok )
|
||||
return RET_CODES::PARSE_FAILED;
|
||||
|
||||
return RET_CODES::OK;
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
# This program source code file is part of KiCad, a free EDA CAD application.
|
||||
#
|
||||
# Copyright (C) 2018 KiCad Developers, see CHANGELOG.TXT for contributors.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, you may find one here:
|
||||
# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
|
||||
# or you may search the http://www.gnu.org website for the version 2 license,
|
||||
# or you may write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
|
||||
|
||||
set( QA_UTIL_COMMON_SRC
|
||||
stdstream_line_reader.cpp
|
||||
)
|
||||
|
||||
# A generic library of useful functions for various testing purposes
|
||||
add_library( qa_utils
|
||||
${QA_UTIL_COMMON_SRC}
|
||||
)
|
||||
|
||||
include_directories( BEFORE ${INC_BEFORE} )
|
||||
|
||||
target_link_libraries( qa_utils
|
||||
common
|
||||
${wxWidgets_LIBRARIES}
|
||||
)
|
||||
|
||||
target_include_directories( qa_utils PUBLIC
|
||||
${CMAKE_CURRENT_SOURCE_DIR}
|
||||
)
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
/*
|
||||
* This program source code file is part of KiCad, a free EDA CAD application.
|
||||
*
|
||||
* Copyright (C) 2018 KiCad Developers, see AUTHORS.txt for contributors.
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, you may find one here:
|
||||
* http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
|
||||
* or you may search the http://www.gnu.org website for the version 2 license,
|
||||
* or you may write to the Free Software Foundation, Inc.,
|
||||
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
|
||||
*/
|
||||
|
||||
#ifndef SCOPED_TIMER_H
|
||||
#define SCOPED_TIMER_H
|
||||
|
||||
#include <chrono>
|
||||
|
||||
/**
|
||||
* A simple RAII class to measure the time of an operation.
|
||||
*
|
||||
* ON construction, a timer is started, and on destruction, the timer is
|
||||
* ended, and the time difference is written into the given duration
|
||||
*/
|
||||
template<typename DURATION>
|
||||
class SCOPED_TIMER
|
||||
{
|
||||
using CLOCK = std::chrono::steady_clock;
|
||||
using TIME_PT = std::chrono::time_point<CLOCK>;
|
||||
|
||||
public:
|
||||
SCOPED_TIMER( DURATION& aDuration ):
|
||||
m_duration( aDuration )
|
||||
{
|
||||
m_start = CLOCK::now();
|
||||
}
|
||||
|
||||
~SCOPED_TIMER()
|
||||
{
|
||||
const auto end = CLOCK::now();
|
||||
|
||||
// update the output
|
||||
m_duration = std::chrono::duration_cast<DURATION>( end - m_start );
|
||||
}
|
||||
|
||||
private:
|
||||
|
||||
DURATION& m_duration;
|
||||
TIME_PT m_start;
|
||||
};
|
||||
|
||||
#endif // SCOPED_TIMER_h
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
/*
|
||||
* This program source code file is part of KiCad, a free EDA CAD application.
|
||||
*
|
||||
* Copyright (C) 2017 KiCad Developers, see AUTHORS.txt for contributors.
|
||||
* Copyright (C) 2018 KiCad Developers, see AUTHORS.txt for contributors.
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
|
|
@ -58,10 +58,10 @@ char* STDISTREAM_LINE_READER::ReadLine()
|
|||
}
|
||||
|
||||
|
||||
void STDISTREAM_LINE_READER::setStream( std::istream& aStream )
|
||||
void STDISTREAM_LINE_READER::SetStream( std::istream& aStream )
|
||||
{
|
||||
// Could be done with a virtual getStream function, but the
|
||||
// virtual function call is a noticable (but minor) penalty within
|
||||
// virtual function call is a noticeable (but minor) penalty within
|
||||
// ReadLine() in tight loops
|
||||
m_stream = &aStream;
|
||||
}
|
||||
|
|
@ -77,7 +77,7 @@ IFSTREAM_LINE_READER::IFSTREAM_LINE_READER( const wxFileName& aFileName ) :
|
|||
THROW_IO_ERROR( msg );
|
||||
}
|
||||
|
||||
setStream( m_fStream );
|
||||
SetStream( m_fStream );
|
||||
|
||||
m_source = aFileName.GetFullName();
|
||||
}
|
||||
|
|
@ -44,9 +44,11 @@ public:
|
|||
|
||||
char* ReadLine() override;
|
||||
|
||||
protected:
|
||||
|
||||
void setStream( std::istream& aStream );
|
||||
/**
|
||||
* Set the stream for this line reader.
|
||||
* @param aStream a stream to read
|
||||
*/
|
||||
void SetStream( std::istream& aStream );
|
||||
|
||||
private:
|
||||
std::string m_buffer;
|
||||
|
|
@ -3,7 +3,6 @@ include_directories( BEFORE ${INC_BEFORE} )
|
|||
|
||||
set( IOBENCHMARK_SRCS
|
||||
io_benchmark.cpp
|
||||
stdstream_line_reader.cpp
|
||||
)
|
||||
|
||||
add_executable( io_benchmark
|
||||
|
|
@ -12,6 +11,7 @@ add_executable( io_benchmark
|
|||
|
||||
target_link_libraries( io_benchmark
|
||||
common
|
||||
qa_utils
|
||||
${wxWidgets_LIBRARIES}
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue