QA: Fix utils.images_are_equal + add basic tests for qa utilities

Note: We allow differences of up to 2 pixel thickness due to erosion operation

qa/tests/cli/test_pcb.py

@@ -38,11 +38,11 @@ def test_pcb_export_svg( kitest: KiTestFixture,
                          test_file: str,
                          output_dir: str,
                          layers_to_test: List[str] ):
-    
+
     input_file = kitest.get_data_file_path( test_file )
-    
+
     for layername in layers_to_test:
-        layerNameFixed = "-" + layername.replace( ".", "_" ) 
+        layerNameFixed = "-" + layername.replace( ".", "_" )
         generated_svg_dir = str( kitest.get_output_path( "cli/{}/".format( output_dir ) ) )
         generated_svg_name = Path( input_file ).stem + layerNameFixed + "-generated.svg"
         generated_svg_path = Path( generated_svg_dir + "/" + generated_svg_name )
@@ -70,4 +70,5 @@ def test_pcb_export_svg( kitest: KiTestFixture,
         svg_source_path = str( Path( input_file ).with_suffix( "" ) )
         svg_source_path += layerNameFixed + ".svg"
 
-        assert utils.svgs_are_equivalent( str( generated_svg_path ), svg_source_path, 1200 )
+        # Comparison DPI = 1270 => 1px == 20um. I.e. allowable error of 60 um after eroding
+        assert utils.svgs_are_equivalent( str( generated_svg_path ), svg_source_path, 1270 )

qa/tests/cli/test_utils.py

@@ -0,0 +1,47 @@
+#
+# This program source code file is part of KiCad, a free EDA CAD application.
+#
+# Copyright (C) 2023 Roberto Fernandez Bautista <roberto.fer.bau@gmail.com>
+# Copyright (C) 2023 KiCad Developers
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+#  MA 02110-1301, USA.
+#
+
+import pytest
+import utils
+import os
+
+@pytest.mark.parametrize("image1,image2,expected",
+                          [
+                            ("3px_base.png", "3px_mid_blk.png",         False),
+                            ("3px_base.png", "3px_mid_1px_lines.png",   True), # Allow 1 px margin of error
+                            ("3px_base.png", "3px_mid_transparent.png", False),
+                            ("3px_base.png", "3px_offBy1_blk.png",      False),
+                            ("3px_base.png", "3px_offBy1_wht.png",      False),
+                            ("square_base.png", "square_blk.png",              False),
+                            ("square_base.png", "square_transparent.png",      False),
+                            ("square_base.png", "square_1px_bigger.png",       True), # Allow 1 px margin of error
+                            ("irregular_base.png", "irregular_1px_eroded.png", True), # Allow 1 px margin of error
+                            ("irregular_base.png", "irregular_2px_eroded.png", False), # 2 px error is too much
+                          ]
+                        )
+def test_images_are_equal(image1: str, image2: str, expected: bool):
+    resources_path = os.path.dirname( __file__ )
+    resources_path = os.path.join( resources_path, "resources" )
+    image1 = os.path.join( resources_path, image1 )
+    image2 = os.path.join( resources_path, image2 )
+    assert utils.images_are_equal( image2, image1 ) == expected
+    assert utils.images_are_equal( image1, image2 ) == expected

qa/tests/cli/utils.py

@@ -41,7 +41,7 @@ def run_and_capture( command: list ) -> Tuple[ str, str, int ]:
     )
 
     out,err = proc.communicate()
-    
+
     return out, err, proc.returncode
 
 def textdiff_files( golden_filepath: str, new_filepath: str, skip: int = 0 ) -> bool:
@@ -63,12 +63,14 @@ def textdiff_files( golden_filepath: str, new_filepath: str, skip: int = 0 ) ->
     return diff_text == ""
 
 
-def images_are_equal( image1: str, image2: str ) -> bool:
+def images_are_equal( image1_path: str, image2_path: str, alpha_colour = (50, 100, 50) ) -> bool:
+    # Note: if modifying this function - please add new tests for it in test_utils.py
+
     # Increase limit to ~500MB uncompressed
     Image.MAX_IMAGE_PIXELS=2 * 1024 * 1024 * 1024 // 4 // 3
 
-    image1 = Image.open( image1 )
+    image1 = Image.open( image1_path )
-    image2 = Image.open( image2 )
+    image2 = Image.open( image2_path )
 
     if image1.size != image2.size:
         return False
@@ -76,36 +78,45 @@ def images_are_equal( image1: str, image2: str ) -> bool:
     if image1.mode != image2.mode:
         return False
 
-    sum = np.sum( ( np.asarray ( image1 ).astype( np.float32 ) - np.asarray( image2 ).astype( np.float32 ) ) ** 2.0 )
+    diff = ImageChops.difference( image1, image2 )
+    sum = np.sum( np.asarray( diff ) )
     retval = True
 
     if sum != 0.0:
-        # images are not identical - lets allow 1 pixel error difference (for curved edges)
+        # Images are not identical - lets allow 1 pixel error difference (for curved edges)
+        diff_multi_bands = diff.split()
+        binary_multi_bands = []
 
-        diff = ImageChops.difference( image1, image2 )
+        for band_diff in diff_multi_bands:
-        diffThresholded = diff.point( lambda x: 255 if x > 1 else 0 )
+            thresholded_band = band_diff.point( lambda x: x > 0 and 255 )
+            binary_multi_bands.append( thresholded_band.convert( "1" ) )
 
-        # erode binary image by 1 pixel
+        binary_result = binary_multi_bands[0]
-        diffEroded = diffThresholded.filter(ImageFilter.MinFilter(3))
-        
-        erodedSum = np.sum( np.asarray( diffEroded ).astype( np.float32 ) )
 
-        retval = erodedSum == 0
+        for i in range( 1, len( binary_multi_bands ) ):
+            binary_result = ImageChops.logical_or( binary_result, binary_multi_bands[i] )
+
+        eroded_result = binary_result.copy().filter( ImageFilter.MinFilter( 3 ) ) # erode once (trim 1 pixel)
+
+        eroded_result_sum = np.sum( np.asarray( eroded_result ) )
+        retval = eroded_result_sum == 0
 
         # Save images
-        diff_name = image1.filename + ".diff1.png"
+        #if not retval:
-        diff.save( diff_name )
+        diff_name = image1.filename + ".diff.png"
-        diff_name = image1.filename + ".diffthresholded.png"
+        diff.save( diff_name ) # Note: if the image has alpha, the diff will be mostly transparent
-        diffThresholded.save( diff_name )
-        diffEroded_name = image1.filename + ".diffEroded_erodedsum" + str(erodedSum)+ ".png"
-        diffEroded.save( diffEroded_name )
 
+        diff_name = image1.filename + ".binary_result.png"
+        binary_result.save( diff_name )
+
+        diff_name = image1.filename + ".eroded_result_" + str( eroded_result_sum )+ ".png"
+        eroded_result.save( diff_name )
 
     return retval
 
 
 def svgs_are_equivalent( svg_generated_path : str, svg_source_path : str, comparison_dpi : int ) -> bool:
-    
+
     png_generated_path = Path( svg_generated_path ).with_suffix( ".png" )
 
     # store source png in same folder as generated, easier to compare
@@ -115,7 +126,7 @@ def svgs_are_equivalent( svg_generated_path : str, svg_source_path : str, compar
     cairosvg.svg2png( url=svg_generated_path,
                       write_to=str( png_generated_path ),
                       dpi=comparison_dpi )
-    
+
     cairosvg.svg2png( url=svg_source_path,
                      write_to=str( png_source_path ),
                       dpi=comparison_dpi )