Add tutorial tests

tests/deploy/test_predictor.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os.path as osp
 import tempfile
 import unittest.mock as mock
 
@@ -28,14 +29,18 @@ class TestPredictor(CommonTest):
 
     @staticmethod
     def add_tests(cls):
+        """
+        Automatically patch testing functions to cls.
+        """
+
         def _test_predictor(trainer_name):
             def _test_predictor_impl(self):
                 trainer_class = getattr(self.MODULE, trainer_name)
                 # Construct trainer with default parameters
                 trainer = trainer_class()
                 with tempfile.TemporaryDirectory() as td:
-                    dynamic_model_dir = f"{td}/dynamic"
-                    static_model_dir = f"{td}/static"
+                    dynamic_model_dir = osp.join(td, "dynamic")
+                    static_model_dir = osp.join(td, "static")
                     # HACK: BaseModel.save_model() requires BaseModel().optimizer to be set
                     optimizer = mock.Mock()
                     optimizer.state_dict.return_value = {'foo': 'bar'}
@@ -114,9 +119,9 @@ class TestCDPredictor(TestPredictor):
                               out_single_file_list_t[0])
 
         # Single input (ndarrays)
-        input_ = (cv2.imread(t1_path).astype('float32'),
-                  cv2.imread(t2_path).astype('float32')
-                  )  # Reuse the name `input_`
+        input_ = (
+            cv2.imread(t1_path).astype('float32'),
+            cv2.imread(t2_path).astype('float32'))  # Reuse the name `input_`
         out_single_array_p = predictor.predict(input_, transforms=transforms)
         self.check_dict_equal(out_single_array_p, out_single_file_p)
         out_single_array_t = trainer.predict(input_, transforms=transforms)
@@ -164,7 +169,7 @@ class TestClasPredictor(TestPredictor):
         trainer.labels = labels
         predictor._model.labels = labels
 
-        # Single input (file paths)
+        # Single input (file path)
         input_ = single_input
         out_single_file_p = predictor.predict(input_, transforms=transforms)
         out_single_file_t = trainer.predict(input_, transforms=transforms)
@@ -178,7 +183,7 @@ class TestClasPredictor(TestPredictor):
         self.check_dict_equal(out_single_file_list_p[0],
                               out_single_file_list_t[0])
 
-        # Single input (ndarrays)
+        # Single input (ndarray)
         input_ = cv2.imread(single_input).astype(
             'float32')  # Reuse the name `input_`
         out_single_array_p = predictor.predict(input_, transforms=transforms)
@@ -227,7 +232,7 @@ class TestDetPredictor(TestPredictor):
         trainer.labels = labels
         predictor._model.labels = labels
 
-        # Single input (file paths)
+        # Single input (file path)
         input_ = single_input
         out_single_file_p = predictor.predict(input_, transforms=transforms)
         out_single_file_t = trainer.predict(input_, transforms=transforms)
@@ -241,23 +246,7 @@ class TestDetPredictor(TestPredictor):
         self.check_dict_equal(out_single_file_list_p[0],
                               out_single_file_list_t[0])
 
-        # Single input (ndarrays)
-        input_ = cv2.imread(single_input).astype(
-            'float32')  # Reuse the name `input_`
-        out_single_array_p = predictor.predict(input_, transforms=transforms)
-        self.check_dict_equal(out_single_array_p, out_single_file_p)
-        out_single_array_t = trainer.predict(input_, transforms=transforms)
-        self.check_dict_equal(out_single_array_p, out_single_array_t)
-        out_single_array_list_p = predictor.predict(
-            [input_], transforms=transforms)
-        self.assertEqual(len(out_single_array_list_p), 1)
-        self.check_dict_equal(out_single_array_list_p[0], out_single_array_p)
-        out_single_array_list_t = trainer.predict(
-            [input_], transforms=transforms)
-        self.check_dict_equal(out_single_array_list_p[0],
-                              out_single_array_list_t[0])
-
-        # Single input (ndarrays)
+        # Single input (ndarray)
         input_ = cv2.imread(single_input).astype(
             'float32')  # Reuse the name `input_`
         out_single_array_p = predictor.predict(input_, transforms=transforms)
@@ -303,7 +292,7 @@ class TestSegPredictor(TestPredictor):
         num_inputs = 2
         transforms = pdrs.transforms.Compose([pdrs.transforms.Normalize()])
 
-        # Single input (file paths)
+        # Single input (file path)
         input_ = single_input
         out_single_file_p = predictor.predict(input_, transforms=transforms)
         out_single_file_t = trainer.predict(input_, transforms=transforms)
@@ -317,7 +306,7 @@ class TestSegPredictor(TestPredictor):
         self.check_dict_equal(out_single_file_list_p[0],
                               out_single_file_list_t[0])
 
-        # Single input (ndarrays)
+        # Single input (ndarray)
         input_ = cv2.imread(single_input).astype(
             'float32')  # Reuse the name `input_`
         out_single_array_p = predictor.predict(input_, transforms=transforms)

tests/test_tutorials.py

@@ -10,4 +10,75 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.
+# limitations under the License.
+
+import os.path as osp
+import tempfile
+import shutil
+from glob import iglob
+
+from testing_utils import run_script, CpuCommonTest
+
+
+class TestTutorial(CpuCommonTest):
+    SUBDIR = "./"
+    TIMEOUT = 300
+    PATTERN = "*.py"
+
+    @classmethod
+    def setUpClass(cls):
+        cls._td = tempfile.TemporaryDirectory(dir='./')
+        # Recursively copy the content of `cls.SUBDIR` to td.
+        # This is necessary for running scripts in td.
+        cls._TSUBDIR = osp.join(cls._td.name, osp.basename(cls.SUBDIR))
+        shutil.copytree(cls.SUBDIR, cls._TSUBDIR)
+        return super().setUpClass()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls._td.cleanup()
+
+    @staticmethod
+    def add_tests(cls):
+        """
+        Automatically patch testing functions to cls.
+        """
+
+        def _test_tutorial(script_name):
+            def _test_tutorial_impl(self):
+                # Set working directory to `cls._TSUBDIR` such that the 
+                # files generated by the script will be automatically cleaned.
+                run_script(f"python {script_name}", wd=cls._TSUBDIR)
+
+            return _test_tutorial_impl
+
+        for script_path in iglob(osp.join(cls.SUBDIR, cls.PATTERN)):
+            script_name = osp.basename(script_path)
+            if osp.normpath(osp.join(cls.SUBDIR, script_name)) != osp.normpath(
+                    script_path):
+                raise ValueError(
+                    f"{script_name} should be directly contained in {cls.SUBDIR}"
+                )
+            setattr(cls, 'test_' + script_name, _test_tutorial(script_name))
+
+        return cls
+
+
+@TestTutorial.add_tests
+class TestCDTutorial(TestTutorial):
+    SUBDIR = "../tutorials/train/change_detection"
+
+
+@TestTutorial.add_tests
+class TestClasTutorial(TestTutorial):
+    SUBDIR = "../tutorials/train/classification"
+
+
+@TestTutorial.add_tests
+class TestDetTutorial(TestTutorial):
+    SUBDIR = "../tutorials/train/object_detection"
+
+
+@TestTutorial.add_tests
+class TestSegTutorial(TestTutorial):
+    SUBDIR = "../tutorials/train/semantic_segmentation"

tests/testing_utils.py

@@ -24,9 +24,9 @@ import paddle
 __all__ = ['CommonTest', 'CpuCommonTest', 'run_script']
 
 
-def run_script(cmd, silent=True, wd=None):
+def run_script(cmd, silent=True, wd=None, timeout=None):
     # XXX: This function is not safe!!!
-    cfg = dict(check=True, shell=True)
+    cfg = dict(check=True, shell=True, timeout=timeout)
     if silent:
         cfg['stdout'] = subprocess.DEVNULL
     if wd is not None: