Upgrade supported paddle version

Dockerfile

@@ -1,5 +1,5 @@
 # 0. set args
-ARG PPTAG=2.4.1  # tags refer to https://hub.docker.com/r/paddlepaddle/paddle/tags
+ARG PPTAG=2.5.1  # tags refer to https://hub.docker.com/r/paddlepaddle/paddle/tags
 
 # 1. pull base image
 FROM paddlepaddle/paddle:${PPTAG}

docs/docker_cn.md

@@ -4,16 +4,16 @@
 
 PaddleRS提供`Dockerfile`，可构建基础镜像用于开发或部署。在镜像构建过程中，默认将拉取PaddleRS develop分支内容，并存放在`/opt/PaddleRS`。在构建镜像时可以通过`PPTAG`参数指定要使用的PaddlePaddle版本，例如：
 
-- 安装CPU版本的PaddlePaddle-2.4.1，未指定`PPTAG`的情况下将默认安装此版本：
+- 安装CPU版本的PaddlePaddle-2.5.1，未指定`PPTAG`的情况下将默认安装此版本：
 
 ```shell
 docker build -t paddlers:latest -f Dockerfile .
 ```
 
-- 安装GPU版本PaddlePaddle-2.4.1，使用CUDA 11.7、cuDNN 8.4以及TensorRT 8.4：
+- 安装GPU版本PaddlePaddle-2.5.1，使用CUDA 11.7、cuDNN 8.4以及TensorRT 8.4：
 
 ```shell
-docker build -t paddlers:latest -f Dockerfile . --build-arg PPTAG=2.4.1-gpu-cuda11.7-cudnn8.4-trt8.4
+docker build -t paddlers:latest -f Dockerfile . --build-arg PPTAG=2.5.1-gpu-cuda11.7-cudnn8.4-trt8.4
 ```
 
 其他环境的`PPTAG`可以参考[此处](https://hub.docker.com/r/paddlepaddle/paddle/tags)。请注意，如果需要安装GPU版本的PaddlePaddle，请确保Docker版本>=19。

docs/docker_en.md

@@ -4,16 +4,16 @@
 
 PaddleRS provides `Dockerfile` to build a base Docker image for development/deployment. By default the develop branch of PaddleRS is fetched and stored in `/opts/PaddleRS` during image build. The `PPTAG` argument can be specified to the PaddlePaddle version you want to install. For example,
 
-- To install CPU version of PaddlePaddle-2.4.1 (which is installed when `docker build` does not receive a `PPTAG` argument), run:
+- To install CPU version of PaddlePaddle-2.5.1 (which is installed when `docker build` does not receive a `PPTAG` argument), run:
 
 ```shell
 docker build -t paddlers:latest -f Dockerfile .
 ```
 
-- To install GPU version of PaddlePaddle-2.4.1, with CUDA 11.8, cuDNN 8.4, and TensorRT 8.4, run:
+- To install GPU version of PaddlePaddle-2.5.1, with CUDA 11.8, cuDNN 8.4, and TensorRT 8.4, run:
 
 ```shell
-docker build -t paddlers:latest -f Dockerfile . --build-arg PPTAG=2.4.1-gpu-cuda11.7-cudnn8.4-trt8.4
+docker build -t paddlers:latest -f Dockerfile . --build-arg PPTAG=2.5.1-gpu-cuda11.7-cudnn8.4-trt8.4
 ```
 
 You can find a full list of available PaddlePaddle versions [here](https://hub.docker.com/r/paddlepaddle/paddle/tags). Please note that if a GPU version of PaddlePaddle is to be used, the version of Docker should >=19.

docs/quick_start_cn.md

@@ -5,14 +5,13 @@
 ## 环境准备
 
 1. [安装PaddlePaddle](https://www.paddlepaddle.org.cn/install/quick)
-  - 版本要求：PaddlePaddle>=2.2.0
+  - 版本要求：PaddlePaddle>=2.5.0
 
 2. 安装PaddleRS
 
 如果希望获取更加稳定的体验，请下载安装[PaddleRS发行版](https://github.com/PaddlePaddle/PaddleRS/releases)。
 
 ```shell
-pip install -r requirements.txt
 pip install .
 ```
 
@@ -22,7 +21,6 @@ PaddleRS代码会跟随开发进度不断更新，如果希望使用最新功能
 git clone https://github.com/PaddlePaddle/PaddleRS
 cd PaddleRS
 git checkout develop
-pip install -r requirements.txt
 pip install .
 ```
 

docs/quick_start_en.md

@@ -5,7 +5,7 @@
 ## Prerequisites
 
 1. [Install PaddlePaddle](https://www.paddlepaddle.org.cn/install/quick)
-  - Version requirements: PaddlePaddle>=2.2.0
+  - Version requirements: PaddlePaddle>=2.5.0
 
 2. Install PaddleRS
 

paddlers/datasets/base.py

@@ -15,7 +15,7 @@
 import copy
 
 from paddle.io import Dataset
-from paddle.fluid.dataloader.collate import default_collate_fn
+from paddle.io.dataloader.collate import default_collate_fn
 
 from paddlers.utils import get_num_workers
 import paddlers.utils.logging as logging

paddlers/models/paddleseg/__init__.py

@@ -12,6 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from . import models, datasets, transforms
+from . import models, datasets, transforms, optimizers
 
-__version__ = '2.7.0'
+__version__ = '2.8.0'

paddlers/models/paddleseg/core/infer.py

@@ -95,8 +95,8 @@ def slide_inference(model, im, crop_size, stride):
     w_crop, h_crop = crop_size
     w_stride, h_stride = stride
     # calculate the crop nums
-    rows = np.int(np.ceil(1.0 * (h_im - h_crop) / h_stride)) + 1
-    cols = np.int(np.ceil(1.0 * (w_im - w_crop) / w_stride)) + 1
+    rows = int(np.ceil(1.0 * (h_im - h_crop) / h_stride)) + 1
+    cols = int(np.ceil(1.0 * (w_im - w_crop) / w_stride)) + 1
     # prevent negative sliding rounds when imgs after scaling << crop_size
     rows = 1 if h_im <= h_crop else rows
     cols = 1 if w_im <= w_crop else cols
@@ -208,12 +208,13 @@ def aug_inference(model,
     final_logit = 0
     h_input, w_input = im.shape[-2], im.shape[-1]
     flip_comb = flip_combination(flip_horizontal, flip_vertical)
+    num_augs = len(scales) * len(flip_comb)
     for scale in scales:
         h = int(h_input * scale + 0.5)
         w = int(w_input * scale + 0.5)
-        im = F.interpolate(im, [h, w], mode='bilinear')
+        im_scale = F.interpolate(im, [h, w], mode='bilinear')
         for flip in flip_comb:
-            im_flip = tensor_flip(im, flip)
+            im_flip = tensor_flip(im_scale, flip)
             logit = inference(
                 model,
                 im_flip,
@@ -222,10 +223,11 @@ def aug_inference(model,
                 stride=stride)
             logit = tensor_flip(logit, flip)
             logit = F.interpolate(logit, [h_input, w_input], mode='bilinear')
-
-            logit = F.softmax(logit, axis=1)
-            final_logit = final_logit + logit
-
+            # Accumulate final logits in place
+            final_logit += logit
+    # We average the accumulated logits to make the numeric values of `final_logit`
+    # comparable to single-scale logits
+    final_logit /= num_augs
     final_logit = reverse_transform(final_logit, trans_info, mode='bilinear')
     pred = paddle.argmax(final_logit, axis=1, keepdim=True, dtype='int32')
 

paddlers/models/paddleseg/core/predict.py

@@ -145,3 +145,6 @@ def predict(model,
             pred_mask.save(pred_saved_path)
 
             progbar_pred.update(i + 1)
+
+    logger.info("Predicted images are saved in {} and {} .".format(
+        added_saved_dir, pred_saved_dir))

paddlers/models/paddleseg/core/train.py

@@ -16,12 +16,14 @@ import os
 import time
 from collections import deque
 import shutil
+from copy import deepcopy
 
 import paddle
 import paddle.nn.functional as F
 
 from paddlers.models.paddleseg.utils import (TimeAverager, calculate_eta, resume, logger,
-                             worker_init_fn, train_profiler, op_flops_funs)
+                             worker_init_fn, train_profiler, op_flops_funs,
+                             init_ema_params, update_ema_model)
 from paddlers.models.paddleseg.core.val import evaluate
 
 
@@ -68,6 +70,7 @@ def train(model,
           log_iters=10,
           num_workers=0,
           use_vdl=False,
+          use_ema=False,
           losses=None,
           keep_checkpoint_max=5,
           test_config=None,
@@ -102,6 +105,13 @@ def train(model,
         profiler_options (str, optional): The option of train profiler.
         to_static_training (bool, optional): Whether to use @to_static for training.
     """
+
+    if use_ema:
+        ema_model = deepcopy(model)
+        ema_model.eval()
+        for param in ema_model.parameters():
+            param.stop_gradient = True
+
     model.train()
     nranks = paddle.distributed.ParallelEnv().nranks
     local_rank = paddle.distributed.ParallelEnv().local_rank
@@ -154,14 +164,16 @@ def train(model,
     avg_loss_list = []
     iters_per_epoch = len(batch_sampler)
     best_mean_iou = -1.0
+    best_ema_mean_iou = -1.0
     best_model_iter = -1
     reader_cost_averager = TimeAverager()
     batch_cost_averager = TimeAverager()
     save_models = deque()
     batch_start = time.time()
-
     iter = start_iter
     while iter < iters:
+        if iter == start_iter and use_ema:
+            init_ema_params(ema_model, model)
         for data in loader:
             iter += 1
             if iter > iters:
@@ -176,6 +188,7 @@ def train(model,
             edges = None
             if 'edge' in data.keys():
                 edges = data['edge'].astype('int64')
+
             if hasattr(model, 'data_format') and model.data_format == 'NHWC':
                 images = images.transpose((0, 2, 3, 1))
 
@@ -189,11 +202,19 @@ def train(model,
                         custom_black_list={'bilinear_interp_v2'}):
                     logits_list = ddp_model(images) if nranks > 1 else model(
                         images)
-                    loss_list = loss_computation(
-                        logits_list=logits_list,
-                        labels=labels,
-                        edges=edges,
-                        losses=losses)
+                    if nranks > 1 and hasattr(ddp_model._layers,
+                                              'loss_computation'):
+                        loss_list = ddp_model._layers.loss_computation(
+                            logits_list, losses, data)
+                    elif nranks == 1 and hasattr(model, 'loss_computation'):
+                        loss_list = model.loss_computation(logits_list, losses,
+                                                           data)
+                    else:
+                        loss_list = loss_computation(
+                            logits_list=logits_list,
+                            labels=labels,
+                            edges=edges,
+                            losses=losses)
                     loss = sum(loss_list)
 
                 scaled = scaler.scale(loss)  # scale the loss
@@ -204,18 +225,23 @@ def train(model,
                     scaler.minimize(optimizer, scaled)  # update parameters
             else:
                 logits_list = ddp_model(images) if nranks > 1 else model(images)
-                loss_list = loss_computation(
-                    logits_list=logits_list,
-                    labels=labels,
-                    edges=edges,
-                    losses=losses)
+
+                if nranks > 1 and hasattr(ddp_model._layers,
+                                          'loss_computation'):
+                    loss_list = ddp_model._layers.loss_computation(logits_list,
+                                                                   losses, data)
+                elif nranks == 1 and hasattr(model, 'loss_computation'):
+                    loss_list = model.loss_computation(logits_list, losses,
+                                                       data)
+                else:
+                    loss_list = loss_computation(
+                        logits_list=logits_list,
+                        labels=labels,
+                        edges=edges,
+                        losses=losses)
                 loss = sum(loss_list)
                 loss.backward()
-                # if the optimizer is ReduceOnPlateau, the loss is the one which has been pass into step.
-                if isinstance(optimizer, paddle.optimizer.lr.ReduceOnPlateau):
-                    optimizer.step(loss)
-                else:
-                    optimizer.step()
+                optimizer.step()
 
             lr = optimizer.get_lr()
 
@@ -225,7 +251,10 @@ def train(model,
             else:
                 lr_sche = optimizer._learning_rate
             if isinstance(lr_sche, paddle.optimizer.lr.LRScheduler):
-                lr_sche.step()
+                if isinstance(lr_sche, paddle.optimizer.lr.ReduceOnPlateau):
+                    lr_sche.step(loss)
+                else:
+                    lr_sche.step()
 
             train_profiler.add_profiler_step(profiler_options)
 
@@ -273,6 +302,9 @@ def train(model,
                 reader_cost_averager.reset()
                 batch_cost_averager.reset()
 
+            if use_ema:
+                update_ema_model(ema_model, model, step=iter)
+
             if (iter % save_interval == 0 or
                     iter == iters) and (val_dataset is not None):
                 num_workers = 1 if num_workers > 0 else 0
@@ -288,6 +320,15 @@ def train(model,
                     amp_level=amp_level,
                     **test_config)
 
+                if use_ema:
+                    ema_mean_iou, ema_acc, _, _, _ = evaluate(
+                        ema_model,
+                        val_dataset,
+                        num_workers=num_workers,
+                        precision=precision,
+                        amp_level=amp_level,
+                        **test_config)
+
                 model.train()
 
             if (iter % save_interval == 0 or iter == iters) and local_rank == 0:
@@ -299,6 +340,12 @@ def train(model,
                             os.path.join(current_save_dir, 'model.pdparams'))
                 paddle.save(optimizer.state_dict(),
                             os.path.join(current_save_dir, 'model.pdopt'))
+
+                if use_ema:
+                    paddle.save(
+                        ema_model.state_dict(),
+                        os.path.join(current_save_dir, 'ema_model.pdparams'))
+
                 save_models.append(current_save_dir)
                 if len(save_models) > keep_checkpoint_max > 0:
                     model_to_remove = save_models.popleft()
@@ -315,10 +362,28 @@ def train(model,
                     logger.info(
                         '[EVAL] The model with the best validation mIoU ({:.4f}) was saved at iter {}.'
                         .format(best_mean_iou, best_model_iter))
+                    if use_ema:
+                        if ema_mean_iou > best_ema_mean_iou:
+                            best_ema_mean_iou = ema_mean_iou
+                            best_ema_model_iter = iter
+                            best_ema_model_dir = os.path.join(save_dir,
+                                                              "ema_best_model")
+                            paddle.save(ema_model.state_dict(),
+                                        os.path.join(best_ema_model_dir,
+                                                     'ema_model.pdparams'))
+                        logger.info(
+                            '[EVAL] The EMA model with the best validation mIoU ({:.4f}) was saved at iter {}.'
+                            .format(best_ema_mean_iou, best_ema_model_iter))
 
                     if use_vdl:
                         log_writer.add_scalar('Evaluate/mIoU', mean_iou, iter)
                         log_writer.add_scalar('Evaluate/Acc', acc, iter)
+
+                        if use_ema:
+                            log_writer.add_scalar('Evaluate/Ema_mIoU',
+                                                  ema_mean_iou, iter)
+                            log_writer.add_scalar('Evaluate/Ema_Acc', ema_acc,
+                                                  iter)
             batch_start = time.time()
 
     # Calculate flops.

paddlers/models/paddleseg/cvlibs/__init__.py

@@ -15,3 +15,4 @@
 from . import manager
 from . import param_init
 from .config import Config
+from .builder import Builder, SegBuilder

paddlers/models/paddleseg/cvlibs/builder.py

@@ -0,0 +1,304 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+from typing import Any, Optional
+
+import yaml
+import paddle
+
+from paddlers.models.paddleseg.cvlibs import manager, Config
+from paddlers.models.paddleseg.utils import utils, logger
+from paddlers.models.paddleseg.utils.utils import CachedProperty as cached_property
+
+
+class Builder(object):
+    """
+    The base class for building components. 
+
+    Args:
+        config (Config): A Config class.
+        comp_list (list, optional): A list of component classes. Default: None
+    """
+
+    def __init__(self, config: Config, comp_list: Optional[list]=None):
+        super().__init__()
+        self.config = config
+        self.comp_list = comp_list
+
+    def build_component(self, cfg):
+        """
+        Create Python object, such as model, loss, dataset, etc.
+        """
+        cfg = copy.deepcopy(cfg)
+        if 'type' not in cfg:
+            raise RuntimeError(
+                "It is not possible to create a component object from {}, as 'type' is not specified.".
+                format(cfg))
+
+        class_type = cfg.pop('type')
+        com_class = self.load_component_class(class_type)
+
+        params = {}
+        for key, val in cfg.items():
+            if self.is_meta_type(val):
+                params[key] = self.build_component(val)
+            elif isinstance(val, list):
+                params[key] = [
+                    self.build_component(item)
+                    if self.is_meta_type(item) else item for item in val
+                ]
+            else:
+                params[key] = val
+
+        try:
+            obj = self.build_component_impl(com_class, **params)
+        except Exception as e:
+            if hasattr(com_class, '__name__'):
+                com_name = com_class.__name__
+            else:
+                com_name = ''
+            raise RuntimeError(
+                f"Tried to create a {com_name} object, but the operation has failed. "
+                "Please double check the arguments used to create the object.\n"
+                f"The error message is: \n{str(e)}")
+
+        return obj
+
+    def build_component_impl(self, component_class, *args, **kwargs):
+        return component_class(*args, **kwargs)
+
+    def load_component_class(self, class_type):
+        for com in self.comp_list:
+            if class_type in com.components_dict:
+                return com[class_type]
+        raise RuntimeError("The specified component ({}) was not found.".format(
+            class_type))
+
+    @classmethod
+    def is_meta_type(cls, obj):
+        # TODO: should we define a protocol (see https://peps.python.org/pep-0544/#defining-a-protocol)
+        # to make it more pythonic?
+        return isinstance(obj, dict) and 'type' in obj
+
+    @classmethod
+    def show_msg(cls, name, cfg):
+        msg = 'Use the following config to build {}\n'.format(name)
+        msg += str(yaml.dump({name: cfg}, Dumper=utils.NoAliasDumper))
+        logger.info(msg[0:-1])
+
+
+class SegBuilder(Builder):
+    """
+    This class is responsible for building components for semantic segmentation. 
+    """
+
+    def __init__(self, config, comp_list=None):
+        if comp_list is None:
+            comp_list = [
+                manager.MODELS, manager.BACKBONES, manager.DATASETS,
+                manager.TRANSFORMS, manager.LOSSES, manager.OPTIMIZERS
+            ]
+        super().__init__(config, comp_list)
+
+    @cached_property
+    def model(self) -> paddle.nn.Layer:
+        model_cfg = self.config.model_cfg
+        assert model_cfg != {}, \
+            'No model specified in the configuration file.'
+
+        if self.config.train_dataset_cfg['type'] != 'Dataset':
+            # check and synchronize the num_classes in model config and dataset class
+            assert hasattr(self.train_dataset_class, 'NUM_CLASSES'), \
+                'If train_dataset class is not `Dataset`, it must have `NUM_CLASSES` attr.'
+            num_classes = getattr(self.train_dataset_class, 'NUM_CLASSES')
+            if 'num_classes' in model_cfg:
+                assert model_cfg['num_classes'] == num_classes, \
+                    'The num_classes is not consistent for model config ({}) ' \
+                    'and train_dataset class ({}) '.format(model_cfg['num_classes'], num_classes)
+            else:
+                logger.warning(
+                    'Add the `num_classes` in train_dataset class to '
+                    'model config. We suggest you manually set `num_classes` in model config.'
+                )
+                model_cfg['num_classes'] = num_classes
+            # check and synchronize the in_channels in model config and dataset class
+            assert hasattr(self.train_dataset_class, 'IMG_CHANNELS'), \
+                'If train_dataset class is not `Dataset`, it must have `IMG_CHANNELS` attr.'
+            in_channels = getattr(self.train_dataset_class, 'IMG_CHANNELS')
+            x = utils.get_in_channels(model_cfg)
+            if x is not None:
+                assert x == in_channels, \
+                    'The in_channels in model config ({}) and the img_channels in train_dataset ' \
+                    'class ({}) is not consistent'.format(x, in_channels)
+            else:
+                model_cfg = utils.set_in_channels(model_cfg, in_channels)
+                logger.warning(
+                    'Add the `in_channels` in train_dataset class to '
+                    'model config. We suggest you manually set `in_channels` in model config.'
+                )
+
+        self.show_msg('model', model_cfg)
+        return self.build_component(model_cfg)
+
+    @cached_property
+    def optimizer(self) -> paddle.optimizer.Optimizer:
+        opt_cfg = self.config.optimizer_cfg
+        assert opt_cfg != {}, \
+            'No optimizer specified in the configuration file.'
+        # For compatibility
+        if opt_cfg['type'] == 'adam':
+            opt_cfg['type'] = 'Adam'
+        if opt_cfg['type'] == 'sgd':
+            opt_cfg['type'] = 'SGD'
+        if opt_cfg['type'] == 'SGD' and 'momentum' in opt_cfg:
+            opt_cfg['type'] = 'Momentum'
+            logger.info('If the type is SGD and momentum in optimizer config, '
+                        'the type is changed to Momentum.')
+        self.show_msg('optimizer', opt_cfg)
+        opt = self.build_component(opt_cfg)
+        opt = opt(self.model, self.lr_scheduler)
+        return opt
+
+    @cached_property
+    def lr_scheduler(self) -> paddle.optimizer.lr.LRScheduler:
+        lr_cfg = self.config.lr_scheduler_cfg
+        assert lr_cfg != {}, \
+            'No lr_scheduler specified in the configuration file.'
+
+        use_warmup = False
+        if 'warmup_iters' in lr_cfg:
+            use_warmup = True
+            warmup_iters = lr_cfg.pop('warmup_iters')
+            assert 'warmup_start_lr' in lr_cfg, \
+                "When use warmup, please set warmup_start_lr and warmup_iters in lr_scheduler"
+            warmup_start_lr = lr_cfg.pop('warmup_start_lr')
+            end_lr = lr_cfg['learning_rate']
+
+        lr_type = lr_cfg.pop('type')
+        if lr_type == 'PolynomialDecay':
+            iters = self.config.iters - warmup_iters if use_warmup else self.config.iters
+            iters = max(iters, 1)
+            lr_cfg.setdefault('decay_steps', iters)
+
+        try:
+            lr_sche = getattr(paddle.optimizer.lr, lr_type)(**lr_cfg)
+        except Exception as e:
+            raise RuntimeError(
+                "Create {} has failed. Please check lr_scheduler in config. "
+                "The error message: {}".format(lr_type, e))
+
+        if use_warmup:
+            lr_sche = paddle.optimizer.lr.LinearWarmup(
+                learning_rate=lr_sche,
+                warmup_steps=warmup_iters,
+                start_lr=warmup_start_lr,
+                end_lr=end_lr)
+
+        return lr_sche
+
+    @cached_property
+    def loss(self) -> dict:
+        loss_cfg = self.config.loss_cfg
+        assert loss_cfg != {}, \
+            'No loss specified in the configuration file.'
+        return self._build_loss('loss', loss_cfg)
+
+    @cached_property
+    def distill_loss(self) -> dict:
+        loss_cfg = self.config.distill_loss_cfg
+        assert loss_cfg != {}, \
+            'No distill_loss specified in the configuration file.'
+        return self._build_loss('distill_loss', loss_cfg)
+
+    def _build_loss(self, loss_name, loss_cfg: dict):
+        def _check_helper(loss_cfg, ignore_index):
+            if 'ignore_index' not in loss_cfg:
+                loss_cfg['ignore_index'] = ignore_index
+                logger.warning('Add the `ignore_index` in train_dataset ' \
+                    'class to {} config. We suggest you manually set ' \
+                    '`ignore_index` in {} config.'.format(loss_name, loss_name)
+                )
+            else:
+                assert loss_cfg['ignore_index'] == ignore_index, \
+                    'the ignore_index in loss and train_dataset must be the same. Currently, loss ignore_index = {}, '\
+                    'train_dataset ignore_index = {}'.format(loss_cfg['ignore_index'], ignore_index)
+
+        # check and synchronize the ignore_index in model config and dataset class
+        if self.config.train_dataset_cfg['type'] != 'Dataset':
+            assert hasattr(self.train_dataset_class, 'IGNORE_INDEX'), \
+                'If train_dataset class is not `Dataset`, it must have `IGNORE_INDEX` attr.'
+            ignore_index = getattr(self.train_dataset_class, 'IGNORE_INDEX')
+            for loss_cfg_i in loss_cfg['types']:
+                if loss_cfg_i['type'] == 'MixedLoss':
+                    for loss_cfg_j in loss_cfg_i['losses']:
+                        _check_helper(loss_cfg_j, ignore_index)
+                else:
+                    _check_helper(loss_cfg_i, ignore_index)
+
+        self.show_msg(loss_name, loss_cfg)
+        loss_dict = {'coef': loss_cfg['coef'], "types": []}
+        for item in loss_cfg['types']:
+            loss_dict['types'].append(self.build_component(item))
+        return loss_dict
+
+    @cached_property
+    def train_dataset(self) -> paddle.io.Dataset:
+        dataset_cfg = self.config.train_dataset_cfg
+        assert dataset_cfg != {}, \
+            'No train_dataset specified in the configuration file.'
+        self.show_msg('train_dataset', dataset_cfg)
+        dataset = self.build_component(dataset_cfg)
+        assert len(dataset) != 0, \
+            'The number of samples in train_dataset is 0. Please check whether the dataset is valid.'
+        return dataset
+
+    @cached_property
+    def val_dataset(self) -> paddle.io.Dataset:
+        dataset_cfg = self.config.val_dataset_cfg
+        assert dataset_cfg != {}, \
+            'No val_dataset specified in the configuration file.'
+        self.show_msg('val_dataset', dataset_cfg)
+        dataset = self.build_component(dataset_cfg)
+        if len(dataset) == 0:
+            logger.warning(
+                'The number of samples in val_dataset is 0. Please ensure this is the desired behavior.'
+            )
+        return dataset
+
+    @cached_property
+    def train_dataset_class(self) -> Any:
+        dataset_cfg = self.config.train_dataset_cfg
+        assert dataset_cfg != {}, \
+            'No train_dataset specified in the configuration file.'
+        dataset_type = dataset_cfg.get('type')
+        return self.load_component_class(dataset_type)
+
+    @cached_property
+    def val_dataset_class(self) -> Any:
+        dataset_cfg = self.config.val_dataset_cfg
+        assert dataset_cfg != {}, \
+            'No val_dataset specified in the configuration file.'
+        dataset_type = dataset_cfg.get('type')
+        return self.load_component_class(dataset_type)
+
+    @cached_property
+    def val_transforms(self) -> list:
+        dataset_cfg = self.config.val_dataset_cfg
+        assert dataset_cfg != {}, \
+            'No val_dataset specified in the configuration file.'
+        transforms = []
+        for item in dataset_cfg.get('transforms', []):
+            transforms.append(self.build_component(item))
+        return transforms

paddlers/models/paddleseg/cvlibs/callbacks.py

@@ -1,279 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import time
-
-import numpy as np
-import paddle
-from paddle.distributed.parallel import ParallelEnv
-from visualdl import LogWriter
-from paddlers.models.paddleseg.utils.progbar import Progbar
-import paddlers.models.paddleseg.utils.logger as logger
-
-
-class CallbackList(object):
-    """
-    Container abstracting a list of callbacks.
-
-    Args:
-        callbacks (list[Callback]): List of `Callback` instances.
-    """
-
-    def __init__(self, callbacks=None):
-        callbacks = callbacks or []
-        self.callbacks = [c for c in callbacks]
-
-    def append(self, callback):
-        self.callbacks.append(callback)
-
-    def set_params(self, params):
-        for callback in self.callbacks:
-            callback.set_params(params)
-
-    def set_model(self, model):
-        for callback in self.callbacks:
-            callback.set_model(model)
-
-    def set_optimizer(self, optimizer):
-        for callback in self.callbacks:
-            callback.set_optimizer(optimizer)
-
-    def on_iter_begin(self, iter, logs=None):
-        """Called right before processing a batch.
-        """
-        logs = logs or {}
-        for callback in self.callbacks:
-            callback.on_iter_begin(iter, logs)
-        self._t_enter_iter = time.time()
-
-    def on_iter_end(self, iter, logs=None):
-        """Called at the end of a batch.
-        """
-        logs = logs or {}
-        for callback in self.callbacks:
-            callback.on_iter_end(iter, logs)
-        self._t_exit_iter = time.time()
-
-    def on_train_begin(self, logs=None):
-        """Called at the beginning of training.
-        """
-        logs = logs or {}
-        for callback in self.callbacks:
-            callback.on_train_begin(logs)
-
-    def on_train_end(self, logs=None):
-        """Called at the end of training.
-        """
-        logs = logs or {}
-        for callback in self.callbacks:
-            callback.on_train_end(logs)
-
-    def __iter__(self):
-        return iter(self.callbacks)
-
-
-class Callback(object):
-    """Abstract base class used to build new callbacks.
-    """
-
-    def __init__(self):
-        self.validation_data = None
-
-    def set_params(self, params):
-        self.params = params
-
-    def set_model(self, model):
-        self.model = model
-
-    def set_optimizer(self, optimizer):
-        self.optimizer = optimizer
-
-    def on_iter_begin(self, iter, logs=None):
-        pass
-
-    def on_iter_end(self, iter, logs=None):
-        pass
-
-    def on_train_begin(self, logs=None):
-        pass
-
-    def on_train_end(self, logs=None):
-        pass
-
-
-class BaseLogger(Callback):
-    def __init__(self, period=10):
-        super(BaseLogger, self).__init__()
-        self.period = period
-
-    def _reset(self):
-        self.totals = {}
-
-    def on_train_begin(self, logs=None):
-        self.totals = {}
-
-    def on_iter_end(self, iter, logs=None):
-        logs = logs or {}
-        #(iter - 1) // iters_per_epoch + 1
-        for k, v in logs.items():
-            if k in self.totals.keys():
-                self.totals[k] += v
-            else:
-                self.totals[k] = v
-
-        if iter % self.period == 0 and ParallelEnv().local_rank == 0:
-
-            for k in self.totals:
-                logs[k] = self.totals[k] / self.period
-            self._reset()
-
-
-class TrainLogger(Callback):
-    def __init__(self, log_freq=10):
-        self.log_freq = log_freq
-
-    def _calculate_eta(self, remaining_iters, speed):
-        if remaining_iters < 0:
-            remaining_iters = 0
-        remaining_time = int(remaining_iters * speed)
-        result = "{:0>2}:{:0>2}:{:0>2}"
-        arr = []
-        for i in range(2, -1, -1):
-            arr.append(int(remaining_time / 60**i))
-            remaining_time %= 60**i
-        return result.format(*arr)
-
-    def on_iter_end(self, iter, logs=None):
-
-        if iter % self.log_freq == 0 and ParallelEnv().local_rank == 0:
-            total_iters = self.params["total_iters"]
-            iters_per_epoch = self.params["iters_per_epoch"]
-            remaining_iters = total_iters - iter
-            eta = self._calculate_eta(remaining_iters, logs["batch_cost"])
-            current_epoch = (iter - 1) // self.params["iters_per_epoch"] + 1
-            loss = logs["loss"]
-            lr = self.optimizer.get_lr()
-            batch_cost = logs["batch_cost"]
-            reader_cost = logs["reader_cost"]
-
-            logger.info(
-                "[TRAIN] epoch={}, iter={}/{}, loss={:.4f}, lr={:.6f}, batch_cost={:.4f}, reader_cost={:.4f} | ETA {}"
-                .format(current_epoch, iter, total_iters, loss, lr, batch_cost,
-                        reader_cost, eta))
-
-
-class ProgbarLogger(Callback):
-    def __init__(self):
-        super(ProgbarLogger, self).__init__()
-
-    def on_train_begin(self, logs=None):
-        self.verbose = self.params["verbose"]
-        self.total_iters = self.params["total_iters"]
-        self.target = self.params["total_iters"]
-        self.progbar = Progbar(target=self.target, verbose=self.verbose)
-        self.seen = 0
-        self.log_values = []
-
-    def on_iter_begin(self, iter, logs=None):
-        #self.seen = 0
-        if self.seen < self.target:
-            self.log_values = []
-
-    def on_iter_end(self, iter, logs=None):
-        logs = logs or {}
-        self.seen += 1
-        for k in self.params['metrics']:
-            if k in logs:
-                self.log_values.append((k, logs[k]))
-
-        #if self.verbose and self.seen < self.target and ParallelEnv.local_rank == 0:
-        #print(self.log_values)
-        if self.seen < self.target:
-            self.progbar.update(self.seen, self.log_values)
-
-
-class ModelCheckpoint(Callback):
-    def __init__(self,
-                 save_dir,
-                 monitor="miou",
-                 save_best_only=False,
-                 save_params_only=True,
-                 mode="max",
-                 period=1):
-
-        super(ModelCheckpoint, self).__init__()
-        self.monitor = monitor
-        self.save_dir = save_dir
-        self.save_best_only = save_best_only
-        self.save_params_only = save_params_only
-        self.period = period
-        self.iters_since_last_save = 0
-
-        if mode == "min":
-            self.monitor_op = np.less
-            self.best = np.Inf
-        elif mode == "max":
-            self.monitor_op = np.greater
-            self.best = -np.Inf
-        else:
-            raise RuntimeError("`mode` is neither \"min\" nor \"max\"!")
-
-    def on_train_begin(self, logs=None):
-        self.verbose = self.params["verbose"]
-        save_dir = self.save_dir
-        if not os.path.isdir(save_dir):
-            if os.path.exists(save_dir):
-                os.remove(save_dir)
-            os.makedirs(save_dir)
-
-    def on_iter_end(self, iter, logs=None):
-        logs = logs or {}
-        self.iters_since_last_save += 1
-        current_save_dir = os.path.join(self.save_dir, "iter_{}".format(iter))
-        current_save_dir = os.path.abspath(current_save_dir)
-        #if self.iters_since_last_save % self.period and ParallelEnv().local_rank == 0:
-        #self.iters_since_last_save = 0
-        if iter % self.period == 0 and ParallelEnv().local_rank == 0:
-            if self.verbose > 0:
-                print("iter {iter_num}: saving model to {path}".format(
-                    iter_num=iter, path=current_save_dir))
-
-            paddle.save(self.model.state_dict(),
-                        os.path.join(current_save_dir, 'model.pdparams'))
-
-            if not self.save_params_only:
-                paddle.save(self.optimizer.state_dict(),
-                            os.path.join(current_save_dir, 'model.pdopt'))
-
-
-class VisualDL(Callback):
-    def __init__(self, log_dir="./log", freq=1):
-        super(VisualDL, self).__init__()
-        self.log_dir = log_dir
-        self.freq = freq
-
-    def on_train_begin(self, logs=None):
-        self.writer = LogWriter(self.log_dir)
-
-    def on_iter_end(self, iter, logs=None):
-        logs = logs or {}
-        if iter % self.freq == 0 and ParallelEnv().local_rank == 0:
-            for k, v in logs.items():
-                self.writer.add_scalar("Train/{}".format(k), v, iter)
-
-        self.writer.flush()
-
-    def on_train_end(self, logs=None):
-        self.writer.close()

paddlers/models/paddleseg/cvlibs/config.py

@@ -12,23 +12,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import six
 import codecs
 import os
-from typing import Any, Dict, Generic
-import warnings
 from ast import literal_eval
+from typing import Any, Dict, Optional
 
-import paddle
 import yaml
-import six
+import paddle
 
+from paddlers.models.paddleseg.cvlibs import config_checker as checker
 from paddlers.models.paddleseg.cvlibs import manager
-from paddlers.models.paddleseg.utils import logger
+from paddlers.models.paddleseg.utils import logger, utils
+
+_INHERIT_KEY = '_inherited_'
+_BASE_KEY = '_base_'
 
 
 class Config(object):
-    '''
-    Training configuration parsing. The only yaml/yml file is supported.
+    """
+    Configuration parsing.
 
     The following hyper-parameters are available in the config file:
         batch_size: The number of samples per gpu.
@@ -37,13 +40,12 @@ class Config(object):
             For data type, please refer to paddleseg.datasets.
             For specific transforms, please refer to paddleseg.transforms.transforms.
         val_dataset: A validation data config including type/data_root/transforms/mode.
-        optimizer: A optimizer config, but currently PaddleSeg only supports sgd with momentum in config file.
-            In addition, weight_decay could be set as a regularization.
-        learning_rate: A learning rate config. If decay is configured, learning _rate value is the starting learning rate,
-             where only poly decay is supported using the config file. In addition, decay power and end_lr are tuned experimentally.
-        loss: A loss config. Multi-loss config is available. The loss type order is consistent with the seg model outputs,
-            where the coef term indicates the weight of corresponding loss. Note that the number of coef must be the same as the number of
-            model outputs, and there could be only one loss type if using the same loss type among the outputs, otherwise the number of
+        optimizer: A optimizer config. Please refer to paddleseg.optimizers.
+        loss: A loss config. Multi-loss config is available. The loss type order is 
+            consistent with the seg model outputs, where the coef term indicates the 
+            weight of corresponding loss. Note that the number of coef must be the 
+            same as the number of model outputs, and there could be only one loss type 
+            if using the same loss type among the outputs, otherwise the number of
             loss type must be consistent with coef.
         model: A model config including type/backbone and model-dependent arguments.
             For model type, please refer to paddleseg.models.
@@ -51,500 +53,181 @@ class Config(object):
 
     Args:
         path (str) : The path of config file, supports yaml format only.
-
-    Examples:
-
-        from paddlers.models.paddleseg.cvlibs.config import Config
-
-        # Create a cfg object with yaml file path.
-        cfg = Config(yaml_cfg_path)
-
-        # Parsing the argument when its property is used.
-        train_dataset = cfg.train_dataset
-
-        # the argument of model should be parsed after dataset,
-        # since the model builder uses some properties in dataset.
-        model = cfg.model
-        ...
-    '''
-
-    def __init__(self,
-                 path: str,
-                 learning_rate: float=None,
-                 batch_size: int=None,
-                 iters: int=None,
-                 opts: list=None):
-        if not path:
-            raise ValueError('Please specify the configuration file path.')
-
-        if not os.path.exists(path):
-            raise FileNotFoundError('File {} does not exist'.format(path))
-
-        self._model = None
-        self._losses = None
-        if path.endswith('yml') or path.endswith('yaml'):
-            self.dic = self._parse_from_yaml(path)
-        else:
-            raise RuntimeError('Config file should in yaml format!')
-
-        self.update(
+        opts (list, optional): Use opts to update the key-value pairs of all options.
+
+    """
+
+    def __init__(
+            self,
+            path: str,
+            learning_rate: Optional[float]=None,
+            batch_size: Optional[int]=None,
+            iters: Optional[int]=None,
+            opts: Optional[list]=None,
+            checker: Optional[checker.ConfigChecker]=None, ):
+        assert os.path.exists(path), \
+            'Config path ({}) does not exist'.format(path)
+        assert path.endswith('yml') or path.endswith('yaml'), \
+            'Config file ({}) should be yaml format'.format(path)
+
+        self.dic = self._parse_from_yaml(path)
+        self.dic = self.update_config_dict(
+            self.dic,
             learning_rate=learning_rate,
             batch_size=batch_size,
             iters=iters,
             opts=opts)
 
-        model_cfg = self.dic.get('model', None)
-        if model_cfg is None:
-            raise RuntimeError('No model specified in the configuration file.')
-        if (not self.train_dataset_config) and (not self.val_dataset_config):
-            raise ValueError(
-                'One of `train_dataset` or `val_dataset should be given, but there are none.'
-            )
-
-    def _update_dic(self, dic, base_dic):
-        """
-        Update config from dic based base_dic
-        """
-        base_dic = base_dic.copy()
-        dic = dic.copy()
-
-        if dic.get('_inherited_', True) == False:
-            dic.pop('_inherited_')
-            return dic
-
-        for key, val in dic.items():
-            if isinstance(val, dict) and key in base_dic:
-                base_dic[key] = self._update_dic(val, base_dic[key])
-            else:
-                base_dic[key] = val
-        dic = base_dic
-        return dic
-
-    def _parse_from_yaml(self, path: str):
-        '''Parse a yaml file and build config'''
-        with codecs.open(path, 'r', 'utf-8') as file:
-            dic = yaml.load(file, Loader=yaml.FullLoader)
-
-        if '_base_' in dic:
-            cfg_dir = os.path.dirname(path)
-            base_path = dic.pop('_base_')
-            base_path = os.path.join(cfg_dir, base_path)
-            base_dic = self._parse_from_yaml(base_path)
-            dic = self._update_dic(dic, base_dic)
-        return dic
-
-    def update(self,
-               learning_rate: float=None,
-               batch_size: int=None,
-               iters: int=None,
-               opts: list=None):
-        '''Update config'''
-        if learning_rate:
-            if 'lr_scheduler' in self.dic:
-                self.dic['lr_scheduler']['learning_rate'] = learning_rate
-            else:
-                self.dic['learning_rate']['value'] = learning_rate
-
-        if batch_size:
-            self.dic['batch_size'] = batch_size
-
-        if iters:
-            self.dic['iters'] = iters
-
-        # fix parameters by --opts of command
-        if opts is not None:
-            if len(opts) % 2 != 0 or len(opts) == 0:
-                raise ValueError(
-                    "Command line options config `--opts` format error! It should be even length like: k1 v1 k2 v2 ... Please check it: {}".
-                    format(opts))
-            for key, value in zip(opts[0::2], opts[1::2]):
-                if isinstance(value, six.string_types):
-                    try:
-                        value = literal_eval(value)
-                    except ValueError:
-                        pass
-                    except SyntaxError:
-                        pass
-                key_list = key.split('.')
-                dic = self.dic
-                for subkey in key_list[:-1]:
-                    dic.setdefault(subkey, dict())
-                    dic = dic[subkey]
-                dic[key_list[-1]] = value
+        if checker is None:
+            checker = self._build_default_checker()
+        checker.apply_all_rules(self)
 
     @property
     def batch_size(self) -> int:
-        return self.dic.get('batch_size', 1)
+        return self.dic.get('batch_size')
 
     @property
     def iters(self) -> int:
-        iters = self.dic.get('iters')
-        if not iters:
-            raise RuntimeError('No iters specified in the configuration file.')
-        return iters
-
-    @property
-    def lr_scheduler(self) -> paddle.optimizer.lr.LRScheduler:
-        if 'lr_scheduler' not in self.dic:
-            raise RuntimeError(
-                'No `lr_scheduler` specified in the configuration file.')
-        params = self.dic.get('lr_scheduler')
-
-        use_warmup = False
-        if 'warmup_iters' in params:
-            use_warmup = True
-            warmup_iters = params.pop('warmup_iters')
-            assert 'warmup_start_lr' in params, \
-                "When use warmup, please set warmup_start_lr and warmup_iters in lr_scheduler"
-            warmup_start_lr = params.pop('warmup_start_lr')
-            end_lr = params['learning_rate']
-
-        lr_type = params.pop('type')
-        if lr_type == 'PolynomialDecay':
-            iters = self.iters - warmup_iters if use_warmup else self.iters
-            iters = max(iters, 1)
-            params.setdefault('decay_steps', iters)
-            params.setdefault('end_lr', 0)
-            params.setdefault('power', 0.9)
-        lr_sche = getattr(paddle.optimizer.lr, lr_type)(**params)
-
-        if use_warmup:
-            lr_sche = paddle.optimizer.lr.LinearWarmup(
-                learning_rate=lr_sche,
-                warmup_steps=warmup_iters,
-                start_lr=warmup_start_lr,
-                end_lr=end_lr)
-
-        return lr_sche
-
-    @property
-    def learning_rate(self) -> paddle.optimizer.lr.LRScheduler:
-        logger.warning(
-            '''`learning_rate` in configuration file will be deprecated, please use `lr_scheduler` instead. E.g
-            lr_scheduler:
-                type: PolynomialDecay
-                learning_rate: 0.01''')
-
-        _learning_rate = self.dic.get('learning_rate', {})
-        if isinstance(_learning_rate, float):
-            return _learning_rate
-
-        _learning_rate = self.dic.get('learning_rate', {}).get('value')
-        if not _learning_rate:
-            raise RuntimeError(
-                'No learning rate specified in the configuration file.')
-
-        args = self.decay_args
-        decay_type = args.pop('type')
-
-        if decay_type == 'poly':
-            lr = _learning_rate
-            return paddle.optimizer.lr.PolynomialDecay(lr, **args)
-        elif decay_type == 'piecewise':
-            values = _learning_rate
-            return paddle.optimizer.lr.PiecewiseDecay(values=values, **args)
-        elif decay_type == 'stepdecay':
-            lr = _learning_rate
-            return paddle.optimizer.lr.StepDecay(lr, **args)
-        else:
-            raise RuntimeError('Only poly and piecewise decay support.')
+        return self.dic.get('iters')
 
     @property
-    def optimizer(self) -> paddle.optimizer.Optimizer:
-        if 'lr_scheduler' in self.dic:
-            lr = self.lr_scheduler
-        else:
-            lr = self.learning_rate
-        args = self.optimizer_args
-        optimizer_type = args.pop('type')
-
-        params = self.model.parameters()
-        if 'backbone_lr_mult' in args:
-            if not hasattr(self.model, 'backbone'):
-                logger.warning('The backbone_lr_mult is not effective because'
-                               ' the model does not have backbone')
-            else:
-                backbone_lr_mult = args.pop('backbone_lr_mult')
-                backbone_params = self.model.backbone.parameters()
-                backbone_params_id = [id(x) for x in backbone_params]
-                other_params = [
-                    x for x in params if id(x) not in backbone_params_id
-                ]
-                params = [{
-                    'params': backbone_params,
-                    'learning_rate': backbone_lr_mult
-                }, {
-                    'params': other_params
-                }]
-
-        if optimizer_type == 'sgd':
-            return paddle.optimizer.Momentum(lr, parameters=params, **args)
-        elif optimizer_type == 'adam':
-            return paddle.optimizer.Adam(lr, parameters=params, **args)
-        elif optimizer_type in paddle.optimizer.__all__:
-            return getattr(paddle.optimizer, optimizer_type)(lr,
-                                                             parameters=params,
-                                                             **args)
-
-        raise RuntimeError('Unknown optimizer type {}.'.format(optimizer_type))
+    def to_static_training(self) -> bool:
+        return self.dic.get('to_static_training', False)
 
     @property
-    def optimizer_args(self) -> dict:
-        args = self.dic.get('optimizer', {}).copy()
-        if args['type'] == 'sgd':
-            args.setdefault('momentum', 0.9)
-
-        return args
+    def model_cfg(self) -> Dict:
+        return self.dic.get('model', {}).copy()
 
     @property
-    def decay_args(self) -> dict:
-        args = self.dic.get('learning_rate', {}).get(
-            'decay', {'type': 'poly',
-                      'power': 0.9}).copy()
-
-        if args['type'] == 'poly':
-            args.setdefault('decay_steps', self.iters)
-            args.setdefault('end_lr', 0)
-
-        return args
+    def loss_cfg(self) -> Dict:
+        return self.dic.get('loss', {}).copy()
 
     @property
-    def loss(self) -> dict:
-        if self._losses is None:
-            self._losses = self._prepare_loss('loss')
-        return self._losses
+    def distill_loss_cfg(self) -> Dict:
+        return self.dic.get('distill_loss', {}).copy()
 
     @property
-    def distill_loss(self) -> dict:
-        if not hasattr(self, '_distill_losses'):
-            self._distill_losses = self._prepare_loss('distill_loss')
-        return self._distill_losses
-
-    def _prepare_loss(self, loss_name):
-        """
-        Parse the loss parameters and load the loss layers.
-
-        Args:
-            loss_name (str): The root name of loss in the yaml file.
-        Returns:
-            dict: A dict including the loss parameters and layers.
-        """
-        args = self.dic.get(loss_name, {}).copy()
-        if 'types' in args and 'coef' in args:
-            len_types = len(args['types'])
-            len_coef = len(args['coef'])
-            if len_types != len_coef:
-                if len_types == 1:
-                    args['types'] = args['types'] * len_coef
-                else:
-                    raise ValueError(
-                        'The length of types should equal to coef or equal to 1 in loss config, but they are {} and {}.'
-                        .format(len_types, len_coef))
-        else:
-            raise ValueError(
-                'Loss config should contain keys of "types" and "coef"')
-
-        losses = dict()
-        for key, val in args.items():
-            if key == 'types':
-                losses['types'] = []
-                for item in args['types']:
-                    if item['type'] != 'MixedLoss':
-                        if 'ignore_index' in item:
-                            assert item['ignore_index'] == self.train_dataset.ignore_index, 'If ignore_index of loss is set, '\
-                            'the ignore_index of loss and train_dataset must be the same. \nCurrently, loss ignore_index = {}, '\
-                            'train_dataset ignore_index = {}. \nIt is recommended not to set loss ignore_index, so it is consistent with '\
-                            'train_dataset by default.'.format(item['ignore_index'], self.train_dataset.ignore_index)
-                        item['ignore_index'] = \
-                            self.train_dataset.ignore_index
-                    losses['types'].append(self._load_object(item))
-            else:
-                losses[key] = val
-        if len(losses['coef']) != len(losses['types']):
-            raise RuntimeError(
-                'The length of coef should equal to types in loss config: {} != {}.'
-                .format(len(losses['coef']), len(losses['types'])))
-        return losses
+    def lr_scheduler_cfg(self) -> Dict:
+        return self.dic.get('lr_scheduler', {}).copy()
 
     @property
-    def model(self) -> paddle.nn.Layer:
-        model_cfg = self.dic.get('model').copy()
-        if not self._model:
-            self._model = self._load_object(model_cfg)
-        return self._model
+    def optimizer_cfg(self) -> Dict:
+        return self.dic.get('optimizer', {}).copy()
 
     @property
-    def train_dataset_config(self) -> Dict:
+    def train_dataset_cfg(self) -> Dict:
         return self.dic.get('train_dataset', {}).copy()
 
     @property
-    def val_dataset_config(self) -> Dict:
+    def val_dataset_cfg(self) -> Dict:
         return self.dic.get('val_dataset', {}).copy()
 
+    # TODO merge test_config into val_dataset
     @property
-    def train_dataset_class(self) -> Generic:
-        dataset_type = self.train_dataset_config['type']
-        return self._load_component(dataset_type)
+    def test_config(self) -> Dict:
+        return self.dic.get('test_config', {}).copy()
+
+    @classmethod
+    def update_config_dict(cls, dic: dict, *args, **kwargs) -> dict:
+        return update_config_dict(dic, *args, **kwargs)
+
+    @classmethod
+    def _parse_from_yaml(cls, path: str, *args, **kwargs) -> dict:
+        return parse_from_yaml(path, *args, **kwargs)
+
+    @classmethod
+    def _build_default_checker(cls):
+        rules = []
+        rules.append(checker.DefaultPrimaryRule())
+        rules.append(checker.DefaultSyncNumClassesRule())
+        rules.append(checker.DefaultSyncImgChannelsRule())
+        # Losses
+        rules.append(checker.DefaultLossRule('loss'))
+        rules.append(checker.DefaultSyncIgnoreIndexRule('loss'))
+        # Distillation losses
+        rules.append(checker.DefaultLossRule('distill_loss'))
+        rules.append(checker.DefaultSyncIgnoreIndexRule('distill_loss'))
+
+        return checker.ConfigChecker(rules, allow_update=True)
 
-    @property
-    def val_dataset_class(self) -> Generic:
-        dataset_type = self.val_dataset_config['type']
-        return self._load_component(dataset_type)
+    def __str__(self) -> str:
+        # Use NoAliasDumper to avoid yml anchor 
+        return yaml.dump(self.dic, Dumper=utils.NoAliasDumper)
 
-    @property
-    def train_dataset(self) -> paddle.io.Dataset:
-        _train_dataset = self.train_dataset_config
-        if not _train_dataset:
-            return None
-        return self._load_object(_train_dataset)
 
-    @property
-    def val_dataset(self) -> paddle.io.Dataset:
-        _val_dataset = self.val_dataset_config
-        if not _val_dataset:
-            return None
-        return self._load_object(_val_dataset)
-
-    def _load_component(self, com_name: str) -> Any:
-        com_list = [
-            manager.MODELS, manager.BACKBONES, manager.DATASETS,
-            manager.TRANSFORMS, manager.LOSSES
-        ]
-
-        for com in com_list:
-            if com_name in com.components_dict:
-                return com[com_name]
-        else:
-            raise RuntimeError(
-                'The specified component was not found {}.'.format(com_name))
-
-    def _load_object(self, cfg: dict) -> Any:
-        cfg = cfg.copy()
-        if 'type' not in cfg:
-            raise RuntimeError('No object information in {}.'.format(cfg))
-
-        component = self._load_component(cfg.pop('type'))
-
-        params = {}
-        for key, val in cfg.items():
-            if self._is_meta_type(val):
-                params[key] = self._load_object(val)
-            elif isinstance(val, list):
-                params[key] = [
-                    self._load_object(item)
-                    if self._is_meta_type(item) else item for item in val
-                ]
-            else:
-                params[key] = val
-
-        return component(**params)
+def parse_from_yaml(path: str):
+    """Parse a yaml file and build config"""
+    with codecs.open(path, 'r', 'utf-8') as file:
+        dic = yaml.load(file, Loader=yaml.FullLoader)
 
-    @property
-    def test_config(self) -> Dict:
-        return self.dic.get('test_config', {})
+    if _BASE_KEY in dic:
+        base_files = dic.pop(_BASE_KEY)
+        if isinstance(base_files, str):
+            base_files = [base_files]
+        for bf in base_files:
+            base_path = os.path.join(os.path.dirname(path), bf)
+            base_dic = parse_from_yaml(base_path)
+            dic = merge_config_dicts(dic, base_dic)
 
-    @property
-    def export_config(self) -> Dict:
-        return self.dic.get('export', {})
+    return dic
 
-    @property
-    def to_static_training(self) -> bool:
-        '''Whether to use @to_static for training'''
-        return self.dic.get('to_static_training', False)
 
-    def _is_meta_type(self, item: Any) -> bool:
-        return isinstance(item, dict) and 'type' in item
+def merge_config_dicts(dic, base_dic):
+    """Merge dic to base_dic and return base_dic."""
+    base_dic = base_dic.copy()
+    dic = dic.copy()
 
-    def __str__(self) -> str:
-        return yaml.dump(self.dic)
+    if not dic.get(_INHERIT_KEY, True):
+        dic.pop(_INHERIT_KEY)
+        return dic
 
-    @property
-    def val_transforms(self) -> list:
-        """Get val_transform from val_dataset"""
-        _val_dataset = self.val_dataset_config
-        if not _val_dataset:
-            return []
-        _transforms = _val_dataset.get('transforms', [])
-        transforms = []
-        for i in _transforms:
-            transforms.append(self._load_object(i))
-        return transforms
-
-    def check_sync_info(self) -> None:
-        """
-        Check and sync the info, such as num_classes and img_channels, 
-        between the config of model, train_dataset and val_dataset.
-        """
-        self._check_sync_num_classes()
-        self._check_sync_img_channels()
-
-    def _check_sync_num_classes(self):
-        num_classes_set = set()
-
-        if self.dic['model'].get('num_classes', None) is not None:
-            num_classes_set.add(self.dic['model'].get('num_classes'))
-        if self.train_dataset_config:
-            if hasattr(self.train_dataset_class, 'NUM_CLASSES'):
-                num_classes_set.add(self.train_dataset_class.NUM_CLASSES)
-            elif 'num_classes' in self.train_dataset_config:
-                num_classes_set.add(self.train_dataset_config['num_classes'])
-        if self.val_dataset_config:
-            if hasattr(self.val_dataset_class, 'NUM_CLASSES'):
-                num_classes_set.add(self.val_dataset_class.NUM_CLASSES)
-            elif 'num_classes' in self.val_dataset_config:
-                num_classes_set.add(self.val_dataset_config['num_classes'])
-
-        if len(num_classes_set) == 0:
-            raise ValueError(
-                '`num_classes` is not found. Please set it in model, train_dataset or val_dataset'
-            )
-        elif len(num_classes_set) > 1:
-            raise ValueError(
-                '`num_classes` is not consistent: {}. Please set it consistently in model or train_dataset or val_dataset'
-                .format(num_classes_set))
-
-        num_classes = num_classes_set.pop()
-        self.dic['model']['num_classes'] = num_classes
-        if self.train_dataset_config and \
-            (not hasattr(self.train_dataset_class, 'NUM_CLASSES')):
-            self.dic['train_dataset']['num_classes'] = num_classes
-        if self.val_dataset_config and \
-            (not hasattr(self.val_dataset_class, 'NUM_CLASSES')):
-            self.dic['val_dataset']['num_classes'] = num_classes
-
-    def _check_sync_img_channels(self):
-        img_channels_set = set()
-        model_cfg = self.dic['model']
-
-        # If the model has backbone, in_channels is the input params of backbone.
-        # Otherwise, in_channels is the input params of the model.
-        if 'backbone' in model_cfg:
-            x = model_cfg['backbone'].get('in_channels', None)
-            if x is not None:
-                img_channels_set.add(x)
-        elif model_cfg.get('in_channels', None) is not None:
-            img_channels_set.add(model_cfg.get('in_channels'))
-        if self.train_dataset_config and \
-            ('img_channels' in self.train_dataset_config):
-            img_channels_set.add(self.train_dataset_config['img_channels'])
-        if self.val_dataset_config and \
-            ('img_channels' in self.val_dataset_config):
-            img_channels_set.add(self.val_dataset_config['img_channels'])
-
-        if len(img_channels_set) > 1:
-            raise ValueError(
-                '`img_channels` is not consistent: {}. Please set it consistently in model or train_dataset or val_dataset'
-                .format(img_channels_set))
-
-        img_channels = 3 if len(img_channels_set) == 0 \
-            else img_channels_set.pop()
-        if 'backbone' in model_cfg:
-            self.dic['model']['backbone']['in_channels'] = img_channels
+    for key, val in dic.items():
+        if isinstance(val, dict) and key in base_dic:
+            base_dic[key] = merge_config_dicts(val, base_dic[key])
         else:
-            self.dic['model']['in_channels'] = img_channels
-        if self.train_dataset_config and \
-            self.train_dataset_config['type'] == "Dataset":
-            self.dic['train_dataset']['img_channels'] = img_channels
-        if self.val_dataset_config and \
-            self.val_dataset_config['type'] == "Dataset":
-            self.dic['val_dataset']['img_channels'] = img_channels
+            base_dic[key] = val
+
+    return base_dic
+
+
+def update_config_dict(dic: dict,
+                       learning_rate: Optional[float]=None,
+                       batch_size: Optional[int]=None,
+                       iters: Optional[int]=None,
+                       opts: Optional[list]=None):
+    """Update config"""
+    # TODO: If the items to update are marked as anchors in the yaml file,
+    # we should synchronize the references.
+    dic = dic.copy()
+
+    if learning_rate:
+        dic['lr_scheduler']['learning_rate'] = learning_rate
+    if batch_size:
+        dic['batch_size'] = batch_size
+    if iters:
+        dic['iters'] = iters
+
+    if opts is not None:
+        for item in opts:
+            assert ('=' in item) and (len(item.split('=')) == 2), "--opts params should be key=value," \
+                " such as `--opts batch_size=1 test_config.scales=0.75,1.0,1.25`, " \
+                "but got ({})".format(opts)
+
+            key, value = item.split('=')
+            if isinstance(value, six.string_types):
+                try:
+                    value = literal_eval(value)
+                except ValueError:
+                    pass
+                except SyntaxError:
+                    pass
+            key_list = key.split('.')
+
+            tmp_dic = dic
+            for subkey in key_list[:-1]:
+                assert subkey in tmp_dic, "Can not update {}, because it is not in config.".format(
+                    key)
+                tmp_dic = tmp_dic[subkey]
+            tmp_dic[key_list[-1]] = value
+
+    return dic

paddlers/models/paddleseg/cvlibs/config_checker.py

@@ -0,0 +1,225 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+
+from paddlers.models.paddleseg.utils import logger
+from paddlers.models.paddleseg.utils import utils
+
+
+class ConfigChecker(object):
+    """
+    This class performs sanity checks on configuration objects and (optionally) updates the configurations
+        (e.g., synchronize specific key-value pairs) based on a set of rules. 
+
+    Args:
+        rule_list (list): A list of rules on which all checks and updates are based.
+        allow_update (bool, optional): Whether or not to allow updating the configuration object.
+    """
+
+    def __init__(self, rule_list, allow_update=True):
+        super().__init__()
+        self.rule_list = rule_list
+        self.allow_update = allow_update
+
+    def apply_rule(self, k, cfg):
+        rule = self.rule_list[k]
+        try:
+            rule.apply(cfg, self.allow_update)
+        except Exception as e:
+            raise RuntimeError(
+                "Sanity check on the configuration file has failed. "
+                "There should be some problems with your config file. "
+                "Please check it carefully.\n"
+                f"The failed rule is {rule.__class__.__name__}, and the error message is: \n{str(e)}"
+            )
+
+    def apply_all_rules(self, cfg):
+        for i in range(len(self.rule_list)):
+            self.apply_rule(i, cfg)
+
+    def add_rule(self, rule):
+        self.rule_list.append(rule)
+
+
+class Rule(object):
+    def check_and_correct(self, cfg):
+        raise NotImplementedError
+
+    def apply(self, cfg, allow_update):
+        if not allow_update:
+            cfg = copy.deepcopy(cfg)
+        self.check_and_correct(cfg)
+
+
+class DefaultPrimaryRule(Rule):
+    def check_and_correct(self, cfg):
+        items = [
+            'batch_size', 'iters', 'train_dataset', 'optimizer', 'lr_scheduler',
+            'loss', 'model'
+        ]
+        for i in items:
+            assert i in cfg.dic, \
+            'No {} specified in the configuration file.'.format(i)
+
+
+class DefaultLossRule(Rule):
+    def __init__(self, loss_name):
+        super().__init__()
+        self.loss_name = loss_name
+
+    def check_and_correct(self, cfg):
+        loss_cfg = cfg.dic.get(self.loss_name, None)
+        if loss_cfg is None:
+            return
+
+        assert 'types' in loss_cfg and 'coef' in loss_cfg, \
+                'Loss config should contain keys of "types" and "coef"'
+        len_types = len(loss_cfg['types'])
+        len_coef = len(loss_cfg['coef'])
+        if len_types != len_coef:
+            if len_types == 1:
+                loss_cfg['types'] = loss_cfg['types'] * len_coef
+            else:
+                raise ValueError(
+                    "For loss config, the length of types should be 1 "
+                    "or be equal to coef , but they are {} and {}.".format(
+                        len_types, len_coef))
+
+
+class DefaultSyncNumClassesRule(Rule):
+    def check_and_correct(self, cfg):
+        # check the num_classes in model, train_dataset and val_dataset config
+        model_cfg = cfg.model_cfg
+        train_dataset_cfg = cfg.train_dataset_cfg
+        val_dataset_cfg = cfg.val_dataset_cfg
+        assert train_dataset_cfg != {}, \
+            'No train_dataset specified in the configuration file.'
+        if train_dataset_cfg['type'] != 'Dataset':
+            return
+        if val_dataset_cfg != {}:
+            assert val_dataset_cfg['type'] == 'Dataset', \
+                'The type of train_dataset and val_dataset must be the same'
+            assert 'num_classes' in val_dataset_cfg, \
+                'No num_classes specified in train_dataset config.'
+        assert 'num_classes' in train_dataset_cfg, \
+            'No num_classes specified in train_dataset config.'
+
+        value_set = set()
+        value_name = 'num_classes'
+        if value_name in model_cfg:
+            value_set.add(model_cfg[value_name])
+        if value_name in train_dataset_cfg:
+            value_set.add(train_dataset_cfg[value_name])
+        if value_name in val_dataset_cfg:
+            value_set.add(val_dataset_cfg[value_name])
+
+        if len(value_set) > 1:
+            raise ValueError(
+                '`num_classes` is not consistent: {}. Please set it '
+                'consistently in model, train_dataset and val_dataset config'.
+                format(value_set))
+        if len(value_set) == 1 and value_name not in model_cfg:
+            logger.warning(
+                'Add the `num_classes` in train_dataset and val_dataset '
+                'config to model config. We suggest you manually set `num_classes` in model config.'
+            )
+            model_cfg[value_name] = value_set.pop()
+            cfg.dic['model'] = model_cfg
+
+
+class DefaultSyncImgChannelsRule(Rule):
+    def check_and_correct(self, cfg):
+        model_cfg = cfg.model_cfg
+        train_dataset_cfg = cfg.train_dataset_cfg
+        val_dataset_cfg = cfg.val_dataset_cfg
+
+        assert train_dataset_cfg != {}, \
+            'No train_dataset specified in the configuration file.'
+        if train_dataset_cfg['type'] != 'Dataset':
+            return
+        if val_dataset_cfg != {}:
+            assert val_dataset_cfg['type'] == 'Dataset', \
+                'The type of train_dataset and val_dataset must be the same'
+
+        # If the model has backbone, in_channels is the input params of backbone.
+        # Otherwise, in_channels is the input params of the model.
+        value_set = set()
+        x = utils.get_in_channels(model_cfg)
+        if x is not None:
+            value_set.add(x)
+        if 'img_channels' in train_dataset_cfg:
+            value_set.add(train_dataset_cfg['img_channels'])
+        if 'img_channels' in val_dataset_cfg:
+            value_set.add(val_dataset_cfg['img_channels'])
+
+        if len(value_set) > 1:
+            raise ValueError('`in_channels` is not consistent: {}. Please set it ' \
+                'consistently in model or train_dataset or val_dataset'.format(value_set))
+        if len(value_set) == 1 and utils.get_in_channels(model_cfg) is None:
+            logger.warning(
+                'Add the `in_channels` in train_dataset and val_dataset '
+                'config to model config. We suggest you manually set `in_channels` in model config.'
+            )
+            model_cfg = utils.set_in_channels(model_cfg, value_set.pop())
+            cfg.dic['model'] = model_cfg
+        # if len(value_set) == 0, model and dataset use the default in_channels (3)
+
+
+class DefaultSyncIgnoreIndexRule(Rule):
+    def __init__(self, loss_name):
+        super().__init__()
+        self.loss_name = loss_name
+
+    def check_and_correct(self, cfg):
+        def _check_helper(loss_cfg, dataset_ignore_index):
+            if 'ignore_index' not in loss_cfg:
+                loss_cfg['ignore_index'] = dataset_ignore_index
+            else:
+                assert loss_cfg['ignore_index'] == dataset_ignore_index, \
+                    'the ignore_index in loss and train_dataset must be the same. Currently, loss ignore_index = {}, '\
+                    'train_dataset ignore_index = {}'.format(loss_cfg['ignore_index'], dataset_ignore_index)
+
+        loss_cfg = cfg.dic.get(self.loss_name, None)
+        if loss_cfg is None:
+            return
+
+        train_dataset_cfg = cfg.train_dataset_cfg
+        val_dataset_cfg = cfg.val_dataset_cfg
+        assert train_dataset_cfg != {}, \
+            'No train_dataset specified in the configuration file.'
+        if train_dataset_cfg['type'] != 'Dataset':
+            return
+        if val_dataset_cfg != {}:
+            assert val_dataset_cfg['type'] == 'Dataset', \
+                'The type of train_dataset and val_dataset must be the same'
+
+        value_set = set()
+        value_name = 'ignore_index'
+        if value_name in train_dataset_cfg:
+            value_set.add(train_dataset_cfg[value_name])
+        if value_name in val_dataset_cfg:
+            value_set.add(val_dataset_cfg[value_name])
+
+        if len(value_set) > 1:
+            raise ValueError('`ignore_index` is not consistent: {}. Please set ' \
+                'it consistently in train_dataset and val_dataset'.format(value_set))
+        if len(value_set) == 1:
+            ignore_index = value_set.pop()
+            for loss_cfg_i in loss_cfg['types']:
+                if loss_cfg_i['type'] == 'MixedLoss':
+                    for loss_cfg_j in loss_cfg_i['losses']:
+                        _check_helper(loss_cfg_j, ignore_index)
+                else:
+                    _check_helper(loss_cfg_i, ignore_index)

paddlers/models/paddleseg/cvlibs/manager.py

@@ -145,3 +145,4 @@ BACKBONES = ComponentManager("backbones")
 DATASETS = ComponentManager("datasets")
 TRANSFORMS = ComponentManager("transforms")
 LOSSES = ComponentManager("losses")
+OPTIMIZERS = ComponentManager("optimizers")

paddlers/models/paddleseg/cvlibs/param_init.py

@@ -13,6 +13,29 @@
 # limitations under the License.
 
 import paddle.nn as nn
+import math
+
+
+def uniform_init(param, **kwargs):
+    """
+    Initialize the `param` with uniform distribution.
+
+    Args:
+        param (Tensor): Tensor that needs to be initialized.
+
+    Examples:
+
+        from paddlers.models.paddleseg.cvlibs import param_init
+        import paddle.nn as nn
+
+        linear = nn.Linear(2, 2)
+        param_init.uniform_init(linear.bias,  low=-0.5, high=0。5)
+        print(linear.bias.numpy())
+        # result is [-0.2734719   0.23939109]
+
+    """
+    initializer = nn.initializer.Uniform(**kwargs)
+    initializer(param, param.block)
 
 
 def constant_init(param, **kwargs):
@@ -163,4 +186,52 @@ def xavier_uniform(param, **kwargs):
         param_init.xavier_uniform(linear.weight)
     """
     initializer = nn.initializer.XavierUniform(**kwargs)
-    initializer(param, param.block)
+    initializer(param, param.block)
+
+
+def multihead_fill(layer, qkv_same_embed_dim=True):
+    """
+    The default initialization of multi-head attention.
+
+    Example:
+        from paddlers.models.paddleseg.cvlibs import param_init
+        import paddle.nn as nn
+        
+        self_attn = nn.MultiHeadAttention(
+            128, 8, dropout=False)
+        param_init.multihead_fill(self_attn, True)
+    """
+
+    def _init_param_as_combined_linear_weight(p):
+        bound = math.sqrt(6 / (3 * p.shape[0] + p.shape[1]))
+        nn.initializer.Uniform(low=-bound, high=bound)(p)
+
+    if qkv_same_embed_dim:
+        _init_param_as_combined_linear_weight(layer.q_proj.weight)
+        _init_param_as_combined_linear_weight(layer.k_proj.weight)
+        _init_param_as_combined_linear_weight(layer.v_proj.weight)
+        xavier_uniform(layer.out_proj.weight)
+    else:
+        for p in layer.parameters():
+            if p.dim() > 1:
+                xavier_uniform(p)
+
+
+def th_linear_fill(layer):
+    """
+    The default way of linear initialization.
+    
+    Example:
+        from paddlers.models.paddleseg.cvlibs import param_init
+        import paddle.nn as nn
+        
+        linear = nn.Linear(128, 128)
+        param_init.linear_fill(linear)
+    """
+    nn.initializer.KaimingUniform(
+        negative_slope=math.sqrt(5), nonlinearity='leaky_relu')(layer.weight)
+
+    if getattr(layer, 'bias', None) is not None:
+        fan_in = layer.weight.shape[0]
+        bound = 1 / math.sqrt(fan_in) if fan_in > 0 else 0
+        nn.initializer.Uniform(low=-bound, high=bound)(layer.bias)

paddlers/models/paddleseg/datasets/ade.py

@@ -39,6 +39,8 @@ class ADE20K(Dataset):
         edge (bool, optional): Whether to compute edge while training. Default: False
     """
     NUM_CLASSES = 150
+    IGNORE_INDEX = 255
+    IMG_CHANNELS = 3
 
     def __init__(self, transforms, dataset_root=None, mode='train', edge=False):
         self.dataset_root = dataset_root
@@ -47,7 +49,7 @@ class ADE20K(Dataset):
         self.mode = mode
         self.file_list = list()
         self.num_classes = self.NUM_CLASSES
-        self.ignore_index = 255
+        self.ignore_index = self.IGNORE_INDEX
         self.edge = edge
 
         if mode not in ['train', 'val']:
@@ -97,23 +99,21 @@ class ADE20K(Dataset):
         ]  # If key in gt_fields, the data[key] have transforms synchronous.
 
         if self.mode == 'val':
-            data = self.transforms(data)
             label = np.asarray(Image.open(label_path))
             # The class 0 is ignored. And it will equal to 255 after
             # subtracted 1, because the dtype of label is uint8.
             label = label - 1
+            data = self.transforms(data)
             label = label[np.newaxis, :, :]
             data['label'] = label
             return data
         else:
-            data['label'] = label_path
+            data['label'] = np.asarray(Image.open(label_path))
             data['gt_fields'].append('label')
-            data = self.transforms(data)
             data['label'] = data['label'] - 1
-            # Recover the ignore pixels adding by transform
-            data['label'][data['label'] == 254] = 255
+            data = self.transforms(data)
             if self.edge:
                 edge_mask = F.mask_to_binary_edge(
                     label, radius=2, num_classes=self.num_classes)
                 data['edge'] = edge_mask
-            return data
+            return data

paddlers/models/paddleseg/datasets/chase_db1.py

@@ -39,6 +39,8 @@ class CHASEDB1(Dataset):
         mode (str, optional): Which part of dataset to use. it is one of ('train', 'val', 'test'). Default: 'train'.
     """
     NUM_CLASSES = 2
+    IGNORE_INDEX = 255
+    IMG_CHANNELS = 3
 
     def __init__(self,
                  dataset_root=None,
@@ -52,7 +54,7 @@ class CHASEDB1(Dataset):
         self.edge = edge
         self.file_list = list()
         self.num_classes = self.NUM_CLASSES
-        self.ignore_index = 255  # labels only have 1/0, thus ignore_index is not necessary
+        self.ignore_index = self.IGNORE_INDEX  # labels only have 1/0, thus ignore_index is not necessary
 
         if mode not in ['train', 'val', 'test']:
             raise ValueError(

paddlers/models/paddleseg/datasets/cityscapes.py

@@ -47,6 +47,8 @@ class Cityscapes(Dataset):
         edge (bool, optional): Whether to compute edge while training. Default: False
     """
     NUM_CLASSES = 19
+    IGNORE_INDEX = 255
+    IMG_CHANNELS = 3
 
     def __init__(self, transforms, dataset_root, mode='train', edge=False):
         self.dataset_root = dataset_root
@@ -55,7 +57,7 @@ class Cityscapes(Dataset):
         mode = mode.lower()
         self.mode = mode
         self.num_classes = self.NUM_CLASSES
-        self.ignore_index = 255
+        self.ignore_index = self.IGNORE_INDEX
         self.edge = edge
 
         if mode not in ['train', 'val', 'test']:

paddlers/models/paddleseg/datasets/cocostuff.py

@@ -44,6 +44,8 @@ class CocoStuff(Dataset):
         edge (bool, optional): Whether to compute edge while training. Default: False
     """
     NUM_CLASSES = 171
+    IGNORE_INDEX = 255
+    IMG_CHANNELS = 3
 
     def __init__(self, transforms, dataset_root, mode='train', edge=False):
         self.dataset_root = dataset_root
@@ -52,7 +54,7 @@ class CocoStuff(Dataset):
         mode = mode.lower()
         self.mode = mode
         self.num_classes = self.NUM_CLASSES
-        self.ignore_index = 255
+        self.ignore_index = self.IGNORE_INDEX
         self.edge = edge
 
         if mode not in ['train', 'val']:

paddlers/models/paddleseg/datasets/drive.py

@@ -37,6 +37,8 @@ class DRIVE(Dataset):
         mode (str, optional): Which part of dataset to use. it is one of ('train', 'val', 'test'). Default: 'train'.
     """
     NUM_CLASSES = 2
+    IGNORE_INDEX = 255
+    IMG_CHANNELS = 3
 
     def __init__(self,
                  dataset_root=None,
@@ -50,7 +52,7 @@ class DRIVE(Dataset):
         self.edge = edge
         self.file_list = list()
         self.num_classes = self.NUM_CLASSES
-        self.ignore_index = 255  # labels only have 1/0, thus ignore_index is not necessary
+        self.ignore_index = self.IGNORE_INDEX  # labels only have 1/0, thus ignore_index is not necessary
 
         if mode not in ['train', 'val', 'test']:
             raise ValueError(

paddlers/models/paddleseg/datasets/eg1800.py

@@ -42,6 +42,8 @@ class EG1800(Dataset):
         edge (bool, optional): Whether to compute edge while training. Default: False
     """
     NUM_CLASSES = 2
+    IGNORE_INDEX = 255
+    IMG_CHANNELS = 3
 
     def __init__(self,
                  common_transforms,
@@ -58,7 +60,7 @@ class EG1800(Dataset):
         if transforms2 is not None:
             self.transforms2 = Compose(transforms2, to_rgb=False)
         mode = mode.lower()
-        self.ignore_index = 255
+        self.ignore_index = self.IGNORE_INDEX
         self.mode = mode
         self.num_classes = self.NUM_CLASSES
         self.input_width = 224
@@ -105,16 +107,19 @@ class EG1800(Dataset):
         label[label > 1] = 0
 
         if self.mode == "val":
-            common_im, label = self.common_transforms(im=im, label=label)
+            common_data = self.common_transforms(dict(img=im, label=label))
+            common_im, label = common_data['img'], common_data['label']
             im = np.float32(common_im[::-1, :, :])  # RGB => BGR
             im_aug = copy.deepcopy(im)
         else:
-            common_im, label = self.common_transforms(im=im, label=label)
+            common_data = self.common_transforms(dict(img=im, label=label))
+            common_im, label = common_data['img'], common_data['label']
             common_im = np.transpose(common_im, [1, 2, 0])
             # add augmentation
-            im, _ = self.transforms1(common_im)
-            im_aug, _ = self.transforms2(common_im)
-
+            data = self.transforms1(dict(img=common_im))
+            im = data['img']
+            data = self.transforms2(dict(img=common_im))
+            im_aug = data['img']
             im = np.float32(im[::-1, :, :])  # RGB => BGR
             im_aug = np.float32(im_aug[::-1, :, :])  # RGB => BGR
 
@@ -130,8 +135,9 @@ class EG1800(Dataset):
         edge_mask = F.mask_to_binary_edge(
             label, radius=4, num_classes=self.num_classes)
         edge_mask = np.transpose(edge_mask, [1, 2, 0]).squeeze(axis=-1)
-        im = np.concatenate([im_aug, im])
+        #im = np.concatenate([im_aug, im])
+        #im = im_aug
         if self.mode == "train":
-            return im, label, edge_mask
+            return dict(img=im, label=label, edge=edge_mask)
         else:
-            return im, label
+            return dict(img=im, label=label)

paddlers/models/paddleseg/datasets/hrf.py

@@ -36,6 +36,8 @@ class HRF(Dataset):
         mode (str, optional): Which part of dataset to use. it is one of ('train', 'val', 'test'). Default: 'train'.
     """
     NUM_CLASSES = 2
+    IGNORE_INDEX = 255
+    IMG_CHANNELS = 3
 
     def __init__(self,
                  dataset_root=None,
@@ -49,7 +51,7 @@ class HRF(Dataset):
         self.edge = edge
         self.file_list = list()
         self.num_classes = self.NUM_CLASSES
-        self.ignore_index = 255
+        self.ignore_index = self.IGNORE_INDEX
 
         if mode not in ['train', 'val', 'test']:
             raise ValueError(

paddlers/models/paddleseg/datasets/mini_deep_globe_road_extraction.py

@@ -37,6 +37,8 @@ class MiniDeepGlobeRoadExtraction(Dataset):
         edge (bool, optional): Whether to compute edge while training. Default: False.
     """
     NUM_CLASSES = 2
+    IGNORE_INDEX = 255
+    IMG_CHANNELS = 3
 
     def __init__(self,
                  dataset_root=None,
@@ -49,7 +51,7 @@ class MiniDeepGlobeRoadExtraction(Dataset):
         self.mode = mode
         self.file_list = list()
         self.num_classes = self.NUM_CLASSES
-        self.ignore_index = 255
+        self.ignore_index = self.IGNORE_INDEX
         self.edge = edge
 
         if mode not in ['train', 'val']:

paddlers/models/paddleseg/datasets/optic_disc_seg.py

@@ -36,6 +36,8 @@ class OpticDiscSeg(Dataset):
         edge (bool, optional): Whether to compute edge while training. Default: False
     """
     NUM_CLASSES = 2
+    IGNORE_INDEX = 255
+    IMG_CHANNELS = 3
 
     def __init__(self,
                  dataset_root=None,
@@ -48,7 +50,7 @@ class OpticDiscSeg(Dataset):
         self.mode = mode
         self.file_list = list()
         self.num_classes = self.NUM_CLASSES
-        self.ignore_index = 255
+        self.ignore_index = self.IGNORE_INDEX
         self.edge = edge
 
         if mode not in ['train', 'val', 'test']:

paddlers/models/paddleseg/datasets/pascal_context.py

@@ -34,6 +34,8 @@ class PascalContext(Dataset):
         edge (bool, optional): Whether to compute edge while training. Default: False
     """
     NUM_CLASSES = 60
+    IGNORE_INDEX = 255
+    IMG_CHANNELS = 3
 
     def __init__(self,
                  transforms=None,
@@ -46,7 +48,7 @@ class PascalContext(Dataset):
         self.mode = mode
         self.file_list = list()
         self.num_classes = self.NUM_CLASSES
-        self.ignore_index = 255
+        self.ignore_index = self.IGNORE_INDEX
         self.edge = edge
 
         if mode not in ['train', 'trainval', 'val']:

paddlers/models/paddleseg/datasets/pp_humanseg14k.py

@@ -36,6 +36,8 @@ class PPHumanSeg14K(Dataset):
         edge (bool, optional): Whether to compute edge while training. Default: False.
     """
     NUM_CLASSES = 2
+    IGNORE_INDEX = 255
+    IMG_CHANNELS = 3
 
     def __init__(self,
                  dataset_root=None,
@@ -48,7 +50,7 @@ class PPHumanSeg14K(Dataset):
         self.mode = mode
         self.file_list = list()
         self.num_classes = self.NUM_CLASSES
-        self.ignore_index = 255
+        self.ignore_index = self.IGNORE_INDEX
         self.edge = edge
 
         if mode not in ['train', 'val', 'test']:

paddlers/models/paddleseg/datasets/pssl.py

@@ -71,6 +71,8 @@ class PSSLDataset(Dataset):
     """
     ignore_index = 1001  # 0~999 is target class, 1000 is bg
     NUM_CLASSES = 1001  # consider target class and bg
+    IGNORE_INDEX = 1001
+    IMG_CHANNELS = 3
 
     def __init__(self,
                  transforms,
@@ -89,7 +91,7 @@ class PSSLDataset(Dataset):
         self.edge = edge
 
         self.num_classes = self.NUM_CLASSES
-        self.ignore_index = self.num_classes  # 1001
+        self.ignore_index = self.IGNORE_INDEX  # 1001
         self.file_list = []
         self.class_id_dict = {}
 

paddlers/models/paddleseg/datasets/stare.py

@@ -36,6 +36,8 @@ class STARE(Dataset):
         mode (str, optional): Which part of dataset to use. it is one of ('train', 'val', 'test'). Default: 'train'.
     """
     NUM_CLASSES = 2
+    IGNORE_INDEX = 255
+    IMG_CHANNELS = 3
 
     def __init__(self,
                  dataset_root=None,
@@ -49,7 +51,7 @@ class STARE(Dataset):
         self.edge = edge
         self.file_list = list()
         self.num_classes = self.NUM_CLASSES
-        self.ignore_index = 255
+        self.ignore_index = self.IGNORE_INDEX
 
         if mode not in ['train', 'val', 'test']:
             raise ValueError(

paddlers/models/paddleseg/datasets/supervisely.py

@@ -42,6 +42,8 @@ class SUPERVISELY(Dataset):
         edge (bool, optional): Whether to compute edge while training. Default: False
     """
     NUM_CLASSES = 2
+    IGNORE_INDEX = 255
+    IMG_CHANNELS = 3
 
     def __init__(self,
                  common_transforms,
@@ -58,7 +60,7 @@ class SUPERVISELY(Dataset):
         if transforms2 is not None:
             self.transforms2 = Compose(transforms2, to_rgb=False)
         mode = mode.lower()
-        self.ignore_index = 255
+        self.ignore_index = self.IGNORE_INDEX
         self.mode = mode
         self.num_classes = self.NUM_CLASSES
         self.input_width = 224
@@ -104,15 +106,19 @@ class SUPERVISELY(Dataset):
         label[label > 0] = 1
 
         if self.mode == "val":
-            common_im, label = self.common_transforms(im=im, label=label)
+            common_data = self.common_transforms(dict(img=im, label=label))
+            common_im, label = common_data['img'], common_data['label']
             im = np.float32(common_im[::-1, :, :])  # RGB => BGR
             im_aug = copy.deepcopy(im)
         else:
-            common_im, label = self.common_transforms(im=im, label=label)
+            common_data = self.common_transforms(dict(img=im, label=label))
+            common_im, label = common_data['img'], common_data['label']
             common_im = np.transpose(common_im, [1, 2, 0])
             # add augmentation
-            im, _ = self.transforms1(common_im)
-            im_aug, _ = self.transforms2(common_im)
+            data = self.transforms1(dict(img=common_im))
+            im = data['img']
+            data = self.transforms2(dict(img=common_im))
+            im_aug = data['img']
 
             im = np.float32(im[::-1, :, :])  # RGB => BGR
             im_aug = np.float32(im_aug[::-1, :, :])  # RGB => BGR
@@ -129,8 +135,8 @@ class SUPERVISELY(Dataset):
         edge_mask = F.mask_to_binary_edge(
             label, radius=4, num_classes=self.num_classes)
         edge_mask = np.transpose(edge_mask, [1, 2, 0]).squeeze(axis=-1)
-        im = np.concatenate([im_aug, im])
+        #im = np.concatenate([im_aug, im])
         if self.mode == "train":
-            return im, label, edge_mask
+            return dict(img=im, label=label, edge=edge_mask)
         else:
-            return im, label
+            return dict(img=im, label=label)

paddlers/models/paddleseg/datasets/voc.py

@@ -37,6 +37,8 @@ class PascalVOC(Dataset):
         edge (bool, optional): Whether to compute edge while training. Default: False
     """
     NUM_CLASSES = 21
+    IGNORE_INDEX = 255
+    IMG_CHANNELS = 3
 
     def __init__(self, transforms, dataset_root=None, mode='train', edge=False):
         self.dataset_root = dataset_root
@@ -45,7 +47,7 @@ class PascalVOC(Dataset):
         self.mode = mode
         self.file_list = list()
         self.num_classes = self.NUM_CLASSES
-        self.ignore_index = 255
+        self.ignore_index = self.IGNORE_INDEX
         self.edge = edge
 
         if mode not in ['train', 'trainval', 'trainaug', 'val']:

paddlers/models/paddleseg/deploy/__init__.py

@@ -0,0 +1,12 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and

paddlers/models/paddleseg/deploy/export.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+
+import paddle
+
+
+class WrappedModel(paddle.nn.Layer):
+    def __init__(self, model, output_op):
+        super().__init__()
+        self.model = model
+        self.output_op = output_op
+        assert output_op in ['argmax', 'softmax'], \
+            "output_op should in ['argmax', 'softmax']"
+
+    def forward(self, x):
+        outs = self.model(x)
+        new_outs = []
+        for out in outs:
+            if self.output_op == 'argmax':
+                out = paddle.argmax(out, axis=1, dtype='int32')
+            elif self.output_op == 'softmax':
+                out = paddle.nn.functional.softmax(out, axis=1)
+            new_outs.append(out)
+        return new_outs

paddlers/models/paddleseg/deploy/infer.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+
+import codecs
+import os
+
+import yaml
+
+import paddlers.models.paddleseg.transforms as T
+from paddlers.models.paddleseg.cvlibs import manager
+
+
+class DeployConfig:
+    def __init__(self, path):
+        with codecs.open(path, 'r', 'utf-8') as file:
+            self.dic = yaml.load(file, Loader=yaml.FullLoader)
+
+        self._transforms = self.load_transforms(self.dic['Deploy'][
+            'transforms'])
+        self._dir = os.path.dirname(path)
+
+    @property
+    def transforms(self):
+        return self._transforms
+
+    @property
+    def model(self):
+        return os.path.join(self._dir, self.dic['Deploy']['model'])
+
+    @property
+    def params(self):
+        return os.path.join(self._dir, self.dic['Deploy']['params'])
+
+    @staticmethod
+    def load_transforms(t_list):
+        com = manager.TRANSFORMS
+        transforms = []
+        for t in t_list:
+            ctype = t.pop('type')
+            transforms.append(com[ctype](**t))
+
+        return T.Compose(transforms)

paddlers/models/paddleseg/hash.txt

@@ -1 +1 @@
-13aac35b31b147d2dd9976821ade13dabebff685
+901428624dff41fe22a6853ed5981c1e603585d9

paddlers/models/paddleseg/models/__init__.py

@@ -61,8 +61,15 @@ from .ddrnet import DDRNet_23
 from .ccnet import CCNet
 from .mobileseg import MobileSeg
 from .upernet import UPerNet
+from .upernet_cae import UPerNetCAE
 from .sinet import SINet
 from .lraspp import LRASPP
 from .mscale_ocrnet import MscaleOCRNet
 from .topformer import TopFormer
 from .rtformer import RTFormer
+from .upernet_vit_adapter import UPerNetViTAdapter
+from .lpsnet import LPSNet
+from .maskformer import MaskFormer
+from .segnext import SegNeXt
+from .knet import KNet
+from .pp_mobileseg import PPMobileSeg

paddlers/models/paddleseg/models/backbones/__init__.py

@@ -24,5 +24,10 @@ from .stdcnet import *
 from .lite_hrnet import *
 from .shufflenetv2 import *
 from .ghostnet import *
+from .cae import *
 from .top_transformer import *
 from .uhrnet import *
+from .strideformer import *
+from .vit_adapter import *
+from .hrformer import *
+from .mscan import *

paddlers/models/paddleseg/models/backbones/cae.py

@@ -0,0 +1,599 @@
+# copyright (c) 2022 PaddlePaddle Authors. All Rights Reserve.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import math
+import numpy as np
+
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+from paddle.nn.initializer import TruncatedNormal, Constant, Normal
+
+from paddlers.models.paddleseg.cvlibs import manager
+from paddlers.models.paddleseg.utils import utils, logger
+from paddlers.models.paddleseg.models.backbones.transformer_utils import to_2tuple, DropPath, Identity
+
+zeros_ = Constant(value=0.)
+
+
+class Mlp(nn.Layer):
+    def __init__(self,
+                 in_features,
+                 hidden_features=None,
+                 out_features=None,
+                 act_layer=nn.GELU,
+                 drop=0.):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = nn.Linear(in_features, hidden_features)
+        self.act = act_layer()
+        self.fc2 = nn.Linear(hidden_features, out_features)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x
+
+
+class Attention(nn.Layer):
+    def __init__(self,
+                 dim,
+                 num_heads=8,
+                 qkv_bias=False,
+                 qk_scale=None,
+                 attn_drop=0.,
+                 proj_drop=0.,
+                 window_size=None):
+        super().__init__()
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        self.scale = qk_scale or head_dim**-0.5
+
+        self.qkv = nn.Linear(dim, dim * 3, bias_attr=False)
+
+        if qkv_bias:
+            self.q_bias = self.create_parameter(
+                shape=([dim]), default_initializer=zeros_)
+            self.v_bias = self.create_parameter(
+                shape=([dim]), default_initializer=zeros_)
+        else:
+            self.q_bias = None
+            self.v_bias = None
+        if window_size:
+            self.window_size = window_size
+            self.num_relative_distance = (2 * window_size[0] - 1) * (
+                2 * window_size[1] - 1) + 3
+            self.relative_position_bias_table = self.create_parameter(
+                shape=(self.num_relative_distance, num_heads),
+                default_initializer=zeros_)  # 2*Wh-1 * 2*Ww-1, nH
+
+            coords_h = paddle.arange(window_size[0])
+            coords_w = paddle.arange(window_size[1])
+            coords = paddle.stack(paddle.meshgrid(
+                [coords_h, coords_w]))  # 2, Wh, Ww
+            coords_flatten = paddle.flatten(coords, 1)  # 2, Wh*Ww 
+            coords_flatten_1 = paddle.unsqueeze(coords_flatten, 2)
+            coords_flatten_2 = paddle.unsqueeze(coords_flatten, 1)
+            relative_coords = coords_flatten_1.clone() - coords_flatten_2.clone(
+            )
+
+            relative_coords = relative_coords.transpose(
+                (1, 2, 0))  # Wh*Ww, Wh*Ww, 2
+            relative_coords[:, :, 0] += window_size[
+                0] - 1  # shift to start from 0
+            relative_coords[:, :, 1] += window_size[1] - 1
+            relative_coords[:, :, 0] *= 2 * window_size[1] - 1
+            relative_position_index = \
+                paddle.zeros(shape=(window_size[0] * window_size[1] + 1, ) * 2, dtype=relative_coords.dtype)
+            relative_position_index[1:, 1:] = relative_coords.sum(
+                -1)  # Wh*Ww, Wh*Ww
+            relative_position_index[0, 0:] = self.num_relative_distance - 3
+            relative_position_index[0:, 0] = self.num_relative_distance - 2
+            relative_position_index[0, 0] = self.num_relative_distance - 1
+
+            self.register_buffer("relative_position_index",
+                                 relative_position_index)
+        else:
+            self.window_size = None
+            self.relative_position_bias_table = None
+            self.relative_position_index = None
+
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+
+    def forward(self, x, rel_pos_bias=None):
+        x_shape = paddle.shape(x)
+        N, C = x_shape[1], x_shape[2]
+
+        qkv_bias = None
+        if self.q_bias is not None:
+            qkv_bias = paddle.concat(
+                (self.q_bias, paddle.zeros_like(self.v_bias), self.v_bias))
+        qkv = F.linear(x, weight=self.qkv.weight, bias=qkv_bias)
+
+        qkv = qkv.reshape((-1, N, 3, self.num_heads,
+                           C // self.num_heads)).transpose((2, 0, 3, 1, 4))
+        q, k, v = qkv[0], qkv[1], qkv[2]
+        attn = (q.matmul(k.transpose((0, 1, 3, 2)))) * self.scale
+
+        if self.relative_position_bias_table is not None:
+            relative_position_bias = self.relative_position_bias_table[
+                self.relative_position_index.reshape([-1])].reshape([
+                    self.window_size[0] * self.window_size[1] + 1,
+                    self.window_size[0] * self.window_size[1] + 1, -1
+                ])  # Wh*Ww,Wh*Ww,nH
+            relative_position_bias = relative_position_bias.transpose(
+                (2, 0, 1))  # nH, Wh*Ww, Wh*Ww
+            attn = attn + relative_position_bias.unsqueeze(0)
+        if rel_pos_bias is not None:
+            attn = attn + rel_pos_bias
+
+        attn = nn.functional.softmax(attn, axis=-1)
+        attn = self.attn_drop(attn)
+
+        x = (attn.matmul(v)).transpose((0, 2, 1, 3)).reshape((-1, N, C))
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+
+class Block(nn.Layer):
+    def __init__(self,
+                 dim,
+                 num_heads,
+                 mlp_ratio=4.,
+                 qkv_bias=False,
+                 qk_scale=None,
+                 drop=0.,
+                 attn_drop=0.,
+                 drop_path=0.,
+                 window_size=None,
+                 init_values=None,
+                 act_layer=nn.GELU,
+                 norm_layer='nn.LayerNorm',
+                 epsilon=1e-5):
+        super().__init__()
+        self.norm1 = nn.LayerNorm(dim, epsilon=1e-6)
+        self.attn = Attention(
+            dim,
+            num_heads=num_heads,
+            qkv_bias=qkv_bias,
+            qk_scale=qk_scale,
+            attn_drop=attn_drop,
+            proj_drop=drop,
+            window_size=window_size)
+        # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here
+        self.drop_path = DropPath(drop_path) if drop_path > 0. else Identity()
+        self.norm2 = eval(norm_layer)(dim, epsilon=epsilon)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        self.mlp = Mlp(in_features=dim,
+                       hidden_features=mlp_hidden_dim,
+                       act_layer=act_layer,
+                       drop=drop)
+
+        if init_values is not None:
+            self.gamma_1 = self.create_parameter(
+                shape=([dim]), default_initializer=Constant(value=init_values))
+            self.gamma_2 = self.create_parameter(
+                shape=([dim]), default_initializer=Constant(value=init_values))
+        else:
+            self.gamma_1, self.gamma_2 = None, None
+
+    def forward(self, x, rel_pos_bias=None):
+        if self.gamma_1 is None:
+            x = x + self.drop_path(
+                self.attn(
+                    self.norm1(x), rel_pos_bias=rel_pos_bias))
+            x = x + self.drop_path(self.mlp(self.norm2(x)))
+        else:
+            x = x + self.drop_path(self.gamma_1 * self.attn(
+                self.norm1(x), rel_pos_bias=rel_pos_bias))
+            x = x + self.drop_path(self.gamma_2 * self.mlp(self.norm2(x)))
+        return x
+
+
+class PatchEmbed(nn.Layer):
+    """ Image to Patch Embedding
+    """
+
+    def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768):
+        super().__init__()
+        self.img_size = to_2tuple(img_size)
+        self.patch_size = to_2tuple(patch_size)
+        self.proj = nn.Conv2D(
+            in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
+
+    @property
+    def num_patches_in_h(self):
+        return self.img_size[1] // self.patch_size[1]
+
+    @property
+    def num_patches_in_w(self):
+        return self.img_size[0] // self.patch_size[0]
+
+    @property
+    def patch_shape(self):
+        return (self.img_size[0] // self.patch_size[0],
+                self.img_size[1] // self.patch_size[1])
+
+    def forward(self, x):
+        x = self.proj(x)
+        return x
+
+
+class RelativePositionBias(nn.Layer):
+    def __init__(self, window_size, num_heads):
+        super().__init__()
+        self.window_size = window_size
+        self.num_relative_distance = (2 * window_size[0] - 1) * (
+            2 * window_size[1] - 1) + 3
+        self.relative_position_bias_table = self.create_parameter(
+            shape=(self.num_relative_distance, num_heads),
+            default_initialize=zeros_)
+
+        # get pair-wise relative position index for each token inside the window
+        coords_h = paddle.arange(window_size[0])
+        coords_w = paddle.arange(window_size[1])
+        coords = paddle.stack(paddle.meshgrid(
+            [coords_h, coords_w]))  # 2, Wh, Ww
+        coords_flatten = coords.flatten(1)  # 2, Wh*Ww
+
+        relative_coords = coords_flatten[:, :,
+                                         None] - coords_flatten[:,
+                                                                None, :]  # 2, Wh*Ww, Wh*Ww
+        relative_coords = relative_coords.transpos(
+            (1, 2, 0))  # Wh*Ww, Wh*Ww, 2 
+        relative_coords[:, :, 0] += window_size[0] - 1  # shift to start from 0
+        relative_coords[:, :, 1] += window_size[1] - 1
+        relative_coords[:, :, 0] *= 2 * window_size[1] - 1
+        relative_position_index = \
+            paddle.zeros(size=(window_size[0] * window_size[1] + 1,) * 2, dtype=relative_coords.dtype)
+        relative_position_index[1:, 1:] = relative_coords.sum(
+            -1)  # Wh*Ww, Wh*Ww
+        relative_position_index[0, 0:] = self.num_relative_distance - 3
+        relative_position_index[0:, 0] = self.num_relative_distance - 2
+        relative_position_index[0, 0] = self.num_relative_distance - 1
+        self.register_buffer("relative_position_index", relative_position_index)
+
+    def forward(self):
+        relative_position_bias = \
+            self.relative_position_bias_table[self.relative_position_index.view(-1)].view(
+                 self.window_size[0] * self.window_size[1] + 1,
+                 self.window_size[0] * self.window_size[1] + 1, -1)  # Wh*Ww,Wh*Ww,nH 
+        return relative_position_bias.transpose((2, 0, 1))  # nH, Wh*Ww, Wh*Ww
+
+
+def get_sinusoid_encoding_table(n_position, d_hid, token=False):
+    ''' Sinusoid position encoding table '''
+
+    def get_position_angle_vec(position):
+        return [
+            position / np.power(10000, 2 * (hid_j // 2) / d_hid)
+            for hid_j in range(d_hid)
+        ]
+
+    sinusoid_table = np.array(
+        [get_position_angle_vec(pos_i) for pos_i in range(n_position)])
+    sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2])  # dim 2i
+    sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2])  # dim 2i+1
+    if token:
+        sinusoid_table = np.concatenate(
+            [sinusoid_table, np.zeros([1, d_hid])], dim=0)
+    return Tensor(sinusoid_table, dtype=float32).unsqueeze(0)
+
+
+@manager.BACKBONES.add_component
+class CAE(nn.Layer):
+    """
+    The Context Autoencoder for Self-Supervised Representation Learning implemetation based on PaddlePaddle
+
+    The original article refers to Chen, Xiaokang, Mingyu Ding, Xiaodi Wang, Ying Xin, Shentong Mo, Yunhao Wang, Shumin Han, Ping Luo, Gang Zeng, and Jingdong Wang. "Context autoencoder for self-supervised representation learning." arXiv preprint arXiv:2202.03026 (2022).
+    (https://arxiv.org/abs/2202.03026)
+
+    Args:
+        img_size (int): Input image size for training the pretrained model, used in absolute postion embedding. Default: 224.
+        patch_size (int | tuple(int)): Patch size. Default: 4.
+        in_chans (int): Number of input image channels. Default: 3.
+        embed_dim (int): Number of linear projection output channels. Default: 96.
+        depths (tuple[int]): Depths of each Swin Transformer stage.
+        num_heads (tuple[int]): Number of attention head of each stage.
+        window_size (int): Window size. Default: 7.
+        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4.
+        qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float): Override default qk scale of head_dim ** -0.5 if set.
+        drop_rate (float): Dropout rate. Default: 0.
+        attn_drop_rate (float): Attention dropout rate. Default: 0.
+        drop_path_rate (float): Stochastic depth rate. Default: 0.0.
+        norm_layer (nn.Layer): Normalization layer. Default: nn.LayerNorm.
+        init_values(float): The initial value of dropout in the block. Default: None.
+        use_rel_pos_bias(bool): Whether to use relative position bias. Default: False.
+        use_shared_rel_pos_bias(bool): Whether to use relative position bias. Default: False.
+        epsilon(float): Epsilon in first norm of block. Default: 1e-5.
+        pretrained (str, optional): The path or url of pretrained model. Default: None.
+    """
+
+    def __init__(self,
+                 img_size=224,
+                 patch_size=16,
+                 in_chans=3,
+                 embed_dim=768,
+                 depth=12,
+                 num_heads=12,
+                 mlp_ratio=4,
+                 qkv_bias=False,
+                 qk_scale=None,
+                 drop_rate=0.,
+                 attn_drop_rate=0.,
+                 drop_path_rate=0.,
+                 norm_layer='nn.LayerNorm',
+                 init_values=None,
+                 use_rel_pos_bias=False,
+                 use_shared_rel_pos_bias=False,
+                 epsilon=1e-5,
+                 pretrained=None,
+                 **args):
+        super().__init__()
+        self.img_size = img_size
+        self.embed_dim = embed_dim
+
+        self.patch_embed = PatchEmbed(
+            img_size=img_size,
+            patch_size=patch_size,
+            in_chans=in_chans,
+            embed_dim=embed_dim)
+        self.pos_w = self.patch_embed.num_patches_in_w
+        self.pos_h = self.patch_embed.num_patches_in_h
+
+        self.pos_embed = self.create_parameter(
+            shape=(1, self.pos_w * self.pos_h + 1, embed_dim),
+            default_initializer=paddle.nn.initializer.TruncatedNormal(std=.02))
+        self.cls_token = self.create_parameter(
+            shape=(1, 1, embed_dim),
+            default_initializer=paddle.nn.initializer.Constant(value=0.))
+        self.pos_drop = nn.Dropout(p=drop_rate)
+
+        if use_shared_rel_pos_bias:
+            self.rel_pos_bias = RelativePositionBias(
+                window_size=self.patch_embed.patch_shape, num_heads=num_heads)
+        else:
+            self.rel_pos_bias = None
+        self.use_rel_pos_bias = use_rel_pos_bias
+
+        dpr = np.linspace(0, drop_path_rate, depth)
+
+        self.blocks = nn.LayerList([
+            Block(
+                dim=embed_dim,
+                num_heads=num_heads,
+                mlp_ratio=mlp_ratio,
+                qkv_bias=qkv_bias,
+                qk_scale=qk_scale,
+                drop=drop_rate,
+                attn_drop=attn_drop_rate,
+                drop_path=dpr[i],
+                norm_layer=norm_layer,
+                init_values=init_values,
+                window_size=self.patch_embed.patch_shape
+                if use_rel_pos_bias else None,
+                epsilon=epsilon) for i in range(depth)
+        ])
+
+        self.pretrained = pretrained
+        self.init_weight()
+
+    def init_weight(self):
+        if self.pretrained:
+            utils.load_pretrained_model(self, self.pretrained)
+
+        model_path = self.pretrained
+        if not os.path.exists(model_path):
+            model_path = utils.download_pretrained_model(model_path)
+
+        load_state_dict = paddle.load(model_path)
+        model_state_dict = self.state_dict()
+        pos_embed_name = "pos_embed"
+        if pos_embed_name in load_state_dict.keys():
+            load_pos_embed = paddle.to_tensor(
+                load_state_dict[pos_embed_name], dtype="float32")
+            if self.pos_embed.shape != load_pos_embed.shape:
+                pos_size = int(math.sqrt(load_pos_embed.shape[1] - 1))
+                model_state_dict[pos_embed_name] = self.resize_pos_embed(
+                    load_pos_embed, (pos_size, pos_size),
+                    (self.pos_h, self.pos_w))
+                self.set_dict(model_state_dict)
+                logger.info("Load pos_embed and resize it from {} to {} .".
+                            format(load_pos_embed.shape, self.pos_embed.shape))
+
+    def resize_pos_embed(self, pos_embed, old_hw, new_hw):
+        """
+        Resize pos_embed weight.
+        Args:
+            pos_embed (Tensor): the pos_embed weight
+            old_hw (list[int]): the height and width of old pos_embed
+            new_hw (list[int]): the height and width of new pos_embed
+        Returns:
+            Tensor: the resized pos_embed weight
+        """
+        cls_pos_embed = pos_embed[:, :1, :]
+        pos_embed = pos_embed[:, 1:, :]
+
+        pos_embed = pos_embed.transpose([0, 2, 1])
+        pos_embed = pos_embed.reshape([1, -1, old_hw[0], old_hw[1]])
+        pos_embed = F.interpolate(
+            pos_embed, new_hw, mode='bicubic', align_corners=False)
+        pos_embed = pos_embed.flatten(2).transpose([0, 2, 1])
+        pos_embed = paddle.concat([cls_pos_embed, pos_embed], axis=1)
+
+        return pos_embed
+
+    def forward(self, x):
+        x = self.patch_embed(x)
+        x_shape = paddle.shape(x)  # b * c * h * w
+
+        cls_tokens = self.cls_token.expand((x_shape[0], -1, -1))
+        x = x.flatten(2).transpose([0, 2, 1])  # b * hw * c
+        x = paddle.concat([cls_tokens, x], axis=1)
+        if paddle.shape(x)[1] == self.pos_embed.shape[1]:
+            x = x + self.pos_embed
+        else:
+            x = x + self.resize_pos_embed(self.pos_embed,
+                                          (self.pos_h, self.pos_w), x_shape[2:])
+        x = self.pos_drop(x)
+
+        rel_pos_bias = self.rel_pos_bias(
+        ) if self.rel_pos_bias is not None else None
+
+        res = []
+        for idx, blk in enumerate(self.blocks):
+            x = blk(x, rel_pos_bias)
+            res.append(x[:, 1:, :])
+        return res, x_shape
+
+    def get_num_layers(self):
+        return len(self.blocks)
+
+    def no_weight_decay(self):
+        return {'pos_embed', 'cls_token'}
+
+
+@manager.BACKBONES.add_component
+def CAE_small_patch16_224(**kwargs):
+    model = VisionTransformer(
+        patch_size=16,
+        embed_dim=768,
+        depth=8,
+        num_heads=8,
+        mlp_ratio=3,
+        qk_scale=768**-0.5,
+        **kwargs)
+    return model
+
+
+@manager.BACKBONES.add_component
+def CAE_base_patch16_224(**kwargs):
+    model = VisionTransformer(
+        patch_size=16,
+        embed_dim=768,
+        depth=12,
+        num_heads=12,
+        mlp_ratio=4,
+        qkv_bias=True,
+        epsilon=1e-6,
+        **kwargs)
+    return model
+
+
+@manager.BACKBONES.add_component
+def CAE_base_patch16_384(**kwargs):
+    model = VisionTransformer(
+        img_size=384,
+        patch_size=16,
+        embed_dim=768,
+        depth=12,
+        num_heads=12,
+        mlp_ratio=4,
+        qkv_bias=True,
+        epsilon=1e-6,
+        **kwargs)
+    return model
+
+
+@manager.BACKBONES.add_component
+def CAE_base_patch32_384(**kwargs):
+    model = VisionTransformer(
+        img_size=384,
+        patch_size=32,
+        embed_dim=768,
+        depth=12,
+        num_heads=12,
+        mlp_ratio=4,
+        qkv_bias=True,
+        epsilon=1e-6,
+        **kwargs)
+    return model
+
+
+@manager.BACKBONES.add_component
+def CAE_large_patch16_224(**kwargs):
+    model = VisionTransformer(
+        patch_size=16,
+        embed_dim=1024,
+        depth=24,
+        num_heads=16,
+        mlp_ratio=4,
+        qkv_bias=True,
+        epsilon=1e-6,
+        **kwargs)
+    return model
+
+
+@manager.BACKBONES.add_component
+def CAE_large_patch16_384(**kwargs):
+    model = VisionTransformer(
+        img_size=384,
+        patch_size=16,
+        embed_dim=1024,
+        depth=24,
+        num_heads=16,
+        mlp_ratio=4,
+        qkv_bias=True,
+        epsilon=1e-6,
+        **kwargs)
+    return model
+
+
+@manager.BACKBONES.add_component
+def CAE_large_patch32_384(**kwargs):
+    model = VisionTransformer(
+        img_size=384,
+        patch_size=32,
+        embed_dim=1024,
+        depth=24,
+        num_heads=16,
+        mlp_ratio=4,
+        qkv_bias=True,
+        epsilon=1e-6,
+        **kwargs)
+    return model
+
+
+@manager.BACKBONES.add_component
+def CAE_huge_patch16_224(**kwargs):
+    model = VisionTransformer(
+        patch_size=16,
+        embed_dim=1280,
+        depth=32,
+        num_heads=16,
+        mlp_ratio=4,
+        **kwargs)
+    return model
+
+
+@manager.BACKBONES.add_component
+def CAE_huge_patch32_384(**kwargs):
+    model = VisionTransformer(
+        img_size=384,
+        patch_size=32,
+        embed_dim=1280,
+        depth=32,
+        num_heads=16,
+        mlp_ratio=4,
+        **kwargs)
+    return model

paddlers/models/paddleseg/models/backbones/hrformer.py

@@ -0,0 +1,1078 @@
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+
+from paddlers.models.paddleseg.cvlibs import manager, param_init
+from paddlers.models.paddleseg.models import layers
+from paddlers.models.paddleseg.utils import utils, logger
+from paddlers.models.paddleseg.models.backbones.transformer_utils import *
+
+
+class PadHelper:
+    """ Make the size of feature map divisible by local group size."""
+
+    def __init__(self, local_group_size=7):
+        self.lgs = local_group_size
+        if not isinstance(self.lgs, (tuple, list)):
+            self.lgs = to_2tuple(self.lgs)
+        assert len(self.lgs) == 2, "The length of self.lgs must be 2."
+
+    def pad_if_needed(self, x, size):
+        n, h, w, c = size
+        pad_h = paddle.cast(
+            paddle.ceil(h / self.lgs[0]) * self.lgs[0] - h, paddle.int32)
+        pad_w = paddle.cast(
+            paddle.ceil(w / self.lgs[1]) * self.lgs[1] - w, paddle.int32)
+        if pad_h > 0 or pad_w > 0:  # center-pad the feature on H and W axes
+            return F.pad(x,
+                         paddle.to_tensor(
+                             [
+                                 pad_w // 2, pad_w - pad_w // 2, pad_h // 2,
+                                 pad_h - pad_h // 2
+                             ],
+                             dtype='int32').reshape([-1]),
+                         data_format='NHWC')
+        return x
+
+    def depad_if_needed(self, x, size):
+        n, h, w, c = size
+        pad_h = paddle.cast(
+            paddle.ceil(h / self.lgs[0]) * self.lgs[0] - h,
+            paddle.int32).reshape([-1])
+        pad_w = paddle.cast(
+            paddle.ceil(w / self.lgs[1]) * self.lgs[1] - w,
+            paddle.int32).reshape([-1])
+        if pad_h > 0 or pad_w > 0:  # remove the center-padding on feature
+            return x[:, pad_h // 2:pad_h // 2 + h, pad_w // 2:pad_w // 2 + w, :]
+        return x
+
+
+class LocalPermuteHelper:
+    """ Permute the feature map to gather pixels in local groups, and then reverse permutation."""
+
+    def __init__(self, local_group_size=7):
+        self.lgs = local_group_size
+        if not isinstance(self.lgs, (tuple, list)):
+            self.lgs = to_2tuple(self.lgs)
+        assert len(self.lgs) == 2, "The length of self.lgs must be 2."
+
+    def permute(self, x, size):
+        n, h, w, c = size
+        qh = h // self.lgs[0]
+        ph = self.lgs[0]
+        qw = w // self.lgs[0]
+        pw = self.lgs[0]
+        c = c
+
+        x = x.reshape([n, qh, ph, qw, pw, c])
+        x = x.transpose([2, 4, 0, 1, 3, 5])
+        x = x.reshape([ph * pw, n * qh * qw, c])
+        return x
+
+    def rev_permute(self, x, size):
+        n, h, w, c = size
+        x = x.reshape([
+            self.lgs[0], self.lgs[0], n, h // self.lgs[0], w // self.lgs[0], c
+        ])
+        x = x.transpose([2, 3, 0, 4, 1, 5])
+        x = x.reshape([n, h, w, c])
+
+        return x
+
+
+class Attention(nn.MultiHeadAttention):
+    """ Multihead Attention with extra flags on the q/k/v and out projections."""
+
+    def __init__(self,
+                 *args,
+                 add_zero_attn=None,
+                 rpe=False,
+                 window_size=7,
+                 **kwargs):
+        super(Attention, self).__init__(*args, **kwargs)
+
+        self.add_zero_attn = add_zero_attn
+        self.rpe = rpe
+        if rpe:
+            self.window_size = [window_size] * 2
+            # define a parameter table of relative position bias
+            parameter_value = paddle.zeros([(2 * self.window_size[0] - 1) * (
+                2 * self.window_size[1] - 1), self.num_heads])
+            self.relative_position_bias_table = self.create_parameter(
+                shape=parameter_value.shape,
+                dtype=str(parameter_value.numpy().dtype),
+                default_initializer=nn.initializer.Assign(parameter_value))
+
+            # get pair-wise relative position index for each token inside the window
+            coords_h = paddle.arange(self.window_size[0])
+            coords_w = paddle.arange(self.window_size[1])
+            coords = paddle.stack(paddle.meshgrid(
+                [coords_h, coords_w]))  # 2, Wh, Ww
+            coords_flatten = paddle.flatten(coords, 1)  # 2, Wh*Ww
+            relative_coords = (
+                coords_flatten[:, :, None] - coords_flatten[:, None, :]
+            )  # 2, Wh*Ww, Wh*Ww
+            relative_coords = relative_coords.transpose(
+                [1, 2, 0])  # Wh*Ww, Wh*Ww, 2
+            relative_coords[:, :, 0] += self.window_size[
+                0] - 1  # shift to start from 0
+            relative_coords[:, :, 1] += self.window_size[1] - 1
+            relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1
+            relative_position_index = relative_coords.sum(-1)  # Wh*Ww, Wh*Ww
+            self.register_buffer("relative_position_index",
+                                 relative_position_index)
+            trunc_normal_(self.relative_position_bias_table)
+
+    def forward(
+            self,
+            query,
+            key,
+            value,
+            key_padding_mask=None,
+            need_weights=False,
+            attn_mask=None,
+            do_qkv_proj=True,
+            do_out_proj=True,
+            rpe=True, ):
+
+        tgt_len, bsz, embed_dim = query.shape
+        head_dim = embed_dim // self.num_heads
+        v_head_dim = self.vdim // self.num_heads
+        assert (head_dim * self.num_heads == embed_dim
+                ), "embed_dim must be divisible by num_heads"
+        scaling = float(head_dim)**-0.5
+
+        # whether or not use the original query/key/value
+        q = self.q_proj(query) * scaling if do_qkv_proj else query
+        k = self.k_proj(key) if do_qkv_proj else key
+        v = self.v_proj(value) if do_qkv_proj else value
+
+        if attn_mask is not None:
+            dtype_lst = [
+                paddle.float32, paddle.float64, paddle.float16, paddle.uint8,
+                paddle.bool
+            ]
+            assert attn_mask.dtype in dtype_lst, \
+                f"Only float, byte, and bool types are supported for attn_mask, not {attn_mask.dtype}"
+            if attn_mask.dtype == paddle.uint8:
+                msg = "Byte tensor for attn_mask in nn.MultiHeadAttention is deprecated. Use bool tensor instead."
+                logger.warning(msg)
+                attn_mask = attn_mask.to(paddle.bool)
+
+            if attn_mask.dim() == 2:
+                attn_mask = attn_mask.unsqueeze(0)
+                if list(attn_mask.shape) != [1, query.shape[0], key.shape[0]]:
+                    raise RuntimeError(
+                        "The size of the 2D attn_mask is not correct.")
+            elif attn_mask.dim() == 3:
+                if attn_mask.shape != [
+                        bsz * self.num_heads, query.shape[0], key.shape[0]
+                ]:
+
+                    raise RuntimeError(
+                        "The size of the 3D attn_mask is not correct.")
+            else:
+                raise RuntimeError(
+                    f"attn_mask's dimension {attn_mask.dim()} is not supported")
+
+        # convert ByteTensor key_padding_mask to bool
+        if key_padding_mask is not None and key_padding_mask.dtype == paddle.uint8:
+            msg = "Byte tensor for key_padding_mask in nn.MultiHeadAttention is deprecated. Use bool tensor instead."
+            logger.warning(msg)
+            key_padding_mask = key_padding_mask.to(paddle.bool)
+
+        q = q.reshape([tgt_len, bsz * self.num_heads,
+                       head_dim]).transpose([1, 0, 2])
+        if k is not None:
+            k = k.reshape([-1, bsz * self.num_heads,
+                           head_dim]).transpose([1, 0, 2])
+        if v is not None:
+            v = v.reshape([-1, bsz * self.num_heads,
+                           v_head_dim]).transpose([1, 0, 2])
+
+        src_len = k.shape[1]
+
+        if key_padding_mask is not None:
+            assert key_padding_mask.shape[0] == bsz
+            assert key_padding_mask.shape[1] == src_len
+
+        if self.add_zero_attn:
+            src_len += 1
+            k = paddle.concat(
+                [
+                    k, paddle.zeros(
+                        (k.shape[0], 1) + k.shape[2:], dtype=k.dtype)
+                ],
+                axis=1)
+            v = paddle.concat(
+                [
+                    v, paddle.zeros(
+                        (v.shape[0], 1) + v.shape[2:], dtype=v.dtype)
+                ],
+                axis=1)
+
+            if attn_mask is not None:
+                attn_mask = F.pad(attn_mask, (0, 1))
+            if key_padding_mask is not None:
+                key_padding_mask = F.pad(key_padding_mask, (0, 1))
+
+        attn_output_weights = paddle.bmm(q, k.transpose([0, 2, 1]))
+        assert list(attn_output_weights.
+                    shape) == [bsz * self.num_heads, tgt_len, src_len]
+        """ Add relative position embedding."""
+        if self.rpe and rpe:
+            # NOTE: for simplicity, we assume src_len == tgt_len == window_size**2 here
+            assert (
+                src_len == self.window_size[0] * self.window_size[1] and
+                tgt_len == self.window_size[0] * self.window_size[1]
+            ), f"src{src_len}, tgt{tgt_len}, window{self.window_size[0]}"
+            relative_position_bias = self.relative_position_bias_table[
+                self.relative_position_index.reshape([-1])].reshape([
+                    self.window_size[0] * self.window_size[1],
+                    self.window_size[0] * self.window_size[1], -1
+                ])  # Wh*Ww,Wh*Ww,nH
+            relative_position_bias = relative_position_bias.transpose(
+                [2, 0, 1])  # nH, Wh*Ww, Wh*Ww
+            attn_output_weights = attn_output_weights.reshape([
+                bsz, self.num_heads, tgt_len, src_len
+            ]) + relative_position_bias.unsqueeze(0)
+            attn_output_weights = attn_output_weights.reshape(
+                [bsz * self.num_heads, tgt_len, src_len])
+        # Attention weight for the invalid region is -inf.
+        if attn_mask is not None:
+            if attn_mask.dtype == paddle.bool:
+                attn_output_weights.masked_fill_(attn_mask, float("-inf"))
+            else:
+                attn_output_weights += attn_mask
+
+        if key_padding_mask is not None:
+            attn_output_weights = attn_output_weights.reshape(
+                [bsz, self.num_heads, tgt_len, src_len])
+            attn_output_weights = attn_output_weights.masked_fill(
+                key_padding_mask.unsqueeze(1).unsqueeze(2),
+                float("-inf"), )
+            attn_output_weights = attn_output_weights.reshape(
+                [bsz * self.num_heads, tgt_len, src_len])
+
+        attn_output_weights = F.softmax(attn_output_weights, axis=-1)
+        attn_output_weights = F.dropout(
+            attn_output_weights, p=self.dropout, training=self.training)
+
+        attn_output = paddle.bmm(attn_output_weights, v)
+        assert list(
+            attn_output.shape) == [bsz * self.num_heads, tgt_len, v_head_dim]
+        attn_output = (
+            attn_output.transpose([1, 0, 2]).reshape([tgt_len, bsz, self.vdim]))
+        if do_out_proj:
+            attn_output = F.linear(attn_output, self.out_proj.weight,
+                                   self.out_proj.bias)
+
+        if need_weights:
+            # average attention weights over heads
+            attn_output_weights = attn_output_weights.reshape(
+                [bsz, self.num_heads, tgt_len, src_len])
+            return attn_output, q, k, attn_output_weights.sum(
+                axis=1) / self.num_heads
+        else:
+            return attn_output, q, k  # additionaly return the query and key
+
+
+class InterlacedPoolAttention(nn.Layer):
+    """ Interlaced sparse multi-head self attention module with relative position bias.
+    Args:
+        embed_dim (int): Number of input channels.
+        num_heads (int): Number of attention heads.
+        window_size (int, optional): Window size. Default: 7.
+        rpe (bool, optional): Whether to use rpe. Default: True.
+    """
+
+    def __init__(self, embed_dim, num_heads, window_size=7, rpe=True, **kwargs):
+        super(InterlacedPoolAttention, self).__init__()
+
+        self.dim = embed_dim
+        self.num_heads = num_heads
+        self.window_size = window_size
+        self.with_rpe = rpe
+        self.attn = Attention(
+            embed_dim, num_heads, rpe=rpe, window_size=window_size, **kwargs)
+        self.pad_helper = PadHelper(window_size)
+        self.permute_helper = LocalPermuteHelper(window_size)
+
+    def forward(self, x, H, W, **kwargs):
+        B, N, C = x.shape
+        x = x.reshape([B, H, W, C])
+        # pad
+        x_pad = self.pad_helper.pad_if_needed(x, paddle.shape(x))
+        # permute
+        x_permute = self.permute_helper.permute(x_pad, x_pad.shape)
+        # attention
+        out, _, _ = self.attn(
+            x_permute, x_permute, x_permute, rpe=self.with_rpe, **kwargs)
+        # reverse permutation
+        out = self.permute_helper.rev_permute(out, x_pad.shape)
+        out = self.pad_helper.depad_if_needed(out, paddle.shape(x))
+        return out.reshape([B, N, C])
+
+
+class MlpDWBN(nn.Layer):
+    def __init__(self,
+                 in_features,
+                 hidden_features=None,
+                 out_features=None,
+                 act_layer=nn.GELU,
+                 dw_act_layer=nn.GELU):
+        super(MlpDWBN, self).__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+
+        self.act1 = act_layer()
+        self.fc1 = layers.ConvBN(in_features, hidden_features, kernel_size=1)
+        self.act2 = dw_act_layer()
+        self.dw3x3 = layers.ConvBN(
+            hidden_features,
+            hidden_features,
+            kernel_size=3,
+            stride=1,
+            groups=hidden_features,
+            padding=1)
+
+        self.fc2 = layers.ConvBN(hidden_features, out_features, kernel_size=1)
+        self.act3 = act_layer()
+
+    def forward(self, x, H, W):
+        if x.dim() == 3:
+            B, N, C = x.shape
+            if N == (H * W + 1):
+                cls_tokens = x[:, 0, :]
+                x_ = x[:, 1:, :].transpose([0, 2, 1]).reshape([B, C, H, W])
+            else:
+                x_ = x.transpose([0, 2, 1]).reshape([B, C, H, W])
+
+            x_ = self.fc1(x_)
+            x_ = self.act1(x_)
+            x_ = self.dw3x3(x_)
+            x_ = self.act2(x_)
+            x_ = self.fc2(x_)
+            x_ = self.act3(x_)
+            x_ = x_.reshape([B, C, -1]).transpose([0, 2, 1])
+            if N == (H * W + 1):
+                x = paddle.concat((cls_tokens.unsqueeze(1), x_), axis=1)
+            else:
+                x = x_
+            return x
+
+        elif x.dim() == 4:
+            x = self.fc1(x)
+            x = self.act1(x)
+            x = self.dw3x3(x)
+            x = self.act2(x)
+            x = self.fc2(x)
+            x = self.act3(x)
+            return x
+
+        else:
+            raise RuntimeError(f"Unsupported input shape: {x.shape}")
+
+
+class Bottleneck(nn.Layer):
+    expansion = 4
+
+    def __init__(self, inplanes, planes, stride=1, downsample=None):
+        super(Bottleneck, self).__init__()
+        self.conv1 = layers.ConvBNReLU(
+            inplanes, planes, kernel_size=1, bias_attr=False)
+        self.conv2 = layers.ConvBNReLU(
+            planes,
+            planes,
+            kernel_size=3,
+            stride=stride,
+            padding=1,
+            bias_attr=False)
+        self.conv3 = layers.ConvBN(
+            planes, planes * self.expansion, kernel_size=1, bias_attr=False)
+        self.relu = nn.ReLU()
+        self.downsample = downsample
+        self.stride = stride
+
+    def forward(self, x):
+        residual = x
+        out = self.conv1(x)
+        out = self.conv2(out)
+        out = self.conv3(out)
+
+        if self.downsample is not None:
+            residual = self.downsample(x)
+
+        out += residual
+        out = self.relu(out)
+
+        return out
+
+
+class GeneralTransformerBlock(nn.Layer):
+    expansion = 1
+
+    def __init__(self,
+                 inplanes,
+                 planes,
+                 num_heads,
+                 window_size=7,
+                 mlp_ratio=4.0,
+                 drop=0.0,
+                 attn_drop=0.0,
+                 drop_path=0.0,
+                 act_layer=nn.GELU,
+                 norm_layer=nn.LayerNorm):
+        super(GeneralTransformerBlock, self).__init__()
+        self.dim = inplanes
+        self.out_dim = planes
+        self.num_heads = num_heads
+        self.window_size = window_size
+        self.mlp_ratio = mlp_ratio
+        self.attn = InterlacedPoolAttention(
+            self.dim,
+            num_heads=num_heads,
+            window_size=window_size,
+            rpe=True,
+            dropout=attn_drop, )
+
+        self.norm1 = norm_layer(self.dim, epsilon=1e-6)
+        self.norm2 = norm_layer(self.dim, epsilon=1e-6)
+        self.drop_path = DropPath(
+            drop_path) if drop_path > 0.0 else nn.Identity()
+        mlp_hidden_dim = int(self.dim * mlp_ratio)
+
+        self.mlp = MlpDWBN(
+            in_features=self.dim,
+            hidden_features=mlp_hidden_dim,
+            out_features=self.out_dim,
+            act_layer=act_layer,
+            dw_act_layer=act_layer)
+
+    def forward(self, x):
+        B, C, H, W = x.shape
+        # reshape
+        x = x.reshape([B, C, -1]).transpose([0, 2, 1])
+        # Attention
+        x = x + self.drop_path(self.attn(self.norm1(x), H, W))
+        # FFN
+        x = x + self.drop_path(self.mlp(self.norm2(x), H, W))
+        # reshape
+        x = x.transpose([0, 2, 1]).reshape([B, C, H, W])
+        return x
+
+
+class HighResolutionTransformerModule(nn.Layer):
+    def __init__(
+            self,
+            num_branches,
+            blocks,
+            num_blocks,
+            num_inchannels,
+            num_channels,
+            num_heads,
+            num_window_sizes,
+            num_mlp_ratios,
+            multi_scale_output=True,
+            drop_path=0.0, ):
+        super(HighResolutionTransformerModule, self).__init__()
+
+        self._check_branches(num_branches, num_blocks, num_inchannels,
+                             num_channels)
+
+        self.num_inchannels = num_inchannels
+        self.num_branches = num_branches
+
+        self.multi_scale_output = multi_scale_output
+        self.branches = self._make_branches(
+            num_branches,
+            blocks,
+            num_blocks,
+            num_channels,
+            num_heads,
+            num_window_sizes,
+            num_mlp_ratios,
+            drop_path, )
+        self.fuse_layers = self._make_fuse_layers()
+        self.relu = nn.ReLU()
+
+        self.num_heads = num_heads
+        self.num_window_sizes = num_window_sizes
+        self.num_mlp_ratios = num_mlp_ratios
+
+    def _check_branches(self, num_branches, num_blocks, num_inchannels,
+                        num_channels):
+
+        if num_branches != len(num_blocks):
+            error_msg = f"Num_branches {num_branches} is not equal\
+                to the length of num_blocks {len(num_blocks)}"
+
+            raise ValueError(error_msg)
+
+        if num_branches != len(num_channels):
+            error_msg = f"Num_branches {num_branches} is not equal\
+                to the length of num_channels {len(num_channels)}"
+
+            raise ValueError(error_msg)
+
+        if num_branches != len(num_inchannels):
+            error_msg = f"Num_branches {num_branches} is not equal\
+                to the length of num_inchannels {len(num_inchannels)}"
+
+            raise ValueError(error_msg)
+
+    def _make_one_branch(self, branch_index, block, num_blocks, num_channels,
+                         num_heads, num_window_sizes, num_mlp_ratios,
+                         drop_paths):
+
+        layers = []
+        layers.append(
+            block(
+                self.num_inchannels[branch_index],
+                num_channels[branch_index],
+                num_heads=num_heads[branch_index],
+                window_size=num_window_sizes[branch_index],
+                mlp_ratio=num_mlp_ratios[branch_index],
+                drop_path=drop_paths[0], ))
+
+        self.num_inchannels[branch_index] = num_channels[
+            branch_index] * block.expansion
+        for i in range(1, num_blocks[branch_index]):
+            layers.append(
+                block(
+                    self.num_inchannels[branch_index],
+                    num_channels[branch_index],
+                    num_heads=num_heads[branch_index],
+                    window_size=num_window_sizes[branch_index],
+                    mlp_ratio=num_mlp_ratios[branch_index],
+                    drop_path=drop_paths[i], ))
+        return nn.Sequential(*layers)
+
+    def _make_branches(
+            self,
+            num_branches,
+            block,
+            num_blocks,
+            num_channels,
+            num_heads,
+            num_window_sizes,
+            num_mlp_ratios,
+            drop_paths, ):
+        branches = []
+
+        for i in range(num_branches):
+            branches.append(
+                self._make_one_branch(
+                    i,
+                    block,
+                    num_blocks,
+                    num_channels,
+                    num_heads,
+                    num_window_sizes,
+                    num_mlp_ratios,
+                    drop_paths=[_ * (2**i) for _ in drop_paths]))
+
+        return nn.LayerList(branches)
+
+    def _make_fuse_layers(self):
+        if self.num_branches == 1:
+            return None
+        num_branches = self.num_branches
+        num_inchannels = self.num_inchannels
+        fuse_layers = []
+        for i in range(num_branches if self.multi_scale_output else 1):
+            fuse_layer = []
+            for j in range(num_branches):
+                if j > i:
+                    fuse_layer.append(
+                        nn.Sequential(
+                            layers.ConvBN(
+                                num_inchannels[j],
+                                num_inchannels[i],
+                                kernel_size=1,
+                                stride=1,
+                                bias_attr=False, ),
+                            nn.Upsample(
+                                scale_factor=2**(j - i), mode="nearest")))
+                elif j == i:
+                    fuse_layer.append(None)
+                else:
+                    conv3x3s = []
+                    for k in range(i - j):
+                        if k == i - j - 1:
+                            num_outchannels_conv3x3 = num_inchannels[i]
+                            conv3x3s.append(
+                                nn.Sequential(
+                                    layers.ConvBN(
+                                        num_inchannels[j],
+                                        num_inchannels[j],
+                                        kernel_size=3,
+                                        stride=2,
+                                        padding=1,
+                                        groups=num_inchannels[j],
+                                        bias_attr=False),
+                                    layers.ConvBN(
+                                        num_inchannels[j],
+                                        num_outchannels_conv3x3,
+                                        kernel_size=1,
+                                        stride=1,
+                                        bias_attr=False)))
+                        else:
+                            num_outchannels_conv3x3 = num_inchannels[j]
+                            conv3x3s.append(
+                                nn.Sequential(
+                                    layers.ConvBN(
+                                        num_inchannels[j],
+                                        num_inchannels[j],
+                                        kernel_size=3,
+                                        stride=2,
+                                        padding=1,
+                                        groups=num_inchannels[j],
+                                        bias_attr=False),
+                                    layers.ConvBNReLU(
+                                        num_inchannels[j],
+                                        num_outchannels_conv3x3,
+                                        kernel_size=1,
+                                        stride=1,
+                                        bias_attr=False)))
+                    fuse_layer.append(nn.Sequential(*conv3x3s))
+            fuse_layers.append(nn.LayerList(fuse_layer))
+
+        return nn.LayerList(fuse_layers)
+
+    def forward(self, x):
+        if self.num_branches == 1:
+            return [self.branches[0](x[0])]
+
+        for i in range(self.num_branches):
+            x[i] = self.branches[i](x[i])
+
+        x_fuse = []
+        for i in range(len(self.fuse_layers)):
+            y = x[0] if i == 0 else self.fuse_layers[i][0](x[0])
+            for j in range(1, self.num_branches):
+                if i == j:
+                    y = y + x[j]
+                elif j > i:
+                    width_output = x[i].shape[-1]
+                    height_output = x[i].shape[-2]
+                    y = y + F.interpolate(
+                        self.fuse_layers[i][j](x[j]),
+                        size=[height_output, width_output],
+                        mode="bilinear",
+                        align_corners=True, )
+                else:
+                    y = y + self.fuse_layers[i][j](x[j])
+            x_fuse.append(self.relu(y))
+
+        return x_fuse
+
+
+class HighResolutionTransformer(nn.Layer):
+    """
+    The HRFormer implementation based on PaddlePaddle.
+
+    The original article refers to
+    Jingdong Wang, et, al. "HRNet：Deep High-Resolution Representation Learning for Visual Recognition"
+    (https://arxiv.org/pdf/1908.07919.pdf).
+
+    Args:
+        drop_path_rate (float, optional): The rate of Drop Path. Default: 0.2.
+        stage1_num_blocks (list[int], optional): Number of blocks per module for stage1. Default: [2].
+        stage1_num_channels (list[int], optional): Number of channels per branch for stage1. Default: [64].
+        stage2_num_modules (int, optional): Number of modules for stage2. Default: 1.
+        stage2_num_branches (int, optional): Number of branches for stage2. Default: 2.
+        stage2_num_blocks (list[int], optional): Number of blocks per module for stage2. Default: [2, 2].
+        stage2_num_channels (list[int], optional): Number of channels per branch for stage2. Default: [32, 64].
+        stage2_num_heads (list[int], optional): Number of heads per multi head attetion for stage2. Default: [1, 2].
+        stage2_num_mlp_ratios (list[int], optional): Number of ratio of mlp per multi head attetion for stage2. Default: [4, 4].
+        stage2_num_window_sizes (list[int], optional): Number of window sizes for stage2. Default: [7, 7].
+        stage3_num_modules (int, optional): Number of modules for stage3. Default: 4.
+        stage3_num_branches (int, optional): Number of branches for stage3. Default: 3.
+        stage3_num_blocks (list[int], optional): Number of blocks per module for stage3. Default: [2, 2, 2].
+        stage3_num_channels (list[int], optional): Number of channels per branch for stage3. Default: [32, 64, 128].
+        stage3_num_heads (list[int], optional): Number of heads per multi head attetion for stage3. Default: [1, 2, 4].
+        stage3_num_mlp_ratios (list[int], optional): Number of ratio of mlp per multi head attetion for stage3. Default: [4, 4, 4].
+        stage3_num_window_sizes (list[int], optional): Number of window sizes for stage3. Default: [7, 7, 7].
+        stage4_num_modules (int, optional): Number of modules for stage4. Default: 2.
+        stage4_num_branches (int, optional): Number of branches for stage4. Default: 4.
+        stage4_num_blocks (list[int], optional): Number of blocks per module for stage4. Default: [2, 2, 2, 2].
+        stage4_num_channels (list[int], optional): Number of channels per branch for stage4. Default: [32, 64, 128, 256].
+        stage4_num_heads (list[int], optional): Number of heads per multi head attetion for stage4. Default: [1, 2, 4, 8].
+        stage4_num_mlp_ratios (list[int], optional): Number of ratio of mlp per multi head attetion for stage4. Default: [4, 4, 4, 4].
+        stage4_num_window_sizes (list[int], optional): Number of window sizes for stage4. Default: [7, 7, 7, 7].
+        in_channels (int, optional): The channels of input image. Default: 3.
+        pretrained (str, optional): The path of pretrained model. Default: None.
+    """
+
+    def __init__(self,
+                 drop_path_rate=0.2,
+                 stage1_num_blocks=[2],
+                 stage1_num_channels=[64],
+                 stage2_num_modules=1,
+                 stage2_num_branches=2,
+                 stage2_num_blocks=[2, 2],
+                 stage2_num_channels=[32, 64],
+                 stage2_num_heads=[1, 2],
+                 stage2_num_mlp_ratios=[4, 4],
+                 stage2_num_window_sizes=[7, 7],
+                 stage3_num_modules=4,
+                 stage3_num_branches=3,
+                 stage3_num_blocks=[2, 2, 2],
+                 stage3_num_channels=[32, 64, 128],
+                 stage3_num_heads=[1, 2, 4],
+                 stage3_num_mlp_ratios=[4, 4, 4],
+                 stage3_num_window_sizes=[7, 7, 7],
+                 stage4_num_modules=2,
+                 stage4_num_branches=4,
+                 stage4_num_blocks=[2, 2, 2, 2],
+                 stage4_num_channels=[32, 64, 128, 256],
+                 stage4_num_heads=[1, 2, 4, 8],
+                 stage4_num_mlp_ratios=[4, 4, 4, 4],
+                 stage4_num_window_sizes=[7, 7, 7, 7],
+                 in_channels=3,
+                 pretrained=None):
+        super(HighResolutionTransformer, self).__init__()
+
+        self.pretrained = pretrained
+        self.drop_path_rate = drop_path_rate
+        self.stage1_num_blocks = stage1_num_blocks
+        self.stage1_num_channels = stage1_num_channels
+        self.stage2_num_modules = stage2_num_modules
+        self.stage2_num_branches = stage2_num_branches
+        self.stage2_num_blocks = stage2_num_blocks
+        self.stage2_num_channels = stage2_num_channels
+        self.stage2_num_heads = stage2_num_heads
+        self.stage2_num_mlp_ratios = stage2_num_mlp_ratios
+        self.stage2_num_window_sizes = stage2_num_window_sizes
+        self.stage3_num_modules = stage3_num_modules
+        self.stage3_num_branches = stage3_num_branches
+        self.stage3_num_blocks = stage3_num_blocks
+        self.stage3_num_channels = stage3_num_channels
+        self.stage3_num_heads = stage3_num_heads
+        self.stage3_num_mlp_ratios = stage3_num_mlp_ratios
+        self.stage3_num_window_sizes = stage3_num_window_sizes
+        self.stage4_num_modules = stage4_num_modules
+        self.stage4_num_branches = stage4_num_branches
+        self.stage4_num_blocks = stage4_num_blocks
+        self.stage4_num_channels = stage4_num_channels
+        self.stage4_num_heads = stage4_num_heads
+        self.stage4_num_mlp_ratios = stage4_num_mlp_ratios
+        self.stage4_num_window_sizes = stage4_num_window_sizes
+
+        self.conv1 = layers.ConvBNReLU(
+            in_channels,
+            64,
+            kernel_size=3,
+            stride=2,
+            padding=1,
+            bias_attr=False)
+
+        self.conv2 = layers.ConvBNReLU(
+            64, 64, kernel_size=3, stride=2, padding=1, bias_attr=False)
+
+        self.feat_channels = [sum(self.stage4_num_channels)]
+
+        depth_s2 = self.stage2_num_blocks[0] * self.stage2_num_modules
+        depth_s3 = self.stage3_num_blocks[0] * self.stage3_num_modules
+        depth_s4 = self.stage4_num_blocks[0] * self.stage4_num_modules
+        depths = [depth_s2, depth_s3, depth_s4]
+        drop_path_rate = self.drop_path_rate
+
+        dpr = [
+            x.item() for x in paddle.linspace(0, drop_path_rate, sum(depths))
+        ]
+
+        num_channels = self.stage1_num_channels[0]
+        block = Bottleneck
+        num_blocks = self.stage1_num_blocks[0]
+        self.layer1 = self._make_layer(block, 64, num_channels, num_blocks)
+        stage1_out_channel = block.expansion * num_channels
+
+        num_channels = self.stage2_num_channels
+        block = GeneralTransformerBlock
+        num_channels = [
+            num_channels[i] * block.expansion
+            for i in range(len(num_channels))
+        ]
+        self.transition1 = self._make_transition_layer([stage1_out_channel],
+                                                       num_channels)
+        self.stage2, pre_stage_channels = self._make_stage(
+            block=block,
+            num_modules=self.stage2_num_modules,
+            num_branches=self.stage2_num_branches,
+            num_blocks=self.stage2_num_blocks,
+            num_channels=self.stage2_num_channels,
+            num_heads=self.stage2_num_heads,
+            num_window_sizes=self.stage2_num_window_sizes,
+            num_mlp_ratios=self.stage2_num_mlp_ratios,
+            num_inchannels=num_channels,
+            drop_path=dpr[0:depth_s2])
+
+        num_channels = self.stage3_num_channels
+        block = GeneralTransformerBlock
+        num_channels = [
+            num_channels[i] * block.expansion
+            for i in range(len(num_channels))
+        ]
+        self.transition2 = self._make_transition_layer(pre_stage_channels,
+                                                       num_channels)
+        self.stage3, pre_stage_channels = self._make_stage(
+            block=block,
+            num_modules=self.stage3_num_modules,
+            num_branches=self.stage3_num_branches,
+            num_blocks=self.stage3_num_blocks,
+            num_channels=self.stage3_num_channels,
+            num_heads=self.stage3_num_heads,
+            num_window_sizes=self.stage3_num_window_sizes,
+            num_mlp_ratios=self.stage3_num_mlp_ratios,
+            num_inchannels=num_channels,
+            drop_path=dpr[depth_s2:depth_s2 + depth_s3])
+
+        num_channels = self.stage4_num_channels
+        block = GeneralTransformerBlock
+        num_channels = [
+            num_channels[i] * block.expansion
+            for i in range(len(num_channels))
+        ]
+        self.transition3 = self._make_transition_layer(pre_stage_channels,
+                                                       num_channels)
+        self.stage4, pre_stage_channels = self._make_stage(
+            block=block,
+            num_modules=self.stage4_num_modules,
+            num_branches=self.stage4_num_branches,
+            num_blocks=self.stage4_num_blocks,
+            num_channels=self.stage4_num_channels,
+            num_heads=self.stage4_num_heads,
+            num_window_sizes=self.stage4_num_window_sizes,
+            num_mlp_ratios=self.stage4_num_mlp_ratios,
+            num_inchannels=num_channels,
+            multi_scale_output=True,
+            drop_path=dpr[depth_s2 + depth_s3:])
+
+        self.init_weight()
+
+    def _make_transition_layer(self, num_channels_pre_layer,
+                               num_channels_cur_layer):
+        num_branches_cur = len(num_channels_cur_layer)
+        num_branches_pre = len(num_channels_pre_layer)
+
+        transition_layers = []
+        for i in range(num_branches_cur):
+            if i < num_branches_pre:
+                if num_channels_cur_layer[i] != num_channels_pre_layer[i]:
+                    transition_layers.append(
+                        nn.Sequential(
+                            layers.ConvBNReLU(
+                                num_channels_pre_layer[i],
+                                num_channels_cur_layer[i],
+                                kernel_size=3,
+                                stride=1,
+                                padding=1,
+                                bias_attr=False)))
+                else:
+                    transition_layers.append(None)
+            else:
+                conv3x3s = []
+                for j in range(i + 1 - num_branches_pre):
+                    inchannels = num_channels_pre_layer[-1]
+                    outchannels = (num_channels_cur_layer[i]
+                                   if j == i - num_branches_pre else inchannels)
+                    conv3x3s.append(
+                        nn.Sequential(
+                            layers.ConvBNReLU(
+                                inchannels,
+                                outchannels,
+                                kernel_size=3,
+                                stride=2,
+                                padding=1,
+                                bias_attr=False)))
+                transition_layers.append(nn.Sequential(*conv3x3s))
+
+        return nn.LayerList(transition_layers)
+
+    def _make_layer(self,
+                    block,
+                    inplanes,
+                    planes,
+                    blocks,
+                    num_heads=1,
+                    stride=1,
+                    window_size=7,
+                    mlp_ratio=4.0):
+        downsample = None
+        if stride != 1 or inplanes != planes * block.expansion:
+            downsample = nn.Sequential(
+                layers.ConvBN(
+                    inplanes,
+                    planes * block.expansion,
+                    kernel_size=1,
+                    stride=stride,
+                    bias_attr=False))
+        modules = []
+
+        if isinstance(block, GeneralTransformerBlock):
+            modules.append(
+                block(
+                    inplanes,
+                    planes,
+                    num_heads,
+                    window_size,
+                    mlp_ratio, ))
+        else:
+            modules.append(block(inplanes, planes, stride, downsample))
+
+        inplanes = planes * block.expansion
+        for _ in range(1, blocks):
+            modules.append(block(inplanes, planes))
+
+        return nn.Sequential(*modules)
+
+    def _make_stage(self,
+                    block,
+                    num_modules,
+                    num_branches,
+                    num_blocks,
+                    num_channels,
+                    num_heads,
+                    num_window_sizes,
+                    num_mlp_ratios,
+                    num_inchannels,
+                    multi_scale_output=True,
+                    drop_path=0.0):
+        modules = []
+        for i in range(num_modules):
+            # multi_scale_output is only used last module
+            if not multi_scale_output and i == num_modules - 1:
+                reset_multi_scale_output = False
+            else:
+                reset_multi_scale_output = True
+
+            modules.append(
+                HighResolutionTransformerModule(
+                    num_branches,
+                    block,
+                    num_blocks,
+                    num_inchannels,
+                    num_channels,
+                    num_heads,
+                    num_window_sizes,
+                    num_mlp_ratios,
+                    reset_multi_scale_output,
+                    drop_path=drop_path[num_blocks[0] * i:num_blocks[0] * (
+                        i + 1)], ))
+            num_inchannels = modules[-1].num_inchannels
+
+        return nn.Sequential(*modules), num_inchannels
+
+    def init_weight(self):
+        for layer in self.sublayers():
+            if isinstance(layer, nn.Conv2D):
+                param_init.normal_init(layer.weight, std=0.001)
+            elif isinstance(layer, (nn.BatchNorm, nn.SyncBatchNorm)):
+                param_init.constant_init(layer.weight, value=1.0)
+                param_init.constant_init(layer.bias, value=0.0)
+            elif isinstance(layer, nn.Linear):
+                trunc_normal_(layer.weight)
+                if layer.bias is not None:
+                    zeros_(layer.bias)
+            elif isinstance(layer, nn.LayerNorm):
+                zeros_(layer.bias)
+                ones_(layer.weight)
+
+        if self.pretrained is not None:
+            utils.load_pretrained_model(self, self.pretrained)
+
+    def forward(self, x):
+        x = self.conv1(x)
+        x = self.conv2(x)
+        x = self.layer1(x)
+
+        x_list = []
+        for i in range(self.stage2_num_branches):
+            if self.transition1[i] is not None:
+                x_list.append(self.transition1[i](x))
+            else:
+                x_list.append(x)
+        y_list = self.stage2(x_list)
+
+        x_list = []
+        for i in range(self.stage3_num_branches):
+            if self.transition2[i] is not None:
+                x_list.append(self.transition2[i](y_list[-1]))
+            else:
+                x_list.append(y_list[i])
+        y_list = self.stage3(x_list)
+
+        x_list = []
+        for i in range(self.stage4_num_branches):
+            if self.transition3[i] is not None:
+                x_list.append(self.transition3[i](y_list[-1]))
+            else:
+                x_list.append(y_list[i])
+        y_list = self.stage4(x_list)
+        _, _, h, w = y_list[0].shape
+        feat1 = y_list[0]
+        feat2 = F.interpolate(
+            y_list[1], size=(h, w), mode="bilinear", align_corners=True)
+        feat3 = F.interpolate(
+            y_list[2], size=(h, w), mode="bilinear", align_corners=True)
+        feat4 = F.interpolate(
+            y_list[3], size=(h, w), mode="bilinear", align_corners=True)
+
+        feats = paddle.concat([feat1, feat2, feat3, feat4], axis=1)
+
+        return [feats]
+
+
+@manager.BACKBONES.add_component
+def HRFormer_small(**kwargs):
+    arch_net = HighResolutionTransformer(**kwargs)
+
+    return arch_net
+
+
+@manager.BACKBONES.add_component
+def HRFormer_base(**kwargs):
+    arch_net = HighResolutionTransformer(
+        stage2_num_channels=[78, 156],
+        stage2_num_heads=[2, 4],
+        stage3_num_channels=[78, 156, 312],
+        stage3_num_heads=[2, 4, 8],
+        stage4_num_channels=[78, 156, 312, 624],
+        stage4_num_heads=[2, 4, 8, 16],
+        **kwargs)
+
+    return arch_net
+
+
+@manager.BACKBONES.add_component
+def HRFormer_base_win_13(**kwargs):
+    arch_net = HighResolutionTransformer(
+        stage2_num_channels=[78, 156],
+        stage2_num_heads=[2, 4],
+        stage2_num_window_sizes=[13, 13],
+        stage3_num_channels=[78, 156, 312],
+        stage3_num_heads=[2, 4, 8],
+        stage4_num_channels=[78, 156, 312, 624],
+        stage4_num_heads=[2, 4, 8, 16],
+        **kwargs)
+
+    return arch_net
+
+
+@manager.BACKBONES.add_component
+def HRFormer_base_win_15(**kwargs):
+    arch_net = HighResolutionTransformer(
+        stage2_num_channels=[78, 156],
+        stage2_num_heads=[2, 4],
+        stage2_num_window_sizes=[15, 15],
+        stage3_num_channels=[78, 156, 312],
+        stage3_num_heads=[2, 4, 8],
+        stage3_num_window_sizes=[15, 15, 15],
+        stage4_num_channels=[78, 156, 312, 624],
+        stage4_num_heads=[2, 4, 8, 16],
+        stage4_num_window_sizes=[15, 15, 15, 15],
+        **kwargs)
+
+    return arch_net

paddlers/models/paddleseg/models/backbones/mobilenetv3.py

@@ -171,7 +171,7 @@ class MobileNetV3(nn.Layer):
             num_groups=1,
             if_act=True,
             act="hardswish")
-        self.blocks = nn.Sequential(*[
+        self.blocks = nn.Sequential(* [
             ResidualUnit(
                 in_c=_make_divisible(inplanes * self.scale if i == 0 else
                                      self.cfg[i - 1][2] * self.scale),

paddlers/models/paddleseg/models/backbones/mscan.py

@@ -0,0 +1,383 @@
+# copyright (c) 2023 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+
+import numpy as np
+import paddle
+import paddle.nn as nn
+from paddle.nn import Conv2D
+from paddle.nn.initializer import Assign, Normal
+
+from paddlers.models.paddleseg.cvlibs import manager
+from paddlers.models.paddleseg.models.backbones.transformer_utils import (DropPath, ones_,
+                                                          to_2tuple, zeros_)
+from paddlers.models.paddleseg.models.layers import SyncBatchNorm
+from paddlers.models.paddleseg.utils import utils
+
+__all__ = ["MSCAN", "MSCAN_T", "MSCAN_S", "MSCAN_B", "MSCAN_L"]
+
+
+def get_depthwise_conv(dim, kernel_size=3):
+    if isinstance(kernel_size, int):
+        kernel_size = to_2tuple(kernel_size)
+    padding = tuple([k // 2 for k in kernel_size])
+    return Conv2D(
+        dim, dim, kernel_size, padding=padding, bias_attr=True, groups=dim)
+
+
+class Mlp(nn.Layer):
+    """Multilayer perceptron."""
+
+    def __init__(self,
+                 in_features,
+                 hidden_features=None,
+                 out_features=None,
+                 act_layer=nn.GELU,
+                 drop=0.0):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = nn.Conv2D(in_features, hidden_features, 1)
+        self.dwconv = get_depthwise_conv(hidden_features)
+        self.act = act_layer()
+        self.fc2 = nn.Conv2D(hidden_features, out_features, 1)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.dwconv(x)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x
+
+
+class StemConv(nn.Layer):
+    def __init__(
+            self,
+            in_channels,
+            out_channels, ):
+        super().__init__()
+        self.proj = nn.Sequential(
+            nn.Conv2D(
+                in_channels,
+                out_channels // 2,
+                kernel_size=3,
+                stride=2,
+                padding=1),
+            SyncBatchNorm(out_channels // 2),
+            nn.GELU(),
+            nn.Conv2D(
+                out_channels // 2,
+                out_channels,
+                kernel_size=3,
+                stride=2,
+                padding=1),
+            SyncBatchNorm(out_channels))
+
+    def forward(self, x):
+        x = self.proj(x)
+        H, W = x.shape[2:]
+        x = x.flatten(2).transpose([0, 2, 1])
+        return x, H, W
+
+
+class AttentionModule(nn.Layer):
+    """
+    AttentionModule Layer, which contains some depth-wise strip convolutions.
+
+    Args:
+        dim (int): Number of input channels.
+        kernel_sizes (list[int], optional): The height or width of each strip convolution kernel. Default: [7, 11, 21].
+    """
+
+    def __init__(self, dim, kernel_sizes=[7, 11, 21]):
+        super().__init__()
+        self.conv0 = nn.Conv2D(dim, dim, 5, padding=2, groups=dim)
+
+        self.dwconvs = nn.LayerList([
+            nn.Sequential((f"conv{i+1}_1", get_depthwise_conv(dim, (1, k))),
+                          (f"conv{i+1}_2", get_depthwise_conv(dim, (k, 1))))
+            for i, k in enumerate(kernel_sizes)
+        ])
+
+        self.conv_out = nn.Conv2D(dim, dim, 1)
+
+    def forward(self, x):
+        u = paddle.clone(x)
+        attn = self.conv0(x)
+
+        attns = [m(attn) for m in self.dwconvs]
+
+        attn += sum(attns)
+
+        attn = self.conv_out(attn)
+
+        return attn * u
+
+
+class SpatialAttention(nn.Layer):
+    """
+    SpatialAttention Layer.
+
+    Args:
+        d_model (int): Number of input channels.
+        atten_kernel_sizes (list[int], optional): The height or width of each strip convolution kernel in attention module.
+            Default: [7, 11, 21].
+    """
+
+    def __init__(self, d_model, atten_kernel_sizes=[7, 11, 21]):
+        super().__init__()
+        self.d_model = d_model
+        self.proj_1 = nn.Conv2D(d_model, d_model, 1)
+        self.activation = nn.GELU()
+        self.spatial_gating_unit = AttentionModule(d_model, atten_kernel_sizes)
+        self.proj_2 = nn.Conv2D(d_model, d_model, 1)
+
+    def forward(self, x):
+        shorcut = paddle.clone(x)
+        x = self.proj_1(x)
+        x = self.activation(x)
+        x = self.spatial_gating_unit(x)
+        x = self.proj_2(x)
+        x = x + shorcut
+        return x
+
+
+class Block(nn.Layer):
+    """
+    MSCAN Block.
+
+    Args:
+        dim (int): Number of feature channels.
+        atten_kernel_sizes (list[int], optional): The height or width of each strip convolution kernel in attention module.
+            Default: [7, 11, 21].
+        mlp_ratio (float, optional): Ratio of mlp hidden dim to embedding dim. Default: 4.0.
+        drop (float, optional): Dropout rate. Default: 0.0
+        attn_drop (float, optional): Attention dropout rate. Default: 0.0.
+        drop_path (float, optional): Stochastic depth rate. Default: 0.0.
+        act_layer (nn.Layer, optional): Activation layer. Default: nn.GELU.
+    """
+
+    def __init__(
+            self,
+            dim,
+            atten_kernel_sizes=[7, 11, 21],
+            mlp_ratio=4.0,
+            drop=0.0,
+            drop_path=0.0,
+            act_layer=nn.GELU, ):
+        super().__init__()
+        self.norm1 = SyncBatchNorm(dim)
+        self.attn = SpatialAttention(dim, atten_kernel_sizes)
+        self.drop_path = DropPath(
+            drop_path) if drop_path > 0.0 else nn.Identity()
+        self.norm2 = SyncBatchNorm(dim)
+        self.mlp = Mlp(in_features=dim,
+                       hidden_features=int(dim * mlp_ratio),
+                       act_layer=act_layer,
+                       drop=drop)
+
+        layer_scale_init_value = paddle.full(
+            [dim, 1, 1], fill_value=1e-2, dtype="float32")
+        self.layer_scale_1 = paddle.create_parameter(
+            [dim, 1, 1], "float32", attr=Assign(layer_scale_init_value))
+        self.layer_scale_2 = paddle.create_parameter(
+            [dim, 1, 1], "float32", attr=Assign(layer_scale_init_value))
+
+    def forward(self, x, H, W):
+        B, N, C = x.shape
+        x = x.transpose([0, 2, 1]).reshape([B, C, H, W])
+        x = x + self.drop_path(self.layer_scale_1 * self.attn(self.norm1(x)))
+        x = x + self.drop_path(self.layer_scale_2 * self.mlp(self.norm2(x)))
+        x = x.reshape([B, C, N]).transpose([0, 2, 1])
+        return x
+
+
+class OverlapPatchEmbed(nn.Layer):
+    """
+    An Opverlaping Image to Patch Embedding Layer.
+
+    Args:
+        patch_size (int, optional): Patch token size. Default: 7.
+        stride (int, optional): Stride of Convolution in OverlapPatchEmbed. Default: 4.
+        in_chans (int, optional): Number of input image channels. Default: 3.
+        embed_dim (int, optional): Number of linear projection output channels. Default: 768.
+    """
+
+    def __init__(self, patch_size=7, stride=4, in_chans=3, embed_dim=768):
+        super().__init__()
+        patch_size = to_2tuple(patch_size)
+
+        self.proj = nn.Conv2D(
+            in_chans,
+            embed_dim,
+            kernel_size=patch_size,
+            stride=stride,
+            padding=(patch_size[0] // 2, patch_size[1] // 2))
+        self.norm = SyncBatchNorm(embed_dim)
+
+    def forward(self, x):
+        x = self.proj(x)
+        H, W = x.shape[2:]
+        x = self.norm(x)
+
+        x = x.flatten(2).transpose([0, 2, 1])
+        return x, H, W
+
+
+def _check_length(*args):
+    target_length = len(args[0])
+    for item in args:
+        if target_length != len(item):
+            return False
+    return True
+
+
+@manager.BACKBONES.add_component
+class MSCAN(nn.Layer):
+    """
+    The MSCAN implementation based on PaddlePaddle.
+
+    The original article refers to
+    Guo, Meng-Hao, et al. "SegNeXt: Rethinking Convolutional Attention Design for Semantic Segmentation"
+    (https://arxiv.org/pdf/2209.08575.pdf)
+
+    Args:
+        in_channels (int, optional): Number of input image channels. Default: 3.
+        embed_dims (list[int], optional): Number of each stage output channels. Default: [32, 64, 160, 256].
+        depths (list[int], optional): Depths of each MSCAN stage.
+        atten_kernel_sizes (list[int], optional): The height or width of each strip convolution kernel in attention module.
+            Default: [7, 11, 21].
+        mlp_ratio (float, optional): Ratio of mlp hidden dim to embedding dim. Default: 4.
+        drop_rate (float, optional): Dropout rate. Default: 0.0.
+        drop_path_rate (float, optional): Stochastic depth rate. Default: 0.1.
+        pretrained (str, optional): The path or url of pretrained model. Default: None.
+    """
+
+    def __init__(self,
+                 in_channels=3,
+                 embed_dims=[32, 64, 160, 256],
+                 depths=[3, 3, 5, 2],
+                 mlp_ratios=[8, 8, 4, 4],
+                 atten_kernel_sizes=[7, 11, 21],
+                 drop_rate=0.0,
+                 drop_path_rate=0.1,
+                 pretrained=None):
+        super().__init__()
+        if not _check_length(embed_dims, mlp_ratios, depths):
+            raise ValueError(
+                "The length of aurgments 'embed_dims', 'mlp_ratios' and 'drop_path_rate' must be same."
+            )
+
+        self.depths = depths
+        self.num_stages = len(embed_dims)
+        self.feat_channels = embed_dims
+
+        drop_path_rates = [
+            x for x in np.linspace(0, drop_path_rate, sum(depths))
+        ]  # stochastic depth decay rule
+        cur = 0
+
+        for i in range(self.num_stages):
+            if i == 0:
+                patch_embed = StemConv(in_channels, embed_dims[0])
+            else:
+                patch_embed = OverlapPatchEmbed(
+                    patch_size=3,
+                    stride=2,
+                    in_chans=embed_dims[i - 1],
+                    embed_dim=embed_dims[i])
+
+            block = nn.LayerList([
+                Block(
+                    dim=embed_dims[i],
+                    atten_kernel_sizes=atten_kernel_sizes,
+                    mlp_ratio=mlp_ratios[i],
+                    drop=drop_rate,
+                    drop_path=drop_path_rates[cur + j])
+                for j in range(depths[i])
+            ])
+            norm = nn.LayerNorm(embed_dims[i])
+            cur += depths[i]
+
+            setattr(self, f"patch_embed{i + 1}", patch_embed)
+            setattr(self, f"block{i + 1}", block)
+            setattr(self, f"norm{i + 1}", norm)
+
+        self.init_weights(pretrained)
+
+    def init_weights(self, pretrained=None):
+        if pretrained is not None:
+            utils.load_pretrained_model(self, pretrained)
+        else:
+            for sublayer in self.sublayers():
+                if isinstance(sublayer, nn.LayerNorm):
+                    zeros_(sublayer.bias)
+                    ones_(sublayer.weight)
+                elif isinstance(sublayer, nn.Conv2D):
+                    fan_out = (sublayer._kernel_size[0] *
+                               sublayer._kernel_size[1] *
+                               sublayer._out_channels)
+                    fan_out //= sublayer._groups
+                    initializer = Normal(mean=0, std=math.sqrt(2.0 / fan_out))
+                    initializer(sublayer.weight)
+                    zeros_(sublayer.bias)
+
+    def forward(self, x):
+        B = x.shape[0]
+        outs = []
+
+        for i in range(self.num_stages):
+            patch_embed = getattr(self, f"patch_embed{i + 1}")
+            block = getattr(self, f"block{i + 1}")
+            norm = getattr(self, f"norm{i + 1}")
+            x, H, W = patch_embed(x)
+            for blk in block:
+                x = blk(x, H, W)
+            x = norm(x)
+            x = x.reshape([B, H, W, -1]).transpose([0, 3, 1, 2])
+            outs.append(x)
+
+        return outs
+
+
+@manager.BACKBONES.add_component
+def MSCAN_T(**kwargs):
+    return MSCAN(**kwargs)
+
+
+@manager.BACKBONES.add_component
+def MSCAN_S(**kwargs):
+    return MSCAN(embed_dims=[64, 128, 320, 512], depths=[2, 2, 4, 2], **kwargs)
+
+
+@manager.BACKBONES.add_component
+def MSCAN_B(**kwargs):
+    return MSCAN(
+        embed_dims=[64, 128, 320, 512],
+        depths=[3, 3, 12, 3],
+        drop_path_rate=0.1,
+        **kwargs)
+
+
+@manager.BACKBONES.add_component
+def MSCAN_L(**kwargs):
+    return MSCAN(
+        embed_dims=[64, 128, 320, 512],
+        depths=[3, 5, 27, 3],
+        drop_path_rate=0.3,
+        **kwargs)

paddlers/models/paddleseg/models/backbones/resnet_vd.py

@@ -297,7 +297,7 @@ class ResNet_vd(nn.Layer):
                         block] if dilation_dict and block in dilation_dict else 1
 
                     # Actually block here is 'stage', and i is 'block' in 'stage'
-                    # At the stage 4, expand the the dilation_rate if given multi_grid
+                    # At the stage 4, expand the dilation_rate if given multi_grid
                     if block == 3:
                         dilation_rate = dilation_rate * multi_grid[i]
                     ###############################################################################

paddlers/models/paddleseg/models/backbones/strideformer.py

@@ -0,0 +1,808 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This file refers to https://github.com/hustvl/TopFormer and https://github.com/fudan-zvg/SeaFormer
+"""
+
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+from paddle import regularizer
+
+from paddlers.models.paddleseg.cvlibs import manager
+from paddlers.models.paddleseg import utils
+from paddlers.models.paddleseg.models.backbones.transformer_utils import DropPath
+from paddlers.models.paddleseg.models.backbones.mobilenetv3 import _make_divisible, _create_act, Hardsigmoid
+from paddlers.models.paddleseg.models.layers import layer_libs
+
+
+class StrideFormer(nn.Layer):
+    def __init__(self,
+                 cfgs,
+                 channels,
+                 embed_dims,
+                 key_dims=[16, 24],
+                 depths=[2, 2],
+                 num_heads=8,
+                 attn_ratios=2,
+                 mlp_ratios=[2, 4],
+                 drop_path_rate=0.1,
+                 act_layer=nn.Sigmoid,
+                 inj_type='AAM',
+                 out_channels=256,
+                 dims=(128, 160),
+                 out_feat_chs=None,
+                 stride_attention=True,
+                 in_channels=3,
+                 pretrained=None):
+        """
+        The StrideFormer implementation based on PaddlePaddle.
+
+        The original article refers to:
+
+        Args:
+            cfgs(list): Each sublist describe the config for a MobileNetV3 block.
+            channels(list): The input channels for each MobileNetV3 block.
+            embed_dims(list): The channels of the features input to the sea attention block.  
+            key_dims(list, optional): The embeding dims for each head in attention.
+            depths(list, optional): describes the depth of the attention block. i,e: M,N.
+            num_heads(int, optional): The number of heads of the attention blocks.
+            attn_ratios(int, optional): The exapend ratio of V.
+            mlp_ratios(list, optional): The ratio of mlp blocks.
+            drop_path_rate(float, optional): The drop path rate in attention block.
+            act_layer(nn.Layer, optional): The activation layer of AAM.
+            inj_type(string, optional): The type of injection/AAM.
+            out_channels(int, optional): The output channels of the AAM.
+            dims(list, optional): The dimension of the fusion block.
+            out_feat_chs(list, optional): The input channels of the AAM.
+            stride_attention(bool, optioal): whether to stride attention in each attention layer.
+            in_channels (int, optional): The channels of input image. Default: 3.
+            pretrained(str, optional): the path of pretrained model.
+        """
+        super().__init__()
+        self.depths = depths
+        self.cfgs = cfgs
+        self.dims = dims
+
+        for i in range(len(cfgs)):
+            smb = StackedMV3Block(
+                cfgs=cfgs[i],
+                stem=True if i == 0 else False,
+                inp_channel=channels[i])
+            setattr(self, f'smb{i+1}', smb)
+
+        for i in range(len(depths)):
+            dpr = [
+                x.item() for x in paddle.linspace(0, drop_path_rate, depths[i])
+            ]
+            trans = BasicLayer(
+                block_num=depths[i],
+                embedding_dim=embed_dims[i],
+                key_dim=key_dims[i],
+                num_heads=num_heads,
+                mlp_ratio=mlp_ratios[i],
+                attn_ratio=attn_ratios,
+                drop=0,
+                attn_drop=0.0,
+                drop_path=dpr,
+                act_layer=act_layer,
+                stride_attention=stride_attention)
+            setattr(self, f"trans{i+1}", trans)
+
+        self.inj_type = inj_type
+        if self.inj_type == "AAM":
+            self.inj_module = InjectionMultiSumallmultiallsum(
+                in_channels=out_feat_chs, out_channels=out_channels)
+            self.feat_channels = [out_channels, ]
+        elif self.inj_type == "AAMSx8":
+            self.inj_module = InjectionMultiSumallmultiallsumSimpx8(
+                in_channels=out_feat_chs, out_channels=out_channels)
+            self.feat_channels = [out_channels, ]
+        elif self.inj_type == 'origin':
+            for i in range(len(dims)):
+                fuse = Fusion_block(
+                    out_feat_chs[0] if i == 0 else dims[i - 1],
+                    out_feat_chs[i + 1],
+                    embed_dim=dims[i])
+                setattr(self, f"fuse{i + 1}", fuse)
+            self.feat_channels = [dims[i], ]
+        else:
+            raise NotImplementedError(self.inj_module + ' is not implemented')
+
+        self.pretrained = pretrained
+        self.init_weight()
+
+    def init_weight(self):
+        if self.pretrained is not None:
+            utils.load_entire_model(self, self.pretrained)
+
+    def forward(self, x):
+        outputs = []
+        num_smb_stage = len(self.cfgs)
+        num_trans_stage = len(self.depths)
+
+        for i in range(num_smb_stage):
+            smb = getattr(self, f"smb{i + 1}")
+            x = smb(x)
+
+            # 1/8 shared feat
+            if i == 1:
+                outputs.append(x)
+            if num_trans_stage + i >= num_smb_stage:
+                trans = getattr(
+                    self, f"trans{i + num_trans_stage - num_smb_stage + 1}")
+                x = trans(x)
+                outputs.append(x)
+
+        if self.inj_type == "origin":
+            x_detail = outputs[0]
+            for i in range(len(self.dims)):
+                fuse = getattr(self, f'fuse{i+1}')
+
+                x_detail = fuse(x_detail, outputs[i + 1])
+            output = x_detail
+        else:
+            output = self.inj_module(outputs)
+
+        return output
+
+
+class HSigmoid(nn.Layer):
+    def __init__(self):
+        super().__init__()
+        self.relu = nn.ReLU6()
+
+    def forward(self, x):
+        return self.relu(x + 3) / 6
+
+
+class ConvBNAct(nn.Layer):
+    def __init__(self,
+                 in_channels,
+                 out_channels,
+                 kernel_size=1,
+                 stride=1,
+                 padding=0,
+                 groups=1,
+                 norm=nn.BatchNorm2D,
+                 act=None,
+                 bias_attr=False):
+        super(ConvBNAct, self).__init__()
+        self.conv = nn.Conv2D(
+            in_channels=in_channels,
+            out_channels=out_channels,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+            groups=groups,
+            bias_attr=None if bias_attr else False)
+        self.act = act() if act is not None else nn.Identity()
+        self.bn = norm(out_channels, bias_attr=None) \
+            if norm is not None else nn.Identity()
+
+    def forward(self, x):
+        x = self.conv(x)
+        x = self.bn(x)
+        x = self.act(x)
+        return x
+
+
+class Conv2DBN(nn.Layer):
+    def __init__(self,
+                 in_channels,
+                 out_channels,
+                 ks=1,
+                 stride=1,
+                 pad=0,
+                 dilation=1,
+                 groups=1,
+                 bn_weight_init=1):
+        super().__init__()
+        self.c = nn.Conv2D(
+            in_channels=in_channels,
+            out_channels=out_channels,
+            kernel_size=ks,
+            stride=stride,
+            padding=pad,
+            dilation=dilation,
+            groups=groups,
+            bias_attr=False)
+        bn_weight_attr = paddle.ParamAttr(
+            initializer=nn.initializer.Constant(bn_weight_init))
+        bn_bias_attr = paddle.ParamAttr(initializer=nn.initializer.Constant(0))
+        self.bn = nn.BatchNorm2D(
+            out_channels, weight_attr=bn_weight_attr, bias_attr=bn_bias_attr)
+
+    def forward(self, inputs):
+        out = self.c(inputs)
+        out = self.bn(out)
+        return out
+
+
+class ConvBNLayer(nn.Layer):
+    def __init__(self,
+                 in_c,
+                 out_c,
+                 filter_size,
+                 stride,
+                 padding,
+                 num_groups=1,
+                 if_act=True,
+                 act=None,
+                 dilation=1):
+        super().__init__()
+
+        self.c = nn.Conv2D(
+            in_channels=in_c,
+            out_channels=out_c,
+            kernel_size=filter_size,
+            stride=stride,
+            padding=padding,
+            groups=num_groups,
+            bias_attr=False,
+            dilation=dilation)
+        self.bn = nn.BatchNorm(
+            num_channels=out_c,
+            act=None,
+            param_attr=paddle.ParamAttr(regularizer=regularizer.L2Decay(0.0)),
+            bias_attr=paddle.ParamAttr(regularizer=regularizer.L2Decay(0.0)))
+        self.if_act = if_act
+        self.act = _create_act(act)
+
+    def forward(self, x):
+        x = self.c(x)
+        x = self.bn(x)
+        if self.if_act:
+            x = self.act(x)
+        return x
+
+
+class ResidualUnit(nn.Layer):
+    def __init__(self,
+                 in_c,
+                 mid_c,
+                 out_c,
+                 filter_size,
+                 stride,
+                 use_se,
+                 act=None,
+                 dilation=1):
+        super().__init__()
+        self.if_shortcut = stride == 1 and in_c == out_c
+        self.if_se = use_se
+
+        self.expand_conv = ConvBNLayer(
+            in_c=in_c,
+            out_c=mid_c,
+            filter_size=1,
+            stride=1,
+            padding=0,
+            if_act=True,
+            act=act)
+        self.bottleneck_conv = ConvBNLayer(
+            in_c=mid_c,
+            out_c=mid_c,
+            filter_size=filter_size,
+            stride=stride,
+            padding=int((filter_size - 1) // 2) * dilation,
+            num_groups=mid_c,
+            if_act=True,
+            act=act,
+            dilation=dilation)
+        if self.if_se:
+            self.mid_se = SEModule(mid_c)
+        self.linear_conv = ConvBNLayer(
+            in_c=mid_c,
+            out_c=out_c,
+            filter_size=1,
+            stride=1,
+            padding=0,
+            if_act=False,
+            act=None)
+
+    def forward(self, x):
+        identity = x
+        x = self.expand_conv(x)
+        x = self.bottleneck_conv(x)
+        if self.if_se:
+            x = self.mid_se(x)
+        x = self.linear_conv(x)
+        if self.if_shortcut:
+            x = paddle.add(identity, x)
+        return x
+
+
+class SEModule(nn.Layer):
+    def __init__(self, channel, reduction=4):
+        super().__init__()
+        self.avg_pool = nn.AdaptiveAvgPool2D(1)
+        self.conv1 = nn.Conv2D(
+            in_channels=channel,
+            out_channels=channel // reduction,
+            kernel_size=1,
+            stride=1,
+            padding=0)
+        self.relu = nn.ReLU()
+        self.conv2 = nn.Conv2D(
+            in_channels=channel // reduction,
+            out_channels=channel,
+            kernel_size=1,
+            stride=1,
+            padding=0)
+        self.hardsigmoid = Hardsigmoid(slope=0.2, offset=0.5)
+
+    def forward(self, x):
+        identity = x
+        x = self.avg_pool(x)
+        x = self.conv1(x)
+        x = self.relu(x)
+        x = self.conv2(x)
+        x = self.hardsigmoid(x)
+        return paddle.multiply(x=identity, y=x)
+
+
+class StackedMV3Block(nn.Layer):
+    """
+    The MobileNetV3 block.
+
+    Args:
+        cfgs (list): The MobileNetV3 config list of a stage.
+        stem (bool): Whether is the first stage or not.
+        in_channels (int, optional): The channels of input image. Default: 3.
+        scale: float=1.0. The coefficient that controls the size of network parameters. 
+    
+    Returns:
+        model: nn.Layer. A stage of specific MobileNetV3 model depends on args.
+    """
+
+    def __init__(self, cfgs, stem, inp_channel, scale=1.0):
+        super().__init__()
+
+        self.scale = scale
+        self.stem = stem
+
+        if self.stem:
+            self.conv = ConvBNLayer(
+                in_c=3,
+                out_c=_make_divisible(inp_channel * self.scale),
+                filter_size=3,
+                stride=2,
+                padding=1,
+                num_groups=1,
+                if_act=True,
+                act="hardswish")
+        self.blocks = nn.LayerList()
+        for i, (k, exp, c, se, act, s) in enumerate(cfgs):
+            self.blocks.append(
+                ResidualUnit(
+                    in_c=_make_divisible(inp_channel * self.scale),
+                    mid_c=_make_divisible(self.scale * exp),
+                    out_c=_make_divisible(self.scale * c),
+                    filter_size=k,
+                    stride=s,
+                    use_se=se,
+                    act=act,
+                    dilation=1))
+            inp_channel = _make_divisible(self.scale * c)
+
+    def forward(self, x):
+        if self.stem:
+            x = self.conv(x)
+
+        for i, block in enumerate(self.blocks):
+            x = block(x)
+
+        return x
+
+
+class SqueezeAxialPositionalEmbedding(nn.Layer):
+    def __init__(self, dim, shape):
+        super().__init__()
+        weight_attr = paddle.ParamAttr(initializer=nn.initializer.Normal())
+        self.pos_embed = paddle.create_parameter(
+            shape=[1, dim, shape], attr=weight_attr, dtype='float32')
+
+    def forward(self, x):
+        B, C, N = x.shape
+        x = x + F.interpolate(
+            self.pos_embed,
+            size=(N, ),
+            mode='linear',
+            align_corners=False,
+            data_format='NCW')
+        return x
+
+
+class Sea_Attention(nn.Layer):
+    def __init__(self,
+                 dim,
+                 key_dim,
+                 num_heads,
+                 attn_ratio=4,
+                 activation=None,
+                 stride_attention=False):
+        super().__init__()
+        self.num_heads = num_heads
+        self.scale = key_dim**-0.5
+        self.nh_kd = nh_kd = key_dim * num_heads
+        self.d = int(attn_ratio * key_dim)
+        self.dh = int(attn_ratio * key_dim) * num_heads
+        self.attn_ratio = attn_ratio
+
+        self.to_q = Conv2DBN(dim, nh_kd, 1)
+        self.to_k = Conv2DBN(dim, nh_kd, 1)
+        self.to_v = Conv2DBN(dim, self.dh, 1)
+
+        self.stride_attention = stride_attention
+
+        if self.stride_attention:
+            self.stride_conv = nn.Sequential(
+                nn.Conv2D(
+                    dim, dim, kernel_size=3, stride=2, padding=1, groups=dim),
+                nn.BatchNorm2D(dim), )
+
+        self.proj = paddle.nn.Sequential(
+            activation(), Conv2DBN(
+                self.dh, dim, bn_weight_init=0))
+        self.proj_encode_row = paddle.nn.Sequential(
+            activation(), Conv2DBN(
+                self.dh, self.dh, bn_weight_init=0))
+        self.pos_emb_rowq = SqueezeAxialPositionalEmbedding(nh_kd, 16)
+        self.pos_emb_rowk = SqueezeAxialPositionalEmbedding(nh_kd, 16)
+        self.proj_encode_column = paddle.nn.Sequential(
+            activation(), Conv2DBN(
+                self.dh, self.dh, bn_weight_init=0))
+        self.pos_emb_columnq = SqueezeAxialPositionalEmbedding(nh_kd, 16)
+        self.pos_emb_columnk = SqueezeAxialPositionalEmbedding(nh_kd, 16)
+
+        self.dwconv = Conv2DBN(
+            2 * self.dh,
+            2 * self.dh,
+            ks=3,
+            stride=1,
+            pad=1,
+            dilation=1,
+            groups=2 * self.dh)
+        self.act = activation()
+        self.pwconv = Conv2DBN(2 * self.dh, dim, ks=1)
+        self.sigmoid = HSigmoid()
+
+    def forward(self, x):
+        B, C, H_ori, W_ori = x.shape
+        if self.stride_attention:
+            x = self.stride_conv(x)
+        B, C, H, W = x.shape
+
+        q = self.to_q(x)  # [B, nhead*dim, H, W]
+        k = self.to_k(x)
+        v = self.to_v(x)
+
+        qkv = paddle.concat([q, k, v], axis=1)
+        qkv = self.act(self.dwconv(qkv))
+        qkv = self.pwconv(qkv)
+
+        qrow = self.pos_emb_rowq(q.mean(-1)).reshape(
+            [B, self.num_heads, -1, H]).transpose(
+                [0, 1, 3, 2])  # [B, nhead, H, dim] 
+        krow = self.pos_emb_rowk(k.mean(-1)).reshape(
+            [B, self.num_heads, -1, H])  # [B, nhead, dim, H]
+        vrow = v.mean(-1).reshape([B, self.num_heads, -1, H]).transpose(
+            [0, 1, 3, 2])  # [B, nhead, H, dim*attn_ratio]
+
+        attn_row = paddle.matmul(qrow, krow) * self.scale  # [B, nhead, H, H]
+        attn_row = F.softmax(attn_row, axis=-1)
+
+        xx_row = paddle.matmul(attn_row, vrow)  # [B, nhead, H, dim*attn_ratio]
+        xx_row = self.proj_encode_row(
+            xx_row.transpose([0, 1, 3, 2]).reshape([B, self.dh, H, 1]))
+
+        ## squeeze column
+        qcolumn = self.pos_emb_columnq(q.mean(-2)).reshape(
+            [B, self.num_heads, -1, W]).transpose([0, 1, 3, 2])
+        kcolumn = self.pos_emb_columnk(k.mean(-2)).reshape(
+            [B, self.num_heads, -1, W])
+        vcolumn = paddle.mean(v, -2).reshape(
+            [B, self.num_heads, -1, W]).transpose([0, 1, 3, 2])
+
+        attn_column = paddle.matmul(qcolumn, kcolumn) * self.scale
+        attn_column = F.softmax(attn_column, axis=-1)
+
+        xx_column = paddle.matmul(attn_column, vcolumn)  # B nH W C
+        xx_column = self.proj_encode_column(
+            xx_column.transpose([0, 1, 3, 2]).reshape([B, self.dh, 1, W]))
+
+        xx = paddle.add(xx_row, xx_column)  # [B, self.dh, H, W] 
+        xx = paddle.add(v, xx)
+
+        xx = self.proj(xx)
+        xx = self.sigmoid(xx) * qkv
+        if self.stride_attention:
+            xx = F.interpolate(xx, size=(H_ori, W_ori), mode='bilinear')
+
+        return xx
+
+
+class MLP(nn.Layer):
+    def __init__(self,
+                 in_features,
+                 hidden_features=None,
+                 out_features=None,
+                 act_layer=nn.ReLU,
+                 drop=0.):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = Conv2DBN(in_features, hidden_features)
+        self.dwconv = nn.Conv2D(
+            hidden_features, hidden_features, 3, 1, 1, groups=hidden_features)
+        self.act = act_layer()
+        self.fc2 = Conv2DBN(hidden_features, out_features)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.dwconv(x)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x
+
+
+class Block(nn.Layer):
+    def __init__(self,
+                 dim,
+                 key_dim,
+                 num_heads,
+                 mlp_ratio=4.,
+                 attn_ratio=2.,
+                 drop=0.,
+                 drop_path=0.,
+                 act_layer=nn.ReLU,
+                 stride_attention=None):
+        super().__init__()
+        self.dim = dim
+        self.num_heads = num_heads
+        self.mlp_ratio = mlp_ratio
+
+        self.attn = Sea_Attention(
+            dim,
+            key_dim=key_dim,
+            num_heads=num_heads,
+            attn_ratio=attn_ratio,
+            activation=act_layer,
+            stride_attention=stride_attention)
+
+        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity(
+        )
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        self.mlp = MLP(in_features=dim,
+                       hidden_features=mlp_hidden_dim,
+                       act_layer=act_layer,
+                       drop=drop)
+
+    def forward(self, x1):
+        x1 = x1 + self.drop_path(self.attn(x1))
+        x1 = x1 + self.drop_path(self.mlp(x1))
+
+        return x1
+
+
+class BasicLayer(nn.Layer):
+    def __init__(self,
+                 block_num,
+                 embedding_dim,
+                 key_dim,
+                 num_heads,
+                 mlp_ratio=4.,
+                 attn_ratio=2.,
+                 drop=0.,
+                 attn_drop=0.,
+                 drop_path=0.,
+                 act_layer=None,
+                 stride_attention=None):
+        super().__init__()
+        self.block_num = block_num
+
+        self.transformer_blocks = nn.LayerList()
+        for i in range(self.block_num):
+            self.transformer_blocks.append(
+                Block(
+                    embedding_dim,
+                    key_dim=key_dim,
+                    num_heads=num_heads,
+                    mlp_ratio=mlp_ratio,
+                    attn_ratio=attn_ratio,
+                    drop=drop,
+                    drop_path=drop_path[i]
+                    if isinstance(drop_path, list) else drop_path,
+                    act_layer=act_layer,
+                    stride_attention=stride_attention))
+
+    def forward(self, x):
+        for i in range(self.block_num):
+            x = self.transformer_blocks[i](x)
+        return x
+
+
+class Fusion_block(nn.Layer):
+    def __init__(self, inp, oup, embed_dim) -> None:
+        super(Fusion_block, self).__init__()
+        self.local_embedding = ConvBNAct(inp, embed_dim, kernel_size=1)
+        self.global_act = ConvBNAct(oup, embed_dim, kernel_size=1)
+        self.act = nn.Sigmoid()
+
+    def forward(self, x_l, x_g):
+        '''
+        x_g: global features
+        x_l: local features
+        '''
+        B, C, H, W = x_l.shape
+        B, C_c, H_c, W_c = x_g.shape
+
+        local_feat = self.local_embedding(x_l)
+        global_act = self.global_act(x_g)
+        sig_act = F.interpolate(
+            self.act(global_act),
+            size=(H, W),
+            mode='bilinear',
+            align_corners=False)
+
+        out = local_feat * sig_act
+
+        return out
+
+
+class InjectionMultiSumallmultiallsum(nn.Layer):
+    def __init__(self, in_channels=(64, 128, 256, 384), out_channels=256):
+        super(InjectionMultiSumallmultiallsum, self).__init__()
+        self.embedding_list = nn.LayerList()
+        self.act_embedding_list = nn.LayerList()
+        self.act_list = nn.LayerList()
+        for i in range(len(in_channels)):
+            self.embedding_list.append(
+                ConvBNAct(
+                    in_channels[i], out_channels, kernel_size=1))
+            self.act_embedding_list.append(
+                ConvBNAct(
+                    in_channels[i], out_channels, kernel_size=1))
+            self.act_list.append(nn.Sigmoid())
+
+    def forward(self, inputs):  # x_x8, x_x16, x_x32, x_x64 
+        low_feat1 = F.interpolate(inputs[0], scale_factor=0.5, mode="bilinear")
+        low_feat1_act = self.act_list[0](self.act_embedding_list[0](low_feat1))
+        low_feat1 = self.embedding_list[0](low_feat1)
+
+        low_feat2 = F.interpolate(
+            inputs[1], size=low_feat1.shape[-2:], mode="bilinear")
+        low_feat2_act = self.act_list[1](
+            self.act_embedding_list[1](low_feat2))  # x16
+        low_feat2 = self.embedding_list[1](low_feat2)
+
+        high_feat_act = F.interpolate(
+            self.act_list[2](self.act_embedding_list[2](inputs[2])),
+            size=low_feat2.shape[2:],
+            mode="bilinear")
+        high_feat = F.interpolate(
+            self.embedding_list[2](inputs[2]),
+            size=low_feat2.shape[2:],
+            mode="bilinear")
+
+        res = low_feat1_act * low_feat2_act * high_feat_act * (
+            low_feat1 + low_feat2) + high_feat
+
+        return res
+
+
+class InjectionMultiSumallmultiallsumSimpx8(nn.Layer):
+    def __init__(self, in_channels=(64, 128, 256, 384), out_channels=256):
+        super(InjectionMultiSumallmultiallsumSimpx8, self).__init__()
+        self.embedding_list = nn.LayerList()
+        self.act_embedding_list = nn.LayerList()
+        self.act_list = nn.LayerList()
+        for i in range(len(in_channels)):
+            if i != 1:
+                self.embedding_list.append(
+                    ConvBNAct(
+                        in_channels[i], out_channels, kernel_size=1))
+            if i != 0:
+                self.act_embedding_list.append(
+                    ConvBNAct(
+                        in_channels[i], out_channels, kernel_size=1))
+                self.act_list.append(nn.Sigmoid())
+
+    def forward(self, inputs):
+        # x_x8, x_x16, x_x32
+        low_feat1 = self.embedding_list[0](inputs[0])
+
+        low_feat2 = F.interpolate(
+            inputs[1], size=low_feat1.shape[-2:], mode="bilinear")
+        low_feat2_act = self.act_list[0](self.act_embedding_list[0](low_feat2))
+
+        high_feat_act = F.interpolate(
+            self.act_list[1](self.act_embedding_list[1](inputs[2])),
+            size=low_feat2.shape[2:],
+            mode="bilinear")
+        high_feat = F.interpolate(
+            self.embedding_list[1](inputs[2]),
+            size=low_feat2.shape[2:],
+            mode="bilinear")
+
+        res = low_feat2_act * high_feat_act * low_feat1 + high_feat
+
+        return res
+
+
+@manager.BACKBONES.add_component
+def MobileSeg_Base(**kwargs):
+    cfg1 = [
+        # k t c, s
+        [3, 16, 16, True, "relu", 1],
+        [3, 64, 32, False, "relu", 2],
+        [3, 96, 32, False, "relu", 1]
+    ]
+    cfg2 = [[5, 128, 64, True, "hardswish", 2],
+            [5, 240, 64, True, "hardswish", 1]]
+    cfg3 = [[5, 384, 128, True, "hardswish", 2],
+            [5, 384, 128, True, "hardswish", 1]]
+    cfg4 = [[5, 768, 192, True, "hardswish", 2],
+            [5, 768, 192, True, "hardswish", 1]]
+
+    channels = [16, 32, 64, 128, 192]
+    depths = [3, 3]
+    emb_dims = [128, 192]
+    num_heads = 8
+
+    model = StrideFormer(
+        cfgs=[cfg1, cfg2, cfg3, cfg4],
+        channels=channels,
+        embed_dims=emb_dims,
+        depths=depths,
+        num_heads=num_heads,
+        act_layer=nn.ReLU6,
+        **kwargs)
+
+    return model
+
+
+@manager.BACKBONES.add_component
+def MobileSeg_Tiny(**kwargs):
+    cfg1 = [
+        # k t c, s
+        [3, 16, 16, True, "relu", 1],
+        [3, 64, 32, False, "relu", 2],
+        [3, 48, 24, False, "relu", 1]
+    ]
+    cfg2 = [[5, 96, 32, True, "hardswish", 2],
+            [5, 96, 32, True, "hardswish", 1]]
+    cfg3 = [[5, 160, 64, True, "hardswish", 2],
+            [5, 160, 64, True, "hardswish", 1]]
+    cfg4 = [[3, 384, 128, True, "hardswish", 2],
+            [3, 384, 128, True, "hardswish", 1]]
+
+    channels = [16, 24, 32, 64, 128]
+    depths = [2, 2]
+    emb_dims = [64, 128]
+    num_heads = 4
+
+    model = StrideFormer(
+        cfgs=[cfg1, cfg2, cfg3, cfg4],
+        channels=channels,
+        embed_dims=emb_dims,
+        depths=depths,
+        num_heads=num_heads,
+        act_layer=nn.ReLU6,
+        **kwargs)
+
+    return model

paddlers/models/paddleseg/models/backbones/swin_transformer.py

@@ -118,7 +118,8 @@ class WindowAttention(nn.Layer):
         self.relative_position_bias_table = self.create_parameter(
             shape=((2 * window_size[0] - 1) * (2 * window_size[1] - 1),
                    num_heads),
-            default_initializer=zeros_)
+            default_initializer=zeros_,
+            dtype='float32')
         self.add_parameter("relative_position_bias_table",
                            self.relative_position_bias_table)
 
@@ -164,7 +165,7 @@ class WindowAttention(nn.Layer):
         q = q * self.scale
         attn = paddle.mm(q, k.transpose([0, 1, 3, 2]))
 
-        index = self.relative_position_index.reshape([-1])
+        index = self.relative_position_index.reshape([-1]).astype('int32')
         relative_position_bias = paddle.index_select(
             self.relative_position_bias_table, index)
 
@@ -713,6 +714,52 @@ class SwinTransformer(nn.Layer):
         self._freeze_stages()
 
 
+@manager.BACKBONES.add_component
+class SwinTransformer_tiny_patch4_window7_224_maskformer(SwinTransformer):
+    def __init__(self, **kwargs):
+        super().__init__(
+            pretrain_img_size=224,
+            embed_dim=96,
+            depths=[2, 2, 6, 2],
+            num_heads=[3, 6, 12, 24],
+            window_size=7,
+            drop_path_rate=0.3,
+            patch_norm=True,
+            **kwargs)
+
+        self._out_features = ["res2", "res3", "res4", "res5"]
+
+        self._out_feature_strides = {
+            "res2": 4,
+            "res3": 8,
+            "res4": 16,
+            "res5": 32,
+        }
+        self._out_feature_channels = {
+            "res2": self.feat_channels[0],
+            "res3": self.feat_channels[1],
+            "res4": self.feat_channels[2],
+            "res5": self.feat_channels[3],
+        }
+
+    def forward(self, x):
+        outputs = {}
+        y = super(SwinTransformer_tiny_patch4_window7_224_maskformer,
+                  self).forward(x)
+        for i, k in enumerate(self._out_features):
+            outputs[k] = y[i]
+        return outputs
+
+    def output_shape(self):
+        return {
+            name: {
+                "channels": self._out_feature_channels[name],
+                "stride": self._out_feature_strides[name]
+            }
+            for name in self._out_features
+        }
+
+
 @manager.BACKBONES.add_component
 def SwinTransformer_tiny_patch4_window7_224(**kwargs):
     model = SwinTransformer(
@@ -726,6 +773,52 @@ def SwinTransformer_tiny_patch4_window7_224(**kwargs):
     return model
 
 
+@manager.BACKBONES.add_component
+class SwinTransformer_small_patch4_window7_224_maskformer(SwinTransformer):
+    def __init__(self, **kwargs):
+        super().__init__(
+            pretrain_img_size=224,
+            embed_dim=96,
+            depths=[2, 2, 18, 2],
+            num_heads=[3, 6, 12, 24],
+            window_size=7,
+            drop_path_rate=0.3,
+            patch_norm=True,
+            **kwargs)
+
+        self._out_features = ["res2", "res3", "res4", "res5"]
+
+        self._out_feature_strides = {
+            "res2": 4,
+            "res3": 8,
+            "res4": 16,
+            "res5": 32,
+        }
+        self._out_feature_channels = {
+            "res2": self.feat_channels[0],
+            "res3": self.feat_channels[1],
+            "res4": self.feat_channels[2],
+            "res5": self.feat_channels[3],
+        }
+
+    def forward(self, x):
+        outputs = {}
+        y = super(SwinTransformer_small_patch4_window7_224_maskformer,
+                  self).forward(x)
+        for i, k in enumerate(self._out_features):
+            outputs[k] = y[i]
+        return outputs
+
+    def output_shape(self):
+        return {
+            name: {
+                "channels": self._out_feature_channels[name],
+                "stride": self._out_feature_strides[name]
+            }
+            for name in self._out_features
+        }
+
+
 @manager.BACKBONES.add_component
 def SwinTransformer_small_patch4_window7_224(**kwargs):
     model = SwinTransformer(
@@ -752,6 +845,52 @@ def SwinTransformer_base_patch4_window7_224(**kwargs):
     return model
 
 
+@manager.BACKBONES.add_component
+class SwinTransformer_base_patch4_window7_384_maskformer(SwinTransformer):
+    def __init__(self, **kwargs):
+        super().__init__(
+            pretrain_img_size=384,
+            embed_dim=128,
+            depths=[2, 2, 18, 2],
+            num_heads=[4, 8, 16, 32],
+            window_size=12,
+            drop_path_rate=0.3,
+            patch_norm=True,
+            **kwargs)
+
+        self._out_features = ["res2", "res3", "res4", "res5"]
+
+        self._out_feature_strides = {
+            "res2": 4,
+            "res3": 8,
+            "res4": 16,
+            "res5": 32,
+        }
+        self._out_feature_channels = {
+            "res2": self.feat_channels[0],
+            "res3": self.feat_channels[1],
+            "res4": self.feat_channels[2],
+            "res5": self.feat_channels[3],
+        }
+
+    def forward(self, x):
+        outputs = {}
+        y = super(SwinTransformer_base_patch4_window7_384_maskformer,
+                  self).forward(x)
+        for i, k in enumerate(self._out_features):
+            outputs[k] = y[i]
+        return outputs
+
+    def output_shape(self):
+        return {
+            name: {
+                "channels": self._out_feature_channels[name],
+                "stride": self._out_feature_strides[name]
+            }
+            for name in self._out_features
+        }
+
+
 @manager.BACKBONES.add_component
 def SwinTransformer_base_patch4_window12_384(**kwargs):
     model = SwinTransformer(
@@ -778,6 +917,52 @@ def SwinTransformer_large_patch4_window7_224(**kwargs):
     return model
 
 
+@manager.BACKBONES.add_component
+class SwinTransformer_large_patch4_window7_384_maskformer(SwinTransformer):
+    def __init__(self, **kwargs):
+        super().__init__(
+            pretrain_img_size=384,
+            embed_dim=192,
+            depths=[2, 2, 18, 2],
+            num_heads=[6, 12, 24, 48],
+            window_size=12,
+            drop_path_rate=0.3,
+            patch_norm=True,
+            **kwargs)
+
+        self._out_features = ["res2", "res3", "res4", "res5"]
+
+        self._out_feature_strides = {
+            "res2": 4,
+            "res3": 8,
+            "res4": 16,
+            "res5": 32,
+        }
+        self._out_feature_channels = {
+            "res2": self.feat_channels[0],
+            "res3": self.feat_channels[1],
+            "res4": self.feat_channels[2],
+            "res5": self.feat_channels[3],
+        }
+
+    def forward(self, x):
+        outputs = {}
+        y = super(SwinTransformer_large_patch4_window7_384_maskformer,
+                  self).forward(x)
+        for i, k in enumerate(self._out_features):
+            outputs[k] = y[i]
+        return outputs
+
+    def output_shape(self):
+        return {
+            name: {
+                "channels": self._out_feature_channels[name],
+                "stride": self._out_feature_strides[name]
+            }
+            for name in self._out_features
+        }
+
+
 @manager.BACKBONES.add_component
 def SwinTransformer_large_patch4_window12_384(**kwargs):
     model = SwinTransformer(

paddlers/models/paddleseg/models/backbones/top_transformer.py

@@ -593,8 +593,8 @@ class TopTransformer(nn.Layer):
             utils.load_entire_model(self, self.pretrained)
 
     def forward(self, x):
-        ouputs = self.tpm(x)
-        out = self.ppa(ouputs)
+        outputs = self.tpm(x)
+        out = self.ppa(outputs)
         out = self.trans(out)
 
         if self.injection:
@@ -602,14 +602,14 @@ class TopTransformer(nn.Layer):
             results = []
             for i in range(len(self.feat_channels)):
                 if i in self.trans_out_indices:
-                    local_tokens = ouputs[i]
+                    local_tokens = outputs[i]
                     global_semantics = xx[i]
                     out_ = self.SIM[i](local_tokens, global_semantics)
                     results.append(out_)
             return results
         else:
-            ouputs.append(out)
-            return ouputs
+            outputs.append(out)
+            return outputs
 
 
 @manager.BACKBONES.add_component

paddlers/models/paddleseg/models/backbones/transformer_utils.py

@@ -33,7 +33,7 @@ def drop_path(x, drop_prob=0., training=False):
     """
     if drop_prob == 0. or not training:
         return x
-    keep_prob = paddle.to_tensor(1 - drop_prob)
+    keep_prob = paddle.to_tensor(1 - drop_prob, dtype=x.dtype)
     shape = (paddle.shape(x)[0], ) + (1, ) * (x.ndim - 1)
     random_tensor = keep_prob + paddle.rand(shape).astype(x.dtype)
     random_tensor = paddle.floor(random_tensor)  # binarize

paddlers/models/paddleseg/models/backbones/uhrnet.py

@@ -646,7 +646,7 @@ class SELayer(nn.Layer):
     def __init__(self, num_channels, num_filters, reduction_ratio, name=None):
         super(SELayer, self).__init__()
 
-        self.pool2d_gap = nn.AdaptiveAvgPool2d(1)
+        self.pool2d_gap = nn.AdaptiveAvgPool2D(1)
 
         self._num_channels = num_channels
 

paddlers/models/paddleseg/models/backbones/vit_adapter.py

@@ -0,0 +1,420 @@
+# This file is heavily based on https://github.com/czczup/ViT-Adapter
+
+import math
+from functools import partial
+
+import numpy as np
+
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+
+from paddlers.models.paddleseg.cvlibs import manager
+from paddlers.models.paddleseg.utils import utils, logger
+from paddlers.models.paddleseg.cvlibs.param_init import normal_init, trunc_normal_init, constant_init
+from paddlers.models.paddleseg.models.backbones.transformer_utils import to_2tuple, DropPath
+from paddlers.models.paddleseg.models.layers.vit_adapter_layers import (
+    SpatialPriorModule, InteractionBlock, deform_inputs)
+from paddlers.models.paddleseg.models.layers.ms_deformable_attention import MSDeformAttn
+
+__all__ = ['ViTAdapter', 'ViTAdapter_Tiny']
+
+
+class PatchEmbed(nn.Layer):
+    """2D Image to Patch Embedding."""
+
+    def __init__(self,
+                 img_size=224,
+                 patch_size=16,
+                 in_chans=3,
+                 embed_dim=768,
+                 norm_layer=None,
+                 flatten=True):
+        super().__init__()
+        img_size = to_2tuple(img_size)
+        patch_size = to_2tuple(patch_size)
+        self.img_size = img_size
+        self.patch_size = patch_size
+        self.grid_size = (img_size[0] // patch_size[0],
+                          img_size[1] // patch_size[1])
+        self.num_patches = self.grid_size[0] * self.grid_size[1]
+        self.flatten = flatten
+
+        self.proj = nn.Conv2D(
+            in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
+        self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
+
+    def forward(self, x):
+        x = self.proj(x)
+        _, _, H, W = x.shape
+        if self.flatten:
+            x = x.flatten(2).transpose([0, 2, 1])  # BCHW -> BNC
+        x = self.norm(x)
+        return x, H, W
+
+
+class Mlp(nn.Layer):
+    def __init__(self,
+                 in_features,
+                 hidden_features=None,
+                 out_features=None,
+                 act_layer=nn.GELU,
+                 drop=0.):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = nn.Linear(in_features, hidden_features)
+        self.act = act_layer()
+        self.fc2 = nn.Linear(hidden_features, out_features)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x
+
+
+class Attention(nn.Layer):
+    def __init__(self,
+                 dim,
+                 num_heads=8,
+                 qkv_bias=False,
+                 attn_drop=0.,
+                 proj_drop=0.):
+        super().__init__()
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        self.scale = head_dim**-0.5
+
+        self.qkv = nn.Linear(dim, dim * 3, bias_attr=qkv_bias)
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+
+    def forward(self, x, H, W):
+        x_shape = paddle.shape(x)
+        N, C = x_shape[1], x_shape[2]
+        qkv = self.qkv(x).reshape((-1, N, 3, self.num_heads, C //
+                                   self.num_heads)).transpose((2, 0, 3, 1, 4))
+        q, k, v = qkv[0], qkv[1], qkv[2]
+
+        attn = (q.matmul(k.transpose((0, 1, 3, 2)))) * self.scale
+        attn = nn.functional.softmax(attn, axis=-1)
+        attn = self.attn_drop(attn)
+
+        x = (attn.matmul(v)).transpose((0, 2, 1, 3)).reshape((-1, N, C))
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+
+class Block(nn.Layer):
+    def __init__(self,
+                 dim,
+                 num_heads,
+                 mlp_ratio=4.,
+                 qkv_bias=False,
+                 drop=0.,
+                 attn_drop=0.,
+                 drop_path=0.,
+                 act_layer=nn.GELU,
+                 norm_layer=nn.LayerNorm,
+                 layer_scale=False):
+        super().__init__()
+        self.norm1 = norm_layer(dim)
+        self.attn = Attention(
+            dim,
+            num_heads=num_heads,
+            qkv_bias=qkv_bias,
+            attn_drop=attn_drop,
+            proj_drop=drop)
+        # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here
+        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity(
+        )
+        self.norm2 = norm_layer(dim)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        self.mlp = Mlp(in_features=dim,
+                       hidden_features=mlp_hidden_dim,
+                       act_layer=act_layer,
+                       drop=drop)
+        self.layer_scale = layer_scale
+        if layer_scale:
+            self.gamma1 = self.create_parameter(
+                shape=(dim, ),
+                default_initializer=paddle.nn.initializer.Constant(value=1.))
+            self.gamma2 = self.create_parameter(
+                shape=(dim, ),
+                default_initializer=paddle.nn.initializer.Constant(value=1.))
+
+    def forward(self, x, H, W):
+        if self.layer_scale:
+            x = x + self.drop_path(self.gamma1 * self.attn(self.norm1(x), H, W))
+            x = x + self.drop_path(self.gamma2 * self.mlp(self.norm2(x)))
+        else:
+            x = x + self.drop_path(self.attn(self.norm1(x), H, W))
+            x = x + self.drop_path(self.mlp(self.norm2(x)))
+        return x
+
+
+class VisionTransformer(nn.Layer):
+    """Vision Transformer.
+
+    A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale`
+        - https://arxiv.org/abs/2010.11929
+
+    Includes distillation token & head support for `DeiT: Data-efficient Image Transformers`
+        - https://arxiv.org/abs/2012.12877
+    """
+
+    def __init__(self,
+                 img_size=224,
+                 patch_size=16,
+                 in_channels=3,
+                 num_classes=1000,
+                 embed_dim=768,
+                 depth=12,
+                 num_heads=12,
+                 mlp_ratio=4.,
+                 qkv_bias=True,
+                 drop_rate=0.,
+                 attn_drop_rate=0.,
+                 drop_path_rate=0.,
+                 layer_scale=True,
+                 embed_layer=PatchEmbed,
+                 norm_layer=partial(
+                     nn.LayerNorm, epsilon=1e-6),
+                 act_layer=nn.GELU,
+                 pretrained=None):
+        """
+        Args:
+            img_size (int, tuple): input image size
+            patch_size (int, tuple): patch size
+            in_channels (int): number of input channels
+            num_classes (int): number of classes for classification head
+            embed_dim (int): embedding dimension
+            depth (int): depth of transformer
+            num_heads (int): number of attention heads
+            mlp_ratio (int): ratio of mlp hidden dim to embedding dim
+            qkv_bias (bool): enable bias for qkv if True
+            drop_rate (float): dropout rate
+            attn_drop_rate (float): attention dropout rate
+            drop_path_rate (float): stochastic depth rate
+            embed_layer (nn.Module): patch embedding layer
+            norm_layer: (nn.Module): normalization layer
+            pretrained: (str): pretrained path
+        """
+        super().__init__()
+        self.num_classes = num_classes
+        self.num_features = self.embed_dim = embed_dim  # num_features for consistency with other models
+        self.num_tokens = 1
+        norm_layer = norm_layer or partial(nn.LayerNorm, epsilon=1e-6)
+        act_layer = act_layer or nn.GELU
+        self.norm_layer = norm_layer
+        self.act_layer = act_layer
+        self.pretrain_size = img_size
+        self.drop_path_rate = drop_path_rate
+        self.drop_rate = drop_rate
+
+        self.patch_embed = embed_layer(
+            img_size=img_size,
+            patch_size=patch_size,
+            in_chans=in_channels,
+            embed_dim=embed_dim)
+        num_patches = self.patch_embed.num_patches
+
+        self.pos_embed = self.create_parameter(
+            shape=(1, num_patches + self.num_tokens, embed_dim),
+            default_initializer=paddle.nn.initializer.Constant(value=0.))
+        self.pos_drop = nn.Dropout(p=drop_rate)
+
+        dpr = np.linspace(0, drop_path_rate,
+                          depth)  # stochastic depth decay rule
+        self.blocks = nn.Sequential(*[
+            Block(
+                dim=embed_dim,
+                num_heads=num_heads,
+                mlp_ratio=mlp_ratio,
+                qkv_bias=qkv_bias,
+                drop=drop_rate,
+                attn_drop=attn_drop_rate,
+                drop_path=dpr[i],
+                norm_layer=norm_layer,
+                act_layer=act_layer,
+                layer_scale=layer_scale) for i in range(depth)
+        ])
+
+        self.pretrained = pretrained
+        self.init_weight()
+
+    def init_weight(self):
+        utils.load_pretrained_model(self, self.pretrained)
+
+
+@manager.BACKBONES.add_component
+class ViTAdapter(VisionTransformer):
+    """ The ViT-Adapter
+    """
+
+    def __init__(self,
+                 pretrain_size=224,
+                 num_heads=12,
+                 conv_inplane=64,
+                 n_points=4,
+                 deform_num_heads=6,
+                 init_values=0.,
+                 interaction_indexes=None,
+                 with_cffn=True,
+                 cffn_ratio=0.25,
+                 deform_ratio=1.0,
+                 add_vit_feature=True,
+                 pretrained=None,
+                 use_extra_extractor=True,
+                 *args,
+                 **kwargs):
+
+        super().__init__(
+            num_heads=num_heads, pretrained=pretrained, *args, **kwargs)
+
+        self.cls_token = None
+        self.num_block = len(self.blocks)
+        self.pretrain_size = (pretrain_size, pretrain_size)
+        self.interaction_indexes = interaction_indexes
+        self.add_vit_feature = add_vit_feature
+        embed_dim = self.embed_dim
+        self.feat_channels = [embed_dim] * 4
+
+        self.level_embed = self.create_parameter(
+            shape=(3, embed_dim),
+            default_initializer=paddle.nn.initializer.Constant(value=0.))
+        self.spm = SpatialPriorModule(
+            inplanes=conv_inplane, embed_dim=embed_dim)
+        self.interactions = nn.Sequential(*[
+            InteractionBlock(
+                dim=embed_dim,
+                num_heads=deform_num_heads,
+                n_points=n_points,
+                init_values=init_values,
+                drop_path=self.drop_path_rate,
+                norm_layer=self.norm_layer,
+                with_cffn=with_cffn,
+                cffn_ratio=cffn_ratio,
+                deform_ratio=deform_ratio,
+                extra_extractor=((True if i == len(interaction_indexes) - 1 else
+                                  False) and use_extra_extractor))
+            for i in range(len(interaction_indexes))
+        ])
+        self.up = nn.Conv2DTranspose(embed_dim, embed_dim, 2, 2)
+        self.norm1 = nn.SyncBatchNorm(embed_dim)
+        self.norm2 = nn.SyncBatchNorm(embed_dim)
+        self.norm3 = nn.SyncBatchNorm(embed_dim)
+        self.norm4 = nn.SyncBatchNorm(embed_dim)
+
+        self.up.apply(self._init_weights)
+        self.spm.apply(self._init_weights)
+        self.interactions.apply(self._init_weights)
+        self.apply(self._init_deform_weights)
+        normal_init(self.level_embed)
+
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_init(m.weight, std=.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                constant_init(m.bias, value=0)
+        elif isinstance(m, nn.LayerNorm) or isinstance(m, (nn.BatchNorm2D,
+                                                           nn.SyncBatchNorm)):
+            constant_init(m.bias, value=0)
+            constant_init(m.weight, value=1.0)
+        elif isinstance(m, nn.Conv2D) or isinstance(m, nn.Conv2DTranspose):
+            fan_out = m._kernel_size[0] * m._kernel_size[1] * m._out_channels
+            fan_out //= m._groups
+            normal_init(m.weight, std=math.sqrt(2.0 / fan_out))
+            if m.bias is not None:
+                constant_init(m.bias, value=0)
+
+    def _get_pos_embed(self, pos_embed, H, W):
+        pos_embed = pos_embed.reshape(
+            [1, self.pretrain_size[0] // 16, self.pretrain_size[1] // 16,
+             -1]).transpose([0, 3, 1, 2])
+        pos_embed = F.interpolate(pos_embed, size=(H, W), mode='bicubic', align_corners=False).\
+            reshape([1, -1, H * W]).transpose([0, 2, 1])
+        return pos_embed
+
+    def _init_deform_weights(self, m):
+        if isinstance(m, MSDeformAttn):
+            m._reset_parameters()
+
+    def _add_level_embed(self, c2, c3, c4):
+        c2 = c2 + self.level_embed[0]
+        c3 = c3 + self.level_embed[1]
+        c4 = c4 + self.level_embed[2]
+        return c2, c3, c4
+
+    def forward(self, x):
+        deform_inputs1, deform_inputs2 = deform_inputs(x)
+
+        # SPM forward
+        c1, c2, c3, c4 = self.spm(x)
+        c2, c3, c4 = self._add_level_embed(c2, c3, c4)
+        c = paddle.concat([c2, c3, c4], axis=1)
+
+        # Patch Embedding forward
+        x, H, W = self.patch_embed(x)
+        bs, n, dim = x.shape
+        pos_embed = self._get_pos_embed(self.pos_embed[:, 1:], H, W)
+        x = self.pos_drop(x + pos_embed)
+
+        # Interaction
+        outs = list()
+        for i, layer in enumerate(self.interactions):
+            indexes = self.interaction_indexes[i]
+            x, c = layer(x, c, self.blocks[indexes[0]:indexes[-1] + 1],
+                         deform_inputs1, deform_inputs2, H, W)
+            outs.append(x.transpose([0, 2, 1]).reshape([bs, dim, H, W]))
+
+        # Split & Reshape
+        c2 = c[:, 0:c2.shape[1], :]
+        c3 = c[:, c2.shape[1]:c2.shape[1] + c3.shape[1], :]
+        c4 = c[:, c2.shape[1] + c3.shape[1]:, :]
+
+        c2 = c2.transpose([0, 2, 1]).reshape([bs, dim, H * 2, W * 2])
+        c3 = c3.transpose([0, 2, 1]).reshape([bs, dim, H, W])
+        c4 = c4.transpose([0, 2, 1]).reshape([bs, dim, H // 2, W // 2])
+        c1 = self.up(c2) + c1
+
+        if self.add_vit_feature:
+            x1, x2, x3, x4 = outs
+            x1 = F.interpolate(
+                x1, scale_factor=4, mode='bilinear', align_corners=False)
+            x2 = F.interpolate(
+                x2, scale_factor=2, mode='bilinear', align_corners=False)
+            x4 = F.interpolate(
+                x4, scale_factor=0.5, mode='bilinear', align_corners=False)
+            c1, c2, c3, c4 = c1 + x1, c2 + x2, c3 + x3, c4 + x4
+
+        # Final Norm
+        f1 = self.norm1(c1)
+        f2 = self.norm2(c2)
+        f3 = self.norm3(c3)
+        f4 = self.norm4(c4)
+        return [f1, f2, f3, f4]
+
+
+@manager.BACKBONES.add_component
+def ViTAdapter_Tiny(**kwargs):
+    return ViTAdapter(
+        num_heads=3,
+        patch_size=16,
+        embed_dim=192,
+        depth=12,
+        mlp_ratio=4,
+        drop_path_rate=0.1,
+        conv_inplane=64,
+        n_points=4,
+        deform_num_heads=6,
+        cffn_ratio=0.25,
+        deform_ratio=1.0,
+        interaction_indexes=[[0, 2], [3, 5], [6, 8], [9, 11]],
+        **kwargs)

paddlers/models/paddleseg/models/backbones/xception_deeplab.py

@@ -98,7 +98,7 @@ class ConvBNLayer(nn.Layer):
         return self._act_op(self._bn(self._conv(inputs)))
 
 
-class Seperate_Conv(nn.Layer):
+class Separate_Conv(nn.Layer):
     def __init__(self,
                  input_channels,
                  output_channels,
@@ -107,7 +107,7 @@ class Seperate_Conv(nn.Layer):
                  dilation=1,
                  act=None,
                  name=None):
-        super(Seperate_Conv, self).__init__()
+        super(Separate_Conv, self).__init__()
 
         self._conv1 = nn.Conv2D(
             in_channels=input_channels,
@@ -168,21 +168,21 @@ class Xception_Block(nn.Layer):
         self.skip_conv = skip_conv
         self.activation_fn_in_separable_conv = activation_fn_in_separable_conv
         if not activation_fn_in_separable_conv:
-            self._conv1 = Seperate_Conv(
+            self._conv1 = Separate_Conv(
                 input_channels,
                 output_channels[0],
                 stride=strides[0],
                 filter=filter_size[0],
                 dilation=dilation,
                 name=name + "/separable_conv1")
-            self._conv2 = Seperate_Conv(
+            self._conv2 = Separate_Conv(
                 output_channels[0],
                 output_channels[1],
                 stride=strides[1],
                 filter=filter_size[1],
                 dilation=dilation,
                 name=name + "/separable_conv2")
-            self._conv3 = Seperate_Conv(
+            self._conv3 = Separate_Conv(
                 output_channels[1],
                 output_channels[2],
                 stride=strides[2],
@@ -190,7 +190,7 @@ class Xception_Block(nn.Layer):
                 dilation=dilation,
                 name=name + "/separable_conv3")
         else:
-            self._conv1 = Seperate_Conv(
+            self._conv1 = Separate_Conv(
                 input_channels,
                 output_channels[0],
                 stride=strides[0],
@@ -198,7 +198,7 @@ class Xception_Block(nn.Layer):
                 act="relu",
                 dilation=dilation,
                 name=name + "/separable_conv1")
-            self._conv2 = Seperate_Conv(
+            self._conv2 = Separate_Conv(
                 output_channels[0],
                 output_channels[1],
                 stride=strides[1],
@@ -206,7 +206,7 @@ class Xception_Block(nn.Layer):
                 act="relu",
                 dilation=dilation,
                 name=name + "/separable_conv2")
-            self._conv3 = Seperate_Conv(
+            self._conv3 = Separate_Conv(
                 output_channels[1],
                 output_channels[2],
                 stride=strides[2],

paddlers/models/paddleseg/models/espnet.py

@@ -341,12 +341,15 @@ class DownSampler(nn.Layer):
             w1 = paddle.shape(avg_out)[2]
             w2 = paddle.shape(inputs)[2]
 
+            inputs = paddle.reshape(inputs, [
+                inputs.shape[0], inputs.shape[1], paddle.shape(inputs)[2],
+                paddle.shape(inputs)[3]
+            ])
             while w2 != w1:
                 inputs = F.avg_pool2d(
                     inputs, kernel_size=3, padding=1, stride=2)
                 w2 = paddle.shape(inputs)[2]
-            # import pdb
-            # pdb.set_trace()
+
             output = output + self.shortcut_layer(inputs)
         return self._act(output)
 

paddlers/models/paddleseg/models/gscnn.py

@@ -210,7 +210,7 @@ class GSCNNHead(nn.Layer):
             input_shape[2:],
             mode='bilinear',
             align_corners=self.align_corners)
-        edge_out = F.sigmoid(cs)  # Ouput of shape stream
+        edge_out = F.sigmoid(cs)  # Output of shape stream
 
         cat = paddle.concat([edge_out, canny], axis=1)
         acts = self.cw(cat)

paddlers/models/paddleseg/models/hrnet_contrast.py

@@ -31,7 +31,7 @@ class HRNetW48Contrast(nn.Layer):
     (https://arxiv.org/abs/2101.11939).
 
     Args:
-        in_channels (int): The output dimensions of backbone.
+        bb_channels (int): The output dimensions of backbone.
         num_classes (int): The unique number of target classes.
         backbone (Paddle.nn.Layer): Backbone network, currently support HRNet_W48.
         drop_prob (float): The probability of dropout.
@@ -42,7 +42,7 @@ class HRNetW48Contrast(nn.Layer):
     """
 
     def __init__(self,
-                 in_channels,
+                 bb_channels,
                  num_classes,
                  backbone,
                  drop_prob,
@@ -50,7 +50,7 @@ class HRNetW48Contrast(nn.Layer):
                  align_corners=False,
                  pretrained=None):
         super().__init__()
-        self.in_channels = in_channels
+        self.bb_channels = bb_channels
         self.backbone = backbone
         self.num_classes = num_classes
         self.proj_dim = proj_dim
@@ -58,16 +58,16 @@ class HRNetW48Contrast(nn.Layer):
 
         self.cls_head = nn.Sequential(
             layers.ConvBNReLU(
-                in_channels, in_channels, kernel_size=3, stride=1, padding=1),
+                bb_channels, bb_channels, kernel_size=3, stride=1, padding=1),
             nn.Dropout2D(drop_prob),
             nn.Conv2D(
-                in_channels,
+                bb_channels,
                 num_classes,
                 kernel_size=1,
                 stride=1,
                 bias_attr=False), )
         self.proj_head = ProjectionHead(
-            dim_in=in_channels, proj_dim=self.proj_dim)
+            dim_in=bb_channels, proj_dim=self.proj_dim)
 
         self.pretrained = pretrained
         self.init_weight()

paddlers/models/paddleseg/models/knet.py

@@ -0,0 +1,531 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+
+from paddlers.models.paddleseg import utils
+from paddlers.models.paddleseg.cvlibs import manager
+from paddlers.models.paddleseg.models import layers
+from .upernet import UPerNetHead
+
+
+@manager.MODELS.add_component
+class KNet(nn.Layer):
+    """
+    The KNet implementation based on PaddlePaddle.
+
+    The original article refers to
+    Wenwei Zhang, et, al. "K-Net: Towards Unified Image Segmentation"
+    (https://arxiv.org/abs/2106.14855).
+
+    Args:
+        num_classes (int): The unique number of target classes.
+        backbone (paddle.nn.Layer): Backbone network.
+        backbone_indices (tuple): Four values in the tuple indicate the indices of output of backbone.
+        kernel_update_head_params (dict): The params to build KernelUpdateHead.
+        kernel_generate_head_params (dict): The params to build KernelGenerateHead.
+        num_stages (int, optional): The num of KernelUpdateHead. Default: 3
+        channels (int, optional): The channels of intermediate layers. Default: 512.
+        enable_auxiliary_loss (bool, optional): A bool value that indicates whether or not to add auxiliary loss. Default: False.
+        align_corners (bool, optional): An argument of "F.interpolate". It should be set to False when the feature size is even,
+            e.g. 1024x512, otherwise it is True, e.g. 769x769. Default: False.
+        dropout_prob (float, optional): Dropout ratio for KNet model. Default: 0.1.
+        pretrained (str, optional): The path or url of pretrained model. Default: None.
+    """
+
+    def __init__(self,
+                 num_classes,
+                 backbone,
+                 backbone_indices,
+                 kernel_update_head_params,
+                 kernel_generate_head_params,
+                 num_stages=3,
+                 channels=512,
+                 enable_auxiliary_loss=False,
+                 align_corners=False,
+                 dropout_prob=0.1,
+                 pretrained=None):
+        super().__init__()
+        assert hasattr(backbone, 'feat_channels'), \
+            "The backbone should has `feat_channels`."
+        assert len(backbone.feat_channels) >= len(backbone_indices), \
+            f"The length of input `backbone_indices` ({len(backbone_indices)}) should not be" \
+            f"greater than the length of `feat_channels` ({len(backbone.feat_channels)})."
+        assert len(backbone.feat_channels) > max(backbone_indices), \
+            f"The maximum value ({max(backbone_indices)}) of `backbone_indices` should be " \
+            f"less than the length of `feat_channels` ({len(backbone.feat_channels)})."
+
+        self.backbone = backbone
+        self.backbone_indices = backbone_indices
+        self.in_channels = [
+            self.backbone.feat_channels[i] for i in backbone_indices
+        ]
+
+        self.align_corners = align_corners
+        self.pretrained = pretrained
+        self.enable_auxiliary_loss = enable_auxiliary_loss
+        self.num_stages = num_stages
+        self.kernel_update_head = nn.LayerList([
+            KernelUpdateHead(**kernel_update_head_params)
+            for _ in range(num_stages)
+        ])
+        self.kernel_generate_head = build_kernel_generate_head(
+            kernel_generate_head_params)
+        if self.enable_auxiliary_loss:
+            self.aux_head = layers.AuxLayer(
+                1024, 256, num_classes, dropout_prob=dropout_prob)
+        self.init_weight()
+
+    def forward(self, x):
+        feats = self.backbone(x)
+        if self.enable_auxiliary_loss:
+            aux_out = self.aux_head(feats[2])
+        feats = [feats[i] for i in self.backbone_indices]
+        sem_seg, feats, seg_kernels = self.kernel_generate_head(feats)
+        stage_segs = [sem_seg]
+        for i in range(self.num_stages):
+            sem_seg, seg_kernels = self.kernel_update_head[i](
+                feats, seg_kernels, sem_seg)
+            stage_segs.append(sem_seg)
+        if self.training:
+            if self.enable_auxiliary_loss:
+                stage_segs.append(aux_out)
+            for i, v in enumerate(stage_segs):
+                stage_segs[i] = F.interpolate(
+                    v,
+                    paddle.shape(x)[2:],
+                    mode='bilinear',
+                    align_corners=self.align_corners)
+            return stage_segs
+        # only return the prediction of the last stage during testing
+        return [
+            F.interpolate(
+                stage_segs[-1],
+                paddle.shape(x)[2:],
+                mode='bilinear',
+                align_corners=self.align_corners)
+        ]
+
+    def init_weight(self):
+        if self.pretrained is not None:
+            utils.load_entire_model(self, self.pretrained)
+
+
+def build_kernel_generate_head(kwargs):
+    support_heads = ['UPerKernelHead', 'FCNKernelHead']
+    head_layer = kwargs.pop('head_layer')
+    assert head_layer in support_heads, f"head layer {head_layer} not supported"
+    if head_layer == 'UPerKernelHead':
+        return UPerKernelHead(**kwargs)
+    if head_layer == 'FCNKernelHead':
+        return FCNKernelHead(**kwargs)
+
+
+class UPerKernelHead(UPerNetHead):
+    def forward(self, inputs):
+        laterals = []
+        for i, lateral_conv in enumerate(self.lateral_convs):
+            laterals.append(lateral_conv(inputs[i]))
+
+        laterals.append(self.ppm(inputs[-1]))
+        fpn_levels = len(laterals)
+        for i in range(fpn_levels - 1, 0, -1):
+            prev_shape = paddle.shape(laterals[i - 1])
+            laterals[i - 1] = laterals[i - 1] + F.interpolate(
+                laterals[i],
+                size=prev_shape[2:],
+                mode='bilinear',
+                align_corners=self.align_corners)
+
+        fpn_outs = []
+        for i in range(fpn_levels - 1):
+            fpn_outs.append(self.fpn_convs[i](laterals[i]))
+        fpn_outs.append(laterals[-1])
+
+        for i in range(fpn_levels - 1, 0, -1):
+            fpn_outs[i] = F.interpolate(
+                fpn_outs[i],
+                size=paddle.shape(fpn_outs[0])[2:],
+                mode='bilinear',
+                align_corners=self.align_corners)
+        fuse_out = paddle.concat(fpn_outs, axis=1)
+        feats = self.fpn_bottleneck(fuse_out)
+        output = self.conv_seg(feats)
+        if self.training:
+            seg_kernels = self.conv_seg.weight.clone()
+        else:
+            # Since tensor.clone() raises error when exporting static model, we use tensor instead,
+            # although this may cause little performance drop in mIoU.
+            seg_kernels = self.conv_seg.weight
+        seg_kernels = seg_kernels[None].expand(
+            [paddle.shape(feats)[0], *paddle.shape(seg_kernels)])
+        return output, feats, seg_kernels
+
+
+class FCNKernelHead(nn.Layer):
+    def __init__(self,
+                 in_channels=2048,
+                 channels=512,
+                 num_convs=2,
+                 concat_input=True,
+                 dropout_prob=0.1,
+                 num_classes=150,
+                 kernel_size=3,
+                 dilation=1):
+        assert num_convs >= 0 and dilation > 0 and isinstance(dilation, int)
+        self.in_channels = in_channels
+        self.channels = channels
+        self.num_convs = num_convs
+        self.concat_input = concat_input
+        self.kernel_size = kernel_size
+        super(FCNKernelHead, self).__init__()
+        if num_convs == 0:
+            assert self.in_channels == self.channels
+
+        conv_padding = (kernel_size // 2) * dilation
+        convs = []
+        for i in range(num_convs):
+            _in_channels = self.in_channels if i == 0 else self.channels
+            convs.append(
+                layers.ConvBNReLU(
+                    _in_channels,
+                    self.channels,
+                    kernel_size=kernel_size,
+                    padding=conv_padding,
+                    dilation=dilation))
+
+        if len(convs) == 0:
+            self.convs = nn.Identity()
+        else:
+            self.convs = nn.Sequential(*convs)
+        if self.concat_input:
+            self.conv_cat = layers.ConvBNReLU(
+                self.in_channels + self.channels,
+                self.channels,
+                kernel_size=kernel_size,
+                padding=kernel_size // 2)
+
+        self.conv_seg = nn.Conv2D(channels, num_classes, kernel_size=1)
+
+        if dropout_prob > 0:
+            self.dropout = nn.Dropout2D(dropout_prob)
+        else:
+            self.dropout = None
+
+    def forward(self, inputs):
+        feats = self.convs(inputs[0])
+        if self.concat_input:
+            feats = self.conv_cat(paddle.concat([inputs[0], feats], axis=1))
+        if self.dropout is not None:
+            feats = self.dropout(feats)
+        output = self.conv_seg(feats)
+        if self.training:
+            seg_kernels = self.conv_seg.weight.clone()
+        else:
+            # Since tensor.clone() raises error when exporting static model, we use tensor.detach() instead,
+            # although this may cause little performance drop in mIoU.
+            seg_kernels = self.conv_seg.weight
+        seg_kernels = seg_kernels[None].expand(
+            [paddle.shape(feats)[0], *paddle.shape(seg_kernels)])
+        return output, feats, seg_kernels
+
+
+class FFN(nn.Layer):
+    def __init__(self,
+                 embed_dims=256,
+                 feedforward_channels=1024,
+                 act_fn=nn.ReLU,
+                 ffn_drop=0.,
+                 dropout_layer=None,
+                 add_identity=True):
+        super().__init__()
+        self.embed_dims = embed_dims
+        self.feedforward_channels = feedforward_channels
+        self.activate = act_fn()
+
+        layers = []
+        in_channels = embed_dims
+        layers.append(
+            nn.Sequential(
+                nn.Linear(in_channels, feedforward_channels), self.activate,
+                nn.Dropout(ffn_drop)))
+        layers.append(nn.Linear(feedforward_channels, embed_dims))
+        layers.append(nn.Dropout(ffn_drop))
+        self.layers = nn.Sequential(*layers)
+        self.dropout_layer = nn.Dropout() if dropout_layer else nn.Identity()
+        self.add_identity = add_identity
+
+    def forward(self, x, identity=None):
+        out = self.layers(x)
+        if not self.add_identity:
+            return self.dropout_layer(out)
+        if identity is None:
+            identity = x
+        return identity + self.dropout_layer(out)
+
+
+class KernelUpdator(nn.Layer):
+    def __init__(
+            self,
+            in_channels=256,
+            feat_channels=64,
+            out_channels=None,
+            input_feat_shape=3,
+            gate_sigmoid=True,
+            gate_norm_act=False,
+            activate_out=False,
+            norm_fn=nn.LayerNorm,
+            act_fn=nn.ReLU, ):
+        super(KernelUpdator, self).__init__()
+        self.in_channels = in_channels
+        self.feat_channels = feat_channels
+        self.out_channels_raw = out_channels
+        self.gate_sigmoid = gate_sigmoid
+        self.gate_norm_act = gate_norm_act
+        self.activate_out = activate_out
+        if isinstance(input_feat_shape, int):
+            input_feat_shape = [input_feat_shape] * 2
+        self.input_feat_shape = input_feat_shape
+        self.act_fn = act_fn
+        self.norm_fn = norm_fn
+        self.out_channels = out_channels if out_channels else in_channels
+
+        self.num_params_in = self.feat_channels
+        self.num_params_out = self.feat_channels
+        self.dynamic_layer = nn.Linear(self.in_channels,
+                                       self.num_params_in + self.num_params_out)
+        self.input_layer = nn.Linear(self.in_channels,
+                                     self.num_params_in + self.num_params_out)
+        self.input_gate = nn.Linear(self.in_channels, self.feat_channels)
+        self.update_gate = nn.Linear(self.in_channels, self.feat_channels)
+        if self.gate_norm_act:
+            self.gate_norm = self.norm_fn(self.feat_channels)
+
+        self.norm_in = self.norm_fn(self.feat_channels)
+        self.norm_out = self.norm_fn(self.feat_channels)
+        self.input_norm_in = self.norm_fn(self.feat_channels)
+        self.input_norm_out = self.norm_fn(self.feat_channels)
+
+        self.activation = self.act_fn()
+
+        self.fc_layer = nn.Linear(self.feat_channels, self.out_channels)
+        self.fc_norm = self.norm_fn(self.out_channels)
+
+    def forward(self, update_feature, input_feature):
+        update_feature = update_feature.reshape([-1, self.in_channels])
+        num_proposals = paddle.shape(update_feature)[0]
+        # dynamic_layer works for
+        # phi_1 and psi_3 in Eq.(4) and (5) of K-Net paper
+        parameters = self.dynamic_layer(update_feature)
+        param_in = parameters[:, :self.num_params_in].reshape(
+            [-1, self.feat_channels])
+        param_out = parameters[:, -self.num_params_out:].reshape(
+            [-1, self.feat_channels])
+
+        # input_layer works for
+        # phi_2 and psi_4 in Eq.(4) and (5) of K-Net paper
+        input_feats = self.input_layer(
+            input_feature.reshape([num_proposals, -1, self.feat_channels]))
+        input_in = input_feats[..., :self.num_params_in]
+        input_out = input_feats[..., -self.num_params_out:]
+
+        # `gate_feats` is F^G in K-Net paper
+        gate_feats = input_in * param_in.unsqueeze(-2)
+        if self.gate_norm_act:
+            gate_feats = self.activation(self.gate_norm(gate_feats))
+
+        input_gate = self.input_norm_in(self.input_gate(gate_feats))
+        update_gate = self.norm_in(self.update_gate(gate_feats))
+        if self.gate_sigmoid:
+            input_gate = F.sigmoid(input_gate)
+            update_gate = F.sigmoid(update_gate)
+        param_out = self.norm_out(param_out)
+        input_out = self.input_norm_out(input_out)
+
+        if self.activate_out:
+            param_out = self.activation(param_out)
+            input_out = self.activation(input_out)
+
+        # Gate mechanism. Eq.(5) in original paper.
+        # param_out has shape (batch_size, feat_channels, out_channels)
+        features = update_gate * param_out.unsqueeze(
+            -2) + input_gate * input_out
+
+        features = self.fc_layer(features)
+        features = self.fc_norm(features)
+        features = self.activation(features)
+
+        return features
+
+
+class KernelUpdateHead(nn.Layer):
+    def __init__(self,
+                 num_classes=150,
+                 num_ffn_fcs=2,
+                 num_heads=8,
+                 num_mask_fcs=3,
+                 feedforward_channels=2048,
+                 in_channels=256,
+                 out_channels=256,
+                 dropout=0.0,
+                 act_fn=nn.ReLU,
+                 ffn_act_fn=nn.ReLU,
+                 conv_kernel_size=3,
+                 feat_transform=False,
+                 kernel_init=False,
+                 with_ffn=True,
+                 feat_gather_stride=1,
+                 mask_transform_stride=1,
+                 kernel_updator_cfg=None):
+        super(KernelUpdateHead, self).__init__()
+        if kernel_updator_cfg is None:
+            kernel_updator_cfg = dict(
+                in_channels=256,
+                feat_channels=256,
+                out_channels=256,
+                input_feat_shape=3)
+        self.num_classes = num_classes
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.dropout = dropout
+        self.num_heads = num_heads
+        self.kernel_init = kernel_init
+        self.with_ffn = with_ffn
+        self.conv_kernel_size = conv_kernel_size
+        self.feat_gather_stride = feat_gather_stride
+        self.mask_transform_stride = mask_transform_stride
+
+        self.attention = nn.MultiHeadAttention(
+            in_channels * conv_kernel_size**2,
+            num_heads,
+            dropout,
+            bias_attr=True)
+        self.attention_norm = nn.LayerNorm(in_channels * conv_kernel_size**2)
+
+        self.kernel_update_conv = KernelUpdator(**kernel_updator_cfg)
+
+        if feat_transform is not None:
+            kernel_size = 1
+            transform_channels = in_channels
+            self.feat_transform = nn.Conv2D(
+                transform_channels,
+                in_channels,
+                kernel_size,
+                stride=feat_gather_stride,
+                padding=int(feat_gather_stride // 2))
+        else:
+            self.feat_transform = None
+
+        if self.with_ffn:
+            self.ffn = FFN(in_channels,
+                           feedforward_channels,
+                           act_fn=ffn_act_fn,
+                           ffn_drop=dropout)
+            self.ffn_norm = nn.LayerNorm(in_channels)
+
+        self.mask_fcs = nn.LayerList()
+        for _ in range(num_mask_fcs):
+            self.mask_fcs.append(
+                nn.Linear(
+                    in_channels, in_channels, bias_attr=False))
+            self.mask_fcs.append(nn.LayerNorm(in_channels))
+            self.mask_fcs.append(act_fn())
+
+        self.fc_mask = nn.Linear(in_channels, out_channels)
+
+    def forward(self, x, proposal_feat, mask_preds, mask_shape=None):
+        N, num_proposals = paddle.shape(proposal_feat)[:2]
+        if self.feat_transform is not None:
+            x = self.feat_transform(x)
+
+        C, H, W = x.shape[-3:]
+
+        mask_h, mask_w = paddle.shape(mask_preds)[-2:]
+        if mask_h != H or mask_w != W:
+            gather_mask = F.interpolate(
+                mask_preds, (H, W), align_corners=False, mode='bilinear')
+        else:
+            gather_mask = mask_preds
+
+        sigmoid_masks = F.softmax(gather_mask, 1)
+
+        # Group Feature Assembling. Eq.(3) in original paper.
+        x_feat = paddle.einsum('bnhw,bchw->bnc', sigmoid_masks, x)
+
+        # obj_feat in shape [B, N, C, K, K] -> [B, N, C, K*K] -> [B, N, K*K, C]
+        proposal_feat = proposal_feat.reshape(
+            [N, num_proposals, self.in_channels, -1]).transpose([0, 1, 3, 2])
+        obj_feat = self.kernel_update_conv(x_feat, proposal_feat)
+
+        # [B, N, K*K, C] -> [B, N, K*K*C] -> [N, B, K*K*C]
+        obj_feat = obj_feat.reshape([N, num_proposals, -1]).transpose([1, 0, 2])
+        obj_feat = self.attention_norm(self.attention(obj_feat))
+        # [N, B, K*K*C] -> [B, N, K*K*C]
+        obj_feat = obj_feat.transpose([1, 0, 2])
+
+        # obj_feat in shape [B, N, K*K*C] -> [B, N, K*K, C]
+        obj_feat = obj_feat.reshape([N, num_proposals, -1, self.in_channels])
+
+        # FFN
+        if self.with_ffn:
+            obj_feat = self.ffn_norm(self.ffn(obj_feat))
+
+        mask_feat = obj_feat
+
+        for reg_layer in self.mask_fcs:
+            mask_feat = reg_layer(mask_feat)
+
+        # [B, N, K*K, C] -> [B, N, C, K*K]
+        mask_feat = self.fc_mask(mask_feat).transpose([0, 1, 3, 2])
+
+        if (self.mask_transform_stride == 2 and self.feat_gather_stride == 1):
+            mask_x = F.interpolate(
+                x, scale_factor=0.5, mode='bilinear', align_corners=False)
+            H, W = paddle.shape(mask_x)[-2:]
+        else:
+            mask_x = x
+        mask_feat = mask_feat.reshape([
+            N, num_proposals, C, self.conv_kernel_size, self.conv_kernel_size
+        ])
+        # [B, C, H, W] -> [1, B*C, H, W]
+        new_mask_preds = []
+        for i in range(N):
+            new_mask_preds.append(
+                F.conv2d(
+                    mask_x[i:i + 1],
+                    mask_feat[i],
+                    padding=int(self.conv_kernel_size // 2)))
+
+        new_mask_preds = paddle.concat(new_mask_preds, axis=0)
+        new_mask_preds = new_mask_preds.reshape([N, num_proposals, H, W])
+        if self.mask_transform_stride == 2:
+            new_mask_preds = F.interpolate(
+                new_mask_preds,
+                scale_factor=2,
+                mode='bilinear',
+                align_corners=False)
+
+        if mask_shape is not None and mask_shape[0] != H:
+            new_mask_preds = F.interpolate(
+                new_mask_preds,
+                mask_shape,
+                align_corners=False,
+                mode='bilinear')
+
+        return new_mask_preds, obj_feat.transpose([0, 1, 3, 2]).reshape([
+            N, num_proposals, self.in_channels, self.conv_kernel_size,
+            self.conv_kernel_size
+        ])

paddlers/models/paddleseg/models/layers/__init__.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .layer_libs import ConvBNReLU, ConvBN, SeparableConvBNReLU, DepthwiseConvBN, AuxLayer, SyncBatchNorm, JPU, ConvBNPReLU, ConvBNAct, ConvBNLeakyReLU
+from .layer_libs import ConvBNReLU, ConvBN, SeparableConvBNReLU, DepthwiseConvBN, AuxLayer, SyncBatchNorm, JPU, ConvBNPReLU, ConvBNAct, ConvBNLeakyReLU, ConvNormAct, ConvGNAct
 from .activation import Activation
 from .pyramid_pool import ASPPModule, PPModule
 from .attention import AttentionBlock
@@ -20,3 +20,4 @@ from .nonlocal2d import NonLocal2D
 from .wrap_functions import *
 from .tensor_fusion import UAFM_SpAtten, UAFM_SpAtten_S, UAFM_ChAtten, UAFM_ChAtten_S, UAFM, UAFMMobile, UAFMMobile_SpAtten
 from .polaried_self_attention import PolarizedSelfAttentionModule
+from .nmf_2d import NMF2D

paddlers/models/paddleseg/models/layers/layer_libs.py

@@ -58,6 +58,71 @@ class ConvBNReLU(nn.Layer):
         return x
 
 
+class ConvGNAct(nn.Layer):
+    def __init__(self,
+                 in_channels,
+                 out_channels,
+                 kernel_size,
+                 padding="same",
+                 num_groups=32,
+                 act_type=None,
+                 **kwargs):
+        super().__init__()
+        self._conv = nn.Conv2D(
+            in_channels, out_channels, kernel_size, padding=padding, **kwargs)
+
+        if "data_format" in kwargs:
+            data_format = kwargs["data_format"]
+        else:
+            data_format = "NCHW"
+        self._group_norm = nn.GroupNorm(
+            num_groups, out_channels, data_format=data_format)
+        self._act_type = act_type
+        if act_type is not None:
+            self._act = layers.Activation(act_type)
+
+    def forward(self, x):
+        x = self._conv(x)
+        x = self._group_norm(x)
+        if self._act_type is not None:
+            x = self._act(x)
+        return x
+
+
+class ConvNormAct(nn.Layer):
+    def __init__(self,
+                 in_channels,
+                 out_channels,
+                 kernel_size,
+                 padding='same',
+                 act_type=None,
+                 norm=None,
+                 **kwargs):
+        super().__init__()
+
+        self._conv = nn.Conv2D(
+            in_channels, out_channels, kernel_size, padding=padding, **kwargs)
+
+        if 'data_format' in kwargs:
+            data_format = kwargs['data_format']
+        else:
+            data_format = 'NCHW'
+
+        self._norm = norm if norm is not None else None
+
+        self._act_type = act_type
+        if act_type is not None:
+            self._act = layers.Activation(act_type)
+
+    def forward(self, x):
+        x = self._conv(x)
+        if self._norm is not None:
+            x = self._norm(x)
+        if self._act_type is not None:
+            x = self._act(x)
+        return x
+
+
 class ConvBNAct(nn.Layer):
     def __init__(self,
                  in_channels,

paddlers/models/paddleseg/models/layers/ms_deformable_attention.py

@@ -0,0 +1,159 @@
+# This file is heavily based on https://github.com/czczup/ViT-Adapter
+import math
+import warnings
+
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+
+from paddlers.models.paddleseg.cvlibs import param_init
+from paddlers.models.paddleseg.cvlibs.param_init import constant_init, xavier_uniform
+
+
+class MSDeformAttn(nn.Layer):
+    def __init__(self,
+                 d_model=256,
+                 n_levels=4,
+                 n_heads=8,
+                 n_points=4,
+                 ratio=1.0):
+        """Multi-Scale Deformable Attention Module.
+        
+        Args:
+            d_model(int, optional): The hidden dimension. Default: 256
+            n_levels(int, optional): The number of feature levels. Default: 4
+            n_heads(int, optional): The number of attention heads. Default: 8
+            n_points(int, optional): The number of sampling points per attention head per feature level. Default: 4
+            ratio (float, optional): The ratio of channels for Linear. Default: 1.0
+        """
+        super().__init__()
+        if d_model % n_heads != 0:
+            raise ValueError('d_model must be divisible by n_heads, '
+                             'but got {} and {}'.format(d_model, n_heads))
+        _d_per_head = d_model // n_heads
+        # you'd better set _d_per_head to a power of 2
+        # which is more efficient in our CUDA implementation
+        if not self._is_power_of_2(_d_per_head):
+            warnings.warn("You'd better set d_model in MSDeformAttn to make "
+                          'the dimension of each attention head a power of 2 '
+                          'which is more efficient in our CUDA implementation.')
+
+        self.im2col_step = 64
+        self.d_model = d_model
+        self.n_levels = n_levels
+        self.n_heads = n_heads
+        self.n_points = n_points
+        self.ratio = ratio
+
+        self.sampling_offsets = nn.Linear(d_model,
+                                          n_heads * n_levels * n_points * 2)
+        self.attention_weights = nn.Linear(d_model,
+                                           n_heads * n_levels * n_points)
+        self.value_proj = nn.Linear(d_model, int(d_model * ratio))
+        self.output_proj = nn.Linear(int(d_model * ratio), d_model)
+
+        self._reset_parameters()
+
+    @staticmethod
+    def _is_power_of_2(n):
+        if (not isinstance(n, int)) or (n < 0):
+            raise ValueError('invalid input for _is_power_of_2: {} (type: {})'.
+                             format(n, type(n)))
+        return (n & (n - 1) == 0) and n != 0
+
+    def _reset_parameters(self):
+        constant_init(self.sampling_offsets.weight, value=0.)
+        thetas = paddle.arange(
+            self.n_heads, dtype='float32') * (2.0 * math.pi / self.n_heads)
+        grid_init = paddle.stack([thetas.cos(), thetas.sin()], -1)
+        grid_init = (grid_init / grid_init.abs().max(
+            -1, keepdim=True)[0]).reshape([self.n_heads, 1, 1, 2]).tile(
+                [1, self.n_levels, self.n_points, 1])
+        for i in range(self.n_points):
+            grid_init[:, :, i, :] *= i + 1
+
+        grid_init = grid_init.reshape([-1])
+        self.sampling_offsets.bias = self.create_parameter(
+            shape=grid_init.shape,
+            default_initializer=paddle.nn.initializer.Assign(grid_init))
+        self.sampling_offsets.bias.stop_gradient = True
+
+        constant_init(self.attention_weights.weight, value=0.)
+        constant_init(self.attention_weights.bias, value=0.)
+        xavier_uniform(self.value_proj.weight)
+        constant_init(self.value_proj.bias, value=0.)
+        xavier_uniform(self.output_proj.weight)
+        constant_init(self.output_proj.bias, value=0.)
+
+    def forward(self,
+                query,
+                reference_points,
+                input_flatten,
+                input_spatial_shapes,
+                input_level_start_index,
+                input_padding_mask=None):
+        """
+        Args:
+            query:                       (N, Length_{query}, C)
+            reference_points:            (N, Length_{query}, n_levels, 2), range in [0, 1], top-left (0,0), bottom-right (1, 1), including padding area
+                                            or (N, Length_{query}, n_levels, 4), add additional (w, h) to form reference boxes
+            input_flatten:               (N, \sum_{l=0}^{L-1} H_l \cdot W_l, C)
+            input_spatial_shapes:        (n_levels, 2), [(H_0, W_0), (H_1, W_1), ..., (H_{L-1}, W_{L-1})]
+            input_level_start_index:     (n_levels, ), [0, H_0*W_0, H_0*W_0+H_1*W_1, H_0*W_0+H_1*W_1+H_2*W_2, ..., H_0*W_0+H_1*W_1+...+H_{L-1}*W_{L-1}]
+            input_padding_mask:          (N, \sum_{l=0}^{L-1} H_l \cdot W_l), True for padding elements, False for non-padding elements
+
+        Returns:
+            output                     (N, Length_{query}, C)
+        """
+
+        def masked_fill(x, mask, value):
+            y = paddle.full(x.shape, value, x.dtype)
+            return paddle.where(mask, y, x)
+
+        N, Len_q, _ = query.shape
+        N, Len_in, _ = input_flatten.shape
+        assert (input_spatial_shapes[:, 0] * input_spatial_shapes[:, 1]
+                ).sum() == Len_in
+
+        value = self.value_proj(input_flatten)
+        if input_padding_mask is not None:
+            value = masked_fill(value, input_padding_mask[..., None], float(0))
+
+        value = value.reshape([
+            N, Len_in, self.n_heads,
+            int(self.ratio * self.d_model) // self.n_heads
+        ])
+        sampling_offsets = self.sampling_offsets(query).reshape(
+            [N, Len_q, self.n_heads, self.n_levels, self.n_points, 2])
+        attention_weights = self.attention_weights(query).reshape(
+            [N, Len_q, self.n_heads, self.n_levels * self.n_points])
+        attention_weights = F.softmax(attention_weights, -1).\
+            reshape([N, Len_q, self.n_heads, self.n_levels, self.n_points])
+
+        if reference_points.shape[-1] == 2:
+            offset_normalizer = paddle.stack(
+                [input_spatial_shapes[..., 1], input_spatial_shapes[..., 0]],
+                -1)
+            sampling_locations = reference_points[:, :, None, :, None, :] \
+                                 + sampling_offsets / offset_normalizer[None, None, None, :, None, :]
+        elif reference_points.shape[-1] == 4:
+            sampling_locations = reference_points[:, :, None, :, None, :2] \
+                                 + sampling_offsets / self.n_points * reference_points[:, :, None, :, None, 2:] * 0.5
+        else:
+            raise ValueError(
+                'Last dim of reference_points must be 2 or 4, but get {} instead.'
+                .format(reference_points.shape[-1]))
+        try:
+            import ms_deform_attn
+        except:
+            print(
+                "Import ms_deform_attn failed. Please download the following file and refer to "
+                "the readme to install ms_deform_attn lib: "
+                "https://paddleseg.bj.bcebos.com/dygraph/customized_ops/ms_deform_attn.zip"
+            )
+            exit()
+        output = ms_deform_attn.ms_deform_attn(
+            value, input_spatial_shapes, input_level_start_index,
+            sampling_locations, attention_weights, self.im2col_step)
+        output = self.output_proj(output)
+        return output

paddlers/models/paddleseg/models/layers/nmf_2d.py

@@ -0,0 +1,158 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import abstractmethod, ABCMeta
+
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+
+
+class _MatrixDecomposition2DBase(nn.Layer, metaclass=ABCMeta):
+    """
+    The base implementation of 2d matrix decomposition.
+
+    The original article refers to
+    Yuanduo Hong, Huihui Pan, Weichao Sun, et al. "Deep Dual-resolution Networks for Real-time and Accurate Semantic Segmentation of Road Scenes"
+    (https://arxiv.org/abs/2101.06085)
+    """
+
+    def __init__(self, args=None):
+        super().__init__()
+        if args is None:
+            args = dict()
+        elif not isinstance(args, dict):
+            raise TypeError("`args` must be a dict, but got {}".foramt(
+                args.__class__.__name__))
+
+        self.spatial = args.setdefault("SPATIAL", True)
+
+        self.S = args.setdefault("MD_S", 1)
+        self.D = args.setdefault("MD_D", 512)
+        self.R = args.setdefault("MD_R", 64)
+
+        self.train_steps = args.setdefault("TRAIN_STEPS", 6)
+        self.eval_steps = args.setdefault("EVAL_STEPS", 7)
+
+        self.inv_t = args.setdefault("INV_T", 100)
+        self.eta = args.setdefault("ETA", 0.9)
+
+        self.rand_init = args.setdefault("RAND_INIT", True)
+
+    @abstractmethod
+    def _build_bases(self, B, S, D, R):
+        raise NotImplementedError
+
+    @abstractmethod
+    def local_step(self, x, bases, coef):
+        raise NotImplementedError
+
+    @abstractmethod
+    def compute_coef(self, x, bases, coef):
+        raise NotImplementedError
+
+    def local_inference(self, x, bases):
+        # (B * S, D, N)^T @ (B * S, D, R) -> (B * S, N, R)
+        coef = paddle.bmm(x.transpose([0, 2, 1]), bases)
+        coef = F.softmax(self.inv_t * coef, axis=-1)
+
+        steps = self.train_steps if self.training else self.eval_steps
+        for _ in range(steps):
+            bases, coef = self.local_step(x, bases, coef)
+
+        return bases, coef
+
+    def forward(self, x):
+        B, C, H, W = x.shape
+
+        # (B, C, H, W) -> (B * S, D, N)
+        if self.spatial:
+            D = C // self.S
+            N = H * W
+            x = x.reshape([B * self.S, D, N])
+        else:
+            D = H * W
+            N = C // self.S
+            x = x.reshape([B * self.S, N, D]).transpose([0, 2, 1])
+
+        if not self.rand_init and not hasattr(self, "bases"):
+            bases = self._build_bases(1, self.S, D, self.R)
+            self.register_buffer("bases", bases)
+
+        # (S, D, R) -> (B * S, D, R)
+        if self.rand_init:
+            bases = self._build_bases(B, self.S, D, self.R)
+        else:
+            bases = paddle.repeat_interleave(self.bases, B, 0)
+
+        bases, coef = self.local_inference(x, bases)
+
+        # (B * S, N, R)
+        coef = self.compute_coef(x, bases, coef)
+
+        # (B * S, D, R) @ (B * S, N, R)^T -> (B * S, D, N)
+        x = paddle.bmm(bases, coef.transpose([0, 2, 1]))
+
+        # (B * S, D, N) -> (B, C, H, W)
+        if self.spatial:
+            x = x.reshape([B, C, H, W])
+        else:
+            x = x.transpose([0, 2, 1]).reshape([B, C, H, W])
+
+        # (B * H, D, R) -> (B, H, N, D)
+        # bases = bases.reshape([B, self.S, D, self.R])
+
+        return x
+
+
+class NMF2D(_MatrixDecomposition2DBase):
+    def __init__(self, args=dict()):
+        super().__init__(args)
+
+        self.inv_t = 1
+
+    def _build_bases(self, B, S, D, R):
+        bases = paddle.rand((B * S, D, R))
+
+        bases = F.normalize(bases, axis=1)
+
+        return bases
+
+    def local_step(self, x, bases, coef):
+        # (B * S, D, N)^T @ (B * S, D, R) -> (B * S, N, R)
+        numerator = paddle.bmm(x.transpose([0, 2, 1]), bases)
+        # (B * S, N, R) @ [(B * S, D, R)^T @ (B * S, D, R)] -> (B * S, N, R)
+        denominator = coef.bmm(bases.transpose([0, 2, 1]).bmm(bases))
+
+        # Multiplicative Update
+        coef = coef * numerator / (denominator + 1e-6)
+
+        # (B * S, D, N) @ (B * S, N, R) -> (B * S, D, R)
+        numerator = paddle.bmm(x, coef)
+        # (B * S, D, R) @ [(B * S, N, R)^T @ (B * S, N, R)] -> (B * S, D, R)
+        denominator = bases.bmm(coef.transpose([0, 2, 1]).bmm(coef))
+        # Multiplicative Update
+        bases = bases * numerator / (denominator + 1e-6)
+
+        return bases, coef
+
+    def compute_coef(self, x, bases, coef):
+        # (B * S, D, N)^T @ (B * S, D, R) -> (B * S, N, R)
+        numerator = paddle.bmm(x.transpose([0, 2, 1]), bases)
+        # (B * S, N, R) @ (B * S, D, R)^T @ (B * S, D, R) -> (B * S, N, R)
+        denominator = coef.bmm(bases.transpose([0, 2, 1]).bmm(bases))
+        # multiplication update
+        coef = coef * numerator / (denominator + 1e-6)
+
+        return coef

paddlers/models/paddleseg/models/layers/vit_adapter_layers.py

@@ -0,0 +1,461 @@
+# This file is heavily based on https://github.com/czczup/ViT-Adapter
+
+import math
+import warnings
+from functools import partial
+
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+from paddlers.models.paddleseg.models.backbones.transformer_utils import DropPath
+from paddlers.models.paddleseg.models.layers.ms_deformable_attention import MSDeformAttn
+
+
+def get_reference_points(spatial_shapes):
+    reference_points_list = []
+    for _, (H_, W_) in enumerate(spatial_shapes):
+        ref_y, ref_x = paddle.meshgrid(
+            paddle.linspace(
+                0.5, H_ - 0.5, H_, dtype='float32'),
+            paddle.linspace(
+                0.5, W_ - 0.5, W_, dtype='float32'))
+        ref_y = ref_y.reshape([1, -1]) / H_
+        ref_x = ref_x.reshape([1, -1]) / W_
+        ref = paddle.stack((ref_x, ref_y), -1)
+        reference_points_list.append(ref)
+    reference_points = paddle.concat(reference_points_list, 1)
+    reference_points = paddle.unsqueeze(reference_points, axis=2)
+    return reference_points
+
+
+def deform_inputs(x):
+    _, _, h, w = x.shape
+    spatial_shapes = paddle.to_tensor(
+        [(h // 8, w // 8), (h // 16, w // 16), (h // 32, w // 32)],
+        dtype='int64')
+    level_start_index = paddle.concat((paddle.zeros(
+        (1, ), dtype='int64'), spatial_shapes.prod(1).cumsum(0)[:-1]))
+    reference_points = get_reference_points([(h // 16, w // 16)])
+    deform_inputs1 = [reference_points, spatial_shapes, level_start_index]
+
+    spatial_shapes = paddle.to_tensor([(h // 16, w // 16)], dtype='int64')
+    level_start_index = paddle.concat((paddle.zeros(
+        (1, ), dtype='int64'), spatial_shapes.prod(1).cumsum(0)[:-1]))
+    reference_points = get_reference_points(
+        [(h // 8, w // 8), (h // 16, w // 16), (h // 32, w // 32)])
+    deform_inputs2 = [reference_points, spatial_shapes, level_start_index]
+
+    return deform_inputs1, deform_inputs2
+
+
+class DWConv(nn.Layer):
+    """
+    The specific DWConv unsed in ConvFFN. 
+    """
+
+    def __init__(self, dim=768):
+        super().__init__()
+        self.dwconv = nn.Conv2D(dim, dim, 3, 1, 1, bias_attr=True, groups=dim)
+
+    def forward(self, x, H, W):
+        B, N, C = x.shape
+        n = N // 21
+        x1 = x[:, 0:16 * n, :].transpose([0, 2, 1]).reshape(
+            [B, C, H * 2, W * 2])
+        x2 = x[:, 16 * n:20 * n, :].transpose([0, 2, 1]).reshape([B, C, H, W])
+        x3 = x[:, 20 * n:, :].transpose([0, 2, 1]).reshape(
+            [B, C, H // 2, W // 2])
+        x1 = self.dwconv(x1).flatten(2).transpose([0, 2, 1])
+        x2 = self.dwconv(x2).flatten(2).transpose([0, 2, 1])
+        x3 = self.dwconv(x3).flatten(2).transpose([0, 2, 1])
+        x = paddle.concat([x1, x2, x3], axis=1)
+        return x
+
+
+class ConvFFN(nn.Layer):
+    """
+    The implementation of ConvFFN unsed in Extractor.
+    """
+
+    def __init__(self,
+                 in_features,
+                 hidden_features=None,
+                 out_features=None,
+                 act_layer=nn.GELU,
+                 drop=0.):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = nn.Linear(in_features, hidden_features)
+        self.dwconv = DWConv(hidden_features)
+        self.act = act_layer()
+        self.fc2 = nn.Linear(hidden_features, out_features)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x, H, W):
+        x = self.fc1(x)
+        x = self.dwconv(x, H, W)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x
+
+
+class Extractor(nn.Layer):
+    """
+    The Extractor module in ViT-Adapter.
+    """
+
+    def __init__(self,
+                 dim,
+                 num_heads=6,
+                 n_points=4,
+                 n_levels=1,
+                 deform_ratio=1.0,
+                 with_cffn=True,
+                 cffn_ratio=0.25,
+                 drop=0.,
+                 drop_path=0.,
+                 norm_layer=partial(
+                     nn.LayerNorm, epsilon=1e-6)):
+        super().__init__()
+        self.query_norm = norm_layer(dim)
+        self.feat_norm = norm_layer(dim)
+        self.attn = MSDeformAttn(
+            d_model=dim,
+            n_levels=n_levels,
+            n_heads=num_heads,
+            n_points=n_points,
+            ratio=deform_ratio)
+        self.with_cffn = with_cffn
+        if with_cffn:
+            self.ffn = ConvFFN(
+                in_features=dim,
+                hidden_features=int(dim * cffn_ratio),
+                drop=drop)
+            self.ffn_norm = norm_layer(dim)
+            self.drop_path = DropPath(
+                drop_path) if drop_path > 0. else nn.Identity()
+
+    def forward(self, query, reference_points, feat, spatial_shapes,
+                level_start_index, H, W):
+        attn = self.attn(
+            self.query_norm(query), reference_points,
+            self.feat_norm(feat), spatial_shapes, level_start_index, None)
+        query = query + attn
+
+        if self.with_cffn:
+            query = query + self.drop_path(self.ffn(self.ffn_norm(query), H, W))
+        return query
+
+
+class Injector(nn.Layer):
+    """
+    The Injector module in ViT-Adapter.
+    """
+
+    def __init__(self,
+                 dim,
+                 num_heads=6,
+                 n_points=4,
+                 n_levels=1,
+                 deform_ratio=1.0,
+                 norm_layer=partial(
+                     nn.LayerNorm, epsilon=1e-6),
+                 init_values=0.):
+        super().__init__()
+        self.query_norm = norm_layer(dim)
+        self.feat_norm = norm_layer(dim)
+        self.attn = MSDeformAttn(
+            d_model=dim,
+            n_levels=n_levels,
+            n_heads=num_heads,
+            n_points=n_points,
+            ratio=deform_ratio)
+        self.gamma = self.create_parameter(
+            shape=(dim, ),
+            default_initializer=paddle.nn.initializer.Constant(
+                value=init_values))
+
+    def forward(self, query, reference_points, feat, spatial_shapes,
+                level_start_index):
+        attn = self.attn(
+            self.query_norm(query), reference_points,
+            self.feat_norm(feat), spatial_shapes, level_start_index, None)
+        return query + self.gamma * attn
+
+
+class InteractionBlock(nn.Layer):
+    """
+    Combine the Extractor, Extractor and ViT Blocks.
+    """
+
+    def __init__(self,
+                 dim,
+                 num_heads=6,
+                 n_points=4,
+                 norm_layer=partial(
+                     nn.LayerNorm, epsilon=1e-6),
+                 drop=0.,
+                 drop_path=0.,
+                 with_cffn=True,
+                 cffn_ratio=0.25,
+                 init_values=0.,
+                 deform_ratio=1.0,
+                 extra_extractor=False):
+        super().__init__()
+
+        self.injector = Injector(
+            dim=dim,
+            n_levels=3,
+            num_heads=num_heads,
+            init_values=init_values,
+            n_points=n_points,
+            norm_layer=norm_layer,
+            deform_ratio=deform_ratio)
+        self.extractor = Extractor(
+            dim=dim,
+            n_levels=1,
+            num_heads=num_heads,
+            n_points=n_points,
+            norm_layer=norm_layer,
+            deform_ratio=deform_ratio,
+            with_cffn=with_cffn,
+            cffn_ratio=cffn_ratio,
+            drop=drop,
+            drop_path=drop_path)
+        if extra_extractor:
+            self.extra_extractors = nn.Sequential(*[
+                Extractor(
+                    dim=dim,
+                    num_heads=num_heads,
+                    n_points=n_points,
+                    norm_layer=norm_layer,
+                    with_cffn=with_cffn,
+                    cffn_ratio=cffn_ratio,
+                    deform_ratio=deform_ratio,
+                    drop=drop,
+                    drop_path=drop_path) for _ in range(2)
+            ])
+        else:
+            self.extra_extractors = None
+
+    def forward(self, x, c, blocks, deform_inputs1, deform_inputs2, H, W):
+        x = self.injector(
+            query=x,
+            reference_points=deform_inputs1[0],
+            feat=c,
+            spatial_shapes=deform_inputs1[1],
+            level_start_index=deform_inputs1[2])
+
+        for _, blk in enumerate(blocks):
+            x = blk(x, H, W)
+
+        c = self.extractor(
+            query=c,
+            reference_points=deform_inputs2[0],
+            feat=x,
+            spatial_shapes=deform_inputs2[1],
+            level_start_index=deform_inputs2[2],
+            H=H,
+            W=W)
+
+        if self.extra_extractors is not None:
+            for extractor in self.extra_extractors:
+                c = extractor(
+                    query=c,
+                    reference_points=deform_inputs2[0],
+                    feat=x,
+                    spatial_shapes=deform_inputs2[1],
+                    level_start_index=deform_inputs2[2],
+                    H=H,
+                    W=W)
+
+        return x, c
+
+
+class InteractionBlockWithCls(nn.Layer):
+    def __init__(self,
+                 dim,
+                 num_heads=6,
+                 n_points=4,
+                 norm_layer=partial(
+                     nn.LayerNorm, eps=1e-6),
+                 drop=0.,
+                 drop_path=0.,
+                 with_cffn=True,
+                 cffn_ratio=0.25,
+                 init_values=0.,
+                 deform_ratio=1.0,
+                 extra_extractor=False):
+        super().__init__()
+
+        self.injector = Injector(
+            dim=dim,
+            n_levels=3,
+            num_heads=num_heads,
+            init_values=init_values,
+            n_points=n_points,
+            norm_layer=norm_layer,
+            deform_ratio=deform_ratio)
+        self.extractor = Extractor(
+            dim=dim,
+            n_levels=1,
+            num_heads=num_heads,
+            n_points=n_points,
+            norm_layer=norm_layer,
+            deform_ratio=deform_ratio,
+            with_cffn=with_cffn,
+            cffn_ratio=cffn_ratio,
+            drop=drop,
+            drop_path=drop_path)
+        if extra_extractor:
+            self.extra_extractors = nn.Sequential(*[
+                Extractor(
+                    dim=dim,
+                    num_heads=num_heads,
+                    n_points=n_points,
+                    norm_layer=norm_layer,
+                    with_cffn=with_cffn,
+                    cffn_ratio=cffn_ratio,
+                    deform_ratio=deform_ratio,
+                    drop=drop,
+                    drop_path=drop_path) for _ in range(2)
+            ])
+        else:
+            self.extra_extractors = None
+
+    def forward(self, x, c, cls, blocks, deform_inputs1, deform_inputs2, H, W):
+        x = self.injector(
+            query=x,
+            reference_points=deform_inputs1[0],
+            feat=c,
+            spatial_shapes=deform_inputs1[1],
+            level_start_index=deform_inputs1[2])
+        x = paddle.concat((cls, x), axis=1)
+        for _, blk in enumerate(blocks):
+            x = blk(x, H, W)
+        cls, x = x[:, :1, ], x[:, 1:, ]
+        c = self.extractor(
+            query=c,
+            reference_points=deform_inputs2[0],
+            feat=x,
+            spatial_shapes=deform_inputs2[1],
+            level_start_index=deform_inputs2[2],
+            H=H,
+            W=W)
+        if self.extra_extractors is not None:
+            for extractor in self.extra_extractors:
+                c = extractor(
+                    query=c,
+                    reference_points=deform_inputs2[0],
+                    feat=x,
+                    spatial_shapes=deform_inputs2[1],
+                    level_start_index=deform_inputs2[2],
+                    H=H,
+                    W=W)
+        return x, c, cls
+
+
+class SpatialPriorModule(nn.Layer):
+    def __init__(self, inplanes=64, embed_dim=384):
+        super().__init__()
+
+        self.stem = nn.Sequential(*[
+            nn.Conv2D(
+                3,
+                inplanes,
+                kernel_size=3,
+                stride=2,
+                padding=1,
+                bias_attr=False), nn.SyncBatchNorm(inplanes), nn.ReLU(),
+            nn.Conv2D(
+                inplanes,
+                inplanes,
+                kernel_size=3,
+                stride=1,
+                padding=1,
+                bias_attr=False), nn.SyncBatchNorm(inplanes), nn.ReLU(),
+            nn.Conv2D(
+                inplanes,
+                inplanes,
+                kernel_size=3,
+                stride=1,
+                padding=1,
+                bias_attr=False), nn.SyncBatchNorm(inplanes), nn.ReLU(),
+            nn.MaxPool2D(
+                kernel_size=3, stride=2, padding=1)
+        ])
+        self.conv2 = nn.Sequential(*[
+            nn.Conv2D(
+                inplanes,
+                2 * inplanes,
+                kernel_size=3,
+                stride=2,
+                padding=1,
+                bias_attr=False), nn.SyncBatchNorm(2 * inplanes), nn.ReLU()
+        ])
+        self.conv3 = nn.Sequential(*[
+            nn.Conv2D(
+                2 * inplanes,
+                4 * inplanes,
+                kernel_size=3,
+                stride=2,
+                padding=1,
+                bias_attr=False), nn.SyncBatchNorm(4 * inplanes), nn.ReLU()
+        ])
+        self.conv4 = nn.Sequential(*[
+            nn.Conv2D(
+                4 * inplanes,
+                4 * inplanes,
+                kernel_size=3,
+                stride=2,
+                padding=1,
+                bias_attr=False), nn.SyncBatchNorm(4 * inplanes), nn.ReLU()
+        ])
+        self.fc1 = nn.Conv2D(
+            inplanes,
+            embed_dim,
+            kernel_size=1,
+            stride=1,
+            padding=0,
+            bias_attr=True)
+        self.fc2 = nn.Conv2D(
+            2 * inplanes,
+            embed_dim,
+            kernel_size=1,
+            stride=1,
+            padding=0,
+            bias_attr=True)
+        self.fc3 = nn.Conv2D(
+            4 * inplanes,
+            embed_dim,
+            kernel_size=1,
+            stride=1,
+            padding=0,
+            bias_attr=True)
+        self.fc4 = nn.Conv2D(
+            4 * inplanes,
+            embed_dim,
+            kernel_size=1,
+            stride=1,
+            padding=0,
+            bias_attr=True)
+
+    def forward(self, x):
+        c1 = self.stem(x)
+        c2 = self.conv2(c1)
+        c3 = self.conv3(c2)
+        c4 = self.conv4(c3)
+        c1 = self.fc1(c1)
+        c2 = self.fc2(c2)
+        c3 = self.fc3(c3)
+        c4 = self.fc4(c4)
+
+        bs, dim, _, _ = c1.shape
+        c2 = c2.reshape([bs, dim, -1]).transpose([0, 2, 1])  # 8s
+        c3 = c3.reshape([bs, dim, -1]).transpose([0, 2, 1])  # 16s
+        c4 = c4.reshape([bs, dim, -1]).transpose([0, 2, 1])  # 32s
+
+        return c1, c2, c3, c4

paddlers/models/paddleseg/models/losses/__init__.py

@@ -34,3 +34,4 @@ from .point_cross_entropy_loss import PointCrossEntropyLoss
 from .pixel_contrast_cross_entropy_loss import PixelContrastCrossEntropyLoss
 from .semantic_encode_cross_entropy_loss import SECrossEntropyLoss
 from .semantic_connectivity_loss import SemanticConnectivityLoss
+from .maskformer_loss import MaskFormerLoss

paddlers/models/paddleseg/models/losses/binary_cross_entropy_loss.py

@@ -133,6 +133,7 @@ class BCELoss(nn.Layer):
         if len(label.shape) != len(logit.shape):
             label = paddle.unsqueeze(label, 1)
         mask = (label != self.ignore_index)
+        label = paddle.where(mask, label, paddle.zeros_like(label))
         mask = paddle.cast(mask, 'float32')
         # label.shape should equal to the logit.shape
         if label.shape[1] != logit.shape[1]:

paddlers/models/paddleseg/models/losses/cross_entropy_loss.py

@@ -33,6 +33,7 @@ class CrossEntropyLoss(nn.Layer):
         top_k_percent_pixels (float, optional): the value lies in [0.0, 1.0].
             When its value < 1.0, only compute the loss for the top k percent pixels
             (e.g., the top 20% pixels). This is useful for hard pixel mining. Default ``1.0``.
+        avg_non_ignore (bool, optional): Whether the loss is only averaged over non-ignored value of pixels. Default: True.
         data_format (str, optional): The tensor format to use, 'NCHW' or 'NHWC'. Default ``'NCHW'``.
     """
 
@@ -40,10 +41,12 @@ class CrossEntropyLoss(nn.Layer):
                  weight=None,
                  ignore_index=255,
                  top_k_percent_pixels=1.0,
+                 avg_non_ignore=True,
                  data_format='NCHW'):
         super(CrossEntropyLoss, self).__init__()
         self.ignore_index = ignore_index
         self.top_k_percent_pixels = top_k_percent_pixels
+        self.avg_non_ignore = avg_non_ignore
         self.EPS = 1e-8
         self.data_format = data_format
         if weight is not None:
@@ -107,10 +110,12 @@ class CrossEntropyLoss(nn.Layer):
         Returns:
             (Tensor): The average loss.
         """
-        mask = label != self.ignore_index
-        mask = paddle.cast(mask, 'float32')
-        label.stop_gradient = True
+        if self.avg_non_ignore:
+            mask = paddle.cast(label != self.ignore_index, dtype='float32')
+        else:
+            mask = paddle.ones(label.shape, dtype='float32')
         mask.stop_gradient = True
+        label.stop_gradient = True
 
         if loss.ndim > mask.ndim:
             loss = paddle.squeeze(loss, axis=-1)

paddlers/models/paddleseg/models/losses/focal_loss.py

@@ -49,7 +49,7 @@ class FocalLoss(nn.Layer):
         Args:
             logit (Tensor): Logit tensor, the data type is float32, float64. Shape is
                 (N, C, H, W), where C is number of classes.
-            label (Tensor): Label tensor, the data type is int64. Shape is (N, W, W),
+            label (Tensor): Label tensor, the data type is int64. Shape is (N, H, W),
                 where each value is 0 <= label[i] <= C-1.
         Returns:
             (Tensor): The average loss.
@@ -62,6 +62,7 @@ class FocalLoss(nn.Layer):
         logit = paddle.transpose(logit, [0, 2, 3, 1])  # N,C,H,W => N,H,W,C
 
         mask = label != self.ignore_index  # N,H,W
+        label = paddle.where(mask, label, paddle.zeros_like(label))
         mask = paddle.unsqueeze(mask, 3)
         mask = paddle.cast(mask, 'float32')
         mask.stop_gradient = True
@@ -110,7 +111,7 @@ class MultiClassFocalLoss(nn.Layer):
         Args:
             logit (Tensor): Logit tensor, the data type is float32, float64. Shape is
                 (N, C, H, W), where C is number of classes.
-            label (Tensor): Label tensor, the data type is int64. Shape is (N, W, W),
+            label (Tensor): Label tensor, the data type is int64. Shape is (N, H, W),
                 where each value is 0 <= label[i] <= C-1.
         Returns:
             (Tensor): The average loss.

paddlers/models/paddleseg/models/losses/maskformer_loss.py

@@ -0,0 +1,461 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# The implementation has referred to :https://github.com/facebookresearch/MaskFormer/blob/main/mask_former/modeling/criterion.py
+
+import copy
+import numpy as np
+from scipy.optimize import linear_sum_assignment
+
+import paddle
+import paddle.nn as nn
+import paddle.distributed as dist
+import paddle.nn.functional as F
+
+from paddlers.models.paddleseg.cvlibs import manager
+
+
+def dice_loss(inputs, targets, num_masks):
+    """
+    Compute the DICE loss, similar to generalized IOU for masks
+    Args:
+        inputs: A float tensor of arbitrary shape.
+                The predictions for each example.
+        targets: A float tensor with the same shape as inputs. Stores the binary
+                 classification label for each element in inputs
+                (0 for the negative class and 1 for the positive class).
+    """
+    inputs = F.sigmoid(inputs)
+    inputs = paddle.flatten(inputs, 1)
+    numerator = 2 * (inputs * targets).sum(-1)
+    denominator = inputs.sum(-1) + targets.sum(-1)
+    loss = 1 - (numerator + 1) / (denominator + 1)
+    num_masks = paddle.full(shape=[1], fill_value=num_masks, dtype='float32')
+    return loss.sum() / num_masks
+
+
+def sigmoid_focal_loss(inputs, targets, num_masks, alpha=0.25, gamma=2):
+    """
+    Loss used in RetinaNet for dense detection: https://arxiv.org/abs/1708.02002.
+    Args:
+        inputs: A float tensor of arbitrary shape.
+                The predictions for each example.
+        targets: A float tensor with the same shape as inputs. Stores the binary
+                 classification label for each element in inputs
+                (0 for the negative class and 1 for the positive class).
+        alpha: (optional) Weighting factor in range (0,1) to balance
+                positive vs negative examples. Default = -1 (no weighting).
+        gamma: Exponent of the modulating factor (1 - p_t) to
+               balance easy vs hard examples.
+    Returns:
+        Loss tensor
+    """
+    prob = F.sigmoid(inputs)
+    ce_loss = F.binary_cross_entropy_with_logits(
+        inputs, targets, reduction="none")
+    p_t = prob * targets + (1 - prob) * (1 - targets)
+    loss = ce_loss * ((1 - p_t)**gamma)
+
+    if alpha >= 0:
+        alpha_t = alpha * targets + (1 - alpha) * (1 - targets)
+        loss = alpha_t * loss
+    num_masks = paddle.full(shape=[1], fill_value=num_masks, dtype='float32')
+    return loss.mean(1).sum() / num_masks
+
+
+def batch_dice_loss(inputs, targets):
+    """
+    Compute the DICE loss, similar to generalized IOU for masks
+    Args:
+        inputs: A float tensor of arbitrary shape.
+                The predictions for each example.
+        targets: A float tensor with the same shape as inputs. Stores the binary
+                 classification label for each element in inputs
+                (0 for the negative class and 1 for the positive class).
+    """
+    inputs = F.sigmoid(inputs)
+    inputs = paddle.flatten(inputs, start_axis=1)
+    numerator = 2 * paddle.einsum("nc,mc->nm", inputs, targets)
+    denominator = paddle.sum(inputs, axis=-1, keepdim=True) + paddle.sum(
+        targets, axis=-1).unsqueeze(0)
+    loss = 1 - (numerator + 1) / (denominator + 1)
+    return loss
+
+
+def batch_sigmoid_focal_loss(inputs, targets, alpha=0.25, gamma=2):
+    """
+    Loss used in RetinaNet for dense detection: https://arxiv.org/abs/1708.02002.
+    Args:
+        inputs: A float tensor of arbitrary shape.
+                The predictions for each example.
+        targets: A float tensor with the same shape as inputs. Stores the binary
+                 classification label for each element in inputs
+                (0 for the negative class and 1 for the positive class).
+        alpha: (optional) Weighting factor in range (0,1) to balance
+                positive vs negative examples. Default = -1 (no weighting).
+        gamma: Exponent of the modulating factor (1 - p_t) to
+               balance easy vs hard examples.
+    Returns:
+        Loss tensor
+    """
+    hw = inputs.shape[1]
+
+    prob = F.sigmoid(inputs)
+    focal_pos = ((1 - prob)**gamma) * F.binary_cross_entropy_with_logits(
+        inputs, paddle.ones_like(inputs), reduction="none")
+    focal_neg = (prob**gamma) * F.binary_cross_entropy_with_logits(
+        inputs, paddle.zeros_like(inputs), reduction="none")
+    if alpha >= 0:
+        focal_pos = focal_pos * alpha
+        focal_neg = focal_neg * (1 - alpha)
+
+    loss = paddle.einsum("nc,mc->nm", focal_pos, targets) + paddle.einsum(
+        "nc,mc->nm", focal_neg, (1 - targets))
+
+    return loss / hw
+
+
+class HungarianMatcher(nn.Layer):
+    """This class computes an assignment between the targets and the predictions of the network
+
+    For efficiency reasons, the targets don't include the no_object. Because of this, in general,
+    there are more predictions than targets. In this case, we do a 1-to-1 matching of the best predictions,
+    while the others are un-matched (and thus treated as non-objects).
+    """
+
+    def __init__(self, cost_class=1, cost_mask=1, cost_dice=1):
+        """Creates the matcher
+
+        Params:
+            cost_class: This is the relative weight of the classification error in the matching cost
+            cost_mask: This is the relative weight of the focal loss of the binary mask in the matching cost
+            cost_dice: This is the relative weight of the dice loss of the binary mask in the matching cost
+        """
+        super().__init__()
+        self.cost_class = cost_class
+        self.cost_mask = cost_mask
+        self.cost_dice = cost_dice
+        assert cost_class != 0 or cost_mask != 0 or cost_dice != 0, "all costs cant be 0"
+
+    @paddle.no_grad()
+    def forward(self, outputs, targets):
+        """Performs the matching More memory-friendly.
+
+        Params:
+            outputs: This is a dict that contains at least these entries:
+                 "pred_logits": Tensor of dim [batch_size, num_queries, num_classes] with the classification logits
+                 "pred_masks": Tensor of dim [batch_size, num_queries, H_pred, W_pred] with the predicted masks
+
+            targets: This is a list of targets (len(targets) = batch_size), where each target is a dict containing:
+                 "labels": Tensor of dim [num_target_boxes] (where num_target_boxes is the number of ground-truth
+                           objects in the target) containing the class labels
+                 "masks": Tensor of dim [num_target_boxes, H_gt, W_gt] containing the target masks
+
+        Returns:
+            A list of size batch_size, containing tuples of (index_i, index_j) where:
+                - index_i is the indices of the selected predictions (in order)
+                - index_j is the indices of the corresponding selected targets (in order)
+            For each batch element, it holds:
+                len(index_i) = len(index_j) = min(num_queries, num_target_boxes)
+        """
+        bs, num_queries = outputs["pred_logits"].shape[:2]
+
+        indices = []
+
+        # Iterate through batch size
+        for b in range(bs):
+            out_prob = F.softmax(
+                outputs["pred_logits"][b],
+                axis=-1)  # [num_queries, num_classes]
+            out_mask = outputs["pred_masks"][b]  # [num_queries, H_pred, W_pred]
+
+            tgt_ids = targets[b]["labels"]
+            # gt masks are already padded when preparing target
+            if targets[b]["labels"].shape[0] == 0:
+                indices.append((np.array(
+                    [], dtype='int64'), np.array(
+                        [], dtype='int64')))
+                continue
+            tgt_mask = paddle.cast(targets[b]["masks"], out_mask.dtype)
+            # Compute the classification cost. Contrary to the loss, we don't use the NLL,
+            # but approximate it in 1 - proba[target class].
+            # The 1 is a constant that doesn't change the matching, it can be ommitted.            
+            cost_class = -paddle.gather(out_prob, index=tgt_ids, axis=1)
+
+            # Downsample gt masks to save memory
+            tgt_mask = F.interpolate(
+                tgt_mask[:, None], size=out_mask.shape[-2:], mode="nearest")
+
+            # Flatten spatial dimension
+            out_mask = out_mask.flatten(1)  # [batch_size * num_queries, H*W]
+            tgt_mask = tgt_mask[:, 0].flatten(1)  # [num_total_targets, H*W]
+
+            # Compute the focal loss between masks
+            cost_mask = batch_sigmoid_focal_loss(out_mask, tgt_mask)
+
+            # Compute the dice loss betwen masks
+            cost_dice = batch_dice_loss(out_mask, tgt_mask)
+
+            # Final cost matrix
+            C = (self.cost_mask * cost_mask + self.cost_class * cost_class +
+                 self.cost_dice * cost_dice)
+            C = C.reshape([num_queries, -1])
+
+            indices.append(linear_sum_assignment(C))
+
+        return [(paddle.to_tensor(
+            i, dtype='int64'), paddle.to_tensor(
+                j, dtype='int64')) for i, j in indices]
+
+
+def nested_tensor_from_tensor_list(tensor_list):
+    def _max_by_axis(the_list):
+        maxes = the_list[0]
+        for sublist in the_list[1:]:
+            for index, item in enumerate(sublist):
+                maxes[index] = max(maxes[index], item)
+        return maxes
+
+    if tensor_list[0].ndim == 3:
+        max_size = _max_by_axis([list(img.shape) for img in tensor_list])
+        batch_shape = [len(tensor_list)] + max_size
+        b, c, h, w = batch_shape
+        tensor = paddle.zeros(batch_shape, dtype=tensor_list[0].dtype)
+        mask = paddle.ones((b, h, w), dtype="bool")
+
+        for i in range(tensor.shape[0]):
+            img = tensor_list[i]
+            tensor[i, :img.shape[0], :img.shape[1], :img.shape[
+                2]] = copy.deepcopy(img)
+            mask[i, :img.shape[1], :img.shape[2]] = False
+    else:
+        raise ValueError("not supported")
+    return tensor, mask
+
+
+@manager.LOSSES.add_component
+class MaskFormerLoss(nn.Layer):
+    """
+    The Maskformer loss implemeted with PaddlePaddle.
+
+    Args:
+        num_classes(int): The number of classes that you want this network to classify. Default:150.
+        eos_coef(float): The weight coefficient of the last class. Default: 0.1.
+        losses(Tuple): The category of losses that you want to compute. Default: ("labels", 'masks').
+        ignore_index(int): The ignored label when we calculate the loss. Default:255.
+
+    """
+
+    def __init__(self,
+                 num_classes=150,
+                 eos_coef=0.1,
+                 losses=("labels", 'masks'),
+                 ignore_index=255):
+        super().__init__()
+        mask_weight = 20.0
+        dice_weight = 1.0
+
+        weight_dict = {
+            "loss_ce": 1,
+            "loss_mask": mask_weight,
+            "loss_dice": dice_weight
+        }
+        eos_coef = 0.1
+        dec_layers = 6
+        aux_weight_dict = {}
+        for i in range(dec_layers - 1):
+            aux_weight_dict.update(
+                {k + f"_{i}": v
+                 for k, v in weight_dict.items()})
+        weight_dict.update(aux_weight_dict)
+        self.num_classes = num_classes
+        self.ignore_index = ignore_index
+        self.weight_dict = weight_dict
+        self.matcher = HungarianMatcher(
+            cost_class=1, cost_mask=mask_weight, cost_dice=dice_weight)
+        self.losses = losses
+        self.empty_weight = paddle.ones(shape=(num_classes + 1, ))
+        self.empty_weight[-1] = eos_coef
+
+    def loss_labels(self, outputs, targets, indices, num_masks):
+        """Classification loss (NLL)
+        targets dicts must contain the key "labels" containing a tensor of dim [nb_target_boxes]
+        """
+        targets_cpt, indices_cpt = [], []
+        for t, indice in zip(targets, indices):
+            if t['labels'].shape[0] != 0:
+                targets_cpt.append(t)
+                indices_cpt.append(indice)
+        else:
+            if indices_cpt == []:
+                losses = {"loss_ce": paddle.to_tensor([0.0])}
+                return losses
+
+        assert "pred_logits" in outputs, "The 'pred_logits' need to be in outputs, but only got keys: {}".format(
+            outputs.keys())
+
+        src_logits = outputs["pred_logits"]
+        idx = self._get_src_permutation_idx(indices)
+        target_classes_o = paddle.concat(
+            [t["labels"][J] for t, (_, J) in zip(targets_cpt, indices_cpt)])
+        target_classes = paddle.full(
+            src_logits.shape[:2], self.num_classes, dtype='int64')
+        target_classes[idx] = target_classes_o
+
+        loss_ce = F.cross_entropy(
+            src_logits.transpose((0, 2, 1)).cast('float32'),
+            target_classes,
+            weight=self.empty_weight,
+            axis=1,
+            use_softmax=True,
+            ignore_index=255)
+        losses = {"loss_ce": loss_ce}
+        return losses
+
+    def loss_masks(self, outputs, targets, indices, num_masks):
+        """Compute the losses related to the masks: the focal loss and the dice loss.
+        targets dicts must contain the key "masks" containing a tensor of dim [nb_target_boxes, h, w]
+        """
+        assert "pred_masks" in outputs, "The 'pred_masks' need to be in outputs, but only got keys: {}".format(
+            outputs.keys())
+
+        targets_cpt, indices_cpt = [], []
+        for t, indice in zip(targets, indices):
+            if t['labels'].shape[0] != 0:
+                targets_cpt.append(t)
+                indices_cpt.append(indice)
+        else:
+            if indices_cpt == []:
+                losses = {
+                    "loss_mask": paddle.to_tensor([0.0]),
+                    "loss_dice": paddle.to_tensor([0.0]),
+                }
+                return losses
+        src_idx = self._get_src_permutation_idx(indices)
+        tgt_idx = self._get_tgt_permutation_idx(indices_cpt)
+        src_masks = outputs["pred_masks"]
+        src_masks = src_masks[src_idx]
+        if src_masks.ndim == 2:
+            src_masks = src_masks.unsqueeze(0)
+        masks = [t["masks"] for t in targets_cpt]
+
+        target_masks, valid = nested_tensor_from_tensor_list(masks)
+        target_masks = paddle.cast(target_masks, src_masks.dtype)
+        target_masks = target_masks[tgt_idx]
+
+        src_masks = F.interpolate(
+            src_masks[:, None],
+            size=target_masks.shape[-2:],
+            mode="bilinear",
+            align_corners=False)
+        src_masks = paddle.flatten(src_masks[:, 0], 1)
+
+        target_masks = paddle.flatten(target_masks, 1)
+        target_masks = target_masks.reshape(src_masks.shape)
+        losses = {
+            "loss_mask": sigmoid_focal_loss(src_masks, target_masks, num_masks),
+            "loss_dice": dice_loss(src_masks, target_masks, num_masks),
+        }
+        return losses
+
+    def _get_src_permutation_idx(self, indices):
+        batch_idx = paddle.concat(
+            [paddle.full_like(src, i) for i, (src, _) in enumerate(indices)])
+        src_idx = paddle.concat([src for (src, _) in indices])
+
+        return batch_idx, src_idx
+
+    def _get_tgt_permutation_idx(self, indices):
+        batch_idx = paddle.concat(
+            [paddle.full_like(tgt, i) for i, (_, tgt) in enumerate(indices)])
+        tgt_idx = paddle.concat([tgt for (_, tgt) in indices])
+
+        return batch_idx, tgt_idx
+
+    def get_loss(self, loss, outputs, targets, indices, num_masks):
+        loss_map = {"labels": self.loss_labels, "masks": self.loss_masks}
+        assert loss in loss_map, f"do you really want to compute {loss} loss?"
+        return loss_map[loss](outputs, targets, indices, num_masks)
+
+    def forward(self, logits, targets):
+        targets_cpt = []
+        batch_size = targets['gt_masks'].shape[0]
+        # split targets in a batch
+        for target_per_image_idx in range(batch_size):
+            gt_masks = targets['gt_masks'][target_per_image_idx, ...]
+            padded_masks = paddle.zeros(
+                (gt_masks.shape[0], gt_masks.shape[-2], gt_masks.shape[-1]),
+                dtype=gt_masks.dtype)
+            padded_masks[:, :gt_masks.shape[1], :gt_masks.shape[2]] = gt_masks
+
+            targets_cpt.append({
+                "labels": targets['gt_classes'][target_per_image_idx, ...],
+                "masks": padded_masks
+            })
+
+        targets = []
+        for item in targets_cpt:
+            item['masks'] = paddle.cast(item['masks'], 'bool')
+            invalid_indices = paddle.nonzero(
+                paddle.cast(item['labels'] == self.ignore_index, 'int64'))
+            if len(invalid_indices) > 0:
+                start_idx = int(invalid_indices[0].numpy())
+            else:
+                start_idx = len(item['labels'])
+            index = paddle.cast(
+                paddle.to_tensor([i for i in range(start_idx)]), 'int64')
+            item['labels'] = paddle.gather(
+                item['labels'], index, axis=0)  # [n] n<150
+            item['masks'] = paddle.gather(
+                item["masks"], index, axis=0)  # [n,512,512]
+            targets.append(item)
+
+        logits_without_aux = {
+            k: v
+            for k, v in logits.items() if k != "aux_outputs"
+        }
+
+        # Retrieve the matching between the outputs of the last layer and the targets
+        indices = self.matcher(logits_without_aux, targets)
+
+        num_masks = sum(len(t['labels']) for t in targets)
+        num_masks = paddle.to_tensor([num_masks], dtype='float32')
+
+        if dist.get_world_size() > 1:
+            dist.all_reduce(num_masks)
+        num_masks = paddle.clip(
+            num_masks / dist.get_world_size(), min=1).detach().numpy()[0]
+
+        losses = {}
+        for loss in self.losses:
+            losses.update(
+                self.get_loss(loss, logits_without_aux, targets, indices,
+                              num_masks))
+
+        if "aux_outputs" in logits:
+            for i in range(len(logits['aux_outputs'])):
+                indices = self.matcher(logits['aux_outputs'][i], targets)
+                for loss in self.losses:
+                    l_dict = self.get_loss(loss, logits['aux_outputs'][i],
+                                           targets, indices, num_masks)
+                    l_dict = {k + f"_{i}": v for k, v in l_dict.items()}
+                    losses.update(l_dict)
+
+        for k in list(losses.keys()):
+            if k in self.weight_dict:
+                losses[k] *= self.weight_dict[k]
+            else:
+                losses.pop(k)
+
+        return sum(losses.values())

paddlers/models/paddleseg/models/lpsnet.py

@@ -0,0 +1,143 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from functools import partial
+
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+
+from paddlers.models.paddleseg.cvlibs import manager
+from paddlers.models.paddleseg.models import layers
+from paddlers.models.paddleseg.utils import utils
+
+__all__ = ["LPSNet", ]
+
+_interpolate = partial(F.interpolate, mode="bilinear", align_corners=True)
+
+
+@manager.MODELS.add_component
+class LPSNet(nn.Layer):
+    """
+    The LPSNet implementation based on PaddlePaddle.
+
+    The original article refers to
+    Zhang, Yiheng and Yao, Ting and Qiu, Zhaofan and Mei, Tao. "Lightweight and Progressively-Scalable Networks for Semantic Segmentation"
+    (https://arxiv.org/pdf/2207.13600)
+
+    Args:
+        depths (list): Depths of each block.
+        channels (list): Channels of each block.
+        scale_ratios (list): Scale ratio for each branch. The number of branches depends on length of scale_ratios.
+        num_classes (int): The unique number of target classes.
+        in_channels (int):  The channels of input image. Default: 3.
+        pretrained (str, optional): The path or url of pretrained model. Default: None.
+    """
+
+    def __init__(
+            self,
+            depths,
+            channels,
+            scale_ratios,
+            num_classes,
+            in_channels=3,
+            pretrained=None, ):
+        super().__init__()
+
+        self.depths = depths
+        self.channels = channels
+        self.scale_ratios = list(filter(lambda x: x > 0, scale_ratios))
+        self.num_classes = num_classes
+        self.in_channels = in_channels
+
+        self.num_paths = len(self.scale_ratios)
+        self.num_blocks = len(depths)
+
+        if self.num_blocks != len(self.channels):
+            raise ValueError(
+                f"Expect depths and channels have same length, but got {self.num_blocks} and {len(self.channels)}"
+            )
+
+        self.nets = nn.LayerList(
+            [self._build_path() for _ in range(self.num_paths)])
+
+        self.head = nn.Conv2D(
+            channels[-1] * self.num_paths, num_classes, 1, bias_attr=True)
+
+        self._init_weight(pretrained)
+
+    def _init_weight(self, pretrained):
+        if pretrained is not None:
+            utils.load_entire_model(self, pretrained)
+
+    def _build_path(self):
+        path = []
+        c_in = self.in_channels
+        for b, (d, c) in enumerate(zip(self.depths, self.channels)):
+            blocks = []
+            for i in range(d):
+                blocks.append(
+                    layers.ConvBNReLU(
+                        in_channels=c_in if i == 0 else c,
+                        out_channels=c,
+                        kernel_size=3,
+                        padding=1,
+                        stride=2
+                        if (i == 0 and b != self.num_blocks - 1) else 1,
+                        bias_attr=False, ))
+                c_in = c
+            path.append(nn.Sequential(*blocks))
+        return nn.LayerList(path)
+
+    def _preprocess_input(self, x):
+        h, w = paddle.shape(x)[-2:]
+        return [
+            _interpolate(x, (int(r * h), int(r * w))) for r in self.scale_ratios
+        ]
+
+    def forward(self, x, interact_begin_idx=2):
+        input_size = paddle.shape(x)[-2:]
+        inputs = self._preprocess_input(x)
+        feats = []
+        for path, x in zip(self.nets, inputs):
+            inp = x
+            for idx in range(interact_begin_idx + 1):
+                inp = path[idx](inp)
+            feats.append(inp)
+
+        for idx in range(interact_begin_idx + 1, self.num_blocks):
+            feats = _multipath_interaction(feats)
+            feats = [path[idx](x) for path, x in zip(self.nets, feats)]
+
+        size = paddle.shape(feats[0])[-2:]
+        feats = [_interpolate(x, size=size) for x in feats]
+
+        out = self.head(paddle.concat(feats, 1))
+
+        return [_interpolate(out, size=input_size)]
+
+
+def _multipath_interaction(feats):
+    length = len(feats)
+    if length == 1:
+        return feats[0]
+    sizes = [paddle.shape(x)[-2:] for x in feats]
+    outs = []
+    looper = list(range(length))
+    for i, s in enumerate(sizes):
+        out = feats[i]
+        for j in filter(lambda x: x != i, looper):
+            out += _interpolate(feats[j], size=s)
+        outs.append(out)
+    return outs

paddlers/models/paddleseg/models/maskformer.py

@@ -0,0 +1,714 @@
+# copyright (c) 2023 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This implementation refers to: https://github.com/facebookresearch/MaskFormer/tree/main/mask_former/modeling
+
+import math
+import copy
+
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+
+from paddlers.models.paddleseg.models import layers
+from paddlers.models.paddleseg.cvlibs import manager, param_init
+from paddlers.models.paddleseg.utils import utils
+from paddlers.models.paddleseg.core.train import check_logits_losses
+
+
+@manager.MODELS.add_component
+class MaskFormer(nn.Layer):
+    """
+    The MaskFormer model implement on PaddlePaddle.
+    
+    The original article please refer to :
+    Cheng, Bowen, Alex Schwing, and Alexander Kirillov. "Per-pixel classification is not all you need for semantic segmentation." Advances in Neural Information Processing Systems 34 (2021): 17864-17875.
+    (https://github.com/facebookresearch/MaskFormer)
+
+    Args:
+        num_classes(int): The number of classes that you want the model to classify.
+        backbone(nn.Layer): The backbone module defined in the paddleseg backbones.
+        sem_seg_postprocess_before_inference(bool): If True, do result postprocess before inference. 
+        pretrained(str): The path to the pretrained model of MaskFormer.
+
+    """
+
+    def __init__(self,
+                 num_classes,
+                 backbone,
+                 sem_seg_postprocess_before_inference=False,
+                 pretrained=None):
+        super(MaskFormer, self).__init__()
+        self.num_classes = num_classes
+        self.backbone = backbone
+        self.sem_seg_postprocess_before_inference = sem_seg_postprocess_before_inference
+        self.seghead = MaskFormerHead(backbone.output_shape(), num_classes)
+        self.pretrained = pretrained
+        self.init_weight()
+
+    def init_weight(self):
+        if self.pretrained is not None:
+            utils.load_entire_model(self, self.pretrained)
+
+    def semantic_inference(self, mask_cls, mask_pred):
+        mask_cls = F.softmax(mask_cls)[..., :-1]
+        mask_pred = F.sigmoid(mask_pred)
+        semseg = paddle.einsum("qc,qhw->chw", mask_cls, mask_pred)
+        return semseg
+
+    def forward(self, x):
+        features = self.backbone(x)
+        outputs = self.seghead(features)
+
+        if self.training:
+            return [outputs]
+        else:
+            mask_cls_results = outputs["pred_logits"]  # [2, 100, 151]
+            mask_pred_results = outputs["pred_masks"]  # [2, 100, 512, 512]
+
+            mask_pred_results = F.interpolate(
+                mask_pred_results,
+                size=(x.shape[-2], x.shape[-1]),
+                mode="bilinear",
+                align_corners=False, )
+            processed_results = []
+
+            for mask_cls_result, mask_pred_result in zip(mask_cls_results,
+                                                         mask_pred_results):
+                image_size = x.shape[-2:]
+                if self.sem_seg_postprocess_before_inference:
+                    mask_pred_result = self.sem_seg_postprocess(
+                        mask_pred_result, image_size, image_size[0],
+                        image_size[1])
+
+                r = self.semantic_inference(mask_cls_result, mask_pred_result)
+
+                if not self.sem_seg_postprocess_before_inference:
+                    r = self.sem_seg_postprocess(r, image_size, image_size[0],
+                                                 image_size[1])
+                processed_results.append({"sem_seg": r})
+
+            r = r[None, ...]
+            return [r]
+
+    def sem_seg_postprocess(self, result, img_size, output_height,
+                            output_width):
+        """
+        Return semantic segmentation predictions in the original resolution.
+
+        The input images are often resized when entering semantic segmentor. Moreover, in same
+        cases, they also padded inside segmentor to be divisible by maximum network stride.
+        As a result, we often need the predictions of the segmentor in a different
+        resolution from its inputs.
+
+        Args:
+            result (Tensor): semantic segmentation prediction logits. A tensor of shape (C, H, W),
+                where C is the number of classes, and H, W are the height and width of the prediction.
+            img_size (tuple): image size that segmentor is taking as input.
+            output_height, output_width: the desired output resolution.
+
+        Returns:
+            semantic segmentation prediction (Tensor): A tensor of the shape
+                (C, output_height, output_width) that contains per-pixel soft predictions.
+        """
+        result = paddle.unsqueeze(result[:, :img_size[0], :img_size[1]], axis=0)
+        result = F.interpolate(
+            result,
+            size=(output_height, output_width),
+            mode="bilinear",
+            align_corners=False)[0]
+        return result
+
+    def loss_computation(self, logits_list, losses, data):
+        check_logits_losses(logits_list, losses)
+        loss_list = []
+        for i in range(len(logits_list)):
+            logits = logits_list[i]
+            loss_i = losses['types'][i]
+            coef_i = losses['coef'][i]
+
+            loss_list.append(coef_i * loss_i(logits, data['instances']))
+        return loss_list
+
+
+class BasePixelDecoder(nn.Layer):
+    def __init__(self, input_shape, conv_dim=256, norm="GN", mask_dim=256):
+        super().__init__()
+        input_shape = sorted(input_shape.items(), key=lambda x: x[1]['stride'])
+        self.in_features = [k for k, v in input_shape]  # "res2" to "res5"
+        feature_channels = [v['channels'] for k, v in input_shape]
+
+        self.lateral_convs, self.output_convs = nn.LayerList(), nn.LayerList()
+        use_bias = norm == ''
+        for idx, in_channels in enumerate(feature_channels):
+            if idx == len(self.in_features) - 1:
+                output_conv = layers.ConvNormAct(
+                    in_channels,
+                    conv_dim,
+                    kernel_size=3,
+                    bias_attr=use_bias,
+                    norm=nn.GroupNorm(
+                        num_groups=32, num_channels=conv_dim),
+                    act_type='relu')
+                self.output_convs.append(output_conv)
+                self.lateral_convs.append(None)
+                for layer in output_conv.sublayers():
+                    if hasattr(layer, "weight"):
+                        param_init.kaiming_uniform(
+                            layer.weight,
+                            negative_slope=1,
+                            nonlinearity='leaky_relu')
+                    if getattr(layer, 'bias', None) is not None:
+                        param_init.constant_init(layer.bias, value=0)
+            else:
+                lateral_norm = nn.GroupNorm(
+                    num_groups=32, num_channels=conv_dim)
+                output_norm = nn.GroupNorm(num_groups=32, num_channels=conv_dim)
+
+                lateral_conv = layers.ConvNormAct(
+                    in_channels,
+                    conv_dim,
+                    kernel_size=1,
+                    bias_attr=False,
+                    norm=lateral_norm)
+                output_conv = layers.ConvNormAct(
+                    conv_dim,
+                    conv_dim,
+                    kernel_size=3,
+                    stride=1,
+                    padding=1,
+                    bias_attr=use_bias,
+                    norm=output_norm,
+                    act_type='relu')
+                self.lateral_convs.append(lateral_conv)
+                self.output_convs.append(output_conv)
+
+                for layer in output_conv.sublayers() + lateral_conv.sublayers():
+                    if hasattr(layer, "weight"):
+                        param_init.kaiming_uniform(
+                            layer.weight,
+                            negative_slope=1,
+                            nonlinearity='leaky_relu')
+                    if getattr(layer, 'bias', None) is not None:
+                        param_init.constant_init(layer.bias, value=0)
+
+        self.lateral_convs = self.lateral_convs[::-1]
+        self.output_convs = self.output_convs[::-1]
+
+        self.mask_features = layers.ConvNormAct(
+            conv_dim, mask_dim, kernel_size=3, stride=1, padding=1)
+        for layer in self.mask_features.sublayers():
+            if hasattr(layer, "weight"):
+                param_init.kaiming_uniform(
+                    layer.weight, negative_slope=1, nonlinearity='leaky_relu')
+            if getattr(layer, 'bias', None) is not None:
+                param_init.constant_init(layer.bias, value=0)
+
+    def forward(self, features):
+        for idx, f in enumerate(self.in_features[::-1]):
+            x = features[f]
+            lateral_conv = self.lateral_convs[idx]
+
+            if lateral_conv is None:
+                y = self.output_convs[idx](x)
+            else:
+                cur_fpn = self.lateral_convs[idx](x)
+                y = cur_fpn + F.interpolate(
+                    y, size=cur_fpn.shape[-2:], mode='nearest')
+                y = self.output_convs[idx](y)
+        return self.mask_features(y), None
+
+
+class PositionEmbeddingSine(nn.Layer):
+    def __init__(self,
+                 num_pos_feats=64,
+                 temperature=10000,
+                 normalize=False,
+                 scale=None):
+        super().__init__()
+        self.num_pos_feats = num_pos_feats
+        self.temperature = temperature
+        self.normalize = normalize
+        if scale is not None and normalize is False:
+            raise ValueError("normalize should be true is scale is not None")
+        if scale is None:
+            scale = 2 * math.pi
+        self.scale = scale
+
+    def forward(self, x, mask=None):
+        if mask is None:
+            mask = paddle.zeros(
+                shape=(x.shape[0], x.shape[2], x.shape[3]), dtype='bool')
+        not_mask = ~mask
+        y_embed = paddle.cumsum(not_mask, axis=1, dtype='float32')
+        x_embed = paddle.cumsum(not_mask, axis=2, dtype='float32')
+
+        if self.normalize:
+            y_embed = y_embed / (y_embed[:, -1:, :] + 1e-6) * self.scale
+            x_embed = x_embed / (x_embed[:, :, -1:] + 1e-6) * self.scale
+
+        dim_t = paddle.arange(self.num_pos_feats, dtype='float32')
+        dim_t = paddle.cast(dim_t, dtype='int64')
+        tmp = paddle.ones_like(dim_t) * 2
+        dim_t = self.temperature**(2 * paddle.floor_divide(dim_t, tmp) /
+                                   self.num_pos_feats)
+        pos_x = x_embed[:, :, :, None] / dim_t
+        pos_y = y_embed[:, :, :, None] / dim_t
+
+        pos_x = paddle.flatten(
+            paddle.stack(
+                (paddle.sin(pos_x[:, :, :, 0::2]),
+                 paddle.cos(pos_x[:, :, :, 1::2])),
+                axis=4),
+            start_axis=3)
+        pos_y = paddle.flatten(
+            paddle.stack(
+                (paddle.sin(pos_y[:, :, :, 0::2]),
+                 paddle.cos(pos_y[:, :, :, 1::2])),
+                axis=4),
+            start_axis=3)
+        pos = paddle.transpose(
+            paddle.concat(
+                (pos_y, pos_x), axis=3), perm=(0, 3, 1, 2))
+        return pos
+
+
+class EncoderLayer(nn.Layer):
+    """
+    The layer to compose the transformer encoder.
+    
+    Args:
+        d_model(int): The input feature's channels.
+        nhead(int): the number of head for MHSA.
+        dim_feedforward(int): The internal channels of linear layer.
+        dropout(int): the dropout probability.
+        activation(str): the kind of activation that used.
+    """
+
+    def __init__(self,
+                 d_model,
+                 nhead,
+                 dim_feedforward=2048,
+                 dropout=0.1,
+                 activation="relu"):
+        super().__init__()
+
+        self.self_attn = nn.MultiHeadAttention(d_model, nhead, dropout)
+
+        self.linear1 = nn.Linear(d_model, dim_feedforward)
+        self.dropout = nn.Dropout(dropout)
+        self.linear2 = nn.Linear(dim_feedforward, d_model)
+
+        self.norm1 = nn.LayerNorm(d_model)
+        self.norm2 = nn.LayerNorm(d_model)
+        self.dropout1 = nn.Dropout(dropout)
+        self.dropout2 = nn.Dropout(dropout)
+
+        self.activation = nn.ReLU()
+        self.init_weight()
+
+    def with_pos_embed(self, tensor, pos=None):
+        return tensor if pos is None else tensor + pos
+
+    def init_weight(self):
+        param_init.multihead_fill(self.self_attn, True)
+        param_init.th_linear_fill(self.linear1)
+        param_init.th_linear_fill(self.linear2)
+
+    def forward(self, src, src_mask, src_key_padding_mask, pos):
+        q = k = self.with_pos_embed(src, pos)
+        if src_key_padding_mask is not None:
+            raise ValueError(
+                "The multihead attention does not support key_padding mask, but got src_key_padding_mask is not None"
+            )
+
+        attn = self.self_attn(q, k, value=src, attn_mask=src_mask)[0]
+        src += self.dropout(attn)
+        src = self.norm1(src)
+        attn = self.linear2(self.dropout(self.activation(self.linear1(src))))
+        src += self.dropout2(attn)
+        src = self.norm2(src)
+
+        return src
+
+
+class TransformerEncoder(nn.Layer):
+    """
+    The transformer encoder.
+    
+    Args:
+        encoder_layer(nn.Layer): The base layer to compose the encoder.
+        num_layers(int): How many layers is used in the encoder.
+        norm(str): the kind of normalization that used before output.
+    """
+
+    def __init__(self, encoder_layer, num_layers, norm=None):
+        super().__init__()
+        self.layers = nn.LayerList()
+        for i in range(num_layers):
+            self.layers.append(encoder_layer)
+        self.norm = norm
+
+    def forward(self, src, mask=None, src_key_padding_mask=None, pos=None):
+        output = src
+
+        for layer in self.layers:
+            # if pos is not none, all the encoder layer will have the position embedding
+            output = layer(
+                output,
+                src_mask=mask,
+                src_key_padding_mask=src_key_padding_mask,
+                pos=pos)
+
+        if self.norm is not None:
+            output = self.norm(output)
+
+        return output
+
+
+class DecoderLayer(nn.Layer):
+    """
+    The layer to compose the transformer decoder.
+    
+    Args:
+        d_model(int): The input feature's channels.
+        nhead(int): the number of head for MHSA.
+        dim_feedforward(int): The internal channels of linear layer.
+        dropout(int): the dropout probability.
+        activation(str): the kind of activation that used.
+    """
+
+    def __init__(self,
+                 d_model,
+                 nhead,
+                 dim_feedforward=2048,
+                 dropout=0.1,
+                 activation="relu"):
+        super().__init__()
+
+        self.self_attn = nn.MultiHeadAttention(d_model, nhead, dropout)
+        self.multihead_attn = nn.MultiHeadAttention(
+            d_model, nhead, dropout=dropout)
+
+        self.linear1 = nn.Linear(d_model, dim_feedforward)
+        self.dropout = nn.Dropout(dropout)
+        self.linear2 = nn.Linear(dim_feedforward, d_model)
+
+        self.norm1 = nn.LayerNorm(d_model)
+        self.norm2 = nn.LayerNorm(d_model)
+        self.norm3 = nn.LayerNorm(d_model)
+        self.dropout1 = nn.Dropout(dropout)
+        self.dropout2 = nn.Dropout(dropout)
+        self.dropout3 = nn.Dropout(dropout)
+
+        self.activation = nn.ReLU()
+
+    def init_weight(self):
+        param_init.multihead_fill(self.self_attn, True)
+        param_init.multihead_fill(self.multihead_attn, True)
+        param_init.th_linear_fill(self.linear1)
+        param_init.th_linear_fill(self.linear2)
+
+    def with_pos_embed(self, tensor, pos=None):
+        return tensor if pos is None else tensor + pos
+
+    def forward(self,
+                tgt,
+                memory,
+                tgt_mask=None,
+                memory_mask=None,
+                tgt_key_padding_mask=None,
+                memory_key_padding_mask=None,
+                pos=None,
+                query_pos=None):
+        if tgt_key_padding_mask or memory_key_padding_mask:
+            raise ValueError(
+                "The multihead attention does not support key_padding_mask")
+
+        q = k = self.with_pos_embed(tgt, query_pos).transpose(perm=(
+            1, 0, 2))  # [2, 100, 256]
+        tgt = tgt.transpose(perm=(1, 0, 2))
+        attn = self.self_attn(
+            q, k, value=tgt,
+            attn_mask=tgt_mask).transpose(perm=(1, 0, 2))  # [100, 2, 256]
+        tgt = tgt.transpose(perm=(1, 0, 2))  # [100, 2, 256]
+
+        tgt += self.dropout1(attn)
+
+        tgt = self.norm1(tgt)  # [100, 2, 256]
+        q = self.with_pos_embed(tgt, query_pos).transpose(perm=(1, 0, 2))
+        k = self.with_pos_embed(memory, pos).transpose(perm=(1, 0, 2))
+        v = memory.transpose(perm=(1, 0, 2))
+        attn = self.multihead_attn(
+            query=q, key=k, value=v,
+            attn_mask=memory_mask).transpose(perm=(1, 0, 2))
+        tgt += self.dropout2(attn)
+        tgt = self.norm2(tgt)  # [100, 2, 256]
+        attn = self.linear2(
+            self.dropout(self.activation(self.linear1(tgt))))  # [100, 2, 256]
+        tgt += self.dropout3(attn)
+        tgt = self.norm3(tgt)
+        return tgt
+
+
+class TransformerDecoder(nn.Layer):
+    """
+    The transformer decoder.
+    
+    Args:
+        encoder_layer(nn.Layer): The base layer to compose the decoder.
+        num_layers(int): How many layers is used in the decoder.
+        norm(str): the kind of normalization that used before output.
+        return_intermediate(bool): Whether to output the intermediate feature.
+    """
+
+    def __init__(self,
+                 decoder_layer,
+                 num_layers,
+                 norm=None,
+                 return_intermediate=True):
+        super().__init__()
+        self.decoder_list = nn.LayerList()
+        for i in range(num_layers):
+            self.decoder_list.append(copy.deepcopy(decoder_layer))
+        self.norm = norm
+        self.return_intermediate = return_intermediate
+
+    def forward(self,
+                tgt,
+                memory,
+                tgt_mask=None,
+                memory_mask=None,
+                tgt_key_padding_mask=None,
+                memory_key_padding_mask=None,
+                pos=None,
+                query_pos=None):
+        output = tgt
+        intermediate = []
+        for layer in self.decoder_list:
+            output = layer(
+                output,
+                memory,
+                tgt_mask=tgt_mask,
+                memory_mask=memory_mask,
+                tgt_key_padding_mask=tgt_key_padding_mask,
+                memory_key_padding_mask=memory_key_padding_mask,
+                pos=pos,
+                query_pos=query_pos)
+            if self.return_intermediate:
+                intermediate.append(self.norm(output))
+
+        if self.norm is not None:
+            output = self.norm(output)
+            if self.return_intermediate:
+                intermediate.pop()
+                intermediate.append(output)
+
+        if self.return_intermediate:
+            return paddle.stack(intermediate)
+
+        return output.unsqueeze(0)
+
+
+class Transformer(nn.Layer):
+    def __init__(self,
+                 d_model=256,
+                 nhead=8,
+                 num_encoder_layers=0,
+                 num_decoder_layers=6,
+                 dim_feedforward=2048,
+                 dropout=0.1,
+                 activation="relu",
+                 normalize_before=False,
+                 return_intermediate_dec=True):
+        super().__init__()
+        self.d_model = d_model
+        self.nhead = nhead
+
+        encoder_layer = EncoderLayer(d_model, nhead, dim_feedforward, dropout,
+                                     activation)
+        encoder_norm = nn.LayerNorm(d_model) if normalize_before else None
+        self.encoder = TransformerEncoder(encoder_layer, num_encoder_layers,
+                                          encoder_norm)
+
+        decoder_layer = DecoderLayer(d_model, nhead, dim_feedforward, dropout,
+                                     activation)
+        decoder_norm = nn.LayerNorm(d_model)
+        self.decoder = TransformerDecoder(
+            decoder_layer,
+            num_decoder_layers,
+            decoder_norm,
+            return_intermediate=return_intermediate_dec)
+
+        self.init_weight()
+
+    def init_weight(self):
+        for name, p in self.named_parameters():
+            if len(p.shape) > 1 and ('attn' not in name):
+                param_init.xavier_uniform(p)
+
+    def forward(self, src, mask, query_embed, pos_embed):
+        # flatten NxCxHxW to HWxNxC 
+        bs, c, h, w = src.shape
+        src = paddle.transpose(paddle.flatten(src, start_axis=2), (2, 0, 1))
+        pos_embed = paddle.transpose(
+            paddle.flatten(
+                pos_embed, start_axis=2), (2, 0, 1))
+        query_embed = paddle.stack([query_embed for i in range(bs)], axis=1)
+        if mask is not None:
+            mask = paddle.flatten(mask, start_axis=1)
+
+        tgt = paddle.zeros_like(query_embed)  # No.querry, N, hdim [100, 2, 256]
+        memory = self.encoder(
+            src, src_key_padding_mask=mask,
+            pos=pos_embed)  # HWxNxC memory = src
+        hs = self.decoder(
+            tgt,
+            memory,
+            memory_key_padding_mask=mask,
+            pos=pos_embed,
+            query_pos=query_embed)
+
+        return paddle.transpose(hs, (0, 2, 1, 3)), paddle.reshape(
+            paddle.transpose(memory, (1, 2, 0)), (bs, c, h, w))
+
+
+class MLP(nn.Layer):
+    def __init__(self, input_dim, hidden_dim, output_dim, num_layers):
+        super().__init__()
+        self.num_layers = num_layers
+        h = [hidden_dim] * (num_layers - 1)
+        self.layers = nn.LayerList(
+            nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim]))
+        self.init_weight()
+
+    def init_weight(self):
+        for layer in self.layers:
+            param_init.th_linear_fill(layer)
+
+    def forward(self, x):
+        for i, layer in enumerate(self.layers):
+            x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x)
+
+        return x
+
+
+class TransformerPredictor(nn.Layer):
+    def __init__(self,
+                 in_channels,
+                 mask_classification,
+                 num_classes=150,
+                 hidden_dim=256,
+                 num_queries=100,
+                 nheads=8,
+                 dropout=0.1,
+                 dim_feedforward=2048,
+                 enc_layers=0,
+                 dec_layers=6,
+                 pre_norm=False,
+                 deep_supervision=True,
+                 mask_dim=256,
+                 enforce_input_project=False):
+        super().__init__()
+        self.mask_classification = mask_classification
+        self.pe_layer = PositionEmbeddingSine(hidden_dim // 2, normalize=True)
+
+        self.transformer = Transformer(
+            d_model=hidden_dim,
+            dropout=dropout,
+            nhead=nheads,
+            dim_feedforward=dim_feedforward,
+            num_encoder_layers=enc_layers,
+            num_decoder_layers=dec_layers,
+            normalize_before=pre_norm,
+            return_intermediate_dec=deep_supervision)
+
+        self.query_embed = nn.Embedding(num_queries, hidden_dim)
+
+        if in_channels != hidden_dim or enforce_input_project:
+            self.input_proj = nn.Conv2D(in_channels, hidden_dim, kernel_size=1)
+            if hasattr(self.input_proj, "weight"):
+                param_init.kaiming_uniform(
+                    self.input_proj.weight,
+                    negative_slope=1,
+                    nonlinearity='leaky_relu')
+            if getattr(self.input_proj, 'bias', None) is not None:
+                param_init.constant_init(self.input_proj.bias, value=0)
+        else:
+            self.input_proj = nn.Sequential()
+
+        self.aux_loss = deep_supervision
+
+        if self.mask_classification:
+            self.class_embed = nn.Linear(hidden_dim, num_classes + 1)
+        self.mask_embed = MLP(hidden_dim, hidden_dim, mask_dim, 3)
+        self.init_weight()
+
+    def init_weight(self, ):
+        param_init.th_linear_fill(self.class_embed)
+        param_init.normal_init(self.query_embed.weight, mean=0.0, std=1.0)
+
+    def forward(self, x, mask_features):
+        pos = self.pe_layer(x)
+
+        mask = None
+        hs, memory = self.transformer(
+            self.input_proj(x), mask, self.query_embed.weight, pos)
+
+        out = {}
+        if self.mask_classification:
+            outputs_class = self.class_embed(hs)
+            out["pred_logits"] = outputs_class[-1]
+
+        if self.aux_loss:
+            mask_embed = self.mask_embed(hs)
+
+            output_seg_masks = paddle.einsum("lbqc,bchw->lbqhw", mask_embed,
+                                             mask_features)
+            out["pred_masks"] = output_seg_masks[-1]
+            if self.mask_classification:
+                out['aux_outputs'] = [{
+                    "pred_logits": a,
+                    "pred_masks": b
+                } for a, b in zip(outputs_class[:-1], output_seg_masks[:-1])]
+            else:
+                out['aux_outputs'] = [{
+                    "pred_masks": b
+                } for b in output_seg_masks[:-1]]
+        else:
+            mask_embed = self.mask_embed(hs[-1])
+            output_seg_masks = paddle.einsum("bqc,bchw->bqhw", mask_embed,
+                                             mask_features)
+            out["pred_masks"] = output_seg_masks
+
+        return out
+
+
+class MaskFormerHead(nn.Layer):
+    def __init__(self, input_shape, num_classes, transformer_in_feature='res5'):
+        super(MaskFormerHead, self).__init__()
+        self.transformer_in_feature = transformer_in_feature
+        self.input_shape = input_shape
+        self.pixel_decoder = BasePixelDecoder(input_shape)
+        self.predictor = TransformerPredictor(
+            input_shape[transformer_in_feature]["channels"],
+            mask_classification=True,
+            num_classes=num_classes)
+
+    def forward(self, x):
+        mask_features, transformer_encoder_features = self.pixel_decoder(x)
+        predictions = self.predictor(x[self.transformer_in_feature],
+                                     mask_features)
+
+        return predictions

paddlers/models/paddleseg/models/pp_mobileseg.py

@@ -0,0 +1,123 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import warnings
+
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+
+from paddlers.models.paddleseg.cvlibs import manager
+from paddlers.models.paddleseg.models import layers
+from paddlers.models.paddleseg.utils import utils
+from paddlers.models.paddleseg.models.backbones.strideformer import ConvBNAct
+
+
+@manager.MODELS.add_component
+class PPMobileSeg(nn.Layer):
+    """
+    The PP_MobileSeg implementation based on PaddlePaddle.
+
+    The original article refers to "Shiyu Tang, Ting Sun, Juncai Peng, Guowei Chen, Yuying Hao, 
+    Manhui Lin, Zhihong Xiao, Jiangbin You, Yi Liu. PP-MobileSeg: Explore the Fast and Accurate 
+    Semantic Segmentation Model on Mobile Devices. https://arxiv.org/abs/2304.05152"
+
+
+    Args:
+        num_classes(int): The unique number of target classes.
+        backbone(nn.Layer): Backbone network.
+        head_use_dw (bool, optional): Whether the head use depthwise convolutions. Default: True.
+        align_corners (bool, optional): Set the align_corners in resizing. Default: False.
+        pretrained (str, optional): The path or url of pretrained model. Default: None.
+        upsample (str, optional): The type of upsample module, valid for VIM is recommend to be used during inference. Default: intepolate.
+    """
+
+    def __init__(self,
+                 num_classes,
+                 backbone,
+                 head_use_dw=True,
+                 align_corners=False,
+                 pretrained=None,
+                 upsample='intepolate'):
+        super().__init__()
+        self.backbone = backbone
+        self.upsample = upsample
+        self.num_classes = num_classes
+
+        self.decode_head = PPMobileSegHead(
+            num_classes=num_classes,
+            in_channels=backbone.feat_channels[0],
+            use_dw=head_use_dw,
+            align_corners=align_corners)
+
+        self.align_corners = align_corners
+        self.pretrained = pretrained
+        self.init_weight()
+
+    def init_weight(self):
+        if self.pretrained is not None:
+            utils.load_entire_model(self, self.pretrained)
+
+    def forward(self, x):
+        x_hw = x.shape[2:]
+        x = self.backbone(x)
+        x = self.decode_head(x)
+        if self.upsample == 'intepolate' or self.training or self.num_classes < 30:
+            x = F.interpolate(
+                x, x_hw, mode='bilinear', align_corners=self.align_corners)
+        elif self.upsample == 'vim':
+            labelset = paddle.unique(paddle.argmax(x, 1))
+            x = paddle.gather(x, labelset, axis=1)
+            x = F.interpolate(
+                x, x_hw, mode='bilinear', align_corners=self.align_corners)
+
+            pred = paddle.argmax(x, 1)
+            pred_retrieve = paddle.zeros(pred.shape, dtype='int32')
+            for i, val in enumerate(labelset):
+                pred_retrieve[pred == i] = labelset[i].cast('int32')
+
+            x = pred_retrieve
+        else:
+            raise NotImplementedError(self.upsample, " is not implemented")
+
+        return [x]
+
+
+class PPMobileSegHead(nn.Layer):
+    def __init__(self,
+                 num_classes,
+                 in_channels,
+                 use_dw=False,
+                 dropout_ratio=0.1,
+                 align_corners=False):
+        super().__init__()
+        self.align_corners = align_corners
+        self.last_channels = in_channels
+
+        self.linear_fuse = ConvBNAct(
+            in_channels=self.last_channels,
+            out_channels=self.last_channels,
+            kernel_size=1,
+            stride=1,
+            groups=self.last_channels if use_dw else 1,
+            act=nn.ReLU)
+        self.dropout = nn.Dropout2D(dropout_ratio)
+        self.conv_seg = nn.Conv2D(
+            self.last_channels, num_classes, kernel_size=1)
+
+    def forward(self, x):
+        x = self.linear_fuse(x)
+        x = self.dropout(x)
+        x = self.conv_seg(x)
+        return x

paddlers/models/paddleseg/models/segnext.py

@@ -0,0 +1,176 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+
+from paddlers.models.paddleseg.cvlibs import manager
+from paddlers.models.paddleseg.models.backbones.mscan import MSCAN
+from paddlers.models.paddleseg.models.layers import NMF2D, ConvGNAct
+from paddlers.models.paddleseg.utils import utils
+
+
+@manager.MODELS.add_component
+class SegNeXt(nn.Layer):
+    """
+    The SegNeXt implementation based on PaddlePaddle.
+
+    The original article refers to
+    Guo, Meng-Hao, et al. "SegNeXt: Rethinking Convolutional Attention Design for Semantic Segmentation"
+    (https://arxiv.org/pdf/2209.08575.pdf)
+
+    Args:
+        backbone (nn.Layer): The backbone must be an instance of MSCAN.
+        decoder_cfg (dict): The arguments of decoder.
+        num_classes (int): The unique number of target classes.
+        backbone_indices (list(int), optional): The values indicate the indices of backbone output 
+           used as the input of the SegNeXt head. Default: [1, 2, 3].
+        pretrained (str, optional): The path or url of pretrained model. Default: None.
+    """
+
+    def __init__(self,
+                 backbone,
+                 decoder_cfg,
+                 num_classes,
+                 backbone_indices=[1, 2, 3],
+                 pretrained=None):
+        super().__init__()
+        self.backbone = backbone
+
+        in_channels = [self.backbone.feat_channels[i] for i in backbone_indices]
+        self.decode_head = LightHamHead(
+            in_channels=in_channels, num_classes=num_classes, **decoder_cfg)
+
+        self.align_corners = self.decode_head.align_corners
+        self.pretrained = pretrained
+        self.init_weights()
+
+    def init_weights(self):
+        if self.pretrained:
+            utils.load_entire_model(self, self.pretrained)
+
+    def forward(self, x):
+        input_size = paddle.shape(x)[2:]
+        feats = self.backbone(x)
+        out = self.decode_head(feats)
+        return [
+            F.interpolate(
+                out,
+                input_size,
+                mode="bilinear",
+                align_corners=self.align_corners)
+        ]
+
+
+class Hamburger(nn.Layer):
+    def __init__(self, ham_channels=512, num_groups=32, ham_kwargs=None):
+        super().__init__()
+        self.ham_in = nn.Conv2D(ham_channels, ham_channels, kernel_size=1)
+
+        self.ham = NMF2D(ham_kwargs)
+
+        self.ham_out = ConvGNAct(
+            ham_channels,
+            ham_channels,
+            kernel_size=1,
+            num_groups=num_groups,
+            bias_attr=False)
+
+    def forward(self, x):
+        enjoy = self.ham_in(x)
+        enjoy = F.relu(enjoy)
+        enjoy = self.ham(enjoy)
+        enjoy = self.ham_out(enjoy)
+        ham = F.relu(x + enjoy)
+
+        return ham
+
+
+class LightHamHead(nn.Layer):
+    """The head implementation of HamNet based on PaddlePaddle.
+    The original article refers to Zhengyang Geng, et al. "Is Attention Better Than Matrix Decomposition?"
+    (https://arxiv.org/abs/2109.04553.pdf)
+
+    Args:
+        in_channels (list[int]): The feature channels from backbone.
+        num_classes (int): The unique number of target classes.
+        channels (int, optional): The intermediate channel of LightHamHead. Default: 256.
+        dropout_rate (float, optional): The rate of dropout. Default: 0.1.
+        align_corners (bool, optional): Whether use align_corners when interpolating. Default: False.
+        ham_channels (int, optional): Input channel of Hamburger. Default: 512.
+        num_groups (int, optional): The num_groups of convolutions in LightHamHead. Default: 32.
+        ham_kwargs (dict, optional): Keyword arguments of Hamburger module.
+    """
+
+    def __init__(self,
+                 in_channels,
+                 num_classes,
+                 channels=256,
+                 dropout_rate=0.1,
+                 align_corners=False,
+                 ham_channels=512,
+                 num_groups=32,
+                 ham_kwargs=None):
+        super().__init__()
+
+        if len(in_channels) != 3:
+            raise ValueError(
+                "The length of `in_channels` must be 3, but got {}".format(
+                    len(in_channels)))
+
+        self.align_corners = align_corners
+
+        self.squeeze = ConvGNAct(
+            sum(in_channels),
+            ham_channels,
+            kernel_size=1,
+            num_groups=num_groups,
+            act_type="relu",
+            bias_attr=False)
+
+        self.hamburger = Hamburger(ham_channels, num_groups, ham_kwargs)
+
+        self.align = ConvGNAct(
+            ham_channels,
+            channels,
+            kernel_size=1,
+            num_groups=num_groups,
+            act_type="relu",
+            bias_attr=False)
+
+        self.dropout = (nn.Dropout2D(dropout_rate)
+                        if dropout_rate > 0.0 else nn.Identity())
+        self.conv_seg = nn.Conv2D(channels, num_classes, kernel_size=1)
+
+    def forward(self, inputs):
+        inputs = inputs[1:]
+        target_shape = paddle.shape(inputs[0])[2:]
+        inputs = [
+            F.interpolate(
+                level,
+                size=target_shape,
+                mode="bilinear",
+                align_corners=self.align_corners) for level in inputs
+        ]
+
+        inputs = paddle.concat(inputs, axis=1)
+        x = self.squeeze(inputs)
+
+        x = self.hamburger(x)
+
+        output = self.align(x)
+        output = self.dropout(output)
+        output = self.conv_seg(output)
+        return output

paddlers/models/paddleseg/models/upernet_cae.py

@@ -0,0 +1,343 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+
+from paddlers.models.paddleseg.utils import utils
+from paddlers.models.paddleseg.cvlibs import manager
+from paddlers.models.paddleseg.models import layers
+from paddlers.models.paddleseg.models.backbones.transformer_utils import Identity
+
+
+@manager.MODELS.add_component
+class UPerNetCAE(nn.Layer):
+    """ 
+    The UPerNet  with CAE as backbone implementation with PaddlePaddle.
+
+    The original paper refers to Unified Perceptual Parsing for Scene Understanding.
+    (https://arxiv.org/abs/1807.10221)
+
+    Args:
+        num_classes (int): The unique number of target classes.
+        backbone (Paddle.nn.Layer): Backbone network, currently support Resnet50/101.
+        backbone_indices (tuple): Four values in the tuple indicate the indices of output of backbone.
+        channels(int): Hidden layer channels of upernet head.
+        fpn_channels(int): The fpn_channels of upernet head.
+        head_channels(int): The inplane of upernet head.
+        channels_fpn(int): The channels_fpn of upernet head.
+        enable_auxiliary_loss (bool, optional): A bool value indicates whether adding auxiliary loss. Default: False.
+        align_corners (bool, optional): An argument of F.interpolate. It should be set to False when the feature size is even,
+            e.g. 1024x512, otherwise it is True, e.g. 769x769. Default: False.
+        dropout_ratio(float): The dropout ratio of upernet head.
+        pretrained (str, optional): The path or url of pretrained model. Default: None.
+    """
+
+    def __init__(self,
+                 num_classes,
+                 backbone,
+                 backbone_indices,
+                 channels,
+                 fpn_channels,
+                 head_channels,
+                 channels_fpn,
+                 enable_auxiliary_loss=True,
+                 align_corners=True,
+                 dropout_ratio=0.1,
+                 pretrained=None):
+        super(UPerNetCAE, self).__init__()
+        self._init_fpn(embed_dim=768, patch_size=16)
+        self.backbone = backbone
+        self.backbone_indices = backbone_indices
+        self.align_corners = align_corners
+        self.pretrained = pretrained
+        self.enable_auxiliary_loss = enable_auxiliary_loss
+
+        self.fpn_channels = fpn_channels
+        self.head_channels = head_channels
+        self.channels_fpn = channels_fpn
+
+        self.decode_head = UPerNetHead(
+            inplane=head_channels,
+            num_class=num_classes,
+            channels_fpn=channels_fpn,
+            dropout_ratio=dropout_ratio,
+            channels=channels,
+            fpn_channels=fpn_channels,
+            enable_auxiliary_loss=self.enable_auxiliary_loss)
+        self.init_weight()
+
+    def _init_fpn(self, embed_dim=768, patch_size=16, out_with_norm=False):
+        if patch_size == 16:
+            self.fpn1 = nn.Sequential(
+                nn.Conv2DTranspose(
+                    embed_dim, embed_dim, kernel_size=2, stride=2),
+                nn.SyncBatchNorm(
+                    embed_dim, momentum=0.1),
+                nn.GELU(),
+                nn.Conv2DTranspose(
+                    embed_dim, embed_dim, kernel_size=2, stride=2), )
+
+            self.fpn2 = nn.Sequential(
+                nn.Conv2DTranspose(
+                    embed_dim, embed_dim, kernel_size=2, stride=2), )
+
+            self.fpn3 = Identity()
+
+            self.fpn4 = nn.MaxPool2D(kernel_size=2, stride=2)
+        elif patch_size == 8:
+            self.fpn1 = nn.Sequential(
+                nn.Conv2DTranspose(
+                    embed_dim, embed_dim, kernel_size=2, stride=2), )
+
+            self.fpn2 = Identity()
+
+            self.fpn3 = nn.Sequential(nn.MaxPool2D(kernel_size=2, stride=2), )
+
+            self.fpn4 = nn.Sequential(nn.MaxPool2D(kernel_size=4, stride=4), )
+
+        if not out_with_norm:
+            self.norm = Identity()
+        else:
+            self.norm = nn.LayerNorm(embed_dim, epsilon=1e-6)
+
+    def forward(self, x):
+        feats, feats_shape = self.backbone(x)  # [1, 1024, 768]
+        B, _, Hp, Wp = feats_shape
+
+        feats = [feats[i] for i in self.backbone_indices]
+
+        for i, feat in enumerate(feats):
+            feats[i] = paddle.reshape(
+                paddle.transpose(
+                    self.norm(feat), perm=[0, 2, 1]),
+                shape=[B, -1, Hp, Wp])
+
+        ops = [self.fpn1, self.fpn2, self.fpn3, self.fpn4]
+        for i in range(len(feats)):
+            feats[i] = ops[i](feats[i])
+
+        logit_list = self.decode_head(feats)
+        logit_list = [
+            F.interpolate(
+                logit,
+                paddle.shape(x)[2:],
+                mode='bilinear',
+                align_corners=False) for logit in logit_list
+        ]
+        return logit_list
+
+    def init_weight(self):
+        if self.pretrained is not None:
+            utils.load_entire_model(self, self.pretrained)
+
+
+class PPModuleCAE(nn.Layer):
+    """
+    Pyramid pooling module originally in PSPNet.
+
+    Args:
+        in_channels (int): The number of intput channels to pyramid pooling module.
+        out_channels (int): The number of output channels after pyramid pooling module.
+        bin_sizes (tuple, optional): The out size of pooled feature maps. Default: (1, 2, 3, 6).
+        dim_reduction (bool, optional): A bool value represents if reducing dimension after pooling. Default: True.
+        align_corners (bool): An argument of F.interpolate. It should be set to False when the output size of feature
+            is even, e.g. 1024x512, otherwise it is True, e.g. 769x769.
+    """
+
+    def __init__(self, in_channels, out_channels, bin_sizes, dim_reduction,
+                 align_corners):
+        super().__init__()
+
+        self.bin_sizes = bin_sizes
+
+        inter_channels = in_channels
+        if dim_reduction:
+            inter_channels = in_channels // len(bin_sizes)
+
+        # we use dimension reduction after pooling mentioned in original implementation.
+        self.stages = nn.LayerList([
+            self._make_stage(in_channels, inter_channels, size)
+            for size in bin_sizes
+        ])
+
+        self.conv_bn_relu2 = layers.ConvBNReLU(
+            in_channels=in_channels + inter_channels * len(bin_sizes),
+            out_channels=out_channels,
+            kernel_size=3,
+            padding=1,
+            bias_attr=False)
+
+        self.align_corners = align_corners
+
+    def _make_stage(self, in_channels, out_channels, size):
+        """
+        Create one pooling layer.
+
+        In our implementation, we adopt the same dimension reduction as the original paper that might be
+        slightly different with other implementations.
+
+        After pooling, the channels are reduced to 1/len(bin_sizes) immediately, while some other implementations
+        keep the channels to be same.
+
+        Args:
+            in_channels (int): The number of intput channels to pyramid pooling module.
+            size (int): The out size of the pooled layer.
+
+        Returns:
+            conv (Tensor): A tensor after Pyramid Pooling Module.
+        """
+
+        prior = nn.AdaptiveAvgPool2D(output_size=(size, size))
+        conv = layers.ConvBNReLU(
+            in_channels=in_channels,
+            out_channels=out_channels,
+            kernel_size=1,
+            bias_attr=False)
+
+        return nn.Sequential(prior, conv)
+
+    def forward(self, input):
+        cat_layers = []
+        for stage in self.stages:
+            x = stage(input)
+            x = F.interpolate(
+                x,
+                paddle.shape(input)[2:],
+                mode='bilinear',
+                align_corners=self.align_corners)
+            cat_layers.append(x)
+        cat_layers = [input] + cat_layers
+        cat = paddle.concat(cat_layers, axis=1)
+        out = self.conv_bn_relu2(cat)
+
+        return out
+
+
+class UPerNetHead(nn.Layer):
+    """
+    The UPerNetHead implementation.
+
+    Args:
+        inplane (int): Input channels of PPM module.
+        num_class (int): The unique number of target classes.
+        channels_fpn (list): The feature channels from backbone.
+        fpn_channels (int, optional): The input channels of FPN module. Default: 512.
+        enable_auxiliary_loss (bool, optional): A bool value indicates whether adding auxiliary loss. Default: False.
+    """
+
+    def __init__(self,
+                 inplane,
+                 num_class,
+                 channels_fpn,
+                 channels,
+                 dropout_ratio=0.1,
+                 fpn_channels=512,
+                 enable_auxiliary_loss=False):
+        super(UPerNetHead, self).__init__()
+        self.psp_modules = PPModuleCAE(
+            in_channels=inplane,
+            out_channels=fpn_channels,
+            bin_sizes=(1, 2, 3, 6),
+            dim_reduction=False,
+            align_corners=False)
+
+        self.enable_auxiliary_loss = enable_auxiliary_loss
+        self.lateral_convs = []
+        self.fpn_convs = []
+
+        for fpn_inplane in channels_fpn[:-1]:
+            self.lateral_convs.append(
+                nn.Sequential(
+                    nn.Conv2D(
+                        fpn_inplane, fpn_channels, 1, bias_attr=False),
+                    layers.SyncBatchNorm(fpn_channels),
+                    nn.ReLU()))
+            self.fpn_convs.append(
+                nn.Sequential(
+                    layers.ConvBNReLU(
+                        fpn_channels, fpn_channels, 3, bias_attr=False)))
+
+        self.lateral_convs = nn.LayerList(self.lateral_convs)
+        self.fpn_convs = nn.LayerList(self.fpn_convs)
+
+        if self.enable_auxiliary_loss:
+            if dropout_ratio is not None:
+                self.dsn = nn.Sequential(
+                    layers.ConvBNReLU(
+                        channels_fpn[2], 256, 3, padding=1, bias_attr=False),
+                    nn.Dropout2D(dropout_ratio),
+                    nn.Conv2D(
+                        256, num_class, kernel_size=1))
+            else:
+                self.dsn = nn.Sequential(
+                    layers.ConvBNReLU(
+                        channels_fpn[2], 256, 3, padding=1, bias_attr=False),
+                    nn.Conv2D(
+                        256, num_class, kernel_size=1))
+
+        if dropout_ratio is not None:
+            self.dropout = nn.Dropout2D(dropout_ratio)
+        else:
+            self.dropout = None
+
+        self.fpn_bottleneck = layers.ConvBNReLU(
+            len(channels_fpn) * channels,
+            channels,
+            3,
+            padding=1,
+            bias_attr=False)
+        self.conv_seg = nn.Conv2D(channels, num_class, kernel_size=1)
+
+    def cls_seg(self, feat):
+        if self.dropout is not None:
+            feat = self.dropout(feat)
+        output = self.conv_seg(feat)
+        return output
+
+    def forward(self, conv_out):
+        psp_out = self.psp_modules(conv_out[-1])
+        f = psp_out
+        fpn_feature_list = [psp_out]
+        out = []
+
+        for i in reversed(range(len(conv_out) - 1)):
+            conv_x = conv_out[i]
+            conv_x = self.lateral_convs[i](conv_x)
+            prev_shape = paddle.shape(conv_x)[2:]
+            f = conv_x + F.interpolate(
+                f, prev_shape, mode='bilinear', align_corners=False)
+            fpn_feature_list.append(self.fpn_convs[i](f))
+
+        fpn_feature_list.reverse()
+        output_size = fpn_feature_list[0].shape[2:]
+        # resize multi-scales feature
+        for index in range(len(conv_out) - 1, 0, -1):
+            fpn_feature_list[index] = F.interpolate(
+                fpn_feature_list[index],
+                size=output_size,
+                mode='bilinear',
+                align_corners=False)
+        fusion_out = paddle.concat(fpn_feature_list, 1)
+        x = self.fpn_bottleneck(fusion_out)
+        x = self.cls_seg(x)
+
+        if self.enable_auxiliary_loss:
+            dsn = self.dsn(conv_out[2])
+            out.append(x)
+            out.append(dsn)
+            return out
+        else:
+            return [x]

paddlers/models/paddleseg/models/upernet_vit_adapter.py

@@ -0,0 +1,277 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+
+from paddlers.models.paddleseg import utils
+from paddlers.models.paddleseg.cvlibs import manager
+from paddlers.models.paddleseg.models import layers
+
+
+@manager.MODELS.add_component
+class UPerNetViTAdapter(nn.Layer):
+    """
+    The UPerNetViTAdapter implementation based on PaddlePaddle.
+
+    The original article refers to
+    Chen, Zhe, Yuchen Duan, Wenhai Wang, Junjun He, Tong Lu, Jifeng Dai, and Yu Qiao. 
+    "Vision Transformer Adapter for Dense Predictions." 
+    (https://arxiv.org/abs/2205.08534).
+
+    The implementation is based on https://github.com/czczup/ViT-Adapter
+
+    Args:
+        num_classes (int): The unique number of target classes.
+        backbone (nn.Layer): The backbone network.
+        backbone_indices (tuple | list): The values indicate the indices of output of backbone.
+        channels (int, optional): The channels of inter layers in upernet head. Default: 512.
+        pool_scales (list, optional): The scales in PPM. Default: [1, 2, 3, 6].
+        dropout_ratio (float, optional): The dropout ratio for upernet head. Default: 0.1.
+        aux_loss (bool, optional): A bool value indicates whether adding auxiliary loss. Default: True.
+        aux_channels (int, optional): The channels of inter layers in auxiliary head. Default: 256.
+        align_corners (bool, optional): An argument of F.interpolate. It should be set to False when the feature size is even,
+            e.g. 1024x512, otherwise it is True, e.g. 769x769. Default: False.
+        pretrained (str, optional): The path or url of pretrained model. Default: None.
+    """
+
+    def __init__(self,
+                 num_classes,
+                 backbone,
+                 backbone_indices,
+                 channels=512,
+                 pool_scales=[1, 2, 3, 6],
+                 dropout_ratio=0.1,
+                 aux_loss=True,
+                 aux_channels=256,
+                 align_corners=False,
+                 pretrained=None):
+        super().__init__()
+        self.backbone = backbone
+        self.backbone_indices = backbone_indices
+        self.align_corners = align_corners
+
+        in_channels = [self.backbone.feat_channels[i] for i in backbone_indices]
+        self.head = UPerNetHead(
+            num_classes=num_classes,
+            in_channels=in_channels,
+            channels=channels,
+            pool_scales=pool_scales,
+            dropout_ratio=dropout_ratio,
+            aux_loss=aux_loss,
+            aux_channels=aux_channels,
+            align_corners=align_corners)
+
+        self.pretrained = pretrained
+        self.init_weight()
+
+    def init_weight(self):
+        if self.pretrained is not None:
+            utils.load_entire_model(self, self.pretrained)
+
+    def forward(self, x):
+        feats = self.backbone(x)
+        feats = [feats[i] for i in self.backbone_indices]
+        logit_list = self.head(feats)
+        logit_list = [
+            F.interpolate(
+                logit,
+                paddle.shape(x)[2:],
+                mode='bilinear',
+                align_corners=self.align_corners) for logit in logit_list
+        ]
+        return logit_list
+
+
+class ConvBNReLU(nn.Layer):
+    def __init__(self,
+                 in_channels,
+                 out_channels,
+                 kernel_size,
+                 bias_attr=False,
+                 **kwargs):
+        super().__init__()
+        self.conv = nn.Conv2D(
+            in_channels,
+            out_channels,
+            kernel_size,
+            bias_attr=bias_attr,
+            **kwargs)
+        self.bn = nn.BatchNorm2D(out_channels)
+        self.relu = nn.ReLU()
+
+    def forward(self, x):
+        x = self.conv(x)
+        x = self.bn(x)
+        x = self.relu(x)
+        return x
+
+
+class PPM(nn.Layer):
+    """Pooling Pyramid Module used in PSPNet.
+
+    Args:
+        pool_scales (tuple | list): Pooling scales used in PPM.
+        in_channels (int): Input channels.
+        channels (int): Output Channels after modules, before conv_seg.
+        act_cfg (dict): Config of activation layers.
+        align_corners (bool): align_corners argument of F.interpolate.
+    """
+
+    def __init__(self, pool_scales, in_channels, channels, align_corners):
+        super().__init__()
+        self.pool_scales = pool_scales
+        self.in_channels = in_channels
+        self.channels = channels
+        self.align_corners = align_corners
+        self.stages = nn.LayerList()
+        for pool_scale in pool_scales:
+            self.stages.append(
+                nn.Sequential(
+                    nn.AdaptiveAvgPool2D(output_size=(pool_scale, pool_scale)),
+                    ConvBNReLU(
+                        in_channels=in_channels,
+                        out_channels=channels,
+                        kernel_size=1)))
+
+    def forward(self, x):
+        ppm_outs = []
+        for ppm in self.stages:
+            ppm_out = ppm(x)
+            upsampled_ppm_out = F.interpolate(
+                ppm_out,
+                paddle.shape(x)[2:],
+                mode='bilinear',
+                align_corners=self.align_corners)
+            ppm_outs.append(upsampled_ppm_out)
+        return ppm_outs
+
+
+class UPerNetHead(nn.Layer):
+    """
+    This head is the implementation of "Unified Perceptual Parsing for Scene Understanding".
+    This is heavily based on https://github.com/czczup/ViT-Adapter
+
+    Args:
+        num_classes (int): The unique number of target classes.
+        in_channels (list[int]): The channels of input features.
+        channels (int, optional): The channels of inter layers in upernet head. Default: 512.
+        pool_scales (list, optional): The scales in PPM. Default: [1, 2, 3, 6].
+        dropout_ratio (float, optional): The dropout ratio for upernet head. Default: 0.1.
+        aux_loss (bool, optional): A bool value indicates whether adding auxiliary loss. Default: True.
+        aux_channels (int, optional): The channels of inter layers in auxiliary head. Default: 256.
+        align_corners (bool, optional): An argument of F.interpolate. It should be set to False when the feature size is even,
+            e.g. 1024x512, otherwise it is True, e.g. 769x769. Default: False.
+    """
+
+    def __init__(self,
+                 num_classes,
+                 in_channels,
+                 channels,
+                 pool_scales=[1, 2, 3, 6],
+                 dropout_ratio=0.1,
+                 aux_loss=False,
+                 aux_channels=256,
+                 align_corners=False):
+        super().__init__()
+        self.align_corners = align_corners
+
+        # PSP Module
+        self.psp_modules = PPM(pool_scales,
+                               in_channels[-1],
+                               channels,
+                               align_corners=align_corners)
+        self.bottleneck = ConvBNReLU(
+            in_channels[-1] + len(pool_scales) * channels,
+            channels,
+            3,
+            padding=1)
+        # FPN Module
+        self.lateral_convs = nn.LayerList()
+        self.fpn_convs = nn.LayerList()
+        for ch in in_channels[:-1]:  # skip the top layer
+            l_conv = ConvBNReLU(ch, channels, 1)
+            fpn_conv = ConvBNReLU(channels, channels, 3, padding=1)
+            self.lateral_convs.append(l_conv)
+            self.fpn_convs.append(fpn_conv)
+
+        self.fpn_bottleneck = ConvBNReLU(
+            len(in_channels) * channels, channels, 3, padding=1)
+        if dropout_ratio > 0:
+            self.dropout = nn.Dropout2D(dropout_ratio)
+        else:
+            self.dropout = None
+        self.conv_seg = nn.Conv2D(channels, num_classes, kernel_size=1)
+
+        self.aux_loss = aux_loss
+        if self.aux_loss:
+            self.aux_conv = ConvBNReLU(
+                in_channels[2], aux_channels, 3, padding=1)
+            self.aux_conv_seg = nn.Conv2D(
+                aux_channels, num_classes, kernel_size=1)
+
+    def psp_forward(self, inputs):
+        x = inputs[-1]
+        psp_outs = [x]
+        psp_outs.extend(self.psp_modules(x))
+        psp_outs = paddle.concat(psp_outs, axis=1)
+        output = self.bottleneck(psp_outs)
+        return output
+
+    def forward(self, inputs):
+        # build laterals
+        laterals = [
+            lateral_conv(inputs[i])
+            for i, lateral_conv in enumerate(self.lateral_convs)
+        ]
+        laterals.append(self.psp_forward(inputs))
+
+        # build top-down path
+        used_backbone_levels = len(laterals)
+        for i in range(used_backbone_levels - 1, 0, -1):
+            upsampled = F.interpolate(
+                laterals[i],
+                paddle.shape(laterals[i - 1])[2:],
+                mode='bilinear',
+                align_corners=self.align_corners)
+            laterals[i - 1] = laterals[i - 1] + upsampled
+
+        # build outputs
+        fpn_outs = [
+            self.fpn_convs[i](laterals[i])
+            for i in range(used_backbone_levels - 1)
+        ]
+        fpn_outs.append(laterals[-1])  # append psp feature
+
+        for i in range(used_backbone_levels - 1, 0, -1):
+            fpn_outs[i] = F.interpolate(
+                fpn_outs[i],
+                size=paddle.shape(fpn_outs[0])[2:],
+                mode='bilinear',
+                align_corners=self.align_corners)
+        fpn_outs = paddle.concat(fpn_outs, axis=1)
+        output = self.fpn_bottleneck(fpn_outs)
+
+        if self.dropout is not None:
+            output = self.dropout(output)
+        output = self.conv_seg(output)
+        logits_list = [output]
+
+        if self.aux_loss and self.training:
+            aux_output = self.aux_conv(inputs[2])
+            aux_output = self.aux_conv_seg(aux_output)
+            logits_list.append(aux_output)
+
+        return logits_list

paddlers/models/paddleseg/optimizers/__init__.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+
+from .optimizer import *
+from .custom_optimizers import *

paddlers/models/paddleseg/optimizers/custom_optimizers.py

@@ -0,0 +1,177 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+import numpy as np
+from functools import partial
+
+import paddle
+from paddle.optimizer import AdamW
+
+
+def layerwise_lr_decay(decay_rate, name_dict, n_layers, param):
+    """
+    Args:
+        decay_rate (float): 
+            The layer-wise decay ratio.
+        name_dict (dict): 
+            The keys of name_dict is dynamic name of model while the value
+            of name_dict is static name.
+            Use model.named_parameters() to get name_dict.
+        n_layers (int):
+            Total number of layers in the transformer encoder.
+    """
+    ratio = 1.0
+    static_name = name_dict[param.name]
+    if "blocks" in static_name:
+        idx = static_name.find("blocks.")
+        layer = int(static_name[idx:].split(".")[1])
+        ratio = decay_rate**(n_layers - layer)
+    elif "embed" in static_name:
+        ratio = decay_rate**(n_layers + 1)
+    param.optimize_attr["learning_rate"] *= ratio
+
+
+class AdamWDL(AdamW):
+    r"""
+    The AdamWDL optimizer is implemented based on the AdamW Optimization with dynamic lr setting.
+    Generally it's used for transformer model.
+    We use "layerwise_lr_decay" as default dynamic lr setting method of AdamWDL.
+    “Layer-wise decay” means exponentially decaying the learning rates of individual 
+    layers in a top-down manner. For example, suppose the 24-th layer uses a learning
+    rate l, and the Layer-wise decay rate is α, then the learning rate of layer m 
+    is lα^(24-m). See more details on: https://arxiv.org/abs/1906.08237.
+    .. math::
+        & t = t + 1
+    
+        & moment\_1\_out = {\beta}_1 * moment\_1 + (1 - {\beta}_1) * grad
+        & moment\_2\_out = {\beta}_2 * moment\_2 + (1 - {\beta}_2) * grad * grad
+        & learning\_rate = learning\_rate * \frac{\sqrt{1 - {\beta}_2^t}}{1 - {\beta}_1^t}
+        & param\_out = param - learning\_rate * (\frac{moment\_1}{\sqrt{moment\_2} + \epsilon} + \lambda * param)
+    Args:
+        learning_rate (float|LRScheduler, optional): The learning rate used to update ``Parameter``.
+            It can be a float value or a LRScheduler. The default value is 0.001.
+        beta1 (float, optional): The exponential decay rate for the 1st moment estimates.
+            It should be a float number or a Tensor with shape [1] and data type as float32.
+            The default value is 0.9.
+        beta2 (float, optional): The exponential decay rate for the 2nd moment estimates.
+            It should be a float number or a Tensor with shape [1] and data type as float32.
+            The default value is 0.999.
+        epsilon (float, optional): A small float value for numerical stability.
+            It should be a float number or a Tensor with shape [1] and data type as float32.
+            The default value is 1e-08.
+        parameters (list|tuple, optional): List/Tuple of ``Tensor`` to update to minimize ``loss``. \
+            This parameter is required in dygraph mode. \
+            The default value is None in static mode, at this time all parameters will be updated.
+        weight_decay (float, optional): The weight decay coefficient, it can be float or Tensor. The default value is 0.01.
+        apply_decay_param_fun (function|None, optional): If it is not None,
+            only tensors that makes apply_decay_param_fun(Tensor.name)==True
+            will be updated. It only works when we want to specify tensors.
+            Default: None.
+        grad_clip (GradientClipBase, optional): Gradient cliping strategy, it's an instance of
+            some derived class of ``GradientClipBase`` . There are three cliping strategies
+            ( :ref:`api_fluid_clip_GradientClipByGlobalNorm` , :ref:`api_fluid_clip_GradientClipByNorm` ,
+            :ref:`api_fluid_clip_GradientClipByValue` ). Default None, meaning there is no gradient clipping.
+        lazy_mode (bool, optional): The official Adam algorithm has two moving-average accumulators.
+            The accumulators are updated at every step. Every element of the two moving-average
+            is updated in both dense mode and sparse mode. If the size of parameter is very large,
+            then the update may be very slow. The lazy mode only update the element that has
+            gradient in current mini-batch, so it will be much more faster. But this mode has
+            different semantics with the original Adam algorithm and may lead to different result.
+            The default value is False.
+        multi_precision (bool, optional): Whether to use multi-precision during weight updating. Default is false.  
+        layerwise_decay (float, optional): The layer-wise decay ratio. Defaults to 1.0.
+        n_layers (int, optional): The total number of encoder layers. Defaults to 12.
+        set_param_lr_fun (function|None, optional): If it's not None, set_param_lr_fun() will set the parameter 
+            learning rate before it executes Adam Operator. Defaults to :ref:`layerwise_lr_decay`.
+        name_dict (dict, optional): The keys of name_dict is dynamic name of model while the value
+            of name_dict is static name. Use model.named_parameters() to get name_dict.
+        name (str, optional): Normally there is no need for user to set this property.
+            For more information, please refer to :ref:`api_guide_Name`.
+            The default value is None.
+    Examples:
+        .. code-block:: python
+            import paddle
+            from paddlenlp.ops.optimizer import AdamWDL
+            def simple_lr_setting(decay_rate, name_dict, n_layers, param):
+                ratio = 1.0
+                static_name = name_dict[param.name]
+                if "weight" in static_name:
+                    ratio = decay_rate**0.5
+                param.optimize_attr["learning_rate"] *= ratio
+            
+            linear = paddle.nn.Linear(10, 10)
+            name_dict = dict()
+            for n, p in linear.named_parameters():
+                name_dict[p.name] = n
+            inp = paddle.rand([10,10], dtype="float32")
+            out = linear(inp)
+            loss = paddle.mean(out)
+            adamwdl = AdamWDL(
+                learning_rate=1e-4,
+                parameters=linear.parameters(),
+                set_param_lr_fun=simple_lr_setting,
+                layerwise_decay=0.8,
+                name_dict=name_dict)
+            
+            loss.backward()
+            adamwdl.step()
+            adamwdl.clear_grad()
+    """
+
+    def __init__(self,
+                 learning_rate=0.001,
+                 beta1=0.9,
+                 beta2=0.999,
+                 epsilon=1e-8,
+                 parameters=None,
+                 weight_decay=0.0,
+                 apply_decay_param_fun=None,
+                 grad_clip=None,
+                 lazy_mode=False,
+                 multi_precision=False,
+                 layerwise_decay=1.0,
+                 n_layers=12,
+                 set_param_lr_fun=layerwise_lr_decay,
+                 name_dict=None,
+                 name=None):
+        self.layerwise_decay = layerwise_decay
+        self.n_layers = n_layers
+        self.set_param_lr_fun = partial(set_param_lr_fun, layerwise_decay,
+                                        name_dict, n_layers)
+        super(AdamWDL, self).__init__(
+            learning_rate=learning_rate,
+            parameters=parameters,
+            beta1=beta1,
+            beta2=beta2,
+            epsilon=epsilon,
+            grad_clip=grad_clip,
+            name=name,
+            apply_decay_param_fun=apply_decay_param_fun,
+            weight_decay=weight_decay,
+            lazy_mode=lazy_mode,
+            multi_precision=multi_precision)
+
+    def _append_optimize_op(self, block, param_and_grad):
+        if self.set_param_lr_fun is None:
+            return super(AdamWDL, self)._append_optimize_op(block,
+                                                            param_and_grad)
+
+        prev_lr = param_and_grad[0].optimize_attr["learning_rate"]
+        self.set_param_lr_fun(param_and_grad[0])
+        # excute Adam op
+        res = super(AdamWDL, self)._append_optimize_op(block, param_and_grad)
+
+        param_and_grad[0].optimize_attr["learning_rate"] = prev_lr
+        return res

paddlers/models/paddleseg/optimizers/optimizer.py

@@ -0,0 +1,309 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paddle
+
+from paddlers.models.paddleseg.cvlibs import manager
+from paddlers.models.paddleseg.utils import logger
+import paddlers.models.paddleseg.optimizers.custom_optimizers as custom_opt
+
+
+class BaseOptimizer(object):
+    """
+    Base optimizer in PaddleSeg.
+
+    Args:
+        weight_decay(float, optional): A float value as coeff of L2 regularization.
+        grad_clip_cfg(dict, optional): A dict to specify grad_clip. It must have the following format: 
+            {'name': 'ClipGradByGlobalNorm', 'clip_norm': float_val},
+            {'name': 'ClipGradByNorm', 'clip_norm': float_val},
+            {'name': 'ClipGradByValue', 'max': float_val, 'min': float_val(optional)}.
+        custom_cfg(list, optional): custom_cfg specify different options for
+            different parameter groups such as the learning rate and weight decay.
+            For example, [{'name': 'backbone', 'lr_mult': 0.1}, {'name': 'norm', 'weight_decay_mult': 0}]
+    
+    An example in config:
+    `
+    optimizer:
+      type: SGD
+      weight_decay: 4.0e-5
+      custom_cfg:
+        - name: backbone
+          lr_mult: 0.1
+        - name: norm
+          weight_decay_mult: 0.0
+      grad_clip_cfg:
+        name: ClipGradByValue
+        max: 1.0
+    `
+    """
+
+    def __init__(self, weight_decay=None, grad_clip_cfg=None, custom_cfg=None):
+        if weight_decay is not None:
+            assert isinstance(weight_decay, float), \
+                "`weight_decay` must be a float."
+        if grad_clip_cfg is not None:
+            assert isinstance(grad_clip_cfg, dict), \
+                "`grad_clip_cfg` must be a dict."
+            assert 'name' in grad_clip_cfg, 'No name specified in grad_clip_cfg'
+            grad_clip_names = [
+                'ClipGradByValue', 'ClipGradByNorm', 'ClipGradByGlobalNorm'
+            ]
+            assert grad_clip_cfg['name'] in grad_clip_names, \
+                'grad_clip name should be {}'.format(grad_clip_names)
+        if custom_cfg is not None:
+            assert isinstance(custom_cfg, list), "`custom_cfg` must be a list."
+            for item in custom_cfg:
+                assert isinstance(
+                    item, dict), "The item of `custom_cfg` must be a dict"
+
+        self.weight_decay = weight_decay
+        self.custom_cfg = custom_cfg
+        self.args = {'weight_decay': weight_decay}
+
+        if grad_clip_cfg is not None:
+            grad_clip_cfg = grad_clip_cfg.copy()
+            grad_clip_name = grad_clip_cfg.pop('name')
+            try:
+                grad_clip = getattr(paddle.nn, grad_clip_name)(**grad_clip_cfg)
+            except Exception as e:
+                raise RuntimeError(
+                    "Create grad_clip has failed. Please check grad_clip_cfg in config. "
+                    f"The error message: \n{str(e)}")
+            self.args.update({'grad_clip': grad_clip})
+
+    def __call__(self, model, lr):
+        # Create optimizer
+        pass
+
+    def _collect_params(self, model):
+        # Collect different parameter groups
+        if self.custom_cfg is None or len(self.custom_cfg) == 0:
+            return model.parameters()
+
+        groups_num = len(self.custom_cfg) + 1
+        params_list = [[] for _ in range(groups_num)]
+        for name, param in model.named_parameters():
+            if param.stop_gradient:
+                continue
+            for idx, item in enumerate(self.custom_cfg):
+                if item['name'] in name:
+                    params_list[idx].append(param)
+                    break
+            else:
+                params_list[-1].append(param)
+
+        res = []
+        for idx, item in enumerate(self.custom_cfg):
+            lr_mult = item.get("lr_mult", 1.0)
+            weight_decay_mult = item.get("weight_decay_mult", None)
+            param_dict = {'params': params_list[idx], 'learning_rate': lr_mult}
+            if self.weight_decay is not None and weight_decay_mult is not None:
+                param_dict[
+                    'weight_decay'] = self.weight_decay * weight_decay_mult
+            res.append(param_dict)
+        res.append({'params': params_list[-1]})
+
+        msg = 'Parameter groups for optimizer: \n'
+        for idx, item in enumerate(self.custom_cfg):
+            params_name = [p.name for p in params_list[idx]]
+            item = item.copy()
+            item['params_name'] = params_name
+            msg += 'Group {}: \n{} \n'.format(idx, item)
+        msg += 'Last group:\n params_name: {}'.format(
+            [p.name for p in params_list[-1]])
+        logger.info(msg)
+
+        return res
+
+
+@manager.OPTIMIZERS.add_component
+class SGD(BaseOptimizer):
+    """
+    SGD optimizer. 
+
+    An example in config:
+    `
+    optimizer:
+      type: SGD
+      weight_decay: 4.0e-5
+      custom_cfg:
+        - name: backbone
+          lr_mult: 0.1
+        - name: norm
+          weight_decay_mult: 0.0
+    `
+    """
+
+    def __init__(self, weight_decay=None, grad_clip_cfg=None, custom_cfg=None):
+        super().__init__(weight_decay, grad_clip_cfg, custom_cfg)
+
+    def __call__(self, model, lr):
+        params = self._collect_params(model)
+        return paddle.optimizer.SGD(learning_rate=lr,
+                                    parameters=params,
+                                    **self.args)
+
+
+@manager.OPTIMIZERS.add_component
+class Momentum(BaseOptimizer):
+    """
+    Momentum optimizer. 
+    """
+
+    def __init__(self,
+                 momentum=0.9,
+                 use_nesterov=False,
+                 weight_decay=None,
+                 grad_clip_cfg=None,
+                 custom_cfg=None):
+        super().__init__(weight_decay, grad_clip_cfg, custom_cfg)
+        self.args.update({'momentum': momentum, 'use_nesterov': use_nesterov})
+
+    def __call__(self, model, lr):
+        params = self._collect_params(model)
+        return paddle.optimizer.Momentum(
+            learning_rate=lr, parameters=params, **self.args)
+
+
+@manager.OPTIMIZERS.add_component
+class Adam(BaseOptimizer):
+    """
+    Adam optimizer. 
+    """
+
+    def __init__(self,
+                 beta1=0.9,
+                 beta2=0.999,
+                 epsilon=1e-08,
+                 lazy_mode=False,
+                 weight_decay=None,
+                 grad_clip_cfg=None,
+                 custom_cfg=None):
+        super().__init__(weight_decay, grad_clip_cfg, custom_cfg)
+        self.args.update({
+            'beta1': beta1,
+            'beta2': beta2,
+            'epsilon': epsilon,
+            'lazy_mode': lazy_mode
+        })
+
+    def __call__(self, model, lr):
+        params = self._collect_params(model)
+        opt = paddle.optimizer.Adam(
+            learning_rate=lr, parameters=params, **self.args)
+        return opt
+
+
+@manager.OPTIMIZERS.add_component
+class AdamW(BaseOptimizer):
+    """
+    AdamW optimizer. 
+    """
+
+    def __init__(self,
+                 beta1=0.9,
+                 beta2=0.999,
+                 epsilon=1e-08,
+                 weight_decay=0.01,
+                 lazy_mode=False,
+                 grad_clip_cfg=None,
+                 custom_cfg=None):
+        super().__init__(weight_decay, grad_clip_cfg, custom_cfg)
+        self.args.update({
+            'beta1': beta1,
+            'beta2': beta2,
+            'epsilon': epsilon,
+            'lazy_mode': lazy_mode
+        })
+
+    def __call__(self, model, lr):
+        params = self._collect_params(model)
+        opt = paddle.optimizer.AdamW(
+            learning_rate=lr, parameters=params, **self.args)
+        return opt
+
+
+@manager.OPTIMIZERS.add_component
+class AdamWDL(BaseOptimizer):
+    """
+    AdamW optimizer. 
+    """
+
+    def __init__(self,
+                 beta1=0.9,
+                 beta2=0.999,
+                 weight_decay=0.01,
+                 layerwise_decay=0.65,
+                 lazy_mode=False,
+                 grad_clip_cfg=None,
+                 custom_cfg=None):
+        super().__init__(weight_decay, grad_clip_cfg, custom_cfg)
+        self.args.update({
+            'beta1': beta1,
+            'beta2': beta2,
+            'weight_decay': weight_decay,
+            'layerwise_decay': layerwise_decay,
+            'lazy_mode': lazy_mode
+        })
+
+    def __call__(self, model, lr):
+        params = self._collect_params(model)
+        opt = custom_opt.AdamWDL(
+            learning_rate=lr, parameters=params, **self.args)
+        return opt
+
+
+@manager.OPTIMIZERS.add_component
+class AdamWDL_CAE(AdamWDL):
+    """
+    AdamW optimizer. 
+    """
+
+    def __init__(self,
+                 beta1=0.9,
+                 beta2=0.999,
+                 weight_decay=0.01,
+                 layerwise_decay=0.65,
+                 lazy_mode=False,
+                 grad_clip_cfg=None,
+                 custom_cfg=None):
+        super().__init__(weight_decay, grad_clip_cfg, custom_cfg)
+        self.args.update({
+            'beta1': beta1,
+            'beta2': beta2,
+            'weight_decay': weight_decay,
+            'layerwise_decay': layerwise_decay,
+            'lazy_mode': lazy_mode
+        })
+
+    def __call__(self, model, lr):
+        params = self._collect_params(model)
+        skip_list = model.backbone.no_weight_decay()
+
+        decay_dict = {
+            param.name: not (len(param.shape) == 1 or name.endswith(".bias") or
+                             name in skip_list)
+            for name, param in model.named_parameters()
+        }
+        self.args['n_layers'] = model.backbone.get_num_layers()
+        self.args['apply_decay_param_fun'] = lambda n: decay_dict[n]
+        name_dict = dict()
+        for n, p in model.named_parameters():
+            name_dict[p.name] = n
+        self.args['name_dict'] = name_dict
+
+        opt = custom_opt.AdamWDL(
+            learning_rate=lr, parameters=params, **self.args)
+        return opt

paddlers/models/paddleseg/transforms/functional.py

@@ -18,6 +18,12 @@ from PIL import Image, ImageEnhance
 from scipy.ndimage import distance_transform_edt
 
 
+def crop(img, crop_coordinate):
+    x1, y1, x2, y2 = crop_coordinate
+    img = img[y1:y2, x1:x2, ...]
+    return img
+
+
 def rescale_size(img_size, target_size):
     scale = min(
         max(target_size) / max(img_size), min(target_size) / min(img_size))

paddlers/models/paddleseg/transforms/transforms.py

@@ -243,13 +243,21 @@ class ResizeByShort:
 
     Args:
         short_size (int): The target size of short side.
+        max_size(int): The maximum length of resized image's long edge, if the resized image's long edge exceed this length, short size will be adjusted.
     """
 
-    def __init__(self, short_size):
+    def __init__(self, short_size, max_size=1e10):
+        if isinstance(short_size, list):
+            short_size = random.choice(short_size)
         self.short_size = short_size
+        self.max_size = max_size
 
     def __call__(self, data):
+        h, w = data['img'].shape[0:2]
         data['trans_info'].append(('resize', data['img'].shape[0:2]))
+        if self.short_size / min(h, w) * max(h, w) > self.max_size:
+            self.short_size = int((self.max_size / max(h, w)) * min(h, w))
+
         data['img'] = functional.resize_short(data['img'], self.short_size)
         for key in data.get('gt_fields', []):
             data[key] = functional.resize_short(data[key], self.short_size,
@@ -550,6 +558,11 @@ class RandomPaddingCrop:
         crop_size (tuple, optional): The target cropping size. Default: (512, 512).
         im_padding_value (float, optional): The padding value of raw image. Default: 127.5.
         label_padding_value (int, optional): The padding value of annotation image. Default: 255.
+        category_max_ratio (float, optional): The maximum ratio that single category could occupy. 
+            Default: 1.0.
+        ignore_index (int, optional): The value that should be ignored in the annotation image. 
+            Default: 255.
+        loop_times (int, optional): The maximum number of attempts to crop an image. Default: 10.
 
     Raises:
         TypeError: When crop_size is neither list nor tuple.
@@ -559,72 +572,95 @@ class RandomPaddingCrop:
     def __init__(self,
                  crop_size=(512, 512),
                  im_padding_value=127.5,
-                 label_padding_value=255):
+                 label_padding_value=255,
+                 category_max_ratio=1.0,
+                 ignore_index=255,
+                 loop_times=10):
         if isinstance(crop_size, list) or isinstance(crop_size, tuple):
             if len(crop_size) != 2:
                 raise ValueError(
-                    'Type of `crop_size` is list or tuple. It should include 2 elements, but it is {}'
+                    'Type of `crop_size` is list or tuple. It should include 2 elements, but it is {}.'
                     .format(crop_size))
         else:
             raise TypeError(
-                "The type of `crop_size` is invalid. It should be list or tuple, but it is {}"
+                "The type of `crop_size` is invalid. It should be list or tuple, but it is {}."
                 .format(type(crop_size)))
-        self.crop_size = crop_size
+        if category_max_ratio <= 0:
+            raise ValueError(
+                "The value of `category_max_ratio` must be greater than 0, but got {}.".
+                format(category_max_ratio))
+        if loop_times <= 0:
+            raise ValueError(
+                "The value of `loop_times` must be greater than 0, but got {}.".
+                format(loop_times))
+
+        self.crop_size = tuple(reversed(crop_size))
         self.im_padding_value = im_padding_value
         self.label_padding_value = label_padding_value
-
-    def __call__(self, data):
-
-        if isinstance(self.crop_size, int):
-            crop_width = self.crop_size
-            crop_height = self.crop_size
-        else:
-            crop_width = self.crop_size[0]
-            crop_height = self.crop_size[1]
-
-        img_height = data['img'].shape[0]
-        img_width = data['img'].shape[1]
-
-        if img_height == crop_height and img_width == crop_width:
-            return data
-        else:
-            pad_height = max(crop_height - img_height, 0)
-            pad_width = max(crop_width - img_width, 0)
-            img_channels = 1 if data['img'].ndim == 2 else data['img'].shape[2]
-            if (pad_height > 0 or pad_width > 0):
-                data['img'] = cv2.copyMakeBorder(
-                    data['img'],
+        self.category_max_ratio = category_max_ratio
+        self.ignore_index = ignore_index
+        self.loop_times = loop_times
+
+    def _get_crop_coordinates(self, origin_size):
+        margin_h = max(origin_size[0] - self.crop_size[0], 0)
+        margin_w = max(origin_size[1] - self.crop_size[1], 0)
+        offset_h = np.random.randint(0, margin_h + 1)
+        offset_w = np.random.randint(0, margin_w + 1)
+        crop_y1, crop_y2 = offset_h, offset_h + self.crop_size[0]
+        crop_x1, crop_x2 = offset_w, offset_w + self.crop_size[1]
+
+        return crop_x1, crop_y1, crop_x2, crop_y2
+
+    def _padding(self, data):
+        img_shape = data['img'].shape[:2]
+        pad_height = max(self.crop_size[0] - img_shape[0], 0)
+        pad_width = max(self.crop_size[1] - img_shape[1], 0)
+        img_channels = 1 if data['img'].ndim == 2 else data['img'].shape[2]
+        if (pad_height > 0 or pad_width > 0):
+            data['img'] = cv2.copyMakeBorder(
+                data['img'],
+                0,
+                pad_height,
+                0,
+                pad_width,
+                cv2.BORDER_CONSTANT,
+                value=(self.im_padding_value, ) * img_channels)
+            for key in data.get('gt_fields', []):
+                data[key] = cv2.copyMakeBorder(
+                    data[key],
                     0,
                     pad_height,
                     0,
                     pad_width,
                     cv2.BORDER_CONSTANT,
-                    value=(self.im_padding_value, ) * img_channels)
-                for key in data.get('gt_fields', []):
-                    data[key] = cv2.copyMakeBorder(
-                        data[key],
-                        0,
-                        pad_height,
-                        0,
-                        pad_width,
-                        cv2.BORDER_CONSTANT,
-                        value=self.label_padding_value)
-                img_height = data['img'].shape[0]
-                img_width = data['img'].shape[1]
-
-            if crop_height > 0 and crop_width > 0:
-                h_off = np.random.randint(img_height - crop_height + 1)
-                w_off = np.random.randint(img_width - crop_width + 1)
-
-                if data['img'].ndim == 2:
-                    data['img'] = data['img'][h_off:(crop_height + h_off),
-                                              w_off:(w_off + crop_width)]
-                else:
-                    data['img'] = data['img'][h_off:(crop_height + h_off),
-                                              w_off:(w_off + crop_width), :]
-                for key in data.get('gt_fields', []):
-                    data[key] = data[key][h_off:(crop_height + h_off), w_off:(
-                        w_off + crop_width)]
+                    value=self.label_padding_value)
+        return data
+
+    def __call__(self, data):
+        img_shape = data['img'].shape[:2]
+        if img_shape[0] == self.crop_size[0] and img_shape[1] == self.crop_size[
+                1]:
+            return data
+
+        data = self._padding(data)
+        img_shape = data['img'].shape[:2]
+        crop_coordinates = self._get_crop_coordinates(img_shape)
+
+        if self.category_max_ratio < 1.0:
+            for _ in range(self.loop_times):
+                seg_temp = functional.crop(data["label"], crop_coordinates)
+                labels, cnt = np.unique(seg_temp, return_counts=True)
+                cnt = cnt[labels != self.ignore_index]
+                if len(cnt) > 1 and np.max(cnt) / np.sum(
+                        cnt) < self.category_max_ratio:
+                    data['img'] = seg_temp
+                    break
+                crop_coordinates = self._get_crop_coordinates(img_shape)
+        else:
+            data['img'] = functional.crop(data['img'], crop_coordinates)
+        for key in data.get("gt_fields", []):
+            data[key] = functional.crop(data[key], crop_coordinates)
+
         return data
 
 
@@ -1126,3 +1162,61 @@ class RandomAffine:
                 borderMode=cv2.BORDER_CONSTANT,
                 borderValue=self.label_padding_value)
         return data
+
+
+@manager.TRANSFORMS.add_component
+class GenerateInstanceTargets:
+    """
+    Generate instance targets from ground-truth labels.
+
+    Args:
+        num_classes (int): The number of classes.
+        ignore_index (int, optional): Specifies a target value that is ignored. Default: 255.
+    """
+
+    def __init__(self, num_classes, ignore_index=255):
+        self.num_classes = num_classes
+        self.ignore_index = ignore_index
+
+    def __call__(self, data):
+        if 'label' in data:
+            sem_seg_gt = data['label']
+            instances = {"image_shape": data['img'].shape[1:]}
+            classes = np.unique(sem_seg_gt)
+            classes = classes[classes != self.ignore_index]
+
+            # To make data compatible with dataloader
+            classes_cpt = np.array([
+                self.ignore_index
+                for _ in range(self.num_classes - len(classes))
+            ])
+            classes_cpt = np.append(classes, classes_cpt)
+            instances["gt_classes"] = np.asarray(classes_cpt).astype('int64')
+
+            masks = []
+            for cid in classes:
+                masks.append(sem_seg_gt == cid)  # [C, H, W] 
+
+            shape = [self.num_classes - len(masks)] + list(data['label'].shape)
+            masks_cpt = np.zeros(shape, dtype='int64')
+
+            if len(masks) == 0:
+                # Some images do not have annotation and will all be ignored
+                instances['gt_masks'] = np.zeros(
+                    (self.num_classes, sem_seg_gt.shape[-2],
+                     sem_seg_gt.shape[-1]),
+                    dtype='int64')
+
+            else:
+                instances['gt_masks'] = np.concatenate(
+                    [
+                        np.stack([
+                            np.ascontiguousarray(x).astype('float32')
+                            for x in masks
+                        ]), masks_cpt
+                    ],
+                    axis=0)
+
+            data['instances'] = instances
+
+        return data

paddlers/models/paddleseg/utils/__init__.py

@@ -19,4 +19,4 @@ from .env import seg_env, get_sys_env
 from .utils import *
 from .timer import TimeAverager, calculate_eta
 from . import visualize
-from .ema import EMA
+from .ema import *

paddlers/models/paddleseg/utils/download.py

@@ -146,21 +146,33 @@ def download_file_and_uncompress(url,
             shutil.rmtree(extraname)
     full_path = os.path.join(extraname,
                              filename) if filename is not None else extraname
+    rank_id_curr_node = int(os.environ.get("PADDLE_RANK_IN_NODE", 0))
     if not os.path.exists(
             full_path):  # If pretrained model exists, skip download process.
-        if not os.path.exists(savename):
-            if not os.path.exists(savepath):
-                _download_file(url, savepath, print_progress)
-
-            if (not tarfile.is_tarfile(savepath)) and (
-                    not zipfile.is_zipfile(savepath)):
-                if not os.path.exists(extraname):
-                    os.makedirs(extraname)
-                shutil.move(savepath, extraname)
-                return extraname
-
-            savename = _uncompress_file(savepath, extrapath, delete_file,
-                                        print_progress)
-            savename = os.path.join(extrapath, savename)
-        shutil.move(savename, extraname)
+        lock_path = extraname + '.download.lock'
+        with open(lock_path, 'w'):  # touch    
+            os.utime(lock_path, None)
+        if rank_id_curr_node == 0:
+            if not os.path.exists(savename):
+                if not os.path.exists(savepath):
+                    _download_file(url, savepath, print_progress)
+
+                if (not tarfile.is_tarfile(savepath)) and (
+                        not zipfile.is_zipfile(savepath)):
+                    if not os.path.exists(extraname):
+                        os.makedirs(extraname)
+                    shutil.move(savepath, extraname)
+
+                else:
+                    savename = _uncompress_file(savepath, extrapath,
+                                                delete_file, print_progress)
+                    savename = os.path.join(extrapath, savename)
+                    shutil.move(savename, extraname)
+
+            os.remove(lock_path)
+
+        else:
+            while os.path.exists(lock_path):
+                time.sleep(0.5)
+
     return extraname

paddlers/models/paddleseg/utils/ema.py

@@ -16,89 +16,33 @@ import numpy as np
 import paddle
 
 
-class EMA(object):
-    """
-    The implementation of Exponential Moving Average for the trainable parameters.
-
-    Args:
-        model (nn.Layer): The model for applying EMA.
-        decay (float, optional): Decay is used to calculate ema_variable by
-            `ema_variable = decay * ema_variable + (1 - decay) * new_variable`.
-            Default: 0.99.
-    
-    Returns:
-        None
-    
-    Examples:
-        .. code-block:: python
-
-            # 1. Define model and dataset
-        
-            # 2. Create EMA
-            ema = EMA(model, decay=0.99)
-
-            # 3. Train stage
-            for data in dataloader():
-                ...
-                optimizer.step()
-                ema.step()
-
-            # 4. Evaluate stage
-            ema.apply()     # Use the EMA data to replace the origin data
-
-            for data in dataloader():
-                ...
-            
-            ema.restore()   # Restore the origin data to the model
-
-    """
-
-    def __init__(self, model, decay=0.99):
-        super().__init__()
-
-        assert isinstance(model, paddle.nn.Layer), \
-            "The model should be the instance of paddle.nn.Layer."
-        assert decay >= 0 and decay <= 1.0, \
-            "The decay = {} should in [0.0, 1.0]".format(decay)
-
-        self._model = model
-        self._decay = decay
-        self._ema_data = {}
-        self._backup_data = {}
-
-        for name, param in self._model.named_parameters():
-            if not param.stop_gradient:
-                self._ema_data[name] = param.numpy()
-
-    def step(self):
-        """
-        Calculate the EMA data for all trainable parameters.
-        """
-        for name, param in self._model.named_parameters():
-            if not param.stop_gradient:
-                assert name in self._ema_data, \
-                    "The param ({}) isn't in the model".format(name)
-                self._ema_data[name] = self._decay * self._ema_data[name] \
-                    + (1.0 - self._decay) * param.numpy()
-
-    def apply(self):
-        """
-        Save the origin data and use the EMA data to replace the origin data.
-        """
-        for name, param in self._model.named_parameters():
-            if not param.stop_gradient:
-                assert name in self._ema_data, \
-                    "The param ({}) isn't in the model".format(name)
-                self._backup_data[name] = param.numpy()
-                param.set_value(self._ema_data[name])
-
-    def restore(self):
-        """
-        Restore the origin data to the model.
-        """
-        for name, param in self._model.named_parameters():
-            if not param.stop_gradient:
-                assert name in self._backup_data, \
-                    "The param ({}) isn't in the model".format(name)
-                param.set_value(self._backup_data[name])
-        self._backup_data = {}
+def judge_params_equal(ema_model, model):
+    for ema_param, param in zip(ema_model.named_parameters(),
+                                model.named_parameters()):
+        if not paddle.equal_all(ema_param[1], param[1]):
+            # print("Difference in", ema_param[0])
+            return False
+    return True
+
+
+def init_ema_params(ema_model, model):
+    state = {}
+    msd = model.state_dict()
+    for k, v in ema_model.state_dict().items():
+        if paddle.is_floating_point(v):
+            v = msd[k].detach()
+        state[k] = v
+    ema_model.set_state_dict(state)
+
+
+def update_ema_model(ema_model, model, step=0, decay=0.999):
+    with paddle.no_grad():
+        state = {}
+        decay = min(1 - 1 / (step + 1), decay)
+        msd = model.state_dict()
+        for k, v in ema_model.state_dict().items():
+            if paddle.is_floating_point(v):
+                v *= decay
+                v += (1.0 - decay) * msd[k].detach()
+            state[k] = v
+        ema_model.set_state_dict(state)

paddlers/models/paddleseg/utils/metrics.py

@@ -50,13 +50,13 @@ def calculate_area(pred, label, num_classes, ignore_index=255):
         pred_i = paddle.logical_and(pred == i, mask)
         label_i = label == i
         intersect_i = paddle.logical_and(pred_i, label_i)
-        pred_area.append(paddle.sum(paddle.cast(pred_i, "int32")))
-        label_area.append(paddle.sum(paddle.cast(label_i, "int32")))
-        intersect_area.append(paddle.sum(paddle.cast(intersect_i, "int32")))
+        pred_area.append(paddle.sum(paddle.cast(pred_i, "int64")))
+        label_area.append(paddle.sum(paddle.cast(label_i, "int64")))
+        intersect_area.append(paddle.sum(paddle.cast(intersect_i, "int64")))
 
-    pred_area = paddle.concat(pred_area)
-    label_area = paddle.concat(label_area)
-    intersect_area = paddle.concat(intersect_area)
+    pred_area = paddle.stack(pred_area)
+    label_area = paddle.stack(label_area)
+    intersect_area = paddle.stack(intersect_area)
 
     return intersect_area, pred_area, label_area
 

paddlers/models/paddleseg/utils/utils.py

@@ -12,20 +12,90 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import contextlib
-import filelock
 import os
+import contextlib
 import tempfile
-import numpy as np
 import random
 from urllib.parse import urlparse, unquote
 
+import yaml
+import numpy as np
 import paddle
+import cv2
 
-from paddlers.models.paddleseg.utils import logger, seg_env
+from paddlers.models.paddleseg.utils import logger, seg_env, get_sys_env
 from paddlers.models.paddleseg.utils.download import download_file_and_uncompress
 
 
+def set_seed(seed=None):
+    if seed is not None:
+        paddle.seed(seed)
+        np.random.seed(seed)
+        random.seed(seed)
+
+
+def show_env_info():
+    env_info = get_sys_env()
+    info = ['{}: {}'.format(k, v) for k, v in env_info.items()]
+    info = '\n'.join(['', format('Environment Information', '-^48s')] + info +
+                     ['-' * 48])
+    logger.info(info)
+
+
+def show_cfg_info(config):
+    msg = '\n---------------Config Information---------------\n'
+    ordered_module = ('batch_size', 'iters', 'train_dataset', 'val_dataset',
+                      'optimizer', 'lr_scheduler', 'loss', 'model')
+    all_module = set(config.dic.keys())
+    for module in ordered_module:
+        if module in config.dic:
+            module_dic = {module: config.dic[module]}
+            msg += str(yaml.dump(module_dic, Dumper=NoAliasDumper))
+            all_module.remove(module)
+    for module in all_module:
+        module_dic = {module: config.dic[module]}
+        msg += str(yaml.dump(module_dic, Dumper=NoAliasDumper))
+    msg += '------------------------------------------------\n'
+    logger.info(msg)
+
+
+def set_device(device):
+    env_info = get_sys_env()
+    if device == 'gpu' and env_info['Paddle compiled with cuda'] \
+        and env_info['GPUs used']:
+        place = 'gpu'
+    elif device == 'xpu' and paddle.is_compiled_with_xpu():
+        place = 'xpu'
+    elif device == 'npu' and paddle.is_compiled_with_custom_device('npu'):
+        place = 'npu'
+    elif device == 'mlu' and paddle.is_compiled_with_custom_device('mlu'):
+        place = 'mlu'
+    else:
+        place = 'cpu'
+    paddle.set_device(place)
+    logger.info("Set device: {}".format(place))
+
+
+def convert_sync_batchnorm(model, device):
+    # Convert bn to sync_bn when use multi GPUs
+    env_info = get_sys_env()
+    if device == 'gpu' and env_info['Paddle compiled with cuda'] \
+        and env_info['GPUs used'] and paddle.distributed.ParallelEnv().nranks > 1:
+        model = paddle.nn.SyncBatchNorm.convert_sync_batchnorm(model)
+        logger.info("Convert bn to sync_bn")
+    return model
+
+
+def set_cv2_num_threads(num_workers):
+    # Limit cv2 threads if too many subprocesses are spawned.
+    # This should reduce resource allocation and thus boost performance.
+    nranks = paddle.distributed.ParallelEnv().nranks
+    if nranks >= 8 and num_workers >= 8:
+        logger.warning("The number of threads used by OpenCV is " \
+            "set to 1 to improve performance.")
+        cv2.setNumThreads(1)
+
+
 @contextlib.contextmanager
 def generate_tempdir(directory: str=None, **kwargs):
     '''Generate a temporary directory'''
@@ -38,8 +108,9 @@ def load_entire_model(model, pretrained):
     if pretrained is not None:
         load_pretrained_model(model, pretrained)
     else:
-        logger.warning('Not all pretrained params of {} are loaded, ' \
-                       'training from scratch or a pretrained backbone.'.format(model.__class__.__name__))
+        logger.warning('Weights are not loaded for {} model since the '
+                       'path of weights is None'.format(
+                           model.__class__.__name__))
 
 
 def download_pretrained_model(pretrained_model):
@@ -62,14 +133,14 @@ def download_pretrained_model(pretrained_model):
         filename = 'model.pdparams'
 
     with generate_tempdir() as _dir:
-        with filelock.FileLock(os.path.join(seg_env.TMP_HOME, savename)):
-            pretrained_model = download_file_and_uncompress(
-                pretrained_model,
-                savepath=_dir,
-                extrapath=seg_env.PRETRAINED_MODEL_HOME,
-                extraname=savename,
-                filename=filename)
-            pretrained_model = os.path.join(pretrained_model, filename)
+        pretrained_model = download_file_and_uncompress(
+            pretrained_model,
+            savepath=_dir,
+            cover=False,
+            extrapath=seg_env.PRETRAINED_MODEL_HOME,
+            extraname=savename,
+            filename=filename)
+        pretrained_model = os.path.join(pretrained_model, filename)
     return pretrained_model
 
 
@@ -177,3 +248,47 @@ def get_image_list(image_path):
             'There are not image file in `--image_path`={}'.format(image_path))
 
     return image_list, image_dir
+
+
+class NoAliasDumper(yaml.SafeDumper):
+    def ignore_aliases(self, data):
+        return True
+
+
+class CachedProperty(object):
+    """
+    A property that is only computed once per instance and then replaces itself with an ordinary attribute.
+
+    The implementation refers to https://github.com/pydanny/cached-property/blob/master/cached_property.py .
+        Note that this implementation does NOT work in multi-thread or coroutine senarios.
+    """
+
+    def __init__(self, func):
+        super().__init__()
+        self.func = func
+        self.__doc__ = getattr(func, '__doc__', '')
+
+    def __get__(self, obj, cls):
+        if obj is None:
+            return self
+        val = self.func(obj)
+        # Hack __dict__ of obj to inject the value
+        # Note that this is only executed once
+        obj.__dict__[self.func.__name__] = val
+        return val
+
+
+def get_in_channels(model_cfg):
+    if 'backbone' in model_cfg:
+        return model_cfg['backbone'].get('in_channels', None)
+    else:
+        return model_cfg.get('in_channels', None)
+
+
+def set_in_channels(model_cfg, in_channels):
+    model_cfg = model_cfg.copy()
+    if 'backbone' in model_cfg:
+        model_cfg['backbone']['in_channels'] = in_channels
+    else:
+        model_cfg['in_channels'] = in_channels
+    return model_cfg

paddlers/models/ppcls/arch/backbone/model_zoo/gvt.py

@@ -324,8 +324,7 @@ class PyramidVisionTransformer(nn.Layer):
             self.pos_drops.append(nn.Dropout(p=drop_rate))
 
         dpr = [
-            x.numpy()[0]
-            for x in paddle.linspace(0, drop_path_rate, sum(depths))
+            float(x) for x in paddle.linspace(0, drop_path_rate, sum(depths))
         ]  # stochastic depth decay rule
 
         cur = 0
@@ -551,8 +550,7 @@ class ALTGVT(PCPVT):
         self.wss = wss
         # transformer encoder
         dpr = [
-            x.numpy()[0]
-            for x in paddle.linspace(0, drop_path_rate, sum(depths))
+            float(x) for x in paddle.linspace(0, drop_path_rate, sum(depths))
         ]  # stochastic depth decay rule
         cur = 0
         self.blocks = nn.LayerList()

paddlers/models/ppcls/engine/evaluation/classification.py

@@ -130,8 +130,7 @@ def classification_eval(engine, epoch_id=0):
             for key in loss_dict:
                 if key not in output_info:
                     output_info[key] = AverageMeter(key, '7.5f')
-                output_info[key].update(loss_dict[key].numpy()[0],
-                                        current_samples)
+                output_info[key].update(float(loss_dict[key]), current_samples)
 
         #  calc metric
         if engine.eval_metric_func is not None:

paddlers/models/ppcls/engine/train/utils.py

@@ -25,8 +25,8 @@ def update_metric(trainer, out, batch, batch_size):
         for key in metric_dict:
             if key not in trainer.output_info:
                 trainer.output_info[key] = AverageMeter(key, '7.5f')
-            trainer.output_info[key].update(metric_dict[key].numpy()[0],
-                                            batch_size)
+            trainer.output_info[key].update(
+                float(metric_dict[key]), batch_size)
 
 
 def update_loss(trainer, loss_dict, batch_size):
@@ -34,7 +34,7 @@ def update_loss(trainer, loss_dict, batch_size):
     for key in loss_dict:
         if key not in trainer.output_info:
             trainer.output_info[key] = AverageMeter(key, '7.5f')
-        trainer.output_info[key].update(loss_dict[key].numpy()[0], batch_size)
+        trainer.output_info[key].update(float(loss_dict[key]), batch_size)
 
 
 def log_info(trainer, batch_size, epoch_id, iter_id):
@@ -53,14 +53,13 @@ def log_info(trainer, batch_size, epoch_id, iter_id):
 
     ips_msg = "ips: {:.5f} samples/s".format(
         batch_size / trainer.time_info["batch_cost"].avg)
-    eta_sec = ((trainer.config["Global"]["epochs"] - epoch_id + 1
-                ) * trainer.max_iter - iter_id
-               ) * trainer.time_info["batch_cost"].avg
+    eta_sec = (
+        (trainer.config["Global"]["epochs"] - epoch_id + 1
+         ) * trainer.max_iter - iter_id) * trainer.time_info["batch_cost"].avg
     eta_msg = "eta: {:s}".format(str(datetime.timedelta(seconds=int(eta_sec))))
     logger.info("[Train][Epoch {}/{}][Iter: {}/{}]{}, {}, {}, {}, {}".format(
         epoch_id, trainer.config["Global"]["epochs"], iter_id,
-        trainer.max_iter, lr_msg, metric_msg, time_msg, ips_msg,
-        eta_msg))
+        trainer.max_iter, lr_msg, metric_msg, time_msg, ips_msg, eta_msg))
 
     for i, lr in enumerate(trainer.lr_sch):
         logger.scaler(

paddlers/models/ppcls/hash.txt

@@ -1 +1 @@
-9a9dac5da035f9b2f54a070cddcda1e32ab72bdd
+9984080a3d7ca52dd9044afaae1df1bed676fd1c

paddlers/models/ppcls/metric/metrics.py

@@ -113,7 +113,7 @@ class mAP(nn.Layer):
         precision_mask = paddle.multiply(equal_flag, precision)
         ap = paddle.sum(precision_mask, axis=1) / paddle.sum(equal_flag,
                                                              axis=1)
-        metric_dict["mAP"] = paddle.mean(ap).numpy()[0]
+        metric_dict["mAP"] = float(paddle.mean(ap))
         return metric_dict
 
 
@@ -157,7 +157,7 @@ class mINP(nn.Layer):
         hard_index = paddle.argmax(auxilary, axis=1).astype("float32")
         all_INP = paddle.divide(paddle.sum(equal_flag, axis=1), hard_index)
         mINP = paddle.mean(all_INP)
-        metric_dict["mINP"] = mINP.numpy()[0]
+        metric_dict["mINP"] = float(mINP)
         return metric_dict
 
 
@@ -207,7 +207,7 @@ class TprAtFpr(nn.Layer):
                 result = "threshold: {}, fpr: 0.0, tpr: {:.5f}".format(
                     threshold, tpr)
                 msg = f"The number of negative samples is 0, please add negative samples."
-                logger.warning(msg)                
+                logger.warning(msg)
             fpr = np.sum(
                 gt_neg_score_list > threshold) / len(gt_neg_score_list)
             if fpr <= self.max_fpr and tpr > max_tpr:
@@ -362,7 +362,7 @@ class HammingDistance(MultiLabelMetric):
         metric_dict["HammingDistance"] = paddle.to_tensor(
             hamming_loss(target, preds))
         self.avg_meters["HammingDistance"].update(
-            metric_dict["HammingDistance"].numpy()[0], output.shape[0])
+            float(metric_dict["HammingDistance"]), output.shape[0])
         return metric_dict
 
 
@@ -402,7 +402,7 @@ class AccuracyScore(MultiLabelMetric):
                 sum(tps) + sum(tns) + sum(fns) + sum(fps))
         metric_dict["AccuracyScore"] = paddle.to_tensor(accuracy)
         self.avg_meters["AccuracyScore"].update(
-            metric_dict["AccuracyScore"].numpy()[0], output.shape[0])
+            float(metric_dict["AccuracyScore"]), output.shape[0])
         return metric_dict
 
 

paddlers/models/ppcls/static/program.py

@@ -153,12 +153,6 @@ def create_strategy(config):
         exec_strategy: exec strategy
     """
     build_strategy = paddle.static.BuildStrategy()
-    exec_strategy = paddle.static.ExecutionStrategy()
-
-    exec_strategy.num_threads = 1
-    exec_strategy.num_iteration_per_drop_scope = (
-        10000
-        if 'AMP' in config and config.AMP.get("level", "O1") == "O2" else 10)
 
     fuse_op = True if 'AMP' in config else False
 
@@ -172,7 +166,7 @@ def create_strategy(config):
     build_strategy.fuse_bn_add_act_ops = fuse_bn_add_act_ops
     build_strategy.enable_addto = enable_addto
 
-    return build_strategy, exec_strategy
+    return build_strategy
 
 
 def dist_optimizer(config, optimizer):
@@ -186,10 +180,9 @@ def dist_optimizer(config, optimizer):
     Returns:
         optimizer: a distributed optimizer
     """
-    build_strategy, exec_strategy = create_strategy(config)
+    build_strategy = create_strategy(config)
 
     dist_strategy = DistributedStrategy()
-    dist_strategy.execution_strategy = exec_strategy
     dist_strategy.build_strategy = build_strategy
 
     dist_strategy.nccl_comm_num = 1
@@ -298,14 +291,10 @@ def compile(config, program, loss_name=None, share_prog=None):
     Returns:
         compiled_program(): a compiled program
     """
-    build_strategy, exec_strategy = create_strategy(config)
+    build_strategy = create_strategy(config)
 
     compiled_program = paddle.static.CompiledProgram(
-        program).with_data_parallel(
-            share_vars_from=share_prog,
-            loss_name=loss_name,
-            build_strategy=build_strategy,
-            exec_strategy=exec_strategy)
+        program, build_strategy=build_strategy)
 
     return compiled_program
 

paddlers/models/ppcls/utils/misc.py

@@ -47,7 +47,7 @@ class AverageMeter(object):
     @property
     def avg_info(self):
         if isinstance(self.avg, paddle.Tensor):
-            self.avg = self.avg.numpy()[0]
+            self.avg = float(self.avg)
         return "{}: {:.5f}".format(self.name, self.avg)
 
     @property

paddlers/models/ppdet/data/crop_utils/__init__.py

@@ -10,4 +10,4 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License. 
+# limitations under the License.

paddlers/models/ppdet/engine/trainer.py

@@ -429,10 +429,8 @@ class Trainer(object):
         model = self.model
         if self.cfg.get('to_static', False):
             model = apply_to_static(self.cfg, model)
-        sync_bn = (
-            getattr(self.cfg, 'norm_type', None) == 'sync_bn' and
-            (self.cfg.use_gpu or self.cfg.use_npu or self.cfg.use_mlu) and
-            self._nranks > 1)
+        sync_bn = (getattr(self.cfg, 'norm_type', None) == 'sync_bn' and
+                   (self.cfg.use_gpu or self.cfg.use_mlu) and self._nranks > 1)
         if sync_bn:
             model = paddle.nn.SyncBatchNorm.convert_sync_batchnorm(model)
 

paddlers/models/ppdet/hash.txt

@@ -1 +1 @@
-00fe2a1c35603b6fb37b73265aecf6282e5e2ad4
+be4ad2f17a898b5f20e35a6de1692e9c5feb3888

paddlers/models/ppdet/metrics/__init__.py

@@ -27,4 +27,4 @@ __all__ = metrics.__all__ + mot_metrics.__all__
 
 from . import mcmot_metrics
 from .mcmot_metrics import *
-__all__ = metrics.__all__ + mcmot_metrics.__all__ 
+__all__ = metrics.__all__ + mcmot_metrics.__all__

paddlers/models/ppdet/modeling/architectures/centertrack.py

@@ -159,4 +159,4 @@ class FasterRCNN(BaseArch):
             this_class = probabilities[labels == i]
             average = np.mean(this_class, axis=0, keepdims=True)
             conditional.append(average)
-        return np.concatenate(conditional) 
+        return np.concatenate(conditional)

paddlers/models/ppdet/modeling/assigners/atss_assigner.py

@@ -222,4 +222,4 @@ class ATSSAssigner(nn.Layer):
                                          paddle.zeros_like(gather_scores))
             assigned_scores *= gather_scores.unsqueeze(-1)
 
-        return assigned_labels, assigned_bboxes, assigned_scores, mask_positive
+        return assigned_labels, assigned_bboxes, assigned_scores

paddlers/models/ppdet/modeling/assigners/fcosr_assigner.py

@@ -224,4 +224,4 @@ class FCOSRAssigner(nn.Layer):
                     (-1, 5))).reshape((b, l, 1))
             assigned_scores = assigned_scores * iou_score
 
-        return assigned_labels, assigned_rboxes, assigned_scores 
+        return assigned_labels, assigned_rboxes, assigned_scores

paddlers/models/ppdet/modeling/assigners/task_aligned_assigner.py

@@ -191,4 +191,4 @@ class TaskAlignedAssigner(nn.Layer):
         alignment_metrics = alignment_metrics.max(-2).unsqueeze(-1)
         assigned_scores = assigned_scores * alignment_metrics
 
-        return assigned_labels, assigned_bboxes, assigned_scores, mask_positive
+        return assigned_labels, assigned_bboxes, assigned_scores

paddlers/models/ppdet/modeling/assigners/task_aligned_assigner_cr.py

@@ -179,4 +179,4 @@ class TaskAlignedAssigner_CR(nn.Layer):
         alignment_metrics = alignment_metrics.max(-2).unsqueeze(-1)
         assigned_scores = assigned_scores * alignment_metrics
 
-        return assigned_labels, assigned_bboxes, assigned_scores, mask_positive
+        return assigned_labels, assigned_bboxes, assigned_scores

paddlers/models/ppdet/modeling/heads/pico_head.py

@@ -651,7 +651,7 @@ class PicoHeadV2(GFLHead):
 
         # label assignment
         if gt_meta['epoch_id'] < self.static_assigner_epoch:
-            assigned_labels, assigned_bboxes, assigned_scores, _ = self.static_assigner(
+            assigned_labels, assigned_bboxes, assigned_scores = self.static_assigner(
                 anchors,
                 num_anchors_list,
                 gt_labels,
@@ -662,7 +662,7 @@ class PicoHeadV2(GFLHead):
                 pred_bboxes=pred_bboxes.detach() * stride_tensor_list)
 
         else:
-            assigned_labels, assigned_bboxes, assigned_scores, _ = self.assigner(
+            assigned_labels, assigned_bboxes, assigned_scores = self.assigner(
                 pred_scores.detach(),
                 pred_bboxes.detach() * stride_tensor_list,
                 centers,

paddlers/models/ppdet/modeling/heads/ppyoloe_contrast_head.py

@@ -121,7 +121,7 @@ class PPYOLOEContrastHead(PPYOLOEHead):
         pad_gt_mask = gt_meta['pad_gt_mask']
         # label assignment
         if gt_meta['epoch_id'] < self.static_assigner_epoch:
-            assigned_labels, assigned_bboxes, assigned_scores, _ = \
+            assigned_labels, assigned_bboxes, assigned_scores = \
                 self.static_assigner(
                     anchors,
                     num_anchors_list,
@@ -133,7 +133,7 @@ class PPYOLOEContrastHead(PPYOLOEHead):
             alpha_l = 0.25
         else:
             if self.sm_use:
-                assigned_labels, assigned_bboxes, assigned_scores, _ = \
+                assigned_labels, assigned_bboxes, assigned_scores = \
                     self.assigner(
                     pred_scores.detach(),
                     pred_bboxes.detach() * stride_tensor,
@@ -144,7 +144,7 @@ class PPYOLOEContrastHead(PPYOLOEHead):
                     pad_gt_mask,
                     bg_index=self.num_classes)
             else:
-                assigned_labels, assigned_bboxes, assigned_scores, _ = \
+                assigned_labels, assigned_bboxes, assigned_scores = \
                     self.assigner(
                     pred_scores.detach(),
                     pred_bboxes.detach() * stride_tensor,

paddlers/models/ppdet/modeling/heads/ppyoloe_head.py

@@ -387,7 +387,7 @@ class PPYOLOEHead(nn.Layer):
         pad_gt_mask = gt_meta['pad_gt_mask']
         # label assignment
         if gt_meta['epoch_id'] < self.static_assigner_epoch:
-            assigned_labels, assigned_bboxes, assigned_scores, mask_positive = \
+            assigned_labels, assigned_bboxes, assigned_scores = \
                 self.static_assigner(
                     anchors,
                     num_anchors_list,
@@ -400,7 +400,7 @@ class PPYOLOEHead(nn.Layer):
         else:
             if self.sm_use:
                 # only used in smalldet of PPYOLOE-SOD model
-                assigned_labels, assigned_bboxes, assigned_scores, mask_positive = \
+                assigned_labels, assigned_bboxes, assigned_scores = \
                     self.assigner(
                     pred_scores.detach(),
                     pred_bboxes.detach() * stride_tensor,
@@ -413,7 +413,7 @@ class PPYOLOEHead(nn.Layer):
             else:
                 if aux_pred is None:
                     if not hasattr(self, "assigned_labels"):
-                        assigned_labels, assigned_bboxes, assigned_scores, mask_positive = \
+                        assigned_labels, assigned_bboxes, assigned_scores = \
                             self.assigner(
                             pred_scores.detach(),
                             pred_bboxes.detach() * stride_tensor,
@@ -427,15 +427,15 @@ class PPYOLOEHead(nn.Layer):
                             self.assigned_labels = assigned_labels
                             self.assigned_bboxes = assigned_bboxes
                             self.assigned_scores = assigned_scores
-                            self.mask_positive = mask_positive
+
                     else:
                         # only used in distill
                         assigned_labels = self.assigned_labels
                         assigned_bboxes = self.assigned_bboxes
                         assigned_scores = self.assigned_scores
-                        mask_positive = self.mask_positive
+
                 else:
-                    assigned_labels, assigned_bboxes, assigned_scores, mask_positive = \
+                    assigned_labels, assigned_bboxes, assigned_scores = \
                             self.assigner(
                             pred_scores_aux.detach(),
                             pred_bboxes_aux.detach() * stride_tensor,
@@ -451,14 +451,12 @@ class PPYOLOEHead(nn.Layer):
 
         assign_out_dict = self.get_loss_from_assign(
             pred_scores, pred_distri, pred_bboxes, anchor_points_s,
-            assigned_labels, assigned_bboxes, assigned_scores, mask_positive,
-            alpha_l)
+            assigned_labels, assigned_bboxes, assigned_scores, alpha_l)
 
         if aux_pred is not None:
             assign_out_dict_aux = self.get_loss_from_assign(
                 aux_pred[0], aux_pred[1], pred_bboxes_aux, anchor_points_s,
-                assigned_labels, assigned_bboxes, assigned_scores,
-                mask_positive, alpha_l)
+                assigned_labels, assigned_bboxes, assigned_scores, alpha_l)
             loss = {}
             for key in assign_out_dict.keys():
                 loss[key] = assign_out_dict[key] + assign_out_dict_aux[key]
@@ -469,7 +467,7 @@ class PPYOLOEHead(nn.Layer):
 
     def get_loss_from_assign(self, pred_scores, pred_distri, pred_bboxes,
                              anchor_points_s, assigned_labels, assigned_bboxes,
-                             assigned_scores, mask_positive, alpha_l):
+                             assigned_scores, alpha_l):
         # cls loss
         if self.use_varifocal_loss:
             one_hot_label = F.one_hot(assigned_labels,
@@ -490,7 +488,6 @@ class PPYOLOEHead(nn.Layer):
             self.distill_pairs['pred_cls_scores'] = pred_scores
             self.distill_pairs['pos_num'] = assigned_scores_sum
             self.distill_pairs['assigned_scores'] = assigned_scores
-            self.distill_pairs['mask_positive'] = mask_positive
             one_hot_label = F.one_hot(assigned_labels,
                                       self.num_classes + 1)[..., :-1]
             self.distill_pairs['target_labels'] = one_hot_label

paddlers/models/ppdet/modeling/heads/sparsercnn_head.py

@@ -317,12 +317,15 @@ class SparseRCNNHead(nn.Layer):
             pooler_scales = [1.0 / 4.0, 1.0 / 8.0, 1.0 / 16.0, 1.0 / 32.0]
             end_level = 3
 
+        aligned = True
+        if paddle.device.is_compiled_with_custom_device('npu'):
+            aligned = False
         box_pooler = RoIAlign(
             resolution=pooler_resolution,
             spatial_scale=pooler_scales,
             sampling_ratio=sampling_ratio,
             end_level=end_level,
-            aligned=True)
+            aligned=aligned)
         return box_pooler
 
     def forward(self, features, input_whwh):