Fixes

.github/workflows/lightning.yml

@@ -0,0 +1,38 @@
+# This workflow will install Python dependencies, run tests with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: lightning
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        lightning: ["2.1.4", "2.2.5", "2.3.3", "2.4.0", "2.5.1", "2.6.5"]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Replace lightning
+        uses: jacobtomlinson/gha-find-replace@v3
+        with:
+          find: "lightning[pytorch-extra]>=2,<2.7"
+          replace: "lightning[pytorch-extra]==${{ matrix.lightning }}"
+          regex: false
+          include: "requirements.txt"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install torch==2.10.0 torchvision==0.25.0 --index-url https://download.pytorch.org/whl/cpu
+          pip install -r requirements.txt
+      - name: Test with pytest
+        run: |
+          pip install pytest==9.0.2
+          python -m pytest tests/

.github/workflows/python.yml

@@ -0,0 +1,35 @@
+# This workflow will install Python dependencies, run tests with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: python
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.10', '3.11', '3.12', '3.13']
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install torch==2.10.0 torchvision==0.25.0 --index-url https://download.pytorch.org/whl/cpu
+          pip install -r requirements.txt
+      - name: Test with pytest
+        run: |
+          pip install pytest==9.0.2
+          python -m pytest tests/

.github/workflows/pytorch.yml

@@ -0,0 +1,38 @@
+# This workflow will install Python dependencies, run tests with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: pytorch
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        pytorch: [
+          'torch==2.13.0 torchvision==0.28.0 --index-url https://download.pytorch.org/whl/cpu',
+          'torch==2.12.1 torchvision==0.27.1 --index-url https://download.pytorch.org/whl/cpu',
+          'torch==2.11.0 torchvision==0.26.0 --index-url https://download.pytorch.org/whl/cpu',
+          'torch==2.10.0 torchvision==0.25.0 --index-url https://download.pytorch.org/whl/cpu',
+          'torch==2.9.1 torchvision==0.24.1 --index-url https://download.pytorch.org/whl/cpu',
+          'torch==2.8.0 torchvision==0.23.0 --index-url https://download.pytorch.org/whl/cpu',
+        ]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install ${{ matrix.pytorch }}
+          pip install -r requirements.txt
+      - name: Test with pytest
+        run: |
+          pip install pytest==9.0.2
+          python -m pytest tests/

README.md

@@ -1,5 +1,9 @@
 # RoCo Spring DevKit
 
+![GitHub CI python status](https://github.com/hmorimitsu/roco-spring-devkit/actions/workflows/python.yml/badge.svg)
+![GitHub CI pytorch status](https://github.com/hmorimitsu/roco-spring-devkit/actions/workflows/pytorch.yml/badge.svg)
+![GitHub CI lightning status](https://github.com/hmorimitsu/roco-spring-devkit/actions/workflows/lightning.yml/badge.svg)
+
 Developer kit for the Robust Correspondence Challenge. This devkit is derived from [PTLFlow](http://github.com/hmorimitsu/ptlflow), so checking [PTLFlow's documentation](https://ptlflow.readthedocs.io/en/latest/) may also help you to customize the code in this devkit.
 
 ## Prerequisites & Requirements

requirements.txt

@@ -0,0 +1,14 @@
+einops<0.9
+h5py<3.17
+kaleido<1.3
+lightning[pytorch-extra]>=2.1,<2.7
+loguru<0.8
+opencv-python<4.14
+pandas<3.1
+plotly<6.7
+pypng==0.20220715.0
+requests<2.34
+scipy<1.18
+tabulate<0.11
+tensorboard<2.21
+timm<1.1

roco_spring_devkit/common/data/optical_flow_datasets.py

@@ -365,7 +365,7 @@ def __init__(
         self.metadata = [
             {
                 "image_paths": [str(p) for p in paths],
-                "is_val": paths[0].stem in val_names,
+                "is_val": paths[0].parent.stem in val_names,
                 "misc": "",
                 "is_seq_start": True,
                 "is_seq_end": True,
@@ -465,7 +465,7 @@ def __init__(
         self.metadata = [
             {
                 "image_paths": [str(p) for p in paths],
-                "is_val": paths[0].stem in val_names,
+                "is_val": paths[0].stem.split("_")[0] in val_names,
                 "misc": "",
                 "is_seq_start": True,
                 "is_seq_end": True,
@@ -1085,12 +1085,15 @@ def __init__(  # noqa: C901
                         )
                         flow_paths = sorted(flow_dir.glob("*.flo"), reverse=rev)
 
-                        # Create groups to separate different sequences
+                        # Create groups to separate different sequences. The
+                        # consecutive-index check must use abs() so that
+                        # descending-sorted (rev=True) flow lists are grouped
+                        # correctly too.
                         flow_groups_paths = [[flow_paths[0]]]
                         prev_idx = int(flow_paths[0].stem)
                         for path in flow_paths[1:]:
                             idx = int(path.stem)
-                            if (idx - 1) == prev_idx:
+                            if abs(idx - prev_idx) == 1:
                                 flow_groups_paths[-1].append(path)
                             else:
                                 flow_groups_paths.append([path])
@@ -1123,7 +1126,7 @@ def __init__(  # noqa: C901
                                     img_dir / (fp.stem + ".png") for fp in flow_paths
                                 ]
                                 if rev:
-                                    idx = int(img_paths[0].stem) - 1
+                                    idx = int(img_paths[-1].stem) - 1
                                 else:
                                     idx = int(img_paths[-1].stem) + 1
                                 img_paths.append(img_dir / f"{idx:07d}.png")
@@ -1171,12 +1174,14 @@ def __init__(  # noqa: C901
                             )
                             flow_paths = sorted(flow_dir.glob("*.flo"), reverse=rev)
 
-                            # Create groups to separate different sequences
+                            # Create groups to separate different sequences. Same abs()
+                            # reasoning as the forward-direction grouping
+                            # above - descending lists need it too.
                             flow_groups_paths = [[flow_paths[0]]]
                             prev_idx = int(flow_paths[0].stem)
                             for path in flow_paths[1:]:
                                 idx = int(path.stem)
-                                if (idx - 1) == prev_idx:
+                                if abs(idx - prev_idx) == 1:
                                     flow_groups_paths[-1].append(path)
                                 else:
                                     flow_groups_paths.append([path])
@@ -2343,7 +2348,6 @@ def __init__(  # noqa: C901
             get_occlusion_mask=False,
             get_motion_boundary_mask=False,
             get_backward=get_backward,
-            get_semantic_segmentation_labels=False,
             get_meta=get_meta,
         )
         self.root_dir = root_dir

roco_spring_devkit/common/data/scene_flow_datasets.py

@@ -665,7 +665,7 @@ def __init__(
                                         "is_seq_start": i == 0,
                                         "is_seq_end": i + step_size
                                         >= len(image_paths) - 1,
-                                        "is_time_reverse": False,
+                                        "is_time_reverse": reverse,
                                         "is_camera_reverse": False,
                                     }
                                 )
@@ -1415,6 +1415,7 @@ def __init__(  # noqa: C901
             [p.stem for p in (Path(root_dir) / split_dir).glob("*")]
         )
 
+        val_seqs = []
         if split == "train" or split == "val":
             split_file = THIS_DIR / "Spring_val.txt"
             with open(split_file, "r") as f:
@@ -1726,7 +1727,7 @@ def __init__(  # noqa: C901
                                             i : i + self.sequence_length
                                         ]
                                     ],
-                                    "is_val": False,
+                                    "is_val": seq_name in val_seqs,
                                     "is_time_reverse": time_rev,
                                     "is_camera_reverse": cam_rev,
                                     "misc": seq_name,

roco_spring_devkit/common/utils/external/selflow.py

@@ -51,5 +51,5 @@ def write_pfm(output_path, flow, scale=1):
         if len(flow.shape) == 3:
             invalid = np.isnan(flow[..., 0]) | np.isnan(flow[..., 1])
             flow = np.dstack([flow, invalid.astype(np.float32)])
-            flow = np.flipud(flow)
+        flow = np.flipud(flow)
         flow.tofile(file)

roco_spring_devkit/common/utils/flow_utils.py

@@ -266,7 +266,6 @@ def spring_epe_to_rgb(
     epe_rgb = cv.applyColorMap(epe, plt_lut)
 
     invalid_mask = ~valid_mask
-    invalid_mask = np.concatenate([invalid_mask, invalid_mask, invalid_mask], -1)
     epe_rgb[invalid_mask] = 0
 
     return epe_rgb

roco_spring_devkit/common/utils/io_adapter.py

@@ -126,11 +126,22 @@ def prepare_inputs(
             inputs.update(kwargs)
             keys_to_remove = []
             for k, v in inputs.items():
-                if v is None or len(v) == 0:
+                if v is None:
                     keys_to_remove.append(k)
+                elif hasattr(v, "__len__") and len(v) == 0:
+                    keys_to_remove.append(k)
+                else:
+                    if not isinstance(v, (np.ndarray, list, tuple)):
+                        inputs[k] = v
             for k in keys_to_remove:
                 del inputs[k]
-            inputs = self.transform(inputs)
+            to_transform = {
+                k: v
+                for k, v in inputs.items()
+                if isinstance(v, (np.ndarray, list, tuple))
+            }
+            transformed = self.transform(to_transform)
+            inputs.update(transformed)
 
         inputs = self._to_cuda(inputs)
 

roco_spring_devkit/common/utils/scene_flow_metrics.py

@@ -171,6 +171,8 @@ def update(
         else:
             valid_flows_target = torch.ones_like(flow_target[:, :1])
         valid_flows_target = valid_flows_target[:, 0]
+        if valid_flows_target.dim() > 3:
+            valid_flows_target = valid_flows_target[:, 0]
 
         valid_disparities_target = targets.get("valid_disparities")
         if valid_disparities_target is not None:
@@ -181,10 +183,10 @@ def update(
             valid_disparities_target = torch.ones_like(disp_target[:, :1])
         valid_disparities_target = valid_disparities_target[:, 0]
 
-        disp1_target = disp_target[:1]
-        valid_disp1_target = valid_disparities_target[:1]
-        disp2_target = disp_target[1:2]
-        valid_disp2_target = valid_disparities_target[1:2]
+        disp1_target = disp_target[0::2]
+        valid_disp1_target = valid_disparities_target[0::2]
+        disp2_target = disp_target[1::2]
+        valid_disp2_target = valid_disparities_target[1::2]
 
         valid_all_target = valid_flows_target * valid_disp1_target * valid_disp2_target
 
@@ -194,33 +196,31 @@ def update(
             target_norm = torch.norm(flow_target, p=2, dim=2)
             target_norm = target_norm.gather(1, min_idx[:, None])[:, 0]
 
-            abs1 = torch.abs(disp1_pred[:, None] - disp1_target)
-            abs1, min_idx = abs1.min(dim=1)
-            abs1 = abs1.gather(1, min_idx[:, None])
-
-            abs2 = torch.abs(disp2_pred[:, None] - disp2_target)
-            abs2, min_idx = abs2.min(dim=1)
-            abs2 = abs2.gather(1, min_idx[:, None])
+            abs1 = torch.abs(disp1_pred - disp1_target)[:, 0]
+            abs2 = torch.abs(disp2_pred - disp2_target)[:, 0]
         else:
             epe_flow = torch.norm(flow_pred - flow_target, p=2, dim=1)
             target_norm = torch.norm(flow_target, p=2, dim=1)
 
             abs1 = torch.abs(disp1_pred - disp1_target)[:, 0]
             abs2 = torch.abs(disp2_pred - disp2_target)[:, 0]
 
+        disp1_target_mag = torch.abs(disp1_target)[:, 0]
+        disp2_target_mag = torch.abs(disp2_target)[:, 0]
+
         px1_flow_mask = (epe_flow > 1).float() * 100
         flall_mask = ((epe_flow > 3) & (epe_flow > (0.05 * target_norm))).float() * 100
 
         px11_mask = (abs1 > 1).float() * 100
-        d1_mask = ((abs1 > 3) & (abs1 > (0.05 * target_norm))).float() * 100
+        d1_mask = ((abs1 > 3) & (abs1 > (0.05 * disp1_target_mag))).float() * 100
 
         px12_mask = (abs2 > 1).float() * 100
-        d2_mask = ((abs2 > 3) & (abs2 > (0.05 * target_norm))).float() * 100
+        d2_mask = ((abs2 > 3) & (abs2 > (0.05 * disp2_target_mag))).float() * 100
 
         px1_all_mask = (
-            (px1_flow_mask / 100) * (px11_mask / 100) * (px12_mask / 100) * 100
-        )
-        sfall_mask = (flall_mask / 100) * (d1_mask / 100) * (d2_mask / 100) * 100
+            (px1_flow_mask > 0) | (px11_mask > 0) | (px12_mask > 0)
+        ).float() * 100
+        sfall_mask = ((flall_mask > 0) | (d1_mask > 0) | (d2_mask > 0)).float() * 100
 
         self.used_keys = [
             ("epe", "epe_flow", "valid_flows_target"),

roco_spring_devkit/common/utils/stereo_metrics.py

@@ -174,13 +174,15 @@ def update(
 
         if len(disp_target.shape) == 5:
             abs = torch.abs(disp_pred[:, None] - disp_target)
-            abs, min_idx = abs.min(dim=1)
-            abs = abs.gather(1, min_idx[:, None])[:, 0]
+            abs, _ = abs.min(dim=1)
+            abs = abs[:, 0]
+            disp_target_mag = torch.abs(disp_target).max(dim=1)[0][:, 0]
         else:
-            abs = torch.abs(disp_pred - disp_target)
+            abs = torch.abs(disp_pred - disp_target)[:, 0]
+            disp_target_mag = torch.abs(disp_target)[:, 0]
 
         px1_mask = (abs > 1).float() * 100
-        d1_mask = ((abs > 3) & (abs > (0.05 * abs))).float() * 100
+        d1_mask = ((abs > 3) & (abs > (0.05 * disp_target_mag))).float() * 100
         self.used_keys = [
             ("abs", "abs", "valid_target"),
             ("1px", "px1_mask", "valid_target"),

roco_spring_devkit/common/utils/stereo_utils.py

@@ -126,7 +126,9 @@ def disparity_to_rgb(
         elif len(input_shape) == 3:
             disparity_rgb = disparity_rgb[0]
         else:
-            disparity_rgb = disparity_rgb.reshape(*input_shape)
+            output_shape = list(input_shape)
+            output_shape[-3] = 3
+            disparity_rgb = disparity_rgb.reshape(output_shape)
     elif len(disparity) == 1:
         disparity_rgb = disparity_rgb[0]
 
@@ -446,7 +448,6 @@ def spring_abs_to_rgb(
     epe_rgb = cv.applyColorMap(epe, plt_lut)
 
     invalid_mask = ~valid_mask
-    invalid_mask = np.concatenate([invalid_mask, invalid_mask, invalid_mask], -1)
     epe_rgb[invalid_mask] = 0
 
     return epe_rgb