minimal testing for A-SOC

Author: thib-s

orthogonium/layers/conv/adaptiveSOC/fast_skew_ortho_conv.py

@@ -138,14 +138,17 @@ def attach_soc_weight(
         weight_name (str): name of the weight
         kernel_shape (tuple): shape of the kernel (out_channels, in_channels/groups, kernel_size, kernel_size)
         groups (int): number of groups
-        bjorck_params (BjorckParams, optional): parameters of the Bjorck orthogonalization. Defaults to BjorckParams().
+        exp_params (ExpParams): parameters for the exponential algorithm.
 
     Returns:
         torch.Tensor: a handle to the attached weight
     """
     out_channels, in_channels, kernel_size, k2 = kernel_shape
     in_channels *= groups  # compute the real number of input channels
-    assert kernel_size == k2, "only square kernels are supported for the moment"
+    assert (
+        kernel_size == k2
+    ), "only square kernels are supported (to compute skew symmetric kernels)"
+    assert kernel_size % 2 == 1, "kernel size must be odd"
     max_channels = max(in_channels, out_channels)
     layer.register_parameter(
         weight_name,
@@ -250,11 +253,6 @@ def __init__(
             groups,
             exp_params=exp_params,
         )
-        if bias:
-            self.bias = nn.Parameter(torch.Tensor(out_channels))
-            nn.init.zeros_(self.bias)
-        else:
-            self.register_parameter("bias", None)
 
     def singular_values(self):
         """Compute the singular values of the convolutional layer using the FFT+SVD method.
@@ -363,12 +361,6 @@ def __init__(
             exp_params=exp_params,
         )
 
-        if bias:
-            self.bias = nn.Parameter(torch.Tensor(out_channels))
-            nn.init.zeros_(self.bias)
-        else:
-            self.register_parameter("bias", None)
-
     def singular_values(self):
         if self.padding_mode != "circular":
             print(
@@ -383,8 +375,8 @@ def singular_values(self):
                 self.groups,
                 self.in_channels // self.groups,
                 self.out_channels // self.groups,
-                self.kernel_size,
-                self.kernel_size,
+                self.weight.shape[-2],
+                self.weight.shape[-1],
             )
             .numpy(),
             self._input_shape,

orthogonium/layers/conv/adaptiveSOC/ortho_conv.py

@@ -40,7 +40,7 @@ def AdaptiveSOCConv2d(
         )
     if kernel_size == stride:
         convclass = RKOConv2d
-    elif (stride == 1) or (in_channels >= out_channels):
+    elif stride == 1:
         convclass = FastSOC
     else:
         convclass = SOCRkoConv2d

orthogonium/layers/conv/adaptiveSOC/soc_x_rko_conv.py

@@ -75,14 +75,26 @@ def __init__(
         self.groups = groups
         self.intermediate_channels = max(in_channels, out_channels // stride**2)
         del self.weight
+        int_kernel_size = kernel_size - (stride - 1)
+        if int_kernel_size % 2 == 0:
+            if int_kernel_size <= 2:
+                int_kernel_size += 1
+            else:
+                int_kernel_size -= 1
+            # warn user that kernel size changed
+            warnings.warn(
+                f"kernel size changed from {kernel_size} to {int_kernel_size} "
+                f"as even kernel size is not supported for SOC.",
+                RuntimeWarning,
+            )
         attach_soc_weight(
             self,
             "weight_1",
             (
                 self.intermediate_channels,
                 in_channels // groups,
-                kernel_size - (stride - 1),
-                kernel_size - (stride - 1),
+                int_kernel_size,
+                int_kernel_size,
             ),
             groups,
             exp_params=exp_params,
@@ -96,12 +108,6 @@ def __init__(
             ortho_params=ortho_params,
         )
 
-        if bias:
-            self.bias = nn.Parameter(torch.Tensor(out_channels))
-            nn.init.zeros_(self.bias)
-        else:
-            self.register_parameter("bias", None)
-
     @property
     def weight(self):
         if self.training:
@@ -237,14 +243,26 @@ def __init__(
             #     RuntimeWarning,
             # )
         del self.weight
+        int_kernel_size = kernel_size - (stride - 1)
+        if int_kernel_size % 2 == 0:
+            if int_kernel_size <= 2:
+                int_kernel_size += 1
+            else:
+                int_kernel_size -= 1
+            # warn user that kernel size changed
+            warnings.warn(
+                f"kernel size changed from {kernel_size} to {int_kernel_size} "
+                f"as even kernel size is not supported for SOC.",
+                RuntimeWarning,
+            )
         attach_soc_weight(
             self,
             "weight_1",
             (
                 self.intermediate_channels,
                 out_channels // groups,
-                kernel_size - (stride - 1),
-                kernel_size - (stride - 1),
+                int_kernel_size,
+                int_kernel_size,
             ),
             groups,
             exp_params=exp_params,
@@ -258,12 +276,6 @@ def __init__(
             ortho_params=ortho_params,
         )
 
-        if bias:
-            self.bias = nn.Parameter(torch.Tensor(out_channels))
-            nn.init.zeros_(self.bias)
-        else:
-            self.register_parameter("bias", None)
-
     def singular_values(self):
         if self.padding_mode != "circular":
             print(
@@ -278,8 +290,8 @@ def singular_values(self):
                 self.groups,
                 self.intermediate_channels // self.groups,
                 self.out_channels // self.groups,
-                self.kernel_size,
-                self.kernel_size,
+                self.weight_1.shape[-2],
+                self.weight_1.shape[-1],
             )
             .numpy(),
             self._input_shape,