[OPIK-906] Updates to support clickhouse replications (#1400)

* add migration change_tables_to_replicated

* update docker-compose

* use better Test reports for backend tests

* add zookeeper to helm chart default

* Update Helm documentation

* update clickhouse config for docker-compose

* Fix tests

* OPIK-906: Fix tests

* fix

* add detach partition from the old table before dropping it

* Add attachments

* update migration script

* add Troubleshooting to docs

* adjust migrations

* fix migartion

---------

Co-authored-by: CometActions <[email protected]>
Co-authored-by: Thiago Hora <[email protected]>

Author: 3 people

.github/workflows/backend_tests.yml

@@ -36,10 +36,10 @@ jobs:
               env:
                 TESTCONTAINERS_REUSE_ENABLE: true
               run: mvn clean test -Dmaven.test.failure.ignore=true
-    
+
             - name: Publish Test Report
               uses: EnricoMi/publish-unit-test-result-action/linux@v2
               if: always()
               with:
                 action_fail: true
-                files: '**/target/surefire-reports/TEST-*.xml'
+                files: '**/target/surefire-reports/TEST-*.xml'

apps/opik-backend/src/main/java/com/comet/opik/domain/FeedbackScoreDAO.java

@@ -138,6 +138,7 @@ AND entity_id IN (
                 WHERE trace_id IN :trace_ids
             )
             AND workspace_id = :workspace_id
+            SETTINGS allow_nondeterministic_mutations = 1
             ;
             """;
 

apps/opik-backend/src/main/resources/liquibase/db-app-analytics/migrations/000017_change_tables_to_replicated.sql

@@ -2,14 +2,18 @@
 
 import com.comet.opik.infrastructure.DatabaseAnalyticsFactory;
 import org.testcontainers.clickhouse.ClickHouseContainer;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.Network;
 import org.testcontainers.utility.DockerImageName;
+import org.testcontainers.utility.MountableFile;
 
 import java.util.Map;
 
 public class ClickHouseContainerUtils {
 
     public static final String DATABASE_NAME = "opik";
     public static final String DATABASE_NAME_VARIABLE = "ANALYTICS_DB_DATABASE_NAME";
+    private static final Network NETWORK = Network.newNetwork();
 
     public static ClickHouseContainer newClickHouseContainer() {
         return newClickHouseContainer(true);
@@ -20,6 +24,49 @@ public static ClickHouseContainer newClickHouseContainer(boolean reusable) {
                 .withReuse(reusable);
     }
 
+    public static GenericContainer<?> newZookeeperContainer() {
+        return newZookeeperContainer(true, NETWORK);
+    }
+
+    public static GenericContainer<?> newZookeeperContainer(boolean reusable, Network network) {
+        return new GenericContainer<>("zookeeper:3.9.3")
+                .withExposedPorts(2181)
+                .withNetworkAliases("zookeeper")
+                .withNetwork(network)
+                .withEnv("ALLOW_ANONYMOUS_LOGIN", "yes")
+                .withEnv("ZOO_MY_ID", "1")
+                .withReuse(reusable);
+    }
+
+    public static ClickHouseContainer newClickHouseContainer(GenericContainer<?> zooKeeperContainer) {
+        return newClickHouseContainer(true, NETWORK, zooKeeperContainer);
+    }
+
+    public static ClickHouseContainer newClickHouseContainer(boolean reusable, Network network,
+            GenericContainer<?> zooKeeperContainer) {
+
+        try {
+
+            ClickHouseContainer container = newClickHouseContainer(reusable);
+
+            if (zooKeeperContainer != null) {
+                container.dependsOn(zooKeeperContainer);
+            }
+
+            return container
+                    .withReuse(reusable)
+                    .withEnv("CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT", "1")
+                    .withNetwork(network)
+                    .withCopyFileToContainer(MountableFile.forClasspathResource("macros.xml"),
+                            "/etc/clickhouse-server/config.d/macros.xml")
+                    .withCopyFileToContainer(MountableFile.forClasspathResource("zookeeper.xml"),
+                            "/etc/clickhouse-server/config.d/zookeeper.xml");
+
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     public static DatabaseAnalyticsFactory newDatabaseAnalyticsFactory(ClickHouseContainer clickHouseContainer,
             String databaseName) {
         var databaseAnalyticsFactory = new DatabaseAnalyticsFactory();

apps/opik-backend/src/test/java/com/comet/opik/api/resources/utils/ClickHouseContainerUtils.java

@@ -13,6 +13,8 @@
 import com.comet.opik.api.resources.utils.TestUtils;
 import com.comet.opik.api.resources.utils.WireMockUtils;
 import com.comet.opik.api.resources.utils.resources.ExperimentResourceClient;
+import com.comet.opik.extensions.DropwizardAppExtensionProvider;
+import com.comet.opik.extensions.RegisterApp;
 import com.comet.opik.infrastructure.DatabaseAnalyticsFactory;
 import com.comet.opik.podam.PodamFactoryUtils;
 import com.redis.testcontainers.RedisContainer;
@@ -26,8 +28,9 @@
 import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInstance;
-import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.testcontainers.clickhouse.ClickHouseContainer;
+import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.containers.MySQLContainer;
 import org.testcontainers.lifecycle.Startables;
 import org.testcontainers.shaded.org.awaitility.Awaitility;
@@ -47,6 +50,7 @@
 
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
 @DisplayName("Dataset Event Listener")
+@ExtendWith(DropwizardAppExtensionProvider.class)
 class DatasetEventListenerTest {
 
     private static final String BASE_RESOURCE_URI = "%s/v1/private/datasets";
@@ -57,19 +61,18 @@ class DatasetEventListenerTest {
     private static final String WORKSPACE_ID = UUID.randomUUID().toString();
     private static final String TEST_WORKSPACE = UUID.randomUUID().toString();
 
-    private static final RedisContainer REDIS = RedisContainerUtils.newRedisContainer();
+    private final RedisContainer REDIS = RedisContainerUtils.newRedisContainer();
+    private final MySQLContainer<?> MYSQL = MySQLContainerUtils.newMySQLContainer();
+    private final GenericContainer<?> ZOOKEEPER_CONTAINER = ClickHouseContainerUtils.newZookeeperContainer();
+    private final ClickHouseContainer CLICKHOUSE = ClickHouseContainerUtils.newClickHouseContainer(ZOOKEEPER_CONTAINER);
 
-    private static final MySQLContainer<?> MYSQL = MySQLContainerUtils.newMySQLContainer();
+    @RegisterApp
+    private final TestDropwizardAppExtension APP;
 
-    private static final ClickHouseContainer CLICKHOUSE = ClickHouseContainerUtils.newClickHouseContainer();
+    private final WireMockUtils.WireMockRuntime wireMock;
 
-    @RegisterExtension
-    private static final TestDropwizardAppExtension APP;
-
-    private static final WireMockUtils.WireMockRuntime wireMock;
-
-    static {
-        Startables.deepStart(MYSQL, CLICKHOUSE, REDIS).join();
+    {
+        Startables.deepStart(MYSQL, CLICKHOUSE, REDIS, ZOOKEEPER_CONTAINER).join();
 
         wireMock = WireMockUtils.startWireMock();
 
@@ -111,7 +114,7 @@ void tearDownAll() {
         wireMock.server().stop();
     }
 
-    private static void mockTargetWorkspace(String apiKey, String workspaceName, String workspaceId) {
+    private void mockTargetWorkspace(String apiKey, String workspaceName, String workspaceId) {
         AuthTestUtils.mockTargetWorkspace(wireMock.server(), apiKey, workspaceName, workspaceId, USER);
     }