CASSANALYTICS-144: Constrain CI resources to prevent error 100

.circleci/config.yml

@@ -62,13 +62,8 @@ commands:
             SPARK_VERSION: "<<parameters.spark>>"
             SCALA_VERSION: "<<parameters.scala>>"
             JDK_VERSION: "<<parameters.jdk>>"
-            INTEGRATION_MAX_PARALLEL_FORKS: 1
-            INTEGRATION_MAX_HEAP_SIZE: "1500M"
-            CORE_MAX_PARALLEL_FORKS: 2
-            CORE_TEST_MAX_HEAP_SIZE: "2048m"
             CASSANDRA_USE_JDK11: <<parameters.use_jdk11>>
           command: |
-            export GRADLE_OPTS="-Xmx2g -Dorg.gradle.jvmargs=-Xmx2g"
             # Run compile/unit tests, skipping integration tests
             ./gradlew --no-daemon --max-workers=2 --stacktrace clean assemble check -x cassandra-analytics-integration-tests:test -Dcassandra.analytics.bridges.sstable_format=<<parameters.sstable_format>>
 
@@ -92,11 +87,8 @@ commands:
             SPARK_VERSION: "<<parameters.spark>>"
             SCALA_VERSION: "<<parameters.scala>>"
             JDK_VERSION: "<<parameters.jdk>>"
-            INTEGRATION_MAX_PARALLEL_FORKS: 1
-            INTEGRATION_MAX_HEAP_SIZE: "2500M"
             CASSANDRA_USE_JDK11: <<parameters.use_jdk11>>
           command: |
-            export GRADLE_OPTS="-Xmx2g -Dorg.gradle.jvmargs=-Xmx2g"
             export DTEST_JAR="dtest-<< parameters.cassandra >>.jar"
             export CASSANDRA_VERSION=$(echo << parameters.cassandra >> | cut -d'.' -f 1,2)
             # Run compile but not unit tests (which are run in run_build)

.github/workflows/test.yaml

@@ -59,7 +59,7 @@ jobs:
 
           CASSANDRA_USE_JDK11=true ./scripts/build-dependencies.sh
 
-          ./gradlew codeCheckTasks
+          ./gradlew --no-daemon codeCheckTasks
       - name: Cache Maven repository
         uses: actions/cache@v4
         with:
@@ -112,11 +112,10 @@ jobs:
           export SPARK_VERSION="3"
           export SCALA_VERSION="${{ matrix.scala }}"
           export JDK_VERSION="11"
-          export INTEGRATION_MAX_PARALLEL_FORKS=1
-          export INTEGRATION_MAX_HEAP_SIZE="1500M"
+          export INTEGRATION_MAX_HEAP_SIZE="3072m"
           export CASSANDRA_USE_JDK11=true
 
-          ./gradlew --stacktrace clean assemble check -x cassandra-analytics-integration-tests:test -Dcassandra.analytics.bridges.sstable_format=${{ matrix.sstable-format }}
+          ./gradlew --no-daemon --stacktrace clean assemble check -x cassandra-analytics-integration-tests:test -Dcassandra.analytics.bridges.sstable_format=${{ matrix.sstable-format }}
 
   integration-test:
     name: Integration test
@@ -164,15 +163,14 @@ jobs:
           export SPARK_VERSION="3"
           export SCALA_VERSION="${{ matrix.scala }}"
           export JDK_VERSION="11"
-          export INTEGRATION_MAX_PARALLEL_FORKS=1
-          export INTEGRATION_MAX_HEAP_SIZE="4096M"
+          export INTEGRATION_MAX_HEAP_SIZE="3072m"
           export CASSANDRA_USE_JDK11=true
 
           export DTEST_JAR="dtest-${{ matrix.cassandra }}.jar"
           export CASSANDRA_VERSION=$(echo ${{ matrix.cassandra }} | cut -d'.' -f 1,2)
 
-          ./gradlew --stacktrace clean assemble
-          
+          ./gradlew --no-daemon --stacktrace clean assemble
+
           cd cassandra-analytics-integration-tests/src/test/java
           CLASSNAMES=$(find . -name '*Test.java' | sort | cut -c 3- | sed 's@/@.@g' | sed 's/.\{5\}$//' | awk 'NR % ${{ matrix.job_total }} == ${{ matrix.job_index }}')
           cd ../../../..

analytics-sidecar-client-common/build.gradle

@@ -36,7 +36,7 @@ sourceCompatibility = 1.8
 
 test {
     useJUnitPlatform()
-    maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
+    maxParallelForks = 2
     reports {
         junitXml.setRequired(true)
         def destDir = Paths.get(rootProject.rootDir.absolutePath, "build", "test-results", "client-common").toFile()

analytics-sidecar-client/build.gradle

@@ -41,7 +41,7 @@ test {
     testLogging {
         events "passed", "skipped", "failed"
     }
-    maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
+    maxParallelForks = 2
     reports {
         junitXml.setRequired(true)
         def destDir = Paths.get(rootProject.rootDir.absolutePath, "build", "test-results", "client").toFile()

analytics-sidecar-vertx-client/build.gradle

@@ -34,7 +34,7 @@ if (propertyWithDefault("artifactType", null) == "common")
 
 test {
     useJUnitPlatform()
-    maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
+    maxParallelForks = 2
     reports {
         junitXml.setRequired(true)
         def destDir = Paths.get(rootProject.rootDir.absolutePath, "build", "test-results", "vertx-client").toFile()

build.gradle

@@ -244,7 +244,10 @@ subprojects {
   codeCheckTasks.dependsOn(tasks.withType(Checkstyle))
   codeCheckTasks.dependsOn(tasks.withType(RatTask))
 
+  // Constrain all tests to non-parallel w/max heap at 2G while we figure out what's causing gradle error 100's
   tasks.withType(Test) {
+    maxParallelForks = 2
+    maxHeapSize = '3072m'
     shouldRunAfter(codeCheckTasks)
     shouldRunAfter(tasks.withType(Checkstyle))
     shouldRunAfter(tasks.withType(RatTask))

cassandra-analytics-cdc/build.gradle

@@ -146,7 +146,7 @@ test {
 
     minHeapSize = '1024m'
     maxHeapSize = '3072m'
-    maxParallelForks = Math.max(Runtime.runtime.availableProcessors() * 2, 8)
+    maxParallelForks = 2
     forkEvery = 1  // Enables different end-to-end test classes use Spark contexts with different configurations
 
     // Make it so unit tests run on a Jar with Cassandra bridge implementations built in

cassandra-analytics-core/build.gradle

@@ -182,8 +182,8 @@ tasks.register('testSequential', Test) {
 
     systemProperty "cassandra.analytics.bridges.sstable_format", System.getProperty("cassandra.analytics.bridges.sstable_format", "big")
     minHeapSize = '1024m'
-    maxHeapSize = System.getenv('CORE_TEST_MAX_HEAP_SIZE') ?: '3072m'
-    maxParallelForks = 1
+    maxHeapSize = '3072m'
+    maxParallelForks = 2
     forkEvery = 1  // Enables different end-to-end test classes use Spark contexts with different configurations
 
     // Make it so unit tests run on a Jar with Cassandra bridge implementations built in
@@ -217,9 +217,8 @@ tasks.register('testSequential', Test) {
 test {
     systemProperty "cassandra.analytics.bridges.sstable_format", System.getProperty("cassandra.analytics.bridges.sstable_format", "big")
     minHeapSize = '1024m'
-    maxHeapSize = System.getenv('CORE_TEST_MAX_HEAP_SIZE') ?: '3072m'
-    maxParallelForks = System.getenv('CORE_MAX_PARALLEL_FORKS')?.toInteger()
-                       ?: Math.max(Runtime.runtime.availableProcessors() * 2, 8)
+    maxHeapSize = '3072m'
+    maxParallelForks = 2
     forkEvery = 1  // Enables different end-to-end test classes use Spark contexts with different configurations
 
     // Make it so unit tests run on a Jar with Cassandra bridge implementations built in

cassandra-analytics-integration-tests/build.gradle

@@ -32,12 +32,9 @@ if (propertyWithDefault("artifactType", null) == "spark")
     apply from: "$rootDir/gradle/common/publishing.gradle"
 }
 
-def integrationMaxHeapSize = System.getenv("INTEGRATION_MAX_HEAP_SIZE") ?: "3000M"
+def integrationMaxHeapSize = '3072m'
 println("Using ${integrationMaxHeapSize} maxHeapSize")
 
-def integrationMaxParallelForks = (System.getenv("INTEGRATION_MAX_PARALLEL_FORKS") ?: "4") as int
-println("Using ${integrationMaxParallelForks} maxParallelForks")
-
 def integrationEnableMtls = (System.getenv("INTEGRATION_MTLS_ENABLED") ?: "true") as boolean
 println("Using mTLS for tests? ${integrationEnableMtls}")
 
@@ -92,7 +89,7 @@ test {
     systemProperty "cassandra.integration.sidecar.test.enable_mtls", integrationEnableMtls
     minHeapSize = '1g'
     maxHeapSize = integrationMaxHeapSize
-    maxParallelForks = integrationMaxParallelForks
+    maxParallelForks = 2
     forkEvery = 1  // Enables different end-to-end test classes use Spark contexts with different configurations
 
     beforeTest { descriptor ->

gradle.properties

@@ -43,7 +43,7 @@ commonsLang3Version=3.12.0
 cassandraDriverVersion=3.11.3
 kafkaClientVersion=3.7.0
 
-org.gradle.jvmargs=-Xmx4096m
+org.gradle.jvmargs=-Xmx3072m
 
 # Avro 1.12.* not compatible with JDK8
 avroVersion=1.11.4