diff --git a/.circleci/config.yml b/.circleci/config.yml index 0e9f576f1..7be2f68a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -62,13 +62,8 @@ commands: SPARK_VERSION: "<>" SCALA_VERSION: "<>" JDK_VERSION: "<>" - INTEGRATION_MAX_PARALLEL_FORKS: 1 - INTEGRATION_MAX_HEAP_SIZE: "1500M" - CORE_MAX_PARALLEL_FORKS: 2 - CORE_TEST_MAX_HEAP_SIZE: "2048m" CASSANDRA_USE_JDK11: <> command: | - export GRADLE_OPTS="-Xmx2g -Dorg.gradle.jvmargs=-Xmx2g" # Run compile/unit tests, skipping integration tests ./gradlew --no-daemon --max-workers=2 --stacktrace clean assemble check -x cassandra-analytics-integration-tests:test -Dcassandra.analytics.bridges.sstable_format=<> @@ -92,11 +87,8 @@ commands: SPARK_VERSION: "<>" SCALA_VERSION: "<>" JDK_VERSION: "<>" - INTEGRATION_MAX_PARALLEL_FORKS: 1 - INTEGRATION_MAX_HEAP_SIZE: "2500M" CASSANDRA_USE_JDK11: <> command: | - export GRADLE_OPTS="-Xmx2g -Dorg.gradle.jvmargs=-Xmx2g" export DTEST_JAR="dtest-<< parameters.cassandra >>.jar" export CASSANDRA_VERSION=$(echo << parameters.cassandra >> | cut -d'.' -f 1,2) # Run compile but not unit tests (which are run in run_build) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 953632909..e106de804 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -59,7 +59,7 @@ jobs: CASSANDRA_USE_JDK11=true ./scripts/build-dependencies.sh - ./gradlew codeCheckTasks + ./gradlew --no-daemon codeCheckTasks - name: Cache Maven repository uses: actions/cache@v4 with: @@ -112,11 +112,10 @@ jobs: export SPARK_VERSION="3" export SCALA_VERSION="${{ matrix.scala }}" export JDK_VERSION="11" - export INTEGRATION_MAX_PARALLEL_FORKS=1 - export INTEGRATION_MAX_HEAP_SIZE="1500M" + export INTEGRATION_MAX_HEAP_SIZE="3072m" export CASSANDRA_USE_JDK11=true - ./gradlew --stacktrace clean assemble check -x cassandra-analytics-integration-tests:test -Dcassandra.analytics.bridges.sstable_format=${{ matrix.sstable-format }} + ./gradlew --no-daemon --stacktrace clean assemble check -x cassandra-analytics-integration-tests:test -Dcassandra.analytics.bridges.sstable_format=${{ matrix.sstable-format }} integration-test: name: Integration test @@ -164,15 +163,14 @@ jobs: export SPARK_VERSION="3" export SCALA_VERSION="${{ matrix.scala }}" export JDK_VERSION="11" - export INTEGRATION_MAX_PARALLEL_FORKS=1 - export INTEGRATION_MAX_HEAP_SIZE="4096M" + export INTEGRATION_MAX_HEAP_SIZE="3072m" export CASSANDRA_USE_JDK11=true export DTEST_JAR="dtest-${{ matrix.cassandra }}.jar" export CASSANDRA_VERSION=$(echo ${{ matrix.cassandra }} | cut -d'.' -f 1,2) - ./gradlew --stacktrace clean assemble - + ./gradlew --no-daemon --stacktrace clean assemble + cd cassandra-analytics-integration-tests/src/test/java CLASSNAMES=$(find . -name '*Test.java' | sort | cut -c 3- | sed 's@/@.@g' | sed 's/.\{5\}$//' | awk 'NR % ${{ matrix.job_total }} == ${{ matrix.job_index }}') cd ../../../.. diff --git a/analytics-sidecar-client-common/build.gradle b/analytics-sidecar-client-common/build.gradle index e742f65ca..68a2c0cea 100644 --- a/analytics-sidecar-client-common/build.gradle +++ b/analytics-sidecar-client-common/build.gradle @@ -36,7 +36,7 @@ sourceCompatibility = 1.8 test { useJUnitPlatform() - maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1 + maxParallelForks = 2 reports { junitXml.setRequired(true) def destDir = Paths.get(rootProject.rootDir.absolutePath, "build", "test-results", "client-common").toFile() diff --git a/analytics-sidecar-client/build.gradle b/analytics-sidecar-client/build.gradle index 202c592d7..fd806b484 100644 --- a/analytics-sidecar-client/build.gradle +++ b/analytics-sidecar-client/build.gradle @@ -41,7 +41,7 @@ test { testLogging { events "passed", "skipped", "failed" } - maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1 + maxParallelForks = 2 reports { junitXml.setRequired(true) def destDir = Paths.get(rootProject.rootDir.absolutePath, "build", "test-results", "client").toFile() diff --git a/analytics-sidecar-vertx-client/build.gradle b/analytics-sidecar-vertx-client/build.gradle index f04aa7329..03f293572 100644 --- a/analytics-sidecar-vertx-client/build.gradle +++ b/analytics-sidecar-vertx-client/build.gradle @@ -34,7 +34,7 @@ if (propertyWithDefault("artifactType", null) == "common") test { useJUnitPlatform() - maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1 + maxParallelForks = 2 reports { junitXml.setRequired(true) def destDir = Paths.get(rootProject.rootDir.absolutePath, "build", "test-results", "vertx-client").toFile() diff --git a/build.gradle b/build.gradle index c556f9948..b96aeff9e 100644 --- a/build.gradle +++ b/build.gradle @@ -244,7 +244,10 @@ subprojects { codeCheckTasks.dependsOn(tasks.withType(Checkstyle)) codeCheckTasks.dependsOn(tasks.withType(RatTask)) + // Constrain all tests to non-parallel w/max heap at 2G while we figure out what's causing gradle error 100's tasks.withType(Test) { + maxParallelForks = 2 + maxHeapSize = '3072m' shouldRunAfter(codeCheckTasks) shouldRunAfter(tasks.withType(Checkstyle)) shouldRunAfter(tasks.withType(RatTask)) diff --git a/cassandra-analytics-cdc/build.gradle b/cassandra-analytics-cdc/build.gradle index 38c284065..8e9a0579b 100644 --- a/cassandra-analytics-cdc/build.gradle +++ b/cassandra-analytics-cdc/build.gradle @@ -146,7 +146,7 @@ test { minHeapSize = '1024m' maxHeapSize = '3072m' - maxParallelForks = Math.max(Runtime.runtime.availableProcessors() * 2, 8) + maxParallelForks = 2 forkEvery = 1 // Enables different end-to-end test classes use Spark contexts with different configurations // Make it so unit tests run on a Jar with Cassandra bridge implementations built in diff --git a/cassandra-analytics-core/build.gradle b/cassandra-analytics-core/build.gradle index ffce66600..9ff3bb1c9 100644 --- a/cassandra-analytics-core/build.gradle +++ b/cassandra-analytics-core/build.gradle @@ -182,8 +182,8 @@ tasks.register('testSequential', Test) { systemProperty "cassandra.analytics.bridges.sstable_format", System.getProperty("cassandra.analytics.bridges.sstable_format", "big") minHeapSize = '1024m' - maxHeapSize = System.getenv('CORE_TEST_MAX_HEAP_SIZE') ?: '3072m' - maxParallelForks = 1 + maxHeapSize = '3072m' + maxParallelForks = 2 forkEvery = 1 // Enables different end-to-end test classes use Spark contexts with different configurations // Make it so unit tests run on a Jar with Cassandra bridge implementations built in @@ -217,9 +217,8 @@ tasks.register('testSequential', Test) { test { systemProperty "cassandra.analytics.bridges.sstable_format", System.getProperty("cassandra.analytics.bridges.sstable_format", "big") minHeapSize = '1024m' - maxHeapSize = System.getenv('CORE_TEST_MAX_HEAP_SIZE') ?: '3072m' - maxParallelForks = System.getenv('CORE_MAX_PARALLEL_FORKS')?.toInteger() - ?: Math.max(Runtime.runtime.availableProcessors() * 2, 8) + maxHeapSize = '3072m' + maxParallelForks = 2 forkEvery = 1 // Enables different end-to-end test classes use Spark contexts with different configurations // Make it so unit tests run on a Jar with Cassandra bridge implementations built in diff --git a/cassandra-analytics-integration-tests/build.gradle b/cassandra-analytics-integration-tests/build.gradle index d8d469f66..a0c22c032 100644 --- a/cassandra-analytics-integration-tests/build.gradle +++ b/cassandra-analytics-integration-tests/build.gradle @@ -32,12 +32,9 @@ if (propertyWithDefault("artifactType", null) == "spark") apply from: "$rootDir/gradle/common/publishing.gradle" } -def integrationMaxHeapSize = System.getenv("INTEGRATION_MAX_HEAP_SIZE") ?: "3000M" +def integrationMaxHeapSize = '3072m' println("Using ${integrationMaxHeapSize} maxHeapSize") -def integrationMaxParallelForks = (System.getenv("INTEGRATION_MAX_PARALLEL_FORKS") ?: "4") as int -println("Using ${integrationMaxParallelForks} maxParallelForks") - def integrationEnableMtls = (System.getenv("INTEGRATION_MTLS_ENABLED") ?: "true") as boolean println("Using mTLS for tests? ${integrationEnableMtls}") @@ -92,7 +89,7 @@ test { systemProperty "cassandra.integration.sidecar.test.enable_mtls", integrationEnableMtls minHeapSize = '1g' maxHeapSize = integrationMaxHeapSize - maxParallelForks = integrationMaxParallelForks + maxParallelForks = 2 forkEvery = 1 // Enables different end-to-end test classes use Spark contexts with different configurations beforeTest { descriptor -> diff --git a/gradle.properties b/gradle.properties index 25e3a00aa..bd5d03fe2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -43,7 +43,7 @@ commonsLang3Version=3.12.0 cassandraDriverVersion=3.11.3 kafkaClientVersion=3.7.0 -org.gradle.jvmargs=-Xmx4096m +org.gradle.jvmargs=-Xmx3072m # Avro 1.12.* not compatible with JDK8 avroVersion=1.11.4