diff --git a/.github/workflows/cron-job-its.yml b/.github/workflows/cron-job-its.yml index 1f555778ab9f..d26ece294c91 100644 --- a/.github/workflows/cron-job-its.yml +++ b/.github/workflows/cron-job-its.yml @@ -60,7 +60,7 @@ jobs: strategy: fail-fast: false matrix: - testing_group: [kafka-index, kafka-index-slow, kafka-transactional-index, kafka-transactional-index-slow, kafka-data-format, realtime-index, append-ingestion] + testing_group: [kafka-index, kafka-index-slow, kafka-transactional-index, kafka-transactional-index-slow, kafka-data-format, realtime-index] uses: ./.github/workflows/reusable-standard-its.yml needs: build with: @@ -74,7 +74,7 @@ jobs: strategy: fail-fast: false matrix: - testing_group: [ input-source, kafka-index, kafka-transactional-index, kafka-index-slow, kafka-transactional-index-slow, kafka-data-format, append-ingestion ] + testing_group: [ kafka-index, kafka-transactional-index, kafka-index-slow, kafka-transactional-index-slow, kafka-data-format ] uses: ./.github/workflows/reusable-standard-its.yml needs: build with: diff --git a/.github/workflows/standard-its.yml b/.github/workflows/standard-its.yml index 3c30c5c91945..e195572b6fb8 100644 --- a/.github/workflows/standard-its.yml +++ b/.github/workflows/standard-its.yml @@ -47,7 +47,7 @@ jobs: strategy: fail-fast: false matrix: - testing_group: [kafka-index, kafka-index-slow, kafka-transactional-index, kafka-transactional-index-slow, realtime-index, append-ingestion, cds-task-schema-publish-disabled, cds-coordinator-metadata-query-disabled] + testing_group: [kafka-index, kafka-index-slow, kafka-transactional-index, kafka-transactional-index-slow, realtime-index] uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: @@ -63,7 +63,7 @@ jobs: strategy: fail-fast: false matrix: - testing_group: [input-source, kafka-index, append-ingestion] + testing_group: [kafka-index] uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: @@ -101,15 +101,3 @@ jobs: mysql_driver: org.mariadb.jdbc.Driver override_config_path: ./environment-configs/test-groups/prepopulated-data group: query - - integration-custom-coordinator-duties-tests: - needs: changes - uses: ./.github/workflows/reusable-standard-its.yml - if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} - with: - build_jdk: 17 - runtime_jdk: 17 - testing_groups: -Dgroups=custom-coordinator-duties - use_indexer: middleManager - override_config_path: ./environment-configs/test-groups/custom-coordinator-duties - group: custom coordinator duties \ No newline at end of file diff --git a/docs/development/modules.md b/docs/development/modules.md index c62a6d4a086f..ff0411b150ab 100644 --- a/docs/development/modules.md +++ b/docs/development/modules.md @@ -326,19 +326,8 @@ The duties will be grouped into multiple groups as per the elements in list `dru All duties in the same group will have the same run period configured by `druid.coordinator..period`. Currently, there is a single thread running the duties sequentially for each group. -For example, see `KillSupervisorsCustomDuty` for a custom coordinator duty implementation and the `custom-coordinator-duties` -integration test group which loads `KillSupervisorsCustomDuty` using the configs set in `integration-tests/docker/environment-configs/test-groups/custom-coordinator-duties`. -This config file adds the configs below to enable a custom coordinator duty. - -```properties -druid.coordinator.dutyGroups=["cleanupMetadata"] -druid.coordinator.cleanupMetadata.duties=["killSupervisors"] -druid.coordinator.cleanupMetadata.duty.killSupervisors.durationToRetain=PT0M -druid.coordinator.cleanupMetadata.period=PT10S -``` - -These configurations create a custom coordinator duty group called `cleanupMetadata` which runs a custom coordinator duty called `killSupervisors` every 10 seconds. -The custom coordinator duty `killSupervisors` also has a config called `durationToRetain` which is set to 0 minute. +For example, see `KillSupervisorsCustomDuty` for a custom coordinator duty implementation and the `KillSupervisorsCustomDutyTest` +for sample properties that may be used to configure the `KillSupervisorsCustomDuty`. ### Routing data through a HTTP proxy for your extension diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionSupervisorTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionSupervisorTest.java index 9019a0cb5ec5..c3796ccaa151 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionSupervisorTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionSupervisorTest.java @@ -29,11 +29,6 @@ import org.apache.druid.indexing.overlord.Segments; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.granularity.Granularity; -import org.apache.druid.msq.guice.IndexerMemoryManagementModule; -import org.apache.druid.msq.guice.MSQDurableStorageModule; -import org.apache.druid.msq.guice.MSQIndexingModule; -import org.apache.druid.msq.guice.MSQSqlModule; -import org.apache.druid.msq.guice.SqlTaskModule; import org.apache.druid.query.DruidMetrics; import org.apache.druid.rpc.UpdateResponse; import org.apache.druid.server.coordinator.ClusterCompactionConfig; @@ -90,15 +85,7 @@ public EmbeddedDruidCluster createCluster() "[\"org.apache.druid.query.policy.NoRestrictionPolicy\"]" ) .addCommonProperty("druid.policy.enforcer.type", "restrictAllTables") - .addExtensions( - CatalogClientModule.class, - CatalogCoordinatorModule.class, - IndexerMemoryManagementModule.class, - MSQDurableStorageModule.class, - MSQIndexingModule.class, - MSQSqlModule.class, - SqlTaskModule.class - ) + .addExtensions(CatalogClientModule.class, CatalogCoordinatorModule.class) .addServer(coordinator) .addServer(overlord) .addServer(indexer) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IndexParallelTaskTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IndexParallelTaskTest.java index 23645dea5942..19ee1061a864 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IndexParallelTaskTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IndexParallelTaskTest.java @@ -129,30 +129,7 @@ public void test_runIndexTask_andReindexIntoAnotherDatasource(PartitionsSpec par { final boolean isRollup = partitionsSpec.isForceGuaranteedRollupCompatible(); - final TaskBuilder.IndexParallel indexTask = - TaskBuilder.ofTypeIndexParallel() - .dataSource(dataSource) - .timestampColumn("timestamp") - .jsonInputFormat() - .localInputSourceWithFiles( - Resources.DataFile.tinyWiki1Json(), - Resources.DataFile.tinyWiki2Json(), - Resources.DataFile.tinyWiki3Json() - ) - .segmentGranularity("DAY") - .dimensions("namespace", "page", "language") - .metricAggregates( - new DoubleSumAggregatorFactory("added", "added"), - new DoubleSumAggregatorFactory("deleted", "deleted"), - new DoubleSumAggregatorFactory("delta", "delta"), - new CountAggregatorFactory("count") - ) - .tuningConfig( - t -> t.withPartitionsSpec(partitionsSpec) - .withForceGuaranteedRollup(isRollup) - .withMaxNumConcurrentSubTasks(10) - .withSplitHintSpec(new MaxSizeSplitHintSpec(1, null)) - ); + final TaskBuilder.IndexParallel indexTask = buildIndexParallelTask(partitionsSpec, false); runTask(indexTask, dataSource); cluster.callApi().waitForAllSegmentsToBeAvailable(dataSource, coordinator, broker); @@ -211,6 +188,60 @@ public void test_runIndexTask_andReindexIntoAnotherDatasource(PartitionsSpec par runQueries(dataSource3); } + @MethodSource("getTestParamPartitionsSpec") + @ParameterizedTest(name = "partitionsSpec={0}") + public void test_runIndexTask_andAppendData(PartitionsSpec partitionsSpec) + { + final TaskBuilder.IndexParallel initialTask = buildIndexParallelTask(partitionsSpec, false); + runTask(initialTask, dataSource); + cluster.callApi().waitForAllSegmentsToBeAvailable(dataSource, coordinator, broker); + cluster.callApi().verifySqlQuery("SELECT COUNT(*) FROM %s", dataSource, "10"); + runGroupByQuery("Crimson Typhoon,1,905.0,9050.0"); + + final TaskBuilder.IndexParallel appendTask + = buildIndexParallelTask(new DynamicPartitionsSpec(null, null), true); + runTask(appendTask, dataSource); + cluster.callApi().waitForAllSegmentsToBeAvailable(dataSource, coordinator, broker); + cluster.callApi().verifySqlQuery("SELECT COUNT(*) FROM %s", dataSource, "20"); + runGroupByQuery("Crimson Typhoon,2,1810.0,18100.0"); + } + + /** + * Creates a builder for an "index_parallel" task to ingest into {@link #dataSource}. + */ + private TaskBuilder.IndexParallel buildIndexParallelTask( + PartitionsSpec partitionsSpec, + boolean appendToExisting + ) + { + final boolean isRollup = partitionsSpec.isForceGuaranteedRollupCompatible(); + + return TaskBuilder.ofTypeIndexParallel() + .dataSource(dataSource) + .timestampColumn("timestamp") + .jsonInputFormat() + .localInputSourceWithFiles( + Resources.DataFile.tinyWiki1Json(), + Resources.DataFile.tinyWiki2Json(), + Resources.DataFile.tinyWiki3Json() + ) + .segmentGranularity("DAY") + .dimensions("namespace", "page", "language") + .metricAggregates( + new DoubleSumAggregatorFactory("added", "added"), + new DoubleSumAggregatorFactory("deleted", "deleted"), + new DoubleSumAggregatorFactory("delta", "delta"), + new CountAggregatorFactory("count") + ) + .appendToExisting(appendToExisting) + .tuningConfig( + t -> t.withPartitionsSpec(partitionsSpec) + .withForceGuaranteedRollup(isRollup) + .withMaxNumConcurrentSubTasks(10) + .withSplitHintSpec(new MaxSizeSplitHintSpec(1, null)) + ); + } + private String runTask(TaskBuilder.IndexParallel taskBuilder, String dataSource) { final String taskId = EmbeddedClusterApis.newTaskId(dataSource); @@ -224,8 +255,13 @@ private void runQueries(String dataSource) "10,2013-09-01T12:41:27.000Z,2013-08-31T01:02:33.000Z", cluster.runSql("SELECT COUNT(*), MAX(__time), MIN(__time) FROM %s", dataSource) ); + runGroupByQuery("Crimson Typhoon,1,905.0,9050.0"); + } + + private void runGroupByQuery(String expectedResult) + { Assertions.assertEquals( - "Crimson Typhoon,1,905.0,9050.0", + expectedResult, cluster.runSql( "SELECT \"page\", COUNT(*) AS \"rows\", SUM(\"added\"), 10 * SUM(\"added\") AS added_times_ten" + " FROM %s" diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/server/KillSupervisorsCustomDutyTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/server/KillSupervisorsCustomDutyTest.java new file mode 100644 index 000000000000..f29d4b305b5a --- /dev/null +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/server/KillSupervisorsCustomDutyTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testing.embedded.server; + +import com.fasterxml.jackson.core.type.TypeReference; +import org.apache.druid.error.ExceptionMatcher; +import org.apache.druid.indexing.compact.CompactionSupervisorSpec; +import org.apache.druid.indexing.overlord.supervisor.NoopSupervisorSpec; +import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; +import org.apache.druid.indexing.overlord.supervisor.VersionedSupervisorSpec; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.rpc.HttpResponseException; +import org.apache.druid.rpc.RequestBuilder; +import org.apache.druid.server.coordinator.InlineSchemaDataSourceCompactionConfig; +import org.apache.druid.testing.embedded.EmbeddedBroker; +import org.apache.druid.testing.embedded.EmbeddedCoordinator; +import org.apache.druid.testing.embedded.EmbeddedDruidCluster; +import org.apache.druid.testing.embedded.EmbeddedOverlord; +import org.apache.druid.testing.embedded.junit5.EmbeddedClusterTestBase; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +public class KillSupervisorsCustomDutyTest extends EmbeddedClusterTestBase +{ + private final EmbeddedCoordinator coordinator = new EmbeddedCoordinator() + .addProperty("druid.coordinator.kill.supervisor.on", "false") + .addProperty("druid.coordinator.dutyGroups", "[\"cleanupMetadata\"]") + .addProperty("druid.coordinator.cleanupMetadata.duties", "[\"killSupervisors\"]") + .addProperty("druid.coordinator.cleanupMetadata.duty.killSupervisors.durationToRetain", "PT0M") + .addProperty("druid.coordinator.cleanupMetadata.period", "PT0.1S"); + + @Override + protected EmbeddedDruidCluster createCluster() + { + return EmbeddedDruidCluster + .withEmbeddedDerbyAndZookeeper() + .useLatchableEmitter() + .addServer(coordinator) + .addServer(new EmbeddedOverlord()) + .addServer(new EmbeddedBroker()); + } + + @Test + public void test_customDuty_removesHistoryOfTerminatedSupervisor() + { + // Create a compaction supervisor + final CompactionSupervisorSpec supervisor = new CompactionSupervisorSpec( + InlineSchemaDataSourceCompactionConfig.builder().forDataSource(dataSource).build(), + false, + null + ); + cluster.callApi().postSupervisor(supervisor); + + // Verify that the history of the supervisor has 1 entry + final List history = getSupervisorHistory(supervisor.getId()); + Assertions.assertEquals(1, history.size()); + + final SupervisorSpec supervisorEntry = history.get(0).getSpec(); + Assertions.assertNotNull(supervisorEntry); + Assertions.assertEquals(List.of(dataSource), supervisorEntry.getDataSources()); + Assertions.assertEquals(supervisor.getId(), supervisorEntry.getId()); + + // Terminate the supervisor + cluster.callApi().onLeaderOverlord(o -> o.terminateSupervisor(supervisor.getId())); + + // Verify that the history now has 2 entries and the latest entry is a tombstone + final List historyAfterTermination = getSupervisorHistory(supervisor.getId()); + Assertions.assertEquals(2, historyAfterTermination.size()); + Assertions.assertInstanceOf(NoopSupervisorSpec.class, historyAfterTermination.get(0).getSpec()); + + // Wait until the cleanup metric has been emitted + coordinator.latchableEmitter().waitForEvent( + event -> event.hasMetricName("metadata/kill/supervisor/count") + .hasValueMatching(Matchers.greaterThanOrEqualTo(1L)) + ); + + // Verify that the history now returns 404 Not Found + MatcherAssert.assertThat( + Assertions.assertThrows( + RuntimeException.class, + () -> getSupervisorHistory(supervisor.getId()) + ), + ExceptionMatcher.of(RuntimeException.class).expectRootCause( + ExceptionMatcher.of(HttpResponseException.class) + .expectMessageContains("404 Not Found") + .expectMessageContains(StringUtils.format("No history for [%s]", supervisor.getId())) + ) + ); + } + + private List getSupervisorHistory(String supervisorId) + { + final String url = StringUtils.format( + "/druid/indexer/v1/supervisor/%s/history", + StringUtils.urlEncode(supervisorId) + ); + return cluster.callApi().serviceClient().onLeaderOverlord( + mapper -> new RequestBuilder(HttpMethod.GET, url), + new TypeReference<>() {} + ); + } +} diff --git a/integration-tests/docker/docker-compose.cds-coordinator-metadata-query-disabled.yml b/integration-tests/docker/docker-compose.cds-coordinator-metadata-query-disabled.yml deleted file mode 100644 index 569cef1d06da..000000000000 --- a/integration-tests/docker/docker-compose.cds-coordinator-metadata-query-disabled.yml +++ /dev/null @@ -1,117 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "2.2" -services: - druid-zookeeper-kafka: - extends: - file: docker-compose.base.yml - service: druid-zookeeper-kafka - - druid-metadata-storage: - extends: - file: docker-compose.base.yml - service: druid-metadata-storage - environment: - - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} - depends_on: - - druid-zookeeper-kafka - - druid-coordinator: - extends: - file: docker-compose.base.yml - service: druid-coordinator - environment: - - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} - - druid_centralizedDatasourceSchema_enabled=true - - druid_centralizedDatasourceSchema_backFillEnabled=true - - druid_centralizedDatasourceSchema_backFillPeriod=15000 - - druid_coordinator_segmentMetadata_metadataRefreshPeriod=PT15S - - druid_coordinator_segmentMetadata_disableSegmentMetadataQueries=true - - druid_manager_segments_useIncrementalCache=always - depends_on: - druid-overlord: - condition: service_started - druid-metadata-storage: - condition: service_healthy - druid-zookeeper-kafka: - condition: service_started - - druid-overlord: - extends: - file: docker-compose.base.yml - service: druid-overlord - environment: - - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} - - druid_centralizedDatasourceSchema_enabled=true - - druid_manager_segments_useIncrementalCache=always - depends_on: - druid-metadata-storage: - condition: service_healthy - druid-zookeeper-kafka: - condition: service_started - - druid-historical: - extends: - file: docker-compose.base.yml - service: druid-historical - environment: - - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} - depends_on: - - druid-zookeeper-kafka - - druid-middlemanager: - extends: - file: docker-compose.base.yml - service: druid-middlemanager - environment: - - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} - - druid_indexer_fork_property_druid_centralizedDatasourceSchema_enabled=true - depends_on: - - druid-zookeeper-kafka - - druid-overlord - - druid-broker: - extends: - file: docker-compose.base.yml - service: druid-broker - environment: - - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} - - druid_sql_planner_metadataRefreshPeriod=PT20S - - druid_sql_planner_disableSegmentMetadataQueries=true - depends_on: - - druid-coordinator - - druid-zookeeper-kafka - - druid-middlemanager - - druid-historical - - druid-router: - extends: - file: docker-compose.base.yml - service: druid-router - environment: - - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} - depends_on: - - druid-zookeeper-kafka - - druid-coordinator - - druid-broker - - druid-overlord - -networks: - druid-it-net: - name: druid-it-net - ipam: - config: - - subnet: 172.172.172.0/24 diff --git a/integration-tests/docker/docker-compose.cds-task-schema-publish-disabled.yml b/integration-tests/docker/docker-compose.cds-task-schema-publish-disabled.yml deleted file mode 100644 index 65bbef6b2384..000000000000 --- a/integration-tests/docker/docker-compose.cds-task-schema-publish-disabled.yml +++ /dev/null @@ -1,118 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "2.2" -services: - druid-zookeeper-kafka: - extends: - file: docker-compose.base.yml - service: druid-zookeeper-kafka - - druid-metadata-storage: - extends: - file: docker-compose.base.yml - service: druid-metadata-storage - environment: - - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} - depends_on: - - druid-zookeeper-kafka - - druid-coordinator: - extends: - file: docker-compose.base.yml - service: druid-coordinator - environment: - - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} - - druid_centralizedDatasourceSchema_enabled=true - - druid_centralizedDatasourceSchema_taskSchemaPublishDisabled=true - - druid_centralizedDatasourceSchema_backFillEnabled=true - - druid_centralizedDatasourceSchema_backFillPeriod=15000 - - druid_coordinator_segmentMetadata_metadataRefreshPeriod=PT15S - - druid_manager_segments_useIncrementalCache=always - depends_on: - druid-overlord: - condition: service_started - druid-metadata-storage: - condition: service_healthy - druid-zookeeper-kafka: - condition: service_started - - druid-overlord: - extends: - file: docker-compose.base.yml - service: druid-overlord - environment: - - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} - - druid_centralizedDatasourceSchema_enabled=true - - druid_centralizedDatasourceSchema_taskSchemaPublishDisabled=true - - druid_manager_segments_useIncrementalCache=always - depends_on: - druid-metadata-storage: - condition: service_healthy - druid-zookeeper-kafka: - condition: service_started - - druid-historical: - extends: - file: docker-compose.base.yml - service: druid-historical - environment: - - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} - depends_on: - - druid-zookeeper-kafka - - druid-middlemanager: - extends: - file: docker-compose.base.yml - service: druid-middlemanager - environment: - - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} - - druid_indexer_fork_property_druid_centralizedDatasourceSchema_enabled=true - depends_on: - - druid-zookeeper-kafka - - druid-overlord - - druid-broker: - extends: - file: docker-compose.base.yml - service: druid-broker - environment: - - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} - - druid_sql_planner_metadataRefreshPeriod=PT20S - - druid_sql_planner_disableSegmentMetadataQueries=true - depends_on: - - druid-coordinator - - druid-zookeeper-kafka - - druid-middlemanager - - druid-historical - - druid-router: - extends: - file: docker-compose.base.yml - service: druid-router - environment: - - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} - depends_on: - - druid-zookeeper-kafka - - druid-coordinator - - druid-broker - - druid-overlord - -networks: - druid-it-net: - name: druid-it-net - ipam: - config: - - subnet: 172.172.172.0/24 diff --git a/integration-tests/docker/druid.sh b/integration-tests/docker/druid.sh index 6d666af1a17e..9478a9e9ef48 100755 --- a/integration-tests/docker/druid.sh +++ b/integration-tests/docker/druid.sh @@ -103,14 +103,9 @@ setupData() # The "query" and "security" test groups require data to be setup before running the tests. # In particular, they requires segments to be download from a pre-existing s3 bucket. # This is done by using the loadSpec put into metadatastore and s3 credientials set below. - if [ "$DRUID_INTEGRATION_TEST_GROUP" = "query" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "security" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "upgrade" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "centralized-datasource-schema" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "cds-task-schema-publish-disabled" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "cds-coordinator-metadata-query-disabled" ]; then + if [ "$DRUID_INTEGRATION_TEST_GROUP" = "query" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "security" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "upgrade" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "centralized-datasource-schema" ]; then cat /test-data/${DRUID_INTEGRATION_TEST_GROUP}-sample-data.sql | mysql -u root druid fi - # The SqlInputSource tests in the "input-source" test group require data to be setup in MySQL before running the tests. - if [ "$DRUID_INTEGRATION_TEST_GROUP" = "input-source" ] ; then - echo "GRANT ALL ON sqlinputsource.* TO 'druid'@'%'; CREATE database sqlinputsource DEFAULT CHARACTER SET utf8mb4;" | mysql -u root druid - cat /test-data/sql-input-source-sample-data.sql | mysql -u root druid - fi service mariadb stop } diff --git a/integration-tests/docker/environment-configs/test-groups/custom-coordinator-duties b/integration-tests/docker/environment-configs/test-groups/custom-coordinator-duties deleted file mode 100644 index cea6370c2914..000000000000 --- a/integration-tests/docker/environment-configs/test-groups/custom-coordinator-duties +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# If you are making a change in load list below, make the necessary changes in github actions too -druid_extensions_loadList=["druid-kafka-indexing-service","mysql-metadata-storage","druid-s3-extensions","druid-basic-security","simple-client-sslcontext","druid-testing-tools","druid-lookups-cached-global","druid-histogram","druid-datasketches"] - -druid_coordinator_period_metadataStoreManagementPeriod=PT5S -druid_sql_planner_authorizeSystemTablesDirectly=false - -#Testing kill supervisor custom coordinator duty -druid_coordinator_kill_supervisor_on=false -druid_coordinator_dutyGroups=["cleanupMetadata"] -druid_coordinator_cleanupMetadata_duties=["killSupervisors"] -druid_coordinator_cleanupMetadata_duty_killSupervisors_durationToRetain=PT0M -druid_coordinator_cleanupMetadata_period=PT10S diff --git a/integration-tests/docker/test-data/cds-coordinator-metadata-query-disabled-sample-data.sql b/integration-tests/docker/test-data/cds-coordinator-metadata-query-disabled-sample-data.sql deleted file mode 100644 index abe0f115189b..000000000000 --- a/integration-tests/docker/test-data/cds-coordinator-metadata-query-disabled-sample-data.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Licensed to the Apache Software Foundation (ASF) under one or more --- contributor license agreements. See the NOTICE file distributed with --- this work for additional information regarding copyright ownership. --- The ASF licenses this file to You under the Apache License, Version 2.0 --- (the "License"); you may not use this file except in compliance with --- the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -INSERT INTO druid_segments (id,dataSource,created_date,start,end,partitioned,version,used,payload,used_status_last_updated) VALUES ('twitterstream_2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z_2013-01-02T04:13:41.980Z_v9','twitterstream','2013-05-13T01:08:18.192Z','2013-01-01T00:00:00.000Z','2013-01-02T00:00:00.000Z',0,'2013-01-02T04:13:41.980Z_v9',1,'{\"dataSource\":\"twitterstream\",\"interval\":\"2013-01-01T00:00:00.000Z/2013-01-02T00:00:00.000Z\",\"version\":\"2013-01-02T04:13:41.980Z_v9\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/twitterstream/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/2013-01-02T04:13:41.980Z_v9/0/index.zip\"},\"dimensions\":\"has_links,first_hashtag,user_time_zone,user_location,has_mention,user_lang,rt_name,user_name,is_retweet,is_viral,has_geo,url_domain,user_mention_name,reply_to_name\",\"metrics\":\"count,tweet_length,num_followers,num_links,num_mentions,num_hashtags,num_favorites,user_total_tweets\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":445235220,\"identifier\":\"twitterstream_2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z_2013-01-02T04:13:41.980Z_v9\"}','1970-01-01T00:00:00.000Z'); -INSERT INTO druid_segments (id,dataSource,created_date,start,end,partitioned,version,used,payload,used_status_last_updated) VALUES ('twitterstream_2013-01-02T00:00:00.000Z_2013-01-03T00:00:00.000Z_2013-01-03T03:44:58.791Z_v9','twitterstream','2013-05-13T00:03:28.640Z','2013-01-02T00:00:00.000Z','2013-01-03T00:00:00.000Z',0,'2013-01-03T03:44:58.791Z_v9',1,'{\"dataSource\":\"twitterstream\",\"interval\":\"2013-01-02T00:00:00.000Z/2013-01-03T00:00:00.000Z\",\"version\":\"2013-01-03T03:44:58.791Z_v9\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/twitterstream/2013-01-02T00:00:00.000Z_2013-01-03T00:00:00.000Z/2013-01-03T03:44:58.791Z_v9/0/index.zip\"},\"dimensions\":\"has_links,first_hashtag,user_time_zone,user_location,has_mention,user_lang,rt_name,user_name,is_retweet,is_viral,has_geo,url_domain,user_mention_name,reply_to_name\",\"metrics\":\"count,tweet_length,num_followers,num_links,num_mentions,num_hashtags,num_favorites,user_total_tweets\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":435325540,\"identifier\":\"twitterstream_2013-01-02T00:00:00.000Z_2013-01-03T00:00:00.000Z_2013-01-03T03:44:58.791Z_v9\"}','1970-01-01T00:00:00.000Z'); -INSERT INTO druid_segments (id,dataSource,created_date,start,end,partitioned,version,used,payload,used_status_last_updated) VALUES ('twitterstream_2013-01-03T00:00:00.000Z_2013-01-04T00:00:00.000Z_2013-01-04T04:09:13.590Z_v9','twitterstream','2013-05-13T00:03:48.807Z','2013-01-03T00:00:00.000Z','2013-01-04T00:00:00.000Z',0,'2013-01-04T04:09:13.590Z_v9',1,'{\"dataSource\":\"twitterstream\",\"interval\":\"2013-01-03T00:00:00.000Z/2013-01-04T00:00:00.000Z\",\"version\":\"2013-01-04T04:09:13.590Z_v9\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/twitterstream/2013-01-03T00:00:00.000Z_2013-01-04T00:00:00.000Z/2013-01-04T04:09:13.590Z_v9/0/index.zip\"},\"dimensions\":\"has_links,first_hashtag,user_time_zone,user_location,has_mention,user_lang,rt_name,user_name,is_retweet,is_viral,has_geo,url_domain,user_mention_name,reply_to_name\",\"metrics\":\"count,tweet_length,num_followers,num_links,num_mentions,num_hashtags,num_favorites,user_total_tweets\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":411651320,\"identifier\":\"twitterstream_2013-01-03T00:00:00.000Z_2013-01-04T00:00:00.000Z_2013-01-04T04:09:13.590Z_v9\"}','1970-01-01T00:00:00.000Z'); -INSERT INTO druid_segments (id,dataSource,created_date,start,end,partitioned,version,used,payload,used_status_last_updated) VALUES ('wikipedia_editstream_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9','wikipedia_editstream','2013-03-15T20:49:52.348Z','2012-12-29T00:00:00.000Z','2013-01-10T08:00:00.000Z',0,'2013-01-10T08:13:47.830Z_v9',1,'{\"dataSource\":\"wikipedia_editstream\",\"interval\":\"2012-12-29T00:00:00.000Z/2013-01-10T08:00:00.000Z\",\"version\":\"2013-01-10T08:13:47.830Z_v9\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/wikipedia_editstream/2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z/2013-01-10T08:13:47.830Z_v9/0/index.zip\"},\"dimensions\":\"anonymous,area_code,city,continent_code,country_name,dma_code,geo,language,namespace,network,newpage,page,postal_code,region_lookup,robot,unpatrolled,user\",\"metrics\":\"added,count,deleted,delta,delta_hist,unique_users,variation\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":446027801,\"identifier\":\"wikipedia_editstream_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9\"}','1970-01-01T00:00:00.000Z'); -INSERT INTO druid_segments (id, dataSource, created_date, start, end, partitioned, version, used, payload,used_status_last_updated) VALUES ('wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z', 'wikipedia', '2013-08-08T21:26:23.799Z', '2013-08-01T00:00:00.000Z', '2013-08-02T00:00:00.000Z', '0', '2013-08-08T21:22:48.989Z', '1', '{\"dataSource\":\"wikipedia\",\"interval\":\"2013-08-01T00:00:00.000Z/2013-08-02T00:00:00.000Z\",\"version\":\"2013-08-08T21:22:48.989Z\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/wikipedia/20130801T000000.000Z_20130802T000000.000Z/2013-08-08T21_22_48.989Z/0/index.zip\"},\"dimensions\":\"dma_code,continent_code,geo,area_code,robot,country_name,network,city,namespace,anonymous,unpatrolled,page,postal_code,language,newpage,user,region_lookup\",\"metrics\":\"count,delta,variation,added,deleted\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":24664730,\"identifier\":\"wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z\"}','1970-01-01T00:00:00.000Z'); diff --git a/integration-tests/docker/test-data/cds-task-schema-publish-disabled-sample-data.sql b/integration-tests/docker/test-data/cds-task-schema-publish-disabled-sample-data.sql deleted file mode 100644 index abe0f115189b..000000000000 --- a/integration-tests/docker/test-data/cds-task-schema-publish-disabled-sample-data.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Licensed to the Apache Software Foundation (ASF) under one or more --- contributor license agreements. See the NOTICE file distributed with --- this work for additional information regarding copyright ownership. --- The ASF licenses this file to You under the Apache License, Version 2.0 --- (the "License"); you may not use this file except in compliance with --- the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -INSERT INTO druid_segments (id,dataSource,created_date,start,end,partitioned,version,used,payload,used_status_last_updated) VALUES ('twitterstream_2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z_2013-01-02T04:13:41.980Z_v9','twitterstream','2013-05-13T01:08:18.192Z','2013-01-01T00:00:00.000Z','2013-01-02T00:00:00.000Z',0,'2013-01-02T04:13:41.980Z_v9',1,'{\"dataSource\":\"twitterstream\",\"interval\":\"2013-01-01T00:00:00.000Z/2013-01-02T00:00:00.000Z\",\"version\":\"2013-01-02T04:13:41.980Z_v9\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/twitterstream/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/2013-01-02T04:13:41.980Z_v9/0/index.zip\"},\"dimensions\":\"has_links,first_hashtag,user_time_zone,user_location,has_mention,user_lang,rt_name,user_name,is_retweet,is_viral,has_geo,url_domain,user_mention_name,reply_to_name\",\"metrics\":\"count,tweet_length,num_followers,num_links,num_mentions,num_hashtags,num_favorites,user_total_tweets\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":445235220,\"identifier\":\"twitterstream_2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z_2013-01-02T04:13:41.980Z_v9\"}','1970-01-01T00:00:00.000Z'); -INSERT INTO druid_segments (id,dataSource,created_date,start,end,partitioned,version,used,payload,used_status_last_updated) VALUES ('twitterstream_2013-01-02T00:00:00.000Z_2013-01-03T00:00:00.000Z_2013-01-03T03:44:58.791Z_v9','twitterstream','2013-05-13T00:03:28.640Z','2013-01-02T00:00:00.000Z','2013-01-03T00:00:00.000Z',0,'2013-01-03T03:44:58.791Z_v9',1,'{\"dataSource\":\"twitterstream\",\"interval\":\"2013-01-02T00:00:00.000Z/2013-01-03T00:00:00.000Z\",\"version\":\"2013-01-03T03:44:58.791Z_v9\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/twitterstream/2013-01-02T00:00:00.000Z_2013-01-03T00:00:00.000Z/2013-01-03T03:44:58.791Z_v9/0/index.zip\"},\"dimensions\":\"has_links,first_hashtag,user_time_zone,user_location,has_mention,user_lang,rt_name,user_name,is_retweet,is_viral,has_geo,url_domain,user_mention_name,reply_to_name\",\"metrics\":\"count,tweet_length,num_followers,num_links,num_mentions,num_hashtags,num_favorites,user_total_tweets\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":435325540,\"identifier\":\"twitterstream_2013-01-02T00:00:00.000Z_2013-01-03T00:00:00.000Z_2013-01-03T03:44:58.791Z_v9\"}','1970-01-01T00:00:00.000Z'); -INSERT INTO druid_segments (id,dataSource,created_date,start,end,partitioned,version,used,payload,used_status_last_updated) VALUES ('twitterstream_2013-01-03T00:00:00.000Z_2013-01-04T00:00:00.000Z_2013-01-04T04:09:13.590Z_v9','twitterstream','2013-05-13T00:03:48.807Z','2013-01-03T00:00:00.000Z','2013-01-04T00:00:00.000Z',0,'2013-01-04T04:09:13.590Z_v9',1,'{\"dataSource\":\"twitterstream\",\"interval\":\"2013-01-03T00:00:00.000Z/2013-01-04T00:00:00.000Z\",\"version\":\"2013-01-04T04:09:13.590Z_v9\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/twitterstream/2013-01-03T00:00:00.000Z_2013-01-04T00:00:00.000Z/2013-01-04T04:09:13.590Z_v9/0/index.zip\"},\"dimensions\":\"has_links,first_hashtag,user_time_zone,user_location,has_mention,user_lang,rt_name,user_name,is_retweet,is_viral,has_geo,url_domain,user_mention_name,reply_to_name\",\"metrics\":\"count,tweet_length,num_followers,num_links,num_mentions,num_hashtags,num_favorites,user_total_tweets\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":411651320,\"identifier\":\"twitterstream_2013-01-03T00:00:00.000Z_2013-01-04T00:00:00.000Z_2013-01-04T04:09:13.590Z_v9\"}','1970-01-01T00:00:00.000Z'); -INSERT INTO druid_segments (id,dataSource,created_date,start,end,partitioned,version,used,payload,used_status_last_updated) VALUES ('wikipedia_editstream_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9','wikipedia_editstream','2013-03-15T20:49:52.348Z','2012-12-29T00:00:00.000Z','2013-01-10T08:00:00.000Z',0,'2013-01-10T08:13:47.830Z_v9',1,'{\"dataSource\":\"wikipedia_editstream\",\"interval\":\"2012-12-29T00:00:00.000Z/2013-01-10T08:00:00.000Z\",\"version\":\"2013-01-10T08:13:47.830Z_v9\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/wikipedia_editstream/2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z/2013-01-10T08:13:47.830Z_v9/0/index.zip\"},\"dimensions\":\"anonymous,area_code,city,continent_code,country_name,dma_code,geo,language,namespace,network,newpage,page,postal_code,region_lookup,robot,unpatrolled,user\",\"metrics\":\"added,count,deleted,delta,delta_hist,unique_users,variation\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":446027801,\"identifier\":\"wikipedia_editstream_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9\"}','1970-01-01T00:00:00.000Z'); -INSERT INTO druid_segments (id, dataSource, created_date, start, end, partitioned, version, used, payload,used_status_last_updated) VALUES ('wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z', 'wikipedia', '2013-08-08T21:26:23.799Z', '2013-08-01T00:00:00.000Z', '2013-08-02T00:00:00.000Z', '0', '2013-08-08T21:22:48.989Z', '1', '{\"dataSource\":\"wikipedia\",\"interval\":\"2013-08-01T00:00:00.000Z/2013-08-02T00:00:00.000Z\",\"version\":\"2013-08-08T21:22:48.989Z\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/wikipedia/20130801T000000.000Z_20130802T000000.000Z/2013-08-08T21_22_48.989Z/0/index.zip\"},\"dimensions\":\"dma_code,continent_code,geo,area_code,robot,country_name,network,city,namespace,anonymous,unpatrolled,page,postal_code,language,newpage,user,region_lookup\",\"metrics\":\"count,delta,variation,added,deleted\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":24664730,\"identifier\":\"wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z\"}','1970-01-01T00:00:00.000Z'); diff --git a/integration-tests/script/docker_compose_args.sh b/integration-tests/script/docker_compose_args.sh index 87475239e623..fcf5bdafa7c5 100644 --- a/integration-tests/script/docker_compose_args.sh +++ b/integration-tests/script/docker_compose_args.sh @@ -49,14 +49,6 @@ getComposeArgs() then # cluster with overriden properties for broker and coordinator echo "-f ${DOCKERDIR}/docker-compose.centralized-datasource-schema.yml" - elif [ "$DRUID_INTEGRATION_TEST_GROUP" = "cds-task-schema-publish-disabled" ] - then - # cluster with overriden properties for broker and coordinator - echo "-f ${DOCKERDIR}/docker-compose.cds-task-schema-publish-disabled.yml" - elif [ "$DRUID_INTEGRATION_TEST_GROUP" = "cds-coordinator-metadata-query-disabled" ] - then - # cluster with overriden properties for broker and coordinator - echo "-f ${DOCKERDIR}/docker-compose.cds-coordinator-metadata-query-disabled.yml" else # default echo "-f ${DOCKERDIR}/docker-compose.yml" diff --git a/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java b/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java index efdd26de20a6..60600d633d2b 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java @@ -33,8 +33,6 @@ public class TestNGGroup public static final String TRANSACTIONAL_KAFKA_INDEX_SLOW = "kafka-transactional-index-slow"; - public static final String APPEND_INGESTION = "append-ingestion"; - /** * This group can only be run individually using -Dgroups=query since it requires specific test data setup. */ @@ -115,11 +113,5 @@ public class TestNGGroup */ public static final String KINESIS_DATA_FORMAT = "kinesis-data-format"; - public static final String CUSTOM_COORDINATOR_DUTIES = "custom-coordinator-duties"; - public static final String CENTRALIZED_DATASOURCE_SCHEMA = "centralized-datasource-schema"; - - public static final String CDS_TASK_SCHEMA_PUBLISH_DISABLED = "cds-task-schema-publish-disabled"; - - public static final String CDS_COORDINATOR_METADATA_QUERY_DISABLED = "cds-coordinator-metadata-query-disabled"; } diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITAppendBatchIndexTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITAppendBatchIndexTest.java deleted file mode 100644 index ced35949626c..000000000000 --- a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITAppendBatchIndexTest.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.tests.indexer; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; -import org.apache.druid.indexer.partitions.HashedPartitionsSpec; -import org.apache.druid.indexer.partitions.PartitionsSpec; -import org.apache.druid.indexer.partitions.SingleDimensionPartitionsSpec; -import org.apache.druid.java.util.common.Pair; -import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.testing.guice.DruidTestModuleFactory; -import org.apache.druid.tests.TestNGGroup; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Guice; -import org.testng.annotations.Test; - -import java.io.Closeable; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.function.Function; - -@Test(groups = {TestNGGroup.APPEND_INGESTION, TestNGGroup.CDS_TASK_SCHEMA_PUBLISH_DISABLED, TestNGGroup.CDS_COORDINATOR_METADATA_QUERY_DISABLED}) -@Guice(moduleFactory = DruidTestModuleFactory.class) -public class ITAppendBatchIndexTest extends AbstractITBatchIndexTest -{ - private static final Logger LOG = new Logger(ITAppendBatchIndexTest.class); - private static final String INDEX_TASK = "/indexer/wikipedia_local_input_source_index_task.json"; - // This query file is for the initial ingestion which is one complete dataset with roll up - private static final String INDEX_QUERIES_INITIAL_INGESTION_RESOURCE = "/indexer/wikipedia_index_queries.json"; - // This query file is for the initial ingestion plus the append ingestion which are two complete dataset with roll - // up within each dataset (roll up within the initial ingestion and roll up within the append ingestion but not - // roll up across both dataset). - private static final String INDEX_QUERIES_POST_APPEND_PRE_COMPACT_RESOURCE = "/indexer/wikipedia_double_ingestion_non_perfect_rollup_index_queries.json"; - // This query file is for the initial ingestion plus the append ingestion plus a compaction task after the two ingestions. - // This is two complete dataset with perfect roll up across both dataset. - private static final String INDEX_QUERIES_POST_APPEND_POST_COMPACT_RESOURCE = "/indexer/wikipedia_double_ingestion_perfect_rollup_index_queries.json"; - - private static final String COMPACTION_TASK = "/indexer/wikipedia_compaction_task.json"; - - @DataProvider - public static Object[][] resources() - { - return new Object[][]{ - // First index with dynamically-partitioned then append dynamically-partitioned - { - ImmutableList.of( - new DynamicPartitionsSpec(null, null), - new DynamicPartitionsSpec(null, null) - ), - ImmutableList.of(4, 8, 2) - }, - // First index with hash-partitioned then append dynamically-partitioned - { - ImmutableList.of( - new HashedPartitionsSpec(null, 3, ImmutableList.of("page", "user")), - new DynamicPartitionsSpec(null, null) - ), - ImmutableList.of(6, 10, 2) - }, - // First index with range-partitioned then append dynamically-partitioned - { - ImmutableList.of( - new SingleDimensionPartitionsSpec(1000, null, "page", false), - new DynamicPartitionsSpec(null, null) - ), - ImmutableList.of(2, 6, 2) - } - }; - } - - @Test(dataProvider = "resources") - public void doIndexTest(List partitionsSpecList, List expectedSegmentCountList) throws Exception - { - final String indexDatasource = "wikipedia_index_test_" + UUID.randomUUID(); - try ( - final Closeable ignored1 = unloader(indexDatasource + config.getExtraDatasourceNameSuffix()); - ) { - // Submit initial ingestion task - submitIngestionTaskAndVerify(indexDatasource, partitionsSpecList.get(0), false, new Pair<>(false, false)); - verifySegmentsCountAndLoaded(indexDatasource, expectedSegmentCountList.get(0)); - doTestQuery(indexDatasource, INDEX_QUERIES_INITIAL_INGESTION_RESOURCE); - // Submit append ingestion task - submitIngestionTaskAndVerify(indexDatasource, partitionsSpecList.get(1), true, new Pair<>(false, false)); - verifySegmentsCountAndLoaded(indexDatasource, expectedSegmentCountList.get(1)); - doTestQuery(indexDatasource, INDEX_QUERIES_POST_APPEND_PRE_COMPACT_RESOURCE); - // Submit compaction task - compactData(indexDatasource, COMPACTION_TASK); - // Verification post compaction - verifySegmentsCountAndLoaded(indexDatasource, expectedSegmentCountList.get(2)); - verifySegmentsCompacted(indexDatasource, expectedSegmentCountList.get(2)); - doTestQuery(indexDatasource, INDEX_QUERIES_POST_APPEND_POST_COMPACT_RESOURCE); - } - } - - private void submitIngestionTaskAndVerify( - String indexDatasource, - PartitionsSpec partitionsSpec, - boolean appendToExisting, - Pair segmentAvailabilityConfirmationPair - ) throws Exception - { - InputFormatDetails inputFormatDetails = InputFormatDetails.JSON; - Map inputFormatMap = new ImmutableMap.Builder().put("type", inputFormatDetails.getInputFormatType()) - .build(); - final Function sqlInputSourcePropsTransform = spec -> { - try { - spec = StringUtils.replace( - spec, - "%%PARTITIONS_SPEC%%", - jsonMapper.writeValueAsString(partitionsSpec) - ); - spec = StringUtils.replace( - spec, - "%%INPUT_SOURCE_FILTER%%", - "*" + inputFormatDetails.getFileExtension() - ); - spec = StringUtils.replace( - spec, - "%%INPUT_SOURCE_BASE_DIR%%", - "/resources/data/batch_index" + inputFormatDetails.getFolderSuffix() - ); - spec = StringUtils.replace( - spec, - "%%INPUT_FORMAT%%", - jsonMapper.writeValueAsString(inputFormatMap) - ); - spec = StringUtils.replace( - spec, - "%%APPEND_TO_EXISTING%%", - jsonMapper.writeValueAsString(appendToExisting) - ); - spec = StringUtils.replace( - spec, - "%%DROP_EXISTING%%", - jsonMapper.writeValueAsString(false) - ); - if (partitionsSpec instanceof DynamicPartitionsSpec) { - spec = StringUtils.replace( - spec, - "%%FORCE_GUARANTEED_ROLLUP%%", - jsonMapper.writeValueAsString(false) - ); - } else if (partitionsSpec instanceof HashedPartitionsSpec || partitionsSpec instanceof SingleDimensionPartitionsSpec) { - spec = StringUtils.replace( - spec, - "%%FORCE_GUARANTEED_ROLLUP%%", - jsonMapper.writeValueAsString(true) - ); - } - return spec; - } - catch (Exception e) { - throw new RuntimeException(e); - } - }; - - doIndexTest( - indexDatasource, - INDEX_TASK, - sqlInputSourcePropsTransform, - null, - false, - false, - true, - segmentAvailabilityConfirmationPair - ); - } -} diff --git a/integration-tests/src/test/java/org/apache/druid/tests/parallelized/ITCustomCoordinatorDuties.java b/integration-tests/src/test/java/org/apache/druid/tests/parallelized/ITCustomCoordinatorDuties.java deleted file mode 100644 index 61635e5c7582..000000000000 --- a/integration-tests/src/test/java/org/apache/druid/tests/parallelized/ITCustomCoordinatorDuties.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.tests.parallelized; - -import org.apache.druid.testing.guice.DruidTestModuleFactory; -import org.apache.druid.tests.TestNGGroup; -import org.apache.druid.tests.indexer.AbstractKafkaIndexingServiceTest; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Guice; -import org.testng.annotations.Test; - -@Test(groups = TestNGGroup.CUSTOM_COORDINATOR_DUTIES) -@Guice(moduleFactory = DruidTestModuleFactory.class) -public class ITCustomCoordinatorDuties extends AbstractKafkaIndexingServiceTest -{ - @Override - public String getTestNamePrefix() - { - return "kafka_non_transactional_parallelized"; - } - - @BeforeClass - public void beforeClass() throws Exception - { - doBeforeClass(); - } - - /** - * This test verify the kill supervisor custom coordinator duties which is enabled using the - * custom coordinator pluggable configurations. - */ - @Test - public void testKafkaTerminatedSupervisorAutoCleanup() throws Exception - { - doTestTerminatedSupervisorAutoCleanup(false); - } -} diff --git a/integration-tests/src/test/resources/indexer/wikipedia_compaction_task.json b/integration-tests/src/test/resources/indexer/wikipedia_compaction_task.json deleted file mode 100644 index fb620c11aa20..000000000000 --- a/integration-tests/src/test/resources/indexer/wikipedia_compaction_task.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "type" : "compact", - "dataSource" : "%%DATASOURCE%%", - "ioConfig" : { - "type": "compact", - "inputSpec": { - "type": "interval", - "interval": "2013-08-31/2013-09-02" - } - }, - "context" : { - "storeCompactionState" : true - } -} \ No newline at end of file diff --git a/integration-tests/src/test/resources/indexer/wikipedia_double_ingestion_non_perfect_rollup_index_queries.json b/integration-tests/src/test/resources/indexer/wikipedia_double_ingestion_non_perfect_rollup_index_queries.json deleted file mode 100644 index 586da63e3dbd..000000000000 --- a/integration-tests/src/test/resources/indexer/wikipedia_double_ingestion_non_perfect_rollup_index_queries.json +++ /dev/null @@ -1,143 +0,0 @@ -[ - { - "description": "timeseries, 1 agg, all", - "query":{ - "queryType" : "timeBoundary", - "dataSource": "%%DATASOURCE%%" - }, - "expectedResults":[ - { - "timestamp" : "2013-08-31T01:02:33.000Z", - "result" : { - "minTime" : "2013-08-31T01:02:33.000Z", - "maxTime" : "2013-09-01T12:41:27.000Z" - } - } - ] - }, - { - "description": "timeseries, datasketch aggs, all", - "query":{ - "queryType" : "timeseries", - "dataSource": "%%DATASOURCE%%", - "granularity":"day", - "intervals":[ - "2013-08-31T00:00/2013-09-01T00:00" - ], - "filter":null, - "aggregations":[ - { - "type": "HLLSketchMerge", - "name": "approxCountHLL", - "fieldName": "HLLSketchBuild", - "lgK": 12, - "tgtHllType": "HLL_4", - "round": true - }, - { - "type":"thetaSketch", - "name":"approxCountTheta", - "fieldName":"thetaSketch", - "size":16384, - "shouldFinalize":true, - "isInputThetaSketch":false, - "errorBoundsStdDev":null - }, - { - "type":"quantilesDoublesSketch", - "name":"quantilesSketch", - "fieldName":"quantilesDoublesSketch", - "k":128 - } - ] - }, - "expectedResults":[ - { - "timestamp" : "2013-08-31T00:00:00.000Z", - "result" : { - "quantilesSketch":10, - "approxCountTheta":5.0, - "approxCountHLL":5 - } - } - ] - }, - { - "description":"having spec on post aggregation", - "query":{ - "queryType":"groupBy", - "dataSource":"%%DATASOURCE%%", - "granularity":"day", - "dimensions":[ - "page" - ], - "filter":{ - "type":"selector", - "dimension":"language", - "value":"zh" - }, - "aggregations":[ - { - "type":"count", - "name":"rows" - }, - { - "type":"longSum", - "fieldName":"added", - "name":"added_count" - } - ], - "postAggregations": [ - { - "type":"arithmetic", - "name":"added_count_times_ten", - "fn":"*", - "fields":[ - {"type":"fieldAccess", "name":"added_count", "fieldName":"added_count"}, - {"type":"constant", "name":"const", "value":10} - ] - } - ], - "having":{"type":"greaterThan", "aggregation":"added_count_times_ten", "value":9000}, - "intervals":[ - "2013-08-31T00:00/2013-09-01T00:00" - ] - }, - "expectedResults":[ { - "version" : "v1", - "timestamp" : "2013-08-31T00:00:00.000Z", - "event" : { - "added_count_times_ten" : 18100.0, - "page" : "Crimson Typhoon", - "added_count" : 1810, - "rows" : 2 - } - } ] - }, - { - "description": "timeseries, count aggs, all", - "query":{ - "queryType" : "timeseries", - "dataSource": "%%DATASOURCE%%", - "granularity":"day", - "intervals":[ - "2013-08-31T00:00/2013-09-01T00:00" - ], - "filter":null, - "aggregations":[ - { - "type": "count", - "name": "rows" - } - ] - }, - "expectedResults":[ - { - "timestamp" : "2013-08-31T00:00:00.000Z", - "result" : { - "rows":10 - } - } - ] - } -] \ No newline at end of file diff --git a/integration-tests/src/test/resources/indexer/wikipedia_double_ingestion_perfect_rollup_index_queries.json b/integration-tests/src/test/resources/indexer/wikipedia_double_ingestion_perfect_rollup_index_queries.json deleted file mode 100644 index eaa9592ca266..000000000000 --- a/integration-tests/src/test/resources/indexer/wikipedia_double_ingestion_perfect_rollup_index_queries.json +++ /dev/null @@ -1,143 +0,0 @@ -[ - { - "description": "timeseries, 1 agg, all", - "query":{ - "queryType" : "timeBoundary", - "dataSource": "%%DATASOURCE%%" - }, - "expectedResults":[ - { - "timestamp" : "2013-08-31T01:02:33.000Z", - "result" : { - "minTime" : "2013-08-31T01:02:33.000Z", - "maxTime" : "2013-09-01T12:41:27.000Z" - } - } - ] - }, - { - "description": "timeseries, datasketch aggs, all", - "query":{ - "queryType" : "timeseries", - "dataSource": "%%DATASOURCE%%", - "granularity":"day", - "intervals":[ - "2013-08-31T00:00/2013-09-01T00:00" - ], - "filter":null, - "aggregations":[ - { - "type": "HLLSketchMerge", - "name": "approxCountHLL", - "fieldName": "HLLSketchBuild", - "lgK": 12, - "tgtHllType": "HLL_4", - "round": true - }, - { - "type":"thetaSketch", - "name":"approxCountTheta", - "fieldName":"thetaSketch", - "size":16384, - "shouldFinalize":true, - "isInputThetaSketch":false, - "errorBoundsStdDev":null - }, - { - "type":"quantilesDoublesSketch", - "name":"quantilesSketch", - "fieldName":"quantilesDoublesSketch", - "k":128 - } - ] - }, - "expectedResults":[ - { - "timestamp" : "2013-08-31T00:00:00.000Z", - "result" : { - "quantilesSketch":10, - "approxCountTheta":5.0, - "approxCountHLL":5 - } - } - ] - }, - { - "description":"having spec on post aggregation", - "query":{ - "queryType":"groupBy", - "dataSource":"%%DATASOURCE%%", - "granularity":"day", - "dimensions":[ - "page" - ], - "filter":{ - "type":"selector", - "dimension":"language", - "value":"zh" - }, - "aggregations":[ - { - "type":"count", - "name":"rows" - }, - { - "type":"longSum", - "fieldName":"added", - "name":"added_count" - } - ], - "postAggregations": [ - { - "type":"arithmetic", - "name":"added_count_times_ten", - "fn":"*", - "fields":[ - {"type":"fieldAccess", "name":"added_count", "fieldName":"added_count"}, - {"type":"constant", "name":"const", "value":10} - ] - } - ], - "having":{"type":"greaterThan", "aggregation":"added_count_times_ten", "value":9000}, - "intervals":[ - "2013-08-31T00:00/2013-09-01T00:00" - ] - }, - "expectedResults":[ { - "version" : "v1", - "timestamp" : "2013-08-31T00:00:00.000Z", - "event" : { - "added_count_times_ten" : 18100.0, - "page" : "Crimson Typhoon", - "added_count" : 1810, - "rows" : 1 - } - } ] - }, - { - "description": "timeseries, count aggs, all", - "query":{ - "queryType" : "timeseries", - "dataSource": "%%DATASOURCE%%", - "granularity":"day", - "intervals":[ - "2013-08-31T00:00/2013-09-01T00:00" - ], - "filter":null, - "aggregations":[ - { - "type": "count", - "name": "rows" - } - ] - }, - "expectedResults":[ - { - "timestamp" : "2013-08-31T00:00:00.000Z", - "result" : { - "rows":5 - } - } - ] - } -] \ No newline at end of file diff --git a/integration-tests/src/test/resources/indexer/wikipedia_local_input_source_index_task.json b/integration-tests/src/test/resources/indexer/wikipedia_local_input_source_index_task.json deleted file mode 100644 index ee0fd73021a8..000000000000 --- a/integration-tests/src/test/resources/indexer/wikipedia_local_input_source_index_task.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "type": "index_parallel", - "spec": { - "dataSchema": { - "dataSource": "%%DATASOURCE%%", - "timestampSpec": { - "column": "timestamp" - }, - "dimensionsSpec": { - "dimensions": [ - "page", - {"type": "string", "name": "language", "createBitmapIndex": false}, - "user", - "unpatrolled", - "newPage", - "robot", - "anonymous", - "namespace", - "continent", - "country", - "region", - "city" - ] - }, - "metricsSpec": [ - { - "type": "count", - "name": "count" - }, - { - "type": "doubleSum", - "name": "added", - "fieldName": "added" - }, - { - "type": "doubleSum", - "name": "deleted", - "fieldName": "deleted" - }, - { - "type": "doubleSum", - "name": "delta", - "fieldName": "delta" - }, - { - "name": "thetaSketch", - "type": "thetaSketch", - "fieldName": "user" - }, - { - "name": "quantilesDoublesSketch", - "type": "quantilesDoublesSketch", - "fieldName": "delta" - }, - { - "name": "HLLSketchBuild", - "type": "HLLSketchBuild", - "fieldName": "user" - } - ], - "granularitySpec": { - "segmentGranularity": "DAY", - "queryGranularity": "second", - "intervals" : [ "2013-08-31/2013-09-02" ] - } - }, - "ioConfig": { - "type": "index_parallel", - "inputSource": { - "type": "local", - "filter" : "%%INPUT_SOURCE_FILTER%%", - "baseDir": "%%INPUT_SOURCE_BASE_DIR%%" - }, - "appendToExisting": %%APPEND_TO_EXISTING%%, - "dropExisting": %%DROP_EXISTING%%, - "inputFormat": %%INPUT_FORMAT%% - }, - "tuningConfig": { - "type": "index_parallel", - "maxNumConcurrentSubTasks": 4, - "splitHintSpec": { - "type": "maxSize", - "maxNumFiles": 1 - }, - "forceGuaranteedRollup": %%FORCE_GUARANTEED_ROLLUP%%, - "partitionsSpec": %%PARTITIONS_SPEC%% - } - } -} \ No newline at end of file