diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CloneHistoricals.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CloneHistoricals.java index da201127f60a..c12193c9b0e5 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CloneHistoricals.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CloneHistoricals.java @@ -68,7 +68,10 @@ public DruidCoordinatorRuntimeParams run(DruidCoordinatorRuntimeParams params) final DruidCluster cluster = params.getDruidCluster(); if (cloneServers.isEmpty()) { - // No servers to be cloned. + if (!cloneStatusManager.getStatusForAllServers().isEmpty()) { + // Clear the status manager if the dynamic config no longer has mappings to avoid showing stale clone statuses. + cloneStatusManager.updateStatus(Map.of()); + } return params; } diff --git a/web-console/src/components/header-bar/header-bar.tsx b/web-console/src/components/header-bar/header-bar.tsx index 63bc0626f7c5..9232dd3e0b70 100644 --- a/web-console/src/components/header-bar/header-bar.tsx +++ b/web-console/src/components/header-bar/header-bar.tsx @@ -405,6 +405,7 @@ export const HeaderBar = React.memo(function HeaderBar(props: HeaderBarProps) { {doctorDialogOpen && setDoctorDialogOpen(false)} />} {coordinatorDynamicConfigDialogOpen && ( setCoordinatorDynamicConfigDialogOpen(false)} /> )} diff --git a/web-console/src/dialogs/coordinator-dynamic-config-dialog/__snapshots__/clone-server-mapping-dialog.spec.tsx.snap b/web-console/src/dialogs/coordinator-dynamic-config-dialog/__snapshots__/clone-server-mapping-dialog.spec.tsx.snap new file mode 100644 index 000000000000..09dac32e148b --- /dev/null +++ b/web-console/src/dialogs/coordinator-dynamic-config-dialog/__snapshots__/clone-server-mapping-dialog.spec.tsx.snap @@ -0,0 +1,526 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`CloneServerMappingDialog matches snapshot when servers are loading 1`] = ` + + + +`; + +exports[`CloneServerMappingDialog matches snapshot with existing mappings 1`] = ` + + +
+ + Each target server clones all segments from its source server. The target does not participate in regular segment assignment or balancing. + + + + + + + + + + + + + + + + +
+ Target + + + Source + +
+ + + clones + + + + +
+ +
+
+
+ + +
+
+ + +`; + +exports[`CloneServerMappingDialog matches snapshot with no existing mappings 1`] = ` + + +
+ + Each target server clones all segments from its source server. The target does not participate in regular segment assignment or balancing. + + +
+
+
+ + +
+
+ + +`; + +exports[`CloneServerMappingDialog renders self-clone mapping (filtered on save by component logic) 1`] = ` + + +
+ + Each target server clones all segments from its source server. The target does not participate in regular segment assignment or balancing. + + + + + + + + + + + + + + + + + + + + + + +
+ Target + + + Source + +
+ + + clones + + + + +
+ + + clones + + + + +
+ +
+
+
+ + +
+
+ + +`; diff --git a/web-console/src/dialogs/coordinator-dynamic-config-dialog/__snapshots__/server-multi-select-dialog.spec.tsx.snap b/web-console/src/dialogs/coordinator-dynamic-config-dialog/__snapshots__/server-multi-select-dialog.spec.tsx.snap new file mode 100644 index 000000000000..c6c657d25a02 --- /dev/null +++ b/web-console/src/dialogs/coordinator-dynamic-config-dialog/__snapshots__/server-multi-select-dialog.spec.tsx.snap @@ -0,0 +1,289 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`ServerMultiSelectDialog matches snapshot when servers are loading 1`] = ` + + + +`; + +exports[`ServerMultiSelectDialog matches snapshot with existing selection 1`] = ` + + +
+ +
+
+ + + hot + + + } + onChange={[Function]} + /> +
+ + +
+
+
+ + + cold + + + } + onChange={[Function]} + /> +
+ +
+
+
+
+
+
+ + 1 + server + + selected + + + +
+
+ + +`; + +exports[`ServerMultiSelectDialog matches snapshot with no selection 1`] = ` + + +
+ +
+
+ + + hot + + + } + onChange={[Function]} + /> +
+ + +
+
+
+ + + cold + + + } + onChange={[Function]} + /> +
+ +
+
+
+
+
+
+ + 0 + server + s + selected + + + +
+
+ + +`; diff --git a/web-console/src/dialogs/coordinator-dynamic-config-dialog/clone-server-mapping-dialog.spec.tsx b/web-console/src/dialogs/coordinator-dynamic-config-dialog/clone-server-mapping-dialog.spec.tsx new file mode 100644 index 000000000000..44246febdc93 --- /dev/null +++ b/web-console/src/dialogs/coordinator-dynamic-config-dialog/clone-server-mapping-dialog.spec.tsx @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { shallow } from '../../utils/shallow-renderer'; + +import { CloneServerMappingDialog } from './clone-server-mapping-dialog'; +import type { TieredServers } from './tiered-servers'; + +const MOCK_SERVERS: TieredServers = { + tiers: ['_default_tier'], + serversByTier: { + _default_tier: ['host1:8083', 'host2:8083', 'host3:8083'], + }, + serverToTier: { + 'host1:8083': '_default_tier', + 'host2:8083': '_default_tier', + 'host3:8083': '_default_tier', + }, + allServers: ['host1:8083', 'host2:8083', 'host3:8083'], +}; + +describe('CloneServerMappingDialog', () => { + it('matches snapshot with no existing mappings', () => { + const dialog = shallow( + {}} + onClose={() => {}} + />, + ); + expect(dialog).toMatchSnapshot(); + }); + + it('matches snapshot with existing mappings', () => { + const dialog = shallow( + {}} + onClose={() => {}} + />, + ); + expect(dialog).toMatchSnapshot(); + }); + + it('matches snapshot when servers are loading', () => { + const dialog = shallow( + {}} + onClose={() => {}} + />, + ); + expect(dialog).toMatchSnapshot(); + }); + + it('renders self-clone mapping (filtered on save by component logic)', () => { + // Verify the dialog renders with a self-clone mapping present. + // The component's handleSave filters out target === source mappings. + const dialog = shallow( + {}} + onClose={() => {}} + />, + ); + expect(dialog).toMatchSnapshot(); + }); +}); diff --git a/web-console/src/dialogs/coordinator-dynamic-config-dialog/clone-server-mapping-dialog.tsx b/web-console/src/dialogs/coordinator-dynamic-config-dialog/clone-server-mapping-dialog.tsx new file mode 100644 index 000000000000..fe77a4eae746 --- /dev/null +++ b/web-console/src/dialogs/coordinator-dynamic-config-dialog/clone-server-mapping-dialog.tsx @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Button, + Callout, + Classes, + Dialog, + Intent, + Menu, + MenuDivider, + MenuItem, +} from '@blueprintjs/core'; +import { IconNames } from '@blueprintjs/icons'; +import { Select } from '@blueprintjs/select'; +import React, { useRef, useState } from 'react'; + +import { Loader } from '../../components'; + +import type { TieredServers } from './tiered-servers'; + +interface CloneMapping { + id: number; + target: string; + source: string; +} + +export interface CloneServerMappingDialogProps { + servers: TieredServers | undefined; + cloneServers: Record; + onSave(mapping: Record): void; + onClose(): void; +} + +export const CloneServerMappingDialog = React.memo(function CloneServerMappingDialog( + props: CloneServerMappingDialogProps, +) { + const { servers, cloneServers: initialMapping, onSave, onClose } = props; + const nextMappingId = useRef(0); + const [mappings, setMappings] = useState(() => + Object.entries(initialMapping || {}).map(([target, source]) => ({ + id: nextMappingId.current++, + target, + source, + })), + ); + + function updateMapping(index: number, field: 'target' | 'source', value: string) { + setMappings(prev => prev.map((m, i) => (i === index ? { ...m, [field]: value } : m))); + } + + function removeMapping(index: number) { + setMappings(prev => prev.filter((_, i) => i !== index)); + } + + function addMapping() { + setMappings(prev => [...prev, { id: nextMappingId.current++, target: '', source: '' }]); + } + + function handleSave() { + const result: Record = {}; + for (const m of mappings) { + if (m.target && m.source && m.target !== m.source) { + result[m.target] = m.source; + } + } + onSave(result); + onClose(); + } + + const usedTargets = new Set(mappings.map(m => m.target).filter(Boolean)); + const usedSources = new Set(mappings.map(m => m.source).filter(Boolean)); + const disabledServers = new Set([...usedTargets, ...usedSources]); + const hasInvalidMapping = mappings.some( + m => + !m.target || + !m.source || + m.target === m.source || + usedSources.has(m.target) || + usedTargets.has(m.source), + ); + + return ( + + {servers ? ( + <> +
+ + Each target server clones all segments from its source server. The target does not + participate in regular segment assignment or balancing. + + {mappings.length > 0 && ( + + + + + + + + + {mappings.map((mapping, i) => ( + + + + + + + ))} + +
Target + Source +
+ updateMapping(i, 'target', v)} + /> + clones + updateMapping(i, 'source', v)} + /> + +
+ )} +
+
+
+
+
+ + ) : ( + + )} + + ); +}); + +// --- Internal server select component --- + +interface ServerSelectProps { + servers: TieredServers; + value: string; + disabledServers?: Set; + onChange(value: string): void; +} + +function ServerSelect(props: ServerSelectProps) { + const { servers, value, disabledServers, onChange } = props; + + // Build a flat list of items with tier headers handled in the renderer + const allItems = servers.allServers; + + return ( + + items={allItems} + itemPredicate={(query, item) => item.toLowerCase().includes(query.toLowerCase())} + itemListRenderer={({ filteredItems, renderItem }) => { + const elements: React.ReactNode[] = []; + let lastTier: string | undefined; + for (let i = 0; i < filteredItems.length; i++) { + const item = filteredItems[i]; + const tier = servers.serverToTier[item] || ''; + if (tier && tier !== lastTier) { + elements.push(); + lastTier = tier; + } + elements.push(renderItem(item, i)); + } + return {elements}; + }} + itemRenderer={(item, { handleClick, handleFocus, modifiers }) => { + if (!modifiers.matchesPredicate) return null; + const disabled = disabledServers ? disabledServers.has(item) && item !== value : false; + return ( + + ); + }} + onItemSelect={onChange} + popoverProps={{ minimal: true }} + filterable + > + + + )} + setSearchText(e.target.value)} + style={{ marginBottom: 10 }} + /> +
+ {filteredTiers.map(({ tier, tierServers, filtered: filteredServers }) => { + if (filteredServers.length === 0) return null; + + const hiddenCount = tierServers.length - filteredServers.length; + const allSelected = filteredServers.every(s => selected.has(s)); + const someSelected = !allSelected && filteredServers.some(s => selected.has(s)); + + return ( +
+ + {tier} + {hiddenCount > 0 && ( + + {' '} + ({hiddenCount} hidden by filter) + + )} + + } + onChange={() => toggleTier(tier, filteredServers)} + /> +
+ {filteredServers.map(server => ( + toggleServer(server)} + /> + ))} +
+
+ ); + })} + {servers.allServers.length === 0 && ( + No historical servers found in the cluster. + )} +
+ +
+
+ + {selected.size} server{selected.size !== 1 ? 's' : ''} selected + +
+
+ + ) : ( + + )} + + ); +}); diff --git a/web-console/src/dialogs/coordinator-dynamic-config-dialog/tiered-servers.spec.ts b/web-console/src/dialogs/coordinator-dynamic-config-dialog/tiered-servers.spec.ts new file mode 100644 index 000000000000..2735dcc66d54 --- /dev/null +++ b/web-console/src/dialogs/coordinator-dynamic-config-dialog/tiered-servers.spec.ts @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { buildTieredServers } from './tiered-servers'; + +describe('buildTieredServers', () => { + it('returns empty structure for empty input', () => { + const result = buildTieredServers([]); + expect(result).toEqual({ + tiers: [], + serversByTier: {}, + serverToTier: {}, + allServers: [], + }); + }); + + it('sorts tiers alphabetically', () => { + const result = buildTieredServers([ + { server: 'host1:8083', tier: '_default_tier' }, + { server: 'host2:8083', tier: 'hot' }, + { server: 'host3:8083', tier: 'cold' }, + ]); + expect(result.tiers).toEqual(['_default_tier', 'cold', 'hot']); + }); + + it('sorts servers within each tier', () => { + const result = buildTieredServers([ + { server: 'host-c:8083', tier: 'hot' }, + { server: 'host-a:8083', tier: 'hot' }, + { server: 'host-b:8083', tier: 'hot' }, + ]); + expect(result.serversByTier['hot']).toEqual(['host-a:8083', 'host-b:8083', 'host-c:8083']); + }); + + it('builds serverToTier map correctly', () => { + const result = buildTieredServers([ + { server: 'host1:8083', tier: 'hot' }, + { server: 'host2:8083', tier: 'cold' }, + ]); + expect(result.serverToTier).toEqual({ + 'host1:8083': 'hot', + 'host2:8083': 'cold', + }); + }); + + it('builds allServers in tier-sorted order', () => { + const result = buildTieredServers([ + { server: 'cold-host:8083', tier: 'cold' }, + { server: 'hot-host2:8083', tier: 'hot' }, + { server: 'hot-host1:8083', tier: 'hot' }, + ]); + expect(result.allServers).toEqual(['cold-host:8083', 'hot-host1:8083', 'hot-host2:8083']); + }); +}); diff --git a/web-console/src/dialogs/coordinator-dynamic-config-dialog/tiered-servers.ts b/web-console/src/dialogs/coordinator-dynamic-config-dialog/tiered-servers.ts new file mode 100644 index 000000000000..b8a5cab9a4f6 --- /dev/null +++ b/web-console/src/dialogs/coordinator-dynamic-config-dialog/tiered-servers.ts @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface TieredServers { + tiers: string[]; + serversByTier: Record; + serverToTier: Record; + allServers: string[]; +} + +export function buildTieredServers(rows: { server: string; tier: string }[]): TieredServers { + const serversByTier: Record = {}; + const serverToTier: Record = {}; + for (const row of rows) { + if (!serversByTier[row.tier]) { + serversByTier[row.tier] = []; + } + serversByTier[row.tier].push(row.server); + serverToTier[row.server] = row.tier; + } + const tiers = Object.keys(serversByTier).sort(); + for (const tier of tiers) { + serversByTier[tier].sort(); + } + const allServers = tiers.flatMap(t => serversByTier[t]); + return { tiers, serversByTier, serverToTier, allServers }; +} diff --git a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.spec.ts b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.spec.ts new file mode 100644 index 000000000000..fa4f5ac238e3 --- /dev/null +++ b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.spec.ts @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { cloneCountSummary, serverCountSummary } from './coordinator-dynamic-config'; + +describe('serverCountSummary', () => { + it('returns None for undefined', () => { + expect(serverCountSummary(undefined)).toBe('None'); + }); + + it('returns None for empty array', () => { + expect(serverCountSummary([])).toBe('None'); + }); + + it('returns None for non-array', () => { + expect(serverCountSummary('not an array')).toBe('None'); + }); + + it('returns singular for one server', () => { + expect(serverCountSummary(['server1'])).toBe('1 server'); + }); + + it('returns plural for multiple servers', () => { + expect(serverCountSummary(['server1', 'server2', 'server3'])).toBe('3 servers'); + }); +}); + +describe('cloneCountSummary', () => { + it('returns None for undefined', () => { + expect(cloneCountSummary(undefined)).toBe('None'); + }); + + it('returns None for non-object', () => { + expect(cloneCountSummary('not an object')).toBe('None'); + }); + + it('returns None for empty object', () => { + expect(cloneCountSummary({})).toBe('None'); + }); + + it('returns singular for one mapping', () => { + expect(cloneCountSummary({ target1: 'source1' })).toBe('1 mapping'); + }); + + it('returns plural for multiple mappings', () => { + expect(cloneCountSummary({ target1: 'source1', target2: 'source2' })).toBe('2 mappings'); + }); +}); diff --git a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx index 3794ecb60376..976553fb571c 100644 --- a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx +++ b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx @@ -39,11 +39,24 @@ export interface CoordinatorDynamicConfig { useRoundRobinSegmentAssignment?: boolean; smartSegmentLoading?: boolean; turboLoadingNodes?: string[]; + cloneServers?: Record; // Undocumented debugDimensions?: any; } +export function serverCountSummary(v: any): string { + if (!v || !Array.isArray(v) || v.length === 0) return 'None'; + return `${v.length} server${v.length !== 1 ? 's' : ''}`; +} + +export function cloneCountSummary(v: any): string { + if (!v || typeof v !== 'object') return 'None'; + const count = Object.keys(v).length; + if (count === 0) return 'None'; + return `${count} mapping${count !== 1 ? 's' : ''}`; +} + export const COORDINATOR_DYNAMIC_CONFIG_FIELDS: Field[] = [ { name: 'pauseCoordination', @@ -146,7 +159,7 @@ export const COORDINATOR_DYNAMIC_CONFIG_FIELDS: Field[ { name: 'decommissioningNodes', - type: 'string-array', + type: 'custom', emptyValue: [], info: ( <> @@ -156,6 +169,50 @@ export const COORDINATOR_DYNAMIC_CONFIG_FIELDS: Field[ maxSegmentsToMove. ), + customSummary: serverCountSummary, + }, + { + name: 'turboLoadingNodes', + type: 'custom', + emptyValue: [], + experimental: true, + info: ( + <> +

+ List of Historical servers to place in turbo loading mode. These servers use a larger + thread-pool to load segments faster but at the cost of query performance. For servers + specified in turboLoadingNodes,{' '} + druid.coordinator.loadqueuepeon.http.batchSize is ignored and the coordinator + uses the value of the respective numLoadingThreads instead. +

+

+ Please use this config with caution. All servers should eventually be removed from this + list once the segment loading on the respective historicals is finished. +

+ + ), + customSummary: serverCountSummary, + }, + { + name: 'cloneServers', + type: 'custom', + emptyValue: {}, + experimental: true, + info: ( + <> +

+ Map from target Historical server to source Historical server. The target clones all + segments from the source, becoming an exact copy. The target does not participate in + regular segment assignment or balancing, and its segments do not count towards replica + counts. +

+

+ If the source server disappears, the target remains in the last known state of the source + until removed from this mapping. +

+ + ), + customSummary: cloneCountSummary, }, { name: 'killDataSourceWhitelist', @@ -190,7 +247,7 @@ export const COORDINATOR_DYNAMIC_CONFIG_FIELDS: Field[ <> Ratio of total available task slots, including autoscaling if applicable that will be allowed for kill tasks. This limit only applies for kill tasks that are spawned - automatically by the Coordinator's auto kill duty, which is enabled when + automatically by the Coordinator's auto kill duty, which is enabled when{' '} druid.coordinator.kill.on is true. ), @@ -245,24 +302,4 @@ export const COORDINATOR_DYNAMIC_CONFIG_FIELDS: Field[ ), }, - { - name: 'turboLoadingNodes', - type: 'string-array', - experimental: true, - info: ( - <> -

- List of Historical servers to place in turbo loading mode. These servers use a larger - thread-pool to load segments faster but at the cost of query performance. For servers - specified in turboLoadingNodes,{' '} - druid.coordinator.loadqueuepeon.http.batchSize is ignored and the coordinator - uses the value of the respective numLoadingThreads instead. -

-

- Please use this config with caution. All servers should eventually be removed from this - list once the segment loading on the respective historicals is finished. -

- - ), - }, ]; diff --git a/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap b/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap index afd053ddc75b..3f265d48186e 100644 --- a/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap +++ b/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap @@ -74,340 +74,344 @@ exports[`ServicesView renders data 1`] = ` - - Build -
- revision - , - "accessor": "build_revision", - "show": true, - "width": 200, - }, - { - "Aggregated": [Function], - "Cell": [Function], - "Header": - Available -
- processors - , - "accessor": "available_processors", - "className": "padded", - "filterable": false, - "show": true, - "width": 100, - }, - { - "Aggregated": [Function], - "Cell": [Function], - "Header": "Total memory", - "accessor": "total_memory", - "className": "padded", - "filterable": false, - "show": true, - "width": 120, - }, - { - "Aggregated": [Function], - "Cell": [Function], - "Header": "Labels", - "accessor": "labels", - "className": "padded", - "filterable": false, - "show": true, - "width": 200, - }, - { - "Aggregated": [Function], - "Cell": [Function], - "Header": "Detail", - "accessor": "service", - "className": "padded", - "filterable": false, - "id": "queue", - "show": true, - "width": 400, - }, - { - "Aggregated": [Function], - "Cell": [Function], - "Header": "Actions", - "accessor": "service", - "filterable": false, - "id": "actions", - "show": true, - "sortable": false, - "width": 70, - }, - ] - } - data={ - [ - [ + + + + Build +
+ revision + , + "accessor": "build_revision", + "show": true, + "width": 200, + }, + { + "Aggregated": [Function], + "Cell": [Function], + "Header": + Available +
+ processors + , + "accessor": "available_processors", + "className": "padded", + "filterable": false, + "show": true, + "width": 100, + }, + { + "Aggregated": [Function], + "Cell": [Function], + "Header": "Total memory", + "accessor": "total_memory", + "className": "padded", + "filterable": false, + "show": true, + "width": 120, + }, + { + "Aggregated": [Function], + "Cell": [Function], + "Header": "Labels", + "accessor": "labels", + "className": "padded", + "filterable": false, + "show": true, + "width": 200, + }, + { + "Aggregated": [Function], + "Cell": [Function], + "Header": "Detail", + "accessor": "service", + "className": "padded wrapped", + "filterable": false, + "id": "queue", + "show": true, + "width": 400, + }, + { + "Aggregated": [Function], + "Cell": [Function], + "Header": "Actions", + "accessor": "service", + "filterable": false, + "id": "actions", + "show": true, + "sortable": false, + "width": 70, + }, + ] + } + data={ + [ + [ + { + "curr_size": 0, + "host": "localhost", + "is_leader": 0, + "max_size": 0, + "plaintext_port": 8082, + "service": "localhost:8082", + "service_type": "broker", + "start_time": 0, + "tier": null, + "tls_port": -1, + }, + { + "curr_size": 179744287, + "host": "localhost", + "is_leader": 0, + "max_size": 3000000000n, + "plaintext_port": 8083, + "segmentsToDrop": 0, + "segmentsToDropSize": 0, + "segmentsToLoad": 0, + "segmentsToLoadSize": 0, + "service": "localhost:8083", + "service_type": "historical", + "start_time": 0, + "tier": "_default_tier", + "tls_port": -1, + }, + ], + ] + } + defaultExpanded={{}} + defaultFilterMethod={[Function]} + defaultFiltered={[]} + defaultPage={0} + defaultPageSize={50} + defaultResized={[]} + defaultSortDesc={false} + defaultSortMethod={[Function]} + defaultSorted={[]} + expanderDefaults={ { - "curr_size": 179744287, - "host": "localhost", - "is_leader": 0, - "max_size": 3000000000n, - "plaintext_port": 8083, - "segmentsToDrop": 0, - "segmentsToDropSize": 0, - "segmentsToLoad": 0, - "segmentsToLoadSize": 0, - "service": "localhost:8083", - "service_type": "historical", - "start_time": 0, - "tier": "_default_tier", - "tls_port": -1, - }, - ], - ] - } - defaultExpanded={{}} - defaultFilterMethod={[Function]} - defaultFiltered={[]} - defaultPage={0} - defaultPageSize={50} - defaultResized={[]} - defaultSortDesc={false} - defaultSortMethod={[Function]} - defaultSorted={[]} - expanderDefaults={ - { - "filterable": false, - "resizable": false, - "sortable": false, - "width": 35, - } - } - filterable={true} - filtered={[]} - freezeWhenExpanded={false} - getLoadingProps={[Function]} - getNoDataProps={[Function]} - getPaginationProps={[Function]} - getProps={[Function]} - getResizerProps={[Function]} - getTableProps={[Function]} - getTbodyProps={[Function]} - getTdProps={[Function]} - getTfootProps={[Function]} - getTfootTdProps={[Function]} - getTfootTrProps={[Function]} - getTheadFilterProps={[Function]} - getTheadFilterThProps={[Function]} - getTheadFilterTrProps={[Function]} - getTheadGroupProps={[Function]} - getTheadGroupThProps={[Function]} - getTheadGroupTrProps={[Function]} - getTheadProps={[Function]} - getTheadThProps={[Function]} - getTheadTrProps={[Function]} - getTrGroupProps={[Function]} - getTrProps={[Function]} - groupedByPivotKey="_groupedByPivot" - indexKey="_index" - loading={false} - loadingText="Loading..." - multiSort={true} - nestingLevelKey="_nestingLevel" - nextText="Next" - noDataText="" - ofText="of" - onFetchData={[Function]} - onFilteredChange={[Function]} - originalKey="_original" - pageJumpText="jump to page" - pageSizeOptions={ - [ - 50, - 100, - 200, - ] - } - pageText="Page" - pivotBy={[]} - pivotDefaults={{}} - pivotIDKey="_pivotID" - pivotValKey="_pivotVal" - previousText="Previous" - resizable={true} - resolveData={[Function]} - rowsSelectorText="rows per page" - rowsText="rows" - showPageJump={true} - showPageSizeOptions={true} - showPagination={false} - showPaginationBottom={true} - showPaginationTop={false} - sortable={true} - style={{}} - subRowsKey="_subRows" - /> + "filterable": false, + "resizable": false, + "sortable": false, + "width": 35, + } + } + filterable={true} + filtered={[]} + freezeWhenExpanded={false} + getLoadingProps={[Function]} + getNoDataProps={[Function]} + getPaginationProps={[Function]} + getProps={[Function]} + getResizerProps={[Function]} + getTableProps={[Function]} + getTbodyProps={[Function]} + getTdProps={[Function]} + getTfootProps={[Function]} + getTfootTdProps={[Function]} + getTfootTrProps={[Function]} + getTheadFilterProps={[Function]} + getTheadFilterThProps={[Function]} + getTheadFilterTrProps={[Function]} + getTheadGroupProps={[Function]} + getTheadGroupThProps={[Function]} + getTheadGroupTrProps={[Function]} + getTheadProps={[Function]} + getTheadThProps={[Function]} + getTheadTrProps={[Function]} + getTrGroupProps={[Function]} + getTrProps={[Function]} + groupedByPivotKey="_groupedByPivot" + indexKey="_index" + loading={false} + loadingText="Loading..." + multiSort={true} + nestingLevelKey="_nestingLevel" + nextText="Next" + noDataText="" + ofText="of" + onFetchData={[Function]} + onFilteredChange={[Function]} + originalKey="_original" + pageJumpText="jump to page" + pageSizeOptions={ + [ + 50, + 100, + 200, + ] + } + pageText="Page" + pivotBy={[]} + pivotDefaults={{}} + pivotIDKey="_pivotID" + pivotValKey="_pivotVal" + previousText="Previous" + resizable={true} + resolveData={[Function]} + rowsSelectorText="rows per page" + rowsText="rows" + showPageJump={true} + showPageSizeOptions={true} + showPagination={false} + showPaginationBottom={true} + showPaginationTop={false} + sortable={true} + style={{}} + subRowsKey="_subRows" + /> + + `; diff --git a/web-console/src/views/services-view/services-view.tsx b/web-console/src/views/services-view/services-view.tsx index 705a8b6f8f60..d23ac2922602 100644 --- a/web-console/src/views/services-view/services-view.tsx +++ b/web-console/src/views/services-view/services-view.tsx @@ -59,6 +59,7 @@ import { formatDurationWithMsIfNeeded, formatInteger, getApiArray, + getApiArrayFromKey, hasOverlayOpen, LocalStorageBackedVisibility, LocalStorageKeys, @@ -167,13 +168,35 @@ interface ServiceResultRow { readonly total_memory: number; } +interface CloneStatusInfo { + readonly sourceServer: string; + readonly state: string; + readonly segmentLoadsRemaining: number; + readonly segmentDropsRemaining: number; + readonly bytesToLoad: number; +} + +interface ServerModeInfo { + readonly turboLoadingNodes: Set; + readonly decommissioningNodes: Set; +} + interface ServicesWithAuxiliaryInfo { readonly services: ServiceResultRow[]; readonly loadQueueInfo: Record; + readonly cloneStatus: Record; + readonly serverMode: ServerModeInfo; readonly workerInfo: Record; } export const LoadQueueInfoContext = createContext>({}); +export const CloneStatusContext = createContext>({}); + +const DEFAULT_SERVER_MODE: ServerModeInfo = { + turboLoadingNodes: new Set(), + decommissioningNodes: new Set(), +}; +export const ServerModeContext = createContext(DEFAULT_SERVER_MODE); interface LoadQueueInfo { readonly segmentsToDrop: NumberLike; @@ -220,6 +243,100 @@ function aggregateLoadQueueInfos(loadQueueInfos: LoadQueueInfo[]): LoadQueueInfo }; } +interface DetailCellProps { + original: ServiceResultRow; + workerInfoLookup: Record; +} + +function DetailCell({ original, workerInfoLookup }: DetailCellProps) { + const { service_type, service, is_leader } = original; + const loadQueueInfoContext = useContext(LoadQueueInfoContext); + const cloneStatusContext = useContext(CloneStatusContext); + const serverModeInfo = useContext(ServerModeContext); + + switch (service_type) { + case 'middle_manager': + case 'indexer': { + const workerInfo = workerInfoLookup[service]; + if (!workerInfo) return null; + + if (workerInfo.worker.version === '') return <>Disabled; + + const details: string[] = []; + if (workerInfo.lastCompletedTaskTime) { + details.push(`Last completed task: ${formatDate(workerInfo.lastCompletedTaskTime)}`); + } + if (workerInfo.blacklistedUntil) { + details.push(`Blacklisted until: ${formatDate(workerInfo.blacklistedUntil)}`); + } + return <>{details.join(' ') || null}; + } + + case 'coordinator': + case 'overlord': + return <>{is_leader === 1 ? 'Leader' : ''}; + + case 'historical': { + const loadQueueInfo = loadQueueInfoContext[service]; + const cloneInfo = cloneStatusContext[service]; + + const parts: string[] = []; + if (serverModeInfo.decommissioningNodes.has(service)) { + parts.push('DECOMMISSIONING'); + } + if (serverModeInfo.turboLoadingNodes.has(service)) { + parts.push('TURBO SEGMENT LOADING'); + } + if (loadQueueInfo) { + parts.push(formatLoadQueueInfo(loadQueueInfo)); + } + if (cloneInfo) { + if (cloneInfo.state === 'SOURCE_SERVER_MISSING') { + parts.push(`Clone of ${cloneInfo.sourceServer} (source missing)`); + } else if (cloneInfo.segmentLoadsRemaining > 0 || cloneInfo.segmentDropsRemaining > 0) { + const details: string[] = []; + if (cloneInfo.segmentLoadsRemaining > 0) { + details.push( + `${pluralIfNeeded( + cloneInfo.segmentLoadsRemaining, + 'segment', + )} to load (${formatBytesCompact(cloneInfo.bytesToLoad)})`, + ); + } + if (cloneInfo.segmentDropsRemaining > 0) { + details.push(`${pluralIfNeeded(cloneInfo.segmentDropsRemaining, 'segment')} to drop`); + } + parts.push(`Cloning from ${cloneInfo.sourceServer}: ${details.join(', ')}`); + } else { + parts.push(`Clone of ${cloneInfo.sourceServer} (synced)`); + } + } + return <>{parts.join('; ') || null}; + } + + default: + return null; + } +} + +interface AggregatedDetailCellProps { + subRows: { _original: ServiceResultRow }[]; +} + +function AggregatedDetailCell({ subRows }: AggregatedDetailCellProps) { + const loadQueueInfoContext = useContext(LoadQueueInfoContext); + const originalRows = subRows.map(r => r._original); + if (!originalRows.some(r => r.service_type === 'historical')) return null; + + const loadQueueInfos: LoadQueueInfo[] = filterMap( + originalRows, + r => loadQueueInfoContext[r.service], + ); + + if (!loadQueueInfos.length) return null; + return <>{formatLoadQueueInfo(aggregateLoadQueueInfos(loadQueueInfos))}; +} + function defaultDisplayFn(value: any): string { if (value === undefined || value === null) return ''; return String(value); @@ -366,6 +483,51 @@ ORDER BY }); } + if (capabilities.hasCoordinatorAccess() && visibleColumns.shown('Detail')) { + auxiliaryQueries.push(async (servicesWithAuxiliaryInfo, signal) => { + try { + const [cloneStatusResp, configResp] = await Promise.all([ + getApiArrayFromKey( + '/druid/coordinator/v1/config/cloneStatus', + 'cloneStatus', + signal, + ).catch(() => [] as (CloneStatusInfo & { targetServer: string })[]), + Api.instance + .get('/druid/coordinator/v1/config', { signal }) + .then(r => r.data) + .catch(() => null), + ]); + + const cloneStatusLookup: Record = lookupBy( + cloneStatusResp, + s => s.targetServer, + ); + + return { + ...servicesWithAuxiliaryInfo, + cloneStatus: cloneStatusLookup, + ...(configResp + ? { + serverMode: { + turboLoadingNodes: new Set(configResp.turboLoadingNodes || []), + decommissioningNodes: new Set( + configResp.decommissioningNodes || [], + ), + }, + } + : {}), + }; + } catch { + AppToaster.show({ + icon: IconNames.ERROR, + intent: Intent.DANGER, + message: 'There was an error getting clone status and server mode info', + }); + return servicesWithAuxiliaryInfo; + } + }); + } + if (capabilities.hasOverlordAccess()) { auxiliaryQueries.push(async (servicesWithAuxiliaryInfo, signal) => { try { @@ -400,7 +562,13 @@ ORDER BY } return new ResultWithAuxiliaryWork( - { services, loadQueueInfo: {}, workerInfo: {} }, + { + services, + loadQueueInfo: {}, + cloneStatus: {}, + serverMode: DEFAULT_SERVER_MODE, + workerInfo: {}, + }, auxiliaryQueries, ); }, @@ -451,30 +619,36 @@ ORDER BY const { filters, onFiltersChange } = this.props; const { servicesState, groupServicesBy, visibleColumns } = this.state; - const { services, loadQueueInfo, workerInfo } = servicesState.data || { + const { services, loadQueueInfo, cloneStatus, serverMode, workerInfo } = servicesState.data || { services: [], loadQueueInfo: {}, + cloneStatus: {}, + serverMode: DEFAULT_SERVER_MODE, workerInfo: {}, }; return ( - onFiltersChange(TableFilters.fromFilters(filters))} - pivotBy={groupServicesBy ? [groupServicesBy] : []} - defaultPageSize={STANDARD_TABLE_PAGE_SIZE} - pageSizeOptions={STANDARD_TABLE_PAGE_SIZE_OPTIONS} - showPagination={services.length > STANDARD_TABLE_PAGE_SIZE} - columns={this.getTableColumns(visibleColumns, filters, onFiltersChange, workerInfo)} - /> + + + onFiltersChange(TableFilters.fromFilters(filters))} + pivotBy={groupServicesBy ? [groupServicesBy] : []} + defaultPageSize={STANDARD_TABLE_PAGE_SIZE} + pageSizeOptions={STANDARD_TABLE_PAGE_SIZE_OPTIONS} + showPagination={services.length > STANDARD_TABLE_PAGE_SIZE} + columns={this.getTableColumns(visibleColumns, filters, onFiltersChange, workerInfo)} + /> + + ); } @@ -817,61 +991,12 @@ ORDER BY id: 'queue', width: 400, filterable: false, - className: 'padded', + className: 'padded wrapped', accessor: 'service', - Cell: ({ original }) => { - const { service_type, service, is_leader } = original; - const loadQueueInfoContext = useContext(LoadQueueInfoContext); - - switch (service_type) { - case 'middle_manager': - case 'indexer': { - const workerInfo = workerInfoLookup[service]; - if (!workerInfo) return null; - - if (workerInfo.worker.version === '') return 'Disabled'; - - const details: string[] = []; - if (workerInfo.lastCompletedTaskTime) { - details.push( - `Last completed task: ${formatDate(workerInfo.lastCompletedTaskTime)}`, - ); - } - if (workerInfo.blacklistedUntil) { - details.push(`Blacklisted until: ${formatDate(workerInfo.blacklistedUntil)}`); - } - return details.join(' ') || null; - } - - case 'coordinator': - case 'overlord': - return is_leader === 1 ? 'Leader' : ''; - - case 'historical': { - const loadQueueInfo = loadQueueInfoContext[service]; - if (!loadQueueInfo) return null; - - return formatLoadQueueInfo(loadQueueInfo); - } - - default: - return null; - } - }, - Aggregated: ({ subRows }) => { - const loadQueueInfoContext = useContext(LoadQueueInfoContext); - const originalRows = subRows.map(r => r._original); - if (!originalRows.some(r => r.service_type === 'historical')) return ''; - - const loadQueueInfos: LoadQueueInfo[] = filterMap( - originalRows, - r => loadQueueInfoContext[r.service], - ); - - return loadQueueInfos.length - ? formatLoadQueueInfo(aggregateLoadQueueInfos(loadQueueInfos)) - : ''; - }, + Cell: ({ original }) => ( + + ), + Aggregated: ({ subRows }) => , }, { Header: ACTION_COLUMN_LABEL,