fix(search): preserve incompatible filters as inactive when switching data sources

.changeset/active-filter-pills-inactive-state.md

@@ -0,0 +1,9 @@
+---
+'@hyperdx/app': patch
+---
+
+feat: preserve incompatible filters as an inactive state when switching data sources on the search page.
+
+Previously, switching from Logs to Traces (or any other schema change) would silently drop filters whose fields don't exist on the new source. Now those filters stay visible in the `ActiveFilterPills` bar with a muted, strikethrough, dashed-border style and a tooltip explaining why they aren't applied. They are automatically excluded from the rendered query so it stays valid, and re-apply if the user switches back to a compatible source.
+
+`ActiveFilterPills` accepts a new `invalidFields?: Set<string>` prop (with optional `invalidFieldReason?: (field: string) => string` for tooltip customization). `useSearchPageFilterState` accepts a new `validFields?: Set<string>` option and exposes `invalidFields` so consumers don't have to compute it themselves.

packages/app/src/DBSearchPage.tsx

@@ -25,10 +25,7 @@ import {
 import { useForm, useWatch } from 'react-hook-form';
 import { z } from 'zod';
 import { zodResolver } from '@hookform/resolvers/zod';
-import {
-  ClickHouseQueryError,
-  ColumnMeta,
-} from '@hyperdx/common-utils/dist/clickhouse';
+import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse';
 import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
 import { buildSearchChartConfig } from '@hyperdx/common-utils/dist/core/searchChartConfig';
 import {
@@ -134,7 +131,6 @@ import {
 } from './components/TimePicker/utils';
 import { useColumns, useTableMetadata } from './hooks/useMetadata';
 import { useSqlSuggestions } from './hooks/useSqlSuggestions';
-import { useStableCallback } from './hooks/useStableCallback';
 import {
   buildDirectTraceWhereClause,
   getDefaultDirectTraceDateRange,
@@ -787,12 +783,6 @@ export function useDefaultOrderBy(sourceID: string | undefined | null) {
   }, [source, tableMetadata]);
 }
 
-function formatDroppedFiltersMessage(count: number): string {
-  const noun = count === 1 ? 'filter' : 'filters';
-  const verb = count === 1 ? 'was' : 'were';
-  return `${count} ${noun} didn't apply to this source and ${verb} removed.`;
-}
-
 // This is outside as it needs to be a stable reference
 const queryStateMap = {
   source: parseAsString,
@@ -1068,10 +1058,6 @@ export function DBSearchPage() {
   );
 
   const filters = useWatch({ name: 'filters', control });
-  const searchFilters = useSearchPageFilterState({
-    searchQuery: filters ?? undefined,
-    onFilterChange: handleSetFilters,
-  });
 
   const watchedSource = useWatch({
     control,
@@ -1080,10 +1066,6 @@ export function DBSearchPage() {
     defaultValue: searchedConfig.source ?? undefined,
   });
   const prevSourceRef = useRef(watchedSource);
-  // Set when the user switches sources via the dropdown. The follow-up
-  // effect waits for the new source's columns to load and then drops any
-  // sidebar filters that don't apply to the new schema.
-  const pendingFilterReconcileRef = useRef<string | null>(null);
 
   const watchedSourceObj = useMemo(
     () => inputSourceObjs?.find(s => s.id === watchedSource),
@@ -1098,9 +1080,30 @@ export function DBSearchPage() {
     { enabled: !!watchedSourceObj },
   );
 
+  // Set of column names for the active source. Filters whose key isn't in
+  // this set are marked inactive (kept in UI state for context but skipped
+  // when serializing the query) so switching sources doesn't lose user
+  // selections. While columns are loading the set is `undefined` and the
+  // hook treats all filters as valid.
+  const validFields = useMemo<Set<string> | undefined>(
+    () =>
+      watchedSourceColumns
+        ? new Set(watchedSourceColumns.map(c => c.name))
+        : undefined,
+    [watchedSourceColumns],
+  );
+
+  const searchFilters = useSearchPageFilterState({
+    searchQuery: filters ?? undefined,
+    onFilterChange: handleSetFilters,
+    validFields,
+  });
+
   useEffect(() => {
-    // If the user changes the source dropdown, reset the select and orderby fields
-    // to match the new source selected
+    // If the user changes the source dropdown, reset the select and orderby
+    // fields to match the new source selected. Filters are reconciled
+    // reactively by `useSearchPageFilterState` via `validFields` — invalid
+    // filters are preserved as inactive instead of being dropped.
     if (watchedSource !== prevSourceRef.current) {
       prevSourceRef.current = watchedSource;
       const newInputSourceObj = inputSourceObjs?.find(
@@ -1114,18 +1117,13 @@ export function DBSearchPage() {
         if (savedSearchId == null || savedSearch?.source !== watchedSource) {
           setValue('select', '');
           setValue('orderBy', '');
-          // Defer filter clearing: wait until the new source's columns load,
-          // then keep filters whose root column exists on the new schema.
-          pendingFilterReconcileRef.current = watchedSource ?? null;
           // If the user is in a saved search, prefer the saved search's select/orderBy if available
         } else {
           setValue('select', savedSearch?.select ?? '');
           setValue('orderBy', savedSearch?.orderBy ?? '');
           // Don't clear filters - we're loading from saved search
         }
         // Push the new source to URL/searchedConfig so the chart re-queries.
-        // Debounced so a later filter reconcile (which also submits) collapses
-        // into a single run.
         debouncedSubmit();
       }
     }
@@ -1139,30 +1137,6 @@ export function DBSearchPage() {
     debouncedSubmit,
   ]);
 
-  const retainCompatibleFilters = useStableCallback((columns: ColumnMeta[]) => {
-    pendingFilterReconcileRef.current = null;
-
-    const allowed = new Set(columns.map(c => c.name));
-
-    const dropped = searchFilters.retainFiltersByColumns(allowed);
-
-    if (dropped.length > 0) {
-      notifications.show({
-        color: 'yellow',
-        message: formatDroppedFiltersMessage(dropped.length),
-      });
-    }
-  });
-
-  useEffect(() => {
-    if (
-      pendingFilterReconcileRef.current === watchedSource &&
-      watchedSourceColumns
-    ) {
-      retainCompatibleFilters(watchedSourceColumns);
-    }
-  }, [watchedSource, watchedSourceColumns, retainCompatibleFilters]);
-
   const onTableScroll = useCallback(
     (scrollTop: number) => {
       // If the user scrolls a bit down, kick out of live mode
@@ -2025,7 +1999,11 @@ export function DBSearchPage() {
             <SearchSubmitButton isFormStateDirty={formState.isDirty} />
           </Flex>
         </Flex>
-        <ActiveFilterPills searchFilters={searchFilters} mt={6} />
+        <ActiveFilterPills
+          searchFilters={searchFilters}
+          invalidFields={searchFilters.invalidFields}
+          mt={6}
+        />
       </form>
       {searchedConfig != null && searchedSource != null && (
         <SaveSearchModal

packages/app/src/__tests__/searchFilters.test.ts

@@ -1155,6 +1155,160 @@ describe('searchFilters', () => {
         warnSpy.mockRestore();
       });
     });
+
+    describe('validFields / invalidFields', () => {
+      it('treats all filters as valid when validFields is not provided', () => {
+        const { result } = renderHook(() =>
+          useSearchPageFilterState({
+            searchQuery: [
+              { type: 'lucene', condition: 'ServiceName:"app"' },
+              { type: 'lucene', condition: 'SeverityText:"info"' },
+            ],
+            onFilterChange: jest.fn(),
+          }),
+        );
+
+        expect(result.current.invalidFields.size).toBe(0);
+      });
+
+      it('marks filter keys absent from validFields as invalid', () => {
+        const { result } = renderHook(() =>
+          useSearchPageFilterState({
+            searchQuery: [
+              { type: 'lucene', condition: 'ServiceName:"app"' },
+              { type: 'lucene', condition: 'SeverityText:"info"' },
+            ],
+            onFilterChange: jest.fn(),
+            validFields: new Set(['ServiceName', 'Timestamp']),
+          }),
+        );
+
+        expect(Array.from(result.current.invalidFields)).toEqual([
+          'SeverityText',
+        ]);
+        // Filter still in UI state — preserved for context.
+        expect(result.current.filters.SeverityText).toBeDefined();
+      });
+
+      it('treats a nested key as valid when its root column is valid', () => {
+        const { result } = renderHook(() =>
+          useSearchPageFilterState({
+            searchQuery: [
+              { type: 'lucene', condition: 'LogAttributes.user:"123"' },
+            ],
+            onFilterChange: jest.fn(),
+            validFields: new Set(['LogAttributes']),
+          }),
+        );
+
+        expect(result.current.invalidFields.size).toBe(0);
+      });
+
+      it('strips invalid filters from the serialized query on user mutation', () => {
+        const onFilterChange = jest.fn();
+        const { result } = renderHook(() =>
+          useSearchPageFilterState({
+            searchQuery: [
+              { type: 'lucene', condition: 'ServiceName:"app"' },
+              { type: 'lucene', condition: 'SeverityText:"info"' },
+            ],
+            onFilterChange,
+            validFields: new Set(['ServiceName']),
+          }),
+        );
+
+        act(() => {
+          result.current.setFilterValue('ServiceName', 'api');
+        });
+
+        const lastCall =
+          onFilterChange.mock.calls[onFilterChange.mock.calls.length - 1][0];
+        expect(
+          lastCall.some((f: { condition: string }) =>
+            f.condition.includes('SeverityText'),
+          ),
+        ).toBe(false);
+        expect(
+          lastCall.some((f: { condition: string }) =>
+            f.condition.includes('ServiceName'),
+          ),
+        ).toBe(true);
+      });
+
+      it('re-emits the query when validFields changes so the URL stays in sync', () => {
+        const onFilterChange = jest.fn();
+        let validFields: Set<string> | undefined = new Set([
+          'ServiceName',
+          'SeverityText',
+        ]);
+        const { rerender } = renderHook(() =>
+          useSearchPageFilterState({
+            searchQuery: [
+              { type: 'lucene', condition: 'ServiceName:"app"' },
+              { type: 'lucene', condition: 'SeverityText:"info"' },
+            ],
+            onFilterChange,
+            validFields,
+          }),
+        );
+
+        onFilterChange.mockClear();
+
+        validFields = new Set(['ServiceName']);
+        rerender();
+
+        // SeverityText is no longer valid → stripped from the emitted query.
+        const lastCall =
+          onFilterChange.mock.calls[onFilterChange.mock.calls.length - 1][0];
+        expect(
+          lastCall.every(
+            (f: { condition: string }) => !f.condition.includes('SeverityText'),
+          ),
+        ).toBe(true);
+      });
+
+      it('preserves invalid filters in state so they re-apply when the field becomes valid again', () => {
+        const onFilterChange = jest.fn();
+        let validFields: Set<string> | undefined = new Set(['ServiceName']);
+        const { result, rerender } = renderHook(() =>
+          useSearchPageFilterState({
+            // URL only has ServiceName (SeverityText was previously stripped)
+            // but UI state will keep SeverityText preserved across rerenders.
+            searchQuery: [{ type: 'lucene', condition: 'ServiceName:"app"' }],
+            onFilterChange,
+            validFields,
+          }),
+        );
+
+        // Seed UI state with an invalid filter directly (simulating an
+        // earlier source where SeverityText was valid).
+        act(() => {
+          result.current.setFilters(prev => ({
+            ...prev,
+            SeverityText: {
+              included: new Set<string | boolean>(['info']),
+              excluded: new Set<string | boolean>(),
+            },
+          }));
+        });
+
+        expect(result.current.invalidFields.has('SeverityText')).toBe(true);
+
+        onFilterChange.mockClear();
+
+        // Switch to a source where SeverityText is valid again.
+        validFields = new Set(['ServiceName', 'SeverityText']);
+        rerender();
+
+        const lastCall =
+          onFilterChange.mock.calls[onFilterChange.mock.calls.length - 1][0];
+        expect(
+          lastCall.some((f: { condition: string }) =>
+            f.condition.includes('SeverityText'),
+          ),
+        ).toBe(true);
+      });
+    });
   });
 
   describe('filters use direct_read optimization', () => {