Skip to content

fix(sql-editor): allow 10k and 50k rows in pivot queries #108434

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mariusandra wants to merge 3 commits into
base: master
Choose a base branch
from
Open

fix(sql-editor): allow 10k and 50k rows in pivot queries #108434

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
00cf31c
fix(sql-editor): allow 10k and 50k rows in pivot queries
mariusandra Sep 29, 2026
5b59481
Merge branch 'master' into fix/bi-pivot-query-limits
mariusandra Sep 29, 2026
c95089f
chore(sql-editor): align pivot limit persistence tests
mariusandra Sep 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 1 addition & 6 deletions frontend/src/scenes/data-warehouse/editor/bi/BIEditor.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@
BIShelf,
BIField,
BISortDirection,
PIVOT_TABLE_QUERY_LIMIT,
getBIDataSourceKey,
isDateTimeBIField,
isNumericBIField,
Expand Down Expand Up @@ -103,7 +102,7 @@
{ value: 'asc', label: 'Ascending' },
]

export function BIEditor({ tabId }: { tabId: string }): JSX.Element {

Check warning on line 105 in frontend/src/scenes/data-warehouse/editor/bi/BIEditor.tsx

View workflow job for this annotation

GitHub Actions / Frontend formatting

lint:complexity 

`BIEditor` has cyclomatic complexity 13 (warn >10)
const logic = biEditorLogic({ tabId })
const { activeDropShelf, activeExpressionEditorId, availableDataSources, config, databaseLoading, sortOptions } =
useValues(logic)
Expand Down Expand Up @@ -131,10 +130,6 @@
setValueAggregation,
setValueCustomExpression,
} = useActions(logic)
const limitOptions =
config.chartType === ChartDisplayType.TwoDimensionalHeatmap
? LIMIT_OPTIONS.filter(({ value }) => value <= PIVOT_TABLE_QUERY_LIMIT)
: LIMIT_OPTIONS

return (
<div
Expand Down Expand Up @@ -197,7 +192,7 @@
/>
<LemonSelect
value={config.limit}
options={limitOptions}
options={LIMIT_OPTIONS}
onChange={setLimit}
renderButtonContent={(option) => `Limit: ${option?.label ?? config.limit}`}
aria-label="Query row limit"
Expand Down
53 changes: 28 additions & 25 deletions frontend/src/scenes/data-warehouse/editor/bi/biEditorTypes.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,32 +108,35 @@ describe('BI editor query generation', () => {
expect(result?.query).toContain('count(*) AS count')
})

it('maps every BI row and column dimension to pivot table axes', () => {
const result = buildBIQuery({
source: { table: 'events' },
chartType: ChartDisplayType.TwoDimensionalHeatmap,
rows: [eventField, timestampField],
columns: [browserField, countryField],
values: [{ field: revenueField, aggregation: 'sum' }],
filters: [],
limit: 50000,
})
it.each([100, 1000, 10000, 50000] as const)(
'maps every BI row and column dimension to pivot table axes with limit %i',
(limit) => {
const result = buildBIQuery({
source: { table: 'events' },
chartType: ChartDisplayType.TwoDimensionalHeatmap,
rows: [eventField, timestampField],
columns: [browserField, countryField],
values: [{ field: revenueField, aggregation: 'sum' }],
filters: [],
limit,
})

expect(result?.query).toContain('toJSONString(tuple(event, timestamp)) AS bi_rows')
expect(result?.query).toContain(
'toJSONString(tuple(properties.$browser, properties.$geoip_country_name)) AS bi_columns'
)
expect(result?.query).toContain('LIMIT 1000')
expect(result?.node.chartSettings).toEqual({
heatmap: {
xAxisColumn: 'bi_columns',
yAxisColumn: 'bi_rows',
valueColumn: 'sum_revenue',
xAxisLabel: 'browser / country',
yAxisLabel: 'event / timestamp',
},
})
})
expect(result?.query).toContain('toJSONString(tuple(event, timestamp)) AS bi_rows')
expect(result?.query).toContain(
'toJSONString(tuple(properties.$browser, properties.$geoip_country_name)) AS bi_columns'
)
expect(result?.query).toContain(`LIMIT ${limit}`)
expect(result?.node.chartSettings).toEqual({
heatmap: {
xAxisColumn: 'bi_columns',
yAxisColumn: 'bi_rows',
valueColumn: 'sum_revenue',
xAxisLabel: 'browser / country',
yAxisLabel: 'event / timestamp',
},
})
}
)

it('ignores blank shelf fields until they are configured', () => {
const blankField: BIField = {
Expand Down
6 changes: 0 additions & 6 deletions frontend/src/scenes/data-warehouse/editor/bi/biEditorTypes.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@

export type BIQueryLimit = (typeof BI_QUERY_LIMITS)[number]

export const PIVOT_TABLE_QUERY_LIMIT: BIQueryLimit = 1000

export type BISortDirection = 'asc' | 'desc'

export interface BISort {
Expand Down Expand Up @@ -115,10 +113,6 @@

export function normalizeBIConfig(config: BIConfig): BIConfig {
let normalized = config
if (normalized.chartType === ChartDisplayType.TwoDimensionalHeatmap && normalized.limit > PIVOT_TABLE_QUERY_LIMIT) {
normalized = { ...normalized, limit: PIVOT_TABLE_QUERY_LIMIT }
}

const sort = normalized.sort
if (sort && !getBISortOptions(normalized).some((option) => option.key === sort.key)) {
normalized = { ...normalized, sort: null }
Expand Down Expand Up @@ -220,7 +214,7 @@
return JSON.stringify(field)
}

function parseBIFieldValue(value: unknown): BIField | null {

Check warning on line 217 in frontend/src/scenes/data-warehouse/editor/bi/biEditorTypes.ts

View workflow job for this annotation

GitHub Actions / Frontend formatting

lint:complexity 

`parseBIFieldValue` has cyclomatic complexity 15 (warn >10)
if (!value || typeof value !== 'object') {
return null
}
Expand Down Expand Up @@ -252,7 +246,7 @@
}
}

export function parseBIEditorState(editorViewValue: unknown, configValue: unknown): BIEditorState | null {

Check warning on line 249 in frontend/src/scenes/data-warehouse/editor/bi/biEditorTypes.ts

View workflow job for this annotation

GitHub Actions / Frontend formatting

lint:complexity 

`parseBIEditorState` has cyclomatic complexity 38 (warn >10)
if (editorViewValue !== BIEditorView.SQL && editorViewValue !== BIEditorView.BI) {
return null
}
Expand Down Expand Up @@ -465,7 +459,7 @@
}
}

function filterExpression(filter: BIFilter): string | null {

Check warning on line 462 in frontend/src/scenes/data-warehouse/editor/bi/biEditorTypes.ts

View workflow job for this annotation

GitHub Actions / Frontend formatting

lint:complexity 

`filterExpression` has cyclomatic complexity 14 (warn >10)
const field = fieldExpression(filter.field)

if (filter.operator === 'custom') {
Expand Down Expand Up @@ -623,7 +617,7 @@
return `${firstValueAlias} DESC`
}

export function buildBIQuery(config: BIConfig): BIQueryBuildResult | null {

Check warning on line 620 in frontend/src/scenes/data-warehouse/editor/bi/biEditorTypes.ts

View workflow job for this annotation

GitHub Actions / Frontend formatting

lint:complexity 

`buildBIQuery` has cyclomatic complexity 14 (warn >10)
if (!config.source) {
return null
}
Expand Down
15 changes: 7 additions & 8 deletions frontend/src/scenes/data-warehouse/editor/sqlEditorLogic.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2189,7 +2189,6 @@ describe('sqlEditorLogic', () => {
chartType: ChartDisplayType.TwoDimensionalHeatmap,
limit: 50000,
}
const restoredConfig: BIConfig = { ...persistedConfig, limit: 1000 }

router.actions.push(urls.sqlEditor(), undefined, {
q: "SELECT event, count(*) FROM events WHERE event = 'signup' GROUP BY event",
Expand All @@ -2201,17 +2200,17 @@ describe('sqlEditorLogic', () => {
.toDispatchActions(['createTab', 'updateTab'])
.toMatchValues({
activeTab: partial({
biEditorState: { editorView: BIEditorView.BI, config: restoredConfig },
biEditorState: { editorView: BIEditorView.BI, config: persistedConfig },
}),
})
await expectLogic(biLogic).toMatchValues({ editorView: BIEditorView.BI, config: restoredConfig })
await expectLogic(biLogic).toMatchValues({ editorView: BIEditorView.BI, config: persistedConfig })

await expectLogic(biLogic, () => biLogic.actions.setFilterValue(0, 'purchase')).toFinishAllListeners()

expect(router.values.hashParams.mode).toEqual(BIEditorView.BI)
expect(router.values.hashParams.bi).toEqual({
...restoredConfig,
filters: [{ ...restoredConfig.filters[0], value: 'purchase' }],
...persistedConfig,
filters: [{ ...persistedConfig.filters[0], value: 'purchase' }],
})

biLogic.unmount()
Expand Down Expand Up @@ -2276,12 +2275,12 @@ describe('sqlEditorLogic', () => {
biLogic.actions.setChartType(ChartDisplayType.TwoDimensionalHeatmap)
).toFinishAllListeners()

expect(biLogic.values.config.limit).toBe(1000)
expect(logic.values.queryInput).toContain('LIMIT 1000')
expect(biLogic.values.config.limit).toBe(50000)
expect(logic.values.queryInput).toContain('LIMIT 50000')
expect(router.values.hashParams.bi).toEqual({
...config,
chartType: ChartDisplayType.TwoDimensionalHeatmap,
limit: 1000,
limit: 50000,
})

biLogic.unmount()
Expand Down
Loading