Skip to content

Commit c7d4d6a

Browse files
ivictborivictbor
authored
Merge pull request #627 from devforth/next
Next
2 parents 7dc653c + 41fc919 commit c7d4d6a

25 files changed

Lines changed: 529 additions & 193 deletions

File tree

‎adminforth/dataConnectors/baseConnector.ts‎

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import { afLogger } from '../modules/logger.js';
1616

1717
type AdminForthFilterNode = IAdminForthSingleFilter | IAdminForthAndOrFilter;
1818
type AdminForthFilterInput = AdminForthFilterNode | AdminForthFilterNode[];
19+
type AggregateGroupByInput = IGroupByRule | IGroupByRule[] | undefined;
1920
type AdminForthFilterNormalizationResult = {
2021
ok: boolean;
2122
error: string;
@@ -286,18 +287,26 @@ export default class AdminForthBaseConnector implements IAdminForthDataSourceCon
286287
resource: AdminForthResource,
287288
filters: IAdminForthAndOrFilter,
288289
aggregations: { [alias: string]: IAggregationRule },
289-
groupBy?: IGroupByRule,
290+
groupBy?: AggregateGroupByInput,
290291
}): Promise<Array<{ group?: string, [key: string]: any }>> {
291292
throw new Error('getAggregateWithOriginalTypes() not implemented for this connector.');
292293
}
293294

295+
normalizeGroupByRules(groupBy?: AggregateGroupByInput): IGroupByRule[] {
296+
return groupBy ? (Array.isArray(groupBy) ? groupBy : [groupBy]) : [];
297+
}
298+
299+
getGroupByResultAlias(groupBy: IGroupByRule, index: number, total: number): string {
300+
return groupBy.as ?? (total === 1 ? 'group' : `group${index + 1}`);
301+
}
302+
294303
private validateAggregateParams(
295304
resource: AdminForthResource,
296305
aggregations: { [alias: string]: IAggregationRule },
297-
groupBy?: IGroupByRule,
306+
groupBy?: AggregateGroupByInput,
298307
): void {
299308
const VALID_ALIAS = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
300-
const VALID_OPERATIONS = ['sum', 'count', 'avg', 'min', 'max', 'median'];
309+
const VALID_OPERATIONS = ['sum', 'count', 'count_distinct', 'avg', 'min', 'max', 'median'];
301310
const VALID_TRUNCATIONS = ['day', 'week', 'month', 'year'];
302311
const VALID_TIMEZONE = /^[a-zA-Z_\/\-\+0-9]+$/;
303312
const columnNames = new Set(resource.dataSourceColumns.map(c => c.name));
@@ -323,20 +332,26 @@ export default class AdminForthBaseConnector implements IAdminForthDataSourceCon
323332
}
324333
}
325334

326-
if (groupBy) {
327-
if (groupBy.type === 'field') {
328-
assertColumn(groupBy.field, 'GroupBy.Field');
329-
} else if (groupBy.type === 'date_trunc') {
330-
const g = groupBy as IGroupByDateTrunc;
335+
for (const groupByRule of this.normalizeGroupByRules(groupBy)) {
336+
if (groupByRule.type === 'field') {
337+
assertColumn(groupByRule.field, 'GroupBy.Field');
338+
if (groupByRule.as && !VALID_ALIAS.test(groupByRule.as)) {
339+
throw new Error(`Invalid groupBy alias "${groupByRule.as}". Must match ${VALID_ALIAS}`);
340+
}
341+
} else if (groupByRule.type === 'date_trunc') {
342+
const g = groupByRule as IGroupByDateTrunc;
331343
assertColumn(g.field, 'GroupBy.DateTrunc');
332344
if (!VALID_TRUNCATIONS.includes(g.truncation)) {
333345
throw new Error(`Invalid truncation "${g.truncation}". Must be one of: ${VALID_TRUNCATIONS.join(', ')}`);
334346
}
335347
if (g.timezone && !VALID_TIMEZONE.test(g.timezone)) {
336348
throw new Error(`Invalid timezone "${g.timezone}". Must be a valid IANA timezone name`);
337349
}
350+
if (g.as && !VALID_ALIAS.test(g.as)) {
351+
throw new Error(`Invalid groupBy alias "${g.as}". Must match ${VALID_ALIAS}`);
352+
}
338353
} else {
339-
throw new Error(`Unknown groupBy type "${(groupBy as any).type}"`);
354+
throw new Error(`Unknown groupBy type "${(groupByRule as any).type}"`);
340355
}
341356
}
342357
}
@@ -345,7 +360,7 @@ export default class AdminForthBaseConnector implements IAdminForthDataSourceCon
345360
resource: AdminForthResource,
346361
filters: IAdminForthAndOrFilter,
347362
aggregations: { [alias: string]: IAggregationRule },
348-
groupBy?: IGroupByRule,
363+
groupBy?: AggregateGroupByInput,
349364
}): Promise<Array<{ group?: string, [key: string]: any }>> {
350365
this.validateAggregateParams(resource, aggregations, groupBy);
351366

‎adminforth/dataConnectors/clickhouse.ts‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -451,16 +451,19 @@ class ClickhouseConnector extends AdminForthBaseConnector implements IAdminForth
451451
resource: AdminForthResource;
452452
filters: IAdminForthAndOrFilter;
453453
aggregations: { [alias: string]: IAggregationRule };
454-
groupBy?: IGroupByRule;
454+
groupBy?: IGroupByRule | IGroupByRule[];
455455
}): Promise <Array<{ group?: string, [key: string]: any }>> {
456456

457457
const tableName = `${this.dbName}.${resource.table}`;
458458

459459
const selectParts: string[] = [];
460-
let groupExpr: string | null = null;
460+
const groupExprs: string[] = [];
461+
const groupByRules = this.normalizeGroupByRules(groupBy);
461462

462-
if (groupBy?.type === 'date_trunc') {
463-
const g = groupBy as IGroupByDateTrunc;
463+
for (const [index, groupByRule] of groupByRules.entries()) {
464+
let groupExpr: string;
465+
if (groupByRule.type === 'date_trunc') {
466+
const g = groupByRule as IGroupByDateTrunc;
464467
const tz = g.timezone ?? 'UTC';
465468

466469
const field = `toTimeZone(${g.field}, '${tz}')`;
@@ -471,18 +474,18 @@ class ClickhouseConnector extends AdminForthBaseConnector implements IAdminForth
471474
case 'week': groupExpr = `toDate(toStartOfWeek(${field}))`; break;
472475
case 'year': groupExpr = `toDate(toStartOfYear(${field}))`; break;
473476
}
474-
475-
selectParts.push(`${groupExpr} AS \`group\``);
476-
477-
} else if (groupBy?.type === 'field') {
478-
const g = groupBy as IGroupByField;
477+
} else {
478+
const g = groupByRule as IGroupByField;
479479
groupExpr = `${g.field}`;
480-
selectParts.push(`${groupExpr} AS \`group\``);
480+
}
481+
groupExprs.push(groupExpr);
482+
selectParts.push(`${groupExpr} AS \`${this.getGroupByResultAlias(groupByRule, index, groupByRules.length)}\``);
481483
}
482484

483485
for (const [alias, rule] of Object.entries(aggregations)) {
484486
switch (rule.operation) {
485487
case 'count': selectParts.push(`count() AS \`${alias}\``); break;
488+
case 'count_distinct': selectParts.push(`uniqExact(${rule.field}) AS \`${alias}\``); break;
486489
case 'sum': selectParts.push(`sum(${rule.field}) AS \`${alias}\``); break;
487490
case 'avg': selectParts.push(`avg(${rule.field}) AS \`${alias}\``); break;
488491
case 'min': selectParts.push(`min(${rule.field}) AS \`${alias}\``); break;
@@ -495,8 +498,8 @@ class ClickhouseConnector extends AdminForthBaseConnector implements IAdminForth
495498

496499
let query = `SELECT ${selectParts.join(', ')} FROM ${tableName} ${where}`;
497500

498-
if (groupExpr) {
499-
query += ` GROUP BY ${groupExpr} ORDER BY ${groupExpr} ASC`;
501+
if (groupExprs.length) {
502+
query += ` GROUP BY ${groupExprs.join(', ')} ORDER BY ${groupExprs.join(', ')} ASC`;
500503
}
501504

502505
const result = await this.client.query({
@@ -664,4 +667,4 @@ class ClickhouseConnector extends AdminForthBaseConnector implements IAdminForth
664667
}
665668
}
666669

667-
export default ClickhouseConnector;
670+
export default ClickhouseConnector;

‎adminforth/dataConnectors/mongo.ts‎

Lines changed: 32 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -310,22 +310,26 @@ class MongoConnector extends AdminForthBaseConnector implements IAdminForthDataS
310310
resource: AdminForthResource;
311311
filters: IAdminForthAndOrFilter;
312312
aggregations: { [alias: string]: IAggregationRule };
313-
groupBy?: IGroupByRule;
313+
groupBy?: IGroupByRule | IGroupByRule[];
314314
}): Promise<Array<{ group?: string, [key: string]: any }>> {
315315

316316
const collection = this.client.db().collection(resource.table);
317317

318318
const match = filters?.subFilters?.length ? this.getFilterQuery(resource, filters) : {};
319319

320+
const groupByRules = this.normalizeGroupByRules(groupBy);
320321
let groupId: any = null;
321-
322-
if (groupBy?.type === 'field') {
323-
const g = groupBy as IGroupByField;
324-
groupId = `$${g.field}`;
322+
if (groupByRules.length) {
323+
groupId = {};
325324
}
326-
327-
if (groupBy?.type === 'date_trunc') {
328-
const g = groupBy as IGroupByDateTrunc;
325+
for (const [index, groupByRule] of groupByRules.entries()) {
326+
const alias = this.getGroupByResultAlias(groupByRule, index, groupByRules.length);
327+
if (groupByRule.type === 'field') {
328+
const g = groupByRule as IGroupByField;
329+
groupId[alias] = `$${g.field}`;
330+
continue;
331+
}
332+
const g = groupByRule as IGroupByDateTrunc;
329333
const tz = g.timezone ?? 'UTC';
330334
const dateTruncSpec: any = {
331335
date: `$${g.field}`,
@@ -335,7 +339,7 @@ class MongoConnector extends AdminForthBaseConnector implements IAdminForthDataS
335339
if (g.truncation === 'week') {
336340
dateTruncSpec.startOfWeek = 'Mon';
337341
}
338-
groupId = { $dateTrunc: dateTruncSpec };
342+
groupId[alias] = { $dateTrunc: dateTruncSpec };
339343
}
340344

341345
const groupStage: Record<string, any> = {
@@ -345,6 +349,7 @@ class MongoConnector extends AdminForthBaseConnector implements IAdminForthDataS
345349
for (const [alias, rule] of Object.entries(aggregations)) {
346350
switch (rule.operation) {
347351
case 'count': groupStage[alias] = { $sum: 1 }; break;
352+
case 'count_distinct': groupStage[alias] = { $addToSet: `$${rule.field}` }; break;
348353
case 'sum': groupStage[alias] = { $sum: { $toDouble: `$${rule.field}` } }; break;
349354
case 'avg': groupStage[alias] = { $avg: { $toDouble: `$${rule.field}` } }; break;
350355
case 'min': groupStage[alias] = { $min: { $toDouble: `$${rule.field}` } }; break;
@@ -364,23 +369,26 @@ class MongoConnector extends AdminForthBaseConnector implements IAdminForthDataS
364369
pipeline.push({
365370
$project: {
366371
_id: 0,
367-
group: !groupBy ? "$$REMOVE" : (groupBy.type === 'date_trunc' ? {
368-
$cond: {
369-
if: { $eq: [{ $type: "$_id" }, "date"] },
370-
then: {
371-
$dateToString: {
372-
format: "%Y-%m-%d",
373-
date: "$_id",
374-
timezone: (groupBy as IGroupByDateTrunc).timezone ?? 'UTC'
375-
}
376-
},
377-
else: "$_id"
378-
}
379-
} : "$_id"),
372+
...Object.fromEntries(groupByRules.map((groupByRule, index) => {
373+
const alias = this.getGroupByResultAlias(groupByRule, index, groupByRules.length);
374+
return [alias, groupByRule.type === 'date_trunc' ? {
375+
$cond: {
376+
if: { $eq: [{ $type: `$_id.${alias}` }, "date"] },
377+
then: {
378+
$dateToString: {
379+
format: "%Y-%m-%d",
380+
date: `$_id.${alias}`,
381+
timezone: (groupByRule as IGroupByDateTrunc).timezone ?? 'UTC'
382+
}
383+
},
384+
else: `$_id.${alias}`
385+
}
386+
} : `$_id.${alias}`];
387+
})),
380388
...Object.fromEntries(
381389
Object.keys(groupStage)
382390
.filter(k => k !== '_id')
383-
.map(k => [k, `$${k}`])
391+
.map(k => [k, aggregations[k]?.operation === 'count_distinct' ? { $size: `$${k}` } : `$${k}`])
384392
),
385393
},
386394
});
@@ -521,4 +529,4 @@ class MongoConnector extends AdminForthBaseConnector implements IAdminForthDataS
521529
}
522530
}
523531

524-
export default MongoConnector;
532+
export default MongoConnector;

‎adminforth/dataConnectors/mysql.ts‎

Lines changed: 41 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -345,37 +345,42 @@ class MysqlConnector extends AdminForthBaseConnector implements IAdminForthDataS
345345
resource: AdminForthResource;
346346
filters: IAdminForthAndOrFilter;
347347
aggregations: { [alias: string]: IAggregationRule };
348-
groupBy?: IGroupByRule;
348+
groupBy?: IGroupByRule | IGroupByRule[];
349349
}): Promise<Array<{ group?: string, [key: string]: any }>> {
350350
const tableName = resource.table;
351351
const selectParts: string[] = [];
352352
const medianFields: { alias: string; field: string }[] = [];
353-
let groupExpr: string | null = null;
354-
355-
if (groupBy?.type === 'field') {
356-
groupExpr = `\`${groupBy.field}\``;
357-
selectParts.push(`${groupExpr} AS \`group\``);
358-
} else if (groupBy?.type === 'date_trunc') {
359-
const g = groupBy as IGroupByDateTrunc;
360-
const tz = g.timezone ?? 'UTC';
361-
if (!/^[A-Za-z0-9/_+\-]+$/.test(tz)) {
362-
throw new Error(`Invalid timezone value: ${tz}`);
363-
}
364-
const innerExpr = `COALESCE(CONVERT_TZ(\`${g.field}\`, 'UTC', '${tz}'), \`${g.field}\`)`;
365-
switch (g.truncation) {
366-
case 'day': groupExpr = `DATE_FORMAT(${innerExpr}, '%Y-%m-%d')`; break;
367-
case 'month': groupExpr = `DATE_FORMAT(${innerExpr}, '%Y-%m-01')`; break;
368-
case 'year': groupExpr = `DATE_FORMAT(${innerExpr}, '%Y-01-01')`; break;
369-
case 'week': groupExpr = `DATE_FORMAT(DATE_SUB(${innerExpr}, INTERVAL WEEKDAY(${innerExpr}) DAY), '%Y-%m-%d')`; break;
353+
const groupExprs: string[] = [];
354+
const groupAliases: string[] = [];
355+
const groupByRules = this.normalizeGroupByRules(groupBy);
356+
357+
for (const [index, groupByRule] of groupByRules.entries()) {
358+
let groupExpr: string;
359+
if (groupByRule.type === 'field') {
360+
groupExpr = `\`${groupByRule.field}\``;
361+
} else {
362+
const g = groupByRule as IGroupByDateTrunc;
363+
const tz = g.timezone ?? 'UTC';
364+
const innerExpr = `COALESCE(CONVERT_TZ(\`${g.field}\`, 'UTC', '${tz}'), \`${g.field}\`)`;
365+
switch (g.truncation) {
366+
case 'day': groupExpr = `DATE_FORMAT(${innerExpr}, '%Y-%m-%d')`; break;
367+
case 'month': groupExpr = `DATE_FORMAT(${innerExpr}, '%Y-%m-01')`; break;
368+
case 'year': groupExpr = `DATE_FORMAT(${innerExpr}, '%Y-01-01')`; break;
369+
case 'week': groupExpr = `DATE_FORMAT(DATE_SUB(${innerExpr}, INTERVAL WEEKDAY(${innerExpr}) DAY), '%Y-%m-%d')`; break;
370+
}
370371
}
371-
selectParts.push(`${groupExpr} AS \`group\``);
372+
const groupAlias = this.getGroupByResultAlias(groupByRule, index, groupByRules.length);
373+
groupExprs.push(groupExpr);
374+
groupAliases.push(groupAlias);
375+
selectParts.push(`${groupExpr} AS \`${groupAlias}\``);
372376
}
373377

374378
for (const [alias, rule] of Object.entries(aggregations)) {
375379
const f = `\`${rule.field}\``;
376380
switch (rule.operation) {
377381
case 'sum': selectParts.push(`SUM(${f}) AS \`${alias}\``); break;
378382
case 'count': selectParts.push(`COUNT(*) AS \`${alias}\``); break;
383+
case 'count_distinct': selectParts.push(`COUNT(DISTINCT ${f}) AS \`${alias}\``); break;
379384
case 'avg': selectParts.push(`AVG(${f}) AS \`${alias}\``); break;
380385
case 'min': selectParts.push(`MIN(${f}) AS \`${alias}\``); break;
381386
case 'max': selectParts.push(`MAX(${f}) AS \`${alias}\``); break;
@@ -389,10 +394,10 @@ class MysqlConnector extends AdminForthBaseConnector implements IAdminForthDataS
389394

390395
// Run non-median aggregations
391396
let rows: AggRow[] = [];
392-
const hasNonMedian = selectParts.length > (groupExpr ? 1 : 0);
397+
const hasNonMedian = selectParts.length > groupExprs.length;
393398
if (hasNonMedian) {
394399
let query = `SELECT ${selectParts.join(', ')} FROM \`${tableName}\` ${where}`;
395-
if (groupExpr) query += ` GROUP BY ${groupExpr} ORDER BY ${groupExpr} ASC`;
400+
if (groupExprs.length) query += ` GROUP BY ${groupExprs.join(', ')} ORDER BY ${groupExprs.join(', ')} ASC`;
396401
dbLogger.trace(`🪲📜 MySQL AGG Q: ${query} values: ${JSON.stringify(filterValues)}`);
397402
const [result] = await this.client.execute(query, filterValues);
398403
rows = result as AggRow[];
@@ -404,18 +409,20 @@ class MysqlConnector extends AdminForthBaseConnector implements IAdminForthDataS
404409
const nullGuard = where ? `${where} AND ${f} IS NOT NULL` : `WHERE ${f} IS NOT NULL`;
405410

406411
let medianQuery: string;
407-
if (groupExpr) {
412+
if (groupExprs.length) {
413+
const groupSelect = groupExprs.map((expr, index) => `${expr} AS \`${groupAliases[index]}\``).join(', ');
414+
const groupColumns = groupAliases.map(alias => `\`${alias}\``).join(', ');
408415
medianQuery = `
409-
SELECT \`group\`, AVG(${f}) AS \`${alias}\`
416+
SELECT ${groupColumns}, AVG(${f}) AS \`${alias}\`
410417
FROM (
411-
SELECT ${groupExpr} AS \`group\`, ${f},
412-
ROW_NUMBER() OVER (PARTITION BY ${groupExpr} ORDER BY ${f}) AS rn,
413-
COUNT(*) OVER (PARTITION BY ${groupExpr}) AS cnt
418+
SELECT ${groupSelect}, ${f},
419+
ROW_NUMBER() OVER (PARTITION BY ${groupExprs.join(', ')} ORDER BY ${f}) AS rn,
420+
COUNT(*) OVER (PARTITION BY ${groupExprs.join(', ')}) AS cnt
414421
FROM \`${tableName}\` ${nullGuard}
415422
) t
416423
WHERE rn IN (FLOOR((cnt + 1) / 2.0), CEIL((cnt + 1) / 2.0))
417-
GROUP BY \`group\`
418-
ORDER BY \`group\` ASC
424+
GROUP BY ${groupColumns}
425+
ORDER BY ${groupColumns} ASC
419426
`;
420427
} else {
421428
medianQuery = `
@@ -434,13 +441,14 @@ class MysqlConnector extends AdminForthBaseConnector implements IAdminForthDataS
434441
const [medianResult] = await this.client.execute(medianQuery, filterValues);
435442
const medianRows = medianResult as AggRow[];
436443

437-
if (groupExpr) {
444+
if (groupExprs.length) {
445+
const groupKey = (row: AggRow) => groupAliases.map(alias => String(row[alias])).join('\u0000');
438446
if (rows.length === 0) {
439-
rows = medianRows.map((r) => ({ group: r.group, [alias]: r[alias] }));
447+
rows = medianRows.map((r) => ({ ...Object.fromEntries(groupAliases.map(groupAlias => [groupAlias, r[groupAlias]])), [alias]: r[alias] }));
440448
} else {
441-
const byGroup = new Map(medianRows.map((r) => [String(r.group), r[alias]]));
449+
const byGroup = new Map(medianRows.map((r) => [groupKey(r), r[alias]]));
442450
for (const row of rows) {
443-
row[alias] = byGroup.get(String(row.group)) ?? null;
451+
row[alias] = byGroup.get(groupKey(row)) ?? null;
444452
}
445453
}
446454
} else {
@@ -557,4 +565,4 @@ class MysqlConnector extends AdminForthBaseConnector implements IAdminForthDataS
557565
}
558566
}
559567

560-
export default MysqlConnector;
568+
export default MysqlConnector;

0 commit comments

Comments
 (0)