Merge pull request #111 from Canner/feature/implement-offset-pagination

Feature: Implement offset and limit of data sources

Author: kokokuo

labs/playground1/sqls/artist/works.yaml

@@ -10,3 +10,5 @@ sample:
     id: '1'
   profile: duck
 profile: duck
+pagination:
+  mode: offset

packages/build/src/lib/document-generator/spec-generator/oas3/oas3SpecGenerator.ts

@@ -89,6 +89,7 @@ export class OAS3SpecGenerator extends SpecGenerator<oas3.OpenAPIObject> {
         in: this.convertFieldInTypeToOASIn(param.fieldIn),
         schema: this.getSchemaObjectFromParameter(param),
         required: this.isParameterRequired(param),
+        description: param.description,
       });
     }
 

packages/build/src/lib/schema-parser/middleware/extractPaginationParams.ts

@@ -0,0 +1,50 @@
+import {
+  APISchema,
+  FieldDataType,
+  FieldInType,
+  PaginationMode,
+  RequestSchema,
+} from '@vulcan-sql/core';
+import { RawAPISchema, SchemaParserMiddleware } from './middleware';
+
+export class ExtractPaginationParams extends SchemaParserMiddleware {
+  public async handle(schemas: RawAPISchema, next: () => Promise<void>) {
+    await next();
+    const transformedSchemas = schemas as APISchema;
+    const paginationParameters: RequestSchema[] = [];
+    if (transformedSchemas.pagination?.mode === PaginationMode.OFFSET) {
+      paginationParameters.push({
+        fieldName: 'limit',
+        fieldIn: FieldInType.QUERY,
+        description:
+          'Offset-based Pagination: The maximum number of rows to return. default: 20',
+        type: FieldDataType.STRING, // We should use STRING type here but not NUMBER because of the issue from normalizeStringValue function (it throw error when input is undefined)
+        validators: [{ name: 'integer', args: { min: 0 } }],
+        constraints: [],
+      });
+      paginationParameters.push({
+        fieldName: 'offset',
+        fieldIn: FieldInType.QUERY,
+        description:
+          'Offset-based Pagination: The offset from the row. default: 0',
+        type: FieldDataType.STRING,
+        validators: [{ name: 'integer', args: { min: 0 } }],
+        constraints: [],
+      });
+    }
+
+    // merge parameters
+    for (const param of paginationParameters) {
+      const existed = transformedSchemas.request.find(
+        (req) =>
+          req.fieldName === param.fieldName && req.fieldIn === param.fieldIn
+      );
+      if (existed) {
+        existed.description = existed.description || param.description;
+        existed.validators = [...existed.validators, ...param.validators];
+      } else {
+        transformedSchemas.request.push(param);
+      }
+    }
+  }
+}

packages/build/src/lib/schema-parser/middleware/index.ts

@@ -15,6 +15,8 @@ import { SetConstraints } from './setConstraints';
 import { SchemaParserMiddleware } from './middleware';
 import { ResponseSampler } from './responseSampler';
 import { CheckProfile } from './checkProfile';
+import { ExtractPaginationParams } from './extractPaginationParams';
+import { TransformPaginationMode } from './transformPaginationMode';
 
 export * from './middleware';
 
@@ -32,7 +34,9 @@ export const SchemaParserMiddlewares: ClassType<SchemaParserMiddleware>[] = [
   NormalizeDataType,
   GeneratePathParameters,
   AddRequiredValidatorForPath,
+  TransformPaginationMode,
   SetConstraints,
+  ExtractPaginationParams, // ExtractPaginationParams should be loaded after SetConstraints
   ResponseSampler,
   CheckProfile,
 ];

packages/build/src/lib/schema-parser/middleware/responseSampler.ts

@@ -38,9 +38,10 @@ export class ResponseSampler extends SchemaParserMiddleware {
         parameters: schema.sample.parameters,
         profileName: schema.sample.profile,
       },
-      // We only need the columns of this query, so we set offset/limit both to 0 here.
+      // We only need the columns of this query, so we set offset=0 and limit=1 here.
+      // Some drivers guess the column types by data, so we should at least query 1 rows.
       {
-        limit: 0,
+        limit: 1,
         offset: 0,
       }
     );

packages/build/src/lib/schema-parser/middleware/transformPaginationMode.ts

@@ -0,0 +1,12 @@
+import { PaginationMode } from '@vulcan-sql/core';
+import { RawAPISchema, SchemaParserMiddleware } from './middleware';
+
+// pagination.mode: offset -> OFFSET
+export class TransformPaginationMode extends SchemaParserMiddleware {
+  public async handle(schemas: RawAPISchema, next: () => Promise<void>) {
+    if (schemas.pagination?.mode)
+      schemas.pagination.mode =
+        schemas.pagination.mode.toUpperCase() as PaginationMode;
+    return next();
+  }
+}

packages/build/test/schema-parser/middleware/extractPaginationParams.spec.ts

@@ -0,0 +1,138 @@
+import { RawAPISchema } from '@vulcan-sql/build/schema-parser';
+import { ExtractPaginationParams } from '@vulcan-sql/build/schema-parser/middleware/extractPaginationParams';
+import {
+  Constraint,
+  FieldDataType,
+  FieldInType,
+  PaginationMode,
+} from '@vulcan-sql/core';
+
+it('Should do nothing when there is no pagination mode configured', async () => {
+  // Arrange
+  const schema: RawAPISchema = {
+    sourceName: 'some-name',
+  };
+  const extractPaginationParams = new ExtractPaginationParams();
+  // Act
+  await extractPaginationParams.handle(schema, async () => Promise.resolve());
+  // Assert
+  expect(schema).toEqual(schema);
+});
+
+it('Should add limit and offset when pagination mode is OFFSET', async () => {
+  // Arrange
+  const schema: RawAPISchema = {
+    sourceName: 'some-name',
+    request: [],
+    pagination: {
+      mode: PaginationMode.OFFSET,
+    },
+  };
+  const extractPaginationParams = new ExtractPaginationParams();
+  // Act
+  await extractPaginationParams.handle(schema, async () => Promise.resolve());
+  // Assert
+  expect(schema.request?.length).toBe(2);
+  expect(schema.request).toContainEqual(
+    expect.objectContaining({
+      fieldName: 'limit',
+      fieldIn: FieldInType.QUERY,
+      description:
+        'Offset-based Pagination: The maximum number of rows to return. default: 20',
+    })
+  );
+  expect(schema.request).toContainEqual(
+    expect.objectContaining({
+      fieldName: 'offset',
+      fieldIn: FieldInType.QUERY,
+      description:
+        'Offset-based Pagination: The offset from the row. default: 0',
+    })
+  );
+});
+
+it('Should merge parameters when some parameters have been defined', async () => {
+  // Arrange
+  const schema: RawAPISchema = {
+    sourceName: 'some-name',
+    request: [
+      {
+        fieldName: 'limit',
+        fieldIn: FieldInType.QUERY,
+        description: 'Existed one',
+        type: FieldDataType.STRING,
+        validators: [{ name: 'integer', args: { min: -4 } }],
+        constraints: [],
+      },
+    ],
+    pagination: {
+      mode: PaginationMode.OFFSET,
+    },
+  };
+  const extractPaginationParams = new ExtractPaginationParams();
+  // Act
+  await extractPaginationParams.handle(schema, async () => Promise.resolve());
+  // Assert
+  expect(schema.request?.length).toBe(2);
+  expect(schema.request).toContainEqual(
+    expect.objectContaining({
+      fieldName: 'limit',
+      fieldIn: FieldInType.QUERY,
+      description: 'Existed one',
+      validators: [
+        { name: 'integer', args: { min: -4 } },
+        { name: 'integer', args: { min: 0 } },
+      ],
+    })
+  );
+  expect(schema.request).toContainEqual(
+    expect.objectContaining({
+      fieldName: 'offset',
+      fieldIn: FieldInType.QUERY,
+      description:
+        'Offset-based Pagination: The offset from the row. default: 0',
+    })
+  );
+});
+
+it('Should override the description of parameters when the existed one is empty', async () => {
+  // Arrange
+  const schema: RawAPISchema = {
+    sourceName: 'some-name',
+    request: [
+      {
+        fieldName: 'limit',
+        fieldIn: FieldInType.QUERY,
+        description: '',
+        type: FieldDataType.STRING,
+        validators: [],
+        constraints: [Constraint.Required()],
+      },
+    ],
+    pagination: {
+      mode: PaginationMode.OFFSET,
+    },
+  };
+  const extractPaginationParams = new ExtractPaginationParams();
+  // Act
+  await extractPaginationParams.handle(schema, async () => Promise.resolve());
+  // Assert
+  expect(schema.request?.length).toBe(2);
+  expect(schema.request).toContainEqual(
+    expect.objectContaining({
+      fieldName: 'limit',
+      fieldIn: FieldInType.QUERY,
+      description:
+        'Offset-based Pagination: The maximum number of rows to return. default: 20',
+      constraints: [Constraint.Required()],
+    })
+  );
+  expect(schema.request).toContainEqual(
+    expect.objectContaining({
+      fieldName: 'offset',
+      fieldIn: FieldInType.QUERY,
+      description:
+        'Offset-based Pagination: The offset from the row. default: 0',
+    })
+  );
+});

packages/build/test/schema-parser/middleware/transformPaginationMode.spec.ts

@@ -0,0 +1,30 @@
+import { RawAPISchema } from '@vulcan-sql/build/schema-parser';
+import { TransformPaginationMode } from '@vulcan-sql/build/schema-parser/middleware/transformPaginationMode';
+import { PaginationMode } from '@vulcan-sql/core';
+
+it('Should do nothing when there is no pagination mode configured', async () => {
+  // Arrange
+  const schema: RawAPISchema = {
+    sourceName: 'some-name',
+  };
+  const transformPaginationMode = new TransformPaginationMode();
+  // Act
+  await transformPaginationMode.handle(schema, async () => Promise.resolve());
+  // Assert
+  expect(schema).toEqual(schema);
+});
+
+it('Should transform pagination mode when it has been configured', async () => {
+  // Arrange
+  const schema: RawAPISchema = {
+    sourceName: 'some-name',
+    pagination: {
+      mode: 'oFfSeT' as any,
+    },
+  };
+  const transformPaginationMode = new TransformPaginationMode();
+  // Act
+  await transformPaginationMode.handle(schema, async () => Promise.resolve());
+  // Assert
+  expect(schema.pagination?.mode).toEqual(PaginationMode.OFFSET);
+});