From 3a8aab8f87ddfdfcd384e4879681e13754d6ea01 Mon Sep 17 00:00:00 2001 From: Benjamin Schubert Date: Sat, 24 Feb 2024 13:53:22 -0500 Subject: [PATCH 1/2] Add endpoint for converting a PDF from a URL --- .eslintrc | 3 +- README.md | 16 +++++- apps/server/src/utilities/file.service.ts | 49 +++++++++++++++++++ .../src/utilities/utilities.controller.ts | 22 +++++++++ 4 files changed, 88 insertions(+), 2 deletions(-) diff --git a/.eslintrc b/.eslintrc index ce5c8035..c186e3ad 100644 --- a/.eslintrc +++ b/.eslintrc @@ -55,7 +55,8 @@ { "singleQuote": true, "parser": "typescript", - "printWidth": 120 + "printWidth": 120, + "endOfLine": "auto" } ], diff --git a/README.md b/README.md index 16652e44..db466a12 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ $ npm install # Build all packages $ npm run build -# Start server, to use interfaces you need to build them beforehand +# Start server; some steps required the first time, see below $ npm run start # Wipe build/ and dist/ folders, build all packages, package to exe and copy dependencies/resources to build folder @@ -35,5 +35,19 @@ $ npm run install:exec ``` +## First time running with npm + +The server needs terrain data before it can run, so first do a full build: + +```bash +$ npm run build:exec +``` + +This will build the terrain data in `build/terrain/`. This needs to be symlinked to the top level before running with npm: + +```bash +mklink /J .\terrain .\build\terrain +``` + ## Documentation Start the server and direct to `localhost:8380/api` for API documentation diff --git a/apps/server/src/utilities/file.service.ts b/apps/server/src/utilities/file.service.ts index 0f4637b3..3aab9471 100644 --- a/apps/server/src/utilities/file.service.ts +++ b/apps/server/src/utilities/file.service.ts @@ -127,6 +127,55 @@ export class FileService { } } + async getConvertedPdfFileFromUrl(url: string, pageNumber: number, scale: number = 4): Promise { + // Some PDFs need external cmaps. + const CMAP_URL = `${join(getExecutablePath(), 'node_modules', 'pdfjs-dist', 'cmaps')}/`; + const CMAP_PACKED = true; + + // Where the standard fonts are located. + const STANDARD_FONT_DATA_URL = `${join(getExecutablePath(), 'node_modules', 'pdfjs-dist', 'standard_fonts')}/`; + + try { + const pngKey = `${url};;${pageNumber};;${scale}`; + if (this.pngCache.has(pngKey)) { + return new StreamableFile(this.pngCache.get(pngKey)); + } + + if (!this.pdfCache.has(url)) { + const resp = await fetch(url); + if (!resp.ok) { + throw new Error('encountered error retrieving PDF file'); + } + + const data = new Uint8Array(await resp.arrayBuffer()); + + // Load the PDF file. + const pdfDocument = await getDocument({ + data, + cMapUrl: CMAP_URL, + cMapPacked: CMAP_PACKED, + standardFontDataUrl: STANDARD_FONT_DATA_URL, + }).promise; + + this.pdfCache.set(url, pdfDocument); + } + + const file = this.pdfCache.get(url); + + const pngBuffer = await pdfToPng(file, pageNumber, scale); + + if (!this.pngCache.has(pngKey)) { + this.pngCache.set(pngKey, pngBuffer); + } + + return new StreamableFile(pngBuffer); + } catch (err) { + const message = `Error converting PDF to PNG: ${url}`; + this.logger.log(message, err); + throw new HttpException(message, HttpStatus.INTERNAL_SERVER_ERROR); + } + } + async getConvertedPdfFile( directory: string, fileName: string, diff --git a/apps/server/src/utilities/utilities.controller.ts b/apps/server/src/utilities/utilities.controller.ts index 19b013db..f9a36267 100644 --- a/apps/server/src/utilities/utilities.controller.ts +++ b/apps/server/src/utilities/utilities.controller.ts @@ -37,6 +37,28 @@ export class UtilityController { return convertedPdfFile; } + @Get('pdf/fromUrl') + @ApiResponse({ + status: 200, + description: 'A streamed converted png image', + type: StreamableFile, + }) + async getPdfFromUrl( + @Query('encodedUrl') encodedUrl: string, + @Query('pagenumber', ParseIntPipe) pagenumber: number, + @Response({ passthrough: true }) res, + ): Promise { + const url = decodeURIComponent(encodedUrl); + const convertedPdfFile = await this.fileService.getConvertedPdfFileFromUrl(`${url}`, pagenumber); + + res.set({ + 'Content-Type': 'image/png', + 'Content-Disposition': `attachment; filename=out-${pagenumber}.png`, + }); + + return convertedPdfFile; + } + @Get('pdf/list') @ApiResponse({ status: 200, From 423dafc0e2049b28d65817e2d0403424a9f868cd Mon Sep 17 00:00:00 2001 From: Benjamin Schubert Date: Sat, 24 Feb 2024 14:35:05 -0500 Subject: [PATCH 2/2] Add page number endpoint for URLs --- apps/server/src/utilities/file.service.ts | 53 +++++++++++-------- .../src/utilities/utilities.controller.ts | 11 ++++ 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/apps/server/src/utilities/file.service.ts b/apps/server/src/utilities/file.service.ts index 3aab9471..66b791ec 100644 --- a/apps/server/src/utilities/file.service.ts +++ b/apps/server/src/utilities/file.service.ts @@ -113,6 +113,11 @@ export class FileService { return getDocument({ data: retrievedFile }).promise.then((document) => document.numPages); } + async getNumberOfPdfPagesFromUrl(url: string): Promise { + const doc = await this.getPdfFromUrl(url); + return doc.numPages; + } + /** * Calling this function checks the safety of the supplied file path and throws an error if it deemed not safe against various potential attacks. * @param filePath @@ -127,7 +132,18 @@ export class FileService { } } - async getConvertedPdfFileFromUrl(url: string, pageNumber: number, scale: number = 4): Promise { + async getPdfFromUrl(url: string): Promise { + if (this.pdfCache.has(url)) { + return this.pdfCache.get(url); + } + + const resp = await fetch(url); + if (!resp.ok) { + throw new Error('encountered error retrieving PDF file'); + } + + const data = new Uint8Array(await resp.arrayBuffer()); + // Some PDFs need external cmaps. const CMAP_URL = `${join(getExecutablePath(), 'node_modules', 'pdfjs-dist', 'cmaps')}/`; const CMAP_PACKED = true; @@ -135,33 +151,26 @@ export class FileService { // Where the standard fonts are located. const STANDARD_FONT_DATA_URL = `${join(getExecutablePath(), 'node_modules', 'pdfjs-dist', 'standard_fonts')}/`; + // Load the PDF file. + const pdfDocument = await getDocument({ + data, + cMapUrl: CMAP_URL, + cMapPacked: CMAP_PACKED, + standardFontDataUrl: STANDARD_FONT_DATA_URL, + }).promise; + + this.pdfCache.set(url, pdfDocument); + return pdfDocument; + } + + async getConvertedPdfFileFromUrl(url: string, pageNumber: number, scale: number = 4): Promise { try { const pngKey = `${url};;${pageNumber};;${scale}`; if (this.pngCache.has(pngKey)) { return new StreamableFile(this.pngCache.get(pngKey)); } - if (!this.pdfCache.has(url)) { - const resp = await fetch(url); - if (!resp.ok) { - throw new Error('encountered error retrieving PDF file'); - } - - const data = new Uint8Array(await resp.arrayBuffer()); - - // Load the PDF file. - const pdfDocument = await getDocument({ - data, - cMapUrl: CMAP_URL, - cMapPacked: CMAP_PACKED, - standardFontDataUrl: STANDARD_FONT_DATA_URL, - }).promise; - - this.pdfCache.set(url, pdfDocument); - } - - const file = this.pdfCache.get(url); - + const file = await this.getPdfFromUrl(url); const pngBuffer = await pdfToPng(file, pageNumber, scale); if (!this.pngCache.has(pngKey)) { diff --git a/apps/server/src/utilities/utilities.controller.ts b/apps/server/src/utilities/utilities.controller.ts index f9a36267..863d8e2f 100644 --- a/apps/server/src/utilities/utilities.controller.ts +++ b/apps/server/src/utilities/utilities.controller.ts @@ -59,6 +59,17 @@ export class UtilityController { return convertedPdfFile; } + @Get('pdf/fromUrl/numpages') + @ApiResponse({ + status: 200, + description: 'Returns the number of pages in the pdf at the URL', + type: Number, + }) + async getNumberOfPagesFromUrl(@Query('encodedUrl') encodedUrl: string): Promise { + const url = decodeURIComponent(encodedUrl); + return this.fileService.getNumberOfPdfPagesFromUrl(url); + } + @Get('pdf/list') @ApiResponse({ status: 200,