import { LogService } from '../abstractions/log.service'; import { Utils } from '../misc/utils'; import { EncArrayBuffer } from '../models/domain/encArrayBuffer'; const MAX_SINGLE_BLOB_UPLOAD_SIZE = 256 * 1024 * 1024; // 256 MiB const MAX_BLOCKS_PER_BLOB = 50000; export class AzureFileUploadService { constructor(private logService: LogService) { } async upload(url: string, data: EncArrayBuffer, renewalCallback: () => Promise) { if (data.buffer.byteLength <= MAX_SINGLE_BLOB_UPLOAD_SIZE) { return await this.azureUploadBlob(url, data); } else { return await this.azureUploadBlocks(url, data, renewalCallback); } } private async azureUploadBlob(url: string, data: EncArrayBuffer) { const urlObject = Utils.getUrl(url); const headers = new Headers({ 'x-ms-date': new Date().toUTCString(), 'x-ms-version': urlObject.searchParams.get('sv'), 'Content-Length': data.buffer.byteLength.toString(), 'x-ms-blob-type': 'BlockBlob', }); const request = new Request(url, { body: data.buffer, cache: 'no-store', method: 'PUT', headers: headers, }); const blobResponse = await fetch(request); if (blobResponse.status !== 201) { throw new Error(`Failed to create Azure blob: ${blobResponse.status}`); } } private async azureUploadBlocks(url: string, data: EncArrayBuffer, renewalCallback: () => Promise) { const baseUrl = Utils.getUrl(url); const blockSize = this.getMaxBlockSize(baseUrl.searchParams.get('sv')); let blockIndex = 0; const numBlocks = Math.ceil(data.buffer.byteLength / blockSize); const blocksStaged: string[] = []; if (numBlocks > MAX_BLOCKS_PER_BLOB) { throw new Error(`Cannot upload file, exceeds maximum size of ${blockSize * MAX_BLOCKS_PER_BLOB}`); } try { while (blockIndex < numBlocks) { url = await this.renewUrlIfNecessary(url, renewalCallback); const blockUrl = Utils.getUrl(url); const blockId = this.encodedBlockId(blockIndex); blockUrl.searchParams.append('comp', 'block'); blockUrl.searchParams.append('blockid', blockId); const start = blockIndex * blockSize; const blockData = data.buffer.slice(start, start + blockSize); const blockHeaders = new Headers({ 'x-ms-date': new Date().toUTCString(), 'x-ms-version': blockUrl.searchParams.get('sv'), 'Content-Length': blockData.byteLength.toString(), }); const blockRequest = new Request(blockUrl.toString(), { body: blockData, cache: 'no-store', method: 'PUT', headers: blockHeaders, }); const blockResponse = await fetch(blockRequest); if (blockResponse.status !== 201) { const message = `Unsuccessful block PUT. Received status ${blockResponse.status}`; this.logService.error(message + '\n' + await blockResponse.json()); throw new Error(message); } blocksStaged.push(blockId); blockIndex++; } url = await this.renewUrlIfNecessary(url, renewalCallback); const blockListUrl = Utils.getUrl(url); const blockListXml = this.blockListXml(blocksStaged); blockListUrl.searchParams.append('comp', 'blocklist'); const headers = new Headers({ 'x-ms-date': new Date().toUTCString(), 'x-ms-version': blockListUrl.searchParams.get('sv'), 'Content-Length': blockListXml.length.toString(), }); const request = new Request(blockListUrl.toString(), { body: blockListXml, cache: 'no-store', method: 'PUT', headers: headers, }); const response = await fetch(request); if (response.status !== 201) { const message = `Unsuccessful block list PUT. Received status ${response.status}`; this.logService.error(message + '\n' + await response.json()); throw new Error(message); } } catch (e) { throw e; } } private async renewUrlIfNecessary(url: string, renewalCallback: () => Promise): Promise { const urlObject = Utils.getUrl(url); const expiry = new Date(urlObject.searchParams.get('se') ?? ''); if (isNaN(expiry.getTime())) { expiry.setTime(Date.now() + 3600000); } if (expiry.getTime() < Date.now() + 1000) { return await renewalCallback(); } return url; } private encodedBlockId(blockIndex: number) { // Encoded blockId max size is 64, so pre-encoding max size is 48 const utfBlockId = ('000000000000000000000000000000000000000000000000' + blockIndex.toString()).slice(-48); return Utils.fromUtf8ToB64(utfBlockId); } private blockListXml(blockIdList: string[]) { let xml = ''; blockIdList.forEach(blockId => { xml += `${blockId}`; }); xml += ''; return xml; } private getMaxBlockSize(version: string) { if (Version.compare(version, '2019-12-12') >= 0) { return 4000 * 1024 * 1024; // 4000 MiB } else if (Version.compare(version, '2016-05-31') >= 0) { return 100 * 1024 * 1024; // 100 MiB } else { return 4 * 1024 * 1024; // 4 MiB } } } class Version { /** * Compares two Azure Versions against each other * @param a Version to compare * @param b Version to compare * @returns a number less than zero if b is newer than a, 0 if equal, * and greater than zero if a is newer than b */ static compare(a: Required | string, b: Required | string) { if (typeof (a) === 'string') { a = new Version(a); } if (typeof (b) === 'string') { b = new Version(b); } return a.year !== b.year ? a.year - b.year : a.month !== b.month ? a.month - b.month : a.day !== b.day ? a.day - b.day : 0; } year = 0; month = 0; day = 0; constructor(version: string) { try { const parts = version.split('-').map(v => Number.parseInt(v, 10)); this.year = parts[0]; this.month = parts[1]; this.day = parts[2]; } catch { // Ignore error } } /** * Compares two Azure Versions against each other * @param compareTo Version to compare against * @returns a number less than zero if compareTo is newer, 0 if equal, * and greater than zero if this is greater than compareTo */ compare(compareTo: Required | string) { return Version.compare(this, compareTo); } }