From c6689e0ad8db8fd86c076d4fcb5485aba5d51151 Mon Sep 17 00:00:00 2001 From: zkldi <20380519+zkldi@users.noreply.github.com> Date: Fri, 12 Nov 2021 17:38:38 +0000 Subject: [PATCH] CDN refactor to support S3> --- server/.github/test.conf.json5 | 10 +- server/package.json | 1 + server/pnpm-lock.yaml | 789 +++++++++++++++++++++- server/src/lib/cdn/cdn.test.ts | 109 ++- server/src/lib/cdn/cdn.ts | 76 +-- server/src/lib/cdn/s3.ts | 64 ++ server/src/lib/setup/config.ts | 41 +- server/src/server/router/ir/usc/router.ts | 7 +- server/src/server/server.ts | 13 +- server/src/test-utils/resets.ts | 20 +- 10 files changed, 1005 insertions(+), 125 deletions(-) create mode 100644 server/src/lib/cdn/s3.ts diff --git a/server/.github/test.conf.json5 b/server/.github/test.conf.json5 index 276f524d8..c134c4fbd 100644 --- a/server/.github/test.conf.json5 +++ b/server/.github/test.conf.json5 @@ -20,14 +20,20 @@ }, ARC_AUTH_TOKEN: "unused", ENABLE_SERVER_HTTPS: false, - RUN_OWN_CDN: true, - CDN_WEB_LOCATION: "/cdn", OUR_URL: "https://example.com", INVITE_CODE_CONFIG: { BATCH_SIZE: 2, INVITE_CAP: 100, BETA_USER_BONUS: 5, }, + CDN_CONFIG: { + WEB_LOCATION: "/cdn", + SAVE_LOCATION: { + TYPE: "LOCAL_FILESYSTEM", + LOCATION: "./test-cdn", + SERVE_OWN_CDN: true, + }, + }, TACHI_CONFIG: { TYPE: "omni", NAME: "Tachi Testing", diff --git a/server/package.json b/server/package.json index f4a9171f8..34602e3de 100644 --- a/server/package.json +++ b/server/package.json @@ -48,6 +48,7 @@ "why-is-node-running": "2.2.0" }, "dependencies": { + "@aws-sdk/client-s3": "^3.41.0", "@types/bull": "^3.15.5", "@types/bunyan": "^1.8.7", "@types/deep-equal": "^1.0.1", diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index fbfee0755..fcabafe06 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -1,6 +1,7 @@ lockfileVersion: 5.3 specifiers: + '@aws-sdk/client-s3': ^3.41.0 '@types/bcrypt': 5.0.0 '@types/bcryptjs': ^2.4.2 '@types/bull': ^3.15.5 @@ -73,6 +74,7 @@ specifiers: winston-transport: ^4.4.0 dependencies: + '@aws-sdk/client-s3': 3.41.0 '@types/bull': 3.15.5 '@types/bunyan': 1.8.7 '@types/deep-equal': 1.0.1 @@ -148,6 +150,780 @@ devDependencies: packages: + /@aws-crypto/crc32/2.0.0: + resolution: {integrity: sha512-TvE1r2CUueyXOuHdEigYjIZVesInd9KN+K/TFFNfkkxRThiNxO6i4ZqqAVMoEjAamZZ1AA8WXJkjCz7YShHPQA==} + dependencies: + '@aws-crypto/util': 2.0.0 + '@aws-sdk/types': 3.40.0 + tslib: 1.14.1 + dev: false + + /@aws-crypto/ie11-detection/2.0.0: + resolution: {integrity: sha512-pkVXf/dq6PITJ0jzYZ69VhL8VFOFoPZLZqtU/12SGnzYuJOOGNfF41q9GxdI1yqC8R13Rq3jOLKDFpUJFT5eTA==} + dependencies: + tslib: 1.14.1 + dev: false + + /@aws-crypto/sha256-browser/2.0.0: + resolution: {integrity: sha512-rYXOQ8BFOaqMEHJrLHul/25ckWH6GTJtdLSajhlqGMx0PmSueAuvboCuZCTqEKlxR8CQOwRarxYMZZSYlhRA1A==} + dependencies: + '@aws-crypto/ie11-detection': 2.0.0 + '@aws-crypto/sha256-js': 2.0.0 + '@aws-crypto/supports-web-crypto': 2.0.0 + '@aws-crypto/util': 2.0.0 + '@aws-sdk/types': 3.40.0 + '@aws-sdk/util-locate-window': 3.37.0 + '@aws-sdk/util-utf8-browser': 3.37.0 + tslib: 1.14.1 + dev: false + + /@aws-crypto/sha256-js/2.0.0: + resolution: {integrity: sha512-VZY+mCY4Nmrs5WGfitmNqXzaE873fcIZDu54cbaDaaamsaTOP1DBImV9F4pICc3EHjQXujyE8jig+PFCaew9ig==} + dependencies: + '@aws-crypto/util': 2.0.0 + '@aws-sdk/types': 3.40.0 + tslib: 1.14.1 + dev: false + + /@aws-crypto/supports-web-crypto/2.0.0: + resolution: {integrity: sha512-Ge7WQ3E0OC7FHYprsZV3h0QIcpdyJLvIeg+uTuHqRYm8D6qCFJoiC+edSzSyFiHtZf+NOQDJ1q46qxjtzIY2nA==} + dependencies: + tslib: 1.14.1 + dev: false + + /@aws-crypto/util/2.0.0: + resolution: {integrity: sha512-YDooyH83m2P5A3h6lNH7hm6mIP93sU/dtzRmXIgtO4BCB7SvtX8ysVKQAE8tVky2DQ3HHxPCjNTuUe7YoAMrNQ==} + dependencies: + '@aws-sdk/types': 3.40.0 + '@aws-sdk/util-utf8-browser': 3.37.0 + tslib: 1.14.1 + dev: false + + /@aws-sdk/abort-controller/3.40.0: + resolution: {integrity: sha512-S7LzLvNuwuf0q7r4q7zqGzxd/W2xYsn8cpZ90MMb3ObolhbkLySrikUJujmXae8k+2/KFCOr+FVC0YLrATSUgQ==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/chunked-blob-reader-native/3.37.0: + resolution: {integrity: sha512-h9OYq6EvDrpb7SKod+Kow+d3aRNFVBYR1a8G8ahEDDQe3AtmA2Smyvni4kt/ABTiKvYdof2//Pq3BL/IUV9n9Q==} + dependencies: + '@aws-sdk/util-base64-browser': 3.37.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/chunked-blob-reader/3.37.0: + resolution: {integrity: sha512-uDacnFaczeO962RnVttwAQddS4rgDfI7nfeY8NV6iZkDv5uxGzHTfH4jT7WvPDM1pSMcOMDx8RJ+Tmtsd1VTsA==} + dependencies: + tslib: 2.3.1 + dev: false + + /@aws-sdk/client-s3/3.41.0: + resolution: {integrity: sha512-PGPy7MbyjkpQECPR02NOxucqJWJEah9IcQcUMyRB1vZBg6irBI1d4GTuWafegr3xKQ+7ZTraupDtUSJ9oRqKOg==} + engines: {node: '>=10.0.0'} + dependencies: + '@aws-crypto/sha256-browser': 2.0.0 + '@aws-crypto/sha256-js': 2.0.0 + '@aws-sdk/client-sts': 3.41.0 + '@aws-sdk/config-resolver': 3.40.0 + '@aws-sdk/credential-provider-node': 3.41.0 + '@aws-sdk/eventstream-serde-browser': 3.40.0 + '@aws-sdk/eventstream-serde-config-resolver': 3.40.0 + '@aws-sdk/eventstream-serde-node': 3.40.0 + '@aws-sdk/fetch-http-handler': 3.40.0 + '@aws-sdk/hash-blob-browser': 3.40.0 + '@aws-sdk/hash-node': 3.40.0 + '@aws-sdk/hash-stream-node': 3.40.0 + '@aws-sdk/invalid-dependency': 3.40.0 + '@aws-sdk/md5-js': 3.40.0 + '@aws-sdk/middleware-apply-body-checksum': 3.40.0 + '@aws-sdk/middleware-bucket-endpoint': 3.41.0 + '@aws-sdk/middleware-content-length': 3.40.0 + '@aws-sdk/middleware-expect-continue': 3.40.0 + '@aws-sdk/middleware-host-header': 3.40.0 + '@aws-sdk/middleware-location-constraint': 3.40.0 + '@aws-sdk/middleware-logger': 3.40.0 + '@aws-sdk/middleware-retry': 3.40.0 + '@aws-sdk/middleware-sdk-s3': 3.41.0 + '@aws-sdk/middleware-serde': 3.40.0 + '@aws-sdk/middleware-signing': 3.40.0 + '@aws-sdk/middleware-ssec': 3.40.0 + '@aws-sdk/middleware-stack': 3.40.0 + '@aws-sdk/middleware-user-agent': 3.40.0 + '@aws-sdk/node-config-provider': 3.40.0 + '@aws-sdk/node-http-handler': 3.40.0 + '@aws-sdk/protocol-http': 3.40.0 + '@aws-sdk/smithy-client': 3.41.0 + '@aws-sdk/types': 3.40.0 + '@aws-sdk/url-parser': 3.40.0 + '@aws-sdk/util-base64-browser': 3.37.0 + '@aws-sdk/util-base64-node': 3.37.0 + '@aws-sdk/util-body-length-browser': 3.37.0 + '@aws-sdk/util-body-length-node': 3.37.0 + '@aws-sdk/util-user-agent-browser': 3.40.0 + '@aws-sdk/util-user-agent-node': 3.40.0 + '@aws-sdk/util-utf8-browser': 3.37.0 + '@aws-sdk/util-utf8-node': 3.37.0 + '@aws-sdk/util-waiter': 3.40.0 + '@aws-sdk/xml-builder': 3.37.0 + entities: 2.2.0 + fast-xml-parser: 3.19.0 + tslib: 2.3.1 + transitivePeerDependencies: + - '@aws-sdk/signature-v4-crt' + dev: false + + /@aws-sdk/client-sso/3.41.0: + resolution: {integrity: sha512-xDvcy7wv3KdHhOpl5fZN+Ydw+dHBmsCZwMFI1ZdJVCSGO+ZKgl5KVWi1LCif6vjZP1pUuGl44oDOZz1ACqOzTg==} + engines: {node: '>=10.0.0'} + dependencies: + '@aws-crypto/sha256-browser': 2.0.0 + '@aws-crypto/sha256-js': 2.0.0 + '@aws-sdk/config-resolver': 3.40.0 + '@aws-sdk/fetch-http-handler': 3.40.0 + '@aws-sdk/hash-node': 3.40.0 + '@aws-sdk/invalid-dependency': 3.40.0 + '@aws-sdk/middleware-content-length': 3.40.0 + '@aws-sdk/middleware-host-header': 3.40.0 + '@aws-sdk/middleware-logger': 3.40.0 + '@aws-sdk/middleware-retry': 3.40.0 + '@aws-sdk/middleware-serde': 3.40.0 + '@aws-sdk/middleware-stack': 3.40.0 + '@aws-sdk/middleware-user-agent': 3.40.0 + '@aws-sdk/node-config-provider': 3.40.0 + '@aws-sdk/node-http-handler': 3.40.0 + '@aws-sdk/protocol-http': 3.40.0 + '@aws-sdk/smithy-client': 3.41.0 + '@aws-sdk/types': 3.40.0 + '@aws-sdk/url-parser': 3.40.0 + '@aws-sdk/util-base64-browser': 3.37.0 + '@aws-sdk/util-base64-node': 3.37.0 + '@aws-sdk/util-body-length-browser': 3.37.0 + '@aws-sdk/util-body-length-node': 3.37.0 + '@aws-sdk/util-user-agent-browser': 3.40.0 + '@aws-sdk/util-user-agent-node': 3.40.0 + '@aws-sdk/util-utf8-browser': 3.37.0 + '@aws-sdk/util-utf8-node': 3.37.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/client-sts/3.41.0: + resolution: {integrity: sha512-XTjmr53kMbXuVhH3B+g2jEYuhNralptsMSd4RcSHCB7BX1NmAMnMFKKTmVlmc5NizWi4x1CzExu86Q0YSqp0og==} + engines: {node: '>=10.0.0'} + dependencies: + '@aws-crypto/sha256-browser': 2.0.0 + '@aws-crypto/sha256-js': 2.0.0 + '@aws-sdk/config-resolver': 3.40.0 + '@aws-sdk/credential-provider-node': 3.41.0 + '@aws-sdk/fetch-http-handler': 3.40.0 + '@aws-sdk/hash-node': 3.40.0 + '@aws-sdk/invalid-dependency': 3.40.0 + '@aws-sdk/middleware-content-length': 3.40.0 + '@aws-sdk/middleware-host-header': 3.40.0 + '@aws-sdk/middleware-logger': 3.40.0 + '@aws-sdk/middleware-retry': 3.40.0 + '@aws-sdk/middleware-sdk-sts': 3.40.0 + '@aws-sdk/middleware-serde': 3.40.0 + '@aws-sdk/middleware-signing': 3.40.0 + '@aws-sdk/middleware-stack': 3.40.0 + '@aws-sdk/middleware-user-agent': 3.40.0 + '@aws-sdk/node-config-provider': 3.40.0 + '@aws-sdk/node-http-handler': 3.40.0 + '@aws-sdk/protocol-http': 3.40.0 + '@aws-sdk/smithy-client': 3.41.0 + '@aws-sdk/types': 3.40.0 + '@aws-sdk/url-parser': 3.40.0 + '@aws-sdk/util-base64-browser': 3.37.0 + '@aws-sdk/util-base64-node': 3.37.0 + '@aws-sdk/util-body-length-browser': 3.37.0 + '@aws-sdk/util-body-length-node': 3.37.0 + '@aws-sdk/util-user-agent-browser': 3.40.0 + '@aws-sdk/util-user-agent-node': 3.40.0 + '@aws-sdk/util-utf8-browser': 3.37.0 + '@aws-sdk/util-utf8-node': 3.37.0 + entities: 2.2.0 + fast-xml-parser: 3.19.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/config-resolver/3.40.0: + resolution: {integrity: sha512-QYy6J2k31QL6J74hPBfptnLW1kQYdN+xjwH4UQ1mv7EUhRoJN9ZY2soStJowFy4at6IIOOVWbyG5dyqvrbEovg==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/signature-v4': 3.40.0 + '@aws-sdk/types': 3.40.0 + '@aws-sdk/util-config-provider': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/credential-provider-env/3.40.0: + resolution: {integrity: sha512-qHZdf2vxhzZkSygjw2I4SEYFL2dMZxxYvO4QlkqQouKY81OVxs/j69oiNCjPasQzGz5jaZZKI8xEAIfkSyr1lg==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/property-provider': 3.40.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/credential-provider-imds/3.40.0: + resolution: {integrity: sha512-Ty/wVa+BQrCFrP06AGl5S1CeLifDt68YrlYXUnkRn603SX4DvxBgVO7XFeDH58G8ziDCiqxfmVl4yjbncPPeSw==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/node-config-provider': 3.40.0 + '@aws-sdk/property-provider': 3.40.0 + '@aws-sdk/types': 3.40.0 + '@aws-sdk/url-parser': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/credential-provider-ini/3.41.0: + resolution: {integrity: sha512-98CGEHg7Tb6HxK5ZIdbAcijvD3IpLe0ddse1xMe/Ilhjz770FS/L2UNprOP6PZTqrSfBffiMrvfThUSuUaTlIQ==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/credential-provider-env': 3.40.0 + '@aws-sdk/credential-provider-imds': 3.40.0 + '@aws-sdk/credential-provider-sso': 3.41.0 + '@aws-sdk/credential-provider-web-identity': 3.41.0 + '@aws-sdk/property-provider': 3.40.0 + '@aws-sdk/shared-ini-file-loader': 3.37.0 + '@aws-sdk/types': 3.40.0 + '@aws-sdk/util-credentials': 3.37.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/credential-provider-node/3.41.0: + resolution: {integrity: sha512-5FW6+wNJgyDCsbAd+mLm/1DBTDkyIYOMVzcxbr6Vi3pM4UrMFdeLdAP62edYW8usg78Xg+c6vaAoEv/M3zkS0Q==} + engines: {node: '>=10.0.0'} + dependencies: + '@aws-sdk/credential-provider-env': 3.40.0 + '@aws-sdk/credential-provider-imds': 3.40.0 + '@aws-sdk/credential-provider-ini': 3.41.0 + '@aws-sdk/credential-provider-process': 3.40.0 + '@aws-sdk/credential-provider-sso': 3.41.0 + '@aws-sdk/credential-provider-web-identity': 3.41.0 + '@aws-sdk/property-provider': 3.40.0 + '@aws-sdk/shared-ini-file-loader': 3.37.0 + '@aws-sdk/types': 3.40.0 + '@aws-sdk/util-credentials': 3.37.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/credential-provider-process/3.40.0: + resolution: {integrity: sha512-qsaNCDesW2GasDbzpeOA371gxugi05JWxt3EKonLbUfkGKBK7kmmL6EgLIxZuNm2/Ve4RS07PKp8yBGm4xIx9w==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/property-provider': 3.40.0 + '@aws-sdk/shared-ini-file-loader': 3.37.0 + '@aws-sdk/types': 3.40.0 + '@aws-sdk/util-credentials': 3.37.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/credential-provider-sso/3.41.0: + resolution: {integrity: sha512-9s7SWu3RVIQ/MTcBCt35EMzxNQm3avivrbpSOKfJwxR5L+oNKPsV+gSqMlkNZGwOVJyUicIsZGcq/4ON6CjrOg==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/client-sso': 3.41.0 + '@aws-sdk/property-provider': 3.40.0 + '@aws-sdk/shared-ini-file-loader': 3.37.0 + '@aws-sdk/types': 3.40.0 + '@aws-sdk/util-credentials': 3.37.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/credential-provider-web-identity/3.41.0: + resolution: {integrity: sha512-VqvVoEh9C8xTXl4stKyJC5IKQhS8g1Gi5k6B9HPHLIxFRRfKxkE73DT4pMN6npnus7o0yi0MTFGQFQGYSrFO2g==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/property-provider': 3.40.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/eventstream-marshaller/3.40.0: + resolution: {integrity: sha512-zHGchfkG3B9M8OOKRpByeS5g1/15YQ0+QQHwxQRtm/CPtKBAIAsCZRQaCNBLu9uQMtBBKj5JsDUcjirhGeSvIg==} + dependencies: + '@aws-crypto/crc32': 2.0.0 + '@aws-sdk/types': 3.40.0 + '@aws-sdk/util-hex-encoding': 3.37.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/eventstream-serde-browser/3.40.0: + resolution: {integrity: sha512-V0AXAfSkhY0hgxDJ0cNA+r42kL8295U7UTCp2Q2fvCaob3wKWh+54KZ2L4IOYTlK3yNzXJ5V6PP1zUuRlsUTew==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/eventstream-marshaller': 3.40.0 + '@aws-sdk/eventstream-serde-universal': 3.40.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/eventstream-serde-config-resolver/3.40.0: + resolution: {integrity: sha512-GgGiJBsQ1/SBTpRM/wCdFBCMo1Nybvy46bNVkH1ujCdp8UTLc5PozzNpH+15V2IQbc9sPDYffMab6HSFjDp5vw==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/eventstream-serde-node/3.40.0: + resolution: {integrity: sha512-CnzX/JZGvhWlg+ooIPVZ78T+5wIm5Ld1BD7jwhlptJa8IjTMvkc8Nh4pAhc7T0ZScy4zZa/oTkqeVYCOVCyd1Q==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/eventstream-marshaller': 3.40.0 + '@aws-sdk/eventstream-serde-universal': 3.40.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/eventstream-serde-universal/3.40.0: + resolution: {integrity: sha512-rkHwVMyZJMhp9iBixkuaAGQNer/DPxZ9kxDDtE+LuAMhepTYQ8c4lUW0QQhYbNMWf48QKD1G4FV3JXIj9JfP9A==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/eventstream-marshaller': 3.40.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/fetch-http-handler/3.40.0: + resolution: {integrity: sha512-w1HiZromoU+/bbEo89uO81l6UO/M+c2uOMnXntZqe6t3ZHUUUo3AbvhKh0QGVFqRQa+Oi0+95KqWmTHa72/9Iw==} + dependencies: + '@aws-sdk/protocol-http': 3.40.0 + '@aws-sdk/querystring-builder': 3.40.0 + '@aws-sdk/types': 3.40.0 + '@aws-sdk/util-base64-browser': 3.37.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/hash-blob-browser/3.40.0: + resolution: {integrity: sha512-l8xyprVVKKH+720VrQ677X6VkvHttDXB4MxkMuxhSvwYBQwsRzP2Wppo7xIAtWGoS+oqlLmD4LCbHdhFRcN5yA==} + dependencies: + '@aws-sdk/chunked-blob-reader': 3.37.0 + '@aws-sdk/chunked-blob-reader-native': 3.37.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/hash-node/3.40.0: + resolution: {integrity: sha512-yOXXK85DdGDktdnQtXgMdaVKii4wtMjEhJ1mrvx2A9nMFNaPhxvERkVVIUKSWlJRa9ZujOw5jWOx8d2R51/Kjg==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/types': 3.40.0 + '@aws-sdk/util-buffer-from': 3.37.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/hash-stream-node/3.40.0: + resolution: {integrity: sha512-4yvRwODMGYtj6qrt+fyydV5MwVwPPoyoeqDoXdLo9x75vRY71DT1pMRt8PDOoY/ZwWbIdEt4+V7x0sLt2uy9WA==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/invalid-dependency/3.40.0: + resolution: {integrity: sha512-axIWtDwCBDDqEgAJipX1FB1ZNpWYXquVwKDMo+7G+ftPBZ4FEq4M1ELhXJL3hhNJ9ZmCQzv+4F6Wnt8dwuzUaQ==} + dependencies: + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/is-array-buffer/3.37.0: + resolution: {integrity: sha512-XLjA/a6AuGnCvcJZLsMTy2jxF2upgGhqCCkoIJgLlzzXHSihur13KcmPvW/zcaGnCRj0SvKWXiJHl4vDlW75VQ==} + engines: {node: '>= 10.0.0'} + dependencies: + tslib: 2.3.1 + dev: false + + /@aws-sdk/md5-js/3.40.0: + resolution: {integrity: sha512-P1tzEljMD/MkjSc00TkVBYvfaVv/7S+04YEwE7tpu/jtxWxMHnk3CMKqq/F2iMhY83DRoqoYy+YqnaF4Bzr1uA==} + dependencies: + '@aws-sdk/types': 3.40.0 + '@aws-sdk/util-utf8-browser': 3.37.0 + '@aws-sdk/util-utf8-node': 3.37.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/middleware-apply-body-checksum/3.40.0: + resolution: {integrity: sha512-gNSFlFu/O8cxAM0X64OwiLLN/NPXvK3FsAIJRsfhIW+dX0bEq4lsGPsdU8Tx+9eenaj/Z01uqgWZ6Izar8zVvQ==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/is-array-buffer': 3.37.0 + '@aws-sdk/protocol-http': 3.40.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/middleware-bucket-endpoint/3.41.0: + resolution: {integrity: sha512-4A0kWHH2qemd4P7CZKS7XB6qtSUP2xMJ7Dn/llxYgvadR0mKEfGPeYPhAss/k7T1JGv+kSTIV30RwUXwdXgE/A==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/protocol-http': 3.40.0 + '@aws-sdk/types': 3.40.0 + '@aws-sdk/util-arn-parser': 3.37.0 + '@aws-sdk/util-config-provider': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/middleware-content-length/3.40.0: + resolution: {integrity: sha512-sybAJb8v7I/vvL08R3+TI/XDAg9gybQTZ2treC24Ap4+jAOz4QBTHJPMKaUlEeFlMUcq4rj6/u2897ebYH6opw==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/protocol-http': 3.40.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/middleware-expect-continue/3.40.0: + resolution: {integrity: sha512-FY6vT0u1ptDZ2bBj1yG/Iyk6HZB7U9fbrpeZNPYzgq8HJxBcTgfLwtB3VLobyhThQm9X2a7R2YZrwtArW8yQfQ==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/middleware-header-default': 3.40.0 + '@aws-sdk/protocol-http': 3.40.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/middleware-header-default/3.40.0: + resolution: {integrity: sha512-eXQ13x/AivPZKoG8/akp9g5xdNHuKftl83GMuk9K6tt4+eAa22TdxiFu4R0UVlKAvo2feqxFrNs5DhhhBeAQWA==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/protocol-http': 3.40.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/middleware-host-header/3.40.0: + resolution: {integrity: sha512-/wocR7JFOLM7/+BQM1DgAd6KCFYcdxYu1P7AhI451GlVNuYa5f89zh7p0gt3SRC6monI5lXgpL7RudhDm8fTrA==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/protocol-http': 3.40.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/middleware-location-constraint/3.40.0: + resolution: {integrity: sha512-9XaVPYsDQVJbWJH96sNdv4HHY3j1raman+lYxMu4528Awp0OdWUeSsGRYRN+CnRPlkHnfNw4m6SKdWYHxdjshw==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/middleware-logger/3.40.0: + resolution: {integrity: sha512-19kx0Xg5ymVRKoupmhdmfTBkROcv3DZj508agpyG2YAo0abOObMlIP4Jltg0VD4PhNjGzNh0jFGJnvhjdwv4/A==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/middleware-retry/3.40.0: + resolution: {integrity: sha512-SMUJrukugLL7YJE5X8B2ToukxMWMPwnf7jAFr84ptycCe8bdWv8x8klQ3EtVWpyqochtNlbTi6J/tTQBniUX7A==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/protocol-http': 3.40.0 + '@aws-sdk/service-error-classification': 3.40.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + uuid: 8.3.2 + dev: false + + /@aws-sdk/middleware-sdk-s3/3.41.0: + resolution: {integrity: sha512-B7JOpmIpm1zxERQEMwZCWj3FisTvwfUgpfQglYdqrB6VpIoCM8fk2pmi5KzU/JDeNBlhTouj6mwnhJL/z5jopA==} + engines: {node: '>= 10.0.0'} + peerDependencies: + '@aws-sdk/signature-v4-crt': ^3.31.0 + dependencies: + '@aws-sdk/protocol-http': 3.40.0 + '@aws-sdk/signature-v4': 3.40.0 + '@aws-sdk/types': 3.40.0 + '@aws-sdk/util-arn-parser': 3.37.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/middleware-sdk-sts/3.40.0: + resolution: {integrity: sha512-TcrbCvj1PkabFZiNczT3yePZtuEm2fAIw1OVnQyLcF2KW+p62Hv5YkK4MPOfx3LA/0lzjOUO1RNl2x7gzV443Q==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/middleware-signing': 3.40.0 + '@aws-sdk/property-provider': 3.40.0 + '@aws-sdk/protocol-http': 3.40.0 + '@aws-sdk/signature-v4': 3.40.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/middleware-serde/3.40.0: + resolution: {integrity: sha512-uOWfZjlAoBy6xPqp0d4ka83WNNbEVCWn9WwfqBUXThyoTdTooYSpXe5y2YzN0BJa8b+tEZTyWpgamnBpFLp47g==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/middleware-signing/3.40.0: + resolution: {integrity: sha512-RqK5nPbfma0qInMvjtpVkDYY/KkFS6EKlOv3DWTdxbXJ4YuOxgKiuUromhmBUoyjFag0JO7LUWod07H+/DawoA==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/property-provider': 3.40.0 + '@aws-sdk/protocol-http': 3.40.0 + '@aws-sdk/signature-v4': 3.40.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/middleware-ssec/3.40.0: + resolution: {integrity: sha512-ZoRpaZeAIQa1Q+NyEh74ATwOR3nFGfcP6Nu0jFzgqoVijCReMnhtlCRx23ccBu1ZLZNUsNk6MhKjY+ZTfNsjEg==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/middleware-stack/3.40.0: + resolution: {integrity: sha512-hby9HvESUYJxpdALX+6Dn2LPmS5jtMVurGB/+j3MWOvIcDYB4bcSXgVRvXzYnTKwbSupIdbX9zOE2ZAx2SJpUQ==} + engines: {node: '>= 10.0.0'} + dependencies: + tslib: 2.3.1 + dev: false + + /@aws-sdk/middleware-user-agent/3.40.0: + resolution: {integrity: sha512-dzC2fxWnanetFJ1oYgil8df3N36bR1yc/OCOpbdfQNiUk1FrXiCXqH5rHNO8zCvnwJAj8GHFwpFGd9a2Qube2w==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/protocol-http': 3.40.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/node-config-provider/3.40.0: + resolution: {integrity: sha512-AmokjgUDECG8osoMfdRsPNweqI+L1pn4bYGk5iTLmzbBi0o4ot0U1FdX8Rf0qJZZwS4t1TXc3s8/PDVknmPxKg==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/property-provider': 3.40.0 + '@aws-sdk/shared-ini-file-loader': 3.37.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/node-http-handler/3.40.0: + resolution: {integrity: sha512-qjda6IbxDhbYr8NHmrMurKkbjgLUkfTMVgagDErDK24Nm3Dn5VaO6J4n6c0Q4OLHlmFaRcUfZSTrOo5DAubqCw==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/abort-controller': 3.40.0 + '@aws-sdk/protocol-http': 3.40.0 + '@aws-sdk/querystring-builder': 3.40.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/property-provider/3.40.0: + resolution: {integrity: sha512-Mx4lkShjsYRwW9ujHA1pcnuubrWQ4kF5/DXWNfUiXuSIO/0Lojp1qTLheyBm4vzkJIlx5umyP6NvRAUkEHSN4Q==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/protocol-http/3.40.0: + resolution: {integrity: sha512-f4ea7/HZkjpvGBrnRIuzc/bhrExWrgDv7eulj4htPukZGHdTqSJD3Jk8lEXWvFuX2vUKQDGhEhCDsqup7YWJQQ==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/querystring-builder/3.40.0: + resolution: {integrity: sha512-gO24oipnNaxJRBXB7lhLfa96vIMOd8gtMBqJTjelTjS2e1ZP1YY12CNKKTWwafSk8Ge021erZAG/YTOaXGpv+g==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/types': 3.40.0 + '@aws-sdk/util-uri-escape': 3.37.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/querystring-parser/3.40.0: + resolution: {integrity: sha512-XZIyaKQIiZAM6zelCBcsLHhVDOLafi7XIOd3jy6SymGN8ajj3HqUJ/vdQ5G6ISTk18OrqgqcCOI9oNzv+nrBcA==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/service-error-classification/3.40.0: + resolution: {integrity: sha512-c8btKmkvjXczWudXubGdbO3JgmjySBUVC/gCrZDNfwNGsG8RYJJQYYcnmt1gWjelUZsgMDl/2PIzxTlxVF91rA==} + engines: {node: '>= 10.0.0'} + dev: false + + /@aws-sdk/shared-ini-file-loader/3.37.0: + resolution: {integrity: sha512-+vRBSlfa48R9KL7DpQt3dsu5/+5atjRgoCISblWo3SLpjrx41pKcjKneo7a1u0aP1Xc2oG2TfIyqTWZuOXsmEQ==} + engines: {node: '>= 10.0.0'} + dependencies: + tslib: 2.3.1 + dev: false + + /@aws-sdk/signature-v4/3.40.0: + resolution: {integrity: sha512-Q1GNZJRCS3W2qsRtDsX/b6EOSfMXfr6TW46N3LnLTGYZ3KAN2SOSJ1DsW59AuGpEZyRmOhJ9L/Q5U403+bZMXQ==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/is-array-buffer': 3.37.0 + '@aws-sdk/types': 3.40.0 + '@aws-sdk/util-hex-encoding': 3.37.0 + '@aws-sdk/util-uri-escape': 3.37.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/smithy-client/3.41.0: + resolution: {integrity: sha512-ldhS0Pf3v6yHCd//kk5DvKcdyeUkKEwxNDRanAp+ekTW68J3XcYgKaPC9sNDhVTDH1zrywTvtEz5zWHEvXjQow==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/middleware-stack': 3.40.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/types/3.40.0: + resolution: {integrity: sha512-KpILcfvRaL88TLvo3SY4OuCCg90SvcNLPyjDwUuBqiOyWODjrKShHtAPJzej4CLp92lofh+ul0UnBfV9Jb/5PA==} + engines: {node: '>= 10.0.0'} + dev: false + + /@aws-sdk/url-parser/3.40.0: + resolution: {integrity: sha512-HwNV+HX7bHgLk5FzTOgdXANsC0SeVz5PMC4Nh+TLz2IoeQnrw4H8dsA4YNonncjern5oC5veKRjQeOoCL5SlSQ==} + dependencies: + '@aws-sdk/querystring-parser': 3.40.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/util-arn-parser/3.37.0: + resolution: {integrity: sha512-njIYn8gzm7Ms17A2oEu0vN/0GJpgq7cNFFtzBrM1cPtrc1jhMRJx5hzS7uX5h6ll8BM92bA3y00evRZFHxQPVQ==} + engines: {node: '>= 10.0.0'} + dependencies: + tslib: 2.3.1 + dev: false + + /@aws-sdk/util-base64-browser/3.37.0: + resolution: {integrity: sha512-o4s/rHVm5k8eC/T7grJQINyYA/mKfDmEWKMA9wk5iBroXlI2rUm7x649TBk5hzoddufk/mffEeNz/1wM7yTmlg==} + dependencies: + tslib: 2.3.1 + dev: false + + /@aws-sdk/util-base64-node/3.37.0: + resolution: {integrity: sha512-1UPxly1GPrGZtlIWvbNCDIAund4Oyp8cFi9neA43TeNACvrmEQu/nG01pDbOoo0ENoVSVJrNAVBeqKEpqjH2GA==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/util-buffer-from': 3.37.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/util-body-length-browser/3.37.0: + resolution: {integrity: sha512-tClmH1uYelqWT43xxmnOsVFbCQJiIwizp6y4E109G2LIof07inxrO0L8nbwBpjhugVplx6NZr9IaqTFqbdM1gA==} + dependencies: + tslib: 2.3.1 + dev: false + + /@aws-sdk/util-body-length-node/3.37.0: + resolution: {integrity: sha512-aY3mXdbEajruRi9CHgq/heM89R+Gectj/Xrs1naewmamaN8NJrvjDm3s+cw//lqqSOW903LYHXDgm7wvCzUnFA==} + engines: {node: '>= 10.0.0'} + dependencies: + tslib: 2.3.1 + dev: false + + /@aws-sdk/util-buffer-from/3.37.0: + resolution: {integrity: sha512-aa3SBwjLwImuJoE4+hxDIWQ9REz3UFb3p7KFPe9qopdXb/yB12RTcbrXVb4whUux4i4mO6KRij0ZNjFZrjrKPg==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/is-array-buffer': 3.37.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/util-config-provider/3.40.0: + resolution: {integrity: sha512-NjZGrA4mqhpr6gkVCAUweurP0Z9d3vFyXJCtulC0BFbpKAnKCf/crSK56NwUaNhAEMCkSuBvjRFzkbfT+HO8bA==} + engines: {node: '>= 10.0.0'} + dependencies: + tslib: 2.3.1 + dev: false + + /@aws-sdk/util-credentials/3.37.0: + resolution: {integrity: sha512-zcLhSZDKgBLhUjSU5HoQpuQiP3v8oE86NmV/tiZVPEaO6YVULEAB2Cfj1hpM/b/JXWzjSHfT06KXT7QUODKS+A==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/shared-ini-file-loader': 3.37.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/util-hex-encoding/3.37.0: + resolution: {integrity: sha512-tn5UpfaeM+rZWqynoNqB8lwtcAXil5YYO3HLGH9himpWAdft/2Z7LK6bsYDpctaAI1WHgMDcL0bw3Id04ZUbhA==} + engines: {node: '>= 10.0.0'} + dependencies: + tslib: 2.3.1 + dev: false + + /@aws-sdk/util-locate-window/3.37.0: + resolution: {integrity: sha512-NvDCfOhLLVHp27oGUUs8EVirhz91aX5gdxGS7J/sh5PF0cNN8rwaR1vSLR7BxPmJHMO7NH7i9EwiELfLfYcq6g==} + engines: {node: '>= 10.0.0'} + dependencies: + tslib: 2.3.1 + dev: false + + /@aws-sdk/util-uri-escape/3.37.0: + resolution: {integrity: sha512-8pKf4YJTELP5lm/CEgYw2atyJBB1RWWqFa0sZx6YJmTlOtLF5G6raUdAi4iDa2hldGt2B6IAdIIyuusT8zeU8Q==} + engines: {node: '>= 10.0.0'} + dependencies: + tslib: 2.3.1 + dev: false + + /@aws-sdk/util-user-agent-browser/3.40.0: + resolution: {integrity: sha512-C69sTI26bV2EprTv3DTXu9XP7kD9Wu4YVPBzqztOYArd2GDYw3w+jS8SEg3XRbjAKY/mOPZ2Thw4StjpZlWZiA==} + dependencies: + '@aws-sdk/types': 3.40.0 + bowser: 2.11.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/util-user-agent-node/3.40.0: + resolution: {integrity: sha512-cjIzd0hRZFTTh7iLJD6Bciu++Em1iaM1clyG02xRl0JD5DEtDSR1zO02uu+AeM7GSLGOxIvwOkK2j8ySPAOmBA==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/node-config-provider': 3.40.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/util-utf8-browser/3.37.0: + resolution: {integrity: sha512-tuiOxzfqet1kKGYzlgpMGfhr64AHJnYsFx2jZiH/O6Yq8XQg43ryjQlbJlim/K/XHGNzY0R+nabeJg34q3Ua1g==} + dependencies: + tslib: 2.3.1 + dev: false + + /@aws-sdk/util-utf8-node/3.37.0: + resolution: {integrity: sha512-fUAgd7UTCULL36j9/vnXHxVhxvswnq23mYgTCIT8NQ7wHN30q2a89ym1e9DwGeQkJEBOkOcKLn6nsMsN7YQMDQ==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/util-buffer-from': 3.37.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/util-waiter/3.40.0: + resolution: {integrity: sha512-jdxwNEZdID49ZvyAnxaeNm5w2moIfMLOwj/q6TxKlxYoXMs16FQWkhyfGue0vEASzchS49ewbyt+KBqpT31Ebg==} + engines: {node: '>= 10.0.0'} + dependencies: + '@aws-sdk/abort-controller': 3.40.0 + '@aws-sdk/types': 3.40.0 + tslib: 2.3.1 + dev: false + + /@aws-sdk/xml-builder/3.37.0: + resolution: {integrity: sha512-Vf0f4ZQ+IBo/l9wUaTOXLqqQO9b/Ll5yPbg+EhHx8zlHbTHIm89ettkVCGyT/taGagC1X+ZeTK9maX6ymEOBow==} + engines: {node: '>= 10.0.0'} + dependencies: + tslib: 2.3.1 + dev: false + /@babel/code-frame/7.12.11: resolution: {integrity: sha512-Zt1yodBx1UcyiePMSkWnU4hPqhwq7hGi2nFL1LeA3EUl+q2LQx16MISgJ0+z7dnmgvP9QtIleuETGOiOH1RcIw==} dependencies: @@ -1071,6 +1847,10 @@ packages: type-is: 1.6.18 dev: false + /bowser/2.11.0: + resolution: {integrity: sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==} + dev: false + /brace-expansion/1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} dependencies: @@ -1610,6 +2390,10 @@ packages: ansi-colors: 4.1.1 dev: true + /entities/2.2.0: + resolution: {integrity: sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==} + dev: false + /error-ex/1.3.2: resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==} dependencies: @@ -4369,7 +5153,10 @@ packages: /tslib/1.14.1: resolution: {integrity: sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==} - dev: true + + /tslib/2.3.1: + resolution: {integrity: sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==} + dev: false /tsutils/3.21.0_typescript@4.3.4: resolution: {integrity: sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==} diff --git a/server/src/lib/cdn/cdn.test.ts b/server/src/lib/cdn/cdn.test.ts index a09fcc8fe..033a0e584 100644 --- a/server/src/lib/cdn/cdn.test.ts +++ b/server/src/lib/cdn/cdn.test.ts @@ -1,12 +1,16 @@ import t from "tap"; -import { CDNDelete, CDNRedirect, CDNRetrieve, CDNStore, CDNStoreOrOverwrite } from "./cdn"; +import { CDNDelete, CDNRedirect, CDNRetrieve, CDNStoreOrOverwrite } from "./cdn"; import fs from "fs"; import { Environment, ServerConfig } from "lib/setup/config"; import path from "path"; import expressRequestMock from "express-request-mock"; -const CDN_FILE_ROOT = Environment.cdnRoot; +if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE !== "LOCAL_FILESYSTEM") { + throw new Error(`Tests must run against LOCAL_FILESYSTEM.`); +} + +const CDN_FILE_ROOT = ServerConfig.CDN_CONFIG.SAVE_LOCATION.LOCATION; function getTestTxt() { return fs.readFileSync(path.join(CDN_FILE_ROOT, "test.txt"), "utf-8"); @@ -29,70 +33,11 @@ const ResetFileRoot = () => { fs.rmSync(CDN_FILE_ROOT, { recursive: true, force: true }); }; -t.test("#CDNStore", (t) => { - t.beforeEach(ResetFileRoot); - - t.test("Should store a value.", async (t) => { - await CDNStore("test.txt", "hello world"); - - const data = fs.readFileSync(path.join(CDN_FILE_ROOT, "test.txt"), "utf8"); - - t.equal(data, "hello world", "Should store the data at the CDN_FILE_ROOT."); - - t.end(); - }); - - t.test("Should generate paths on the way to the file if they do not exist.", async (t) => { - await CDNStore("a/b/c/d/e/f/g.txt", "hello"); - - const data = fs.readFileSync(path.join(CDN_FILE_ROOT, "a/b/c/d/e/f/g.txt"), "utf8"); - - t.equal(data, "hello", "Should store the data at the deeply nested location."); - - t.end(); - }); - - t.test("Should not overwrite files.", async (t) => { - await CDNStore("test.txt", "1"); - t.rejects( - () => CDNStore("test.txt", "2"), - "Should reject an overwrite to an existing file." - ); - - const data = getTestTxt(); - - t.equal(data, "1", "Should not have been overwrote."); - t.end(); - }); - - t.end(); -}); - -t.test("#CDNDelete", (t) => { - t.beforeEach(ResetFileRoot); - - t.test("Should delete the file at the given location.", async (t) => { - await CDNStore("test.txt", "1"); - - const data = getTestTxt(); - - t.equal(data, "1"); - - await CDNDelete("test.txt"); - - t.not(fs.existsSync(path.join(CDN_FILE_ROOT, "test.txt")), "File should not exist."); - - t.end(); - }); - - t.end(); -}); - t.test("#CDNRetrieve", (t) => { t.beforeEach(ResetFileRoot); t.test("Should retrieve the file at the given location.", async (t) => { - await CDNStore("test.txt", "1"); + await CDNStoreOrOverwrite("test.txt", "1"); const data = await CDNRetrieve("test.txt"); @@ -113,6 +58,26 @@ t.test("#CDNRetrieve", (t) => { t.test("#CDNStoreOrOverwrite", (t) => { t.beforeEach(ResetFileRoot); + t.test("Should store a value.", async (t) => { + await CDNStoreOrOverwrite("test.txt", "hello world"); + + const data = fs.readFileSync(path.join(CDN_FILE_ROOT, "test.txt"), "utf8"); + + t.equal(data, "hello world", "Should store the data at the CDN_FILE_ROOT."); + + t.end(); + }); + + t.test("Should generate paths on the way to the file if they do not exist.", async (t) => { + await CDNStoreOrOverwrite("a/b/c/d/e/f/g.txt", "hello"); + + const data = fs.readFileSync(path.join(CDN_FILE_ROOT, "a/b/c/d/e/f/g.txt"), "utf8"); + + t.equal(data, "hello", "Should store the data at the deeply nested location."); + + t.end(); + }); + t.test("Should store a file if one doesn't exist", async (t) => { await CDNStoreOrOverwrite("test.txt", "1"); @@ -134,6 +99,26 @@ t.test("#CDNStoreOrOverwrite", (t) => { t.end(); }); +t.test("#CDNDelete", (t) => { + t.beforeEach(ResetFileRoot); + + t.test("Should delete the file at the given location.", async (t) => { + await CDNStoreOrOverwrite("test.txt", "1"); + + const data = getTestTxt(); + + t.equal(data, "1"); + + await CDNDelete("test.txt"); + + t.not(fs.existsSync(path.join(CDN_FILE_ROOT, "test.txt")), "File should not exist."); + + t.end(); + }); + + t.end(); +}); + t.test("#CDNRedirect", (t) => { t.beforeEach(ResetFileRoot); diff --git a/server/src/lib/cdn/cdn.ts b/server/src/lib/cdn/cdn.ts index 5e1c7d7bc..a83996c59 100644 --- a/server/src/lib/cdn/cdn.ts +++ b/server/src/lib/cdn/cdn.ts @@ -1,10 +1,11 @@ -import fs from "fs"; -import path from "path"; -import CreateLogCtx from "lib/logger/logger"; -import { promisify } from "util"; -import mkdirp from "mkdirp"; import { Response } from "express"; -import { Environment, ServerConfig } from "lib/setup/config"; +import fs from "fs"; +import CreateLogCtx from "lib/logger/logger"; +import { ServerConfig } from "lib/setup/config"; +import mkdirp from "mkdirp"; +import path from "path"; +import { promisify } from "util"; +import { DeleteFromS3, PushToS3 } from "./s3"; const readFilePromise = promisify(fs.readFile); const writeFilePromise = promisify(fs.writeFile); @@ -18,8 +19,19 @@ const logger = CreateLogCtx(__filename); * @danger - This function should **NEVER** be called with unsanitised user input! * Path directory traversal *is* possible, and *will* ruin your day. */ -function CDNRoot(fileLoc: string) { - return path.join(Environment.cdnRoot, fileLoc); +function CDNFileSystemRoot(fileLoc: string) { + if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE !== "LOCAL_FILESYSTEM") { + logger.severe( + `Attempted to run CDNFileSystemRoot, but was not using LOCAL_FILESYSTEM as a CDN.`, + { fileLoc, conf: ServerConfig.CDN_CONFIG } + ); + + throw new Error( + `Attempted to run CDNFileSystemRoot, but was not using LOCAL_FILESYSTEM as a CDN.` + ); + } + + return path.join(ServerConfig.CDN_CONFIG.SAVE_LOCATION.LOCATION, fileLoc); } /** @@ -31,7 +43,7 @@ function CDNRoot(fileLoc: string) { export function CDNRetrieve(fileLoc: string) { logger.debug(`Retrieving path ${fileLoc} locally.`); - return readFilePromise(CDNRoot(fileLoc)); + return readFilePromise(CDNFileSystemRoot(fileLoc)); } /** @@ -42,50 +54,38 @@ export function CDNRedirect(res: Response, fileLoc: string) { throw new Error(`Invalid fileLoc - did not start with /.`); } - return res.redirect(`${ServerConfig.CDN_WEB_LOCATION}${fileLoc}`); -} - -/** - * Flag for fs.open to write a file but NOT overwrite it if it already exists. - */ -const WRITE_NO_OVERWRITE = "wx"; - -/** - * Stores the provided buffer or string as a file at the given location. - * @returns Nothing on success. Throws on error. - */ -export async function CDNStore(fileLoc: string, data: Buffer | string) { - logger.debug(`Storing path ${fileLoc}.`); - - const loc = CDNRoot(fileLoc); - - // make the parent folders if they dont exist. else, mkdirp is a no-op. - await mkdirp(path.dirname(loc)); - - return writeFilePromise(loc, data, { flag: WRITE_NO_OVERWRITE }); + return res.redirect(`${ServerConfig.CDN_CONFIG.WEB_LOCATION}${fileLoc}`); } /** * Stores a file at fileLoc. If it already exists, overwrite it. */ -export async function CDNStoreOrOverwrite(fileLoc: string, data: Buffer | string) { +export async function CDNStoreOrOverwrite(fileLoc: string, data: Buffer | string): Promise { logger.debug(`Storing or overwriting path ${fileLoc}.`); - const loc = CDNRoot(fileLoc); + if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE === "LOCAL_FILESYSTEM") { + const loc = CDNFileSystemRoot(fileLoc); - // make the parent folders if they dont exist. else, mkdirp is a no-op. - await mkdirp(path.dirname(loc)); + // make the parent folders if they dont exist. else, mkdirp is a no-op. + await mkdirp(path.dirname(loc)); - return writeFilePromise(loc, data); + return writeFilePromise(loc, data); + } + + await PushToS3(fileLoc, data); } /** * Removes a file at this CDN location. */ -export function CDNDelete(fileLoc: string) { +export async function CDNDelete(fileLoc: string) { logger.verbose(`Deleting path ${fileLoc}.`); - const loc = CDNRoot(fileLoc); + if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE === "LOCAL_FILESYSTEM") { + const loc = CDNFileSystemRoot(fileLoc); - return rmFilePromise(loc, { force: true }); + return rmFilePromise(loc, { force: true }); + } + + await DeleteFromS3(fileLoc); } diff --git a/server/src/lib/cdn/s3.ts b/server/src/lib/cdn/s3.ts new file mode 100644 index 000000000..8ee269535 --- /dev/null +++ b/server/src/lib/cdn/s3.ts @@ -0,0 +1,64 @@ +import { S3Client, PutObjectCommand, DeleteObjectCommand } from "@aws-sdk/client-s3"; +import CreateLogCtx from "lib/logger/logger"; +import { ServerConfig } from "lib/setup/config"; + +const logger = CreateLogCtx("S3 Client"); + +let s3: null | S3Client = null; + +if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE === "S3_BUCKET") { + const data = ServerConfig.CDN_CONFIG.SAVE_LOCATION; + logger.info(`Using S3_BUCKET as CDN location.`, { bootInfo: true }); + s3 = new S3Client({ + endpoint: data.ENDPOINT, + credentials: { + accessKeyId: data.ACCESS_KEY, + secretAccessKey: data.SECRET_ACCESS_KEY, + }, + }); +} + +/** + * Pushes a file to the configured S3 Bucket. Overwrites if already exists. + */ +export function PushToS3(path: string, content: Buffer | string) { + if (!s3 || ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE !== "S3_BUCKET") { + logger.severe( + `Attempted to push to S3, but CDN_CONFIG.SAVE_LOCATION.TYPE was not S3_BUCKET?`, + ServerConfig.CDN_CONFIG + ); + throw new Error( + `Attempted to push to S3, but CDN_CONFIG.SAVE_LOCATION.TYPE was not S3_BUCKET?` + ); + } + + return s3.send( + new PutObjectCommand({ + Bucket: ServerConfig.CDN_CONFIG.SAVE_LOCATION.BUCKET, + Key: path, + Body: content, + }) + ); +} + +/** + * Deletes the provided file from the configured S3 bucket. + */ +export function DeleteFromS3(path: string) { + if (!s3 || ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE !== "S3_BUCKET") { + logger.severe( + `Attempted to delete from S3, but CDN_CONFIG.SAVE_LOCATION.TYPE was not S3_BUCKET?`, + ServerConfig.CDN_CONFIG + ); + throw new Error( + `Attempted to delete from S3, but CDN_CONFIG.SAVE_LOCATION.TYPE was not S3_BUCKET?` + ); + } + + return s3.send( + new DeleteObjectCommand({ + Bucket: ServerConfig.CDN_CONFIG.SAVE_LOCATION.BUCKET, + Key: path, + }) + ); +} diff --git a/server/src/lib/setup/config.ts b/server/src/lib/setup/config.ts index 8ca042689..9ac4f1665 100644 --- a/server/src/lib/setup/config.ts +++ b/server/src/lib/setup/config.ts @@ -59,7 +59,6 @@ export interface TachiServerConfig { MIN_OAUTH2_INFO?: OAuth2Info; ARC_AUTH_TOKEN?: string; ENABLE_SERVER_HTTPS?: boolean; - RUN_OWN_CDN?: boolean; CLIENT_DEV_SERVER?: string | null; RATE_LIMIT: integer; OAUTH_CLIENT_CAP: integer; @@ -78,7 +77,6 @@ export interface TachiServerConfig { USC_QUEUE_SIZE: integer; BEATORAJA_QUEUE_SIZE: integer; OUR_URL: string; - CDN_WEB_LOCATION: string; INVITE_CODE_CONFIG?: { BATCH_SIZE: integer; INVITE_CAP: integer; @@ -100,6 +98,18 @@ export interface TachiServerConfig { WHO_TO_TAG: string[]; }; }; + CDN_CONFIG: { + WEB_LOCATION: string; + SAVE_LOCATION: + | { TYPE: "LOCAL_FILESYSTEM"; LOCATION: string; RUN_OWN_CDN?: boolean } + | { + TYPE: "S3_BUCKET"; + ENDPOINT: string; + ACCESS_KEY: string; + SECRET_ACCESS_KEY: string; + BUCKET: string; + }; + }; } const isValidOauth2 = p.optional({ @@ -121,7 +131,6 @@ const err = p(config, { MIN_OAUTH2_INFO: isValidOauth2, ARC_AUTH_TOKEN: "*string", ENABLE_SERVER_HTTPS: "*boolean", - RUN_OWN_CDN: "*boolean", CLIENT_DEV_SERVER: "*?string", RATE_LIMIT: p.optional(p.isPositiveInteger), OAUTH_CLIENT_CAP: p.optional(p.isPositiveInteger), @@ -138,7 +147,6 @@ const err = p(config, { USC_QUEUE_SIZE: p.optional(p.gteInt(2)), BEATORAJA_QUEUE_SIZE: p.optional(p.gteInt(2)), OUR_URL: "string", - CDN_WEB_LOCATION: "string", INVITE_CODE_CONFIG: p.optional({ BATCH_SIZE: p.isPositiveInteger, INVITE_CAP: p.isPositiveInteger, @@ -162,6 +170,23 @@ const err = p(config, { WHO_TO_TAG: ["string"], }), }), + CDN_CONFIG: { + WEB_LOCATION: "string", + SAVE_LOCATION: p.or( + { + TYPE: p.is("LOCAL_FILESYSTEM"), + SERVE_OWN_CDN: "*boolean", + LOCATION: "string", + }, + { + TYPE: p.is("S3_BUCKET"), + ENDPOINT: "string", + ACCESS_KEY: "string", + SECRET_ACCESS_KEY: "string", + BUCKET: "string", + } + ), + }, }); if (err) { @@ -216,12 +241,6 @@ if (!seqUrl && tachiServerConfig.LOGGER_CONFIG.SEQ_API_KEY) { ); } -const cdnRoot = process.env.CDN_FILE_ROOT; -if (!cdnRoot) { - logger.error(`No CDN_FILE_ROOT specified in environment. Terminating.`); - process.exit(1); -} - const nodeEnv = process.env.NODE_ENV; if (!nodeEnv) { logger.error(`No NODE_ENV specified in environment. Terminating.`); @@ -241,8 +260,6 @@ export const Environment = { port, redisUrl, mongoUrl, - // If node_env is test, force to ./test-cdn. - cdnRoot: nodeEnv === "test" ? "./test-cdn" : cdnRoot, nodeEnv, replicaIdentity, seqUrl, diff --git a/server/src/server/router/ir/usc/router.ts b/server/src/server/router/ir/usc/router.ts index 6a9f004a2..866406aa3 100644 --- a/server/src/server/router/ir/usc/router.ts +++ b/server/src/server/router/ir/usc/router.ts @@ -1,6 +1,6 @@ import { RequestHandler, Router } from "express"; import db from "external/mongo/db"; -import { CDNStore } from "lib/cdn/cdn"; +import { CDNStoreOrOverwrite } from "lib/cdn/cdn"; import { GetUSCIRReplayURL } from "lib/cdn/url-format"; import { ONE_MEGABYTE } from "lib/constants/filesize"; import { SYMBOL_TachiAPIAuth, SYMBOL_TachiData } from "lib/constants/tachi"; @@ -403,7 +403,10 @@ router.post( } try { - await CDNStore(GetUSCIRReplayURL(correspondingScore.scoreID), req.file.buffer); + await CDNStoreOrOverwrite( + GetUSCIRReplayURL(correspondingScore.scoreID), + req.file.buffer + ); return res.status(200).json({ statusCode: STATUS_CODES.SUCCESS, diff --git a/server/src/server/server.ts b/server/src/server/server.ts index 09b30fd42..7a1fb4242 100644 --- a/server/src/server/server.ts +++ b/server/src/server/server.ts @@ -111,17 +111,22 @@ app.use("/", mainRouter); // The RUN_OWN_CDN option means that our /cdn path has to be hosted by us. In production, // this is not the case (we have a dedicated nginx box for it running in a separate process). // In dev, this is a pain to setup, so we can just run it locally. -if (ServerConfig.RUN_OWN_CDN) { +if ( + ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE === "LOCAL_FILESYSTEM" && + ServerConfig.CDN_CONFIG.SAVE_LOCATION.RUN_OWN_CDN +) { if (Environment.nodeEnv === "production") { logger.warn( - `Running OWN_CDN in production. Consider making a separate process handle your CDN for performance.`, + `Running LOCAL_FILESYSTEM OWN_CDN in production. Consider making a separate process handle your CDN for performance.`, { bootInfo: true } ); } else if (Environment.nodeEnv !== "test") { - logger.info(`Running own CDN at ${Environment.cdnRoot}.`, { bootInfo: true }); + logger.info(`Running own CDN at ${ServerConfig.CDN_CONFIG.SAVE_LOCATION.LOCATION}.`, { + bootInfo: true, + }); } - app.use("/cdn", express.static(Environment.cdnRoot)); + app.use("/cdn", express.static(ServerConfig.CDN_CONFIG.SAVE_LOCATION.LOCATION)); app.get("/cdn/*", (req, res) => res.status(404).send("No content here.")); } diff --git a/server/src/test-utils/resets.ts b/server/src/test-utils/resets.ts index 8806cc601..e5ef362e1 100644 --- a/server/src/test-utils/resets.ts +++ b/server/src/test-utils/resets.ts @@ -10,6 +10,12 @@ import rimraf from "rimraf"; import { SetIndexes } from "external/mongo/indexes"; import { Environment, ServerConfig } from "lib/setup/config"; +if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE !== "LOCAL_FILESYSTEM") { + throw new Error( + `Cannot run tests when CDN_CONFIG.SAVE_LOCATION.TYPE is not LOCAL_FILESYSTEM! (Got ${ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE}.)` + ); +} + const logger = CreateLogCtx(__filename); const DATA_DIR = path.join(__dirname, "./mock-db"); @@ -78,14 +84,20 @@ export default async function ResetDBState() { } export function ResetCDN() { - return new Promise((resolve, reject) => - rimraf(Environment.cdnRoot, (err) => { + return new Promise((resolve, reject) => { + if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE !== "LOCAL_FILESYSTEM") { + throw new Error( + `Cannot run tests when CDN_CONFIG.SAVE_LOCATION.TYPE is not LOCAL_FILESYSTEM! (Got ${ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE}.)` + ); + } + + rimraf(ServerConfig.CDN_CONFIG.SAVE_LOCATION.LOCATION, (err) => { if (err) { reject(err); } resolve(); - }) - ); + }); + }); } export async function SetIndexesForDB() {