block on download

Signed-off-by: Eli Uriegas <eliuriegas@fb.com>
This commit is contained in:
Eli Uriegas 2021-05-19 14:54:47 -07:00
parent b3d4dd0b18
commit 35ec0229c4
No known key found for this signature in database
GPG key ID: EE2C3B4893010973
2 changed files with 86 additions and 48 deletions

93
dist/index.js vendored
View file

@ -19,7 +19,13 @@ module.exports =
/******/ };
/******/
/******/ // Execute the module function
/******/ modules[moduleId].call(module.exports, module, module.exports, __webpack_require__);
/******/ var threw = true;
/******/ try {
/******/ modules[moduleId].call(module.exports, module, module.exports, __webpack_require__);
/******/ threw = false;
/******/ } finally {
/******/ if(threw) delete installedModules[moduleId];
/******/ }
/******/
/******/ // Flag the module as loaded
/******/ module.l = true;
@ -5444,6 +5450,7 @@ module.exports = {"metadata":{"apiVersion":"2017-07-25","endpointPrefix":"dataex
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Outputs = exports.Inputs = void 0;
var Inputs;
(function (Inputs) {
Inputs["Name"] = "name";
@ -24756,7 +24763,7 @@ Object.defineProperty(exports, '__esModule', { value: true });
var request = __webpack_require__(1753);
var universalUserAgent = __webpack_require__(6796);
const VERSION = "4.6.1";
const VERSION = "4.6.2";
class GraphqlError extends Error {
constructor(request, response) {
@ -29852,6 +29859,7 @@ exports.getInput = getInput;
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
function setOutput(name, value) {
process.stdout.write(os.EOL);
command_1.issueCommand('set-output', { name }, value);
}
exports.setOutput = setOutput;
@ -31121,6 +31129,25 @@ exports.getUserAgent = getUserAgent;
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
@ -31130,13 +31157,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k];
result["default"] = mod;
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
@ -31148,6 +31168,25 @@ const os = __importStar(__webpack_require__(2087));
const fs = __importStar(__webpack_require__(5747));
const path_1 = __importDefault(__webpack_require__(5622));
const constants_1 = __webpack_require__(1211);
function doDownload(s3, s3Bucket, fileKey, localKey) {
return __awaiter(this, void 0, void 0, function* () {
return new Promise(function (resolve, reject) {
const localKeyDir = path_1.default.dirname(localKey);
if (!fs.existsSync(localKeyDir)) {
core.debug(`Creating directory (${localKeyDir}) since it did not exist`);
fs.mkdirSync(localKeyDir, { recursive: true });
}
const getObjectParams = { Bucket: s3Bucket, Key: fileKey };
const writeStream = fs.createWriteStream(localKey);
core.info(`Started download: ${localKey}`);
core.debug(`S3 download uri: s3://${s3Bucket}/${fileKey}`);
const readStream = s3.getObject(getObjectParams).createReadStream();
readStream.pipe(writeStream);
readStream.on('close', resolve);
readStream.on('error', reject);
});
});
}
function run() {
return __awaiter(this, void 0, void 0, function* () {
try {
@ -31173,31 +31212,21 @@ function run() {
};
core.debug(JSON.stringify(s3Params));
s3.listObjects(s3Params, function (err, data) {
if (err) {
throw err;
}
if (!data.Contents) {
throw new Error(`Could not find objects with ${s3Prefix}`);
}
for (const fileObject of data.Contents) {
if (!fileObject.Key) {
continue;
return __awaiter(this, void 0, void 0, function* () {
if (err) {
throw err;
}
const getObjectParams = { Bucket: s3Bucket, Key: fileObject.Key };
const localKey = path_1.default.join(resolvedPath, fileObject.Key.replace(s3Prefix, ''));
const localKeyDir = path_1.default.dirname(localKey);
if (!fs.existsSync(localKeyDir)) {
core.debug(`Creating directory (${localKeyDir}) since it did not exist`);
fs.mkdirSync(localKeyDir, { recursive: true });
if (!data.Contents) {
throw new Error(`Could not find objects with ${s3Prefix}`);
}
const writeStream = fs.createWriteStream(localKey);
core.info(`Started download: ${localKey}`);
core.debug(`S3 download uri: s3://${s3Bucket}/${fileObject.Key}`);
const readStream = s3.getObject(getObjectParams).createReadStream();
readStream.pipe(writeStream);
readStream.unpipe();
core.info(`Finished download for ${localKey}`);
}
for (const fileObject of data.Contents) {
if (!fileObject.Key) {
continue;
}
const localKey = path_1.default.join(resolvedPath, fileObject.Key.replace(s3Prefix, ''));
yield doDownload(s3, s3Bucket, fileObject.Key, localKey);
}
});
});
// output the directory that the artifact(s) was/were downloaded to
// if no path is provided, an empty string resolves to the current working directory

View file

@ -6,6 +6,29 @@ import * as fs from 'fs'
import path from 'path'
import {Inputs, Outputs} from './constants'
async function doDownload(
s3: AWS.S3,
s3Bucket: string,
fileKey: string,
localKey: string
): Promise<void> {
return new Promise(function (resolve, reject) {
const localKeyDir = path.dirname(localKey)
if (!fs.existsSync(localKeyDir)) {
core.debug(`Creating directory (${localKeyDir}) since it did not exist`)
fs.mkdirSync(localKeyDir, {recursive: true})
}
const getObjectParams = {Bucket: s3Bucket, Key: fileKey}
const writeStream = fs.createWriteStream(localKey)
core.info(`Started download: ${localKey}`)
core.debug(`S3 download uri: s3://${s3Bucket}/${fileKey}`)
const readStream = s3.getObject(getObjectParams).createReadStream()
readStream.pipe(writeStream)
readStream.on('close', resolve)
readStream.on('error', reject)
})
}
async function run(): Promise<void> {
try {
const name = core.getInput(Inputs.Name, {required: false})
@ -29,7 +52,7 @@ async function run(): Promise<void> {
Prefix: s3Prefix
}
core.debug(JSON.stringify(s3Params))
s3.listObjects(s3Params, function (err, data) {
s3.listObjects(s3Params, async function (err, data) {
if (err) {
throw err
}
@ -40,25 +63,11 @@ async function run(): Promise<void> {
if (!fileObject.Key) {
continue
}
const getObjectParams = {Bucket: s3Bucket, Key: fileObject.Key}
const localKey = path.join(
resolvedPath,
fileObject.Key.replace(s3Prefix, '')
)
const localKeyDir = path.dirname(localKey)
if (!fs.existsSync(localKeyDir)) {
core.debug(
`Creating directory (${localKeyDir}) since it did not exist`
)
fs.mkdirSync(localKeyDir, {recursive: true})
}
const writeStream = fs.createWriteStream(localKey)
core.info(`Started download: ${localKey}`)
core.debug(`S3 download uri: s3://${s3Bucket}/${fileObject.Key}`)
const readStream = s3.getObject(getObjectParams).createReadStream()
readStream.pipe(writeStream)
readStream.unpipe()
core.info(`Finished download for ${localKey}`)
await doDownload(s3, s3Bucket, fileObject.Key, localKey)
}
})
// output the directory that the artifact(s) was/were downloaded to