feat: add @actions/cache

This commit is contained in:
xHyroM 2022-07-12 09:00:22 +02:00
parent b15fb7d098
commit 16e8c96a41
1932 changed files with 261172 additions and 10 deletions

View file

@ -0,0 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
export const AVRO_SYNC_MARKER_SIZE = 16;
export const AVRO_INIT_BYTES = new Uint8Array([79, 98, 106, 1]);
export const AVRO_CODEC_KEY = "avro.codec";
export const AVRO_SCHEMA_KEY = "avro.schema";
//# sourceMappingURL=AvroConstants.js.map

View file

@ -0,0 +1 @@
{"version":3,"file":"AvroConstants.js","sourceRoot":"","sources":["../../../../storage-internal-avro/src/AvroConstants.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,MAAM,CAAC,MAAM,qBAAqB,GAAW,EAAE,CAAC;AAChD,MAAM,CAAC,MAAM,eAAe,GAAe,IAAI,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;AAC5E,MAAM,CAAC,MAAM,cAAc,GAAW,YAAY,CAAC;AACnD,MAAM,CAAC,MAAM,eAAe,GAAW,aAAa,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nexport const AVRO_SYNC_MARKER_SIZE: number = 16;\nexport const AVRO_INIT_BYTES: Uint8Array = new Uint8Array([79, 98, 106, 1]);\nexport const AVRO_CODEC_KEY: string = \"avro.codec\";\nexport const AVRO_SCHEMA_KEY: string = \"avro.schema\";\n"]}

View file

@ -0,0 +1,311 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
export class AvroParser {
/**
* Reads a fixed number of bytes from the stream.
*
* @param stream -
* @param length -
* @param options -
*/
static async readFixedBytes(stream, length, options = {}) {
const bytes = await stream.read(length, { abortSignal: options.abortSignal });
if (bytes.length !== length) {
throw new Error("Hit stream end.");
}
return bytes;
}
/**
* Reads a single byte from the stream.
*
* @param stream -
* @param options -
*/
static async readByte(stream, options = {}) {
const buf = await AvroParser.readFixedBytes(stream, 1, options);
return buf[0];
}
// int and long are stored in variable-length zig-zag coding.
// variable-length: https://lucene.apache.org/core/3_5_0/fileformats.html#VInt
// zig-zag: https://developers.google.com/protocol-buffers/docs/encoding?csw=1#types
static async readZigZagLong(stream, options = {}) {
let zigZagEncoded = 0;
let significanceInBit = 0;
let byte, haveMoreByte, significanceInFloat;
do {
byte = await AvroParser.readByte(stream, options);
haveMoreByte = byte & 0x80;
zigZagEncoded |= (byte & 0x7f) << significanceInBit;
significanceInBit += 7;
} while (haveMoreByte && significanceInBit < 28); // bitwise operation only works for 32-bit integers
if (haveMoreByte) {
// Switch to float arithmetic
// eslint-disable-next-line no-self-assign
zigZagEncoded = zigZagEncoded;
significanceInFloat = 268435456; // 2 ** 28.
do {
byte = await AvroParser.readByte(stream, options);
zigZagEncoded += (byte & 0x7f) * significanceInFloat;
significanceInFloat *= 128; // 2 ** 7
} while (byte & 0x80);
const res = (zigZagEncoded % 2 ? -(zigZagEncoded + 1) : zigZagEncoded) / 2;
if (res < Number.MIN_SAFE_INTEGER || res > Number.MAX_SAFE_INTEGER) {
throw new Error("Integer overflow.");
}
return res;
}
return (zigZagEncoded >> 1) ^ -(zigZagEncoded & 1);
}
static async readLong(stream, options = {}) {
return AvroParser.readZigZagLong(stream, options);
}
static async readInt(stream, options = {}) {
return AvroParser.readZigZagLong(stream, options);
}
static async readNull() {
return null;
}
static async readBoolean(stream, options = {}) {
const b = await AvroParser.readByte(stream, options);
if (b === 1) {
return true;
}
else if (b === 0) {
return false;
}
else {
throw new Error("Byte was not a boolean.");
}
}
static async readFloat(stream, options = {}) {
const u8arr = await AvroParser.readFixedBytes(stream, 4, options);
const view = new DataView(u8arr.buffer, u8arr.byteOffset, u8arr.byteLength);
return view.getFloat32(0, true); // littleEndian = true
}
static async readDouble(stream, options = {}) {
const u8arr = await AvroParser.readFixedBytes(stream, 8, options);
const view = new DataView(u8arr.buffer, u8arr.byteOffset, u8arr.byteLength);
return view.getFloat64(0, true); // littleEndian = true
}
static async readBytes(stream, options = {}) {
const size = await AvroParser.readLong(stream, options);
if (size < 0) {
throw new Error("Bytes size was negative.");
}
return stream.read(size, { abortSignal: options.abortSignal });
}
static async readString(stream, options = {}) {
const u8arr = await AvroParser.readBytes(stream, options);
const utf8decoder = new TextDecoder();
return utf8decoder.decode(u8arr);
}
static async readMapPair(stream, readItemMethod, options = {}) {
const key = await AvroParser.readString(stream, options);
// FUTURE: this won't work with readFixed (currently not supported) which needs a length as the parameter.
const value = await readItemMethod(stream, options);
return { key, value };
}
static async readMap(stream, readItemMethod, options = {}) {
const readPairMethod = (s, opts = {}) => {
return AvroParser.readMapPair(s, readItemMethod, opts);
};
const pairs = await AvroParser.readArray(stream, readPairMethod, options);
const dict = {};
for (const pair of pairs) {
dict[pair.key] = pair.value;
}
return dict;
}
static async readArray(stream, readItemMethod, options = {}) {
const items = [];
for (let count = await AvroParser.readLong(stream, options); count !== 0; count = await AvroParser.readLong(stream, options)) {
if (count < 0) {
// Ignore block sizes
await AvroParser.readLong(stream, options);
count = -count;
}
while (count--) {
const item = await readItemMethod(stream, options);
items.push(item);
}
}
return items;
}
}
var AvroComplex;
(function (AvroComplex) {
AvroComplex["RECORD"] = "record";
AvroComplex["ENUM"] = "enum";
AvroComplex["ARRAY"] = "array";
AvroComplex["MAP"] = "map";
AvroComplex["UNION"] = "union";
AvroComplex["FIXED"] = "fixed";
})(AvroComplex || (AvroComplex = {}));
var AvroPrimitive;
(function (AvroPrimitive) {
AvroPrimitive["NULL"] = "null";
AvroPrimitive["BOOLEAN"] = "boolean";
AvroPrimitive["INT"] = "int";
AvroPrimitive["LONG"] = "long";
AvroPrimitive["FLOAT"] = "float";
AvroPrimitive["DOUBLE"] = "double";
AvroPrimitive["BYTES"] = "bytes";
AvroPrimitive["STRING"] = "string";
})(AvroPrimitive || (AvroPrimitive = {}));
export class AvroType {
/**
* Determines the AvroType from the Avro Schema.
*/
static fromSchema(schema) {
if (typeof schema === "string") {
return AvroType.fromStringSchema(schema);
}
else if (Array.isArray(schema)) {
return AvroType.fromArraySchema(schema);
}
else {
return AvroType.fromObjectSchema(schema);
}
}
static fromStringSchema(schema) {
switch (schema) {
case AvroPrimitive.NULL:
case AvroPrimitive.BOOLEAN:
case AvroPrimitive.INT:
case AvroPrimitive.LONG:
case AvroPrimitive.FLOAT:
case AvroPrimitive.DOUBLE:
case AvroPrimitive.BYTES:
case AvroPrimitive.STRING:
return new AvroPrimitiveType(schema);
default:
throw new Error(`Unexpected Avro type ${schema}`);
}
}
static fromArraySchema(schema) {
return new AvroUnionType(schema.map(AvroType.fromSchema));
}
static fromObjectSchema(schema) {
const type = schema.type;
// Primitives can be defined as strings or objects
try {
return AvroType.fromStringSchema(type);
}
catch (err) {
// eslint-disable-line no-empty
}
switch (type) {
case AvroComplex.RECORD:
if (schema.aliases) {
throw new Error(`aliases currently is not supported, schema: ${schema}`);
}
if (!schema.name) {
throw new Error(`Required attribute 'name' doesn't exist on schema: ${schema}`);
}
// eslint-disable-next-line no-case-declarations
const fields = {};
if (!schema.fields) {
throw new Error(`Required attribute 'fields' doesn't exist on schema: ${schema}`);
}
for (const field of schema.fields) {
fields[field.name] = AvroType.fromSchema(field.type);
}
return new AvroRecordType(fields, schema.name);
case AvroComplex.ENUM:
if (schema.aliases) {
throw new Error(`aliases currently is not supported, schema: ${schema}`);
}
if (!schema.symbols) {
throw new Error(`Required attribute 'symbols' doesn't exist on schema: ${schema}`);
}
return new AvroEnumType(schema.symbols);
case AvroComplex.MAP:
if (!schema.values) {
throw new Error(`Required attribute 'values' doesn't exist on schema: ${schema}`);
}
return new AvroMapType(AvroType.fromSchema(schema.values));
case AvroComplex.ARRAY: // Unused today
case AvroComplex.FIXED: // Unused today
default:
throw new Error(`Unexpected Avro type ${type} in ${schema}`);
}
}
}
class AvroPrimitiveType extends AvroType {
constructor(primitive) {
super();
this._primitive = primitive;
}
read(stream, options = {}) {
switch (this._primitive) {
case AvroPrimitive.NULL:
return AvroParser.readNull();
case AvroPrimitive.BOOLEAN:
return AvroParser.readBoolean(stream, options);
case AvroPrimitive.INT:
return AvroParser.readInt(stream, options);
case AvroPrimitive.LONG:
return AvroParser.readLong(stream, options);
case AvroPrimitive.FLOAT:
return AvroParser.readFloat(stream, options);
case AvroPrimitive.DOUBLE:
return AvroParser.readDouble(stream, options);
case AvroPrimitive.BYTES:
return AvroParser.readBytes(stream, options);
case AvroPrimitive.STRING:
return AvroParser.readString(stream, options);
default:
throw new Error("Unknown Avro Primitive");
}
}
}
class AvroEnumType extends AvroType {
constructor(symbols) {
super();
this._symbols = symbols;
}
async read(stream, options = {}) {
const value = await AvroParser.readInt(stream, options);
return this._symbols[value];
}
}
class AvroUnionType extends AvroType {
constructor(types) {
super();
this._types = types;
}
async read(stream, options = {}) {
const typeIndex = await AvroParser.readInt(stream, options);
return this._types[typeIndex].read(stream, options);
}
}
class AvroMapType extends AvroType {
constructor(itemType) {
super();
this._itemType = itemType;
}
read(stream, options = {}) {
const readItemMethod = (s, opts) => {
return this._itemType.read(s, opts);
};
return AvroParser.readMap(stream, readItemMethod, options);
}
}
class AvroRecordType extends AvroType {
constructor(fields, name) {
super();
this._fields = fields;
this._name = name;
}
async read(stream, options = {}) {
const record = {};
record["$schema"] = this._name;
for (const key in this._fields) {
if (Object.prototype.hasOwnProperty.call(this._fields, key)) {
record[key] = await this._fields[key].read(stream, options);
}
}
return record;
}
}
//# sourceMappingURL=AvroParser.js.map

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,5 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
export class AvroReadable {
}
//# sourceMappingURL=AvroReadable.js.map

View file

@ -0,0 +1 @@
{"version":3,"file":"AvroReadable.js","sourceRoot":"","sources":["../../../../storage-internal-avro/src/AvroReadable.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAelC,MAAM,OAAgB,YAAY;CAGjC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { AbortSignalLike } from \"@azure/abort-controller\";\n\n/**\n * Options to configure the {@link AvroReadable.read} operation.\n */\nexport interface AvroReadableReadOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the &commat;azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\nexport abstract class AvroReadable {\n public abstract get position(): number;\n public abstract read(size: number, options?: AvroReadableReadOptions): Promise<Uint8Array>;\n}\n"]}

View file

@ -0,0 +1,47 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
import { AvroReadable } from "./AvroReadable";
import { AbortError } from "@azure/abort-controller";
const ABORT_ERROR = new AbortError("Reading from the avro blob was aborted.");
export class AvroReadableFromBlob extends AvroReadable {
constructor(blob) {
super();
this._blob = blob;
this._position = 0;
}
get position() {
return this._position;
}
async read(size, options = {}) {
size = Math.min(size, this._blob.size - this._position);
if (size <= 0) {
return new Uint8Array();
}
const fileReader = new FileReader();
return new Promise((resolve, reject) => {
function cleanUp() {
if (options.abortSignal) {
options.abortSignal.removeEventListener("abort", abortHandler);
}
}
function abortHandler() {
fileReader.abort();
cleanUp();
reject(ABORT_ERROR);
}
if (options.abortSignal) {
options.abortSignal.addEventListener("abort", abortHandler);
}
fileReader.onloadend = (ev) => {
cleanUp();
resolve(new Uint8Array(ev.target.result));
};
fileReader.onerror = () => {
cleanUp();
reject();
};
fileReader.readAsArrayBuffer(this._blob.slice(this._position, (this._position += size)));
});
}
}
//# sourceMappingURL=AvroReadableFromBlob.js.map

View file

@ -0,0 +1 @@
{"version":3,"file":"AvroReadableFromBlob.js","sourceRoot":"","sources":["../../../../storage-internal-avro/src/AvroReadableFromBlob.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,EAAE,YAAY,EAA2B,MAAM,gBAAgB,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAErD,MAAM,WAAW,GAAG,IAAI,UAAU,CAAC,yCAAyC,CAAC,CAAC;AAE9E,MAAM,OAAO,oBAAqB,SAAQ,YAAY;IAIpD,YAAY,IAAU;QACpB,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;IACrB,CAAC;IAED,IAAW,QAAQ;QACjB,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAEM,KAAK,CAAC,IAAI,CAAC,IAAY,EAAE,UAAmC,EAAE;QACnE,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;QACxD,IAAI,IAAI,IAAI,CAAC,EAAE;YACb,OAAO,IAAI,UAAU,EAAE,CAAC;SACzB;QAED,MAAM,UAAU,GAAG,IAAI,UAAU,EAAE,CAAC;QACpC,OAAO,IAAI,OAAO,CAAa,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAEjD,SAAS,OAAO;gBACd,IAAI,OAAO,CAAC,WAAW,EAAE;oBACvB,OAAO,CAAC,WAAY,CAAC,mBAAmB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;iBACjE;YACH,CAAC;YAED,SAAS,YAAY;gBACnB,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,OAAO,EAAE,CAAC;gBACV,MAAM,CAAC,WAAW,CAAC,CAAC;YACtB,CAAC;YAED,IAAI,OAAO,CAAC,WAAW,EAAE;gBACvB,OAAO,CAAC,WAAW,CAAC,gBAAgB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;aAC7D;YAED,UAAU,CAAC,SAAS,GAAG,CAAC,EAAO,EAAE,EAAE;gBACjC,OAAO,EAAE,CAAC;gBACV,OAAO,CAAC,IAAI,UAAU,CAAC,EAAE,CAAC,MAAO,CAAC,MAAM,CAAC,CAAC,CAAC;YAC7C,CAAC,CAAC;YAEF,UAAU,CAAC,OAAO,GAAG,GAAG,EAAE;gBACxB,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE,CAAC;YACX,CAAC,CAAC;YAEF,UAAU,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3F,CAAC,CAAC,CAAC;IACL,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { AvroReadable, AvroReadableReadOptions } from \"./AvroReadable\";\nimport { AbortError } from \"@azure/abort-controller\";\n\nconst ABORT_ERROR = new AbortError(\"Reading from the avro blob was aborted.\");\n\nexport class AvroReadableFromBlob extends AvroReadable {\n private _position: number;\n private _blob: Blob;\n\n constructor(blob: Blob) {\n super();\n this._blob = blob;\n this._position = 0;\n }\n\n public get position(): number {\n return this._position;\n }\n\n public async read(size: number, options: AvroReadableReadOptions = {}): Promise<Uint8Array> {\n size = Math.min(size, this._blob.size - this._position);\n if (size <= 0) {\n return new Uint8Array();\n }\n\n const fileReader = new FileReader();\n return new Promise<Uint8Array>((resolve, reject) => {\n\n function cleanUp(): void {\n if (options.abortSignal) {\n options.abortSignal!.removeEventListener(\"abort\", abortHandler);\n }\n }\n\n function abortHandler(): void {\n fileReader.abort();\n cleanUp();\n reject(ABORT_ERROR);\n }\n\n if (options.abortSignal) {\n options.abortSignal.addEventListener(\"abort\", abortHandler);\n }\n\n fileReader.onloadend = (ev: any) => {\n cleanUp();\n resolve(new Uint8Array(ev.target!.result));\n };\n\n fileReader.onerror = () => {\n cleanUp();\n reject();\n };\n\n fileReader.readAsArrayBuffer(this._blob.slice(this._position, (this._position += size)));\n });\n }\n}\n"]}

View file

@ -0,0 +1,84 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
import { AvroReadable } from "./AvroReadable";
import { AbortError } from "@azure/abort-controller";
const ABORT_ERROR = new AbortError("Reading from the avro stream was aborted.");
export class AvroReadableFromStream extends AvroReadable {
constructor(readable) {
super();
this._readable = readable;
this._position = 0;
}
toUint8Array(data) {
if (typeof data === "string") {
return Buffer.from(data);
}
return data;
}
get position() {
return this._position;
}
async read(size, options = {}) {
var _a;
if ((_a = options.abortSignal) === null || _a === void 0 ? void 0 : _a.aborted) {
throw ABORT_ERROR;
}
if (size < 0) {
throw new Error(`size parameter should be positive: ${size}`);
}
if (size === 0) {
return new Uint8Array();
}
if (!this._readable.readable) {
throw new Error("Stream no longer readable.");
}
// See if there is already enough data.
const chunk = this._readable.read(size);
if (chunk) {
this._position += chunk.length;
// chunk.length maybe less than desired size if the stream ends.
return this.toUint8Array(chunk);
}
else {
// register callback to wait for enough data to read
return new Promise((resolve, reject) => {
/* eslint-disable @typescript-eslint/no-use-before-define */
const cleanUp = () => {
this._readable.removeListener("readable", readableCallback);
this._readable.removeListener("error", rejectCallback);
this._readable.removeListener("end", rejectCallback);
this._readable.removeListener("close", rejectCallback);
if (options.abortSignal) {
options.abortSignal.removeEventListener("abort", abortHandler);
}
};
const readableCallback = () => {
const callbackChunk = this._readable.read(size);
if (callbackChunk) {
this._position += callbackChunk.length;
cleanUp();
// callbackChunk.length maybe less than desired size if the stream ends.
resolve(this.toUint8Array(callbackChunk));
}
};
const rejectCallback = () => {
cleanUp();
reject();
};
const abortHandler = () => {
cleanUp();
reject(ABORT_ERROR);
};
this._readable.on("readable", readableCallback);
this._readable.once("error", rejectCallback);
this._readable.once("end", rejectCallback);
this._readable.once("close", rejectCallback);
if (options.abortSignal) {
options.abortSignal.addEventListener("abort", abortHandler);
}
/* eslint-enable @typescript-eslint/no-use-before-define */
});
}
}
}
//# sourceMappingURL=AvroReadableFromStream.js.map

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,107 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
import { __asyncGenerator, __await } from "tslib";
// TODO: Do a review of non-interfaces
/* eslint-disable @azure/azure-sdk/ts-use-interface-parameters */
import "@azure/core-paging";
import { AVRO_CODEC_KEY, AVRO_INIT_BYTES, AVRO_SCHEMA_KEY, AVRO_SYNC_MARKER_SIZE, } from "./AvroConstants";
import { AvroParser, AvroType } from "./AvroParser";
import { arraysEqual } from "./utils/utils.common";
export class AvroReader {
constructor(dataStream, headerStream, currentBlockOffset, indexWithinCurrentBlock) {
this._dataStream = dataStream;
this._headerStream = headerStream || dataStream;
this._initialized = false;
this._blockOffset = currentBlockOffset || 0;
this._objectIndex = indexWithinCurrentBlock || 0;
this._initialBlockOffset = currentBlockOffset || 0;
}
get blockOffset() {
return this._blockOffset;
}
get objectIndex() {
return this._objectIndex;
}
async initialize(options = {}) {
const header = await AvroParser.readFixedBytes(this._headerStream, AVRO_INIT_BYTES.length, {
abortSignal: options.abortSignal,
});
if (!arraysEqual(header, AVRO_INIT_BYTES)) {
throw new Error("Stream is not an Avro file.");
}
// File metadata is written as if defined by the following map schema:
// { "type": "map", "values": "bytes"}
this._metadata = await AvroParser.readMap(this._headerStream, AvroParser.readString, {
abortSignal: options.abortSignal,
});
// Validate codec
const codec = this._metadata[AVRO_CODEC_KEY];
if (!(codec === undefined || codec === null || codec === "null")) {
throw new Error("Codecs are not supported");
}
// The 16-byte, randomly-generated sync marker for this file.
this._syncMarker = await AvroParser.readFixedBytes(this._headerStream, AVRO_SYNC_MARKER_SIZE, {
abortSignal: options.abortSignal,
});
// Parse the schema
const schema = JSON.parse(this._metadata[AVRO_SCHEMA_KEY]);
this._itemType = AvroType.fromSchema(schema);
if (this._blockOffset === 0) {
this._blockOffset = this._initialBlockOffset + this._dataStream.position;
}
this._itemsRemainingInBlock = await AvroParser.readLong(this._dataStream, {
abortSignal: options.abortSignal,
});
// skip block length
await AvroParser.readLong(this._dataStream, { abortSignal: options.abortSignal });
this._initialized = true;
if (this._objectIndex && this._objectIndex > 0) {
for (let i = 0; i < this._objectIndex; i++) {
await this._itemType.read(this._dataStream, { abortSignal: options.abortSignal });
this._itemsRemainingInBlock--;
}
}
}
hasNext() {
return !this._initialized || this._itemsRemainingInBlock > 0;
}
parseObjects(options = {}) {
return __asyncGenerator(this, arguments, function* parseObjects_1() {
if (!this._initialized) {
yield __await(this.initialize(options));
}
while (this.hasNext()) {
const result = yield __await(this._itemType.read(this._dataStream, {
abortSignal: options.abortSignal,
}));
this._itemsRemainingInBlock--;
this._objectIndex++;
if (this._itemsRemainingInBlock === 0) {
const marker = yield __await(AvroParser.readFixedBytes(this._dataStream, AVRO_SYNC_MARKER_SIZE, {
abortSignal: options.abortSignal,
}));
this._blockOffset = this._initialBlockOffset + this._dataStream.position;
this._objectIndex = 0;
if (!arraysEqual(this._syncMarker, marker)) {
throw new Error("Stream is not a valid Avro file.");
}
try {
this._itemsRemainingInBlock = yield __await(AvroParser.readLong(this._dataStream, {
abortSignal: options.abortSignal,
}));
}
catch (err) {
// We hit the end of the stream.
this._itemsRemainingInBlock = 0;
}
if (this._itemsRemainingInBlock > 0) {
// Ignore block size
yield __await(AvroParser.readLong(this._dataStream, { abortSignal: options.abortSignal }));
}
}
yield yield __await(result);
}
});
}
}
//# sourceMappingURL=AvroReader.js.map

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
export { AvroReader } from "./AvroReader";
export { AvroReadable } from "./AvroReadable";
export { AvroReadableFromBlob } from "./AvroReadableFromBlob";
//# sourceMappingURL=index.browser.js.map

View file

@ -0,0 +1 @@
{"version":3,"file":"index.browser.js","sourceRoot":"","sources":["../../../../storage-internal-avro/src/index.browser.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nexport { AvroReader } from \"./AvroReader\";\nexport { AvroReadable } from \"./AvroReadable\";\nexport { AvroReadableFromBlob } from \"./AvroReadableFromBlob\";\n"]}

View file

@ -0,0 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
export { AvroReader } from "./AvroReader";
export { AvroReadable } from "./AvroReadable";
export { AvroReadableFromStream } from "./AvroReadableFromStream";
//# sourceMappingURL=index.js.map

View file

@ -0,0 +1 @@
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../storage-internal-avro/src/index.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nexport { AvroReader } from \"./AvroReader\";\nexport { AvroReadable } from \"./AvroReadable\";\nexport { AvroReadableFromStream } from \"./AvroReadableFromStream\";\n"]}

View file

@ -0,0 +1,17 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
export function arraysEqual(a, b) {
if (a === b)
return true;
// eslint-disable-next-line eqeqeq
if (a == null || b == null)
return false;
if (a.length !== b.length)
return false;
for (let i = 0; i < a.length; ++i) {
if (a[i] !== b[i])
return false;
}
return true;
}
//# sourceMappingURL=utils.common.js.map

View file

@ -0,0 +1 @@
{"version":3,"file":"utils.common.js","sourceRoot":"","sources":["../../../../../storage-internal-avro/src/utils/utils.common.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAOlC,MAAM,UAAU,WAAW,CAAC,CAAa,EAAE,CAAa;IACtD,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACzB,kCAAkC;IAClC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI;QAAE,OAAO,KAAK,CAAC;IACzC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE;QACjC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;KACjC;IACD,OAAO,IAAI,CAAC;AACd,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nexport interface KeyValuePair<T> {\n key: string;\n value: T;\n}\n\nexport function arraysEqual(a: Uint8Array, b: Uint8Array): boolean {\n if (a === b) return true;\n // eslint-disable-next-line eqeqeq\n if (a == null || b == null) return false;\n if (a.length !== b.length) return false;\n\n for (let i = 0; i < a.length; ++i) {\n if (a[i] !== b[i]) return false;\n }\n return true;\n}\n"]}