Streaming new records into an existing parquet file in S3 about parquetjs HOT 5 OPEN

import { S3Client } from '@aws-sdk/client-s3';
import { Upload } from '@aws-sdk/lib-storage';

const createUploadStream = (
  fileName: string,
): { passStream: stream.PassThrough; parallelUploadS3: Upload } => {
  const passStream = new stream.PassThrough();
  const uploadKeyName = fileName + '.parquet';
  const parallelUploadS3 = new Upload({
    client: new S3Client({}),
    queueSize: 4,
    params: {
      Bucket: downloadBucketName,
      Key: uploadKeyName,
      Body: passStream,
    },
  });

  parallelUploadS3.on('httpUploadProgress', (progress) => {
    console.log(`Download part: ${progress.part}`);
  });

  return { passStream, parallelUploadS3 };
};

export const handler = async (event: DownloadLambdaEvent) => {
  console.log('Processing event: ', JSON.stringify(event, null, 2));
  try {
  
const { parallelUploadS3, passStream } = createUploadStream(event.fileName);

https.get(`${DownloadUrlPrefix}${event.fileName}.zip`, (fileStream) => {
      fileStream.pipe(unzipper.ParseOne()).pipe(ndjson.parse()).pipe(new parquet.ParquetTransformer(schema, opts)).pipe(passStream);
    });
    ...