xbc30 / large-file-upload Goto Github PK

const blobSlice =
  File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice;

blobSlice.call(file, start, end) // 赋值到FormData示例的file属性

fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
// readAsArrayBuffer() 方法用于启动读取指定的 Blob 或 File 内容

const input = fs.createReadStream(path, {
    highWaterMark: chunkSize
});
input.pipe(output);

// 计算文件哈希
calFileHash(file) {
    return new Promise((resolve, reject) => {
        const chunks = Math.ceil(file.size / this.chunkSize);

        const spark = new SparkMD5.ArrayBuffer();
        const fileReader = new FileReader();
        const that = this;
        let currentChunk = 0;

        fileReader.onload = function (e) {
            console.log("read chunk nr", currentChunk + 1, "of", chunks);
            spark.append(e.target.result); // Append array buffer
            currentChunk++;

            if (currentChunk < chunks) {
                loadNext();
            } else {
                that.chunkTotal = currentChunk;
                const hash = spark.end();
                that.updateTips("success", "加载文件成功，文件哈希为" + hash);
                resolve(hash);
            }
        };

        fileReader.onerror = function () {
            that.updateTips("error", "读取切分文件失败，请重试");
            reject("读取切分文件失败，请重试");
        };

        function loadNext() {
            var start = currentChunk * that.chunkSize,
                end =
                start + that.chunkSize >= file.size
                    ? file.size
                    : start + that.chunkSize;

            fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
        }

        loadNext();
    }).catch((err) => {
        console.log(err);
    });
}

// 跟后台校验当前文件是否已经上传过 or 是否需要切换断点续传 返回数据中 res.data == 2(文件已上传过) 1(断点续传) 0(从未上传)
const data = await that.postFileHash();

async postFileHash() {
    const that = this;
    return new Promise((resolve, reject) => {
            axios.post("/hash_check", {
                hash: that.fileHash,
                chunkSize: that.chunkSize,
                total: that.chunkTotal,
            })
            ...
        });
    }

// 构建HTTP FormData数据并请求，最后请求合并分片
async buildFormDataToSend(chunkCount, file, hash, res) {
    const that = this;
    const chunkReqArr = [];
    for (let i = 0; i < chunkCount; i++) {
    if (
        res.type == 0 ||
        (res.type == 1 &&
        res.index &&
        res.index.length > 0 &&
        !res.index.includes(i.toString()))
    ) {
        // 构建需要上传的分片数据
        const start = i * this.chunkSize;
        const end = Math.min(file.size, start + this.chunkSize);
        const form = new FormData();
        form.append("file", blobSlice.call(file, start, end));
        form.append("name", file.name);
        form.append("total", chunkCount);
        form.append("chunkSize", this.chunkSize);
        form.append("index", i);
        form.append("size", file.size);
        form.append("hash", hash);
        const chunkReqItem = axios.post("/chunks_upload", form, {
        onUploadProgress: (e) => {
            // e为ProgressEvent，当loaded===total表明该分片上传完成
            if (e.loaded === e.total) {
            that.chunkDoneTotal += 1;
            }
        },
        });
        chunkReqArr.push(chunkReqItem);
    }
    }
    if (chunkReqArr.length <= 10) {
        Promise.all(chunkReqArr)
        .then(() => {
            that.updateTips("success", "分片上传完成，正在请求合并分片");
            // 合并chunks
            const data = {
                chunkSize: that.chunkSize,
                name: file.name,
                total: that.chunkTotal,
                hash: that.fileHash,
            };
            that.postChunkMerge(data);
        })
        .catch((err) => {
            console.log(err);
            this.updateTips("error", "上传任务中断");
        });
    } else {
        for (let item of chunkReqArr) {
            await item;
        }
        that.updateTips("success", "分片上传完成，正在请求合并分片");
        // 合并chunks
        const data = {
            chunkSize: that.chunkSize,
            name: file.name,
            total: that.chunkTotal,
            hash: that.fileHash,
        };
        that.postChunkMerge(data);
    }
},

// 构建Socket.IO Blob数据并请求，最后请求合并分片
async buildBlobData(chunkCount, file, hash, res) {
    const that = this;
    const blobData = [];
    for (let i = 0; i < chunkCount; i++) {
        if (
            res.type == 0 ||
            (res.type == 1 &&
            res.index &&
            res.index.length > 0 &&
            !res.index.includes(i.toString()))
        ) {
            const start = i * that.chunkSize;
            const end = Math.min(file.size, start + that.chunkSize);
            const blobItem = Object.create({});
            blobItem.file = blobSlice.call(file, start, end);
            blobItem.name = file.name;
            blobItem.total = chunkCount;
            blobItem.chunkSize = that.chunkSize;
            blobItem.index = i;
            blobItem.size = file.size;
            blobItem.hash = hash;
            blobData.push(blobItem);
        }
    }
    for (let item of blobData) {
        await that.socket.emit("upload", item);
    }
},

const streamMerge = (inputPathList, outputPath, chunkSize = 2 * 1024 * 1024) => {
  // Validate inputPathList.
  if (inputPathList.length <= 0) {
    return Promise.reject(new Error("Please input an array with files path!"));
  }

  // create writable stream for output
  const output = fs.createWriteStream(outputPath, {
    encoding: null
  });

  return Promise.mapSeries(inputPathList, function (item) {
    return new Promise(function (resolve, reject) {
      const input = fs.createReadStream(item, {
        encoding: null
      });

      const inputStream = new stream.Readable({
        // equivalent to controlling the size of a bucket
        highWaterMark: chunkSize // the size of each on data of the control flow, the default is 16kb
      }).wrap(input)

      // pipeline data flow
      inputStream.pipe(output, {
        end: false
      });
      inputStream.on('error', reject);
      inputStream.on('end', resolve);
    });
  }).then(function () {
    // close the stream to prevent memory leaks
    output.close();
    return Promise.resolve(outputPath);
  });
}

this.$refs.fileElem.dispatchEvent(new MouseEvent("click"));

const files = this.$refs.fileElem.files[0];

const droppedFiles = e.dataTransfer.files;
if (!droppedFiles) return;

config.multipart = {
    mode: 'file', // 上传文件模式

    fileExtensions: [
        '', // 支持无扩展文件上传
    ],
};