Fade away

Appearance

大文件上传 ​

大文件上传的主要思想是分片，把一个大文件分成若干的片，依次上传

其中需要用到spark-md5.js这个库来给每一片生成唯一的MD5值，框架环境直接npm i，若是浏览器环境找到code界面相应文件，复制即可

image-20250804093856712

获取文件信息 ​

首先前端获取文件信息的方式有好几种，包括使用input元素选择文件，还有使用拖拽api获取，本次就使用拖拽上传来获取

拖拽api获取文件信息主要方式如下，在drop事件的监听中，可以获取到file文件信息

js
box.addEventListener("drop", async (e) => {
    e.preventDefault();
    e.stopPropagation();

    const target = e.target;
    target.classList.remove("cover");

    const file = e.dataTransfer.files[0];
});

分片函数 ​

我们需要写一个cutFile函数用来将file分成若干个片

文件总大小/每一片大小=分片数，需要向上取整，多余的零头也得是一片

把分好的每一片使用createChunks函数生成每一片的具体信息，如开始结束下标，md5值

js
// 文件分片
const cutFile = async (file, CHUNK_SIZE = 1024 * 1024 * 5) => {
    const chunkCount = Math.ceil(file.size / CHUNK_SIZE);		
    const result = [];
    for (let i = 0; i < chunkCount; i++) {
        const chunk = await createChunks(file, i, CHUNK_SIZE);
        result.push(chunk);
    }
    return result;
};

box.addEventListener("drop", async (e) => {
    e.preventDefault();
    e.stopPropagation();

    const target = e.target;
    target.classList.remove("cover");

    const file = e.dataTransfer.files[0];

    const res = await cutFile(file);
    console.log(res);
});

具体分片信息 ​

接下来需要写一个createChunks函数，来获取每一片的详情数据

此处使用spark-md5.js给每一片生成md5值，因为接口需要

createChunks需要单独写在一个文件中

js
// 给每片创建分片信息
import "./SparkMd5.js";

export const createChunks = (file, index, chunkSize) => {
  return new Promise((resolve) => {
    const start = index * chunkSize;
    const end = start + chunkSize;
    const spark = new SparkMD5.ArrayBuffer();
    const fileReader = new FileReader();
    const blob = file.slice(start, end);
    fileReader.onload = (e) => {
      spark.append(e.target.result);
      resolve({
        start,
        end,
        index,
        hash: spark.end(),
        blob,
      });
    };
    fileReader.readAsArrayBuffer(blob);
  });
};

计算分片时间 ​

来计算一下分片的时间

js
box.addEventListener("drop", async (e) => {
    e.preventDefault();
    e.stopPropagation();

    const target = e.target;
    target.classList.remove("cover");

    const file = e.dataTransfer.files[0];
    
    console.time("cutFile");
    const res = await cutFile(file);
    console.log(res);
    console.timeEnd("cutFile");
});

image-20250804095219791

一个200多mb的文件，花费了两秒多，非常消耗时间

消耗时间的点主要在于createChunks函数中MD5值的计算

我们需要采用webWork的方式，减少这个时间

使用WebWork ​

开多少个线程？ ​

通过navigator.hardwareConcurrency可以获取电脑的cpu内核数，内核数有多少就可以开多少个线程，最低可以给4个

js
const Therd_Chout = navigator.hardwareConcurrency || 4;

之后使用for循环，开同Therd_Chout个线程

js
for (let i = 0; i < Therd_Chout; i++) {
    // 创建多线程
    const work = new Worker("./work.js", {
        type: "module",
    });
 	
    // 发送数据
    work.postMessage();
    
    // 获取返回来的文件分片信息，并且最后关闭多线程
    work.onmessage = (e) => {};
}

修改cutFile函数 ​

填充内容如下

js
const Therd_Chout = navigator.hardwareConcurrency || 4; // 线程数

const cutFile = async (file, CHUNK_SIZE = 1024 * 1024 * 5) => {
    return new Promise((resolve) => {
        const chunkCount = Math.ceil(file.size / CHUNK_SIZE);	// 一共分多少片
        let finishCount = 0;	// 用于计算多线程返回个数，全部返回了，就把分片结果resolve出去
        const threadChunkCount = Math.ceil(chunkCount / Therd_Chout); // 每个线程分多少片
        
        const result = [];
        for (let i = 0; i < Therd_Chout; i++) {
            const work = new Worker("./work.js", {
                type: "module",
            });
            const start = i * threadChunkCount;
            let end = (i + 1) * threadChunkCount;
            if (end > chunkCount) {
                end = chunkCount;
            }
            work.postMessage({
                file,
                CHUNK_SIZE,
                startChunkIndex: start,
                endChunkIndex: end,
            });
            work.onmessage = (e) => {
                for (let i = start; i < end; i++) {
                    result[i] = e.data[i - start];
                }
                work.terminate();   // 每个线程完成后要关闭
                finishCount++;
                if (finishCount === Therd_Chout) {
                    resolve(result);
                }
            };
        }
    });
};

然后在work.js中使用createChunks

js
import { createChunks } from "./createChunks.js";

self.onmessage = async (e) => {
  const {
    file,
    CHUNK_SIZE,
    startChunkIndex: start,
    endChunkIndex: end,
  } = e.data;

  const proms = [];
  for (let i = start; i < end; i++) {
    proms.push(createChunks(file, i, CHUNK_SIZE));
  }
  const chunks = await Promise.all(proms);
  postMessage(chunks);
};

image-20250804105104186

现在可以看到只花费了457ms，相比之前的2488ms提升了许多

完整代码 ​

html
<!DOCTYPE html>
<html lang="en">
  <head>
    <meta charset="UTF-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
    <title>Document</title>
    <style>
      .box {
        width: 500px;
        height: 300px;
        border: 1px dashed gray;
        border-radius: 10px;
        display: flex;
        justify-content: center;
        align-items: center;
      }
      .text {
        display: none;
      }
      .cover {
        background-color: rgba(0, 0, 0, 0.1);
        border: 1px dashed #333;
      }
    </style>
  </head>
  <body>
    <div class="box">
      <span class="text">松开鼠标即可开始上传</span>
    </div>
    <script>
      const box = document.querySelector(".box");
      const text = document.querySelector(".text");

      box.addEventListener("dragenter", (e) => {
        e.preventDefault();
        const target = e.target;
        target.classList.add("cover");
        text.style.display = "inline";
      });
      box.addEventListener("dragover", (e) => {
        e.preventDefault(); // 必须阻止默认行为才能触发drop
      });
      box.addEventListener("dragleave", (e) => {
        e.preventDefault();
        const target = e.target;
        target.classList.remove("cover");
        text.style.display = "none";
      });

      // 文件分片

      const Therd_Chout = navigator.hardwareConcurrency || 4; // 线程数

      const cutFile = async (file, CHUNK_SIZE = 1024 * 1024 * 5) => {
        return new Promise((resolve) => {
          const chunkCount = Math.ceil(file.size / CHUNK_SIZE);
          let finishCount = 0;
          const threadChunkCount = Math.ceil(chunkCount / Therd_Chout); // 每个线程分多少片
          const result = [];
          for (let i = 0; i < Therd_Chout; i++) {
            const work = new Worker("./work.js", {
              type: "module",
            });
            const start = i * threadChunkCount;
            let end = (i + 1) * threadChunkCount;
            if (end > chunkCount) {
              end = chunkCount;
            }
            work.postMessage({
              file,
              CHUNK_SIZE,
              startChunkIndex: start,
              endChunkIndex: end,
            });
            work.onmessage = (e) => {
              for (let i = start; i < end; i++) {
                result[i] = e.data[i - start];
              }
              work.terminate();   // 每个线程完成后要关闭
              finishCount++;
              if (finishCount === Therd_Chout) {
                resolve(result);
              }
            };
          }
        });
      };

      box.addEventListener("drop", async (e) => {
        e.preventDefault();
        e.stopPropagation();

        const target = e.target;
        text.style.display = "none";
        target.classList.remove("cover");

        const file = e.dataTransfer.files[0];

        console.time("cutFile");
        const res = await cutFile(file);
        console.log(res);
        console.timeEnd("cutFile");
      });
    </script>
  </body>
</html>
js
// 给每片创建分片信息
import "./SparkMd5.js";

export const createChunks = (file, index, chunkSize) => {
  return new Promise((resolve) => {
    const start = index * chunkSize;
    const end = start + chunkSize;
    const spark = new SparkMD5.ArrayBuffer();
    const fileReader = new FileReader();
    const blob = file.slice(start, end);
    fileReader.onload = (e) => {
      spark.append(e.target.result);
      resolve({
        start,
        end,
        index,
        hash: spark.end(),
        blob,
      });
    };
    fileReader.readAsArrayBuffer(blob);
  });
};
js
import { createChunks } from "./createChunks.js";

self.onmessage = async (e) => {
  const {
    file,
    CHUNK_SIZE,
    startChunkIndex: start,
    endChunkIndex: end,
  } = e.data;

  const proms = [];
  for (let i = start; i < end; i++) {
    proms.push(createChunks(file, i, CHUNK_SIZE));
  }
  const chunks = await Promise.all(proms);

  postMessage(chunks);
};

MIT Licensed

Copyright © 2024-present xarzhi