# big-file-upload-base-express

**Repository Path**: axtlive/big-file-upload-base-express

## Basic Information

- **Project Name**: big-file-upload-base-express
- **Description**: No description available
- **Primary Language**: Unknown
- **License**: Not specified
- **Default Branch**: master
- **Homepage**: None
- **GVP Project**: No

## Statistics

- **Stars**: 0
- **Forks**: 0
- **Created**: 2024-11-06
- **Last Updated**: 2024-11-06

## Categories & Tags

**Categories**: Uncategorized

**Tags**: None

## README

[TOC]



录了视频：[基于 express 大文件上传](https://www.zhihu.com/zvideo/1418697983638126592)


 **查看优化，可切换到 dev2 分支** 


# 1. 初始化 server

创建文件夹 test

## I. 初始化

```bash
npm init -y
yarn add express    # express
yarn add nodemon    # nodejs 热更新
yarn add cli-color  # 命令行输出颜色
```

## II. 创建服务

```javascript
// src/index.js
const express = require("express");

const server = express();

// browser test
server.get("/", async (req, res) => {
  res.send(`hi,express.`);
});

server.listen(3000, (_) => {
  console.log(clc.bold.blue.underline("http://localhost:3000/"));
  console.log(clc.bold.blue.underline("http://127.0.0.1:3000/"));
});
```

## III. 配置启动

package.json

```json
{
  "scripts": {
    "serve": "nodemon ./src/index.js"
  }
}
```

```bash
yarn serve     # 启动服务
```

# 2. 前端部分

index.html

## I. html

```html
<div id="app">
  <form action="" method="post">
    <input type="file" name="" id="fileInput" />
    <button id="uploadBtn">上传</button>
  </form>
</div>
```

## II. 全局变量

```javascript
var file = null;
const SIZE = 1024 * 1024 * 100; // 每个切片 10MB
```

## III. input:file 的 onchange 事件

```javascript
// 文件改变
function handleFileChange(event) {
  const file = event.target.files[0];
  if (!file) return;
  window.file = file;
}

document
  .getElementById("fileInput")
  .addEventListener("change", handleFileChange);
```

## IV. button 的 onclick 事件

```javascript
document.getElementById("uploadBtn").addEventListener("click", handleUpload);
```

handleUpload 是大文件上传的内容

## V. 大文件上传 handleUpload

1. 大文件切片
2. 上传切片（并发）
3. 合并切片

```JavaScript
// 大文件上传
async function handleUpload(event) {
    event.preventDefault();

    const file = window.file;
    if (!file) return;

    // 切片
    const createFileChunks = function (file, size = SIZE) {...}

    // 上传 切片
    const uploadFileChunks = async function (fileChunks, filename) {...}

    // 合并切片
    async function handleMerge(filename) {...}

    const fileChunks = createFileChunks(file);
    await uploadFileChunks(fileChunks, file.name);
    await mergeFileChunks(file.name);
    console.log('上传完毕');
}
```

### a. 大文件切片 createFileChunks

```javascript
// 切片
const createFileChunks = function (file, size = SIZE) {
  const fileChunks = [];
  let cur = 0;
  while (cur < file.size) {
    fileChunks.push({
      chunk: file.slice(cur, cur + size),
    });
    cur += size;
  }
  return fileChunks;
};
```

### b. 上传切片 uploadFileChunks

引入 axios

```html
<script src="https://unpkg.com/axios/dist/axios.min.js"></script>
```

设置 baseURL

```javascript
axios.defaults.baseURL = `http://localhost:3000`;
```

上传切片（FormData）

```javascript
// 上传 切片
const uploadFileChunks = async function (fileChunks, filename) {
  const formDataList = fileChunks.map(({ chunk }, index) => {
    const formData = new FormData();

    console.log(
      `${filename}-${index} is instanceof Blob?`,
      chunk instanceof Blob
    );

    formData.append("filename", filename);
    formData.append("hash", index);
    formData.append("chunk", chunk);
    return {
      formData,
    };
  });

  const requestList = formDataList.map(({ formData }) =>
    axios({
      method: "post",
      url: `/upload`,
      data: formData,
    })
  );

  await Promise.all(requestList);
};
```

### c. 合并切片 mergeFileChunks

```javascript
// 合并切片
async function mergeFileChunks(filename) {
  try {
    await axios({
      method: "get",
      url: `/merge`,
      params: {
        filename,
      },
    });
  } catch (error) {
    console.error(error);
  }
}
```

# 3. 后端部分

## I. 处理跨域

```javascript
// 设置响应头处理跨域
server.use((req, res, next) => {
  res.header("Access-Control-Allow-Origin", "*");
  next();
});
```

## II. server:post 上传切片 '/upload'

```
yarn add multiparty
```

- 通过 [multiparty](https://www.npmjs.com/package/multiparty) 获取 FormData 内容

  - 对于 multipart.parse 可以捕获普通的数据
  - 对于 切片 Blob 文件，需要通过 EventEmitter 监听捕获
  - 因此，对于上述两者不能同时得到，故而 EventEmitter 发布订阅来触发保存切片

- 到这里已经开始保存切片
  - fs.readFileSync 读取文件 Buffer
  - 创建写入流 fs.createWriteStream
  - cw.wirte(buffer) 写入二进制数据

```javascript
const multiparty = require("multiparty");
const path = require("path");
const fs = require("fs");
const EventEmitter = require("events");
const { Buffer } = require("buffer");

const STATIC_TEMPORARY = path.resolve(__dirname, "../static/temporary"); // 创建 test/static/temporary 临时存放切片

// 上传切片
server.post("/upload", async (req, res) => {
  const multipart = new multiparty.Form();

  try {
    let formData = {
      filename: "",
      hash: "",
      chunk: null,
    };

    let isFileOk = false, // 是否拿到 file
      isFieldOk = false; // 是否拿到 fields

    const myEmitter = new EventEmitter();

    // 获取参数
    multipart.parse(req, function (err, fields, files) {
      formData.filename = fields["filename"];
      formData.hash = fields["hash"];

      isFieldOk = true;
      myEmitter.emit("start");
    });

    // 获取文件
    multipart.on("file", async function (name, file) {
      formData["chunk"] = file;
      isFileOk = true;
      myEmitter.emit("start");
    });

    // 保存文件
    myEmitter.on("start", function () {
      if (isFileOk && isFieldOk) {
        const { filename, hash, chunk } = formData;

        // 如果没有文件夹则新建
        const chunkDir = `${STATIC_TEMPORARY}/${filename}`;
        if (!fs.existsSync(chunkDir)) fs.mkdirSync(chunkDir);

        // 获取 Buffer File
        const buffer = fs.readFileSync(chunk.path);

        // 创建写入流 写入Buffer
        const ws = fs.createWriteStream(`${chunkDir}/${hash}`);
        ws.end(buffer);

        isFileOk = false;
        isFieldOk = false;

        res.send(`${chunk} 上传完成!`);
      }
    });
  } catch (error) {
    console.error(error);
  }
});
```

## III. server:get 合并切片 '/merge'

这其实就是合并 Buffer，将各个切片的的二进制数据按顺序进行合并，得到文件

```javascript
const STATIC_FILES = path.resolve(__dirname, "../static/files"); // 合并后存放位置

// 合并切片
server.get("/merge", async (req, res) => {
  const { filename } = req.query;

  try {
    const wsPath = `${STATIC_FILES}/${filename}`;
    const temporaryPath = `${STATIC_TEMPORARY}/${filename}`;

    let len = 0;
    const bufferList = fs.readdirSync(temporaryPath).map((name) => {
      const buffer = fs.readFileSync(`${temporaryPath}/${name}`);
      len += buffer.length;
      return buffer;
    });

    // 合并写入
    const buffer = Buffer.concat(bufferList, len);
    const ws = fs.createWriteStream(wsPath);
    ws.end(buffer);

    res.send(`${filename} 合并成功`);
  } catch (error) {
    console.error(error);
  }
});
```

# 4. about

Author: Axtlive