FormData/Go分片/分块文件上传
程序员文章站
2022-03-25 16:25:12
FormData 接口提供了一种表示表单数据的键值对的构造方式，经过它的数据可以使用 XMLHttpRequest.send() 方法送出，本接口和此方法都相当简单直接。如果送出时的编码类型被设为 "multipart/form-data"，它会使用和表单一样的格式。 如果你想构建一个简单的GET请 ......
formdata 接口提供了一种表示表单数据的键值对的构造方式，经过它的数据可以使用 xmlhttprequest.send() 方法送出，本接口和此方法都相当简单直接。如果送出时的编码类型被设为 "multipart/form-data"，它会使用和表单一样的格式。
如果你想构建一个简单的get请求，并且通过<form>的形式带有查询参数，可以将它直接传递给urlsearchparams。
更多解释mdn: https://developer.mozilla.org/zh-cn/docs/web/api/formdata
分块(分片，统称分块了，确实只是发送一块数据)文件上传主要分2部分。
1. 前端js用file.slice可以从文件中切出一块一块的数据，然后用formdata包装一下，用xmlhttprequest把切出来的数据块，一块一块send到server.
2. server接收到的每一块都是一个multipart/form-data form表单。可以在表单里放很多附属信息，文件名，大小，块大小，块索引，最总带上这块切出来的二进制数据。
multipart/form-data 数据
post /upload http/1.1 host: localhost:8080 content-length: 2098072 user-agent: mozilla/5.0 (macintosh; intel mac os x 10_15_4) applewebkit/537.36 (khtml, like gecko) chrome/80.0.3987.149 safari/537.36 content-type: multipart/form-data; boundary=----webkitformboundarymtng0xrr3asr7wx7 ------webkitformboundaryhdbeczab5xbq6d55 content-disposition: form-data; name="file_name" apache-maven-3.6.3-bin.zip ------webkitformboundaryhdbeczab5xbq6d55 content-disposition: form-data; name="file_size" 9602303 ------webkitformboundaryhdbeczab5xbq6d55 content-disposition: form-data; name="block_size" 2097152 ------webkitformboundaryhdbeczab5xbq6d55 content-disposition: form-data; name="total_blocks" 5 ------webkitformboundaryhdbeczab5xbq6d55 content-disposition: form-data; name="break_error" true ------webkitformboundaryhdbeczab5xbq6d55 content-disposition: form-data; name="index" 3 ------webkitformboundaryhdbeczab5xbq6d55 content-disposition: form-data; name="data"; filename="blob" content-type: application/octet-stream (binary)
在server存储文件，基本也就2种方案：
1. 直接创建一个对应大小的文件，按照每块数据的offset位置，写进去。
2. 每个传过来的数据块，保存成一个单独的数据块文件，最后把所有文件块合并成文件。
我这里只是做了一份简单的演示代码，基本上是不能用于生产环境的。
index.html，直接把js写进去了
1 <!doctype html>
2 <html>
3 <head>
4 <meta charset="utf8">
5 <title>multil-blocks upload</title>
6 </head>
7
8 <body>
9 <h2>multil-blocks upload</h2>
10
11 <input id="file" type="file" />
12
13 <input type="checkbox" id="multil_block_file">multil block file</input>
14 <button type="button" onclick="on_block_upload()">block upload</button>
15 <button type="button" onclick="on_concurrency_upload()">concurrency upload</button>
16 <hr/>
17
18 <div>
19 <label>file name: </label><span id="file_name"></span>
20 </div>
21 <div>
22 <label>file size: </label><span id="file_size"></span>
23 </div>
24 <div>
25 <label>split blocks: </label><span id="block_count"></span>
26 </div>
27
28 <hr/>
29
30 <p id="upload_info"></p>
31
32 <script>
33 var block_size = 1024 * 1024 * 2;
34
35 var el_file = document.getelementbyid('file');
36 var el_multil_block_file = document.getelementbyid('multil_block_file');
37 var el_file_name = document.getelementbyid('file_name');
38 var el_file_size = document.getelementbyid('file_size');
39 var el_block_count = document.getelementbyid('block_count');
40 var el_upload_info = document.getelementbyid('upload_info');
41
42 var file = null;
43 var total_blocks = 0;
44 var block_index = -1;
45 var block_index_random_arr = [];
46 var form_data = null;
47
48
49 el_file.onchange = function() {
50 if (this.files.length === 0) return;
51
52 file = this.files[0];
53 total_blocks = math.ceil( file.size / block_size );
54
55 el_file_name.innertext = file.name;
56 el_file_size.innertext = file.size;
57 el_block_count.innertext = total_blocks;
58 }
59
60 function print_info(msg) {
61 el_upload_info.innerhtml += `${msg}<br/>`;
62 }
63
64 function done() {
65 file = null;
66 total_blocks = 0;
67 block_index = -1;
68 form_data = null;
69
70 el_file.value = '';
71 }
72
73
74 function get_base_form_data() {
75 var base_data = new formdata();
76 base_data.append('file_name', file.name);
77 base_data.append('file_size', file.size);
78 base_data.append('block_size', block_size);
79 base_data.append('total_blocks', total_blocks);
80 base_data.append('break_error', true);
81 base_data.append('index', 0);
82 base_data.append('data', null);
83
84 return base_data
85 }
86
87
88 function build_block_index_random_arr() {
89 block_index_random_arr = new array(total_blocks).fill(0).map((v,i) => i);
90 block_index_random_arr.sort((n, m) => math.random() > .5 ? -1 : 1);
91
92 print_info(`upload sequence: ${block_index_random_arr}`);
93 }
94
95
96 function post(index, success_cb, failed_cb) {
97 if (!form_data) {
98 form_data = get_base_form_data();
99 }
100 var start = index * block_size;
101 var end = math.min(file.size, start + block_size);
102
103 form_data.set('index', index);
104 form_data.set('data', file.slice(start, end));
105
106 print_info(`post ${index}/${total_blocks}, offset: ${start} -- ${end}`);
107
108
109 var xhr = new xmlhttprequest();
110 xhr.open('post', '/upload', true);
111 /*
112 browser-based general content types
113 content-type: multipart/form-data; boundary=----webkitformboundarysxh5dies2xfmulxl
114
115 error content type:
116 xhr.setrequestheader('content-type', 'multipart/form-data');
117 content-type: multipart/form-data;
118 */
119 xhr.onreadystatechange = function() {
120
121 if (xhr.readystate === xmlhttprequest.done) {
122
123 if (xhr.status >= 200 && xhr.status < 300 && success_cb) {
124 return success_cb();
125 }
126
127 if (xhr.status >= 400 && failed_cb) {
128 failed_cb();
129 }
130 }
131 }
132
133 // xhr.onerror event
134 xhr.send(form_data);
135 }
136
137
138 function block_upload() {
139 if (!file) {
140 return;
141 }
142 if (block_index + 1 >= total_blocks) {
143 return done();
144 }
145
146 block_index++;
147 var index = block_index_random_arr[block_index];
148
149 post(index, block_upload);
150 }
151
152
153 function concurrency_upload() {
154 if (!file || total_blocks === 0) {
155 return;
156 }
157
158 build_block_index_random_arr();
159
160 form_data = get_base_form_data();
161 form_data.set('break_error', false);
162 form_data.set('multil_block', el_multil_block_file.checked);
163
164 for (var i of block_index_random_arr) {
165 ((idx) => {
166 post(idx, null, function() {
167 print_info(`failed: ${idx}`);
168 settimeout(() => post(idx), 1000);
169 });
170 })(i);
171 }
172 }
173
174
175 function on_block_upload() {
176 if (file) {
177 print_info('block upload');
178
179 form_data = get_base_form_data();
180 form_data.set('multil_block', el_multil_block_file.checked);
181
182 build_block_index_random_arr();
183
184 block_index = -1;
185 block_upload();
186 }
187 }
188
189 function on_concurrency_upload() {
190 if (file) {
191 print_info('concurrency upload');
192 concurrency_upload();
193 }
194 }
195 </script>
196
197 </body>
198 </html>
简单的go server和保存文件，基本忽略所有的错误处理
1 package main
2
3 import (
4 "fmt"
5 "io/ioutil"
6 "log"
7 "net/http"
8 "os"
9 "path"
10 "path/filepath"
11 "regexp"
12 "strconv"
13 "strings"
14 "syscall"
15 "text/template"
16 )
17
18 type multilblockfile struct {
19 filename string
20 size int64
21 blocksize int64
22 totalblocks int
23 index int
24 bufs []byte
25 breakerror bool
26 }
27
28 func fileisexist(f string) bool {
29 _, err := os.stat(f)
30 return err == nil || os.isexist(err)
31 }
32
33 func lockfile(f *os.file) error {
34 err := syscall.flock(int(f.fd()), syscall.lock_ex|syscall.lock_nb)
35 if err != nil {
36 return fmt.errorf("get flock failed. err: %s", err)
37 }
38
39 return nil
40 }
41
42 func unlockfile(f *os.file) error {
43 defer f.close()
44 return syscall.flock(int(f.fd()), syscall.lock_un)
45 }
46
47 func singlefilesave(mbf *multilblockfile) error {
48
49 dir, _ := filepath.abs(filepath.dir(os.args[0]))
50 filepath := path.join(dir, "tmp", mbf.filename)
51
52 offset := int64(mbf.index) * mbf.blocksize
53
54 fmt.println(">>> single file save ---------------------")
55 fmt.printf("save file: %s \n", filepath)
56 fmt.printf("file offset: %d \n", offset)
57
58 var f *os.file
59 var needtruncate bool = false
60 if !fileisexist(filepath) {
61 needtruncate = true
62 }
63
64 f, _ = os.openfile(filepath, syscall.o_creat|syscall.o_wronly, 0777)
65
66 err := lockfile(f)
67 if err != nil {
68 if mbf.breakerror {
69 log.fatalf("get flock failed. err: %s", err)
70 } else {
71 return err
72 }
73 }
74
75 if needtruncate {
76 f.truncate(mbf.size)
77 }
78
79 f.writeat(mbf.bufs, offset)
80
81 unlockfile(f)
82
83 return nil
84 }
85
86 func multilblockssave(mbf *multilblockfile) error {
87 dir, _ := filepath.abs(filepath.dir(os.args[0]))
88 tmpfolderpath := path.join(dir, "tmp")
89 tmpfilename := fmt.sprintf("%s.%d", mbf.filename, mbf.index)
90 fileblockpath := path.join(tmpfolderpath, tmpfilename)
91
92 f, _ := os.openfile(fileblockpath, syscall.o_creat|syscall.o_wronly|syscall.o_trunc, 0777)
93 defer f.close()
94
95 f.write(mbf.bufs)
96 f.close()
97
98 re := regexp.mustcompile(`(?i:^` + mbf.filename + `).\d$`)
99
100 files, _ := ioutil.readdir(tmpfolderpath)
101 matchfiles := make(map[string]bool)
102
103 for _, file := range files {
104 if file.isdir() {
105 continue
106 }
107
108 fname := file.name()
109 if re.matchstring(fname) {
110 matchfiles[fname] = true
111 }
112 }
113
114 if len(matchfiles) >= mbf.totalblocks {
115 lastfile, _ := os.openfile(path.join(tmpfolderpath, mbf.filename), syscall.o_creat|syscall.o_wronly, 0777)
116 lockfile(lastfile)
117
118 lastfile.truncate(mbf.size)
119
120 for name := range matchfiles {
121 tmppath := path.join(tmpfolderpath, name)
122
123 idxstr := name[strings.lastindex(name, ".")+1:]
124 idx, _ := strconv.parseint(idxstr, 10, 32)
125
126 fmt.printf("match file: %s index: %d \n", name, idx)
127
128 data, _ := ioutil.readfile(tmppath)
129
130 lastfile.writeat(data, idx*mbf.blocksize)
131
132 os.remove(tmppath)
133 }
134 unlockfile(lastfile)
135 }
136
137 return nil
138 }
139
140 func indexhandle(w http.responsewriter, r *http.request) {
141 tmp, _ := template.parsefiles("./static/index.html")
142 tmp.execute(w, "index")
143 }
144
145 func uploadhandle(w http.responsewriter, r *http.request) {
146
147 var mbf multilblockfile
148 mbf.filename = r.formvalue("file_name")
149 mbf.size, _ = strconv.parseint(r.formvalue("file_size"), 10, 64)
150 mbf.blocksize, _ = strconv.parseint(r.formvalue("block_size"), 10, 64)
151 mbf.breakerror, _ = strconv.parsebool(r.formvalue("break_error"))
152
153 var i int64
154 i, _ = strconv.parseint(r.formvalue("total_blocks"), 10, 32)
155 mbf.totalblocks = int(i)
156
157 i, _ = strconv.parseint(r.formvalue("index"), 10, 32)
158 mbf.index = int(i)
159
160 d, _, _ := r.formfile("data")
161 mbf.bufs, _ = ioutil.readall(d)
162
163 fmt.printf(">>> upload --------------------- \n")
164 fmt.printf("file name: %s \n", mbf.filename)
165 fmt.printf("size: %d \n", mbf.size)
166 fmt.printf("block size: %d \n", mbf.blocksize)
167 fmt.printf("total blocks: %d \n", mbf.totalblocks)
168 fmt.printf("index: %d \n", mbf.index)
169 fmt.println("bufs len:", len(mbf.bufs))
170
171 multilblockfile, _ := strconv.parsebool(r.formvalue("multil_block"))
172
173 var err error
174 if multilblockfile {
175 err = multilblockssave(&mbf)
176 } else {
177 err = singlefilesave(&mbf)
178 }
179
180 if !mbf.breakerror && err != nil {
181 w.writeheader(400)
182 fmt.fprintf(w, fmt.sprintf("%s", err))
183 return
184 }
185
186 fmt.fprintf(w, "ok")
187 }
188
189 func main() {
190 println("listen on 8080")
191
192 http.handlefunc("/", indexhandle)
193 http.handlefunc("/upload", uploadhandle)
194
195 log.fatal(http.listenandserve(":8080", nil))
196 }
目录截图，比较乱来
上一篇： Python-反射机制