架构设计：文件服务的设计与实现 - 今日头条

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

// 配置线程池，Spring默认线程池没有设置大小，如果出现阻塞，可能会出现OOM@Bean("eventThread")
 public TaskExecutor taskExecutor() {
 ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
 // 设置核心线程数，转换是个很耗时的过程，所以直接排队执行
 executor.setCorePoolSize(1);
 // 设置最大线程数
 executor.setMaxPoolSize(1);
 // 设置队列容量
 executor.setQueueCapacity(100);
 // 设置线程活跃时间（秒）
 executor.setKeepAliveSeconds(60);
 // 设置默认线程名称
 executor.setThreadNamePrefix("eventThread-");
 // 设置拒绝策略
 executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
 // 等待所有任务结束后再关闭线程池
 executor.setWaitForTasksToCompleteOnShutdown(true);
 return executor;
 }
/**
 * 内部消息总线
 */@Service@EnableAsyncpublic class EventBus implements ApplicationEventPublisherAware {
 private ApplicationEventPublisher publisher;
 @Override
 public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
 this.publisher = applicationEventPublisher;
 }
 public void add(ApplicationEvent event) {
 publisher.publishEvent(event);
 }
}
// 事件类public class UploadEvent extends ApplicationEvent {
 public UploadEvent(Object source) {
 super(source);
 }
}
public class ConvertEvent extends ApplicationEvent {
 public ConvertEvent(Object source) {
 super(source);
 }
}
// 监听类@Componentpublic class UploadListener {
 @EventListener
 @Async("eventThread") // 使用自定义的线程池
 public void process(UploadEvent event) {
 }
}
@Componentpublic class ConvertListener {
 @EventListener
 @Async("eventThread")
 public void process(ConvertEvent event) {
 }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

@Configuration@ConfigurationProperties(prefix = "fileupload.config")
public class VideoConfigRespository {
 private List<VideoConfig> videoConfigList;
 /**
 * 根据分组(系统)找到对应的视频配置
 *
 * @param group
 * @return
 */
 public List<VideoConfig> find(String group) {
 if (videoConfigList == null) {
 return new ArrayList<>();
 } else {
 return videoConfigList.stream().filter(it -> it.getGroup().equals(group)).collect(Collectors.toList());
 }
 }
 public List<VideoConfig> getVideoConfigList() {
 return videoConfigList;
 }
 public void setVideoConfigList(List<VideoConfig> videoConfigList) {
 this.videoConfigList = videoConfigList;
 }
}

1
2
3
4
5
6
7

# 视频配置
fileupload.config.videoConfigList[0].group=GROUP1 
# 默认配置
fileupload.config.videoConfigList[1].group=GROUP2
fileupload.config.videoConfigList[1].type=webm 
 # 转换为webm
fileupload.config.videoConfigList[1].frameSecondList[0]=3 # 取第3秒的图片

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

@Componentpublic class ConvertResultRespository {
 ......
 /**
 * 保存转换结果
 *
 * @param result
 * @return
 */
 public void save(ConvertResult result) {
 Path savePath = Paths.get(tokenPath, result.getToken());
 try {
 if(!Files.exists(savePath.getParent())) {
 Files.createDirectories(savePath.getParent());
 }
 Files.write(savePath, gson.toJson(result).getBytes(UTF8_CHARSET));
 } catch (IOException e) {
 logger.error("save ConvertResult[{}} error!", result, e);
 }
 }
 /**
 * 查找转换结果
 *
 * @param token
 * @return
 */
 public ConvertResult find(String token) {
 Path findPath = Paths.get(tokenPath, token);
 try {
 if (Files.exists(findPath)) {
 String result = new String(Files.readAllBytes(findPath), UTF8_CHARSET);
 return gson.fromJson(result, ConvertResult.class);
 }
 } catch (IOException e) {
 logger.error("find ConvertResult by token[{}} error!", token, e);
 }
 return null;
 }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

/**
* 获取转换后的信息
*/@ResponseBody@GetMapping(value = "/realUrl/{token}")
public ResponseEntity realUrl(@PathVariable String token) {
 .....
}
/**
* 上传文件
*/@ResponseBody@PostMapping(value = {"/partupload/{group}"})
public ResponseEntity upload(HttpServletRequest request, @PathVariable String group) {
 .....
}

1
2
3
4
5
6
7
8

{
 "code": 1,
 "message": "maps.mp4",
 "token": "key_286400710002612",
 "group": "GROUP1",
 "fileType": "VIDEO",
 "filePath": "http://www.abc.com/1556172522968_maps.mp4"
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

{
 "token": "key_282816586380196",
 "group": "SHILU",
 "fileType": "IMAGE",
 "filePath": "http://www.abc.com/SHILU/1/1556164891252_0.jpeg",
 "convertFileInfoList": [
 {
 "fileLength": 0,
 "fileType": "IMAGE",
 "filePath": null,
 "imgPaths": [
 "http://www.abc.com/SHILU/1/1556164891252_0_100_200.jpeg"
 ]
 }
 ]
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

# 对外提供服务的域名
fileupload.server.name=http://www.abc.com # libreoffice home路径
office.home=/snap/libreoffice/115/lib/libreoffice
# 文件上传保存路径
fileupload.upload.root=/home/files
# 文件服务器动态配置 图片配置，切100*200的图fileupload.config.imageConfigList[0].group=group1
fileupload.config.imageConfigList[0].width=100
fileupload.config.imageConfigList[0].height=200
# 视频配置
# 默认配置，转换m3u8
fileupload.config.videoConfigList[0].group=group1
# 转换webm，切第3秒的图
fileupload.config.videoConfigList[1].group=group2
fileupload.config.videoConfigList[1].type=webm
fileupload.config.videoConfigList[1].frameSecondList[0]=3
# office配置，默认转png
fileupload.config.officeConfigList[0].group=group1
# 转PDF
fileupload.config.officeConfigList[0].type=PDF
# pdf配置，转png
fileupload.config.pdfConfigList[0].group=group1
# 上传文件大小，当前端不支持分片上传时设置
spring.servlet.multipart.max-file-size=1024MB
spring.servlet.multipart.max-request-size=1024MB