使用代理和线程池下载

src/main/java/top/yuchat/crawler/video/models/service/MadouVideoService.java

@@ -23,6 +23,8 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 @Slf4j
 @Service
@@ -55,6 +57,8 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
 
     private final ClassifyInfoService classifyInfoService;
 
+    ExecutorService scheduledThreadPool = Executors.newScheduledThreadPool(16);
+
 
     public void getVideoList() {
         List<ClassifyInfo> list = classifyInfoService.list();
@@ -130,34 +134,46 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
     }
 
     @PostConstruct
-    public void download() {
+    public void download() throws IOException {
         QueryWrapper<MadouVideoInfo> wrapper = new QueryWrapper<>();
         wrapper.eq("m3u8", true);
         List<MadouVideoInfo> madouVideoInfos = list(wrapper);
         log.info("开始处理数据，数据总量: {}", madouVideoInfos.size());
 
+        Path imgPath = Paths.get(downloadBasePath, "img");
+        if (!Files.exists(imgPath)) {
+            Files.createDirectories(imgPath);
+        }
         for (MadouVideoInfo videoInfo : madouVideoInfos) {
-            try {
+            scheduledThreadPool.submit(() -> {
-                log.info("开始下载：{}，m3u8地址：{}", videoInfo.getTitle(), videoInfo.getM3u8Url());
+                try {
-                String result = HttpUtils.get(videoInfo.getM3u8Url());
+                    log.info("下载封面图片，名称：{}，地址：{}", videoInfo.getTitle(), videoInfo.getCoverUrl());
-                Path path = Paths.get(downloadBasePath, videoInfo.getId().toString());
+                    HttpUtils.download(videoInfo.getCoverUrl(), Paths.get(imgPath.toString(), videoInfo.getId().toString() + videoInfo.getCoverUrl().substring(videoInfo.getCoverUrl().lastIndexOf("."))));
-                if (!Files.exists(path)) {
-                    Files.createDirectories(path);
-                }
-                Path m3u8 = Paths.get(path.toString(), "index.m3u8");
-                Files.writeString(m3u8, result);
 
-                List<String> tss = Arrays.stream(result.split("\n")).filter(t -> t.contains(".ts")).toList();
+                    log.info("开始下载：{}，m3u8地址：{}", videoInfo.getTitle(), videoInfo.getM3u8Url());
-                for (String ts : tss) {
+                    String result = HttpUtils.get(videoInfo.getM3u8Url());
-                    log.info("开始下载ts文件，文件名称：{}", ts);
+                    Path path = Paths.get(downloadBasePath, "m3u8", videoInfo.getId().toString());
-                    Path tsPath = Paths.get(path.toString(), ts);
+                    if (!Files.exists(path)) {
-                    HttpUtils.download(videoInfo.getM3u8Url().replace("index.m3u8", ts), tsPath);
+                        Files.createDirectories(path);
+                    }
+                    Path m3u8 = Paths.get(path.toString(), "index.m3u8");
+                    Files.writeString(m3u8, result);
+
+                    List<String> tss = Arrays.stream(result.split("\n")).filter(t -> t.contains(".ts")).toList();
+                    int size = tss.size();
+                    for (int i = 0; i < size; i++) {
+
+                        String ts = tss.get(i);
+                        log.info("正在下载ts文件，已下载：{}，共：{}，当前进度：{}%", i, size, String.format("%.2f", (i * 100.0 / size)));
+                        Path tsPath = Paths.get(path.toString(), ts);
+                        HttpUtils.download(videoInfo.getM3u8Url().replace("index.m3u8", ts), tsPath);
+                    }
+                    videoInfo.setM3u8(false);
+                    updateById(videoInfo);
+                } catch (Exception e) {
+                    log.error("下载失败，标题：{}，失败原因：{}，地址：{}", videoInfo.getTitle(), videoInfo.getId(), e.getMessage());
                 }
-                videoInfo.setM3u8(false);
+            });
-                updateById(videoInfo);
-            } catch (Exception e) {
-                log.error("下载失败，标题：{}，失败原因：{}，地址：{}", videoInfo.getTitle(), videoInfo.getId(), e.getMessage());
-            }
         }
     }
 }

src/main/java/top/yuchat/crawler/video/utils/HttpUtils.java

@@ -2,11 +2,15 @@ package top.yuchat.crawler.video.utils;
 
 import lombok.extern.slf4j.Slf4j;
 import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
 import org.apache.http.HttpStatus;
 import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.DefaultProxyRoutePlanner;
 import org.apache.http.util.EntityUtils;
 
 import java.io.IOException;
@@ -16,10 +20,13 @@ import java.nio.file.Paths;
 
 @Slf4j
 public class HttpUtils {
-    private static final HttpClient HTTP_CLIENT = new DefaultHttpClient();
+    private static final HttpHost PROXY = new HttpHost("172.20.0.1", 1080);
+    private static final RequestConfig CONFIG = RequestConfig.custom().setProxy(PROXY).build();
+    private static final HttpClient HTTP_CLIENT = HttpClients.createDefault();
 
     public static String get(String url) throws IOException {
         HttpGet httpGet = new HttpGet(url);
+        httpGet.setConfig(CONFIG);
         CloseableHttpResponse response = (CloseableHttpResponse) HTTP_CLIENT.execute(httpGet);
         int statusCode = response.getStatusLine().getStatusCode();
         if (HttpStatus.SC_OK != statusCode) {
@@ -32,6 +39,7 @@ public class HttpUtils {
 
     public static void download(String url, Path path) throws IOException {
         HttpGet httpGet = new HttpGet(url);
+        httpGet.setConfig(CONFIG);
         CloseableHttpResponse response = (CloseableHttpResponse) HTTP_CLIENT.execute(httpGet);
         int statusCode = response.getStatusLine().getStatusCode();
         if (HttpStatus.SC_OK != statusCode) {