视频文件的下载

.gitignore

@@ -26,3 +26,4 @@ replay_pid*
 
 .idea
 target
+video

src/main/java/top/yuchat/crawler/video/models/service/MadouVideoService.java

@@ -1,6 +1,5 @@
 package top.yuchat.crawler.video.models.service;
 
-import com.alibaba.fastjson2.JSON;
 import com.alibaba.fastjson2.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
@@ -11,6 +10,7 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import top.yuchat.crawler.video.utils.HttpUtils;
 import top.yuchat.crawler.video.models.entity.ClassifyInfo;
@@ -18,6 +18,10 @@ import top.yuchat.crawler.video.models.entity.MadouVideoInfo;
 import top.yuchat.crawler.video.models.mapper.MadouVideoMapper;
 
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
 import java.util.List;
 
 @Slf4j
@@ -36,9 +40,19 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
      * https://yutujx.com/?url=https://t20a.cdn2020.com/video/m3u8/2022/10/23/cc234c9c/index.m3u8
      */
 
+    /**
+     * 网页基础路径
+     */
     public static final String BASE_URL = "https://madou.io";
+
+    /**
+     * 分页路径
+     */
     public static final String VIDEO_PACKAGE = BASE_URL + "/index.php/vod/type/id/{id}/page/{page_size}.html";
 
+    @Value("${top.yuchat.download.path:null}")
+    public String downloadBasePath;
+
     private final ClassifyInfoService classifyInfoService;
 
 
@@ -87,7 +101,7 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
         }
     }
 
-    @PostConstruct
+
     public void processingData() {
         QueryWrapper<MadouVideoInfo> wrapper = new QueryWrapper<>();
         wrapper.eq("m3u8", false);
@@ -113,7 +127,37 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
                 log.error("处理失败，失败信息：{}", e.getMessage());
             }
         }
+    }
 
+    @PostConstruct
+    public void download() {
+        QueryWrapper<MadouVideoInfo> wrapper = new QueryWrapper<>();
+        wrapper.eq("m3u8", true);
+        List<MadouVideoInfo> madouVideoInfos = list(wrapper);
+        log.info("开始处理数据，数据总量: {}", madouVideoInfos.size());
 
+        for (MadouVideoInfo videoInfo : madouVideoInfos) {
+            try {
+                log.info("开始下载：{}，m3u8地址：{}", videoInfo.getTitle(), videoInfo.getM3u8Url());
+                String result = HttpUtils.get(videoInfo.getM3u8Url());
+                Path path = Paths.get(downloadBasePath, videoInfo.getId().toString());
+                if (!Files.exists(path)) {
+                    Files.createDirectories(path);
+                }
+                Path m3u8 = Paths.get(path.toString(), "index.m3u8");
+                Files.writeString(m3u8, result);
+
+                List<String> tss = Arrays.stream(result.split("\n")).filter(t -> t.contains(".ts")).toList();
+                for (String ts : tss) {
+                    log.info("开始下载ts文件，文件名称：{}", ts);
+                    Path tsPath = Paths.get(path.toString(), ts);
+                    HttpUtils.download(videoInfo.getM3u8Url().replace("index.m3u8", ts), tsPath);
+                }
+                videoInfo.setM3u8(false);
+                updateById(videoInfo);
+            } catch (Exception e) {
+                log.error("下载失败，标题：{}，失败原因：{}，地址：{}", videoInfo.getTitle(), videoInfo.getId(), e.getMessage());
+            }
+        }
     }
 }

src/main/java/top/yuchat/crawler/video/utils/HttpUtils.java

@@ -10,6 +10,9 @@ import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.util.EntityUtils;
 
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 
 @Slf4j
 public class HttpUtils {
@@ -20,10 +23,23 @@ public class HttpUtils {
         CloseableHttpResponse response = (CloseableHttpResponse) HTTP_CLIENT.execute(httpGet);
         int statusCode = response.getStatusLine().getStatusCode();
         if (HttpStatus.SC_OK != statusCode) {
-            throw new RuntimeException("请求失败，状态码：" + statusCode + "，请求地址：" + url);
+            log.error("请求失败，状态码：{}", statusCode);
+            throw new RuntimeException("请求失败，状态码：" + statusCode);
         }
         HttpEntity entity = response.getEntity();
         return EntityUtils.toString(entity);
     }
 
+    public static void download(String url, Path path) throws IOException {
+        HttpGet httpGet = new HttpGet(url);
+        CloseableHttpResponse response = (CloseableHttpResponse) HTTP_CLIENT.execute(httpGet);
+        int statusCode = response.getStatusLine().getStatusCode();
+        if (HttpStatus.SC_OK != statusCode) {
+            log.error("文件下载失败，状态码：{}", statusCode);
+            throw new RuntimeException("文件下载失败");
+        }
+        HttpEntity entity = response.getEntity();
+        Files.copy(entity.getContent(), path);
+    }
+
 }

src/main/resources/application.yml

@@ -4,3 +4,8 @@ spring:
     url: jdbc:postgresql://pgsql.yuchat.top:5432/postgres
     username: postgres
     password: longqi@1314
+
+top:
+  yuchat:
+    download:
+      path: F:\videos