diff --git a/src/main/java/top/yuchat/crawler/video/models/service/MadouVideoService.java b/src/main/java/top/yuchat/crawler/video/models/service/MadouVideoService.java index 58476dc..c9d0fba 100644 --- a/src/main/java/top/yuchat/crawler/video/models/service/MadouVideoService.java +++ b/src/main/java/top/yuchat/crawler/video/models/service/MadouVideoService.java @@ -23,6 +23,8 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; @Slf4j @Service @@ -55,6 +57,8 @@ public class MadouVideoService extends ServiceImpl list = classifyInfoService.list(); @@ -130,34 +134,46 @@ public class MadouVideoService extends ServiceImpl wrapper = new QueryWrapper<>(); wrapper.eq("m3u8", true); List madouVideoInfos = list(wrapper); log.info("开始处理数据,数据总量: {}", madouVideoInfos.size()); + Path imgPath = Paths.get(downloadBasePath, "img"); + if (!Files.exists(imgPath)) { + Files.createDirectories(imgPath); + } for (MadouVideoInfo videoInfo : madouVideoInfos) { - try { - log.info("开始下载:{},m3u8地址:{}", videoInfo.getTitle(), videoInfo.getM3u8Url()); - String result = HttpUtils.get(videoInfo.getM3u8Url()); - Path path = Paths.get(downloadBasePath, videoInfo.getId().toString()); - if (!Files.exists(path)) { - Files.createDirectories(path); - } - Path m3u8 = Paths.get(path.toString(), "index.m3u8"); - Files.writeString(m3u8, result); + scheduledThreadPool.submit(() -> { + try { + log.info("下载封面图片,名称:{},地址:{}", videoInfo.getTitle(), videoInfo.getCoverUrl()); + HttpUtils.download(videoInfo.getCoverUrl(), Paths.get(imgPath.toString(), videoInfo.getId().toString() + videoInfo.getCoverUrl().substring(videoInfo.getCoverUrl().lastIndexOf(".")))); - List tss = Arrays.stream(result.split("\n")).filter(t -> t.contains(".ts")).toList(); - for (String ts : tss) { - log.info("开始下载ts文件,文件名称:{}", ts); - Path tsPath = Paths.get(path.toString(), ts); - HttpUtils.download(videoInfo.getM3u8Url().replace("index.m3u8", ts), tsPath); + log.info("开始下载:{},m3u8地址:{}", videoInfo.getTitle(), videoInfo.getM3u8Url()); + String result = HttpUtils.get(videoInfo.getM3u8Url()); + Path path = Paths.get(downloadBasePath, "m3u8", videoInfo.getId().toString()); + if (!Files.exists(path)) { + Files.createDirectories(path); + } + Path m3u8 = Paths.get(path.toString(), "index.m3u8"); + Files.writeString(m3u8, result); + + List tss = Arrays.stream(result.split("\n")).filter(t -> t.contains(".ts")).toList(); + int size = tss.size(); + for (int i = 0; i < size; i++) { + + String ts = tss.get(i); + log.info("正在下载ts文件,已下载:{},共:{},当前进度:{}%", i, size, String.format("%.2f", (i * 100.0 / size))); + Path tsPath = Paths.get(path.toString(), ts); + HttpUtils.download(videoInfo.getM3u8Url().replace("index.m3u8", ts), tsPath); + } + videoInfo.setM3u8(false); + updateById(videoInfo); + } catch (Exception e) { + log.error("下载失败,标题:{},失败原因:{},地址:{}", videoInfo.getTitle(), videoInfo.getId(), e.getMessage()); } - videoInfo.setM3u8(false); - updateById(videoInfo); - } catch (Exception e) { - log.error("下载失败,标题:{},失败原因:{},地址:{}", videoInfo.getTitle(), videoInfo.getId(), e.getMessage()); - } + }); } } } diff --git a/src/main/java/top/yuchat/crawler/video/utils/HttpUtils.java b/src/main/java/top/yuchat/crawler/video/utils/HttpUtils.java index 5407836..c2468a9 100644 --- a/src/main/java/top/yuchat/crawler/video/utils/HttpUtils.java +++ b/src/main/java/top/yuchat/crawler/video/utils/HttpUtils.java @@ -2,11 +2,15 @@ package top.yuchat.crawler.video.utils; import lombok.extern.slf4j.Slf4j; import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; import org.apache.http.HttpStatus; import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.DefaultProxyRoutePlanner; import org.apache.http.util.EntityUtils; import java.io.IOException; @@ -16,10 +20,13 @@ import java.nio.file.Paths; @Slf4j public class HttpUtils { - private static final HttpClient HTTP_CLIENT = new DefaultHttpClient(); + private static final HttpHost PROXY = new HttpHost("172.20.0.1", 1080); + private static final RequestConfig CONFIG = RequestConfig.custom().setProxy(PROXY).build(); + private static final HttpClient HTTP_CLIENT = HttpClients.createDefault(); public static String get(String url) throws IOException { HttpGet httpGet = new HttpGet(url); + httpGet.setConfig(CONFIG); CloseableHttpResponse response = (CloseableHttpResponse) HTTP_CLIENT.execute(httpGet); int statusCode = response.getStatusLine().getStatusCode(); if (HttpStatus.SC_OK != statusCode) { @@ -32,6 +39,7 @@ public class HttpUtils { public static void download(String url, Path path) throws IOException { HttpGet httpGet = new HttpGet(url); + httpGet.setConfig(CONFIG); CloseableHttpResponse response = (CloseableHttpResponse) HTTP_CLIENT.execute(httpGet); int statusCode = response.getStatusLine().getStatusCode(); if (HttpStatus.SC_OK != statusCode) {