yanlongqi/crawler-video
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

使用代理和线程池下载

Browse Source
This commit is contained in:
燕陇琪 2024-10-14 23:18:49 +08:00
parent bd9c9c7baa
commit 54f605382b
2 changed files with 45 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
src/main/java/top/yuchat/crawler/video/models/service/MadouVideoService.java
View File

@ -23,6 +23,8 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@Slf4j
@Service
@ -55,6 +57,8 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
private final ClassifyInfoService classifyInfoService;
ExecutorService scheduledThreadPool = Executors.newScheduledThreadPool(16);
public void getVideoList() {
List<ClassifyInfo> list = classifyInfoService.list();
@ -130,34 +134,46 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
}
@PostConstruct
public void download() {
public void download() throws IOException {
QueryWrapper<MadouVideoInfo> wrapper = new QueryWrapper<>();
wrapper.eq("m3u8", true);
List<MadouVideoInfo> madouVideoInfos = list(wrapper);
log.info("开始处理数据，数据总量: {}", madouVideoInfos.size());
Path imgPath = Paths.get(downloadBasePath, "img");
if (!Files.exists(imgPath)) {
Files.createDirectories(imgPath);
}
for (MadouVideoInfo videoInfo : madouVideoInfos) {
try {
log.info("开始下载：{}，m3u8地址：{}", videoInfo.getTitle(), videoInfo.getM3u8Url());
String result = HttpUtils.get(videoInfo.getM3u8Url());
Path path = Paths.get(downloadBasePath, videoInfo.getId().toString());
if (!Files.exists(path)) {
Files.createDirectories(path);
}
Path m3u8 = Paths.get(path.toString(), "index.m3u8");
Files.writeString(m3u8, result);
scheduledThreadPool.submit(() -> {
try {
log.info("下载封面图片，名称：{}，地址：{}", videoInfo.getTitle(), videoInfo.getCoverUrl());
HttpUtils.download(videoInfo.getCoverUrl(), Paths.get(imgPath.toString(), videoInfo.getId().toString() + videoInfo.getCoverUrl().substring(videoInfo.getCoverUrl().lastIndexOf("."))));
List<String> tss = Arrays.stream(result.split("\n")).filter(t -> t.contains(".ts")).toList();
for (String ts : tss) {
log.info("开始下载ts文件，文件名称：{}", ts);
Path tsPath = Paths.get(path.toString(), ts);
HttpUtils.download(videoInfo.getM3u8Url().replace("index.m3u8", ts), tsPath);
log.info("开始下载：{}，m3u8地址：{}", videoInfo.getTitle(), videoInfo.getM3u8Url());
String result = HttpUtils.get(videoInfo.getM3u8Url());
Path path = Paths.get(downloadBasePath, "m3u8", videoInfo.getId().toString());
if (!Files.exists(path)) {
Files.createDirectories(path);
}
Path m3u8 = Paths.get(path.toString(), "index.m3u8");
Files.writeString(m3u8, result);
List<String> tss = Arrays.stream(result.split("\n")).filter(t -> t.contains(".ts")).toList();
int size = tss.size();
for (int i = 0; i < size; i++) {
String ts = tss.get(i);
log.info("正在下载ts文件，已下载：{}，共：{}，当前进度：{}%", i, size, String.format("%.2f", (i * 100.0 / size)));
Path tsPath = Paths.get(path.toString(), ts);
HttpUtils.download(videoInfo.getM3u8Url().replace("index.m3u8", ts), tsPath);
}
videoInfo.setM3u8(false);
updateById(videoInfo);
} catch (Exception e) {
log.error("下载失败，标题：{}，失败原因：{}，地址：{}", videoInfo.getTitle(), videoInfo.getId(), e.getMessage());
}
videoInfo.setM3u8(false);
updateById(videoInfo);
} catch (Exception e) {
log.error("下载失败，标题：{}，失败原因：{}，地址：{}", videoInfo.getTitle(), videoInfo.getId(), e.getMessage());
}
});
}
}
}

10
src/main/java/top/yuchat/crawler/video/utils/HttpUtils.java
View File

@ -2,11 +2,15 @@ package top.yuchat.crawler.video.utils;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.DefaultProxyRoutePlanner;
import org.apache.http.util.EntityUtils;
import java.io.IOException;
@ -16,10 +20,13 @@ import java.nio.file.Paths;
@Slf4j
public class HttpUtils {
private static final HttpClient HTTP_CLIENT = new DefaultHttpClient();
private static final HttpHost PROXY = new HttpHost("172.20.0.1", 1080);
private static final RequestConfig CONFIG = RequestConfig.custom().setProxy(PROXY).build();
private static final HttpClient HTTP_CLIENT = HttpClients.createDefault();
public static String get(String url) throws IOException {
HttpGet httpGet = new HttpGet(url);
httpGet.setConfig(CONFIG);
CloseableHttpResponse response = (CloseableHttpResponse) HTTP_CLIENT.execute(httpGet);
int statusCode = response.getStatusLine().getStatusCode();
if (HttpStatus.SC_OK != statusCode) {
@ -32,6 +39,7 @@ public class HttpUtils {
public static void download(String url, Path path) throws IOException {
HttpGet httpGet = new HttpGet(url);
httpGet.setConfig(CONFIG);
CloseableHttpResponse response = (CloseableHttpResponse) HTTP_CLIENT.execute(httpGet);
int statusCode = response.getStatusLine().getStatusCode();
if (HttpStatus.SC_OK != statusCode) {