diff --git a/.gitignore b/.gitignore index e0bb4ad..d188a6a 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ replay_pid* .idea target +video \ No newline at end of file diff --git a/src/main/java/top/yuchat/crawler/video/models/service/MadouVideoService.java b/src/main/java/top/yuchat/crawler/video/models/service/MadouVideoService.java index f470779..58476dc 100644 --- a/src/main/java/top/yuchat/crawler/video/models/service/MadouVideoService.java +++ b/src/main/java/top/yuchat/crawler/video/models/service/MadouVideoService.java @@ -1,6 +1,5 @@ package top.yuchat.crawler.video.models.service; -import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSONObject; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; @@ -11,6 +10,7 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import top.yuchat.crawler.video.utils.HttpUtils; import top.yuchat.crawler.video.models.entity.ClassifyInfo; @@ -18,6 +18,10 @@ import top.yuchat.crawler.video.models.entity.MadouVideoInfo; import top.yuchat.crawler.video.models.mapper.MadouVideoMapper; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; import java.util.List; @Slf4j @@ -36,9 +40,19 @@ public class MadouVideoService extends ServiceImpl wrapper = new QueryWrapper<>(); wrapper.eq("m3u8", false); @@ -113,7 +127,37 @@ public class MadouVideoService extends ServiceImpl wrapper = new QueryWrapper<>(); + wrapper.eq("m3u8", true); + List madouVideoInfos = list(wrapper); + log.info("开始处理数据,数据总量: {}", madouVideoInfos.size()); + for (MadouVideoInfo videoInfo : madouVideoInfos) { + try { + log.info("开始下载:{},m3u8地址:{}", videoInfo.getTitle(), videoInfo.getM3u8Url()); + String result = HttpUtils.get(videoInfo.getM3u8Url()); + Path path = Paths.get(downloadBasePath, videoInfo.getId().toString()); + if (!Files.exists(path)) { + Files.createDirectories(path); + } + Path m3u8 = Paths.get(path.toString(), "index.m3u8"); + Files.writeString(m3u8, result); + + List tss = Arrays.stream(result.split("\n")).filter(t -> t.contains(".ts")).toList(); + for (String ts : tss) { + log.info("开始下载ts文件,文件名称:{}", ts); + Path tsPath = Paths.get(path.toString(), ts); + HttpUtils.download(videoInfo.getM3u8Url().replace("index.m3u8", ts), tsPath); + } + videoInfo.setM3u8(false); + updateById(videoInfo); + } catch (Exception e) { + log.error("下载失败,标题:{},失败原因:{},地址:{}", videoInfo.getTitle(), videoInfo.getId(), e.getMessage()); + } + } } } diff --git a/src/main/java/top/yuchat/crawler/video/utils/HttpUtils.java b/src/main/java/top/yuchat/crawler/video/utils/HttpUtils.java index 7ea0fdd..5407836 100644 --- a/src/main/java/top/yuchat/crawler/video/utils/HttpUtils.java +++ b/src/main/java/top/yuchat/crawler/video/utils/HttpUtils.java @@ -10,6 +10,9 @@ import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.util.EntityUtils; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; @Slf4j public class HttpUtils { @@ -20,10 +23,23 @@ public class HttpUtils { CloseableHttpResponse response = (CloseableHttpResponse) HTTP_CLIENT.execute(httpGet); int statusCode = response.getStatusLine().getStatusCode(); if (HttpStatus.SC_OK != statusCode) { - throw new RuntimeException("请求失败,状态码:" + statusCode + ",请求地址:" + url); + log.error("请求失败,状态码:{}", statusCode); + throw new RuntimeException("请求失败,状态码:" + statusCode); } HttpEntity entity = response.getEntity(); return EntityUtils.toString(entity); } + public static void download(String url, Path path) throws IOException { + HttpGet httpGet = new HttpGet(url); + CloseableHttpResponse response = (CloseableHttpResponse) HTTP_CLIENT.execute(httpGet); + int statusCode = response.getStatusLine().getStatusCode(); + if (HttpStatus.SC_OK != statusCode) { + log.error("文件下载失败,状态码:{}", statusCode); + throw new RuntimeException("文件下载失败"); + } + HttpEntity entity = response.getEntity(); + Files.copy(entity.getContent(), path); + } + } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index c4d96da..d68bc1f 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -3,4 +3,9 @@ spring: driver-class-name: org.postgresql.Driver url: jdbc:postgresql://pgsql.yuchat.top:5432/postgres username: postgres - password: longqi@1314 \ No newline at end of file + password: longqi@1314 + +top: + yuchat: + download: + path: F:\videos \ No newline at end of file