开发视频播放的接口
This commit is contained in:
parent
112110946d
commit
6ddda86621
3
.vscode/settings.json
vendored
Normal file
3
.vscode/settings.json
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
{
|
||||
"java.configuration.updateBuildConfiguration": "interactive"
|
||||
}
|
||||
16
pom.xml
16
pom.xml
@ -1,6 +1,7 @@
|
||||
<?xml version="1.0" encoding="UTF-8" ?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>crawler-video</artifactId>
|
||||
<groupId>top.yuchat</groupId>
|
||||
@ -30,7 +31,7 @@
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter</artifactId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
@ -71,6 +72,13 @@
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<parameters>true</parameters>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
||||
@ -1,11 +1,15 @@
|
||||
package top.yuchat.crawler.video.models.service;
|
||||
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
@ -60,7 +64,6 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
|
||||
|
||||
ExecutorService scheduledThreadPool = Executors.newScheduledThreadPool(72);
|
||||
|
||||
|
||||
public void getVideoList() {
|
||||
List<ClassifyInfo> list = classifyInfoService.list();
|
||||
int count = 0;
|
||||
@ -68,7 +71,8 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
|
||||
for (ClassifyInfo classifyInfo : list) {
|
||||
log.info("开始处理分类：{}", classifyInfo.getName());
|
||||
for (int i = 1; i <= classifyInfo.getPageSize(); i++) {
|
||||
String url = VIDEO_PACKAGE.replace("{id}", classifyInfo.getId()).replace("{page_size}", String.valueOf(i));
|
||||
String url = VIDEO_PACKAGE.replace("{id}", classifyInfo.getId()).replace("{page_size}",
|
||||
String.valueOf(i));
|
||||
log.info("正在处理第{}页，页面地址：{}", i, url);
|
||||
try {
|
||||
saveVideo(url, classifyInfo.getName());
|
||||
@ -82,7 +86,6 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
|
||||
log.info("数据处理完成，共处理 {} 页数据", count);
|
||||
}
|
||||
|
||||
|
||||
public void saveVideo(String url, String classify) throws IOException {
|
||||
String html = httpRequestComponent.get(url);
|
||||
Document document = Jsoup.parse(html);
|
||||
@ -106,7 +109,6 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void processingData() {
|
||||
QueryWrapper<MadouVideoInfo> wrapper = new QueryWrapper<>();
|
||||
wrapper.eq("m3u8", false);
|
||||
@ -148,7 +150,9 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
|
||||
scheduledThreadPool.submit(() -> {
|
||||
try {
|
||||
log.info("下载封面图片，名称：{}，地址：{}", videoInfo.getTitle(), videoInfo.getCoverUrl());
|
||||
httpRequestComponent.download(videoInfo.getCoverUrl(), Paths.get(imgPath.toString(), videoInfo.getId().toString() + videoInfo.getCoverUrl().substring(videoInfo.getCoverUrl().lastIndexOf("."))));
|
||||
httpRequestComponent.download(videoInfo.getCoverUrl(),
|
||||
Paths.get(imgPath.toString(), videoInfo.getId().toString()
|
||||
+ videoInfo.getCoverUrl().substring(videoInfo.getCoverUrl().lastIndexOf("."))));
|
||||
|
||||
log.info("开始下载：{}，m3u8地址：{}", videoInfo.getTitle(), videoInfo.getM3u8Url());
|
||||
String result = httpRequestComponent.get(videoInfo.getM3u8Url());
|
||||
@ -176,7 +180,7 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
|
||||
}
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
// @PostConstruct
|
||||
public void getVideoSize() {
|
||||
QueryWrapper<MadouVideoInfo> wrapper = new QueryWrapper<>();
|
||||
wrapper.isNull("size");
|
||||
@ -189,7 +193,8 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
|
||||
long size = 0;
|
||||
for (int i = 0; i < tss.size(); i++) {
|
||||
String ts = tss.get(i);
|
||||
log.info("正在处理ts文件，已下载：{}，共：{}，当前进度：{}%", i, tss.size(), String.format("%.2f", (i * 100.0 / tss.size())));
|
||||
log.info("正在处理ts文件，已下载：{}，共：{}，当前进度：{}%", i, tss.size(),
|
||||
String.format("%.2f", (i * 100.0 / tss.size())));
|
||||
size += httpRequestComponent.getFileSize(videoInfo.getM3u8Url().replace("index.m3u8", ts));
|
||||
}
|
||||
videoInfo.setSize(size);
|
||||
@ -202,4 +207,18 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public IPage<MadouVideoInfo> pageList(Integer page, Integer size, String classify, String title) {
|
||||
LambdaQueryWrapper<MadouVideoInfo> queryWrapper = new LambdaQueryWrapper<>();
|
||||
|
||||
if (StringUtils.isNotBlank(classify)) {
|
||||
queryWrapper.eq(MadouVideoInfo::getClassify, classify);
|
||||
}
|
||||
|
||||
if (StringUtils.isNotBlank(title)) {
|
||||
queryWrapper.like(MadouVideoInfo::getTitle, title);
|
||||
}
|
||||
|
||||
return this.page(new Page<>(page, size), queryWrapper);
|
||||
}
|
||||
}
|
||||
|
||||
@ -2,7 +2,6 @@ package top.yuchat.crawler.video.utils;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.HttpEntity;
|
||||
import org.apache.http.HttpHost;
|
||||
import org.apache.http.HttpStatus;
|
||||
import org.apache.http.client.config.RequestConfig;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
@ -31,7 +30,7 @@ import java.security.NoSuchAlgorithmException;
|
||||
@Slf4j
|
||||
@Component
|
||||
public class HttpRequestComponent {
|
||||
private static final HttpHost PROXY = new HttpHost("172.20.0.1", 1080);
|
||||
// private static final HttpHost PROXY = new HttpHost("172.20.0.1", 1080);
|
||||
|
||||
private CloseableHttpClient closeableHttpClient;
|
||||
private RequestConfig requestConfig;
|
||||
@ -48,7 +47,7 @@ public class HttpRequestComponent {
|
||||
|
||||
public void createHttpClients() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
|
||||
SSLContextBuilder builder = new SSLContextBuilder();
|
||||
builder.loadTrustMaterial(null, (_, _) -> true);
|
||||
builder.loadTrustMaterial(null, (_2, _1) -> true);
|
||||
SSLConnectionSocketFactory sslref = new SSLConnectionSocketFactory(builder.build(), NoopHostnameVerifier.INSTANCE);
|
||||
Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create().register("http", new PlainConnectionSocketFactory()).register("https", sslref).build();
|
||||
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registry);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user