开发视频播放的接口
This commit is contained in:
parent
112110946d
commit
6ddda86621
3
.vscode/settings.json
vendored
Normal file
3
.vscode/settings.json
vendored
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
{
|
||||||
|
"java.configuration.updateBuildConfiguration": "interactive"
|
||||||
|
}
|
||||||
12
pom.xml
12
pom.xml
@ -1,5 +1,6 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
<modelVersion>4.0.0</modelVersion>
|
<modelVersion>4.0.0</modelVersion>
|
||||||
<artifactId>crawler-video</artifactId>
|
<artifactId>crawler-video</artifactId>
|
||||||
@ -30,7 +31,7 @@
|
|||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.springframework.boot</groupId>
|
<groupId>org.springframework.boot</groupId>
|
||||||
<artifactId>spring-boot-starter</artifactId>
|
<artifactId>spring-boot-starter-web</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
@ -71,6 +72,13 @@
|
|||||||
<groupId>org.springframework.boot</groupId>
|
<groupId>org.springframework.boot</groupId>
|
||||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||||
</plugin>
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-compiler-plugin</artifactId>
|
||||||
|
<configuration>
|
||||||
|
<parameters>true</parameters>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
</plugins>
|
</plugins>
|
||||||
</build>
|
</build>
|
||||||
|
|
||||||
|
|||||||
@ -1,11 +1,15 @@
|
|||||||
package top.yuchat.crawler.video.models.service;
|
package top.yuchat.crawler.video.models.service;
|
||||||
|
|
||||||
import com.alibaba.fastjson2.JSONObject;
|
import com.alibaba.fastjson2.JSONObject;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||||
|
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||||
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
|
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
|
||||||
import jakarta.annotation.PostConstruct;
|
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.jsoup.Jsoup;
|
import org.jsoup.Jsoup;
|
||||||
import org.jsoup.nodes.Document;
|
import org.jsoup.nodes.Document;
|
||||||
import org.jsoup.nodes.Element;
|
import org.jsoup.nodes.Element;
|
||||||
@ -60,7 +64,6 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
|
|||||||
|
|
||||||
ExecutorService scheduledThreadPool = Executors.newScheduledThreadPool(72);
|
ExecutorService scheduledThreadPool = Executors.newScheduledThreadPool(72);
|
||||||
|
|
||||||
|
|
||||||
public void getVideoList() {
|
public void getVideoList() {
|
||||||
List<ClassifyInfo> list = classifyInfoService.list();
|
List<ClassifyInfo> list = classifyInfoService.list();
|
||||||
int count = 0;
|
int count = 0;
|
||||||
@ -68,7 +71,8 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
|
|||||||
for (ClassifyInfo classifyInfo : list) {
|
for (ClassifyInfo classifyInfo : list) {
|
||||||
log.info("开始处理分类:{}", classifyInfo.getName());
|
log.info("开始处理分类:{}", classifyInfo.getName());
|
||||||
for (int i = 1; i <= classifyInfo.getPageSize(); i++) {
|
for (int i = 1; i <= classifyInfo.getPageSize(); i++) {
|
||||||
String url = VIDEO_PACKAGE.replace("{id}", classifyInfo.getId()).replace("{page_size}", String.valueOf(i));
|
String url = VIDEO_PACKAGE.replace("{id}", classifyInfo.getId()).replace("{page_size}",
|
||||||
|
String.valueOf(i));
|
||||||
log.info("正在处理第{}页,页面地址:{}", i, url);
|
log.info("正在处理第{}页,页面地址:{}", i, url);
|
||||||
try {
|
try {
|
||||||
saveVideo(url, classifyInfo.getName());
|
saveVideo(url, classifyInfo.getName());
|
||||||
@ -82,7 +86,6 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
|
|||||||
log.info("数据处理完成,共处理 {} 页数据", count);
|
log.info("数据处理完成,共处理 {} 页数据", count);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public void saveVideo(String url, String classify) throws IOException {
|
public void saveVideo(String url, String classify) throws IOException {
|
||||||
String html = httpRequestComponent.get(url);
|
String html = httpRequestComponent.get(url);
|
||||||
Document document = Jsoup.parse(html);
|
Document document = Jsoup.parse(html);
|
||||||
@ -106,7 +109,6 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public void processingData() {
|
public void processingData() {
|
||||||
QueryWrapper<MadouVideoInfo> wrapper = new QueryWrapper<>();
|
QueryWrapper<MadouVideoInfo> wrapper = new QueryWrapper<>();
|
||||||
wrapper.eq("m3u8", false);
|
wrapper.eq("m3u8", false);
|
||||||
@ -148,7 +150,9 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
|
|||||||
scheduledThreadPool.submit(() -> {
|
scheduledThreadPool.submit(() -> {
|
||||||
try {
|
try {
|
||||||
log.info("下载封面图片,名称:{},地址:{}", videoInfo.getTitle(), videoInfo.getCoverUrl());
|
log.info("下载封面图片,名称:{},地址:{}", videoInfo.getTitle(), videoInfo.getCoverUrl());
|
||||||
httpRequestComponent.download(videoInfo.getCoverUrl(), Paths.get(imgPath.toString(), videoInfo.getId().toString() + videoInfo.getCoverUrl().substring(videoInfo.getCoverUrl().lastIndexOf("."))));
|
httpRequestComponent.download(videoInfo.getCoverUrl(),
|
||||||
|
Paths.get(imgPath.toString(), videoInfo.getId().toString()
|
||||||
|
+ videoInfo.getCoverUrl().substring(videoInfo.getCoverUrl().lastIndexOf("."))));
|
||||||
|
|
||||||
log.info("开始下载:{},m3u8地址:{}", videoInfo.getTitle(), videoInfo.getM3u8Url());
|
log.info("开始下载:{},m3u8地址:{}", videoInfo.getTitle(), videoInfo.getM3u8Url());
|
||||||
String result = httpRequestComponent.get(videoInfo.getM3u8Url());
|
String result = httpRequestComponent.get(videoInfo.getM3u8Url());
|
||||||
@ -176,7 +180,7 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostConstruct
|
// @PostConstruct
|
||||||
public void getVideoSize() {
|
public void getVideoSize() {
|
||||||
QueryWrapper<MadouVideoInfo> wrapper = new QueryWrapper<>();
|
QueryWrapper<MadouVideoInfo> wrapper = new QueryWrapper<>();
|
||||||
wrapper.isNull("size");
|
wrapper.isNull("size");
|
||||||
@ -189,7 +193,8 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
|
|||||||
long size = 0;
|
long size = 0;
|
||||||
for (int i = 0; i < tss.size(); i++) {
|
for (int i = 0; i < tss.size(); i++) {
|
||||||
String ts = tss.get(i);
|
String ts = tss.get(i);
|
||||||
log.info("正在处理ts文件,已下载:{},共:{},当前进度:{}%", i, tss.size(), String.format("%.2f", (i * 100.0 / tss.size())));
|
log.info("正在处理ts文件,已下载:{},共:{},当前进度:{}%", i, tss.size(),
|
||||||
|
String.format("%.2f", (i * 100.0 / tss.size())));
|
||||||
size += httpRequestComponent.getFileSize(videoInfo.getM3u8Url().replace("index.m3u8", ts));
|
size += httpRequestComponent.getFileSize(videoInfo.getM3u8Url().replace("index.m3u8", ts));
|
||||||
}
|
}
|
||||||
videoInfo.setSize(size);
|
videoInfo.setSize(size);
|
||||||
@ -202,4 +207,18 @@ public class MadouVideoService extends ServiceImpl<MadouVideoMapper, MadouVideoI
|
|||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public IPage<MadouVideoInfo> pageList(Integer page, Integer size, String classify, String title) {
|
||||||
|
LambdaQueryWrapper<MadouVideoInfo> queryWrapper = new LambdaQueryWrapper<>();
|
||||||
|
|
||||||
|
if (StringUtils.isNotBlank(classify)) {
|
||||||
|
queryWrapper.eq(MadouVideoInfo::getClassify, classify);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StringUtils.isNotBlank(title)) {
|
||||||
|
queryWrapper.like(MadouVideoInfo::getTitle, title);
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.page(new Page<>(page, size), queryWrapper);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -2,7 +2,6 @@ package top.yuchat.crawler.video.utils;
|
|||||||
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.http.HttpEntity;
|
import org.apache.http.HttpEntity;
|
||||||
import org.apache.http.HttpHost;
|
|
||||||
import org.apache.http.HttpStatus;
|
import org.apache.http.HttpStatus;
|
||||||
import org.apache.http.client.config.RequestConfig;
|
import org.apache.http.client.config.RequestConfig;
|
||||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||||
@ -31,7 +30,7 @@ import java.security.NoSuchAlgorithmException;
|
|||||||
@Slf4j
|
@Slf4j
|
||||||
@Component
|
@Component
|
||||||
public class HttpRequestComponent {
|
public class HttpRequestComponent {
|
||||||
private static final HttpHost PROXY = new HttpHost("172.20.0.1", 1080);
|
// private static final HttpHost PROXY = new HttpHost("172.20.0.1", 1080);
|
||||||
|
|
||||||
private CloseableHttpClient closeableHttpClient;
|
private CloseableHttpClient closeableHttpClient;
|
||||||
private RequestConfig requestConfig;
|
private RequestConfig requestConfig;
|
||||||
@ -48,7 +47,7 @@ public class HttpRequestComponent {
|
|||||||
|
|
||||||
public void createHttpClients() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
|
public void createHttpClients() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
|
||||||
SSLContextBuilder builder = new SSLContextBuilder();
|
SSLContextBuilder builder = new SSLContextBuilder();
|
||||||
builder.loadTrustMaterial(null, (_, _) -> true);
|
builder.loadTrustMaterial(null, (_2, _1) -> true);
|
||||||
SSLConnectionSocketFactory sslref = new SSLConnectionSocketFactory(builder.build(), NoopHostnameVerifier.INSTANCE);
|
SSLConnectionSocketFactory sslref = new SSLConnectionSocketFactory(builder.build(), NoopHostnameVerifier.INSTANCE);
|
||||||
Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create().register("http", new PlainConnectionSocketFactory()).register("https", sslref).build();
|
Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create().register("http", new PlainConnectionSocketFactory()).register("https", sslref).build();
|
||||||
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registry);
|
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registry);
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user