114 lines
4.8 KiB
Java
114 lines
4.8 KiB
Java
package top.yuchat.crawler.video.utils;
|
||
|
||
import lombok.extern.slf4j.Slf4j;
|
||
import org.apache.http.HttpEntity;
|
||
import org.apache.http.HttpHost;
|
||
import org.apache.http.HttpStatus;
|
||
import org.apache.http.client.config.RequestConfig;
|
||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||
import org.apache.http.client.methods.HttpGet;
|
||
import org.apache.http.config.Registry;
|
||
import org.apache.http.config.RegistryBuilder;
|
||
import org.apache.http.conn.socket.ConnectionSocketFactory;
|
||
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
|
||
import org.apache.http.conn.ssl.NoopHostnameVerifier;
|
||
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
|
||
import org.apache.http.impl.client.CloseableHttpClient;
|
||
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
|
||
import org.apache.http.impl.client.HttpClients;
|
||
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
|
||
import org.apache.http.ssl.SSLContextBuilder;
|
||
import org.apache.http.util.EntityUtils;
|
||
import org.springframework.stereotype.Component;
|
||
|
||
import java.io.IOException;
|
||
import java.nio.file.Files;
|
||
import java.nio.file.Path;
|
||
import java.security.KeyManagementException;
|
||
import java.security.KeyStoreException;
|
||
import java.security.NoSuchAlgorithmException;
|
||
|
||
@Slf4j
|
||
@Component
|
||
public class HttpRequestComponent {
|
||
private static final HttpHost PROXY = new HttpHost("172.20.0.1", 1080);
|
||
|
||
private CloseableHttpClient closeableHttpClient;
|
||
private RequestConfig requestConfig;
|
||
private int socketTimeout = 5000;
|
||
private int connectTimeout = 5000;
|
||
private int connectRequestTimeout = 5000;
|
||
private int maxTotal = 500;
|
||
private int maxPerRoute = 120;
|
||
|
||
public HttpRequestComponent() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
|
||
init();
|
||
createHttpClients();
|
||
}
|
||
|
||
public void createHttpClients() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
|
||
SSLContextBuilder builder = new SSLContextBuilder();
|
||
builder.loadTrustMaterial(null, (_, _) -> true);
|
||
SSLConnectionSocketFactory sslref = new SSLConnectionSocketFactory(builder.build(), NoopHostnameVerifier.INSTANCE);
|
||
Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create().register("http", new PlainConnectionSocketFactory()).register("https", sslref).build();
|
||
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registry);
|
||
cm.setMaxTotal(maxTotal);
|
||
cm.setDefaultMaxPerRoute(maxPerRoute);
|
||
closeableHttpClient = HttpClients.custom()
|
||
.setSSLSocketFactory(sslref)
|
||
.setConnectionManager(cm)
|
||
.setRetryHandler(new DefaultHttpRequestRetryHandler(5, true))
|
||
.setConnectionManagerShared(true)
|
||
.build();
|
||
}
|
||
|
||
public void init() {
|
||
this.requestConfig = RequestConfig.custom()
|
||
.setSocketTimeout(socketTimeout)
|
||
.setConnectTimeout(connectTimeout)
|
||
.setConnectionRequestTimeout(connectRequestTimeout)
|
||
// .setProxy(PROXY)
|
||
.build();
|
||
}
|
||
|
||
public String get(String url) throws IOException {
|
||
HttpGet httpGet = new HttpGet(url);
|
||
httpGet.setConfig(requestConfig);
|
||
CloseableHttpResponse response = closeableHttpClient.execute(httpGet);
|
||
int statusCode = response.getStatusLine().getStatusCode();
|
||
if (HttpStatus.SC_OK != statusCode) {
|
||
log.error("请求失败，状态码：{}", statusCode);
|
||
throw new RuntimeException("请求失败，状态码：" + statusCode);
|
||
}
|
||
HttpEntity entity = response.getEntity();
|
||
return EntityUtils.toString(entity);
|
||
}
|
||
|
||
public void download(String url, Path path) throws IOException {
|
||
HttpGet httpGet = new HttpGet(url);
|
||
httpGet.setConfig(requestConfig);
|
||
CloseableHttpResponse response = closeableHttpClient.execute(httpGet);
|
||
int statusCode = response.getStatusLine().getStatusCode();
|
||
if (HttpStatus.SC_OK != statusCode) {
|
||
log.error("文件下载失败，状态码：{}", statusCode);
|
||
throw new RuntimeException("文件下载失败");
|
||
}
|
||
HttpEntity entity = response.getEntity();
|
||
|
||
if (Files.exists(path)) {
|
||
long contentLength = entity.getContentLength();
|
||
long size = Files.size(path);
|
||
log.warn("文件已存在，请求文件大小：{}，本地文件大小：{}", contentLength, size);
|
||
if (size >= contentLength) {
|
||
log.warn("文件已存在，跳过下载，文件路径：{}", path);
|
||
response.close();
|
||
return;
|
||
}
|
||
log.warn("文件不完整，删除重新下载，PATH：{}", path);
|
||
Files.delete(path);
|
||
}
|
||
Files.copy(entity.getContent(), path);
|
||
}
|
||
|
||
}
|