package top.yuchat.crawler.video.utils; import lombok.extern.slf4j.Slf4j; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpStatus; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.config.Registry; import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.conn.socket.PlainConnectionSocketFactory; import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.util.EntityUtils; import org.springframework.stereotype.Component; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.security.KeyManagementException; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; @Slf4j @Component public class HttpRequestComponent { private static final HttpHost PROXY = new HttpHost("172.20.0.1", 1080); private CloseableHttpClient closeableHttpClient; private RequestConfig requestConfig; private int socketTimeout = 5000; private int connectTimeout = 5000; private int connectRequestTimeout = 5000; private int maxTotal = 500; private int maxPerRoute = 120; public HttpRequestComponent() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { init(); createHttpClients(); } public void createHttpClients() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { SSLContextBuilder builder = new SSLContextBuilder(); builder.loadTrustMaterial(null, (_, _) -> true); SSLConnectionSocketFactory sslref = new SSLConnectionSocketFactory(builder.build(), NoopHostnameVerifier.INSTANCE); Registry registry = RegistryBuilder.create().register("http", new PlainConnectionSocketFactory()).register("https", sslref).build(); PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registry); cm.setMaxTotal(maxTotal); cm.setDefaultMaxPerRoute(maxPerRoute); closeableHttpClient = HttpClients.custom() .setSSLSocketFactory(sslref) .setConnectionManager(cm) .setRetryHandler(new DefaultHttpRequestRetryHandler(5, true)) .setConnectionManagerShared(true) .build(); } public void init() { this.requestConfig = RequestConfig.custom() .setSocketTimeout(socketTimeout) .setConnectTimeout(connectTimeout) .setConnectionRequestTimeout(connectRequestTimeout) // .setProxy(PROXY) .build(); } public String get(String url) throws IOException { HttpGet httpGet = new HttpGet(url); httpGet.setConfig(requestConfig); CloseableHttpResponse response = closeableHttpClient.execute(httpGet); int statusCode = response.getStatusLine().getStatusCode(); if (HttpStatus.SC_OK != statusCode) { log.error("请求失败,状态码:{}", statusCode); throw new RuntimeException("请求失败,状态码:" + statusCode); } HttpEntity entity = response.getEntity(); return EntityUtils.toString(entity); } public void download(String url, Path path) throws IOException { HttpGet httpGet = new HttpGet(url); httpGet.setConfig(requestConfig); CloseableHttpResponse response = closeableHttpClient.execute(httpGet); int statusCode = response.getStatusLine().getStatusCode(); if (HttpStatus.SC_OK != statusCode) { log.error("文件下载失败,状态码:{}", statusCode); throw new RuntimeException("文件下载失败"); } HttpEntity entity = response.getEntity(); if (Files.exists(path)) { long contentLength = entity.getContentLength(); long size = Files.size(path); log.warn("文件已存在,请求文件大小:{},本地文件大小:{}", contentLength, size); if (size >= contentLength) { log.warn("文件已存在,跳过下载,文件路径:{}", path); response.close(); return; } log.warn("文件不完整,删除重新下载,PATH:{}", path); Files.delete(path); } Files.copy(entity.getContent(), path); } }